diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d7e55963..ef0a0c69 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -12,8 +12,8 @@ on: - 'services/gateway/**' - 'services/avatars/**' - '.github/workflows/rust.yml' - - 'Dockerfile.rust' - - 'Dockerfile.bin' + - 'ci/Dockerfile.rust' + - 'ci/rust-docker-target.sh' - 'Cargo.toml' - 'Cargo.lock' @@ -40,7 +40,7 @@ jobs: with: # https://github.com/docker/build-push-action/issues/378 context: . - file: Dockerfile.rust + file: ci/Dockerfile.rust push: false cache-from: type=registry,ref=ghcr.io/pluralkit/docker-cache:rust cache-to: type=registry,ref=ghcr.io/pluralkit/docker-cache:rust,mode=max @@ -48,13 +48,7 @@ jobs: # add more binaries here - run: | - for binary in "api" "gateway" "avatars"; do - for tag in latest ${{ env.BRANCH_NAME }} ${{ github.sha }}; do - cat Dockerfile.bin | sed "s/__BINARY__/$binary/g" | docker build -t ghcr.io/pluralkit/$binary:$tag -f - . - done - if [ "${{ github.repository }}" == "PluralKit/PluralKit" ]; then - docker push ghcr.io/pluralkit/$binary:${{ env.BRANCH_NAME }} - docker push ghcr.io/pluralkit/$binary:${{ github.sha }} - [ "${{ env.BRANCH_NAME }}" == "main" ] && docker push ghcr.io/pluralkit/$binary:latest - fi - done + tag=${{ github.sha }} \ + branch=${{ env.BRANCH_NAME }} \ + push=$([ "${{ github.repository }}" == "PluralKit/PluralKit" ] && echo true || echo false) \ + ci/rust-docker-target.sh diff --git a/.github/workflows/scheduled_tasks.yml b/.github/workflows/scheduled_tasks.yml index 7bbd4450..8a232e04 100644 --- a/.github/workflows/scheduled_tasks.yml +++ b/.github/workflows/scheduled_tasks.yml @@ -2,7 +2,6 @@ name: Build scheduled tasks runner Docker image on: push: - branches: [main, gateway-service] paths: - .github/workflows/scheduled_tasks.yml - 'services/scheduled_tasks/**' diff --git a/Dockerfile.bin b/Dockerfile.bin deleted file mode 100644 index f12cd703..00000000 --- a/Dockerfile.bin +++ /dev/null @@ -1,5 +0,0 @@ -FROM alpine:latest - -COPY /.docker-bin/__BINARY__ /bin/__BINARY__ - -CMD ["/bin/__BINARY__"] diff --git a/Dockerfile.rust b/ci/Dockerfile.rust similarity index 87% rename from Dockerfile.rust rename to ci/Dockerfile.rust index 2d94fe66..9a620e22 100644 --- a/Dockerfile.rust +++ b/ci/Dockerfile.rust @@ -33,9 +33,11 @@ COPY services/avatars/ /build/services/avatars RUN cargo build --bin api --release --target x86_64-unknown-linux-musl RUN cargo build --bin gateway --release --target x86_64-unknown-linux-musl RUN cargo build --bin avatars --release --target x86_64-unknown-linux-musl +RUN cargo build --bin avatar_cleanup --release --target x86_64-unknown-linux-musl -FROM scratch +FROM scratch COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/api /api COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/gateway /gateway COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/avatars /avatars +COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/avatar_cleanup /avatar_cleanup diff --git a/ci/rust-docker-target.sh b/ci/rust-docker-target.sh new file mode 100755 index 00000000..19748c4c --- /dev/null +++ b/ci/rust-docker-target.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e + +#tag= +#branch= +#push= + +build() { + bin=$1 + extra=$2 + + f=$(mktemp) + + cat > $f << EOF +FROM alpine:latest +COPY .docker-bin/$bin /bin/$bin +$extra +CMD ["/bin/$bin"] +EOF + + echo "building $dockerfile" + + $dockerfile | docker build -t ghcr.io/pluralkit/$bin:$tag -f $f . + + rm $f + + if [ "$push" == "true" ]; then + docker push ghcr.io/pluralkit/$bin:$tag + docker image tag ghcr.io/pluralkit/$bin:$tag ghcr.io/pluralkit/$bin:$branch + docker push ghcr.io/pluralkit/$bin:$branch + if [ "$branch" == "main" ]; then + docker image tag ghcr.io/pluralkit/$bin:$tag ghcr.io/pluralkit/$bin:latest + docker push ghcr.io/pluralkit/$bin:latest + fi + fi +} + +# add rust binaries here to build +build api +build gateway +build avatars "COPY .docker-bin/avatar_cleanup /bin/avatar_cleanup" diff --git a/go.work.sum b/go.work.sum index c0f58b37..20bb40cf 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,2 +1,3 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= diff --git a/lib/libpk/src/_config.rs b/lib/libpk/src/_config.rs index 298e603c..d112eacd 100644 --- a/lib/libpk/src/_config.rs +++ b/lib/libpk/src/_config.rs @@ -61,6 +61,11 @@ pub struct AvatarsConfig { #[serde(default)] pub migrate_worker_count: u32, + + #[serde(default)] + pub cloudflare_zone_id: Option, + #[serde(default)] + pub cloudflare_token: Option, } #[derive(Deserialize, Clone, Debug)] diff --git a/lib/libpk/src/db/repository/avatars.rs b/lib/libpk/src/db/repository/avatars.rs index cb77ee8c..6732ac67 100644 --- a/lib/libpk/src/db/repository/avatars.rs +++ b/lib/libpk/src/db/repository/avatars.rs @@ -2,6 +2,13 @@ use sqlx::{PgPool, Postgres, Transaction}; use crate::db::types::avatars::*; +pub async fn get_by_id(pool: &PgPool, id: String) -> anyhow::Result> { + Ok(sqlx::query_as("select * from images where id = $1") + .bind(id) + .fetch_optional(pool) + .await?) +} + pub async fn get_by_original_url( pool: &PgPool, original_url: &str, diff --git a/services/avatars/Cargo.toml b/services/avatars/Cargo.toml index bb664f6e..b4f9e22b 100644 --- a/services/avatars/Cargo.toml +++ b/services/avatars/Cargo.toml @@ -3,6 +3,10 @@ name = "avatars" version = "0.1.0" edition = "2021" +[[bin]] +name = "avatar_cleanup" +path = "src/cleanup.rs" + [dependencies] libpk = { path = "../../lib/libpk" } anyhow = { workspace = true } diff --git a/services/avatars/Dockerfile b/services/avatars/Dockerfile new file mode 100644 index 00000000..21864cc2 --- /dev/null +++ b/services/avatars/Dockerfile @@ -0,0 +1,88 @@ +# syntax=docker/dockerfile:1 + +# Comments are provided throughout this file to help you get started. +# If you need more help, visit the Dockerfile reference guide at +# https://docs.docker.com/go/dockerfile-reference/ + +ARG RUST_VERSION=1.75.0 +ARG APP_NAME=pluralkit-avatars + +################################################################################ +# xx is a helper for cross-compilation. +# See https://github.com/tonistiigi/xx/ for more information. +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.3.0 AS xx + +################################################################################ +# Create a stage for building the application. +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-alpine AS build +ARG APP_NAME +WORKDIR /app + +# Copy cross compilation utilities from the xx stage. +COPY --from=xx / / + +# Install host build dependencies. +RUN apk add --no-cache clang lld musl-dev git file + +# This is the architecture you’re building for, which is passed in by the builder. +# Placing it here allows the previous steps to be cached across architectures. +ARG TARGETPLATFORM + +# Install cross compilation build dependencies. +RUN xx-apk add --no-cache musl-dev gcc + +# Build the application. +# Leverage a cache mount to /usr/local/cargo/registry/ +# for downloaded dependencies, a cache mount to /usr/local/cargo/git/db +# for git repository dependencies, and a cache mount to /app/target/ for +# compiled dependencies which will speed up subsequent builds. +# Leverage a bind mount to the src directory to avoid having to copy the +# source code into the container. Once built, copy the executable to an +# output directory before the cache mounted /app/target is unmounted. +# XXX: removed `id` from target mount, see: https://github.com/reproducible-containers/buildkit-cache-dance/issues/12 +RUN --mount=type=bind,source=src,target=src \ + --mount=type=bind,source=Cargo.toml,target=Cargo.toml \ + --mount=type=bind,source=Cargo.lock,target=Cargo.lock \ + --mount=type=cache,target=/app/target/$TARGETPLATFORM/ \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/usr/local/cargo/registry/ \ + < anyhow::Result<()> { + libpk::init_logging("avatar_cleanup")?; + libpk::init_metrics()?; + info!("hello world"); + + let config = libpk::config + .avatars + .as_ref() + .expect("missing avatar service config"); + + let bucket = { + let region = s3::Region::Custom { + region: "s3".to_string(), + endpoint: config.s3.endpoint.to_string(), + }; + + let credentials = s3::creds::Credentials::new( + Some(&config.s3.application_id), + Some(&config.s3.application_key), + None, + None, + None, + ) + .unwrap(); + + let bucket = s3::Bucket::new(&config.s3.bucket, region, credentials)?; + + Arc::new(bucket) + }; + + let pool = libpk::db::init_data_db().await?; + + loop { + // no infinite loops + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + match cleanup_job(pool.clone(), bucket.clone()).await { + Ok(()) => {} + Err(err) => { + error!("failed to run avatar cleanup job: {}", err); + // sentry + } + } + } +} + +#[derive(FromRow)] +struct CleanupJobEntry { + id: String, +} + +async fn cleanup_job(pool: sqlx::PgPool, bucket: Arc) -> anyhow::Result<()> { + let mut tx = pool.begin().await?; + + let image_id: Option = + sqlx::query_as("select id from image_cleanup_jobs for update skip locked limit 1;") + .fetch_optional(&mut *tx) + .await?; + if image_id.is_none() { + info!("no job to run, sleeping for 1 minute"); + tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; + return Ok(()); + } + let image_id = image_id.unwrap().id; + info!("got image {image_id}, cleaning up..."); + + let image_data = libpk::db::repository::avatars::get_by_id(&pool, image_id.clone()).await?; + if image_data.is_none() { + info!("image {image_id} was already deleted, skipping"); + sqlx::query("delete from image_cleanup_jobs where id = $1") + .bind(image_id) + .execute(&mut *tx) + .await?; + return Ok(()); + } + let image_data = image_data.unwrap(); + + let config = libpk::config + .avatars + .as_ref() + .expect("missing avatar service config"); + + let path = image_data + .url + .strip_prefix(config.cdn_url.as_str()) + .unwrap(); + + let s3_resp = bucket.delete_object(path).await?; + match s3_resp.status_code() { + 204 => { + info!("successfully deleted image {image_id} from s3"); + } + _ => { + anyhow::bail!("s3 returned bad error code {}", s3_resp.status_code()); + } + } + + if let Some(zone_id) = config.cloudflare_zone_id.as_ref() { + let client = ClientBuilder::new() + .connect_timeout(Duration::from_secs(3)) + .timeout(Duration::from_secs(3)) + .build() + .context("error making client")?; + + let cf_resp = client + .post(format!( + "https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache" + )) + .header( + "Authorization", + format!("Bearer {}", config.cloudflare_token.as_ref().unwrap()), + ) + .body(format!(r#"{{"files":["{}"]}}"#, image_data.url)) + .send() + .await?; + + match cf_resp.status() { + StatusCode::OK => { + info!( + "successfully purged url {} from cloudflare cache", + image_data.url + ); + } + _ => { + let status = cf_resp.status(); + println!("{:#?}", cf_resp.text().await?); + anyhow::bail!("cloudflare returned bad error code {}", status); + } + } + } + + sqlx::query("delete from images where id = $1") + .bind(image_id.clone()) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(()) +} diff --git a/services/avatars/src/init.sql b/services/avatars/src/init.sql index 854c065b..09e7ed54 100644 --- a/services/avatars/src/init.sql +++ b/services/avatars/src/init.sql @@ -21,4 +21,6 @@ create index if not exists images_uploaded_by_account_idx on images (uploaded_by create table if not exists image_queue (itemid serial primary key, url text not null, kind text not null); alter table images add column if not exists uploaded_by_system uuid; -alter table images add column if not exists content_type text default 'image/webp'; \ No newline at end of file +alter table images add column if not exists content_type text default 'image/webp'; + +create table image_cleanup_jobs(id text references images(id) on delete cascade); diff --git a/services/scheduled_tasks/go.mod b/services/scheduled_tasks/go.mod index 830dc19f..7cea5024 100644 --- a/services/scheduled_tasks/go.mod +++ b/services/scheduled_tasks/go.mod @@ -3,10 +3,17 @@ module scheduled_tasks go 1.18 require ( - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/getsentry/sentry-go v0.15.0 + github.com/go-redis/redis/v8 v8.11.5 + github.com/jackc/pgx/v4 v4.16.1 + github.com/prometheus/client_golang v1.20.5 + golang.org/x/text v0.16.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/getsentry/sentry-go v0.15.0 // indirect - github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.12.1 // indirect github.com/jackc/pgio v1.0.0 // indirect @@ -14,11 +21,13 @@ require ( github.com/jackc/pgproto3/v2 v2.3.0 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.11.0 // indirect - github.com/jackc/pgx v3.6.2+incompatible // indirect - github.com/jackc/pgx/v4 v4.16.1 // indirect github.com/jackc/puddle v1.2.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect - golang.org/x/text v0.5.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/sys v0.22.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/services/scheduled_tasks/go.sum b/services/scheduled_tasks/go.sum index c44602df..e4e20bdf 100644 --- a/services/scheduled_tasks/go.sum +++ b/services/scheduled_tasks/go.sum @@ -1,25 +1,32 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/getsentry/sentry-go v0.15.0 h1:CP9bmA7pralrVUedYZsmIHWpq/pBtXTSew7xvVpfLaA= github.com/getsentry/sentry-go v0.15.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -36,10 +43,10 @@ github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -57,8 +64,6 @@ github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrU github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= -github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= @@ -71,31 +76,50 @@ github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -107,6 +131,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -128,10 +153,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -140,6 +163,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -153,8 +177,8 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -162,10 +186,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -179,10 +202,15 @@ golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/services/scheduled_tasks/main.go b/services/scheduled_tasks/main.go index 263e2a93..5e337c62 100644 --- a/services/scheduled_tasks/main.go +++ b/services/scheduled_tasks/main.go @@ -7,12 +7,24 @@ import ( "runtime/debug" "strings" "time" + "net/http" "github.com/getsentry/sentry-go" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var set_guild_count = false +var ( + cleanupQueueLength = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "pluralkit_image_cleanup_queue_length", + Help: "Remaining image cleanup jobs", + }) +) + func main() { if _, ok := os.LookupEnv("SET_GUILD_COUNT"); ok { set_guild_count = true @@ -29,9 +41,19 @@ func main() { connect_dbs() log.Println("starting scheduled tasks runner") - wait_until_next_minute() - go doforever(time.Minute*10, withtime("message stats updater", update_db_message_meta)) - doforever(time.Minute, withtime("scheduled tasks", task_main)) + go func() { + wait_until_next_minute() + + go doforever(time.Minute, withtime("stats updater", update_db_meta)) + go doforever(time.Minute*10, withtime("message stats updater", update_db_message_meta)) + go doforever(time.Minute, withtime("discord stats updater", update_discord_stats)) + go doforever(time.Minute*30, withtime("queue deleted image cleanup job", queue_deleted_image_cleanup)) + }() + + go doforever(time.Second * 10, withtime("prometheus updater", update_prom)) + log.Println("listening for prometheus on :9000") + http.Handle("/metrics", promhttp.Handler()) + http.ListenAndServe(":9000", nil) } func wait_until_next_minute() { @@ -49,6 +71,7 @@ func get_env_var(key string) string { func withtime(name string, todo func()) func() { return func() { + log.Println("running", name) timeBefore := time.Now() todo() timeAfter := time.Now() diff --git a/services/scheduled_tasks/repo.go b/services/scheduled_tasks/repo.go index dd637af0..ece7e0dd 100644 --- a/services/scheduled_tasks/repo.go +++ b/services/scheduled_tasks/repo.go @@ -131,6 +131,15 @@ func get_message_count() int { return count } +func get_image_cleanup_queue_length() int { + var count int + row := data_db.QueryRow(context.Background(), "select count(*) as count from image_cleanup_jobs") + if err := row.Scan(&count); err != nil { + panic(err) + } + return count +} + func run_data_stats_query() map[string]interface{} { s := map[string]interface{}{} @@ -142,6 +151,9 @@ func run_data_stats_query() map[string]interface{} { for rows.Next() { for i, column := range descs { + if string(column.Name) == "message_count" { + continue + } values, err := rows.Values() if err != nil { panic(err) diff --git a/services/scheduled_tasks/tasks.go b/services/scheduled_tasks/tasks.go index acb164fe..a1404682 100644 --- a/services/scheduled_tasks/tasks.go +++ b/services/scheduled_tasks/tasks.go @@ -9,15 +9,7 @@ import ( "golang.org/x/text/message" ) -func task_main() { - log.Println("running per-minute scheduled tasks") - - update_db_meta() - update_stats() - update_bot_status() -} - -var table_stat_keys = []string{"system", "member", "group", "switch", "message"} +var table_stat_keys = []string{"system", "member", "group", "switch"} func plural(key string) string { if key[len(key)-1] == 'h' { @@ -26,17 +18,24 @@ func plural(key string) string { return key + "s" } +func update_prom() { + count := get_image_cleanup_queue_length() + cleanupQueueLength.Set(float64(count)) +} + func update_db_meta() { - log.Println("updating database stats") for _, key := range table_stat_keys { - if key == "message" { - // updating message count from data db takes way too long, so we do it on a separate timer (every 10 minutes) - continue - } q := fmt.Sprintf("update info set %s_count = (select count(*) from %s)", key, plural(key)) log.Println("data db query:", q) run_simple_pg_query(data_db, q) } + + data_stats := run_data_stats_query() + for _, key := range table_stat_keys { + val := data_stats[key+"_count"].(int64) + log.Printf("%v: %v\n", key+"_count", val) + do_stats_insert(plural(key), val) + } } func update_db_message_meta() { @@ -46,9 +45,11 @@ func update_db_message_meta() { if err != nil { panic(err) } + + do_stats_insert("messages", int64(count)) } -func get_discord_counts() (int, int) { +func update_discord_stats() { redisStats := query_http_cache() guild_count := 0 @@ -60,28 +61,13 @@ func get_discord_counts() (int, int) { channel_count += v.ChannelCount } - return guild_count, channel_count -} - -func update_stats() { - data_stats := run_data_stats_query() - for _, key := range table_stat_keys { - val := data_stats[key+"_count"].(int64) - do_stats_insert(plural(key), val) - } - guild_count, channel_count := get_discord_counts() - do_stats_insert("guilds", int64(guild_count)) do_stats_insert("channels", int64(channel_count)) -} - -func update_bot_status() { if !set_guild_count { return } - guild_count, _ := get_discord_counts() p := message.NewPrinter(language.English) s := p.Sprintf("%d", guild_count) @@ -90,3 +76,28 @@ func update_bot_status() { panic(err) } } + +// MUST add new image columns here +var deletedImageCleanupQuery = ` +insert into image_cleanup_jobs +select id from images where + not exists (select from image_cleanup_jobs j where j.id = images.id) + and not exists (select from systems where avatar_url = images.url) + and not exists (select from systems where banner_image = images.url) + and not exists (select from system_guild where avatar_url = images.url) + + and not exists (select from members where avatar_url = images.url) + and not exists (select from members where banner_image = images.url) + and not exists (select from members where webhook_avatar_url = images.url) + and not exists (select from member_guild where avatar_url = images.url) + + and not exists (select from groups where icon = images.url) + and not exists (select from groups where banner_image = images.url); + ` + +func queue_deleted_image_cleanup() { + _, err := data_db.Exec(context.Background(), deletedImageCleanupQuery) + if err != nil { + panic(err) + } +}