diff --git a/.github/workflows/scheduled_tasks.yml b/.github/workflows/scheduled_tasks.yml deleted file mode 100644 index 12d4f112..00000000 --- a/.github/workflows/scheduled_tasks.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build scheduled tasks runner Docker image - -on: - push: - paths: - - .github/workflows/scheduled_tasks.yml - - 'services/scheduled_tasks/**' - -jobs: - build: - runs-on: ubuntu-latest - permissions: - packages: write - if: github.repository == 'PluralKit/PluralKit' - steps: - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.CR_PAT }} - - uses: actions/checkout@v2 - - run: echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - uses: docker/build-push-action@v2 - with: - # https://github.com/docker/build-push-action/issues/378 - context: services/scheduled_tasks/ - push: true - tags: | - ghcr.io/pluralkit/scheduled_tasks:${{ env.BRANCH_NAME }} - ghcr.io/pluralkit/scheduled_tasks:${{ github.sha }} - ghcr.io/pluralkit/scheduled_tasks:latest - cache-from: type=registry,ref=ghcr.io/pluralkit/scheduledtasks:${{ env.BRANCH_NAME }} - cache-to: type=inline diff --git a/Cargo.lock b/Cargo.lock index 7532dde2..23e0fcab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,6 +110,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-trait" version = "0.1.80" @@ -1906,6 +1912,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2811,6 +2827,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "scheduled_tasks" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "fred", + "libpk", + "metrics", + "num-format", + "reqwest 0.12.8", + "serde", + "sqlx", + "tokio", + "tracing", +] + [[package]] name = "scopeguard" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 366e2c60..f598c431 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,8 @@ members = [ "./services/api", "./services/dispatch", "./services/gateway", - "./services/avatars" + "./services/avatars", + "./services/scheduled_tasks", ] [workspace.dependencies] diff --git a/ci/Dockerfile.rust b/ci/Dockerfile.rust index e0156e5c..8a3d5a67 100644 --- a/ci/Dockerfile.rust +++ b/ci/Dockerfile.rust @@ -25,16 +25,14 @@ COPY Cargo.lock /build/ # this needs to match workspaces in Cargo.toml COPY lib/libpk /build/lib/libpk -COPY services/api/ /build/services/api -COPY services/dispatch/ /build/services/dispatch -COPY services/gateway/ /build/services/gateway -COPY services/avatars/ /build/services/avatars +COPY services/ /build/services RUN cargo build --bin api --release --target x86_64-unknown-linux-musl RUN cargo build --bin dispatch --release --target x86_64-unknown-linux-musl RUN cargo build --bin gateway --release --target x86_64-unknown-linux-musl RUN cargo build --bin avatars --release --target x86_64-unknown-linux-musl RUN cargo build --bin avatar_cleanup --release --target x86_64-unknown-linux-musl +RUN cargo build --bin scheduled_tasks --release --target x86_64-unknown-linux-musl FROM scratch @@ -43,3 +41,4 @@ COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/dispa COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/gateway /gateway COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/avatars /avatars COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/avatar_cleanup /avatar_cleanup +COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/scheduled_tasks /scheduled_tasks diff --git a/ci/rust-docker-target.sh b/ci/rust-docker-target.sh index 9932abf8..8e87dbca 100755 --- a/ci/rust-docker-target.sh +++ b/ci/rust-docker-target.sh @@ -41,3 +41,4 @@ build api build dispatch build gateway build avatars "COPY .docker-bin/avatar_cleanup /bin/avatar_cleanup" +build scheduled_tasks diff --git a/go.work b/go.work deleted file mode 100644 index a9e9d1ec..00000000 --- a/go.work +++ /dev/null @@ -1,5 +0,0 @@ -go 1.23 - -toolchain go1.23.2 - -use ./services/scheduled_tasks diff --git a/go.work.sum b/go.work.sum deleted file mode 100644 index 72bcf761..00000000 --- a/go.work.sum +++ /dev/null @@ -1,4 +0,0 @@ -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= diff --git a/lib/libpk/src/_config.rs b/lib/libpk/src/_config.rs index 6f69399b..b182f359 100644 --- a/lib/libpk/src/_config.rs +++ b/lib/libpk/src/_config.rs @@ -31,7 +31,8 @@ pub struct DatabaseConfig { pub(crate) data_db_uri: String, pub(crate) data_db_max_connections: Option, pub(crate) data_db_min_connections: Option, - // pub(crate) _messages_db_uri: String, + pub(crate) messages_db_uri: Option, + pub(crate) stats_db_uri: Option, pub(crate) db_password: Option, pub data_redis_addr: String, } @@ -79,6 +80,13 @@ pub struct S3Config { pub endpoint: String, } +#[derive(Deserialize, Debug)] +pub struct ScheduledTasksConfig { + pub set_guild_count: bool, + pub expected_gateway_count: usize, + pub gateway_url: String, +} + fn _metrics_default() -> bool { false } @@ -96,6 +104,8 @@ pub struct PKConfig { pub api: Option, #[serde(default)] pub avatars: Option, + #[serde(default)] + pub scheduled_tasks: Option, #[serde(default = "_metrics_default")] pub run_metrics_server: bool, diff --git a/lib/libpk/src/db/mod.rs b/lib/libpk/src/db/mod.rs index 49bd1524..4a5093f4 100644 --- a/lib/libpk/src/db/mod.rs +++ b/lib/libpk/src/db/mod.rs @@ -44,3 +44,53 @@ pub async fn init_data_db() -> anyhow::Result { Ok(pool.connect_with(options).await?) } + +pub async fn init_messages_db() -> anyhow::Result { + info!("connecting to messages database"); + + let mut options = PgConnectOptions::from_str( + &crate::config + .db + .messages_db_uri + .as_ref() + .expect("missing messages db uri"), + )?; + + if let Some(password) = crate::config.db.db_password.clone() { + options = options.password(&password); + } + + let mut pool = PgPoolOptions::new(); + + if let Some(max_conns) = crate::config.db.data_db_max_connections { + pool = pool.max_connections(max_conns); + } + + if let Some(min_conns) = crate::config.db.data_db_min_connections { + pool = pool.min_connections(min_conns); + } + + Ok(pool.connect_with(options).await?) +} + +pub async fn init_stats_db() -> anyhow::Result { + info!("connecting to stats database"); + + let mut options = PgConnectOptions::from_str( + &crate::config + .db + .stats_db_uri + .as_ref() + .expect("missing messages db uri"), + )?; + + if let Some(password) = crate::config.db.db_password.clone() { + options = options.password(&password); + } + + Ok(PgPoolOptions::new() + .max_connections(1) + .min_connections(1) + .connect_with(options) + .await?) +} diff --git a/lib/libpk/src/db/repository/stats.rs b/lib/libpk/src/db/repository/stats.rs index 0ef81ac8..122472dd 100644 --- a/lib/libpk/src/db/repository/stats.rs +++ b/lib/libpk/src/db/repository/stats.rs @@ -3,6 +3,19 @@ pub async fn get_stats(pool: &sqlx::postgres::PgPool) -> anyhow::Result Ok(counts) } +pub async fn insert_stats( + pool: &sqlx::postgres::PgPool, + table: &str, + value: i64, +) -> anyhow::Result<()> { + // danger sql injection + sqlx::query(format!("insert into {table} values (now(), $1)").as_str()) + .bind(value) + .execute(pool) + .await?; + Ok(()) +} + #[derive(serde::Serialize, sqlx::FromRow)] pub struct Counts { pub system_count: i64, diff --git a/services/scheduled_tasks/Cargo.toml b/services/scheduled_tasks/Cargo.toml new file mode 100644 index 00000000..07d0d33d --- /dev/null +++ b/services/scheduled_tasks/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "scheduled_tasks" +version = "0.1.0" +edition = "2021" + +[dependencies] +libpk = { path = "../../lib/libpk" } + +anyhow = { workspace = true } +chrono = { workspace = true } +fred = { workspace = true } +metrics = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +sqlx = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +num-format = "0.4.4" diff --git a/services/scheduled_tasks/Dockerfile b/services/scheduled_tasks/Dockerfile deleted file mode 100644 index 8f790bd1..00000000 --- a/services/scheduled_tasks/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM alpine:latest AS builder - -RUN apk add go - -WORKDIR /build - -COPY ./ /build - -RUN GOTOOLCHAIN=auto go build . - -FROM alpine:latest - -COPY --from=builder /build/scheduled_tasks /bin/runner - -ENTRYPOINT ["/bin/runner"] diff --git a/services/scheduled_tasks/db.go b/services/scheduled_tasks/db.go deleted file mode 100644 index b27a2a75..00000000 --- a/services/scheduled_tasks/db.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "context" - "os" - - redis "github.com/go-redis/redis/v8" - "github.com/jackc/pgx/v4/pgxpool" -) - -var data_db *pgxpool.Pool -var messages_db *pgxpool.Pool -var stats_db *pgxpool.Pool -var rdb *redis.Client - -func run_simple_pg_query(c *pgxpool.Pool, sql string) { - _, err := c.Exec(context.Background(), sql) - if err != nil { - panic(err) - } -} - -func connect_dbs() { - data_db = pg_connect(get_env_var("DATA_DB_URI")) - messages_db = pg_connect(get_env_var("MESSAGES_DB_URI")) - rdb = redis_connect(get_env_var("REDIS_ADDR")) - - if uri, ok := os.LookupEnv("STATS_DB_URI"); ok { - stats_db = pg_connect(uri) - } -} - -func pg_connect(url string) *pgxpool.Pool { - conn, err := pgxpool.Connect(context.Background(), url) - if err != nil { - panic(err) - } - - return conn -} - -func redis_connect(url string) *redis.Client { - return redis.NewClient(&redis.Options{ - Addr: url, - DB: 0, - }) -} diff --git a/services/scheduled_tasks/go.mod b/services/scheduled_tasks/go.mod deleted file mode 100644 index 74020db5..00000000 --- a/services/scheduled_tasks/go.mod +++ /dev/null @@ -1,33 +0,0 @@ -module scheduled_tasks - -go 1.23 - -require ( - github.com/getsentry/sentry-go v0.15.0 - github.com/go-redis/redis/v8 v8.11.5 - github.com/jackc/pgx/v4 v4.16.1 - github.com/prometheus/client_golang v1.20.5 - golang.org/x/text v0.16.0 -) - -require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.12.1 // indirect - github.com/jackc/pgio v1.0.0 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.11.0 // indirect - github.com/jackc/puddle v1.2.1 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sys v0.22.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect -) diff --git a/services/scheduled_tasks/go.sum b/services/scheduled_tasks/go.sum deleted file mode 100644 index e4e20bdf..00000000 --- a/services/scheduled_tasks/go.sum +++ /dev/null @@ -1,216 +0,0 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/getsentry/sentry-go v0.15.0 h1:CP9bmA7pralrVUedYZsmIHWpq/pBtXTSew7xvVpfLaA= -github.com/getsentry/sentry-go v0.15.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8= -github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= -github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= -github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y= -github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= -github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/services/scheduled_tasks/main.go b/services/scheduled_tasks/main.go deleted file mode 100644 index 5e337c62..00000000 --- a/services/scheduled_tasks/main.go +++ /dev/null @@ -1,106 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "runtime/debug" - "strings" - "time" - "net/http" - - "github.com/getsentry/sentry-go" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var set_guild_count = false - -var ( - cleanupQueueLength = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "pluralkit_image_cleanup_queue_length", - Help: "Remaining image cleanup jobs", - }) -) - -func main() { - if _, ok := os.LookupEnv("SET_GUILD_COUNT"); ok { - set_guild_count = true - } - - err := sentry.Init(sentry.ClientOptions{ - Dsn: os.Getenv("SENTRY_DSN"), - }) - if err != nil { - panic(err) - } - - log.Println("connecting to databases") - connect_dbs() - - log.Println("starting scheduled tasks runner") - go func() { - wait_until_next_minute() - - go doforever(time.Minute, withtime("stats updater", update_db_meta)) - go doforever(time.Minute*10, withtime("message stats updater", update_db_message_meta)) - go doforever(time.Minute, withtime("discord stats updater", update_discord_stats)) - go doforever(time.Minute*30, withtime("queue deleted image cleanup job", queue_deleted_image_cleanup)) - }() - - go doforever(time.Second * 10, withtime("prometheus updater", update_prom)) - log.Println("listening for prometheus on :9000") - http.Handle("/metrics", promhttp.Handler()) - http.ListenAndServe(":9000", nil) -} - -func wait_until_next_minute() { - now := time.Now().UTC().Add(time.Minute) - after := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), 0, 0, time.UTC) - time.Sleep(after.Sub(time.Now().UTC())) -} - -func get_env_var(key string) string { - if val, ok := os.LookupEnv(key); ok { - return val - } - panic(fmt.Errorf("missing `%s` in environment", key)) -} - -func withtime(name string, todo func()) func() { - return func() { - log.Println("running", name) - timeBefore := time.Now() - todo() - timeAfter := time.Now() - log.Println("ran", name, "in", timeAfter.Sub(timeBefore).String()) - } -} - -func doforever(dur time.Duration, todo func()) { - for { - go wrapRecover(todo) - time.Sleep(dur) - } -} - -func wrapRecover(todo func()) { - defer func() { - if err := recover(); err != nil { - if val, ok := err.(error); ok { - sentry.CaptureException(val) - } else { - sentry.CaptureMessage(fmt.Sprint("unknown error", err)) - } - - stack := strings.Split(string(debug.Stack()), "\n") - stack = stack[7:] - log.Printf("error running tasks: %v\n", err) - fmt.Println(strings.Join(stack, "\n")) - } - }() - - todo() -} diff --git a/services/scheduled_tasks/repo.go b/services/scheduled_tasks/repo.go deleted file mode 100644 index 31554850..00000000 --- a/services/scheduled_tasks/repo.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "log" - "io" - "os" - "net/http" - "strconv" -) - -type httpstats struct { - Up bool `json:"up"` - GuildCount int `json:"guild_count"` - ChannelCount int `json:"channel_count"` -} - -func query_http_cache() []httpstats { - var values []httpstats - - http_cache_url := os.Getenv("HTTP_CACHE_URL") - if http_cache_url == "" { - panic("missing HTTP_CACHE_URL in environment") - } - - cluster_count, err := strconv.Atoi(os.Getenv("CLUSTER_COUNT")) - if err != nil { - panic(fmt.Sprintf("missing or invalid CLUSTER_COUNT in environment")) - } - - for i := range cluster_count { - log.Printf("querying gateway cluster %v for discord stats\n", i) - url := fmt.Sprintf("http://cluster%v.%s:5000/stats", i, http_cache_url) - resp, err := http.Get(url) - if err != nil { - panic(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusFound { - panic(fmt.Sprintf("got status %v trying to query %v.%s:5000", resp.Status, i, http_cache_url)) - } - var s httpstats - data, err := io.ReadAll(resp.Body) - if err != nil { - panic(err) - } - err = json.Unmarshal(data, &s) - if err != nil { - panic(err) - } - if s.Up == false { - panic("gateway is not up yet, skipping stats collection") - } - values = append(values, s) - } - - return values -} - -type rstatval struct { - GuildCount int `json:"GuildCount"` - ChannelCount int `json:"ChannelCount"` -} - -func run_redis_query() []rstatval { - cmd := rdb.HGetAll(context.Background(), "pluralkit:cluster_stats") - if err := cmd.Err(); err != nil { - panic(err) - } - - res, err := cmd.Result() - if err != nil { - panic(err) - } - - var values []rstatval - - for _, data := range res { - var tmp rstatval - if err = json.Unmarshal([]byte(data), &tmp); err != nil { - panic(err) - } - - values = append(values, tmp) - } - - return values -} - -func get_message_count() int { - var count int - row := messages_db.QueryRow(context.Background(), "select count(*) as count from messages") - if err := row.Scan(&count); err != nil { - panic(err) - } - return count -} - -func get_image_cleanup_queue_length() int { - var count int - row := data_db.QueryRow(context.Background(), "select count(*) as count from image_cleanup_jobs") - if err := row.Scan(&count); err != nil { - panic(err) - } - return count -} - -func run_data_stats_query() map[string]interface{} { - s := map[string]interface{}{} - - rows, err := data_db.Query(context.Background(), "select * from info") - if err != nil { - panic(err) - } - descs := rows.FieldDescriptions() - - for rows.Next() { - for i, column := range descs { - if string(column.Name) == "message_count" { - continue - } - values, err := rows.Values() - if err != nil { - panic(err) - } - - s[string(column.Name)] = values[i] - } - } - - return s -} - -func do_stats_insert(table string, value int64) { - if stats_db == nil { - return - } - - sql := fmt.Sprintf("insert into %s values (now(), $1)", table) - log.Println("stats db query:", sql, "value:", value) - _, err := stats_db.Exec(context.Background(), sql, value) - if err != nil { - panic(err) - } -} diff --git a/services/scheduled_tasks/src/main.rs b/services/scheduled_tasks/src/main.rs new file mode 100644 index 00000000..10aa2ea8 --- /dev/null +++ b/services/scheduled_tasks/src/main.rs @@ -0,0 +1,136 @@ +use chrono::{TimeDelta, Timelike}; +use fred::prelude::RedisPool; +use sqlx::PgPool; +use std::time::Duration; +use tokio::task::JoinSet; +use tracing::{error, info}; + +mod tasks; +use tasks::*; + +#[derive(Clone)] +pub struct AppCtx { + pub data: PgPool, + pub messages: PgPool, + pub stats: PgPool, + pub redis: RedisPool, +} + +libpk::main!("scheduled_tasks"); +async fn real_main() -> anyhow::Result<()> { + let ctx = AppCtx { + data: libpk::db::init_data_db().await?, + messages: libpk::db::init_messages_db().await?, + stats: libpk::db::init_stats_db().await?, + redis: libpk::db::init_redis().await?, + }; + + info!("starting scheduled tasks runner"); + + let mut set = JoinSet::new(); + + // i couldn't be bothered to figure out the types of passing in an async + // function to another function... so macro it is + macro_rules! doforever { + ($timeout:expr, $desc:expr, $fn:ident) => { + let ctx = ctx.clone(); + set.spawn(tokio::spawn(async move { + loop { + let ctx = ctx.clone(); + wait_interval($timeout).await; + info!("running {}", $desc); + let before = std::time::Instant::now(); + if let Err(error) = $fn(ctx).await { + error!("failed to run {}: {}", $desc, error); + // sentry + } + let duration = before.elapsed(); + info!("ran {} in {duration:?}", $desc); + // add prometheus log + } + })) + }; + } + + doforever!(10, "prometheus updater", update_prometheus); + doforever!(60, "database stats updater", update_db_meta); + doforever!(600, "message stats updater", update_db_message_meta); + doforever!(60, "discord stats updater", update_discord_stats); + doforever!( + 1800, + "queue deleted image cleanup job", + queue_deleted_image_cleanup + ); + + set.join_next() + .await + .ok_or(anyhow::anyhow!("could not join_next"))???; + + Ok(()) +} + +// cron except not really +async fn wait_interval(interval_secs: u32) { + let now = chrono::Utc::now(); + + // for now, only supports hardcoded intervals + let next_iter_time = match interval_secs { + // 10 sec + // at every [x]0th second (00, 10, 20, 30, 40, 50) + 10 => { + let mut minute = 0; + let mut second = (now.second() - (now.second() % 10)) + 10; + if second == 60 { + minute += 1; + second = 0; + } + now.checked_add_signed(TimeDelta::minutes(minute)) + .expect("invalid time") + .with_second(second) + .expect("invalid time") + } + // 1 minute + // every minute at :00 seconds + 60 => now + .checked_add_signed(TimeDelta::minutes(1)) + .expect("invalid time") + .with_second(0) + .expect("invalid time"), + // 10 minutes + // at every [x]0 minute (00, 10, 20, 30, 40, 50) + 600 => { + let mut minute = (now.minute() + 10) % 10; + let mut hour = 0; + if minute == 60 { + minute = 0; + hour = 1; + } + + now.checked_add_signed(TimeDelta::hours(hour)) + .expect("invalid time") + .with_minute(minute) + .expect("invalid time") + } + // 30 minutes + // at :00 and :30 + 1800 => { + let mut minute = (now.minute() + 30) % 30; + let mut hour = 0; + if minute == 60 { + minute = 0; + hour = 1; + } + + now.checked_add_signed(TimeDelta::hours(hour)) + .expect("invalid time") + .with_minute(minute) + .expect("invalid time") + } + + _ => unreachable!(), + }; + + let dur = next_iter_time - now; + + tokio::time::sleep(Duration::from_secs(dur.num_seconds() as u64)).await; +} diff --git a/services/scheduled_tasks/src/tasks.rs b/services/scheduled_tasks/src/tasks.rs new file mode 100644 index 00000000..0d7f5b9c --- /dev/null +++ b/services/scheduled_tasks/src/tasks.rs @@ -0,0 +1,151 @@ +use std::time::Duration; + +use anyhow::anyhow; +use fred::prelude::KeysInterface; +use libpk::{ + config, + db::repository::{get_stats, insert_stats}, +}; +use metrics::gauge; +use num_format::{Locale, ToFormattedString}; +use reqwest::ClientBuilder; +use sqlx::Executor; + +use crate::AppCtx; + +pub async fn update_prometheus(ctx: AppCtx) -> anyhow::Result<()> { + #[derive(sqlx::FromRow)] + struct Count { + count: i64, + } + let count: Count = sqlx::query_as("select count(*) from image_cleanup_jobs") + .fetch_one(&ctx.data) + .await?; + + gauge!("pluralkit_image_cleanup_queue_length").set(count.count as f64); + + // todo: remaining shard session_start_limit + Ok(()) +} + +pub async fn update_db_meta(ctx: AppCtx) -> anyhow::Result<()> { + ctx.data + .execute( + r#" + update info set + system_count = (select count(*) from systems), + member_count = (select count(*) from systems), + group_count = (select count(*) from systems), + switch_count = (select count(*) from systems) + "#, + ) + .await?; + let new_stats = get_stats(&ctx.data).await?; + insert_stats(&ctx.stats, "systems", new_stats.system_count).await?; + insert_stats(&ctx.stats, "members", new_stats.member_count).await?; + insert_stats(&ctx.stats, "groups", new_stats.group_count).await?; + insert_stats(&ctx.stats, "switches", new_stats.switch_count).await?; + Ok(()) +} + +pub async fn update_db_message_meta(ctx: AppCtx) -> anyhow::Result<()> { + #[derive(sqlx::FromRow)] + struct MessageCount { + count: i64, + } + let message_count: MessageCount = sqlx::query_as("select count(*) from messages") + .fetch_one(&ctx.messages) + .await?; + sqlx::query("update info set message_count = $1") + .bind(message_count.count) + .execute(&ctx.data) + .await?; + insert_stats(&ctx.stats, "messages", message_count.count).await?; + + Ok(()) +} + +pub async fn update_discord_stats(ctx: AppCtx) -> anyhow::Result<()> { + let client = ClientBuilder::new() + .connect_timeout(Duration::from_secs(3)) + .timeout(Duration::from_secs(3)) + .build() + .expect("error making client"); + + let cfg = config + .scheduled_tasks + .as_ref() + .expect("missing scheduled_tasks config"); + + #[derive(serde::Deserialize)] + struct GatewayStatus { + up: bool, + guild_count: i64, + channel_count: i64, + } + + let mut guild_count = 0; + let mut channel_count = 0; + + for idx in 0..=cfg.expected_gateway_count { + let res = client + .get(format!("http://cluster{idx}.{}/stats", cfg.gateway_url)) + .send() + .await?; + + let stat: GatewayStatus = res.json().await?; + + if !stat.up { + return Err(anyhow!("cluster {idx} is not up")); + } + + guild_count += stat.guild_count; + channel_count += stat.channel_count; + } + + insert_stats(&ctx.stats, "guilds", guild_count).await?; + insert_stats(&ctx.stats, "channels", channel_count).await?; + + if cfg.set_guild_count { + ctx.redis + .set::<(), &str, String>( + "pluralkit:botstatus", + format!( + "in {} servers", + guild_count.to_formatted_string(&Locale::en) + ), + None, + None, + false, + ) + .await?; + } + + Ok(()) +} + +pub async fn queue_deleted_image_cleanup(ctx: AppCtx) -> anyhow::Result<()> { + // todo: we want to delete immediately when system is deleted, but after a + // delay if member is deleted + ctx.data + .execute( + r#" +insert into image_cleanup_jobs +select id from images where + not exists (select from image_cleanup_jobs j where j.id = images.id) + and not exists (select from systems where avatar_url = images.url) + and not exists (select from systems where banner_image = images.url) + and not exists (select from system_guild where avatar_url = images.url) + + and not exists (select from members where avatar_url = images.url) + and not exists (select from members where banner_image = images.url) + and not exists (select from members where webhook_avatar_url = images.url) + and not exists (select from member_guild where avatar_url = images.url) + + and not exists (select from groups where icon = images.url) + and not exists (select from groups where banner_image = images.url); + "#, + ) + .await?; + Ok(()) +} diff --git a/services/scheduled_tasks/tasks.go b/services/scheduled_tasks/tasks.go deleted file mode 100644 index a1404682..00000000 --- a/services/scheduled_tasks/tasks.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log" - - "golang.org/x/text/language" - "golang.org/x/text/message" -) - -var table_stat_keys = []string{"system", "member", "group", "switch"} - -func plural(key string) string { - if key[len(key)-1] == 'h' { - return key + "es" - } - return key + "s" -} - -func update_prom() { - count := get_image_cleanup_queue_length() - cleanupQueueLength.Set(float64(count)) -} - -func update_db_meta() { - for _, key := range table_stat_keys { - q := fmt.Sprintf("update info set %s_count = (select count(*) from %s)", key, plural(key)) - log.Println("data db query:", q) - run_simple_pg_query(data_db, q) - } - - data_stats := run_data_stats_query() - for _, key := range table_stat_keys { - val := data_stats[key+"_count"].(int64) - log.Printf("%v: %v\n", key+"_count", val) - do_stats_insert(plural(key), val) - } -} - -func update_db_message_meta() { - count := get_message_count() - - _, err := data_db.Exec(context.Background(), "update info set message_count = $1", count) - if err != nil { - panic(err) - } - - do_stats_insert("messages", int64(count)) -} - -func update_discord_stats() { - redisStats := query_http_cache() - - guild_count := 0 - channel_count := 0 - - for _, v := range redisStats { - log.Println(v.GuildCount, v.ChannelCount) - guild_count += v.GuildCount - channel_count += v.ChannelCount - } - - do_stats_insert("guilds", int64(guild_count)) - do_stats_insert("channels", int64(channel_count)) - - if !set_guild_count { - return - } - - p := message.NewPrinter(language.English) - s := p.Sprintf("%d", guild_count) - - cmd := rdb.Set(context.Background(), "pluralkit:botstatus", "in "+s+" servers", 0) - if err := cmd.Err(); err != nil { - panic(err) - } -} - -// MUST add new image columns here -var deletedImageCleanupQuery = ` -insert into image_cleanup_jobs -select id from images where - not exists (select from image_cleanup_jobs j where j.id = images.id) - and not exists (select from systems where avatar_url = images.url) - and not exists (select from systems where banner_image = images.url) - and not exists (select from system_guild where avatar_url = images.url) - - and not exists (select from members where avatar_url = images.url) - and not exists (select from members where banner_image = images.url) - and not exists (select from members where webhook_avatar_url = images.url) - and not exists (select from member_guild where avatar_url = images.url) - - and not exists (select from groups where icon = images.url) - and not exists (select from groups where banner_image = images.url); - ` - -func queue_deleted_image_cleanup() { - _, err := data_db.Exec(context.Background(), deletedImageCleanupQuery) - if err != nil { - panic(err) - } -}