From 17f5561293a4d08736262cd58759d603040700d5 Mon Sep 17 00:00:00 2001 From: alyssa Date: Mon, 21 Oct 2024 11:42:32 +0900 Subject: [PATCH] chore: merge avatars service into monorepo --- .github/workflows/rust.yml | 3 +- Cargo.lock | 746 ++++++++++++++++-- Cargo.toml | 14 +- Dockerfile.rust | 3 + lib/libpk/Cargo.toml | 2 + lib/libpk/src/_config.rs | 37 +- lib/libpk/src/db/mod.rs | 1 + lib/libpk/src/db/repository/avatars.rs | 87 ++ lib/libpk/src/db/repository/mod.rs | 2 + lib/libpk/src/db/types/avatars.rs | 53 ++ lib/libpk/src/db/types/mod.rs | 1 + lib/libpk/src/lib.rs | 2 +- services/api/src/main.rs | 4 +- services/api/src/middleware/ratelimit.rs | 55 +- services/avatars/Cargo.toml | 25 + services/avatars/src/hash.rs | 21 + services/avatars/src/init.sql | 24 + services/avatars/src/main.rs | 259 ++++++ services/avatars/src/migrate.rs | 146 ++++ services/avatars/src/process.rs | 257 ++++++ services/avatars/src/pull.rs | 166 ++++ services/avatars/src/store.rs | 60 ++ services/dispatch/Cargo.toml | 2 +- services/gateway/src/cache_api.rs | 8 +- services/gateway/src/discord/cache.rs | 34 +- services/gateway/src/discord/gateway.rs | 18 +- .../gateway/src/discord/identify_queue.rs | 6 +- 27 files changed, 1925 insertions(+), 111 deletions(-) create mode 100644 lib/libpk/src/db/repository/avatars.rs create mode 100644 lib/libpk/src/db/types/avatars.rs create mode 100644 lib/libpk/src/db/types/mod.rs create mode 100644 services/avatars/Cargo.toml create mode 100644 services/avatars/src/hash.rs create mode 100644 services/avatars/src/init.sql create mode 100644 services/avatars/src/main.rs create mode 100644 services/avatars/src/migrate.rs create mode 100644 services/avatars/src/process.rs create mode 100644 services/avatars/src/pull.rs create mode 100644 services/avatars/src/store.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 584ab179..d7e55963 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -10,6 +10,7 @@ on: - 'lib/libpk/**' - 'services/api/**' - 'services/gateway/**' + - 'services/avatars/**' - '.github/workflows/rust.yml' - 'Dockerfile.rust' - 'Dockerfile.bin' @@ -47,7 +48,7 @@ jobs: # add more binaries here - run: | - for binary in "api" "gateway"; do + for binary in "api" "gateway" "avatars"; do for tag in latest ${{ env.BRANCH_NAME }} ${{ github.sha }}; do cat Dockerfile.bin | sed "s/__BINARY__/$binary/g" | docker build -t ghcr.io/pluralkit/$binary:$tag -f - . done diff --git a/Cargo.lock b/Cargo.lock index f2936d41..b5e711b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,23 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.11" @@ -127,6 +144,22 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attohttpc" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" +dependencies = [ + "http 0.2.8", + "log", + "rustls 0.20.9", + "serde", + "serde_json", + "url", + "webpki", + "webpki-roots 0.22.6", +] + [[package]] name = "atty" version = "0.2.14" @@ -144,6 +177,57 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "avatars" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum 0.7.5", + "data-encoding", + "form_urlencoded", + "futures", + "gif", + "image", + "libpk", + "reqwest 0.12.8", + "rust-s3", + "serde", + "sha2", + "sqlx", + "thiserror", + "time", + "tokio", + "tracing", + "uuid", + "webp", +] + +[[package]] +name = "aws-creds" +version = "0.34.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" +dependencies = [ + "attohttpc", + "dirs", + "log", + "quick-xml", + "rust-ini 0.18.0", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-region" +version = "0.25.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9aed3f9c7eac9be28662fdb3b0f4d1951e812f7c64fed4f0327ba702f459b3b" +dependencies = [ + "thiserror", +] + [[package]] name = "axum" version = "0.6.7" @@ -259,11 +343,17 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -321,6 +411,12 @@ version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +[[package]] +name = "bytemuck" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" + [[package]] name = "byteorder" version = "1.5.0" @@ -348,6 +444,11 @@ name = "cc" version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -369,6 +470,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "config" version = "0.14.0" @@ -382,7 +489,7 @@ dependencies = [ "nom", "pathdiff", "ron", - "rust-ini", + "rust-ini 0.19.0", "serde", "serde_json", "toml", @@ -566,6 +673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -589,6 +697,26 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dispatch" version = "0.1.0" @@ -596,7 +724,7 @@ dependencies = [ "anyhow", "axum 0.7.5", "hickory-client", - "reqwest", + "reqwest 0.12.8", "serde", "serde_json", "tokio", @@ -604,6 +732,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "dlv-list" version = "0.5.2" @@ -628,6 +762,15 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -698,6 +841,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +[[package]] +name = "fdeflate" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8090f921a24b04994d9929e204f50b498a33ea6ba559ffaa05e04f7ee7fb5ab" +dependencies = [ + "simd-adler32", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -712,7 +864,7 @@ checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.7.4", ] [[package]] @@ -934,12 +1086,47 @@ dependencies = [ "wasi", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.8", + "indexmap", + "slab", + "tokio", + "tokio-util 0.7.12", + "tracing", +] + [[package]] name = "h2" version = "0.4.6" @@ -964,6 +1151,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" @@ -977,7 +1167,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash", + "ahash 0.8.11", "allocator-api2", ] @@ -1209,6 +1399,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2 0.3.26", "http 0.2.8", "http-body 0.4.5", "httparse", @@ -1231,7 +1422,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", + "h2 0.4.6", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -1243,6 +1434,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.8", + "hyper 0.14.24", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.26.0" @@ -1276,7 +1481,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.0", "tower-service", - "webpki-roots", + "webpki-roots 0.26.6", ] [[package]] @@ -1342,6 +1547,22 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-traits", + "png", + "tiff", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -1382,6 +1603,21 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.69" @@ -1413,9 +1649,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libm" @@ -1439,11 +1675,23 @@ dependencies = [ "prost-types", "serde", "sqlx", + "time", "tokio", "tracing", "tracing-gelf", "tracing-subscriber", "twilight-model", + "uuid", +] + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.5.0", + "libc", ] [[package]] @@ -1457,6 +1705,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libwebp-sys" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54cd30df7c7165ce74a456e4ca9732c603e8dc5e60784558c1c6dc047f876733" +dependencies = [ + "cc", + "glob", +] + [[package]] name = "libz-sys" version = "1.1.18" @@ -1520,6 +1778,17 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +[[package]] +name = "maybe-async" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1530,6 +1799,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.5.0" @@ -1551,7 +1826,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884adb57038347dfbaf2d5065887b6cf4312330dc8e94bc30a1a839bd79d3261" dependencies = [ - "ahash", + "ahash 0.8.11", "portable-atomic", ] @@ -1611,6 +1886,16 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.0.2" @@ -1757,13 +2042,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list 0.3.0", + "hashbrown 0.12.3", +] + [[package]] name = "ordered-multimap" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" dependencies = [ - "dlv-list", + "dlv-list 0.5.2", "hashbrown 0.13.2", ] @@ -1961,6 +2256,19 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "png" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.0", +] + [[package]] name = "portable-atomic" version = "1.8.0" @@ -2083,10 +2391,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] -name = "quinn" -version = "0.11.3" +name = "quick-xml" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", @@ -2102,13 +2420,13 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand", - "ring", + "ring 0.17.8", "rustc-hash", "rustls 0.23.10", "slab", @@ -2119,15 +2437,15 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ "libc", "once_cell", "socket2 0.5.7", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2220,6 +2538,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.9.4" @@ -2266,9 +2595,52 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" -version = "0.12.7" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.8", + "http-body 0.4.5", + "hyper 0.14.24", + "hyper-rustls 0.24.2", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-rustls 0.24.1", + "tokio-util 0.7.12", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 0.25.4", + "winreg", +] + +[[package]] +name = "reqwest" +version = "0.12.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64 0.22.1", "bytes", @@ -2289,7 +2661,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.10", - "rustls-pemfile", + "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", "serde_json", @@ -2302,7 +2674,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.26.6", "windows-registry", ] @@ -2320,6 +2692,21 @@ dependencies = [ "tower-service", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + [[package]] name = "ring" version = "0.17.8" @@ -2331,7 +2718,7 @@ dependencies = [ "getrandom", "libc", "spin 0.9.8", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -2367,6 +2754,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap 0.4.3", +] + [[package]] name = "rust-ini" version = "0.19.0" @@ -2374,7 +2771,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" dependencies = [ "cfg-if", - "ordered-multimap", + "ordered-multimap 0.6.0", +] + +[[package]] +name = "rust-s3" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.1", + "bytes", + "cfg-if", + "futures", + "hex", + "hmac", + "http 0.2.8", + "log", + "maybe-async", + "md5", + "percent-encoding", + "quick-xml", + "reqwest 0.11.27", + "serde", + "serde_derive", + "sha2", + "thiserror", + "time", + "tokio", + "tokio-stream", + "url", ] [[package]] @@ -2402,15 +2831,39 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +dependencies = [ + "log", + "ring 0.16.20", + "sct", + "webpki", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ - "ring", + "ring 0.17.8", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.4", "subtle", "zeroize", ] @@ -2422,9 +2875,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" dependencies = [ "once_cell", - "ring", + "ring 0.17.8", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.4", "subtle", "zeroize", ] @@ -2436,12 +2889,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.2", "rustls-pki-types", "schannel", "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.1.2" @@ -2454,9 +2916,19 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] [[package]] name = "rustls-webpki" @@ -2464,9 +2936,9 @@ version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ - "ring", + "ring 0.17.8", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -2496,6 +2968,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + [[package]] name = "security-framework" version = "2.11.0" @@ -2639,9 +3121,9 @@ checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -2686,6 +3168,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simdutf8" version = "0.1.4" @@ -2787,11 +3275,10 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" dependencies = [ - "ahash", + "ahash 0.8.11", "atoi", "byteorder", "bytes", - "chrono", "crc", "crossbeam-queue", "either", @@ -2815,10 +3302,12 @@ dependencies = [ "smallvec", "sqlformat", "thiserror", + "time", "tokio", "tokio-stream", "tracing", "url", + "uuid", ] [[package]] @@ -2871,7 +3360,6 @@ dependencies = [ "bitflags 2.5.0", "byteorder", "bytes", - "chrono", "crc", "digest 0.10.7", "dotenvy", @@ -2899,7 +3387,9 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", + "time", "tracing", + "uuid", "whoami", ] @@ -2913,7 +3403,6 @@ dependencies = [ "base64 0.21.7", "bitflags 2.5.0", "byteorder", - "chrono", "crc", "dotenvy", "etcetera", @@ -2938,7 +3427,9 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", + "time", "tracing", + "uuid", "whoami", ] @@ -2949,7 +3440,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" dependencies = [ "atoi", - "chrono", "flume", "futures-channel", "futures-core", @@ -2961,9 +3451,11 @@ dependencies = [ "percent-encoding", "serde", "sqlx-core", + "time", "tracing", "url", "urlencoding", + "uuid", ] [[package]] @@ -3020,6 +3512,27 @@ dependencies = [ "futures-core", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -3043,22 +3556,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.66", ] [[package]] @@ -3070,6 +3583,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.36" @@ -3077,6 +3601,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde", @@ -3153,6 +3678,16 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.25.0" @@ -3226,7 +3761,7 @@ dependencies = [ "futures-sink", "http 1.1.0", "httparse", - "ring", + "ring 0.17.8", "rustls-native-certs", "rustls-pki-types", "sha1_smol", @@ -3338,11 +3873,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -3351,20 +3885,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.66", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -3597,6 +4131,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -3620,6 +4160,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "uuid" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +dependencies = [ + "serde", +] + [[package]] name = "valuable" version = "0.1.0" @@ -3727,24 +4276,78 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] -name = "web-sys" -version = "0.3.61" +name = "wasm-streams" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", ] [[package]] -name = "webpki-roots" -version = "0.26.3" +name = "webp" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "4bb5d8e7814e92297b0e1c773ce43d290bef6c17452dafd9fc49e5edb5beba71" +dependencies = [ + "image", + "libwebp-sys", +] + +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "webpki-roots" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "whoami" version = "1.5.1" @@ -3867,6 +4470,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.42.1" @@ -4054,6 +4666,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index 9428e560..11ef9be6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,8 @@ members = [ "./lib/libpk", "./services/api", "./services/dispatch", - "./services/gateway" + "./services/gateway", + "./services/avatars" ] [workspace.dependencies] @@ -16,13 +17,16 @@ fred = { version = "5.2.0", default-features = false, features = ["tracing", "po futures = "0.3.30" lazy_static = "1.4.0" metrics = "0.23.0" -serde = "1.0.152" +reqwest = { version = "0.12.7" , default-features = false, features = ["rustls-tls", "trust-dns"]} +serde = { version = "1.0.196", features = ["derive"] } serde_json = "1.0.117" signal-hook = "0.3.17" -sqlx = { version = "0.7.4", features = ["runtime-tokio", "postgres", "chrono", "macros"] } -tokio = { version = "1.25.0", features = ["full"] } -tracing = "0.1.37" +sqlx = { version = "0.7.4", features = ["runtime-tokio", "postgres", "time", "macros", "uuid"] } +time = "0.3.34" +tokio = { version = "1.36.0", features = ["full"] } +tracing = "0.1.40" tracing-subscriber = { version = "0.3.16", features = ["env-filter", "json"] } +uuid = { version = "1.7.0", features = ["serde"] } twilight-gateway = { git = "https://github.com/pluralkit/twilight" } twilight-cache-inmemory = { git = "https://github.com/pluralkit/twilight", features = ["permission-calculator"] } diff --git a/Dockerfile.rust b/Dockerfile.rust index 84a33453..d1fdd997 100644 --- a/Dockerfile.rust +++ b/Dockerfile.rust @@ -28,11 +28,14 @@ COPY proto/ /build/proto COPY lib/libpk /build/lib/libpk COPY services/api/ /build/services/api COPY services/gateway/ /build/services/gateway +COPY services/avatars/ /build/services/avatars RUN cargo build --bin api --release --target x86_64-unknown-linux-musl RUN cargo build --bin gateway --release --target x86_64-unknown-linux-musl +RUN cargo build --bin avatars --release --target x86_64-unknown-linux-musl FROM scratch COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/api /api COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/gateway /gateway +COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/avatars /avatars diff --git a/lib/libpk/Cargo.toml b/lib/libpk/Cargo.toml index 4372a68c..e3fb52da 100644 --- a/lib/libpk/Cargo.toml +++ b/lib/libpk/Cargo.toml @@ -13,11 +13,13 @@ metrics = { workspace = true } metrics-exporter-prometheus = { version = "0.15.3", default-features = false, features = ["tokio", "http-listener", "tracing"] } serde = { workspace = true } sqlx = { workspace = true } +time = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-gelf = "0.7.1" tracing-subscriber = { workspace = true} twilight-model = { workspace = true } +uuid = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } diff --git a/lib/libpk/src/_config.rs b/lib/libpk/src/_config.rs index 7c1364c1..d138df1b 100644 --- a/lib/libpk/src/_config.rs +++ b/lib/libpk/src/_config.rs @@ -20,6 +20,9 @@ pub struct DiscordConfig { pub max_concurrency: u32, pub cluster: Option, pub api_base_url: Option, + + #[serde(default = "_default_api_addr")] + pub cache_api_addr: String, } #[derive(Deserialize, Debug)] @@ -36,7 +39,7 @@ fn _default_api_addr() -> String { "0.0.0.0:5000".to_string() } -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Clone, Debug)] pub struct ApiConfig { #[serde(default = "_default_api_addr")] pub addr: String, @@ -50,6 +53,23 @@ pub struct ApiConfig { pub temp_token2: Option, } +#[derive(Deserialize, Clone, Debug)] +pub struct AvatarsConfig { + pub s3: S3Config, + pub cdn_url: String, + + #[serde(default)] + pub migrate_worker_count: u32, +} + +#[derive(Deserialize, Clone, Debug)] +pub struct S3Config { + pub bucket: String, + pub application_id: String, + pub application_key: String, + pub endpoint: String, +} + fn _metrics_default() -> bool { false } @@ -61,8 +81,9 @@ fn _json_log_default() -> bool { pub struct PKConfig { pub db: DatabaseConfig, - pub discord: DiscordConfig, - pub api: ApiConfig, + pub discord: Option, + pub api: Option, + pub avatars: Option, #[serde(default = "_metrics_default")] pub run_metrics_server: bool, @@ -71,6 +92,16 @@ pub struct PKConfig { pub(crate) json_log: bool, } +impl PKConfig { + pub fn api(self) -> ApiConfig { + self.api.expect("missing api config") + } + + pub fn discord_config(self) -> DiscordConfig { + self.discord.expect("missing discord config") + } +} + lazy_static! { #[derive(Debug)] pub static ref CONFIG: Arc = { diff --git a/lib/libpk/src/db/mod.rs b/lib/libpk/src/db/mod.rs index 207849bd..56feaa4f 100644 --- a/lib/libpk/src/db/mod.rs +++ b/lib/libpk/src/db/mod.rs @@ -4,6 +4,7 @@ use std::str::FromStr; use tracing::info; pub mod repository; +pub mod types; pub async fn init_redis() -> anyhow::Result { info!("connecting to redis"); diff --git a/lib/libpk/src/db/repository/avatars.rs b/lib/libpk/src/db/repository/avatars.rs new file mode 100644 index 00000000..cb77ee8c --- /dev/null +++ b/lib/libpk/src/db/repository/avatars.rs @@ -0,0 +1,87 @@ +use sqlx::{PgPool, Postgres, Transaction}; + +use crate::db::types::avatars::*; + +pub async fn get_by_original_url( + pool: &PgPool, + original_url: &str, +) -> anyhow::Result> { + Ok( + sqlx::query_as("select * from images where original_url = $1") + .bind(original_url) + .fetch_optional(pool) + .await?, + ) +} + +pub async fn get_by_attachment_id( + pool: &PgPool, + attachment_id: u64, +) -> anyhow::Result> { + Ok( + sqlx::query_as("select * from images where original_attachment_id = $1") + .bind(attachment_id as i64) + .fetch_optional(pool) + .await?, + ) +} + +pub async fn pop_queue( + pool: &PgPool, +) -> anyhow::Result, ImageQueueEntry)>> { + let mut tx = pool.begin().await?; + let res: Option = sqlx::query_as("delete from image_queue where itemid = (select itemid from image_queue order by itemid for update skip locked limit 1) returning *") + .fetch_optional(&mut *tx).await?; + Ok(res.map(|x| (tx, x))) +} + +pub async fn get_queue_length(pool: &PgPool) -> anyhow::Result { + Ok(sqlx::query_scalar("select count(*) from image_queue") + .fetch_one(pool) + .await?) +} + +pub async fn get_stats(pool: &PgPool) -> anyhow::Result { + Ok(sqlx::query_as( + "select count(*) as total_images, sum(file_size) as total_file_size from images", + ) + .fetch_one(pool) + .await?) +} + +pub async fn add_image(pool: &PgPool, meta: ImageMeta) -> anyhow::Result { + let kind_str = match meta.kind { + ImageKind::Avatar => "avatar", + ImageKind::Banner => "banner", + }; + + let res = sqlx::query("insert into images (id, url, content_type, original_url, file_size, width, height, original_file_size, original_type, original_attachment_id, kind, uploaded_by_account, uploaded_by_system, uploaded_at) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, (now() at time zone 'utc')) on conflict (id) do nothing") + .bind(meta.id) + .bind(meta.url) + .bind(meta.content_type) + .bind(meta.original_url) + .bind(meta.file_size) + .bind(meta.width) + .bind(meta.height) + .bind(meta.original_file_size) + .bind(meta.original_type) + .bind(meta.original_attachment_id) + .bind(kind_str) + .bind(meta.uploaded_by_account) + .bind(meta.uploaded_by_system) + .execute(pool).await?; + Ok(res.rows_affected() > 0) +} + +pub async fn push_queue( + conn: &mut sqlx::PgConnection, + url: &str, + kind: ImageKind, +) -> anyhow::Result<()> { + sqlx::query("insert into image_queue (url, kind) values ($1, $2)") + .bind(url) + .bind(kind) + .execute(conn) + .await?; + Ok(()) +} diff --git a/lib/libpk/src/db/repository/mod.rs b/lib/libpk/src/db/repository/mod.rs index ae0ae7b9..9c6ba214 100644 --- a/lib/libpk/src/db/repository/mod.rs +++ b/lib/libpk/src/db/repository/mod.rs @@ -1,5 +1,7 @@ mod stats; pub use stats::*; +pub mod avatars; + mod auth; pub use auth::*; diff --git a/lib/libpk/src/db/types/avatars.rs b/lib/libpk/src/db/types/avatars.rs new file mode 100644 index 00000000..928c3e99 --- /dev/null +++ b/lib/libpk/src/db/types/avatars.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use time::OffsetDateTime; +use uuid::Uuid; + +#[derive(FromRow)] +pub struct ImageMeta { + pub id: String, + pub kind: ImageKind, + pub content_type: String, + pub url: String, + pub file_size: i32, + pub width: i32, + pub height: i32, + pub uploaded_at: Option, + + pub original_url: Option, + pub original_attachment_id: Option, + pub original_file_size: Option, + pub original_type: Option, + pub uploaded_by_account: Option, + pub uploaded_by_system: Option, +} + +#[derive(FromRow, Serialize)] +pub struct Stats { + pub total_images: i64, + pub total_file_size: i64, +} + +#[derive(Serialize, Deserialize, Clone, Copy, Debug, sqlx::Type, PartialEq)] +#[serde(rename_all = "snake_case")] +#[sqlx(rename_all = "snake_case", type_name = "text")] +pub enum ImageKind { + Avatar, + Banner, +} + +impl ImageKind { + pub fn size(&self) -> (u32, u32) { + match self { + Self::Avatar => (512, 512), + Self::Banner => (1024, 1024), + } + } +} + +#[derive(FromRow)] +pub struct ImageQueueEntry { + pub itemid: i32, + pub url: String, + pub kind: ImageKind, +} diff --git a/lib/libpk/src/db/types/mod.rs b/lib/libpk/src/db/types/mod.rs new file mode 100644 index 00000000..03f04d0a --- /dev/null +++ b/lib/libpk/src/db/types/mod.rs @@ -0,0 +1 @@ +pub mod avatars; diff --git a/lib/libpk/src/lib.rs b/lib/libpk/src/lib.rs index 9b945db4..c474bada 100644 --- a/lib/libpk/src/lib.rs +++ b/lib/libpk/src/lib.rs @@ -1,5 +1,5 @@ use metrics_exporter_prometheus::PrometheusBuilder; -use tracing_subscriber::{EnvFilter, Registry}; +use tracing_subscriber::EnvFilter; pub mod db; pub mod proto; diff --git a/services/api/src/main.rs b/services/api/src/main.rs index a8b5a9ff..b47c9218 100644 --- a/services/api/src/main.rs +++ b/services/api/src/main.rs @@ -63,7 +63,7 @@ async fn main() -> anyhow::Result<()> { let db = libpk::db::init_data_db().await?; let redis = libpk::db::init_redis().await?; - let rproxy_uri = Uri::from_static(&libpk::config.api.remote_url).to_string(); + let rproxy_uri = Uri::from_static(&libpk::config.api.as_ref().expect("missing api config").remote_url).to_string(); let rproxy_client = hyper_util::client::legacy::Client::<(), ()>::builder(TokioExecutor::new()) .build(HttpConnector::new()); @@ -145,7 +145,7 @@ async fn main() -> anyhow::Result<()> { .route("/", get(|| async { axum::response::Redirect::to("https://pluralkit.me/api") })); - let addr: &str = libpk::config.api.addr.as_ref(); + let addr: &str = libpk::config.api.as_ref().expect("missing api config").addr.as_ref(); let listener = tokio::net::TcpListener::bind(addr).await?; info!("listening on {}", addr); axum::serve(listener, app).await?; diff --git a/services/api/src/middleware/ratelimit.rs b/services/api/src/middleware/ratelimit.rs index 05e55815..e5da214d 100644 --- a/services/api/src/middleware/ratelimit.rs +++ b/services/api/src/middleware/ratelimit.rs @@ -20,32 +20,38 @@ lazy_static::lazy_static! { // this is awful but it works pub fn ratelimiter(f: F) -> FromFnLayer, T> { - let redis = libpk::config.api.ratelimit_redis_addr.as_ref().map(|val| { - let r = fred::pool::RedisPool::new( - fred::types::RedisConfig::from_url_centralized(val.as_ref()) - .expect("redis url is invalid"), - 10, - ) - .expect("failed to connect to redis"); + let redis = libpk::config + .api + .as_ref() + .expect("missing api config") + .ratelimit_redis_addr + .as_ref() + .map(|val| { + let r = fred::pool::RedisPool::new( + fred::types::RedisConfig::from_url_centralized(val.as_ref()) + .expect("redis url is invalid"), + 10, + ) + .expect("failed to connect to redis"); - let handle = r.connect(Some(ReconnectPolicy::default())); + let handle = r.connect(Some(ReconnectPolicy::default())); - tokio::spawn(async move { handle }); + tokio::spawn(async move { handle }); - let rscript = r.clone(); - tokio::spawn(async move { - if let Ok(()) = rscript.wait_for_connect().await { - match rscript.script_load(LUA_SCRIPT).await { - Ok(_) => info!("connected to redis for request rate limiting"), - Err(err) => error!("could not load redis script: {}", err), + let rscript = r.clone(); + tokio::spawn(async move { + if let Ok(()) = rscript.wait_for_connect().await { + match rscript.script_load(LUA_SCRIPT).await { + Ok(_) => info!("connected to redis for request rate limiting"), + Err(err) => error!("could not load redis script: {}", err), + } + } else { + error!("could not wait for connection to load redis script!"); } - } else { - error!("could not wait for connection to load redis script!"); - } - }); + }); - r - }); + r + }); if redis.is_none() { warn!("running without request rate limiting!"); @@ -95,7 +101,12 @@ pub async fn do_request_ratelimited( // https://github.com/rust-lang/rust/issues/53667 let is_temp_token2 = if let Some(header) = request.headers().clone().get("X-PluralKit-App") { - if let Some(token2) = &libpk::config.api.temp_token2 { + if let Some(token2) = &libpk::config + .api + .as_ref() + .expect("missing api config") + .temp_token2 + { if header.to_str().unwrap_or("invalid") == token2 { true } else { diff --git a/services/avatars/Cargo.toml b/services/avatars/Cargo.toml new file mode 100644 index 00000000..bb664f6e --- /dev/null +++ b/services/avatars/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "avatars" +version = "0.1.0" +edition = "2021" + +[dependencies] +libpk = { path = "../../lib/libpk" } +anyhow = { workspace = true } +axum = { workspace = true } +data-encoding = "2.5.0" +form_urlencoded = "1.2.1" +futures = { workspace = true } +gif = "0.13.1" +image = { version = "0.24.8", default-features = false, features = ["gif", "jpeg", "png", "webp", "tiff"] } +reqwest = { workspace = true } +rust-s3 = { version = "0.33.0", default-features = false, features = ["tokio-rustls-tls"] } +sha2 = "0.10.8" +serde = { workspace = true } +sqlx = { workspace = true } +thiserror = "1.0.56" +time = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } +webp = "0.2.6" diff --git a/services/avatars/src/hash.rs b/services/avatars/src/hash.rs new file mode 100644 index 00000000..e025c2fa --- /dev/null +++ b/services/avatars/src/hash.rs @@ -0,0 +1,21 @@ +use std::fmt::Display; + +use sha2::{Digest, Sha256}; + +#[derive(Debug)] +pub struct Hash([u8; 32]); + +impl Hash { + pub fn sha256(data: &[u8]) -> Hash { + let mut hasher = Sha256::new(); + hasher.update(data); + Hash(hasher.finalize().into()) + } +} + +impl Display for Hash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let encoding = data_encoding::BASE32_NOPAD; + write!(f, "{}", encoding.encode(&self.0[..16]).to_lowercase()) + } +} diff --git a/services/avatars/src/init.sql b/services/avatars/src/init.sql new file mode 100644 index 00000000..854c065b --- /dev/null +++ b/services/avatars/src/init.sql @@ -0,0 +1,24 @@ +create table if not exists images +( + id text primary key, + url text not null, + original_url text, + original_file_size int, + original_type text, + original_attachment_id bigint, + file_size int not null, + width int not null, + height int not null, + kind text not null, + uploaded_at timestamptz not null, + uploaded_by_account bigint +); + +create index if not exists images_original_url_idx on images (original_url); +create index if not exists images_original_attachment_id_idx on images (original_attachment_id); +create index if not exists images_uploaded_by_account_idx on images (uploaded_by_account); + +create table if not exists image_queue (itemid serial primary key, url text not null, kind text not null); + +alter table images add column if not exists uploaded_by_system uuid; +alter table images add column if not exists content_type text default 'image/webp'; \ No newline at end of file diff --git a/services/avatars/src/main.rs b/services/avatars/src/main.rs new file mode 100644 index 00000000..1f364fcc --- /dev/null +++ b/services/avatars/src/main.rs @@ -0,0 +1,259 @@ +mod hash; +mod migrate; +mod process; +mod pull; +mod store; + +use anyhow::Context; +use axum::extract::State; +use axum::routing::get; +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + routing::post, + Json, Router, +}; +use libpk::_config::AvatarsConfig; +use libpk::db::repository::avatars as db; +use libpk::db::types::avatars::*; +use reqwest::{Client, ClientBuilder}; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; +use std::error::Error; +use std::sync::Arc; +use std::time::Duration; +use thiserror::Error; +use tracing::{error, info}; +use uuid::Uuid; + +#[derive(Error, Debug)] +pub enum PKAvatarError { + // todo: split off into logical groups (cdn/url error, image format error, etc) + #[error("invalid cdn url")] + InvalidCdnUrl, + + #[error("discord cdn responded with status code: {0}")] + BadCdnResponse(reqwest::StatusCode), + + #[error("network error: {0}")] + NetworkError(reqwest::Error), + + #[error("response is missing header: {0}")] + MissingHeader(&'static str), + + #[error("unsupported content type: {0}")] + UnsupportedContentType(String), + + #[error("image file size too large ({0} > {1})")] + ImageFileSizeTooLarge(u64, u64), + + #[error("unsupported image format: {0:?}")] + UnsupportedImageFormat(image::ImageFormat), + + #[error("could not detect image format")] + UnknownImageFormat, + + #[error("original image dimensions too large: {0:?} > {1:?}")] + ImageDimensionsTooLarge((u32, u32), (u32, u32)), + + #[error("could not decode image, is it corrupted?")] + ImageFormatError(#[from] image::ImageError), + + #[error("unknown error")] + InternalError(#[from] anyhow::Error), +} + +#[derive(Deserialize, Debug)] +pub struct PullRequest { + url: String, + kind: ImageKind, + uploaded_by: Option, // should be String? serde makes this hard :/ + system_id: Option, + + #[serde(default)] + force: bool, +} + +#[derive(Serialize)] +pub struct PullResponse { + url: String, + new: bool, +} + +async fn pull( + State(state): State, + Json(req): Json, +) -> Result, PKAvatarError> { + let parsed = pull::parse_url(&req.url) // parsing beforehand to "normalize" + .map_err(|_| PKAvatarError::InvalidCdnUrl)?; + + if !req.force { + if let Some(existing) = db::get_by_attachment_id(&state.pool, parsed.attachment_id).await? { + return Ok(Json(PullResponse { + url: existing.url, + new: false, + })); + } + } + + let result = crate::pull::pull(state.pull_client, &parsed).await?; + + let original_file_size = result.data.len(); + let encoded = process::process_async(result.data, req.kind).await?; + + let store_res = crate::store::store(&state.bucket, &encoded).await?; + let final_url = format!("{}{}", state.config.cdn_url, store_res.path); + let is_new = db::add_image( + &state.pool, + ImageMeta { + id: store_res.id, + url: final_url.clone(), + content_type: encoded.format.mime_type().to_string(), + original_url: Some(parsed.full_url), + original_type: Some(result.content_type), + original_file_size: Some(original_file_size as i32), + original_attachment_id: Some(parsed.attachment_id as i64), + file_size: encoded.data.len() as i32, + width: encoded.width as i32, + height: encoded.height as i32, + kind: req.kind, + uploaded_at: None, + uploaded_by_account: req.uploaded_by.map(|x| x as i64), + uploaded_by_system: req.system_id, + }, + ) + .await?; + + Ok(Json(PullResponse { + url: final_url, + new: is_new, + })) +} + +pub async fn stats(State(state): State) -> Result, PKAvatarError> { + Ok(Json(db::get_stats(&state.pool).await?)) +} + +#[derive(Clone)] +pub struct AppState { + bucket: Arc, + pull_client: Arc, + pool: PgPool, + config: Arc, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + libpk::init_logging("avatars")?; + libpk::init_metrics()?; + info!("hello world"); + + let config = libpk::config + .avatars + .as_ref() + .expect("missing avatar service config"); + + let bucket = { + let region = s3::Region::Custom { + region: "s3".to_string(), + endpoint: config.s3.endpoint.to_string(), + }; + + let credentials = s3::creds::Credentials::new( + Some(&config.s3.application_id), + Some(&config.s3.application_key), + None, + None, + None, + ) + .unwrap(); + + let bucket = s3::Bucket::new(&config.s3.bucket, region, credentials)?; + + Arc::new(bucket) + }; + + let pull_client = Arc::new( + ClientBuilder::new() + .connect_timeout(Duration::from_secs(3)) + .timeout(Duration::from_secs(3)) + .user_agent("PluralKit-Avatars/0.1") + .build() + .context("error making client")?, + ); + + let pool = libpk::db::init_data_db().await?; + + let state = AppState { + bucket, + pull_client, + pool, + config: Arc::new(config.clone()), + }; + + // migrations are done, disable this + // migrate::spawn_migrate_workers(Arc::new(state.clone()), state.config.migrate_worker_count); + + let app = Router::new() + .route("/pull", post(pull)) + .route("/stats", get(stats)) + .with_state(state); + + let host = "0.0.0.0:3000"; + info!("starting server on {}!", host); + let listener = tokio::net::TcpListener::bind(host).await.unwrap(); + axum::serve(listener, app).await.unwrap(); + + Ok(()) +} + +struct AppError(anyhow::Error); + +#[derive(Serialize)] +struct ErrorResponse { + error: String, +} + +impl IntoResponse for AppError { + fn into_response(self) -> Response { + error!("error handling request: {}", self.0); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: self.0.to_string(), + }), + ) + .into_response() + } +} + +impl IntoResponse for PKAvatarError { + fn into_response(self) -> Response { + let status_code = match self { + PKAvatarError::InternalError(_) | PKAvatarError::NetworkError(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } + _ => StatusCode::BAD_REQUEST, + }; + + // print inner error if otherwise hidden + error!("error: {}", self.source().unwrap_or(&self)); + + ( + status_code, + Json(ErrorResponse { + error: self.to_string(), + }), + ) + .into_response() + } +} + +impl From for AppError +where + E: Into, +{ + fn from(err: E) -> Self { + Self(err.into()) + } +} diff --git a/services/avatars/src/migrate.rs b/services/avatars/src/migrate.rs new file mode 100644 index 00000000..87a5747d --- /dev/null +++ b/services/avatars/src/migrate.rs @@ -0,0 +1,146 @@ +use crate::pull::parse_url; +use crate::{db, process, AppState, PKAvatarError}; +use libpk::db::types::avatars::{ImageMeta, ImageQueueEntry}; +use reqwest::StatusCode; +use std::error::Error; +use std::sync::Arc; +use std::time::Duration; +use time::Instant; +use tokio::sync::Semaphore; +use tracing::{error, info, instrument, warn}; + +static PROCESS_SEMAPHORE: Semaphore = Semaphore::const_new(100); + +pub async fn handle_item_inner( + state: &AppState, + item: &ImageQueueEntry, +) -> Result<(), PKAvatarError> { + let parsed = parse_url(&item.url).map_err(|_| PKAvatarError::InvalidCdnUrl)?; + + if let Some(_) = db::get_by_attachment_id(&state.pool, parsed.attachment_id).await? { + info!( + "attachment {} already migrated, skipping", + parsed.attachment_id + ); + return Ok(()); + } + + let pulled = crate::pull::pull(state.pull_client.clone(), &parsed).await?; + let data_len = pulled.data.len(); + + let encoded = { + // Trying to reduce CPU load/potentially blocking the worker by adding a bottleneck on parallel encodes + // no semaphore on the main api though, that one should ideally be low latency + // todo: configurable? + let time_before_semaphore = Instant::now(); + let permit = PROCESS_SEMAPHORE + .acquire() + .await + .map_err(|e| PKAvatarError::InternalError(e.into()))?; + let time_after_semaphore = Instant::now(); + let semaphore_time = time_after_semaphore - time_before_semaphore; + if semaphore_time.whole_milliseconds() > 100 { + warn!( + "waited more than {} ms for process semaphore", + semaphore_time.whole_milliseconds() + ); + } + + let encoded = process::process_async(pulled.data, item.kind).await?; + drop(permit); + encoded + }; + let store_res = crate::store::store(&state.bucket, &encoded).await?; + let final_url = format!("{}{}", state.config.cdn_url, store_res.path); + + db::add_image( + &state.pool, + ImageMeta { + id: store_res.id, + url: final_url.clone(), + content_type: encoded.format.mime_type().to_string(), + original_url: Some(parsed.full_url), + original_type: Some(pulled.content_type), + original_file_size: Some(data_len as i32), + original_attachment_id: Some(parsed.attachment_id as i64), + file_size: encoded.data.len() as i32, + width: encoded.width as i32, + height: encoded.height as i32, + kind: item.kind, + uploaded_at: None, + uploaded_by_account: None, + uploaded_by_system: None, + }, + ) + .await?; + + info!( + "migrated {} ({}k -> {}k)", + final_url, + data_len, + encoded.data.len() + ); + Ok(()) +} + +pub async fn handle_item(state: &AppState) -> Result<(), PKAvatarError> { + // let queue_length = db::get_queue_length(&state.pool).await?; + // info!("migrate queue length: {}", queue_length); + + if let Some((mut tx, item)) = db::pop_queue(&state.pool).await? { + match handle_item_inner(state, &item).await { + Ok(_) => { + tx.commit().await.map_err(Into::::into)?; + Ok(()) + } + Err( + // Errors that mean the image can't be migrated and doesn't need to be retried + e @ (PKAvatarError::ImageDimensionsTooLarge(_, _) + | PKAvatarError::UnknownImageFormat + | PKAvatarError::UnsupportedImageFormat(_) + | PKAvatarError::UnsupportedContentType(_) + | PKAvatarError::ImageFileSizeTooLarge(_, _) + | PKAvatarError::InvalidCdnUrl + | PKAvatarError::BadCdnResponse(StatusCode::NOT_FOUND | StatusCode::FORBIDDEN)), + ) => { + warn!("error migrating {}, skipping: {}", item.url, e); + tx.commit().await.map_err(Into::::into)?; + Ok(()) + } + Err(e @ PKAvatarError::ImageFormatError(_)) => { + // will add this item back to the end of the queue + db::push_queue(&mut *tx, &item.url, item.kind).await?; + tx.commit().await.map_err(Into::::into)?; + Err(e) + } + Err(e) => Err(e), + } + } else { + tokio::time::sleep(Duration::from_secs(5)).await; + Ok(()) + } +} + +#[instrument(skip(state))] +pub async fn worker(worker_id: u32, state: Arc) { + info!("spawned migrate worker with id {}", worker_id); + loop { + match handle_item(&state).await { + Ok(()) => {} + Err(e) => { + error!( + "error in migrate worker {}: {}", + worker_id, + e.source().unwrap_or(&e) + ); + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + } +} + +pub fn spawn_migrate_workers(state: Arc, count: u32) { + for i in 0..count { + tokio::spawn(worker(i, state.clone())); + } +} diff --git a/services/avatars/src/process.rs b/services/avatars/src/process.rs new file mode 100644 index 00000000..a5d5a752 --- /dev/null +++ b/services/avatars/src/process.rs @@ -0,0 +1,257 @@ +use image::{DynamicImage, ImageFormat}; +use std::borrow::Cow; +use std::io::Cursor; +use time::Instant; +use tracing::{debug, error, info, instrument}; + +use crate::{hash::Hash, ImageKind, PKAvatarError}; + +const MAX_DIMENSION: u32 = 4000; + +pub struct ProcessOutput { + pub width: u32, + pub height: u32, + pub hash: Hash, + pub format: ProcessedFormat, + pub data: Vec, +} + +#[derive(Copy, Clone, Debug)] +pub enum ProcessedFormat { + Webp, + Gif, +} + +impl ProcessedFormat { + pub fn mime_type(&self) -> &'static str { + match self { + ProcessedFormat::Gif => "image/gif", + ProcessedFormat::Webp => "image/webp", + } + } + + pub fn extension(&self) -> &'static str { + match self { + ProcessedFormat::Webp => "webp", + ProcessedFormat::Gif => "gif", + } + } +} + +// Moving Vec in here since the thread needs ownership of it now, it's fine, don't need it after +pub async fn process_async(data: Vec, kind: ImageKind) -> Result { + tokio::task::spawn_blocking(move || process(&data, kind)) + .await + .map_err(|je| PKAvatarError::InternalError(je.into()))? +} + +#[instrument(skip_all)] +pub fn process(data: &[u8], kind: ImageKind) -> Result { + let time_before = Instant::now(); + let reader = reader_for(data); + match reader.format() { + Some(ImageFormat::Png | ImageFormat::WebP | ImageFormat::Jpeg | ImageFormat::Tiff) => {} // ok :) + Some(ImageFormat::Gif) => { + // animated gifs will need to be handled totally differently + // so split off processing here and come back if it's not applicable + // (non-banner gifs + 1-frame animated gifs still need to be webp'd) + if let Some(output) = process_gif(data, kind)? { + return Ok(output); + } + } + Some(other) => return Err(PKAvatarError::UnsupportedImageFormat(other)), + None => return Err(PKAvatarError::UnknownImageFormat), + } + + // want to check dimensions *before* decoding so we don't accidentally end up with a memory bomb + // eg. a 16000x16000 png file is only 31kb and expands to almost a gig of memory + let (width, height) = assert_dimensions(reader.into_dimensions()?)?; + + // need to make a new reader??? why can't it just use the same one. reduce duplication? + let reader = reader_for(data); + + let time_after_parse = Instant::now(); + + // apparently `image` sometimes decodes webp images wrong/weird. + // see: https://discord.com/channels/466707357099884544/667795132971614229/1209925940835262464 + // instead, for webp, we use libwebp itself to decode, as well. + // (pls no cve) + let image = if reader.format() == Some(ImageFormat::WebP) { + let webp_image = webp::Decoder::new(data).decode().ok_or_else(|| { + PKAvatarError::InternalError(anyhow::anyhow!("webp decode failed").into()) + })?; + webp_image.to_image() + } else { + reader.decode().map_err(|e| { + // print the ugly error, return the nice error + error!("error decoding image: {}", e); + PKAvatarError::ImageFormatError(e) + })? + }; + + let time_after_decode = Instant::now(); + let image = resize(image, kind); + let time_after_resize = Instant::now(); + + let encoded = encode(image); + let time_after = Instant::now(); + + info!( + "{}: lossy size {}K (parse: {} ms, decode: {} ms, resize: {} ms, encode: {} ms)", + encoded.hash, + encoded.data.len() / 1024, + (time_after_parse - time_before).whole_milliseconds(), + (time_after_decode - time_after_parse).whole_milliseconds(), + (time_after_resize - time_after_decode).whole_milliseconds(), + (time_after - time_after_resize).whole_milliseconds(), + ); + + debug!( + "processed image {}: {} bytes, {}x{} -> {} bytes, {}x{}", + encoded.hash, + data.len(), + width, + height, + encoded.data.len(), + encoded.width, + encoded.height + ); + Ok(encoded) +} + +fn assert_dimensions((width, height): (u32, u32)) -> Result<(u32, u32), PKAvatarError> { + if width > MAX_DIMENSION || height > MAX_DIMENSION { + return Err(PKAvatarError::ImageDimensionsTooLarge( + (width, height), + (MAX_DIMENSION, MAX_DIMENSION), + )); + } + return Ok((width, height)); +} +fn process_gif(input_data: &[u8], kind: ImageKind) -> Result, PKAvatarError> { + // gifs only supported for banners + if kind != ImageKind::Banner { + return Ok(None); + } + + // and we can't rescale gifs (i tried :/) so the max size is the real limit + if kind != ImageKind::Banner { + return Ok(None); + } + + let reader = gif::Decoder::new(Cursor::new(input_data)).map_err(Into::::into)?; + let (max_width, max_height) = kind.size(); + if reader.width() as u32 > max_width || reader.height() as u32 > max_height { + return Err(PKAvatarError::ImageDimensionsTooLarge( + (reader.width() as u32, reader.height() as u32), + (max_width, max_height), + )); + } + Ok(process_gif_inner(reader).map_err(Into::::into)?) +} + +fn process_gif_inner( + mut reader: gif::Decoder>, +) -> Result, anyhow::Error> { + let time_before = Instant::now(); + + let (width, height) = (reader.width(), reader.height()); + + let mut writer = gif::Encoder::new( + Vec::new(), + width as u16, + height as u16, + reader.global_palette().unwrap_or(&[]), + )?; + writer.set_repeat(reader.repeat())?; + + let mut frame_buf = Vec::new(); + + let mut frame_count = 0; + while let Some(frame) = reader.next_frame_info()? { + let mut frame = frame.clone(); + assert_dimensions((frame.width as u32, frame.height as u32))?; + frame_buf.clear(); + frame_buf.resize(reader.buffer_size(), 0); + reader.read_into_buffer(&mut frame_buf)?; + frame.buffer = Cow::Borrowed(&frame_buf); + + frame.make_lzw_pre_encoded(); + writer.write_lzw_pre_encoded_frame(&frame)?; + frame_count += 1; + } + + if frame_count == 1 { + // If there's only one frame, then this doesn't need to be a gif. webp it + // (unfortunately we can't tell if there's only one frame until after the first frame's been decoded...) + return Ok(None); + } + + let data = writer.into_inner()?; + let time_after = Instant::now(); + + let hash = Hash::sha256(&data); + + let original_data = reader.into_inner(); + info!( + "processed gif {}: {}K -> {}K ({} ms, frames: {})", + hash, + original_data.buffer().len() / 1024, + data.len() / 1024, + (time_after - time_before).whole_milliseconds(), + frame_count + ); + + Ok(Some(ProcessOutput { + data, + format: ProcessedFormat::Gif, + hash, + width: width as u32, + height: height as u32, + })) +} + +fn reader_for(data: &[u8]) -> image::io::Reader> { + image::io::Reader::new(Cursor::new(data)) + .with_guessed_format() + .expect("cursor i/o is infallible") +} + +#[instrument(skip_all)] +fn resize(image: DynamicImage, kind: ImageKind) -> DynamicImage { + let (target_width, target_height) = kind.size(); + if image.width() <= target_width && image.height() <= target_height { + // don't resize if already smaller + return image; + } + + // todo: best filter? + let resized = image.resize( + target_width, + target_height, + image::imageops::FilterType::Lanczos3, + ); + return resized; +} + +#[instrument(skip_all)] +// can't believe this is infallible +fn encode(image: DynamicImage) -> ProcessOutput { + let (width, height) = (image.width(), image.height()); + let image_buf = image.to_rgba8(); + + let encoded_lossy = webp::Encoder::new(&*image_buf, webp::PixelLayout::Rgba, width, height) + .encode_simple(false, 90.0) + .expect("encode should be infallible") + .to_vec(); + + let hash = Hash::sha256(&encoded_lossy); + + ProcessOutput { + data: encoded_lossy, + format: ProcessedFormat::Webp, + hash, + width, + height, + } +} diff --git a/services/avatars/src/pull.rs b/services/avatars/src/pull.rs new file mode 100644 index 00000000..e93fa577 --- /dev/null +++ b/services/avatars/src/pull.rs @@ -0,0 +1,166 @@ +use std::time::Duration; +use std::{str::FromStr, sync::Arc}; + +use crate::PKAvatarError; +use anyhow::Context; +use reqwest::{Client, ClientBuilder, StatusCode, Url}; +use time::Instant; +use tracing::{error, instrument}; + +const MAX_SIZE: u64 = 8 * 1024 * 1024; + +pub struct PullResult { + pub data: Vec, + pub content_type: String, + pub last_modified: Option, +} + +#[instrument(skip_all)] +pub async fn pull( + client: Arc, + parsed_url: &ParsedUrl, +) -> Result { + let time_before = Instant::now(); + let mut trimmed_url = trim_url_query(&parsed_url.full_url)?; + if trimmed_url.host_str() == Some("media.discordapp.net") { + trimmed_url + .set_host(Some("cdn.discordapp.com")) + .expect("set_host should not fail"); + } + let response = client.get(trimmed_url.clone()).send().await.map_err(|e| { + error!("network error for {}: {}", parsed_url.full_url, e); + PKAvatarError::NetworkError(e) + })?; + let time_after_headers = Instant::now(); + let status = response.status(); + + if status != StatusCode::OK { + return Err(PKAvatarError::BadCdnResponse(status)); + } + + let size = match response.content_length() { + None => return Err(PKAvatarError::MissingHeader("Content-Length")), + Some(size) if size > MAX_SIZE => { + return Err(PKAvatarError::ImageFileSizeTooLarge(size, MAX_SIZE)) + } + Some(size) => size, + }; + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|x| x.to_str().ok()) // invalid (non-unicode) header = missing, why not + .map(|mime| mime.split(';').next().unwrap_or("")) // cut off at ; + .ok_or(PKAvatarError::MissingHeader("Content-Type"))? + .to_owned(); + let mime = match content_type.as_str() { + mime @ ("image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/tiff") => mime, + _ => return Err(PKAvatarError::UnsupportedContentType(content_type)), + }; + + let last_modified = response + .headers() + .get(reqwest::header::LAST_MODIFIED) + .and_then(|x| x.to_str().ok()) + .map(|x| x.to_string()); + + let body = response.bytes().await.map_err(|e| { + error!("network error for {}: {}", parsed_url.full_url, e); + PKAvatarError::NetworkError(e) + })?; + if body.len() != size as usize { + // ???does this ever happen? + return Err(PKAvatarError::InternalError(anyhow::anyhow!( + "server responded with wrong length" + ))); + } + let time_after_body = Instant::now(); + + let headers_time = time_after_headers - time_before; + let body_time = time_after_body - time_after_headers; + + // can't do dynamic log level lmao + if status != StatusCode::OK { + tracing::warn!( + "{}: {} (headers: {}ms, body: {}ms)", + status, + &trimmed_url, + headers_time.whole_milliseconds(), + body_time.whole_milliseconds() + ); + } else { + tracing::info!( + "{}: {} (headers: {}ms, body: {}ms)", + status, + &trimmed_url, + headers_time.whole_milliseconds(), + body_time.whole_milliseconds() + ); + }; + + Ok(PullResult { + data: body.to_vec(), + content_type: mime.to_string(), + last_modified, + }) +} + +#[derive(Debug)] +pub struct ParsedUrl { + pub channel_id: u64, + pub attachment_id: u64, + pub filename: String, + pub full_url: String, +} + +pub fn parse_url(url: &str) -> anyhow::Result { + // todo: should this return PKAvatarError::InvalidCdnUrl? + let url = Url::from_str(url).context("invalid url")?; + + match (url.scheme(), url.domain()) { + ("https", Some("media.discordapp.net" | "cdn.discordapp.com")) => {} + _ => anyhow::bail!("not a discord cdn url"), + } + + match url + .path_segments() + .map(|x| x.collect::>()) + .as_deref() + { + Some([_, channel_id, attachment_id, filename]) => { + let channel_id = u64::from_str(channel_id).context("invalid channel id")?; + let attachment_id = u64::from_str(attachment_id).context("invalid channel id")?; + + Ok(ParsedUrl { + channel_id, + attachment_id, + filename: filename.to_string(), + full_url: url.to_string(), + }) + } + _ => anyhow::bail!("invaild discord cdn url"), + } +} + +fn trim_url_query(url: &str) -> anyhow::Result { + let mut parsed = Url::parse(url)?; + + let mut qs = form_urlencoded::Serializer::new(String::new()); + for (key, value) in parsed.query_pairs() { + match key.as_ref() { + "ex" | "is" | "hm" => { + qs.append_pair(key.as_ref(), value.as_ref()); + } + _ => {} + } + } + + let new_query = qs.finish(); + parsed.set_query(if new_query.len() > 0 { + Some(&new_query) + } else { + None + }); + + Ok(parsed) +} diff --git a/services/avatars/src/store.rs b/services/avatars/src/store.rs new file mode 100644 index 00000000..4232ebd8 --- /dev/null +++ b/services/avatars/src/store.rs @@ -0,0 +1,60 @@ +use crate::process::ProcessOutput; +use tracing::error; + +pub struct StoreResult { + pub id: String, + pub path: String, +} + +pub async fn store(bucket: &s3::Bucket, res: &ProcessOutput) -> anyhow::Result { + // errors here are all going to be internal + let encoded_hash = res.hash.to_string(); + let path = format!( + "images/{}/{}.{}", + &encoded_hash[..2], + &encoded_hash[2..], + res.format.extension() + ); + + // todo: something better than these retries + let mut retry_count = 0; + loop { + if retry_count == 2 { + tokio::time::sleep(tokio::time::Duration::new(2, 0)).await; + } + if retry_count > 2 { + anyhow::bail!("error uploading image to cdn, too many retries") // nicer user-facing error? + } + retry_count += 1; + + let resp = bucket + .put_object_with_content_type(&path, &res.data, res.format.mime_type()) + .await?; + match resp.status_code() { + 200 => { + tracing::debug!("uploaded image to {}", &path); + + return Ok(StoreResult { + id: encoded_hash, + path, + }); + } + 500 | 503 => { + tracing::warn!( + "got 503 uploading image to {} ({}), retrying... (try {}/3)", + &path, + resp.as_str()?, + retry_count + ); + continue; + } + _ => { + error!( + "storage backend responded status code {}", + resp.status_code() + ); + anyhow::bail!("error uploading image to cdn") // nicer user-facing error? + } + } + } +} diff --git a/services/dispatch/Cargo.toml b/services/dispatch/Cargo.toml index 68e1bdd4..656cd50c 100644 --- a/services/dispatch/Cargo.toml +++ b/services/dispatch/Cargo.toml @@ -13,4 +13,4 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } hickory-client = "0.24.1" -reqwest = { version = "0.12.7", default-features = false, features = ["rustls-tls"] } +reqwest = { workspace = true } diff --git a/services/gateway/src/cache_api.rs b/services/gateway/src/cache_api.rs index e09a0b12..efbca0fc 100644 --- a/services/gateway/src/cache_api.rs +++ b/services/gateway/src/cache_api.rs @@ -36,7 +36,7 @@ pub async fn run_server(cache: Arc) -> anyhow::Result<()> { .route( "/guilds/:guild_id/members/@me", get(|State(cache): State>, Path(guild_id): Path| async move { - match cache.0.member(Id::new(guild_id), libpk::config.discord.client_id) { + match cache.0.member(Id::new(guild_id), libpk::config.discord.as_ref().expect("missing discord config").client_id) { Some(member) => status_code(StatusCode::FOUND, to_string(member.value()).unwrap()), None => status_code(StatusCode::NOT_FOUND, "".to_string()), } @@ -45,7 +45,7 @@ pub async fn run_server(cache: Arc) -> anyhow::Result<()> { .route( "/guilds/:guild_id/permissions/@me", get(|State(cache): State>, Path(guild_id): Path| async move { - match cache.guild_permissions(Id::new(guild_id), libpk::config.discord.client_id).await { + match cache.guild_permissions(Id::new(guild_id), libpk::config.discord.as_ref().expect("missing discord config").client_id).await { Ok(val) => { println!("hh {}", Permissions::all().bits()); status_code(StatusCode::FOUND, to_string(&val.bits()).unwrap()) @@ -114,7 +114,7 @@ pub async fn run_server(cache: Arc) -> anyhow::Result<()> { if guild_id == 0 { return status_code(StatusCode::FOUND, to_string(&*DM_PERMISSIONS).unwrap()); } - match cache.channel_permissions(Id::new(channel_id), libpk::config.discord.client_id).await { + match cache.channel_permissions(Id::new(channel_id), libpk::config.discord.as_ref().expect("missing discord config").client_id).await { Ok(val) => status_code(StatusCode::FOUND, to_string(&val).unwrap()), Err(err) => { error!(?err, ?channel_id, ?guild_id, "failed to get own channelpermissions"); @@ -176,7 +176,7 @@ pub async fn run_server(cache: Arc) -> anyhow::Result<()> { .layer(axum::middleware::from_fn(crate::logger::logger)) .with_state(cache); - let addr: &str = libpk::config.api.addr.as_ref(); + let addr: &str = libpk::config.discord.as_ref().expect("missing discord config").cache_api_addr.as_ref(); let listener = tokio::net::TcpListener::bind(addr).await?; info!("listening on {}", addr); axum::serve(listener, app).await?; diff --git a/services/gateway/src/discord/cache.rs b/services/gateway/src/discord/cache.rs index 38a43e40..5f7d6e2c 100644 --- a/services/gateway/src/discord/cache.rs +++ b/services/gateway/src/discord/cache.rs @@ -89,10 +89,22 @@ fn member_to_cached_member(item: Member, id: Id) -> CachedMember { } pub fn new() -> DiscordCache { - let mut client_builder = - twilight_http::Client::builder().token(libpk::config.discord.bot_token.clone()); + let mut client_builder = twilight_http::Client::builder().token( + libpk::config + .discord + .as_ref() + .expect("missing discord config") + .bot_token + .clone(), + ); - if let Some(base_url) = libpk::config.discord.api_base_url.clone() { + if let Some(base_url) = libpk::config + .discord + .as_ref() + .expect("missing discord config") + .api_base_url + .clone() + { client_builder = client_builder.proxy(base_url, true); } @@ -136,7 +148,13 @@ impl DiscordCache { return Ok(Permissions::all()); } - let member = if user_id == libpk::config.discord.client_id { + let member = if user_id + == libpk::config + .discord + .as_ref() + .expect("missing discord config") + .client_id + { self.0 .member(guild_id, user_id) .ok_or(format_err!("self member not found"))? @@ -202,7 +220,13 @@ impl DiscordCache { return Ok(Permissions::all()); } - let member = if user_id == libpk::config.discord.client_id { + let member = if user_id + == libpk::config + .discord + .as_ref() + .expect("missing discord config") + .client_id + { self.0 .member(guild_id, user_id) .ok_or_else(|| { diff --git a/services/gateway/src/discord/gateway.rs b/services/gateway/src/discord/gateway.rs index 97711069..723bcc89 100644 --- a/services/gateway/src/discord/gateway.rs +++ b/services/gateway/src/discord/gateway.rs @@ -17,6 +17,8 @@ use super::{cache::DiscordCache, shard_state::ShardStateManager}; pub fn cluster_config() -> ClusterSettings { libpk::config .discord + .as_ref() + .expect("missing discord config") .cluster .clone() .unwrap_or(libpk::_config::ClusterSettings { @@ -51,10 +53,18 @@ pub fn create_shards(redis: fred::pool::RedisPool) -> anyhow::Result RedisQueue { RedisQueue { redis, - concurrency: libpk::config.discord.max_concurrency, + concurrency: libpk::config + .discord + .as_ref() + .expect("missing discord config") + .max_concurrency, } }