From a72afb35a0bb30e17c00c96be86d8452f93a9df8 Mon Sep 17 00:00:00 2001 From: alyssa Date: Mon, 10 Mar 2025 15:12:56 +0000 Subject: [PATCH 01/80] feat: add remote config over http/redis --- PluralKit.Bot/BotConfig.cs | 2 + PluralKit.Bot/Init.cs | 7 ++ PluralKit.Bot/Modules.cs | 2 + PluralKit.Bot/PluralKit.Bot.csproj | 1 + PluralKit.Bot/Services/HttpListenerService.cs | 56 +++++++++++++++ .../Services/RuntimeConfigService.cs | 58 +++++++++++++++ PluralKit.Bot/packages.lock.json | 47 ++++++++++++ PluralKit.Tests/packages.lock.json | 49 ++++++++++++- crates/gateway/src/cache_api.rs | 23 +++++- crates/gateway/src/main.rs | 12 +++- crates/libpk/src/lib.rs | 1 + crates/libpk/src/runtime_config.rs | 72 +++++++++++++++++++ 12 files changed, 326 insertions(+), 4 deletions(-) create mode 100644 PluralKit.Bot/Services/HttpListenerService.cs create mode 100644 PluralKit.Bot/Services/RuntimeConfigService.cs create mode 100644 crates/libpk/src/runtime_config.rs diff --git a/PluralKit.Bot/BotConfig.cs b/PluralKit.Bot/BotConfig.cs index fdc54ad2..a9db7233 100644 --- a/PluralKit.Bot/BotConfig.cs +++ b/PluralKit.Bot/BotConfig.cs @@ -24,6 +24,8 @@ public class BotConfig public string? HttpCacheUrl { get; set; } public bool HttpUseInnerCache { get; set; } = false; + public string? HttpListenerAddr { get; set; } + public string? DiscordBaseUrl { get; set; } public string? AvatarServiceUrl { get; set; } diff --git a/PluralKit.Bot/Init.cs b/PluralKit.Bot/Init.cs index c4b9ed65..f19576de 100644 --- a/PluralKit.Bot/Init.cs +++ b/PluralKit.Bot/Init.cs @@ -76,6 +76,13 @@ public class Init // Init the bot instance itself, register handlers and such to the client before beginning to connect bot.Init(); + // load runtime config from redis + await services.Resolve().LoadConfig(); + + // Start HTTP server + if (config.HttpListenerAddr != null) + services.Resolve().Start(config.HttpListenerAddr); + // Start the Discord shards themselves (handlers already set up) logger.Information("Connecting to Discord"); await StartCluster(services); diff --git a/PluralKit.Bot/Modules.cs b/PluralKit.Bot/Modules.cs index 6ceea629..ae6f85ec 100644 --- a/PluralKit.Bot/Modules.cs +++ b/PluralKit.Bot/Modules.cs @@ -153,6 +153,8 @@ public class BotModule: Module builder.RegisterType().AsSelf().SingleInstance(); builder.RegisterType().AsSelf().SingleInstance(); builder.RegisterType().AsSelf().SingleInstance(); + builder.RegisterType().AsSelf().SingleInstance(); + builder.RegisterType().AsSelf().SingleInstance(); // Sentry stuff builder.Register(_ => new Scope(null)).AsSelf().InstancePerLifetimeScope(); diff --git a/PluralKit.Bot/PluralKit.Bot.csproj b/PluralKit.Bot/PluralKit.Bot.csproj index 01d89e1d..753d1f30 100644 --- a/PluralKit.Bot/PluralKit.Bot.csproj +++ b/PluralKit.Bot/PluralKit.Bot.csproj @@ -24,5 +24,6 @@ + diff --git a/PluralKit.Bot/Services/HttpListenerService.cs b/PluralKit.Bot/Services/HttpListenerService.cs new file mode 100644 index 00000000..73a1f907 --- /dev/null +++ b/PluralKit.Bot/Services/HttpListenerService.cs @@ -0,0 +1,56 @@ +using Serilog; + +using Newtonsoft.Json; + +using WatsonWebserver.Lite; +using WatsonWebserver.Core; + +namespace PluralKit.Bot; + +public class HttpListenerService +{ + private readonly ILogger _logger; + private readonly RuntimeConfigService _runtimeConfig; + + public HttpListenerService(ILogger logger, RuntimeConfigService runtimeConfig) + { + _logger = logger.ForContext(); + _runtimeConfig = runtimeConfig; + } + + public void Start(string host) + { + var server = new WebserverLite(new WebserverSettings(host, 5002), DefaultRoute); + + server.Routes.PreAuthentication.Static.Add(WatsonWebserver.Core.HttpMethod.GET, "/runtime_config", RuntimeConfigGet); + server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.POST, "/runtime_config/{key}", RuntimeConfigSet); + server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.DELETE, "/runtime_config/{key}", RuntimeConfigDelete); + + server.Start(); + } + + private async Task DefaultRoute(HttpContextBase ctx) + => await ctx.Response.Send("hellorld"); + + private async Task RuntimeConfigGet(HttpContextBase ctx) + { + var config = _runtimeConfig.GetAll(); + ctx.Response.Headers.Add("content-type", "application/json"); + await ctx.Response.Send(JsonConvert.SerializeObject(config)); + } + + private async Task RuntimeConfigSet(HttpContextBase ctx) + { + var key = ctx.Request.Url.Parameters["key"]; + var value = ctx.Request.DataAsString; + await _runtimeConfig.Set(key, value); + await RuntimeConfigGet(ctx); + } + + private async Task RuntimeConfigDelete(HttpContextBase ctx) + { + var key = ctx.Request.Url.Parameters["key"]; + await _runtimeConfig.Delete(key); + await RuntimeConfigGet(ctx); + } +} \ No newline at end of file diff --git a/PluralKit.Bot/Services/RuntimeConfigService.cs b/PluralKit.Bot/Services/RuntimeConfigService.cs new file mode 100644 index 00000000..2d71a2a2 --- /dev/null +++ b/PluralKit.Bot/Services/RuntimeConfigService.cs @@ -0,0 +1,58 @@ +using Newtonsoft.Json; + +using Serilog; + +using StackExchange.Redis; + +using PluralKit.Core; + +namespace PluralKit.Bot; + +public class RuntimeConfigService +{ + private readonly RedisService _redis; + private readonly ILogger _logger; + + private Dictionary settings = new(); + + private string RedisKey; + + public RuntimeConfigService(ILogger logger, RedisService redis, BotConfig config) + { + _logger = logger.ForContext(); + _redis = redis; + + var clusterId = config.Cluster?.NodeIndex ?? 0; + RedisKey = $"remote_config:dotnet_bot:{clusterId}"; + } + + public async Task LoadConfig() + { + var redisConfig = await _redis.Connection.GetDatabase().HashGetAllAsync(RedisKey); + foreach (var entry in redisConfig) + settings.Add(entry.Name, entry.Value); + + var configStr = JsonConvert.SerializeObject(settings); + _logger.Information($"starting with runtime config: {configStr}"); + } + + public async Task Set(string key, string value) + { + await _redis.Connection.GetDatabase().HashSetAsync(RedisKey, new[] { new HashEntry(key, new RedisValue(value)) }); + settings.Add(key, value); + _logger.Information($"updated runtime config: {key}={value}"); + } + + public async Task Delete(string key) + { + await _redis.Connection.GetDatabase().HashDeleteAsync(RedisKey, key); + settings.Remove(key); + _logger.Information($"updated runtime config: {key} removed"); + } + + public object? Get(string key) => settings.GetValueOrDefault(key); + + public bool Exists(string key) => settings.ContainsKey(key); + + public Dictionary GetAll() => settings; +} \ No newline at end of file diff --git a/PluralKit.Bot/packages.lock.json b/PluralKit.Bot/packages.lock.json index e2eda930..fca78f6e 100644 --- a/PluralKit.Bot/packages.lock.json +++ b/PluralKit.Bot/packages.lock.json @@ -14,6 +14,16 @@ "resolved": "4.13.0", "contentHash": "Wfw3M1WpFcrYaGzPm7QyUTfIOYkVXQ1ry6p4WYjhbLz9fPwV23SGQZTFDpdox67NHM0V0g1aoQ4YKLm4ANtEEg==" }, + "Watson.Lite": { + "type": "Direct", + "requested": "[6.3.5, )", + "resolved": "6.3.5", + "contentHash": "YF8+se3IVenn8YlyNeb4wSJK6QMnVD0QHIOEiZ22wS4K2wkwoSDzWS+ZAjk1MaPeB+XO5gRoENUN//pOc+wI2g==", + "dependencies": { + "CavemanTcp": "2.0.5", + "Watson.Core": "6.3.5" + } + }, "App.Metrics": { "type": "Transitive", "resolved": "4.3.0", @@ -107,6 +117,11 @@ "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.1" } }, + "CavemanTcp": { + "type": "Transitive", + "resolved": "2.0.5", + "contentHash": "90wywmGpjrj26HMAkufYZwuZI8sVYB1mRwEdqugSR3kgDnPX+3l0jO86gwtFKsPvsEpsS4Dn/1EbhguzUxMU8Q==" + }, "Dapper": { "type": "Transitive", "resolved": "2.1.35", @@ -130,6 +145,11 @@ "System.Diagnostics.DiagnosticSource": "5.0.0" } }, + "IpMatcher": { + "type": "Transitive", + "resolved": "1.0.5", + "contentHash": "WXNlWERj+0GN699AnMNsuJ7PfUAbU4xhOHP3nrNXLHqbOaBxybu25luSYywX1133NSlitA4YkSNmJuyPvea4sw==" + }, "IPNetwork2": { "type": "Transitive", "resolved": "3.0.667", @@ -391,6 +411,11 @@ "resolved": "8.5.0", "contentHash": "VYYMZNitZ85UEhwOKkTQI63WEMvzUqwQc74I2mm8h/DBVAMcBBxqYPni4DmuRtbCwngmuONuK2yBJfWNRKzI+A==" }, + "RegexMatcher": { + "type": "Transitive", + "resolved": "1.0.9", + "contentHash": "RkQGXIrqHjD5h1mqefhgCbkaSdRYNRG5rrbzyw5zeLWiS0K1wq9xR3cNhQdzYR2MsKZ3GN523yRUsEQIMPxh3Q==" + }, "Serilog": { "type": "Transitive", "resolved": "4.2.0", @@ -714,6 +739,28 @@ "System.Runtime": "4.3.0" } }, + "Timestamps": { + "type": "Transitive", + "resolved": "1.0.11", + "contentHash": "SnWhXm3FkEStQGgUTfWMh9mKItNW032o/v8eAtFrOGqG0/ejvPPA1LdLZx0N/qqoY0TH3x11+dO00jeVcM8xNQ==" + }, + "UrlMatcher": { + "type": "Transitive", + "resolved": "3.0.1", + "contentHash": "hHBZVzFSfikrx4XsRsnCIwmGLgbNKtntnlqf4z+ygcNA6Y/L/J0x5GiZZWfXdTfpxhy5v7mlt2zrZs/L9SvbOA==" + }, + "Watson.Core": { + "type": "Transitive", + "resolved": "6.3.5", + "contentHash": "Y5YxKOCSLe2KDmfwvI/J0qApgmmZR77LwyoufRVfKH7GLdHiE7fY0IfoNxWTG7nNv8knBfgwyOxdehRm+4HaCg==", + "dependencies": { + "IpMatcher": "1.0.5", + "RegexMatcher": "1.0.9", + "System.Text.Json": "8.0.5", + "Timestamps": "1.0.11", + "UrlMatcher": "3.0.1" + } + }, "myriad": { "type": "Project", "dependencies": { diff --git a/PluralKit.Tests/packages.lock.json b/PluralKit.Tests/packages.lock.json index 2ba02316..c11cedb5 100644 --- a/PluralKit.Tests/packages.lock.json +++ b/PluralKit.Tests/packages.lock.json @@ -128,6 +128,11 @@ "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.1" } }, + "CavemanTcp": { + "type": "Transitive", + "resolved": "2.0.5", + "contentHash": "90wywmGpjrj26HMAkufYZwuZI8sVYB1mRwEdqugSR3kgDnPX+3l0jO86gwtFKsPvsEpsS4Dn/1EbhguzUxMU8Q==" + }, "Dapper": { "type": "Transitive", "resolved": "2.1.35", @@ -156,6 +161,11 @@ "resolved": "2.14.1", "contentHash": "lQKvtaTDOXnoVJ20ibTuSIOf2i0uO0MPbDhd1jm238I+U/2ZnRENj0cktKZhtchBMtCUSRQ5v4xBCUbKNmyVMw==" }, + "IpMatcher": { + "type": "Transitive", + "resolved": "1.0.5", + "contentHash": "WXNlWERj+0GN699AnMNsuJ7PfUAbU4xhOHP3nrNXLHqbOaBxybu25luSYywX1133NSlitA4YkSNmJuyPvea4sw==" + }, "IPNetwork2": { "type": "Transitive", "resolved": "3.0.667", @@ -510,6 +520,11 @@ "resolved": "8.5.0", "contentHash": "VYYMZNitZ85UEhwOKkTQI63WEMvzUqwQc74I2mm8h/DBVAMcBBxqYPni4DmuRtbCwngmuONuK2yBJfWNRKzI+A==" }, + "RegexMatcher": { + "type": "Transitive", + "resolved": "1.0.9", + "contentHash": "RkQGXIrqHjD5h1mqefhgCbkaSdRYNRG5rrbzyw5zeLWiS0K1wq9xR3cNhQdzYR2MsKZ3GN523yRUsEQIMPxh3Q==" + }, "Sentry": { "type": "Transitive", "resolved": "4.13.0", @@ -887,6 +902,37 @@ "System.Runtime": "4.3.0" } }, + "Timestamps": { + "type": "Transitive", + "resolved": "1.0.11", + "contentHash": "SnWhXm3FkEStQGgUTfWMh9mKItNW032o/v8eAtFrOGqG0/ejvPPA1LdLZx0N/qqoY0TH3x11+dO00jeVcM8xNQ==" + }, + "UrlMatcher": { + "type": "Transitive", + "resolved": "3.0.1", + "contentHash": "hHBZVzFSfikrx4XsRsnCIwmGLgbNKtntnlqf4z+ygcNA6Y/L/J0x5GiZZWfXdTfpxhy5v7mlt2zrZs/L9SvbOA==" + }, + "Watson.Core": { + "type": "Transitive", + "resolved": "6.3.5", + "contentHash": "Y5YxKOCSLe2KDmfwvI/J0qApgmmZR77LwyoufRVfKH7GLdHiE7fY0IfoNxWTG7nNv8knBfgwyOxdehRm+4HaCg==", + "dependencies": { + "IpMatcher": "1.0.5", + "RegexMatcher": "1.0.9", + "System.Text.Json": "8.0.5", + "Timestamps": "1.0.11", + "UrlMatcher": "3.0.1" + } + }, + "Watson.Lite": { + "type": "Transitive", + "resolved": "6.3.5", + "contentHash": "YF8+se3IVenn8YlyNeb4wSJK6QMnVD0QHIOEiZ22wS4K2wkwoSDzWS+ZAjk1MaPeB+XO5gRoENUN//pOc+wI2g==", + "dependencies": { + "CavemanTcp": "2.0.5", + "Watson.Core": "6.3.5" + } + }, "xunit.abstractions": { "type": "Transitive", "resolved": "2.0.3", @@ -954,7 +1000,8 @@ "Humanizer.Core": "[2.14.1, )", "Myriad": "[1.0.0, )", "PluralKit.Core": "[1.0.0, )", - "Sentry": "[4.13.0, )" + "Sentry": "[4.13.0, )", + "Watson.Lite": "[6.3.5, )" } }, "pluralkit.core": { diff --git a/crates/gateway/src/cache_api.rs b/crates/gateway/src/cache_api.rs index c73424f9..e3de4696 100644 --- a/crates/gateway/src/cache_api.rs +++ b/crates/gateway/src/cache_api.rs @@ -2,9 +2,10 @@ use axum::{ extract::{Path, State}, http::StatusCode, response::{IntoResponse, Response}, - routing::get, + routing::{delete, get, post}, Router, }; +use libpk::runtime_config::RuntimeConfig; use serde_json::{json, to_string}; use tracing::{error, info}; use twilight_model::id::Id; @@ -21,7 +22,11 @@ fn status_code(code: StatusCode, body: String) -> Response { // this function is manually formatted for easier legibility of route_services #[rustfmt::skip] -pub async fn run_server(cache: Arc) -> anyhow::Result<()> { +pub async fn run_server(cache: Arc, runtime_config: Arc) -> anyhow::Result<()> { + // hacky fix for `move` + let runtime_config_for_post = runtime_config.clone(); + let runtime_config_for_delete = runtime_config.clone(); + let app = Router::new() .route( "/guilds/:guild_id", @@ -171,6 +176,20 @@ pub async fn run_server(cache: Arc) -> anyhow::Result<()> { status_code(StatusCode::FOUND, to_string(&stats).unwrap()) })) + .route("/runtime_config", get(|| async move { + status_code(StatusCode::FOUND, to_string(&runtime_config.get_all().await).unwrap()) + })) + .route("/runtime_config/:key", post(|Path(key): Path, body: String| async move { + let runtime_config = runtime_config_for_post; + runtime_config.set(key, body).await.expect("failed to update runtime config"); + status_code(StatusCode::FOUND, to_string(&runtime_config.get_all().await).unwrap()) + })) + .route("/runtime_config/:key", delete(|Path(key): Path| async move { + let runtime_config = runtime_config_for_delete; + runtime_config.delete(key).await.expect("failed to update runtime config"); + status_code(StatusCode::FOUND, to_string(&runtime_config.get_all().await).unwrap()) + })) + .layer(axum::middleware::from_fn(crate::logger::logger)) .with_state(cache); diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 6bc33e13..254ce504 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -2,7 +2,9 @@ #![feature(if_let_guard)] use chrono::Timelike; +use discord::gateway::cluster_config; use fred::{clients::RedisPool, interfaces::*}; +use libpk::runtime_config::RuntimeConfig; use signal_hook::{ consts::{SIGINT, SIGTERM}, iterator::Signals, @@ -28,6 +30,14 @@ async fn real_main() -> anyhow::Result<()> { let redis = libpk::db::init_redis().await?; + let runtime_config = Arc::new( + RuntimeConfig::new( + redis.clone(), + format!("gateway:{}", cluster_config().node_id), + ) + .await?, + ); + let shard_state = discord::shard_state::new(redis.clone()); let cache = Arc::new(discord::cache::new()); @@ -57,7 +67,7 @@ async fn real_main() -> anyhow::Result<()> { // todo: probably don't do it this way let api_shutdown_tx = shutdown_tx.clone(); set.spawn(tokio::spawn(async move { - match cache_api::run_server(cache).await { + match cache_api::run_server(cache, runtime_config).await { Err(error) => { tracing::error!(?error, "failed to serve cache api"); let _ = api_shutdown_tx.send(()); diff --git a/crates/libpk/src/lib.rs b/crates/libpk/src/lib.rs index af967728..1864e332 100644 --- a/crates/libpk/src/lib.rs +++ b/crates/libpk/src/lib.rs @@ -8,6 +8,7 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilte use sentry_tracing::event_from_event; pub mod db; +pub mod runtime_config; pub mod state; pub mod _config; diff --git a/crates/libpk/src/runtime_config.rs b/crates/libpk/src/runtime_config.rs new file mode 100644 index 00000000..739818e4 --- /dev/null +++ b/crates/libpk/src/runtime_config.rs @@ -0,0 +1,72 @@ +use fred::{clients::RedisPool, interfaces::HashesInterface}; +use std::collections::HashMap; +use tokio::sync::RwLock; +use tracing::info; + +pub struct RuntimeConfig { + redis: RedisPool, + settings: RwLock>, + redis_key: String, +} + +impl RuntimeConfig { + pub async fn new(redis: RedisPool, component_key: String) -> anyhow::Result { + let redis_key = format!("remote_config:{component_key}"); + + let mut c = RuntimeConfig { + redis, + settings: RwLock::new(HashMap::new()), + redis_key, + }; + + c.load().await?; + + Ok(c) + } + + pub async fn load(&mut self) -> anyhow::Result<()> { + let redis_config: HashMap = self.redis.hgetall(&self.redis_key).await?; + + let mut settings = self.settings.write().await; + + for (key, value) in redis_config { + settings.insert(key, value); + } + + info!("starting with runtime config: {:?}", self.settings); + Ok(()) + } + + pub async fn set(&self, key: String, value: String) -> anyhow::Result<()> { + self.redis + .hset::<(), &str, (String, String)>(&self.redis_key, (key.clone(), value.clone())) + .await?; + self.settings + .write() + .await + .insert(key.clone(), value.clone()); + info!("updated runtime config: {key}={value}"); + Ok(()) + } + + pub async fn delete(&self, key: String) -> anyhow::Result<()> { + self.redis + .hdel::<(), &str, String>(&self.redis_key, key.clone()) + .await?; + self.settings.write().await.remove(&key.clone()); + info!("updated runtime config: {key} removed"); + Ok(()) + } + + pub async fn get(&self, key: String) -> Option { + self.settings.read().await.get(&key).cloned() + } + + pub async fn exists(&self, key: &str) -> bool { + self.settings.read().await.contains_key(key) + } + + pub async fn get_all(&self) -> HashMap { + self.settings.read().await.clone() + } +} From 2578eb0e3ca759f75e8257f7fc980e6dfa1bdbaf Mon Sep 17 00:00:00 2001 From: alyssa Date: Fri, 4 Apr 2025 11:06:16 +0000 Subject: [PATCH 02/80] feat: send events from gateway to bot over http --- Cargo.lock | 1 + PluralKit.Bot/Bot.cs | 8 ++- PluralKit.Bot/Services/HttpListenerService.cs | 54 +++++++++++++++++-- crates/gateway/Cargo.toml | 1 + crates/gateway/src/discord/gateway.rs | 42 +++++++++++++-- crates/gateway/src/main.rs | 36 ++++++++++++- crates/libpk/src/lib.rs | 1 + crates/libpk/src/runtime_config.rs | 6 +-- 8 files changed, 134 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee181b20..a4a5e80c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1088,6 +1088,7 @@ dependencies = [ "lazy_static", "libpk", "metrics", + "reqwest 0.12.8", "serde_json", "serde_variant", "signal-hook", diff --git a/PluralKit.Bot/Bot.cs b/PluralKit.Bot/Bot.cs index d74428ee..46e4e4a1 100644 --- a/PluralKit.Bot/Bot.cs +++ b/PluralKit.Bot/Bot.cs @@ -32,13 +32,14 @@ public class Bot private readonly DiscordApiClient _rest; private readonly RedisService _redis; private readonly ILifetimeScope _services; + private readonly RuntimeConfigService _runtimeConfig; private Timer _periodicTask; // Never read, just kept here for GC reasons public Bot(ILifetimeScope services, ILogger logger, PeriodicStatCollector collector, IMetrics metrics, BotConfig config, RedisService redis, ErrorMessageService errorMessageService, CommandMessageService commandMessageService, - Cluster cluster, DiscordApiClient rest, IDiscordCache cache) + Cluster cluster, DiscordApiClient rest, IDiscordCache cache, RuntimeConfigService runtimeConfig) { _logger = logger.ForContext(); _services = services; @@ -51,6 +52,7 @@ public class Bot _rest = rest; _redis = redis; _cache = cache; + _runtimeConfig = runtimeConfig; } private string BotStatus => $"{(_config.Prefixes ?? BotConfig.DefaultPrefixes)[0]}help" @@ -97,13 +99,15 @@ public class Bot private async Task OnEventReceived(int shardId, IGatewayEvent evt) { + if (_runtimeConfig.Exists("disable_events")) return; + // we HandleGatewayEvent **before** getting the own user, because the own user is set in HandleGatewayEvent for ReadyEvent await _cache.HandleGatewayEvent(evt); await _cache.TryUpdateSelfMember(_config.ClientId, evt); await OnEventReceivedInner(shardId, evt); } - private async Task OnEventReceivedInner(int shardId, IGatewayEvent evt) + public async Task OnEventReceivedInner(int shardId, IGatewayEvent evt) { // HandleEvent takes a type parameter, automatically inferred by the event type // It will then look up an IEventHandler in the DI container and call that object's handler method diff --git a/PluralKit.Bot/Services/HttpListenerService.cs b/PluralKit.Bot/Services/HttpListenerService.cs index 73a1f907..79573625 100644 --- a/PluralKit.Bot/Services/HttpListenerService.cs +++ b/PluralKit.Bot/Services/HttpListenerService.cs @@ -1,21 +1,26 @@ -using Serilog; +using System.Text.Json; -using Newtonsoft.Json; +using Serilog; using WatsonWebserver.Lite; using WatsonWebserver.Core; +using Myriad.Gateway; +using Myriad.Serialization; + namespace PluralKit.Bot; public class HttpListenerService { private readonly ILogger _logger; private readonly RuntimeConfigService _runtimeConfig; + private readonly Bot _bot; - public HttpListenerService(ILogger logger, RuntimeConfigService runtimeConfig) + public HttpListenerService(ILogger logger, RuntimeConfigService runtimeConfig, Bot bot) { _logger = logger.ForContext(); _runtimeConfig = runtimeConfig; + _bot = bot; } public void Start(string host) @@ -26,6 +31,8 @@ public class HttpListenerService server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.POST, "/runtime_config/{key}", RuntimeConfigSet); server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.DELETE, "/runtime_config/{key}", RuntimeConfigDelete); + server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.POST, "/events/{shard_id}", GatewayEvent); + server.Start(); } @@ -36,7 +43,7 @@ public class HttpListenerService { var config = _runtimeConfig.GetAll(); ctx.Response.Headers.Add("content-type", "application/json"); - await ctx.Response.Send(JsonConvert.SerializeObject(config)); + await ctx.Response.Send(JsonSerializer.Serialize(config)); } private async Task RuntimeConfigSet(HttpContextBase ctx) @@ -53,4 +60,43 @@ public class HttpListenerService await _runtimeConfig.Delete(key); await RuntimeConfigGet(ctx); } + + private JsonSerializerOptions _jsonSerializerOptions = new JsonSerializerOptions().ConfigureForMyriad(); + + private async Task GatewayEvent(HttpContextBase ctx) + { + var shardIdString = ctx.Request.Url.Parameters["shard_id"]; + if (!int.TryParse(shardIdString, out var shardId)) return; + + var packet = JsonSerializer.Deserialize(ctx.Request.DataAsString, _jsonSerializerOptions); + var evt = DeserializeEvent(shardId, packet.EventType!, (JsonElement)packet.Payload!); + if (evt != null) + { + await _bot.OnEventReceivedInner(shardId, evt); + } + await ctx.Response.Send("a"); + } + + private IGatewayEvent? DeserializeEvent(int shardId, string eventType, JsonElement payload) + { + if (!IGatewayEvent.EventTypes.TryGetValue(eventType, out var clrType)) + { + _logger.Debug("Shard {ShardId}: Received unknown event type {EventType}", shardId, eventType); + return null; + } + + try + { + _logger.Verbose("Shard {ShardId}: Deserializing {EventType} to {ClrType}", shardId, eventType, + clrType); + return JsonSerializer.Deserialize(payload.GetRawText(), clrType, _jsonSerializerOptions) + as IGatewayEvent; + } + catch (JsonException e) + { + _logger.Error(e, "Shard {ShardId}: Error deserializing event {EventType} to {ClrType}", shardId, + eventType, clrType); + return null; + } + } } \ No newline at end of file diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index bde62f22..e9fd444e 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -13,6 +13,7 @@ futures = { workspace = true } lazy_static = { workspace = true } libpk = { path = "../libpk" } metrics = { workspace = true } +reqwest = { workspace = true } serde_json = { workspace = true } signal-hook = { workspace = true } tokio = { workspace = true } diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index 6aedca23..c4a13a81 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -1,7 +1,8 @@ use futures::StreamExt; -use libpk::_config::ClusterSettings; +use libpk::{_config::ClusterSettings, runtime_config::RuntimeConfig}; use metrics::counter; -use std::sync::{mpsc::Sender, Arc}; +use std::sync::Arc; +use tokio::sync::mpsc::Sender; use tracing::{error, info, warn}; use twilight_gateway::{ create_iterator, ConfigBuilder, Event, EventTypeFlags, Message, Shard, ShardId, @@ -12,7 +13,10 @@ use twilight_model::gateway::{ Intents, }; -use crate::discord::identify_queue::{self, RedisQueue}; +use crate::{ + discord::identify_queue::{self, RedisQueue}, + RUNTIME_CONFIG_KEY_EVENT_TARGET, +}; use super::{cache::DiscordCache, shard_state::ShardStateManager}; @@ -78,13 +82,20 @@ pub fn create_shards(redis: fred::clients::RedisPool) -> anyhow::Result, - _tx: Sender<(ShardId, String)>, + tx: Sender<(ShardId, String)>, shard_state: ShardStateManager, cache: Arc, + runtime_config: Arc, ) { // let _span = info_span!("shard_runner", shard_id = shard.id().number()).entered(); let shard_id = shard.id().number(); + let our_user_id = libpk::config + .discord + .as_ref() + .expect("missing discord config") + .client_id; + info!("waiting for events"); while let Some(item) = shard.next().await { let raw_event = match item { @@ -165,7 +176,28 @@ pub async fn runner( cache.0.update(&event); // okay, we've handled the event internally, let's send it to consumers - // tx.send((shard.id(), raw_event)).unwrap(); + + // some basic filtering here is useful + // we can't use if matching using the | operator, so anything matched does nothing + // and the default match skips the next block (continues to the next event) + match event { + Event::InteractionCreate(_) => {} + Event::MessageCreate(m) if m.author.id != our_user_id => {} + Event::MessageUpdate(m) + if let Some(author) = m.author.clone() + && author.id != our_user_id + && !author.bot => {} + Event::MessageDelete(_) => {} + Event::MessageDeleteBulk(_) => {} + Event::ReactionAdd(r) if r.user_id != our_user_id => {} + _ => { + continue; + } + } + + if runtime_config.exists(RUNTIME_CONFIG_KEY_EVENT_TARGET).await { + tx.send((shard.id(), raw_event)).await.unwrap(); + } } } diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 254ce504..cc360606 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -5,6 +5,7 @@ use chrono::Timelike; use discord::gateway::cluster_config; use fred::{clients::RedisPool, interfaces::*}; use libpk::runtime_config::RuntimeConfig; +use reqwest::ClientBuilder; use signal_hook::{ consts::{SIGINT, SIGTERM}, iterator::Signals, @@ -15,7 +16,7 @@ use std::{ vec::Vec, }; use tokio::task::JoinSet; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use twilight_gateway::{MessageSender, ShardId}; use twilight_model::gateway::payload::outgoing::UpdatePresence; @@ -23,6 +24,8 @@ mod cache_api; mod discord; mod logger; +const RUNTIME_CONFIG_KEY_EVENT_TARGET: &'static str = "event_target"; + libpk::main!("gateway"); async fn real_main() -> anyhow::Result<()> { let (shutdown_tx, shutdown_rx) = channel::<()>(); @@ -57,9 +60,40 @@ async fn real_main() -> anyhow::Result<()> { event_tx.clone(), shard_state.clone(), cache.clone(), + runtime_config.clone(), ))); } + set.spawn(tokio::spawn({ + let runtime_config = runtime_config.clone(); + async move { + let client = Arc::new(ClientBuilder::new() + .connect_timeout(Duration::from_secs(1)) + .timeout(Duration::from_secs(1)) + .build() + .expect("error making client")); + + while let Some((shard_id, event)) = event_rx.recv().await { + let target = runtime_config.get(RUNTIME_CONFIG_KEY_EVENT_TARGET).await; + if let Some(target) = target { + tokio::spawn({ + let client = client.clone(); + async move { + if let Err(error) = client + .post(format!("{target}/{}", shard_id.number())) + .body(event) + .send() + .await + { + error!(error = ?error, "failed to request event target") + } + } + }); + } + } + } + })); + set.spawn(tokio::spawn( async move { scheduled_task(redis, senders).await }, )); diff --git a/crates/libpk/src/lib.rs b/crates/libpk/src/lib.rs index 1864e332..10d174d0 100644 --- a/crates/libpk/src/lib.rs +++ b/crates/libpk/src/lib.rs @@ -43,6 +43,7 @@ pub fn init_logging(component: &str) { tracing_subscriber::registry() .with(sentry_layer) .with(tracing_subscriber::fmt::layer()) + .with(EnvFilter::from_default_env()) .init(); } } diff --git a/crates/libpk/src/runtime_config.rs b/crates/libpk/src/runtime_config.rs index 739818e4..e1fbea7a 100644 --- a/crates/libpk/src/runtime_config.rs +++ b/crates/libpk/src/runtime_config.rs @@ -33,7 +33,7 @@ impl RuntimeConfig { settings.insert(key, value); } - info!("starting with runtime config: {:?}", self.settings); + info!("starting with runtime config: {:?}", settings); Ok(()) } @@ -58,8 +58,8 @@ impl RuntimeConfig { Ok(()) } - pub async fn get(&self, key: String) -> Option { - self.settings.read().await.get(&key).cloned() + pub async fn get(&self, key: &str) -> Option { + self.settings.read().await.get(key).cloned() } pub async fn exists(&self, key: &str) -> bool { From cc7122e389b2d994259aea43ab3b6ab294907811 Mon Sep 17 00:00:00 2001 From: alyssa Date: Fri, 4 Apr 2025 11:09:41 +0000 Subject: [PATCH 03/80] fix(gateway): use tokio channels as to not block gateway on exit --- crates/gateway/src/main.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index cc360606..4213b235 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -10,12 +10,8 @@ use signal_hook::{ consts::{SIGINT, SIGTERM}, iterator::Signals, }; -use std::{ - sync::{mpsc::channel, Arc}, - time::Duration, - vec::Vec, -}; -use tokio::task::JoinSet; +use std::{sync::Arc, time::Duration, vec::Vec}; +use tokio::{sync::mpsc::channel, task::JoinSet}; use tracing::{error, info, warn}; use twilight_gateway::{MessageSender, ShardId}; use twilight_model::gateway::payload::outgoing::UpdatePresence; @@ -28,7 +24,7 @@ const RUNTIME_CONFIG_KEY_EVENT_TARGET: &'static str = "event_target"; libpk::main!("gateway"); async fn real_main() -> anyhow::Result<()> { - let (shutdown_tx, shutdown_rx) = channel::<()>(); + let (shutdown_tx, mut shutdown_rx) = channel::<()>(1); let shutdown_tx = Arc::new(shutdown_tx); let redis = libpk::db::init_redis().await?; @@ -46,7 +42,8 @@ async fn real_main() -> anyhow::Result<()> { let shards = discord::gateway::create_shards(redis.clone())?; - let (event_tx, _event_rx) = channel(); + // arbitrary + let (event_tx, mut event_rx) = channel(1000); let mut senders = Vec::new(); let mut signal_senders = Vec::new(); @@ -126,19 +123,19 @@ async fn real_main() -> anyhow::Result<()> { let _ = sender.command(&presence); } - let _ = shutdown_tx.send(()); + let _ = shutdown_tx.send(()).await; break; } })); - let _ = shutdown_rx.recv(); + let _ = shutdown_rx.recv().await; - // sleep 500ms to allow everything to clean up properly - tokio::time::sleep(Duration::from_millis(500)).await; + info!("gateway exiting, have a nice day!"); set.abort_all(); - info!("gateway exiting, have a nice day!"); + // sleep 500ms to allow everything to clean up properly + tokio::time::sleep(Duration::from_millis(500)).await; Ok(()) } From 7cd3939f954f49ab12d516c211aa889f9d648be1 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 22 Mar 2025 15:38:21 +0000 Subject: [PATCH 04/80] feat(bot): add config setting to disable connecting to discord gateway --- PluralKit.Bot/BotConfig.cs | 1 + PluralKit.Bot/Init.cs | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/PluralKit.Bot/BotConfig.cs b/PluralKit.Bot/BotConfig.cs index a9db7233..0be554a1 100644 --- a/PluralKit.Bot/BotConfig.cs +++ b/PluralKit.Bot/BotConfig.cs @@ -25,6 +25,7 @@ public class BotConfig public bool HttpUseInnerCache { get; set; } = false; public string? HttpListenerAddr { get; set; } + public bool DisableGateway { get; set; } = false; public string? DiscordBaseUrl { get; set; } public string? AvatarServiceUrl { get; set; } diff --git a/PluralKit.Bot/Init.cs b/PluralKit.Bot/Init.cs index f19576de..c3046aa9 100644 --- a/PluralKit.Bot/Init.cs +++ b/PluralKit.Bot/Init.cs @@ -84,9 +84,11 @@ public class Init services.Resolve().Start(config.HttpListenerAddr); // Start the Discord shards themselves (handlers already set up) - logger.Information("Connecting to Discord"); - await StartCluster(services); - + if (!config.DisableGateway) + { + logger.Information("Connecting to Discord"); + await StartCluster(services); + } logger.Information("Connected! All is good (probably)."); // Lastly, we just... wait. Everything else is handled in the DiscordClient event loop From 64ff69723cc588ce58eb2696799a145641128d37 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 29 Mar 2025 12:20:11 +0000 Subject: [PATCH 05/80] fix(bot): prefix command_message keys --- PluralKit.Bot/Services/CommandMessageService.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/PluralKit.Bot/Services/CommandMessageService.cs b/PluralKit.Bot/Services/CommandMessageService.cs index 796f7f0a..440670ab 100644 --- a/PluralKit.Bot/Services/CommandMessageService.cs +++ b/PluralKit.Bot/Services/CommandMessageService.cs @@ -27,7 +27,7 @@ public class CommandMessageService messageId, authorId, channelId ); - await _redis.Connection.GetDatabase().StringSetAsync(messageId.ToString(), $"{authorId}-{channelId}-{guildId}", expiry: CommandMessageRetention); + await _redis.Connection.GetDatabase().StringSetAsync("command_message:" + messageId.ToString(), $"{authorId}-{channelId}-{guildId}", expiry: CommandMessageRetention); } public async Task GetCommandMessage(ulong messageId) @@ -38,6 +38,12 @@ public class CommandMessageService var split = ((string)str).Split("-"); return new CommandMessage(ulong.Parse(split[0]), ulong.Parse(split[1]), ulong.Parse(split[2])); } + str = await _redis.Connection.GetDatabase().StringGetAsync("command_message:" + messageId.ToString()); + if (str.HasValue) + { + var split = ((string)str).Split("-"); + return new CommandMessage(ulong.Parse(split[0]), ulong.Parse(split[1]), ulong.Parse(split[2])); + } return null; } } From 15c992c572147b266859ed4a6c8c18afb4744b86 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 29 Mar 2025 17:55:01 +0000 Subject: [PATCH 06/80] feat: remote await events from gateway --- Cargo.lock | 1 + Myriad/Cache/HTTPDiscordCache.cs | 69 +++++- Myriad/Myriad.csproj | 2 + Myriad/packages.lock.json | 21 ++ PluralKit.Bot/BotConfig.cs | 1 + PluralKit.Bot/Interactive/BaseInteractive.cs | 4 +- PluralKit.Bot/Interactive/YesNoPrompt.cs | 15 ++ PluralKit.Bot/Modules.cs | 11 +- .../Services/InteractionDispatchService.cs | 15 +- PluralKit.Bot/Utils/ContextUtils.cs | 23 +- PluralKit.Bot/packages.lock.json | 2 + PluralKit.Tests/packages.lock.json | 2 + crates/gateway/Cargo.toml | 1 + crates/gateway/src/cache_api.rs | 25 +- crates/gateway/src/discord/gateway.rs | 10 +- crates/gateway/src/event_awaiter.rs | 223 ++++++++++++++++++ crates/gateway/src/main.rs | 44 +++- 17 files changed, 439 insertions(+), 30 deletions(-) create mode 100644 crates/gateway/src/event_awaiter.rs diff --git a/Cargo.lock b/Cargo.lock index a4a5e80c..75afc365 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1089,6 +1089,7 @@ dependencies = [ "libpk", "metrics", "reqwest 0.12.8", + "serde", "serde_json", "serde_variant", "signal-hook", diff --git a/Myriad/Cache/HTTPDiscordCache.cs b/Myriad/Cache/HTTPDiscordCache.cs index 31035660..311d697d 100644 --- a/Myriad/Cache/HTTPDiscordCache.cs +++ b/Myriad/Cache/HTTPDiscordCache.cs @@ -1,7 +1,10 @@ using Serilog; using System.Net; +using System.Text; using System.Text.Json; +using NodaTime; + using Myriad.Serialization; using Myriad.Types; @@ -12,6 +15,7 @@ public class HttpDiscordCache: IDiscordCache private readonly ILogger _logger; private readonly HttpClient _client; private readonly Uri _cacheEndpoint; + private readonly string? _eventTarget; private readonly int _shardCount; private readonly ulong _ownUserId; @@ -21,11 +25,12 @@ public class HttpDiscordCache: IDiscordCache public EventHandler<(bool?, string)> OnDebug; - public HttpDiscordCache(ILogger logger, HttpClient client, string cacheEndpoint, int shardCount, ulong ownUserId, bool useInnerCache) + public HttpDiscordCache(ILogger logger, HttpClient client, string cacheEndpoint, string? eventTarget, int shardCount, ulong ownUserId, bool useInnerCache) { _logger = logger; _client = client; _cacheEndpoint = new Uri(cacheEndpoint); + _eventTarget = eventTarget; _shardCount = shardCount; _ownUserId = ownUserId; _jsonSerializerOptions = new JsonSerializerOptions().ConfigureForMyriad(); @@ -65,6 +70,68 @@ public class HttpDiscordCache: IDiscordCache return JsonSerializer.Deserialize(plaintext, _jsonSerializerOptions); } + private Task AwaitEvent(ulong guildId, object data) + => AwaitEventShard((int)((guildId >> 22) % (ulong)_shardCount), data); + + private async Task AwaitEventShard(int shardId, object data) + { + if (_eventTarget == null) + throw new Exception("missing event target for remote await event"); + + var cluster = _cacheEndpoint.Authority; + // todo: there should not be infra-specific code here + if (cluster.Contains(".service.consul") || cluster.Contains("process.pluralkit-gateway.internal")) + // int(((guild_id >> 22) % shard_count) / 16) + cluster = $"cluster{shardId / 16}.{cluster}"; + + var response = await _client.PostAsync( + $"{_cacheEndpoint.Scheme}://{cluster}/await_event", + new StringContent(JsonSerializer.Serialize(data), Encoding.UTF8) + ); + + if (response.StatusCode != HttpStatusCode.NoContent) + throw new Exception($"failed to await event from gateway: {response.StatusCode}"); + } + + public async Task AwaitReaction(ulong guildId, ulong messageId, ulong userId, Duration? timeout) + { + var obj = new + { + message_id = messageId, + user_id = userId, + target = _eventTarget!, + timeout = timeout?.TotalSeconds, + }; + + await AwaitEvent(guildId, obj); + } + + public async Task AwaitMessage(ulong guildId, ulong channelId, ulong authorId, Duration? timeout, string[] options = null) + { + var obj = new + { + channel_id = channelId, + author_id = authorId, + target = _eventTarget!, + timeout = timeout?.TotalSeconds, + options = options, + }; + + await AwaitEvent(guildId, obj); + } + + public async Task AwaitInteraction(int shardId, string id, Duration? timeout) + { + var obj = new + { + id = id, + target = _eventTarget!, + timeout = timeout?.TotalSeconds, + }; + + await AwaitEventShard(shardId, obj); + } + public async Task TryGetGuild(ulong guildId) { var hres = await QueryCache($"/guilds/{guildId}", guildId); diff --git a/Myriad/Myriad.csproj b/Myriad/Myriad.csproj index 3b92ea30..0a6ed588 100644 --- a/Myriad/Myriad.csproj +++ b/Myriad/Myriad.csproj @@ -21,6 +21,8 @@ + + diff --git a/Myriad/packages.lock.json b/Myriad/packages.lock.json index 945321a9..7680637b 100644 --- a/Myriad/packages.lock.json +++ b/Myriad/packages.lock.json @@ -2,6 +2,22 @@ "version": 1, "dependencies": { "net8.0": { + "NodaTime": { + "type": "Direct", + "requested": "[3.2.0, )", + "resolved": "3.2.0", + "contentHash": "yoRA3jEJn8NM0/rQm78zuDNPA3DonNSZdsorMUj+dltc1D+/Lc5h9YXGqbEEZozMGr37lAoYkcSM/KjTVqD0ow==" + }, + "NodaTime.Serialization.JsonNet": { + "type": "Direct", + "requested": "[3.1.0, )", + "resolved": "3.1.0", + "contentHash": "eEr9lXUz50TYr4rpeJG4TDAABkpxjIKr5mDSi/Zav8d6Njy6fH7x4ZtNwWFj0Vd+vIvEZNrHFQ4Gfy8j4BqRGg==", + "dependencies": { + "Newtonsoft.Json": "13.0.3", + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Polly": { "type": "Direct", "requested": "[8.5.0, )", @@ -52,6 +68,11 @@ "resolved": "6.0.0", "contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.3", + "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" + }, "Pipelines.Sockets.Unofficial": { "type": "Transitive", "resolved": "2.2.8", diff --git a/PluralKit.Bot/BotConfig.cs b/PluralKit.Bot/BotConfig.cs index 0be554a1..1e6e0f0b 100644 --- a/PluralKit.Bot/BotConfig.cs +++ b/PluralKit.Bot/BotConfig.cs @@ -26,6 +26,7 @@ public class BotConfig public string? HttpListenerAddr { get; set; } public bool DisableGateway { get; set; } = false; + public string? EventAwaiterTarget { get; set; } public string? DiscordBaseUrl { get; set; } public string? AvatarServiceUrl { get; set; } diff --git a/PluralKit.Bot/Interactive/BaseInteractive.cs b/PluralKit.Bot/Interactive/BaseInteractive.cs index c2ae33ee..779058fe 100644 --- a/PluralKit.Bot/Interactive/BaseInteractive.cs +++ b/PluralKit.Bot/Interactive/BaseInteractive.cs @@ -28,7 +28,7 @@ public abstract class BaseInteractive ButtonStyle style = ButtonStyle.Secondary, bool disabled = false) { var dispatch = _ctx.Services.Resolve(); - var customId = dispatch.Register(handler, Timeout); + var customId = dispatch.Register(_ctx.ShardId, handler, Timeout); var button = new Button { @@ -89,7 +89,7 @@ public abstract class BaseInteractive { var dispatch = ctx.Services.Resolve(); foreach (var button in _buttons) - button.CustomId = dispatch.Register(button.Handler, Timeout); + button.CustomId = dispatch.Register(_ctx.ShardId, button.Handler, Timeout); } public abstract Task Start(); diff --git a/PluralKit.Bot/Interactive/YesNoPrompt.cs b/PluralKit.Bot/Interactive/YesNoPrompt.cs index 110e4bb9..194dd1f1 100644 --- a/PluralKit.Bot/Interactive/YesNoPrompt.cs +++ b/PluralKit.Bot/Interactive/YesNoPrompt.cs @@ -1,5 +1,6 @@ using Autofac; +using Myriad.Cache; using Myriad.Gateway; using Myriad.Rest.Types; using Myriad.Types; @@ -69,6 +70,9 @@ public class YesNoPrompt: BaseInteractive return true; } + // no need to reawait message + // gateway will already have sent us only matching messages + return false; } @@ -88,6 +92,17 @@ public class YesNoPrompt: BaseInteractive { try { + // check if http gateway and set listener + // todo: this one needs to handle options for message + if (_ctx.Cache is HttpDiscordCache) + await (_ctx.Cache as HttpDiscordCache).AwaitMessage( + _ctx.Guild?.Id ?? 0, + _ctx.Channel.Id, + _ctx.Author.Id, + Timeout, + options: new[] { "yes", "y", "no", "n" } + ); + await queue.WaitFor(MessagePredicate, Timeout, cts.Token); } catch (TimeoutException e) diff --git a/PluralKit.Bot/Modules.cs b/PluralKit.Bot/Modules.cs index ae6f85ec..668ea9c3 100644 --- a/PluralKit.Bot/Modules.cs +++ b/PluralKit.Bot/Modules.cs @@ -49,8 +49,15 @@ public class BotModule: Module if (botConfig.HttpCacheUrl != null) { - var cache = new HttpDiscordCache(c.Resolve(), - c.Resolve(), botConfig.HttpCacheUrl, botConfig.Cluster?.TotalShards ?? 1, botConfig.ClientId, botConfig.HttpUseInnerCache); + var cache = new HttpDiscordCache( + c.Resolve(), + c.Resolve(), + botConfig.HttpCacheUrl, + botConfig.EventAwaiterTarget, + botConfig.Cluster?.TotalShards ?? 1, + botConfig.ClientId, + botConfig.HttpUseInnerCache + ); var metrics = c.Resolve(); diff --git a/PluralKit.Bot/Services/InteractionDispatchService.cs b/PluralKit.Bot/Services/InteractionDispatchService.cs index 968ea35a..f900a792 100644 --- a/PluralKit.Bot/Services/InteractionDispatchService.cs +++ b/PluralKit.Bot/Services/InteractionDispatchService.cs @@ -1,5 +1,7 @@ using System.Collections.Concurrent; +using Myriad.Cache; + using NodaTime; using Serilog; @@ -16,9 +18,12 @@ public class InteractionDispatchService: IDisposable private readonly ConcurrentDictionary _handlers = new(); private readonly ILogger _logger; - public InteractionDispatchService(IClock clock, ILogger logger) + private readonly IDiscordCache _cache; + + public InteractionDispatchService(IClock clock, ILogger logger, IDiscordCache cache) { _clock = clock; + _cache = cache; _logger = logger.ForContext(); _cleanupWorker = CleanupLoop(_cts.Token); @@ -50,9 +55,15 @@ public class InteractionDispatchService: IDisposable _handlers.TryRemove(customIdGuid, out _); } - public string Register(Func callback, Duration? expiry = null) + public string Register(int shardId, Func callback, Duration? expiry = null) { var key = Guid.NewGuid(); + + // if http_cache, return RegisterRemote + // not awaited here, it's probably fine + if (_cache is HttpDiscordCache) + (_cache as HttpDiscordCache).AwaitInteraction(shardId, key.ToString(), expiry); + var handler = new RegisteredInteraction { Callback = callback, diff --git a/PluralKit.Bot/Utils/ContextUtils.cs b/PluralKit.Bot/Utils/ContextUtils.cs index ce353472..7cd3de42 100644 --- a/PluralKit.Bot/Utils/ContextUtils.cs +++ b/PluralKit.Bot/Utils/ContextUtils.cs @@ -1,6 +1,7 @@ using Autofac; using Myriad.Builders; +using Myriad.Cache; using Myriad.Gateway; using Myriad.Rest.Exceptions; using Myriad.Rest.Types.Requests; @@ -40,8 +41,12 @@ public static class ContextUtils } public static async Task AwaitReaction(this Context ctx, Message message, - User user = null, Func predicate = null, Duration? timeout = null) + User user, Func predicate = null, Duration? timeout = null) { + // check if http gateway and set listener + if (ctx.Cache is HttpDiscordCache) + await (ctx.Cache as HttpDiscordCache).AwaitReaction(ctx.Guild?.Id ?? 0, message.Id, user!.Id, timeout); + bool ReactionPredicate(MessageReactionAddEvent evt) { if (message.Id != evt.MessageId) return false; // Ignore reactions for different messages @@ -57,11 +62,17 @@ public static class ContextUtils public static async Task ConfirmWithReply(this Context ctx, string expectedReply, bool treatAsHid = false) { + var timeout = Duration.FromMinutes(1); + + // check if http gateway and set listener + if (ctx.Cache is HttpDiscordCache) + await (ctx.Cache as HttpDiscordCache).AwaitMessage(ctx.Guild?.Id ?? 0, ctx.Channel.Id, ctx.Author.Id, timeout); + bool Predicate(MessageCreateEvent e) => e.Author.Id == ctx.Author.Id && e.ChannelId == ctx.Channel.Id; var msg = await ctx.Services.Resolve>() - .WaitFor(Predicate, Duration.FromMinutes(1)); + .WaitFor(Predicate, timeout); var content = msg.Content; if (treatAsHid) @@ -96,11 +107,17 @@ public static class ContextUtils async Task PromptPageNumber() { + var timeout = Duration.FromMinutes(0.5); + + // check if http gateway and set listener + if (ctx.Cache is HttpDiscordCache) + await (ctx.Cache as HttpDiscordCache).AwaitMessage(ctx.Guild?.Id ?? 0, ctx.Channel.Id, ctx.Author.Id, timeout); + bool Predicate(MessageCreateEvent e) => e.Author.Id == ctx.Author.Id && e.ChannelId == ctx.Channel.Id; var msg = await ctx.Services.Resolve>() - .WaitFor(Predicate, Duration.FromMinutes(0.5)); + .WaitFor(Predicate, timeout); int.TryParse(msg.Content, out var num); diff --git a/PluralKit.Bot/packages.lock.json b/PluralKit.Bot/packages.lock.json index fca78f6e..79da3d52 100644 --- a/PluralKit.Bot/packages.lock.json +++ b/PluralKit.Bot/packages.lock.json @@ -764,6 +764,8 @@ "myriad": { "type": "Project", "dependencies": { + "NodaTime": "[3.2.0, )", + "NodaTime.Serialization.JsonNet": "[3.1.0, )", "Polly": "[8.5.0, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Serilog": "[4.2.0, )", diff --git a/PluralKit.Tests/packages.lock.json b/PluralKit.Tests/packages.lock.json index c11cedb5..b9e49ba9 100644 --- a/PluralKit.Tests/packages.lock.json +++ b/PluralKit.Tests/packages.lock.json @@ -976,6 +976,8 @@ "myriad": { "type": "Project", "dependencies": { + "NodaTime": "[3.2.0, )", + "NodaTime.Serialization.JsonNet": "[3.1.0, )", "Polly": "[8.5.0, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Serilog": "[4.2.0, )", diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index e9fd444e..420aef1c 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -14,6 +14,7 @@ lazy_static = { workspace = true } libpk = { path = "../libpk" } metrics = { workspace = true } reqwest = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } signal-hook = { workspace = true } tokio = { workspace = true } diff --git a/crates/gateway/src/cache_api.rs b/crates/gateway/src/cache_api.rs index e3de4696..d22fb113 100644 --- a/crates/gateway/src/cache_api.rs +++ b/crates/gateway/src/cache_api.rs @@ -10,9 +10,12 @@ use serde_json::{json, to_string}; use tracing::{error, info}; use twilight_model::id::Id; -use crate::discord::{ - cache::{dm_channel, DiscordCache, DM_PERMISSIONS}, - gateway::cluster_config, +use crate::{ + discord::{ + cache::{dm_channel, DiscordCache, DM_PERMISSIONS}, + gateway::cluster_config, + }, + event_awaiter::{AwaitEventRequest, EventAwaiter}, }; use std::sync::Arc; @@ -22,10 +25,11 @@ fn status_code(code: StatusCode, body: String) -> Response { // this function is manually formatted for easier legibility of route_services #[rustfmt::skip] -pub async fn run_server(cache: Arc, runtime_config: Arc) -> anyhow::Result<()> { +pub async fn run_server(cache: Arc, runtime_config: Arc, awaiter: Arc) -> anyhow::Result<()> { // hacky fix for `move` let runtime_config_for_post = runtime_config.clone(); let runtime_config_for_delete = runtime_config.clone(); + let awaiter_for_clear = awaiter.clone(); let app = Router::new() .route( @@ -190,6 +194,19 @@ pub async fn run_server(cache: Arc, runtime_config: Arc(&body) else { + return status_code(StatusCode::BAD_REQUEST, "".to_string()); + }; + awaiter.handle_request(req).await; + status_code(StatusCode::NO_CONTENT, "".to_string()) + })) + .route("/clear_awaiter", post(|| async move { + awaiter_for_clear.clear().await; + status_code(StatusCode::NO_CONTENT, "".to_string()) + })) + .layer(axum::middleware::from_fn(crate::logger::logger)) .with_state(cache); diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index c4a13a81..89dfc26f 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -82,7 +82,7 @@ pub fn create_shards(redis: fred::clients::RedisPool) -> anyhow::Result, - tx: Sender<(ShardId, String)>, + tx: Sender<(ShardId, Event, String)>, shard_state: ShardStateManager, cache: Arc, runtime_config: Arc, @@ -182,21 +182,21 @@ pub async fn runner( // and the default match skips the next block (continues to the next event) match event { Event::InteractionCreate(_) => {} - Event::MessageCreate(m) if m.author.id != our_user_id => {} - Event::MessageUpdate(m) + Event::MessageCreate(ref m) if m.author.id != our_user_id => {} + Event::MessageUpdate(ref m) if let Some(author) = m.author.clone() && author.id != our_user_id && !author.bot => {} Event::MessageDelete(_) => {} Event::MessageDeleteBulk(_) => {} - Event::ReactionAdd(r) if r.user_id != our_user_id => {} + Event::ReactionAdd(ref r) if r.user_id != our_user_id => {} _ => { continue; } } if runtime_config.exists(RUNTIME_CONFIG_KEY_EVENT_TARGET).await { - tx.send((shard.id(), raw_event)).await.unwrap(); + tx.send((shard.id(), event, raw_event)).await.unwrap(); } } } diff --git a/crates/gateway/src/event_awaiter.rs b/crates/gateway/src/event_awaiter.rs new file mode 100644 index 00000000..9196170d --- /dev/null +++ b/crates/gateway/src/event_awaiter.rs @@ -0,0 +1,223 @@ +// - reaction: (message_id, user_id) +// - message: (author_id, channel_id, ?options) +// - interaction: (custom_id where not_includes "help-menu") + +use std::{ + collections::{hash_map::Entry, HashMap}, + time::Duration, +}; + +use serde::Deserialize; +use tokio::{sync::RwLock, time::Instant}; +use tracing::info; +use twilight_gateway::Event; +use twilight_model::{ + application::interaction::InteractionData, + id::{ + marker::{ChannelMarker, MessageMarker, UserMarker}, + Id, + }, +}; + +static DEFAULT_TIMEOUT: Duration = Duration::from_mins(15); + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum AwaitEventRequest { + Reaction { + message_id: Id, + user_id: Id, + target: String, + timeout: Option, + }, + Message { + channel_id: Id, + author_id: Id, + target: String, + timeout: Option, + options: Option>, + }, + Interaction { + id: String, + target: String, + timeout: Option, + }, +} + +pub struct EventAwaiter { + reactions: RwLock, Id), (Instant, String)>>, + messages: RwLock< + HashMap<(Id, Id), (Instant, String, Option>)>, + >, + interactions: RwLock>, +} + +impl EventAwaiter { + pub fn new() -> Self { + let v = Self { + reactions: RwLock::new(HashMap::new()), + messages: RwLock::new(HashMap::new()), + interactions: RwLock::new(HashMap::new()), + }; + + v + } + + pub async fn cleanup_loop(&self) { + loop { + tokio::time::sleep(Duration::from_secs(30)).await; + info!("running event_awaiter cleanup loop"); + let mut counts = (0, 0, 0); + let now = Instant::now(); + { + let mut reactions = self.reactions.write().await; + for key in reactions.clone().keys() { + if let Entry::Occupied(entry) = reactions.entry(key.clone()) + && entry.get().0 < now + { + counts.0 += 1; + entry.remove(); + } + } + } + { + let mut messages = self.messages.write().await; + for key in messages.clone().keys() { + if let Entry::Occupied(entry) = messages.entry(key.clone()) + && entry.get().0 < now + { + counts.1 += 1; + entry.remove(); + } + } + } + { + let mut interactions = self.interactions.write().await; + for key in interactions.clone().keys() { + if let Entry::Occupied(entry) = interactions.entry(key.clone()) + && entry.get().0 < now + { + counts.2 += 1; + entry.remove(); + } + } + } + info!("ran event_awaiter cleanup loop, took {}us, {} reactions, {} messages, {} interactions", Instant::now().duration_since(now).as_micros(), counts.0, counts.1, counts.2); + } + } + + pub async fn target_for_event(&self, event: Event) -> Option { + match event { + Event::MessageCreate(message) => { + let mut messages = self.messages.write().await; + + messages + .remove(&(message.channel_id, message.author.id)) + .map(|(timeout, target, options)| { + if let Some(options) = options + && !options.contains(&message.content) + { + messages.insert( + (message.channel_id, message.author.id), + (timeout, target, Some(options)), + ); + return None; + } + Some((*target).to_string()) + })? + } + Event::ReactionAdd(reaction) + if let Some((_, target)) = self + .reactions + .write() + .await + .remove(&(reaction.message_id, reaction.user_id)) => + { + Some((*target).to_string()) + } + Event::InteractionCreate(interaction) + if let Some(data) = interaction.data.clone() + && let InteractionData::MessageComponent(component) = data + && !component.custom_id.contains("help-menu") + && let Some((_, target)) = + self.interactions.write().await.remove(&component.custom_id) => + { + Some((*target).to_string()) + } + + _ => None, + } + } + + pub async fn handle_request(&self, req: AwaitEventRequest) { + match req { + AwaitEventRequest::Reaction { + message_id, + user_id, + target, + timeout, + } => { + self.reactions.write().await.insert( + (message_id, user_id), + ( + Instant::now() + .checked_add( + timeout + .map(|i| Duration::from_secs(i)) + .unwrap_or(DEFAULT_TIMEOUT), + ) + .expect("invalid time"), + target, + ), + ); + } + AwaitEventRequest::Message { + channel_id, + author_id, + target, + timeout, + options, + } => { + self.messages.write().await.insert( + (channel_id, author_id), + ( + Instant::now() + .checked_add( + timeout + .map(|i| Duration::from_secs(i)) + .unwrap_or(DEFAULT_TIMEOUT), + ) + .expect("invalid time"), + target, + options, + ), + ); + } + AwaitEventRequest::Interaction { + id, + target, + timeout, + } => { + self.interactions.write().await.insert( + id, + ( + Instant::now() + .checked_add( + timeout + .map(|i| Duration::from_secs(i)) + .unwrap_or(DEFAULT_TIMEOUT), + ) + .expect("invalid time"), + target, + ), + ); + } + } + } + + pub async fn clear(&self) { + self.reactions.write().await.clear(); + self.messages.write().await.clear(); + self.interactions.write().await.clear(); + } +} diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 4213b235..ab679c06 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -1,8 +1,10 @@ #![feature(let_chains)] #![feature(if_let_guard)] +#![feature(duration_constructors)] use chrono::Timelike; use discord::gateway::cluster_config; +use event_awaiter::EventAwaiter; use fred::{clients::RedisPool, interfaces::*}; use libpk::runtime_config::RuntimeConfig; use reqwest::ClientBuilder; @@ -12,12 +14,13 @@ use signal_hook::{ }; use std::{sync::Arc, time::Duration, vec::Vec}; use tokio::{sync::mpsc::channel, task::JoinSet}; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; use twilight_gateway::{MessageSender, ShardId}; use twilight_model::gateway::payload::outgoing::UpdatePresence; mod cache_api; mod discord; +mod event_awaiter; mod logger; const RUNTIME_CONFIG_KEY_EVENT_TARGET: &'static str = "event_target"; @@ -39,6 +42,11 @@ async fn real_main() -> anyhow::Result<()> { let shard_state = discord::shard_state::new(redis.clone()); let cache = Arc::new(discord::cache::new()); + let awaiter = Arc::new(EventAwaiter::new()); + tokio::spawn({ + let awaiter = awaiter.clone(); + async move { awaiter.cleanup_loop().await } + }); let shards = discord::gateway::create_shards(redis.clone())?; @@ -63,22 +71,36 @@ async fn real_main() -> anyhow::Result<()> { set.spawn(tokio::spawn({ let runtime_config = runtime_config.clone(); - async move { - let client = Arc::new(ClientBuilder::new() - .connect_timeout(Duration::from_secs(1)) - .timeout(Duration::from_secs(1)) - .build() - .expect("error making client")); + let awaiter = awaiter.clone(); + + async move { + let client = Arc::new( + ClientBuilder::new() + .connect_timeout(Duration::from_secs(1)) + .timeout(Duration::from_secs(1)) + .build() + .expect("error making client"), + ); + + while let Some((shard_id, parsed_event, raw_event)) = event_rx.recv().await { + let target = if let Some(target) = awaiter.target_for_event(parsed_event).await { + debug!("sending event to awaiter"); + Some(target) + } else if let Some(target) = + runtime_config.get(RUNTIME_CONFIG_KEY_EVENT_TARGET).await + { + Some(target) + } else { + None + }; - while let Some((shard_id, event)) = event_rx.recv().await { - let target = runtime_config.get(RUNTIME_CONFIG_KEY_EVENT_TARGET).await; if let Some(target) = target { tokio::spawn({ let client = client.clone(); async move { if let Err(error) = client .post(format!("{target}/{}", shard_id.number())) - .body(event) + .body(raw_event) .send() .await { @@ -98,7 +120,7 @@ async fn real_main() -> anyhow::Result<()> { // todo: probably don't do it this way let api_shutdown_tx = shutdown_tx.clone(); set.spawn(tokio::spawn(async move { - match cache_api::run_server(cache, runtime_config).await { + match cache_api::run_server(cache, runtime_config, awaiter.clone()).await { Err(error) => { tracing::error!(?error, "failed to serve cache api"); let _ = api_shutdown_tx.send(()); From a8664665a693f9fc67038c9c597518dccb97ec06 Mon Sep 17 00:00:00 2001 From: alyssa Date: Tue, 1 Apr 2025 10:48:20 +0000 Subject: [PATCH 07/80] feat: add last message cache to gateway --- Myriad/Cache/HTTPDiscordCache.cs | 3 + PluralKit.Bot/Commands/Message.cs | 2 +- PluralKit.Bot/Handlers/MessageCreated.cs | 4 +- PluralKit.Bot/Handlers/MessageEdited.cs | 2 +- PluralKit.Bot/Proxy/ProxyService.cs | 6 +- .../Services/LastMessageCacheService.cs | 36 ++++- crates/gateway/src/cache_api.rs | 7 +- crates/gateway/src/discord/cache.rs | 126 +++++++++++++++++- crates/gateway/src/discord/gateway.rs | 2 +- crates/gateway/src/event_awaiter.rs | 2 +- 10 files changed, 172 insertions(+), 18 deletions(-) diff --git a/Myriad/Cache/HTTPDiscordCache.cs b/Myriad/Cache/HTTPDiscordCache.cs index 311d697d..b70fcf12 100644 --- a/Myriad/Cache/HTTPDiscordCache.cs +++ b/Myriad/Cache/HTTPDiscordCache.cs @@ -70,6 +70,9 @@ public class HttpDiscordCache: IDiscordCache return JsonSerializer.Deserialize(plaintext, _jsonSerializerOptions); } + public Task GetLastMessage(ulong guildId, ulong channelId) + => QueryCache($"/guilds/{guildId}/channels/{channelId}/last_message", guildId); + private Task AwaitEvent(ulong guildId, object data) => AwaitEventShard((int)((guildId >> 22) % (ulong)_shardCount), data); diff --git a/PluralKit.Bot/Commands/Message.cs b/PluralKit.Bot/Commands/Message.cs index 692f75b1..94602a06 100644 --- a/PluralKit.Bot/Commands/Message.cs +++ b/PluralKit.Bot/Commands/Message.cs @@ -305,7 +305,7 @@ public class ProxiedMessage throw new PKError(error); } - var lastMessage = _lastMessageCache.GetLastMessage(ctx.Message.ChannelId); + var lastMessage = await _lastMessageCache.GetLastMessage(ctx.Message.GuildId ?? 0, ctx.Message.ChannelId); var isLatestMessage = lastMessage?.Current.Id == ctx.Message.Id ? lastMessage?.Previous?.Id == msg.Mid diff --git a/PluralKit.Bot/Handlers/MessageCreated.cs b/PluralKit.Bot/Handlers/MessageCreated.cs index 2bb6bfcf..a14d024b 100644 --- a/PluralKit.Bot/Handlers/MessageCreated.cs +++ b/PluralKit.Bot/Handlers/MessageCreated.cs @@ -55,7 +55,9 @@ public class MessageCreated: IEventHandler public (ulong?, ulong?) ErrorChannelFor(MessageCreateEvent evt, ulong userId) => (evt.GuildId, evt.ChannelId); private bool IsDuplicateMessage(Message msg) => // We consider a message duplicate if it has the same ID as the previous message that hit the gateway - _lastMessageCache.GetLastMessage(msg.ChannelId)?.Current.Id == msg.Id; + // use only the local cache here + // http gateway sets last message before forwarding the message here, so this will always return true + _lastMessageCache._GetLastMessage(msg.ChannelId)?.Current.Id == msg.Id; public async Task Handle(int shardId, MessageCreateEvent evt) { diff --git a/PluralKit.Bot/Handlers/MessageEdited.cs b/PluralKit.Bot/Handlers/MessageEdited.cs index 8e131c0c..6ffcd00e 100644 --- a/PluralKit.Bot/Handlers/MessageEdited.cs +++ b/PluralKit.Bot/Handlers/MessageEdited.cs @@ -65,7 +65,7 @@ public class MessageEdited: IEventHandler var guild = await _cache.TryGetGuild(channel.GuildId!.Value); if (guild == null) throw new Exception("could not find self guild in MessageEdited event"); - var lastMessage = _lastMessageCache.GetLastMessage(evt.ChannelId)?.Current; + var lastMessage = (await _lastMessageCache.GetLastMessage(evt.GuildId.HasValue ? evt.GuildId.Value ?? 0 : 0, evt.ChannelId))?.Current; // Only react to the last message in the channel if (lastMessage?.Id != evt.Id) diff --git a/PluralKit.Bot/Proxy/ProxyService.cs b/PluralKit.Bot/Proxy/ProxyService.cs index 43e1ed5e..8a59957d 100644 --- a/PluralKit.Bot/Proxy/ProxyService.cs +++ b/PluralKit.Bot/Proxy/ProxyService.cs @@ -246,7 +246,7 @@ public class ProxyService ChannelId = rootChannel.Id, ThreadId = threadId, MessageId = trigger.Id, - Name = await FixSameName(messageChannel.Id, ctx, match.Member), + Name = await FixSameName(trigger.GuildId!.Value, messageChannel.Id, ctx, match.Member), AvatarUrl = AvatarUtils.TryRewriteCdnUrl(match.Member.ProxyAvatar(ctx)), Content = content, Attachments = trigger.Attachments, @@ -458,11 +458,11 @@ public class ProxyService }; } - private async Task FixSameName(ulong channelId, MessageContext ctx, ProxyMember member) + private async Task FixSameName(ulong guildId, ulong channelId, MessageContext ctx, ProxyMember member) { var proxyName = member.ProxyName(ctx); - var lastMessage = _lastMessage.GetLastMessage(channelId)?.Previous; + var lastMessage = (await _lastMessage.GetLastMessage(guildId, channelId))?.Previous; if (lastMessage == null) // cache is out of date or channel is empty. return proxyName; diff --git a/PluralKit.Bot/Services/LastMessageCacheService.cs b/PluralKit.Bot/Services/LastMessageCacheService.cs index a06da7fa..46a51c64 100644 --- a/PluralKit.Bot/Services/LastMessageCacheService.cs +++ b/PluralKit.Bot/Services/LastMessageCacheService.cs @@ -1,6 +1,7 @@ #nullable enable using System.Collections.Concurrent; +using Myriad.Cache; using Myriad.Types; namespace PluralKit.Bot; @@ -9,9 +10,18 @@ public class LastMessageCacheService { private readonly IDictionary _cache = new ConcurrentDictionary(); + private readonly IDiscordCache _maybeHttp; + + public LastMessageCacheService(IDiscordCache cache) + { + _maybeHttp = cache; + } + public void AddMessage(Message msg) { - var previous = GetLastMessage(msg.ChannelId); + if (_maybeHttp is HttpDiscordCache) return; + + var previous = _GetLastMessage(msg.ChannelId); var current = ToCachedMessage(msg); _cache[msg.ChannelId] = new CacheEntry(current, previous?.Current); } @@ -19,12 +29,26 @@ public class LastMessageCacheService private CachedMessage ToCachedMessage(Message msg) => new(msg.Id, msg.ReferencedMessage.Value?.Id, msg.Author.Username); - public CacheEntry? GetLastMessage(ulong channel) => - _cache.TryGetValue(channel, out var message) ? message : null; + public async Task GetLastMessage(ulong guild, ulong channel) + { + if (_maybeHttp is HttpDiscordCache) + return await (_maybeHttp as HttpDiscordCache).GetLastMessage(guild, channel); + + return _cache.TryGetValue(channel, out var message) ? message : null; + } + + public CacheEntry? _GetLastMessage(ulong channel) + { + if (_maybeHttp is HttpDiscordCache) return null; + + return _cache.TryGetValue(channel, out var message) ? message : null; + } public void HandleMessageDeletion(ulong channel, ulong message) { - var storedMessage = GetLastMessage(channel); + if (_maybeHttp is HttpDiscordCache) return; + + var storedMessage = _GetLastMessage(channel); if (storedMessage == null) return; @@ -39,7 +63,9 @@ public class LastMessageCacheService public void HandleMessageDeletion(ulong channel, List messages) { - var storedMessage = GetLastMessage(channel); + if (_maybeHttp is HttpDiscordCache) return; + + var storedMessage = _GetLastMessage(channel); if (storedMessage == null) return; diff --git a/crates/gateway/src/cache_api.rs b/crates/gateway/src/cache_api.rs index d22fb113..2e91f465 100644 --- a/crates/gateway/src/cache_api.rs +++ b/crates/gateway/src/cache_api.rs @@ -8,7 +8,7 @@ use axum::{ use libpk::runtime_config::RuntimeConfig; use serde_json::{json, to_string}; use tracing::{error, info}; -use twilight_model::id::Id; +use twilight_model::id::{marker::ChannelMarker, Id}; use crate::{ discord::{ @@ -136,7 +136,10 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path((_guild_id, channel_id)): Path<(u64, Id)>| async move { + let lm = cache.get_last_message(channel_id).await; + status_code(StatusCode::FOUND, to_string(&lm).unwrap()) + }), ) .route( diff --git a/crates/gateway/src/discord/cache.rs b/crates/gateway/src/discord/cache.rs index b4a81664..2b5b22b8 100644 --- a/crates/gateway/src/discord/cache.rs +++ b/crates/gateway/src/discord/cache.rs @@ -1,6 +1,7 @@ use anyhow::format_err; use lazy_static::lazy_static; -use std::sync::Arc; +use serde::Serialize; +use std::{collections::HashMap, sync::Arc}; use tokio::sync::RwLock; use twilight_cache_inmemory::{ model::CachedMember, @@ -8,11 +9,12 @@ use twilight_cache_inmemory::{ traits::CacheableChannel, InMemoryCache, ResourceType, }; +use twilight_gateway::Event; use twilight_model::{ channel::{Channel, ChannelType}, guild::{Guild, Member, Permissions}, id::{ - marker::{ChannelMarker, GuildMarker, UserMarker}, + marker::{ChannelMarker, GuildMarker, MessageMarker, UserMarker}, Id, }, }; @@ -123,16 +125,134 @@ pub fn new() -> DiscordCache { .build(), ); - DiscordCache(cache, client, RwLock::new(Vec::new())) + DiscordCache( + cache, + client, + RwLock::new(Vec::new()), + RwLock::new(HashMap::new()), + ) +} + +#[derive(Clone, Serialize)] +pub struct CachedMessage { + id: Id, + referenced_message: Option>, + author_username: String, +} + +#[derive(Clone, Serialize)] +pub struct LastMessageCacheEntry { + pub current: CachedMessage, + pub previous: Option, } pub struct DiscordCache( pub Arc, pub Arc, pub RwLock>, + pub RwLock, LastMessageCacheEntry>>, ); impl DiscordCache { + pub async fn get_last_message( + &self, + channel: Id, + ) -> Option { + self.3.read().await.get(&channel).cloned() + } + + pub async fn update(&self, event: &twilight_gateway::Event) { + self.0.update(event); + + match event { + Event::MessageCreate(m) => match self.3.write().await.entry(m.channel_id) { + std::collections::hash_map::Entry::Occupied(mut e) => { + let cur = e.get(); + e.insert(LastMessageCacheEntry { + current: CachedMessage { + id: m.id, + referenced_message: m.referenced_message.as_ref().map(|v| v.id), + author_username: m.author.name.clone(), + }, + previous: Some(cur.current.clone()), + }); + } + std::collections::hash_map::Entry::Vacant(e) => { + e.insert(LastMessageCacheEntry { + current: CachedMessage { + id: m.id, + referenced_message: m.referenced_message.as_ref().map(|v| v.id), + author_username: m.author.name.clone(), + }, + previous: None, + }); + } + }, + Event::MessageDelete(m) => { + self.handle_message_deletion(m.channel_id, vec![m.id]).await; + } + Event::MessageDeleteBulk(m) => { + self.handle_message_deletion(m.channel_id, m.ids.clone()) + .await; + } + _ => {} + }; + } + + async fn handle_message_deletion( + &self, + channel_id: Id, + mids: Vec>, + ) { + let mut lm = self.3.write().await; + + let Some(entry) = lm.get(&channel_id) else { + return; + }; + + let mut entry = entry.clone(); + + // if none of the deleted messages are relevant, just return + if !mids.contains(&entry.current.id) + && entry + .previous + .clone() + .map(|v| !mids.contains(&v.id)) + .unwrap_or(false) + { + return; + } + + // remove "previous" entry if it was deleted + if let Some(prev) = entry.previous.clone() + && mids.contains(&prev.id) + { + entry.previous = None; + } + + // set "current" entry to "previous" if current entry was deleted + // (if the "previous" entry still exists, it was not deleted) + if let Some(prev) = entry.previous.clone() + && mids.contains(&entry.current.id) + { + entry.current = prev; + entry.previous = None; + } + + // if the current entry was already deleted, but previous wasn't, + // we would've set current to previous + // so if current is deleted this means both current and previous have + // been deleted + // so just drop the cache entry here + if mids.contains(&entry.current.id) && entry.previous.is_none() { + lm.remove(&channel_id); + return; + } + + // ok, update the entry + lm.insert(channel_id, entry.clone()); + } + pub async fn guild_permissions( &self, guild_id: Id, diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index 89dfc26f..fc881f34 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -173,7 +173,7 @@ pub async fn runner( cache.2.write().await.push(shard_id); } } - cache.0.update(&event); + cache.update(&event).await; // okay, we've handled the event internally, let's send it to consumers diff --git a/crates/gateway/src/event_awaiter.rs b/crates/gateway/src/event_awaiter.rs index 9196170d..10f1888d 100644 --- a/crates/gateway/src/event_awaiter.rs +++ b/crates/gateway/src/event_awaiter.rs @@ -115,7 +115,7 @@ impl EventAwaiter { .remove(&(message.channel_id, message.author.id)) .map(|(timeout, target, options)| { if let Some(options) = options - && !options.contains(&message.content) + && !options.contains(&message.content.to_lowercase()) { messages.insert( (message.channel_id, message.author.id), From 2710d1fb2be4f7ba70493107a5c994b1a5330cbe Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 22 Mar 2025 15:50:08 +0000 Subject: [PATCH 08/80] feat: admin requests --- ci/Dockerfile.rust | 2 + ci/rust-docker-target.sh | 1 + crates/gdpr_worker/Cargo.toml | 15 ++++ crates/gdpr_worker/src/main.rs | 147 +++++++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+) create mode 100644 crates/gdpr_worker/Cargo.toml create mode 100644 crates/gdpr_worker/src/main.rs diff --git a/ci/Dockerfile.rust b/ci/Dockerfile.rust index 0452cb4b..fdc276c9 100644 --- a/ci/Dockerfile.rust +++ b/ci/Dockerfile.rust @@ -31,6 +31,7 @@ RUN cargo build --bin gateway --release --target x86_64-unknown-linux-musl RUN cargo build --bin avatars --release --target x86_64-unknown-linux-musl RUN cargo build --bin avatar_cleanup --release --target x86_64-unknown-linux-musl RUN cargo build --bin scheduled_tasks --release --target x86_64-unknown-linux-musl +RUN cargo build --bin gdpr_worker --release --target x86_64-unknown-linux-musl FROM scratch @@ -40,3 +41,4 @@ COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/gatew COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/avatars /avatars COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/avatar_cleanup /avatar_cleanup COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/scheduled_tasks /scheduled_tasks +COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/gdpr_worker /gdpr_worker diff --git a/ci/rust-docker-target.sh b/ci/rust-docker-target.sh index 8e87dbca..501f65c2 100755 --- a/ci/rust-docker-target.sh +++ b/ci/rust-docker-target.sh @@ -42,3 +42,4 @@ build dispatch build gateway build avatars "COPY .docker-bin/avatar_cleanup /bin/avatar_cleanup" build scheduled_tasks +build gdpr_worker diff --git a/crates/gdpr_worker/Cargo.toml b/crates/gdpr_worker/Cargo.toml new file mode 100644 index 00000000..a30751f9 --- /dev/null +++ b/crates/gdpr_worker/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "gdpr_worker" +version = "0.1.0" +edition = "2021" + +[dependencies] +libpk = { path = "../libpk" } +anyhow = { workspace = true } +axum = { workspace = true } +futures = { workspace = true } +sqlx = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +twilight-http = { workspace = true } +twilight-model = { workspace = true } diff --git a/crates/gdpr_worker/src/main.rs b/crates/gdpr_worker/src/main.rs new file mode 100644 index 00000000..7409bdff --- /dev/null +++ b/crates/gdpr_worker/src/main.rs @@ -0,0 +1,147 @@ +#![feature(let_chains)] + +use sqlx::prelude::FromRow; +use std::{sync::Arc, time::Duration}; +use tracing::{error, info, warn}; +use twilight_http::api_error::{ApiError, GeneralApiError}; +use twilight_model::id::{ + marker::{ChannelMarker, MessageMarker}, + Id, +}; + +// create table messages_gdpr_jobs (mid bigint not null references messages(mid) on delete cascade, channel bigint not null); + +libpk::main!("messages_gdpr_worker"); +async fn real_main() -> anyhow::Result<()> { + let db = libpk::db::init_messages_db().await?; + + let mut client_builder = twilight_http::Client::builder() + .token( + libpk::config + .discord + .as_ref() + .expect("missing discord config") + .bot_token + .clone(), + ) + .timeout(Duration::from_secs(30)); + + if let Some(base_url) = libpk::config + .discord + .as_ref() + .expect("missing discord config") + .api_base_url + .clone() + { + client_builder = client_builder.proxy(base_url, true).ratelimiter(None); + } + + let client = Arc::new(client_builder.build()); + + loop { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + match run_job(db.clone(), client.clone()).await { + Ok(()) => {} + Err(err) => { + error!("failed to run messages gdpr job: {:?}", err); + } + } + } +} + +#[derive(FromRow)] +struct GdprJobEntry { + mid: i64, + channel_id: i64, +} + +async fn run_job(pool: sqlx::PgPool, discord: Arc) -> anyhow::Result<()> { + let mut tx = pool.begin().await?; + + let message: Option = sqlx::query_as( + "select mid, channel_id from messages_gdpr_jobs for update skip locked limit 1;", + ) + .fetch_optional(&mut *tx) + .await?; + + let Some(message) = message else { + info!("no job to run, sleeping for 1 minute"); + tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; + return Ok(()); + }; + + info!("got mid={}, cleaning up...", message.mid); + + // naively delete message on discord's end + let res = discord + .delete_message( + Id::::new(message.channel_id as u64), + Id::::new(message.mid as u64), + ) + .await; + + if res.is_ok() { + sqlx::query("delete from messages_gdpr_jobs where mid = $1") + .bind(message.mid) + .execute(&mut *tx) + .await?; + } + + if let Err(err) = res { + if let twilight_http::error::ErrorType::Response { error, status, .. } = err.kind() + && let ApiError::General(GeneralApiError { code, .. }) = error + { + match (status.get(), code) { + (403, _) => { + warn!( + "got 403 while deleting message in channel {}, failing fast", + message.channel_id + ); + sqlx::query("delete from messages_gdpr_jobs where channel_id = $1") + .bind(message.channel_id) + .execute(&mut *tx) + .await?; + } + (_, 10003) => { + warn!( + "deleting message in channel {}: channel not found, failing fast", + message.channel_id + ); + sqlx::query("delete from messages_gdpr_jobs where channel_id = $1") + .bind(message.channel_id) + .execute(&mut *tx) + .await?; + } + (_, 10008) => { + warn!("deleting message {}: message not found", message.mid); + sqlx::query("delete from messages_gdpr_jobs where mid = $1") + .bind(message.mid) + .execute(&mut *tx) + .await?; + } + (_, 50083) => { + warn!( + "could not delete message in thread {}: thread is archived, failing fast", + message.channel_id + ); + sqlx::query("delete from messages_gdpr_jobs where channel_id = $1") + .bind(message.channel_id) + .execute(&mut *tx) + .await?; + } + _ => { + error!( + "got unknown error deleting message {}: status={status}, code={code}", + message.mid + ); + } + } + } else { + return Err(err.into()); + } + } + + tx.commit().await?; + + return Ok(()); +} From 19616b6bbb7e1d3932f1a1d96011b0b1eb54b761 Mon Sep 17 00:00:00 2001 From: alyssa Date: Fri, 4 Apr 2025 12:20:15 +0000 Subject: [PATCH 09/80] chore: bump twilight --- Cargo.lock | 2514 ++++++++++++++++--------- Cargo.toml | 20 +- crates/gateway/src/discord/cache.rs | 4 +- crates/gateway/src/discord/gateway.rs | 5 +- 4 files changed, 1617 insertions(+), 926 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75afc365..ab8a5c1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,13 @@ version = 4 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -29,7 +23,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -43,7 +37,7 @@ dependencies = [ "cfg-if", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -57,9 +51,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -78,32 +72,32 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.69" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" [[package]] name = "api" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.5", + "axum 0.7.9", "fred", - "hyper 1.5.0", + "hyper 1.6.0", "hyper-util", "lazy_static", "libpk", "metrics", "pluralkit_models", - "reqwest 0.12.8", + "reqwest 0.12.15", "reverse-proxy-service", "serde", "serde_json", "serde_urlencoded", "sqlx", "tokio", - "tower", - "tower-http 0.5.2", + "tower 0.4.13", + "tower-http", "tracing", "twilight-http", ] @@ -114,6 +108,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayvec" version = "0.7.6" @@ -122,13 +122,13 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] @@ -152,7 +152,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" dependencies = [ - "http 0.2.8", + "http 0.2.12", "log", "rustls 0.20.9", "serde", @@ -164,28 +164,28 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "avatars" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.5", + "axum 0.7.9", "data-encoding", "form_urlencoded", "futures", "gif", "image", "libpk", - "reqwest 0.12.8", + "reqwest 0.12.15", "rust-s3", "serde", "sha2", "sqlx", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", "uuid", @@ -204,34 +204,57 @@ dependencies = [ "quick-xml", "rust-ini 0.18.0", "serde", - "thiserror", + "thiserror 1.0.69", "time", "url", ] +[[package]] +name = "aws-lc-rs" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-region" version = "0.25.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9aed3f9c7eac9be28662fdb3b0f4d1951e812f7c64fed4f0327ba702f459b3b" dependencies = [ - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "axum" -version = "0.6.7" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fb79c228270dcf2426e74864cabc94babb5dbab01a4314e702d2f16540e1591" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core 0.3.4", "bitflags 1.3.2", "bytes", "futures-util", - "http 0.2.8", - "http-body 0.4.5", - "hyper 0.14.24", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "itoa", "matchit", "memchr", @@ -245,26 +268,25 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 0.1.2", "tokio", - "tower", - "tower-http 0.3.5", + "tower 0.4.13", "tower-layer", "tower-service", ] [[package]] name = "axum" -version = "0.7.5" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core 0.4.3", + "axum-core 0.4.5", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper 1.5.0", + "hyper 1.6.0", "hyper-util", "itoa", "matchit", @@ -277,9 +299,9 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -294,8 +316,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.8", - "http-body 0.4.5", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -304,20 +326,20 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", "tracing", @@ -325,17 +347,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -358,9 +380,32 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.6.0" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", + "which", +] [[package]] name = "bitflags" @@ -370,33 +415,33 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" dependencies = [ "serde", ] [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" [[package]] name = "byteorder" @@ -406,15 +451,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "bytes-utils" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" dependencies = [ "bytes", "either", @@ -422,15 +467,30 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" dependencies = [ "jobserver", "libc", "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -438,10 +498,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "chrono" -version = "0.4.38" +name = "cfg_aliases" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ "android-tzdata", "iana-time-zone", @@ -449,7 +515,27 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-link", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", ] [[package]] @@ -458,6 +544,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -469,22 +565,21 @@ dependencies = [ [[package]] name = "config" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be" +checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" dependencies = [ "async-trait", "convert_case", "json5", - "lazy_static", "nom", "pathdiff", "ron", - "rust-ini 0.19.0", + "rust-ini 0.20.0", "serde", "serde_json", "toml", - "yaml-rust", + "yaml-rust2", ] [[package]] @@ -508,7 +603,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] @@ -539,16 +634,26 @@ dependencies = [ ] [[package]] -name = "core-foundation-sys" -version = "0.8.6" +name = "core-foundation" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -594,37 +699,33 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-common" @@ -638,9 +739,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -648,36 +749,37 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] name = "darling_macro" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.66", + "syn", ] [[package]] name = "dashmap" -version = "5.4.0" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "crossbeam-utils", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -685,9 +787,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" [[package]] name = "debugid" @@ -712,9 +814,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" dependencies = [ "powerfmt", "serde", @@ -757,9 +859,9 @@ name = "dispatch" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.5", + "axum 0.7.9", "hickory-client", - "reqwest 0.12.8", + "reqwest 0.12.15", "serde", "serde_json", "tokio", @@ -767,6 +869,17 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dlv-list" version = "0.3.0" @@ -789,19 +902,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] -name = "either" -version = "1.8.1" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" dependencies = [ "serde", ] [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -814,30 +933,30 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "enum-as-inner" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -853,9 +972,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", @@ -864,15 +983,15 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fdeflate" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8090f921a24b04994d9929e204f50b498a33ea6ba559ffaa05e04f7ee7fb5ab" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" dependencies = [ "simd-adler32", ] @@ -891,13 +1010,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.30" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "libz-sys", - "miniz_oxide 0.7.4", + "miniz_oxide", ] [[package]] @@ -911,9 +1029,9 @@ dependencies = [ [[package]] name = "flume" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", @@ -926,6 +1044,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -951,11 +1075,11 @@ dependencies = [ "futures", "log", "parking_lot", - "rand", + "rand 0.8.5", "redis-protocol", "semver", "sha-1", - "socket2 0.5.7", + "socket2", "tokio", "tokio-stream", "tokio-util", @@ -972,14 +1096,20 @@ checksum = "1458c6e22d36d61507034d5afecc64f105c1d39712b7ac6ec3b352c423f715cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] -name = "futures" -version = "0.3.30" +name = "fs_extra" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -992,9 +1122,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1002,15 +1132,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1030,38 +1160,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1080,7 +1210,7 @@ name = "gateway" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.5", + "axum 0.7.9", "bytes", "chrono", "fred", @@ -1088,7 +1218,7 @@ dependencies = [ "lazy_static", "libpk", "metrics", - "reqwest 0.12.8", + "reqwest 0.12.15", "serde", "serde_json", "serde_variant", @@ -1102,11 +1232,26 @@ dependencies = [ "twilight-util", ] +[[package]] +name = "gdpr_worker" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum 0.7.9", + "futures", + "libpk", + "sqlx", + "tokio", + "tracing", + "twilight-http", + "twilight-model", +] + [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -1119,8 +1264,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1135,15 +1296,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" @@ -1156,7 +1317,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.8", + "http 0.2.12", "indexmap", "slab", "tokio", @@ -1166,16 +1327,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.3.1", "indexmap", "slab", "tokio", @@ -1192,12 +1353,6 @@ dependencies = [ "ahash 0.7.8", ] -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" - [[package]] name = "hashbrown" version = "0.14.5" @@ -1209,14 +1364,34 @@ dependencies = [ ] [[package]] -name = "hashlink" -version = "0.9.1" +name = "hashbrown" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "heck" version = "0.4.1" @@ -1229,15 +1404,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.9" @@ -1252,9 +1418,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-client" -version = "0.24.1" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab9683b08d8f8957a857b0236455d80e1886eaa8c6178af556aa7871fb61b55" +checksum = "156579a5cd8d1fc6f0df87cc21b6ee870db978a163a1ba484acd98a4eff5a6de" dependencies = [ "cfg-if", "data-encoding", @@ -1263,17 +1429,17 @@ dependencies = [ "hickory-proto", "once_cell", "radix_trie", - "rand", - "thiserror", + "rand 0.8.5", + "thiserror 1.0.69", "tokio", "tracing", ] [[package]] name = "hickory-proto" -version = "0.24.1" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" +checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248" dependencies = [ "async-trait", "cfg-if", @@ -1282,11 +1448,11 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna 0.4.0", + "idna", "ipnet", "once_cell", - "rand", - "thiserror", + "rand 0.8.5", + "thiserror 1.0.69", "tinyvec", "tokio", "tracing", @@ -1313,11 +1479,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1333,9 +1499,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -1344,9 +1510,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -1355,74 +1521,68 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.8", + "http 0.2.12", "pin-project-lite", ] [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.3.1", ] [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] -[[package]] -name = "http-range-header" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" - [[package]] name = "httparse" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.24" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", "h2 0.3.26", - "http 0.2.8", - "http-body 0.4.5", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.7", + "socket2", "tokio", "tower-service", "tracing", @@ -1431,16 +1591,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.6", - "http 1.1.0", - "http-body 1.0.0", + "h2 0.4.8", + "http 1.3.1", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1457,8 +1617,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http 0.2.8", - "hyper 0.14.24", + "http 0.2.12", + "hyper 0.14.32", "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", @@ -1466,54 +1626,38 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.26.0" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http 1.1.0", - "hyper 1.5.0", + "http 1.3.1", + "hyper 1.6.0", "hyper-util", - "rustls 0.22.4", + "rustls 0.23.25", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.2", "tower-service", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" -dependencies = [ - "futures-util", - "http 1.1.0", - "hyper 1.5.0", - "hyper-util", - "rustls 0.23.23", - "rustls-pki-types", - "tokio", - "tokio-rustls 0.26.0", - "tower-service", - "webpki-roots 0.26.6", + "webpki-roots 0.26.8", ] [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", - "http-body 1.0.0", - "hyper 1.5.0", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", + "libc", "pin-project-lite", - "socket2 0.5.7", + "socket2", "tokio", "tower-service", "tracing", @@ -1521,16 +1665,17 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "windows-core", + "windows-core 0.61.0", ] [[package]] @@ -1542,6 +1687,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1550,22 +1813,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", ] [[package]] -name = "idna" -version = "0.5.0" +name = "idna_adapter" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1586,43 +1850,75 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] name = "inherent" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0122b7114117e64a63ac49f752a5ca4624d534c7b1c7de796ac196381cd2d947" +checksum = "6c38228f24186d9cc68c729accb4d413be9eaed6ad07ff79e0270d9e56f3de13" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] name = "ipnet" -version = "2.7.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -1634,24 +1930,25 @@ checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "json-subscriber" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d0a86fd2fba3a8721e7086b2c9fceb0994f71cdbd64ad2dfc1b202a5c062b4" +checksum = "a04191d0d2a8409d99dccd5642abf91197a53fd661c6611ddb1f87751802d449" dependencies = [ "serde", "serde_json", "tracing", "tracing-core", - "tracing-serde", + "tracing-serde 0.1.3", "tracing-subscriber", "uuid", ] @@ -1669,24 +1966,40 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.5.2", + "spin 0.9.8", ] [[package]] -name = "libc" -version = "0.2.161" +name = "lazycell" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" + +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.48.5", +] [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libpk" @@ -1717,7 +2030,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.9.0", "libc", ] @@ -1727,7 +2040,6 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ - "cc", "pkg-config", "vcpkg", ] @@ -1743,33 +2055,28 @@ dependencies = [ ] [[package]] -name = "libz-sys" -version = "1.1.18" +name = "linux-raw-sys" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" + +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1777,9 +2084,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "matchers" @@ -1792,9 +2099,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "maybe-async" @@ -1804,7 +2111,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] @@ -1825,18 +2132,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" -dependencies = [ - "autocfg", -] +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "metrics" @@ -1856,14 +2154,14 @@ checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" dependencies = [ "base64 0.22.1", "http-body-util", - "hyper 1.5.0", + "hyper 1.6.0", "hyper-util", "indexmap", "ipnet", "metrics", "metrics-util", "quanta", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -1885,9 +2183,9 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minimal-lexical" @@ -1897,18 +2195,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "ff70ce3e48ae43fa075863cef62e8b43b71a4f2382229920e0df362592919430" dependencies = [ "adler2", "simd-adler32", @@ -1916,13 +2205,12 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi 0.3.9", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -1932,7 +2220,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] @@ -1976,7 +2264,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -2019,9 +2307,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -2029,34 +2317,34 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] [[package]] name = "object" -version = "0.36.3" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "ordered-float" @@ -2079,19 +2367,19 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.6.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" dependencies = [ "dlv-list 0.5.2", - "hashbrown 0.13.2", + "hashbrown 0.14.5", ] [[package]] name = "os_info" -version = "3.8.2" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae99c7fa6dd38c7cafe1ec085e804f8f555a2f8659b0dbe03f1f9963a9b51092" +checksum = "2a604e53c24761286860eba4e2c8b23a0161526476b1de520139d69cdb85a6b5" dependencies = [ "log", "serde", @@ -2112,9 +2400,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -2122,28 +2410,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-targets 0.52.6", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pathdiff" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pem-rfc7468" @@ -2162,19 +2444,20 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.5.5" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028accff104c4e513bad663bbcd2ad7cfd5304144404c31ed0a77ac103d00660" +checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" dependencies = [ - "thiserror", + "memchr", + "thiserror 2.0.12", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.5.5" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac3922aac69a40733080f53c1ce7f91dcf57e1a5f6c52f421fadec7fbdc4b69" +checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5" dependencies = [ "pest", "pest_generator", @@ -2182,22 +2465,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.5.5" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d06646e185566b5961b4058dd107e0a7f56e77c3f484549fb119867773c0f202" +checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 1.0.107", + "syn", ] [[package]] name = "pest_meta" -version = "2.5.5" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6f60b2ba541577e2a0c307c8f39d1439108120eb7903adeb6497fa880c59616" +checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" dependencies = [ "once_cell", "pest", @@ -2206,29 +2489,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2259,9 +2542,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "pluralkit_models" @@ -2278,22 +2561,22 @@ dependencies = [ [[package]] name = "png" -version = "0.17.14" +version = "0.17.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" dependencies = [ "bitflags 1.3.2", "crc32fast", "fdeflate", "flate2", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] name = "portable-atomic" -version = "1.8.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "powerfmt" @@ -2303,30 +2586,43 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy 0.8.24", +] + +[[package]] +name = "prettyplease" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +dependencies = [ + "proc-macro2", + "syn", +] [[package]] name = "proc-macro2" -version = "1.0.84" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] [[package]] name = "quanta" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" dependencies = [ "crossbeam-utils", "libc", "once_cell", "raw-cpuid", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "web-sys", "winapi", ] @@ -2343,61 +2639,73 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" dependencies = [ "bytes", + "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", - "rustls 0.23.23", - "socket2 0.5.7", - "thiserror", + "rustc-hash 2.1.1", + "rustls 0.23.25", + "socket2", + "thiserror 2.0.12", "tokio", "tracing", + "web-time", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" dependencies = [ "bytes", - "rand", - "ring 0.17.8", - "rustc-hash", - "rustls 0.23.23", + "getrandom 0.3.2", + "rand 0.9.0", + "ring 0.17.14", + "rustc-hash 2.1.1", + "rustls 0.23.25", + "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.12", "tinyvec", "tracing", + "web-time", ] [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" dependencies = [ + "cfg_aliases", "libc", "once_cell", - "socket2 0.5.7", + "socket2", "tracing", "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radix_trie" version = "0.2.1" @@ -2415,8 +2723,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.24", ] [[package]] @@ -2426,7 +2745,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -2435,16 +2764,25 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", ] [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.9.0", ] [[package]] @@ -2463,20 +2801,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", ] [[package]] @@ -2485,21 +2814,21 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "regex" -version = "1.9.4" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.7", - "regex-syntax 0.7.5", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] @@ -2508,31 +2837,31 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax 0.6.28", + "regex-syntax 0.6.29", ] [[package]] name = "regex-automata" -version = "0.3.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.5", ] [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" @@ -2546,9 +2875,9 @@ dependencies = [ "futures-core", "futures-util", "h2 0.3.26", - "http 0.2.8", - "http-body 0.4.5", - "hyper 0.14.24", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-rustls 0.24.2", "ipnet", "js-sys", @@ -2579,20 +2908,20 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.8" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper 1.5.0", - "hyper-rustls 0.27.3", + "hyper 1.6.0", + "hyper-rustls 0.27.5", "hyper-util", "ipnet", "js-sys", @@ -2602,21 +2931,22 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.23", - "rustls-pemfile 2.1.2", + "rustls 0.23.25", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.2", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.26.6", + "webpki-roots 0.26.8", "windows-registry", ] @@ -2626,9 +2956,9 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7c5828ba3be8842e97b1c96b9df9449b326e5d252c5fb76d73605288b326b75" dependencies = [ - "axum 0.6.7", - "http 0.2.8", - "hyper 0.14.24", + "axum 0.6.20", + "http 0.2.12", + "hyper 0.14.32", "log", "regex", "tower-service", @@ -2651,15 +2981,14 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.8" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", - "spin 0.9.8", "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -2671,16 +3000,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64 0.21.7", - "bitflags 2.5.0", + "bitflags 2.9.0", "serde", "serde_derive", ] [[package]] name = "rsa" -version = "0.9.6" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" dependencies = [ "const-oid", "digest", @@ -2689,7 +3018,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "spki", "subtle", @@ -2708,12 +3037,12 @@ dependencies = [ [[package]] name = "rust-ini" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a" dependencies = [ "cfg-if", - "ordered-multimap 0.6.0", + "ordered-multimap 0.7.3", ] [[package]] @@ -2731,7 +3060,7 @@ dependencies = [ "futures", "hex", "hmac", - "http 0.2.8", + "http 0.2.12", "log", "maybe-async", "md5", @@ -2741,7 +3070,7 @@ dependencies = [ "serde", "serde_derive", "sha2", - "thiserror", + "thiserror 1.0.69", "time", "tokio", "tokio-stream", @@ -2756,9 +3085,15 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -2771,17 +3106,30 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.15", "windows-sys 0.52.0", ] +[[package]] +name = "rustix" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +dependencies = [ + "bitflags 2.9.0", + "errno", + "libc", + "linux-raw-sys 0.9.3", + "windows-sys 0.59.0", +] + [[package]] name = "rustls" version = "0.20.9" @@ -2801,47 +3149,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.8", + "ring 0.17.14", "rustls-webpki 0.101.7", "sct", ] [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "ring 0.17.8", - "rustls-pki-types", - "rustls-webpki 0.102.8", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls" -version = "0.23.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "aws-lc-rs", "log", "once_cell", - "ring 0.17.8", + "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.1", "subtle", "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.2", "rustls-pki-types", "schannel", "security-framework", @@ -2858,19 +3193,48 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +dependencies = [ + "web-time", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5467026f437b4cb2a533865eaa73eb840019a0916f4b9ec563c6e617e086c9" +dependencies = [ + "core-foundation 0.10.0", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.25", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.103.1", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" @@ -2878,40 +3242,50 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.8", + "ring 0.17.14", "untrusted 0.9.0", ] [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ - "ring 0.17.8", + "aws-lc-rs", + "ring 0.17.14", "rustls-pki-types", "untrusted 0.9.0", ] [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2925,7 +3299,7 @@ dependencies = [ "libpk", "metrics", "num-format", - "reqwest 0.12.8", + "reqwest 0.12.15", "serde", "serde_json", "sqlx", @@ -2936,9 +3310,9 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -2946,15 +3320,15 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.8", + "ring 0.17.14", "untrusted 0.9.0", ] [[package]] name = "sea-query" -version = "0.32.1" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "085e94f7d7271c0393ac2d164a39994b1dff1b06bc40cd9a0da04f3d672b0fee" +checksum = "f5a24d8b9fcd2674a6c878a3d871f4f1380c6c43cc3718728ac96864d888458e" dependencies = [ "inherent", "sea-query-derive", @@ -2962,26 +3336,26 @@ dependencies = [ [[package]] name = "sea-query-derive" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9834af2c4bd8c5162f00c89f1701fb6886119a88062cf76fe842ea9e232b9839" +checksum = "bae0cbad6ab996955664982739354128c58d16e126114fe88c2a493642502aab" dependencies = [ "darling", "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.66", - "thiserror", + "syn", + "thiserror 2.0.12", ] [[package]] name = "security-framework" -version = "2.11.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.5.0", - "core-foundation", + "bitflags 2.9.0", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -2989,9 +3363,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -2999,9 +3373,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.16" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "sentry" @@ -3010,8 +3384,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a7332159e544e34db06b251b1eda5e546bd90285c3f58d9c8ff8450b484e0da" dependencies = [ "httpdate", - "reqwest 0.12.8", - "rustls 0.23.23", + "reqwest 0.12.15", + "rustls 0.23.25", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -3020,7 +3394,7 @@ dependencies = [ "sentry-tracing", "tokio", "ureq", - "webpki-roots 0.26.6", + "webpki-roots 0.26.8", ] [[package]] @@ -3056,7 +3430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "653942e6141f16651273159f4b8b1eaeedf37a7554c00cd798953e64b8a9bf72" dependencies = [ "once_cell", - "rand", + "rand 0.8.5", "sentry-types", "serde", "serde_json", @@ -3103,10 +3477,10 @@ checksum = "2d4203359e60724aa05cf2385aaf5d4f147e837185d7dd2b9ccf1ee77f4420c8" dependencies = [ "debugid", "hex", - "rand", + "rand 0.8.5", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", "time", "url", "uuid", @@ -3114,9 +3488,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] @@ -3133,45 +3507,47 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "indexmap", "itoa", + "memchr", "ryu", "serde", ] [[package]] name = "serde_path_to_error" -version = "0.1.9" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" dependencies = [ + "itoa", "serde", ] [[package]] name = "serde_repr" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] @@ -3228,9 +3604,9 @@ dependencies = [ [[package]] name = "sha1_smol" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" [[package]] name = "sha2" @@ -3245,9 +3621,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -3270,9 +3646,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -3284,7 +3660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3295,49 +3671,39 @@ checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "sketches-ddsketch" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" [[package]] name = "slab" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" dependencies = [ "serde", ] [[package]] name = "socket2" -version = "0.4.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -3368,21 +3734,11 @@ dependencies = [ "der", ] -[[package]] -name = "sqlformat" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f" -dependencies = [ - "nom", - "unicode_categories", -] - [[package]] name = "sqlx" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93334716a037193fac19df402f8571269c84a00852f6a7066b5d2616dcd64d3e" +checksum = "4410e73b3c0d8442c5f99b425d7a435b5ee0ae4167b3196771dd3f7a01be745f" dependencies = [ "sqlx-core", "sqlx-macros", @@ -3393,38 +3749,32 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e" +checksum = "6a007b6936676aa9ab40207cde35daab0a04b823be8ae004368c0793b96a61e0" dependencies = [ - "atoi", - "byteorder", "bytes", "chrono", "crc", "crossbeam-queue", "either", "event-listener", - "futures-channel", "futures-core", "futures-intrusive", "futures-io", "futures-util", - "hashbrown 0.14.5", - "hashlink", - "hex", + "hashbrown 0.15.2", + "hashlink 0.10.0", "indexmap", "log", "memchr", "once_cell", - "paste", "percent-encoding", "serde", "serde_json", "sha2", "smallvec", - "sqlformat", - "thiserror", + "thiserror 2.0.12", "time", "tokio", "tokio-stream", @@ -3435,22 +3785,22 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac0692bcc9de3b073e8d747391827297e075c7710ff6276d9f7a1f3d58c6657" +checksum = "3112e2ad78643fef903618d78cf0aec1cb3134b019730edb039b69eaf531f310" dependencies = [ "proc-macro2", "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.66", + "syn", ] [[package]] name = "sqlx-macros-core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5" +checksum = "4e9f90acc5ab146a99bf5061a7eb4976b573f560bc898ef3bf8435448dd5e7ad" dependencies = [ "dotenvy", "either", @@ -3466,7 +3816,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.66", + "syn", "tempfile", "tokio", "url", @@ -3474,13 +3824,13 @@ dependencies = [ [[package]] name = "sqlx-mysql" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64bb4714269afa44aef2755150a0fc19d756fb580a67db8885608cf02f47d06a" +checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.5.0", + "bitflags 2.9.0", "byteorder", "bytes", "chrono", @@ -3502,7 +3852,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand", + "rand 0.8.5", "rsa", "serde", "sha1", @@ -3510,7 +3860,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 2.0.12", "time", "tracing", "uuid", @@ -3519,13 +3869,13 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa91a732d854c5d7726349bb4bb879bb9478993ceb764247660aee25f67c2f8" +checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.5.0", + "bitflags 2.9.0", "byteorder", "chrono", "crc", @@ -3533,7 +3883,6 @@ dependencies = [ "etcetera", "futures-channel", "futures-core", - "futures-io", "futures-util", "hex", "hkdf", @@ -3544,14 +3893,14 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha2", "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 2.0.12", "time", "tracing", "uuid", @@ -3560,9 +3909,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5b2cf34a45953bfd3daaf3db0f7a7878ab9b7a6b91b422d24a7a9e4c857b680" +checksum = "f85ca71d3a5b24e64e1d08dd8fe36c6c95c339a896cc33068148906784620540" dependencies = [ "atoi", "chrono", @@ -3584,6 +3933,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "stringprep" version = "0.1.5" @@ -3597,26 +3952,15 @@ dependencies = [ [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "1.0.107" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -3631,13 +3975,24 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -3645,7 +4000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -3661,42 +4016,64 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.1" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" dependencies = [ - "cfg-if", "fastrand", - "rustix", - "windows-sys 0.52.0", + "getrandom 0.3.2", + "once_cell", + "rustix 1.0.5", + "windows-sys 0.59.0", ] [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ + "cfg-if", "once_cell", ] @@ -3713,9 +4090,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -3728,15 +4105,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -3752,10 +4129,20 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.6.0" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -3768,9 +4155,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.3" +version = "1.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" dependencies = [ "backtrace", "bytes", @@ -3779,20 +4166,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.7", + "socket2", "tokio-macros", "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] @@ -3807,31 +4194,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" -dependencies = [ - "rustls 0.23.23", - "rustls-pki-types", + "rustls 0.23.25", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -3840,9 +4215,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", @@ -3853,33 +4228,32 @@ dependencies = [ [[package]] name = "tokio-websockets" -version = "0.7.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "988c6e20955aa5043e0822cb27093ebaabb430a126cda0223824b6d65ea900c1" +checksum = "fbc46f9dc832c663a5db08513162001a29ac820913275d58943f942c2bc1c435" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes", "fastrand", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.3.1", "httparse", - "ring 0.17.8", - "rustls-native-certs", + "ring 0.17.14", "rustls-pki-types", + "rustls-platform-verifier", "sha1_smol", "simdutf8", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.2", "tokio-util", - "tracing", ] [[package]] name = "toml" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" dependencies = [ "serde", "serde_spanned", @@ -3898,9 +4272,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap", "serde", @@ -3926,22 +4300,19 @@ dependencies = [ ] [[package]] -name = "tower-http" -version = "0.3.5" +name = "tower" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ - "bitflags 1.3.2", - "bytes", "futures-core", "futures-util", - "http 0.2.8", - "http-body 0.4.5", - "http-range-header", "pin-project-lite", - "tower", + "sync_wrapper 1.0.2", + "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -3950,11 +4321,11 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.9.0", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "pin-project-lite", "tower-layer", @@ -3964,21 +4335,21 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -3988,20 +4359,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -4029,10 +4400,20 @@ dependencies = [ ] [[package]] -name = "tracing-subscriber" -version = "0.3.18" +name = "tracing-serde" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -4046,36 +4427,34 @@ dependencies = [ "tracing", "tracing-core", "tracing-log", - "tracing-serde", + "tracing-serde 0.2.0", ] [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twilight-cache-inmemory" -version = "0.16.0-rc.1" -source = "git+https://github.com/pluralkit/twilight#b346757b3054d461f5652b5ba0148a70eda5697e" +version = "0.16.0" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.9.0", "dashmap", "serde", - "tracing", "twilight-model", "twilight-util", ] [[package]] name = "twilight-gateway" -version = "0.16.0-rc.1" -source = "git+https://github.com/pluralkit/twilight#b346757b3054d461f5652b5ba0148a70eda5697e" +version = "0.16.0" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.9.0", "fastrand", - "flate2", "futures-core", "futures-sink", "serde", @@ -4086,12 +4465,13 @@ dependencies = [ "twilight-gateway-queue", "twilight-http", "twilight-model", + "zstd-safe", ] [[package]] name = "twilight-gateway-queue" -version = "0.16.0-rc.1" -source = "git+https://github.com/pluralkit/twilight#b346757b3054d461f5652b5ba0148a70eda5697e" +version = "0.16.0" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" dependencies = [ "tokio", "tracing", @@ -4099,16 +4479,17 @@ dependencies = [ [[package]] name = "twilight-http" -version = "0.16.0-rc.1" -source = "git+https://github.com/pluralkit/twilight#b346757b3054d461f5652b5ba0148a70eda5697e" +version = "0.16.0" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" dependencies = [ "fastrand", - "http 1.1.0", + "http 1.3.1", "http-body-util", - "hyper 1.5.0", - "hyper-rustls 0.26.0", + "hyper 1.6.0", + "hyper-rustls 0.27.5", "hyper-util", "percent-encoding", + "rustls 0.23.25", "serde", "serde_json", "tokio", @@ -4120,8 +4501,8 @@ dependencies = [ [[package]] name = "twilight-http-ratelimiting" -version = "0.16.0-rc.1" -source = "git+https://github.com/pluralkit/twilight#b346757b3054d461f5652b5ba0148a70eda5697e" +version = "0.16.0" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" dependencies = [ "tokio", "tracing", @@ -4129,10 +4510,10 @@ dependencies = [ [[package]] name = "twilight-model" -version = "0.16.0-rc.1" -source = "git+https://github.com/pluralkit/twilight#b346757b3054d461f5652b5ba0148a70eda5697e" +version = "0.16.0" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.9.0", "serde", "serde-value", "serde_repr", @@ -4141,31 +4522,31 @@ dependencies = [ [[package]] name = "twilight-util" -version = "0.16.0-rc.1" -source = "git+https://github.com/pluralkit/twilight#b346757b3054d461f5652b5ba0148a70eda5697e" +version = "0.16.0" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" dependencies = [ "twilight-model", ] [[package]] name = "twilight-validate" -version = "0.16.0-rc.1" -source = "git+https://github.com/pluralkit/twilight#b346757b3054d461f5652b5ba0148a70eda5697e" +version = "0.16.0" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" dependencies = [ "twilight-model", ] [[package]] name = "typenum" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "ucd-trie" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "uname" @@ -4178,42 +4559,36 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-properties" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" - -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "untrusted" @@ -4229,27 +4604,27 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.1" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.23", + "rustls 0.23.25", "rustls-pki-types", "url", - "webpki-roots 0.26.6", + "webpki-roots 0.26.8", ] [[package]] name = "url" -version = "2.5.2" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna", "percent-encoding", "serde", ] @@ -4261,20 +4636,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] -name = "uuid" -version = "1.11.0" +name = "utf16_iter" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom", + "getrandom 0.3.2", "serde", ] [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vcpkg" @@ -4284,17 +4671,26 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -4304,6 +4700,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -4312,46 +4717,48 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4359,28 +4766,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -4391,9 +4801,19 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", @@ -4415,10 +4835,19 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.8", + "ring 0.17.14", "untrusted 0.9.0", ] +[[package]] +name = "webpki-root-certs" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09aed61f5e8d2c18344b3faa33a4c837855fe56642757754775548fee21386c4" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.22.6" @@ -4436,9 +4865,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" dependencies = [ "rustls-pki-types", ] @@ -4450,12 +4879,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" [[package]] -name = "whoami" -version = "1.5.1" +name = "which" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ - "redox_syscall 0.4.1", + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + +[[package]] +name = "whoami" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" +dependencies = [ + "redox_syscall", "wasite", ] @@ -4475,6 +4916,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -4487,7 +4937,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core", + "windows-core 0.52.0", "windows-targets 0.52.6", ] @@ -4501,33 +4951,82 @@ dependencies = [ ] [[package]] -name = "windows-registry" -version = "0.2.0" +name = "windows-core" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings 0.4.0", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-registry" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", - "windows-strings", - "windows-targets 0.52.6", + "windows-strings 0.3.1", + "windows-targets 0.53.0", ] [[package]] name = "windows-result" -version = "0.2.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" dependencies = [ - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "windows-strings" -version = "0.1.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-result", - "windows-targets 0.52.6", + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", ] [[package]] @@ -4536,7 +5035,7 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets 0.42.1", + "windows-targets 0.42.2", ] [[package]] @@ -4568,17 +5067,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm 0.42.1", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", - "windows_x86_64_gnullvm 0.42.1", - "windows_x86_64_msvc 0.42.1", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -4605,7 +5104,7 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", @@ -4613,10 +5112,26 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.1" +name = "windows-targets" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" @@ -4631,10 +5146,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] -name = "windows_aarch64_msvc" -version = "0.42.1" +name = "windows_aarch64_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" @@ -4649,10 +5170,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] -name = "windows_i686_gnu" -version = "0.42.1" +name = "windows_aarch64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" @@ -4666,6 +5193,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" @@ -4673,10 +5206,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] -name = "windows_i686_msvc" -version = "0.42.1" +name = "windows_i686_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" @@ -4691,10 +5230,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] -name = "windows_x86_64_gnu" -version = "0.42.1" +name = "windows_i686_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" @@ -4709,10 +5254,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.1" +name = "windows_x86_64_gnu" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" @@ -4727,10 +5278,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.1" +name = "windows_x86_64_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" @@ -4745,10 +5302,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winnow" -version = "0.6.20" +name = "windows_x86_64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "winnow" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" dependencies = [ "memchr", ] @@ -4764,32 +5327,120 @@ dependencies = [ ] [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "wit-bindgen-rt" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "linked-hash-map", + "bitflags 2.9.0", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yaml-rust2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink 0.8.4", +] + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", ] [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive 0.8.24", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", ] [[package]] @@ -4797,3 +5448,44 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index c605303a..cb9c4dfa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,14 +25,14 @@ tracing = "0.1" tracing-subscriber = { version = "0.3.16", features = ["env-filter", "json"] } uuid = { version = "1.7.0", features = ["serde"] } -twilight-gateway = { git = "https://github.com/pluralkit/twilight" } -twilight-cache-inmemory = { git = "https://github.com/pluralkit/twilight", features = ["permission-calculator"] } -twilight-util = { git = "https://github.com/pluralkit/twilight", features = ["permission-calculator"] } -twilight-model = { git = "https://github.com/pluralkit/twilight" } -twilight-http = { git = "https://github.com/pluralkit/twilight", default-features = false, features = ["rustls-native-roots"] } +twilight-gateway = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590" } +twilight-cache-inmemory = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590", features = ["permission-calculator"] } +twilight-util = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590", features = ["permission-calculator"] } +twilight-model = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590" } +twilight-http = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590", default-features = false, features = ["rustls-aws_lc_rs", "rustls-native-roots"] } -#twilight-gateway = { path = "../twilight/twilight-gateway" } -#twilight-cache-inmemory = { path = "../twilight/twilight-cache-inmemory", features = ["permission-calculator"] } -#twilight-util = { path = "../twilight/twilight-util", features = ["permission-calculator"] } -#twilight-model = { path = "../twilight/twilight-model" } -#twilight-http = { path = "../twilight/twilight-http", default-features = false, features = ["rustls-native-roots"] } +# twilight-gateway = { path = "../twilight/twilight-gateway" } +# twilight-cache-inmemory = { path = "../twilight/twilight-cache-inmemory", features = ["permission-calculator"] } +# twilight-util = { path = "../twilight/twilight-util", features = ["permission-calculator"] } +# twilight-model = { path = "../twilight/twilight-model" } +# twilight-http = { path = "../twilight/twilight-http", default-features = false, features = ["rustls-aws_lc_rs", "rustls-native-roots"] } diff --git a/crates/gateway/src/discord/cache.rs b/crates/gateway/src/discord/cache.rs index 2b5b22b8..e0a4aacf 100644 --- a/crates/gateway/src/discord/cache.rs +++ b/crates/gateway/src/discord/cache.rs @@ -476,12 +476,14 @@ impl DiscordCache { system_channel_flags: guild.system_channel_flags(), system_channel_id: guild.system_channel_id(), threads: vec![], - unavailable: false, + unavailable: Some(false), vanity_url_code: guild.vanity_url_code().map(ToString::to_string), verification_level: guild.verification_level(), voice_states: vec![], widget_channel_id: guild.widget_channel_id(), widget_enabled: guild.widget_enabled(), + guild_scheduled_events: guild.guild_scheduled_events().to_vec(), + max_stage_video_channel_users: guild.max_stage_video_channel_users(), } }) } diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index fc881f34..4c30e6a9 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -183,10 +183,7 @@ pub async fn runner( match event { Event::InteractionCreate(_) => {} Event::MessageCreate(ref m) if m.author.id != our_user_id => {} - Event::MessageUpdate(ref m) - if let Some(author) = m.author.clone() - && author.id != our_user_id - && !author.bot => {} + Event::MessageUpdate(ref m) if m.author.id != our_user_id && !m.author.bot => {} Event::MessageDelete(_) => {} Event::MessageDeleteBulk(_) => {} Event::ReactionAdd(ref r) if r.user_id != our_user_id => {} From eca5cbff97dad43c119c794576be3e814d2b6c01 Mon Sep 17 00:00:00 2001 From: alyssa Date: Mon, 7 Apr 2025 00:56:18 +0000 Subject: [PATCH 10/80] chore: move docs to fly --- docs/Dockerfile | 18 ++++++++++++++++++ docs/fly.toml | 3 +++ netlify.toml | 6 ------ 3 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 docs/Dockerfile create mode 100644 docs/fly.toml delete mode 100644 netlify.toml diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 00000000..a13a59fb --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,18 @@ +FROM alpine:latest AS builder + +RUN apk add nodejs-current yarn + +WORKDIR /build +COPY . . +RUN yarn +ENV NODE_OPTIONS=--openssl-legacy-provider +RUN yarn build + +FROM alpine:latest + +RUN apk add caddy + +WORKDIR /app +COPY --from=builder /build/content/.vuepress/dist /app + +ENTRYPOINT [ "/usr/sbin/caddy", "file-server", "-l", "0.0.0.0:8000", "-r", "/app" ] diff --git a/docs/fly.toml b/docs/fly.toml new file mode 100644 index 00000000..c85d5812 --- /dev/null +++ b/docs/fly.toml @@ -0,0 +1,3 @@ +app = "pluralkit-docs" +primary_region = "arn" +http_service.internal_port = 8000 diff --git a/netlify.toml b/netlify.toml deleted file mode 100644 index 9a00fb7f..00000000 --- a/netlify.toml +++ /dev/null @@ -1,6 +0,0 @@ -# Configuration for Netlify (website deployment, etc) - -[build] -base = "docs/" -publish = "content/.vuepress/dist" -command = "npm run build" From 4acc51195d23401997ca3e25ba8b91130e54636a Mon Sep 17 00:00:00 2001 From: alyssa Date: Tue, 8 Apr 2025 17:34:50 +0000 Subject: [PATCH 11/80] fix(scheduled_tasks): don't crash on missing data --- crates/scheduled_tasks/src/tasks.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/crates/scheduled_tasks/src/tasks.rs b/crates/scheduled_tasks/src/tasks.rs index e0247768..bdcb51be 100644 --- a/crates/scheduled_tasks/src/tasks.rs +++ b/crates/scheduled_tasks/src/tasks.rs @@ -10,7 +10,6 @@ use metrics::gauge; use num_format::{Locale, ToFormattedString}; use reqwest::ClientBuilder; use sqlx::Executor; -use tracing::error; use crate::AppCtx; @@ -190,28 +189,25 @@ pub async fn update_stats_api(ctx: AppCtx) -> anyhow::Result<()> { let data = resp.json::().await?; let error_handler = || { - error!("missing data at {}", $q); + anyhow::anyhow!("missing data at {}", $q) }; data.data .result .get(0) - .ok_or_else(error_handler) - .unwrap() + .ok_or_else(error_handler)? .value .clone() .get(1) - .ok_or_else(error_handler) - .unwrap() + .ok_or_else(error_handler)? .as_str() - .ok_or_else(error_handler) - .unwrap() + .ok_or_else(error_handler)? .parse::<$t>()? }}; ($t:ty, $q:expr, $wrap:expr) => {{ let val = prom_instant_query!($t, $q); let val = (val * $wrap).round() / $wrap; - format!("{:.2}", val).parse::().unwrap() + format!("{:.2}", val).parse::()? }}; } From 9506833abf27451c28692f0ec8d7167aa0041934 Mon Sep 17 00:00:00 2001 From: asleepyskye Date: Sun, 13 Apr 2025 11:01:12 -0400 Subject: [PATCH 12/80] fix(nix): change rust build command --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index dc5d9f9c..8cb2c505 100644 --- a/flake.nix +++ b/flake.nix @@ -160,7 +160,7 @@ ${sourceDotenv} set -x ${pluralkitConfCheck} - exec cargo build --package ${name} + exec cargo build --bin ${name} ''; }; }; From 437ea72ed43fe810ea88e07d978b2bb89b9ade19 Mon Sep 17 00:00:00 2001 From: alyssa Date: Thu, 24 Apr 2025 10:42:40 +0000 Subject: [PATCH 13/80] chore: update readme, example config, and running documentation Co-authored-by: asleepyskye --- README.md | 69 ++----------------- BRANCHRENAME.md => dev-docs/BRANCHRENAME.md | 0 LEGACYMIGRATE.md => dev-docs/LEGACYMIGRATE.md | 0 dev-docs/README.md | 21 ++++++ dev-docs/RUNNING.md | 69 +++++++++++++++++++ pluralkit.conf.example | 4 +- 6 files changed, 96 insertions(+), 67 deletions(-) rename BRANCHRENAME.md => dev-docs/BRANCHRENAME.md (100%) rename LEGACYMIGRATE.md => dev-docs/LEGACYMIGRATE.md (100%) create mode 100644 dev-docs/README.md create mode 100644 dev-docs/RUNNING.md diff --git a/README.md b/README.md index 5060756a..af6d24fe 100644 --- a/README.md +++ b/README.md @@ -5,75 +5,14 @@ PluralKit is a Discord bot meant for plural communities. It has features like me PluralKit has a Discord server for support, feedback, and discussion: https://discord.gg/PczBt78 -# Requirements -Running the bot requires [.NET 5](https://dotnet.microsoft.com/download), a PostgreSQL database and a Redis database. It should function on any system where the prerequisites are set up (including Windows). - -Optionally, it can integrate with [Sentry](https://sentry.io/welcome/) for error reporting and [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) for aggregate statistics. - -# Configuration -Configuring the bot is done through a JSON configuration file. An example of the configuration format can be seen in [`pluralkit.conf.example`](https://github.com/PluralKit/PluralKit/blob/master/pluralkit.conf.example). -The configuration file needs to be placed in the bot's working directory (usually the repository root) and must be called `pluralkit.conf`. - -The configuration file is in JSON format (albeit with a `.conf` extension). The following keys are available (using `.` to indicate a nested object level), bolded key names are required: -* **`PluralKit.Bot.Token`**: the Discord bot token to connect with -* **`PluralKit.Database`**: the URI of the database to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) -* **`PluralKit.RedisAddr`**: the `host:port` of a Redis database to connect to -* `PluralKit.Bot.Prefixes`: an array of command prefixes to use (default `["pk;", "pk!"]`). -* **`PluralKit.Bot.ClientId`**: the ID of the bot's user account, used for calculating the bot's own permissions and for the link in `pk;invite`. -* `PluralKit.SentryUrl` *(optional)*: the [Sentry](https://sentry.io/welcome/) client key/DSN to report runtime errors to. If absent, disables Sentry integration. -* `PluralKit.InfluxUrl` *(optional)*: the URL to an [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) server to report aggregate statistics to. An example of these stats can be seen on [the public stats page](https://stats.pluralkit.me). -* `PluralKit.InfluxDb` *(optional)*: the name of an InfluxDB database to report statistics to. If either this field or `PluralKit.InfluxUrl` are absent, InfluxDB reporting will be disabled. -* `PluralKit.LogDir` *(optional)*: the directory to save information and error logs to. If left blank, will default to `logs/` in the current working directory. - -The bot can also take configuration from environment variables, which will override the values read from the file. Here, use `:` (colon) or `__` (double underscore) as a level separator (eg. `export PluralKit__Bot__Token=foobar123`) as per [ASP.NET config](https://docs.microsoft.com/en-us/aspnet/core/fundamentals/configuration/?view=aspnetcore-3.1#environment-variables). - # Running +See [RUNNING.md](./RUNNING.md). -## Docker -The easiest way to get the bot running is with Docker. The repository contains a `docker-compose.yml` file ready to use. - -* Clone this repository: `git clone https://github.com/PluralKit/PluralKit` -* Create a `pluralkit.conf` file in the same directory as `docker-compose.yml` containing at least `PluralKit.Bot.Token` and `PluralKit.Bot.ClientId` fields - * (`PluralKit.Database` is overridden in `docker-compose.yml` to point to the Postgres container) -* Build the bot: `docker-compose build` -* Run the bot: `docker-compose up` - -In other words: -``` -$ git clone https://github.com/PluralKit/PluralKit -$ cd PluralKit -$ cp pluralkit.conf.example pluralkit.conf -$ nano pluralkit.conf # (or vim, or whatever) -$ docker-compose up -d -``` - -## Manually -* Install the .NET 6 SDK (see https://dotnet.microsoft.com/download) -* Clone this repository: `git clone https://github.com/PluralKit/PluralKit` -* Create and fill in a `pluralkit.conf` file in the same directory as `docker-compose.yml` -* Run the bot: `dotnet run --project PluralKit.Bot` - * Alternatively, `dotnet build -c Release -o build/`, then `dotnet build/PluralKit.Bot.dll` - -(tip: use `scripts/run-test-db.sh` to run a temporary PostgreSQL database on your local system. Requires Docker.) - -## Scheduled Tasks worker - -There is a scheduled tasks worker that needs to be ran separately from the bot. This handles cleaning up the database, and updating statistics (system/member/etc counts, shown in the `pk;stats` embed). - -Note: This worker is *not required*, and the bot will function correctly without it. - -If you are running the bot via docker-compose, this is set up automatically. - -If you run the bot manually you can run the worker as such: -* `dotnet run --project PluralKit.ScheduledTasks` -* or if you used `dotnet build` rather than `dotnet run` to run the bot: `dotnet build/PluralKit.ScheduledTasks.dll` - -# Upgrading database from legacy version -If you have an instance of the Python version of the bot (from the `legacy` branch), you may need to take extra database migration steps. -For more information, see [LEGACYMIGRATE.md](./LEGACYMIGRATE.md). +# Development +See [the dev-docs/ directory](./dev-docs/README.md) # User documentation See [the docs/ directory](./docs/README.md) # License -This project is under the GNU Affero General Public License, Version 3. It is available at the following link: https://www.gnu.org/licenses/agpl-3.0.en.html \ No newline at end of file +This project is under the GNU Affero General Public License, Version 3. It is available at the following link: https://www.gnu.org/licenses/agpl-3.0.en.html diff --git a/BRANCHRENAME.md b/dev-docs/BRANCHRENAME.md similarity index 100% rename from BRANCHRENAME.md rename to dev-docs/BRANCHRENAME.md diff --git a/LEGACYMIGRATE.md b/dev-docs/LEGACYMIGRATE.md similarity index 100% rename from LEGACYMIGRATE.md rename to dev-docs/LEGACYMIGRATE.md diff --git a/dev-docs/README.md b/dev-docs/README.md new file mode 100644 index 00000000..e712685f --- /dev/null +++ b/dev-docs/README.md @@ -0,0 +1,21 @@ +# PluralKit development documentation + +Most of PluralKit's code is written in C#, and is split into 3 projects: `PluralKit.Core` (supporting libraries), `PluralKit.Bot` (the Discord bot), and `PluralKit.API` (ASP.NET webserver with controllers for most API endpoints). + +There is an ongoing effort to port this code to Rust, and we have a handful of crates already. Currently, the main Rust services are `gateway` (connects to Discord to receive events), `api` (handles authentication and rate-limiting for the public API, as well as a couple of private endpoints), and `scheduled_tasks` (background cron job runner, for statistics and miscellaneous cleanup). + +At the very least, `PluralKit.Bot` and `gateway` are required for the bot to run. While code still exists to connect to the Discord gateway directly from the C# bot, this is no longer a supported configuration and may break in the future. + +Service-specific documentation can be found for the C# services in [dotnet.md](./dotnet.md), and for the Rust services in [rust.md](./rust.md) (todo; there is a temporary mostly-accurate document at [RUNNING.md](./RUNNING.md)). + +## Building/running + +PluralKit uses a PostgreSQL database and a Redis database to store data. User-provided data is stored in Postgres; Redis is used for internal state and transient data such as the command execution cache. It's generally easy to run these in Docker or with the Nix `process-compose`, but any install method should work. + +The production instance of PluralKit uses Docker images built in CI. These take a long time to rebuild and aren't good for development (they're production builds, so it's not possible to hook up a debugger). Instead, it's preferable to install build dependencies locally. This is easy with the provided Nix flake: run `nix develop .#bot` to drop into a shell with all the C# build dependencies available, and `nix develop .#services` to get a shell with the Rust build dependencies. It's also okay to manually install build dependencies if you prefer. + +PluralKit services are configured with environment variables; see service-specific documentation for details. Generally, the configuration from the self-host `docker-compose.yml` should get you started. + +## Upgrading database from legacy version +If you have an instance of the Python version of the bot (from the `legacy` branch), you may need to take extra database migration steps. +For more information, see [LEGACYMIGRATE.md](./LEGACYMIGRATE.md). diff --git a/dev-docs/RUNNING.md b/dev-docs/RUNNING.md new file mode 100644 index 00000000..9dc5de7e --- /dev/null +++ b/dev-docs/RUNNING.md @@ -0,0 +1,69 @@ +# Technical Overview: + +PluralKit is composed of several different parts, some of which optional and not needed for the bot to function in a testing environment. +##### Required: +- Bot (*PluralKit.Bot*) +- Gateway +- PostgreSQL Database +- Redis Database +##### Optional: +- API (*PluralKit.API*) +- Scheduled Tasks (*scheduled_tasks*) + +*Optionally, it can also integrate with Sentry for error reporting, and InfluxDB for aggregate statistics. In production, we use [VictoriaMetrics](https://victoriametrics.com/) InfluxDB competible endpoint, to query metrics in a Prometheus-compatible format. + +The bot and API are built using .NET 8, with other optional components used for scaling (ex. scheduled_tasks) built using Rust. +# Building + Running + +**The below procedures are intended for development and testing use only! + +Newer versions of the bot have moved to using [Nix](https://nixos.org/) for simplified builds. A docker compose file is also provided, but not recommended as it is not actively maintained. + +See [Configuration](./CONFIGURATION.md) for full details on configuration as only the basics will be covered here. +Configuration is done through a JSON configuration file `pluralkit.conf` placed in the bot's working directory. An example of the configuration format can be seen in [pluralkit.conf.example](pluralkit.conf.example). +The minimum configuration needed for the bot to function must include the following: +- **`PluralKit.Bot.Token`**: the Discord bot token to connect with +- **`PluralKit.Bot.ClientId`**: the ID of the bot's user account, used for calculating the bot's own permissions and for the link in `pk;invite` +- **`PluralKit.Database`**: the URI of the PostgreSQL database to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) +- **`PluralKit.RedisAddr`**: the `host:port` of the Redis database to connect to + +**When running using Docker, you do not need to specify the Postgres or Redis URLs as these will be overwritten by environment variables in the compose file.** + +**When using Nix, the Database URI Username, Password, and Database fields must match what the database was setup with in the `flake.nix` file!** + +The bot can also take configuration from environment variables, which will override the values read from the file. Here, use `:` (colon) or `__` (double underscore) as a level separator (eg. `export PluralKit__Bot__Token=foobar123`) as per [ASP.NET config](https://docs.microsoft.com/en-us/aspnet/core/fundamentals/configuration/?view=aspnetcore-3.1#environment-variables). +## Nix (recommended) +The bot, databases, and services are available to run all in one as a Nix flake (using process-compose-flake). +As of the writing of this document, there are a few caveats with the current flake file. +- The database URI in the config must match the username, password, and database specified during database creation (currently `postgres`, `postgres`, and `pluralkit` respectively). +- Not all services are added to the flake file yet. + +**Additionally, as of the writing of this document, the `pluralkit-gateway` service reads environment variables to get the bot token and database URLs, so you also need to create a `.env` file in the `PluralKit` directory with the following variables:** +``` +pluralkit__db__db_password="postgres" +pluralkit__db__data_db_uri="postgresql://postgres@localhost:5432/pluralkit" +pluralkit__db__data_redis_addr="redis://localhost:6379" +pluralkit__discord__bot_token="BOT_TOKEN_GOES_HERE" +pluralkit__discord__client_id="BOT_CLIENT_ID_GOES_HERE" +pluralkit__discord__client_secret=1 +pluralkit__discord__max_concurrency=1 +``` +**(This should match the username/password/database specified in the flake file and the configuration file)** + +*(assuming you already have Git installed, if not, you can start a shell with git by running `nix-shell -p git`)* +1. Clone the repository: `git clone https://github.com/PluralKit/PluralKit` +2. Create a `pluralkit.conf` configuration file in the `PluralKit` directory + - Again, the DB URI parameters must match what's in the `flake.nix` file +3. Create a `.env` configuration file in the `PluralKit` directory *(see above)* +4. Build and run: `nix run .#dev` + - This will download the dependencies, build, and run PluralKit + - If Nix is not setup to allow flakes, you may need to add `--extra-experimental-features nix-command --extra-experimental-features flakes` to the command + - If the `pluralkit-bot` process fails to run, you can restart it by selecting it and pressing `Ctrl-R` +``` +[nix-shell:~]$ git clone https://github.com/PluralKit/PluralKit +[nix-shell:~]$ cd PluralKit +[nix-shell:~/PluralKit]$ cp pluralkit.conf.example pluralkit.conf +[nix-shell:~/PluralKit]$ nano pluralkit.conf +[nix-shell:~/PluralKit]$ nano .env +[nix-shell:~/PluralKit]$ nix run .#dev +``` diff --git a/pluralkit.conf.example b/pluralkit.conf.example index 44ce1ef0..a705265b 100644 --- a/pluralkit.conf.example +++ b/pluralkit.conf.example @@ -4,7 +4,7 @@ "Token": "BOT_TOKEN_GOES_HERE", "ClientId": 466707357099884544, }, - "Database": "Host=localhost;Port=5432;Username=myusername;Password=mypassword;Database=mydatabasename", - "RedisAddr": "host:port" + "Database": "Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=pluralkit", + "RedisAddr": "127.0.0.1:6379" } } \ No newline at end of file From 292c182eb22d7a8254ae45b70dd9ef7a24f60731 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 13 Apr 2025 22:05:14 +0000 Subject: [PATCH 14/80] feat: update docker-compose.yml for selfhost --- PluralKit.Core/CoreConfig.cs | 1 - PluralKit.Core/Modules/LoggingModule.cs | 24 -------- README.md | 22 +++++++- ci/Dockerfile.rust | 2 +- crates/gateway/src/main.rs | 13 +++++ crates/libpk/src/_config.rs | 3 + crates/scheduled_tasks/src/tasks.rs | 4 +- docker-compose.yml | 73 ++++++++++--------------- pluralkit.conf.example | 10 ---- 9 files changed, 68 insertions(+), 84 deletions(-) delete mode 100644 pluralkit.conf.example diff --git a/PluralKit.Core/CoreConfig.cs b/PluralKit.Core/CoreConfig.cs index 4adf815b..1e77271b 100644 --- a/PluralKit.Core/CoreConfig.cs +++ b/PluralKit.Core/CoreConfig.cs @@ -19,5 +19,4 @@ public class CoreConfig public LogEventLevel ConsoleLogLevel { get; set; } = LogEventLevel.Debug; public LogEventLevel ElasticLogLevel { get; set; } = LogEventLevel.Information; - public LogEventLevel FileLogLevel { get; set; } = LogEventLevel.Information; } \ No newline at end of file diff --git a/PluralKit.Core/Modules/LoggingModule.cs b/PluralKit.Core/Modules/LoggingModule.cs index 18ffa58b..f02bfe69 100644 --- a/PluralKit.Core/Modules/LoggingModule.cs +++ b/PluralKit.Core/Modules/LoggingModule.cs @@ -73,30 +73,6 @@ public class LoggingModule: Module .Destructure.AsScalar() .Destructure.ByTransforming(t => new { t.Prefix, t.Suffix }) .Destructure.With() - .WriteTo.Async(a => - { - // Both the same output, except one is raw compact JSON and one is plain text. - // Output simultaneously. May remove the JSON formatter later, keeping it just in cast. - // Flush interval is 50ms (down from 10s) to make "tail -f" easier. May be too low? - a.File( - (config.LogDir ?? "logs") + $"/pluralkit.{_component}.log", - outputTemplate: outputTemplate, - retainedFileCountLimit: 10, - rollingInterval: RollingInterval.Day, - fileSizeLimitBytes: null, - flushToDiskInterval: TimeSpan.FromMilliseconds(50), - restrictedToMinimumLevel: config.FileLogLevel, - formatProvider: new UTCTimestampFormatProvider(), - buffered: true); - - a.File( - new RenderedCompactJsonFormatter(new ScalarFormatting.JsonValue()), - (config.LogDir ?? "logs") + $"/pluralkit.{_component}.json", - rollingInterval: RollingInterval.Day, - flushToDiskInterval: TimeSpan.FromMilliseconds(50), - restrictedToMinimumLevel: config.FileLogLevel, - buffered: true); - }) .WriteTo.Async(a => a.Console( theme: AnsiConsoleTheme.Code, diff --git a/README.md b/README.md index af6d24fe..84cc3f56 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,27 @@ PluralKit is a Discord bot meant for plural communities. It has features like me PluralKit has a Discord server for support, feedback, and discussion: https://discord.gg/PczBt78 # Running -See [RUNNING.md](./RUNNING.md). +In production, we run PluralKit using Kubernetes (soon). The configuration can be found in the infra repo. + +For self-hosting, it's simpler to use Docker, with the provided [docker-compose](./docker-compose.yml) file. + +Create a `.env` file with the Discord client ID and bot token: +``` +CLIENT_ID=198622483471925248 +BOT_TOKEN=MTk4NjIyNDgzNDcxOTI1MjQ4.Cl2FMQ.ZnCjm1XVW7vRze4b7Cq4se7kKWs +``` + +If you want to use `pk;admin` commands (to raise member limits and such), set `ADMIN_ROLE` to a Discord role ID: + +``` +ADMIN_ROLE=682632767057428509 +``` + +Run `docker compose build`, then `docker compose up -d`. + +To view logs, use `docker compose logs`. + +Postgres data is stored in a `pluralkit_data` [Docker volume](https://docs.docker.com/engine/storage/volumes/). # Development See [the dev-docs/ directory](./dev-docs/README.md) diff --git a/ci/Dockerfile.rust b/ci/Dockerfile.rust index fdc276c9..b035cfcf 100644 --- a/ci/Dockerfile.rust +++ b/ci/Dockerfile.rust @@ -33,7 +33,7 @@ RUN cargo build --bin avatar_cleanup --release --target x86_64-unknown-linux-mus RUN cargo build --bin scheduled_tasks --release --target x86_64-unknown-linux-musl RUN cargo build --bin gdpr_worker --release --target x86_64-unknown-linux-musl -FROM scratch +FROM alpine:latest COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/api /api COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/dispatch /dispatch diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index ab679c06..7a2a0721 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -40,6 +40,19 @@ async fn real_main() -> anyhow::Result<()> { .await?, ); + // hacky, but needed for selfhost for now + if let Some(target) = libpk::config + .discord + .as_ref() + .unwrap() + .gateway_target + .clone() + { + runtime_config + .set(RUNTIME_CONFIG_KEY_EVENT_TARGET.to_string(), target) + .await?; + } + let shard_state = discord::shard_state::new(redis.clone()); let cache = Arc::new(discord::cache::new()); let awaiter = Arc::new(EventAwaiter::new()); diff --git a/crates/libpk/src/_config.rs b/crates/libpk/src/_config.rs index b182f359..aa6a1f3b 100644 --- a/crates/libpk/src/_config.rs +++ b/crates/libpk/src/_config.rs @@ -24,6 +24,9 @@ pub struct DiscordConfig { #[serde(default = "_default_api_addr")] pub cache_api_addr: String, + + #[serde(default)] + pub gateway_target: Option, } #[derive(Deserialize, Debug)] diff --git a/crates/scheduled_tasks/src/tasks.rs b/crates/scheduled_tasks/src/tasks.rs index bdcb51be..98396a3f 100644 --- a/crates/scheduled_tasks/src/tasks.rs +++ b/crates/scheduled_tasks/src/tasks.rs @@ -188,9 +188,7 @@ pub async fn update_stats_api(ctx: AppCtx) -> anyhow::Result<()> { let data = resp.json::().await?; - let error_handler = || { - anyhow::anyhow!("missing data at {}", $q) - }; + let error_handler = || anyhow::anyhow!("missing data at {}", $q); data.data .result diff --git a/docker-compose.yml b/docker-compose.yml index 161fd133..41468461 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,52 +1,46 @@ -# API port: 2838, InfluxDB port: 2839, both listen on localhost only -# Reads `pluralkit.conf` from current directory, and logs to `/var/log/pluralkit`. - version: "3" services: bot: - image: pluralkit # This image is reused in the other containers due to the - build: . # build instruction right here + build: . command: ["bin/PluralKit.Bot.dll"] environment: - - "PluralKit:Database=Host=db;Username=postgres;Password=postgres;Database=postgres;Maximum Pool Size=1000" - - "PluralKit:RedisAddr=redis" - - "PluralKit:InfluxUrl=http://influx:8086" - - "PluralKit:InfluxDb=pluralkit" - - "PluralKit:LogDir=/var/log/pluralkit" - volumes: - - "./pluralkit.conf:/app/pluralkit.conf:ro" - - "/var/log/pluralkit:/var/log/pluralkit" + - "PluralKit__Database=Host=db;Username=postgres;Password=postgres;Database=postgres" + - "PluralKit__RedisAddr=redis" + - "PluralKit__Bot__Token=${BOT_TOKEN}" + - "PluralKit__Bot__ClientId=${CLIENT_ID}" + - "PluralKit__Bot__AdminRole=${ADMIN_ROLE}" + - "PluralKit__Bot__HttpCacheUrl=http://gateway:5000" + - "PluralKit__Bot__HttpListenerAddr=0.0.0.0" + - "PluralKit__Bot__EventAwaiterTarget=http://bot:5002/events" + - "PluralKit__Bot__DisableGateway=true" restart: unless-stopped - api: - image: pluralkit - command: ["bin/PluralKit.API.dll"] + gateway: + build: + context: . + dockerfile: ci/Dockerfile.rust + command: ["/gateway"] environment: - - "PluralKit:Database=Host=db;Username=postgres;Password=postgres;Database=postgres;Maximum Pool Size=1000" - - "PluralKit:RedisAddr=redis" - ports: - - "127.0.0.1:2838:5000" - restart: unless-stopped - - scheduled_tasks: - image: pluralkit - command: ["bin/PluralKit.ScheduledTasks.dll"] - environment: - - "PluralKit:Database=Host=db;Username=postgres;Password=postgres;Database=postgres;Maximum Pool Size=1000" + - RUST_LOG=info + - pluralkit__discord__client_id=${CLIENT_ID} + - pluralkit__discord__bot_token=${BOT_TOKEN} + - pluralkit__discord__max_concurrency=1 + - pluralkit__discord__gateway_target=http://bot:5002/events + - pluralkit__db__data_db_uri=postgresql://postgres:postgres@db:5432/postgres + - pluralkit__db__data_redis_addr=redis://redis:6379 + - pluralkit__api__temp_token2=1 + - pluralkit__api__remote_url=1 + - pluralkit__api__ratelimit_redis_addr=1 + - pluralkit__discord__client_secret=1 + depends_on: + - redis restart: unless-stopped db: - image: postgres:12-alpine + image: postgres:17-alpine volumes: - "db_data:/var/lib/postgresql/data" - - "/var/run/postgresql:/var/run/postgresql" - command: ["postgres", - "-c", "max-connections=1000", - "-c", "timezone=Etc/UTC", - "-c", "max_wal_size=1GB", - "-c", "min_wal_size=80MB", - "-c", "shared_buffers=128MB"] environment: - "POSTGRES_PASSWORD=postgres" restart: unless-stopped @@ -55,14 +49,5 @@ services: image: redis:alpine restart: unless-stopped - influx: - image: influxdb:1.8 - volumes: - - "influx_data:/var/lib/influxdb" - ports: - - 127.0.0.1:2839:8086 - restart: unless-stopped - volumes: db_data: - influx_data: diff --git a/pluralkit.conf.example b/pluralkit.conf.example deleted file mode 100644 index a705265b..00000000 --- a/pluralkit.conf.example +++ /dev/null @@ -1,10 +0,0 @@ -{ - "PluralKit": { - "Bot": { - "Token": "BOT_TOKEN_GOES_HERE", - "ClientId": 466707357099884544, - }, - "Database": "Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=pluralkit", - "RedisAddr": "127.0.0.1:6379" - } -} \ No newline at end of file From 795e4f43b4f25ef24a36f2cecda7e9fc1d605438 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 19 Apr 2025 18:26:44 +0000 Subject: [PATCH 15/80] fix(bot): get rid of infra-specific code in HttpDiscordCache --- Myriad/Cache/HTTPDiscordCache.cs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/Myriad/Cache/HTTPDiscordCache.cs b/Myriad/Cache/HTTPDiscordCache.cs index b70fcf12..2314b957 100644 --- a/Myriad/Cache/HTTPDiscordCache.cs +++ b/Myriad/Cache/HTTPDiscordCache.cs @@ -14,7 +14,7 @@ public class HttpDiscordCache: IDiscordCache { private readonly ILogger _logger; private readonly HttpClient _client; - private readonly Uri _cacheEndpoint; + private readonly string _cacheEndpoint; private readonly string? _eventTarget; private readonly int _shardCount; private readonly ulong _ownUserId; @@ -29,7 +29,7 @@ public class HttpDiscordCache: IDiscordCache { _logger = logger; _client = client; - _cacheEndpoint = new Uri(cacheEndpoint); + _cacheEndpoint = cacheEndpoint; _eventTarget = eventTarget; _shardCount = shardCount; _ownUserId = ownUserId; @@ -52,13 +52,12 @@ public class HttpDiscordCache: IDiscordCache private async Task QueryCache(string endpoint, ulong guildId) { - var cluster = _cacheEndpoint.Authority; - // todo: there should not be infra-specific code here - if (cluster.Contains(".service.consul") || cluster.Contains("process.pluralkit-gateway.internal")) - // int(((guild_id >> 22) % shard_count) / 16) - cluster = $"cluster{(int)(((guildId >> 22) % (ulong)_shardCount) / 16)}.{cluster}"; + var cluster = _cacheEndpoint; - var response = await _client.GetAsync($"{_cacheEndpoint.Scheme}://{cluster}{endpoint}"); + if (cluster.Contains("{clusterid}")) + cluster = cluster.Replace("{clusterid}", $"{(int)(((guildId >> 22) % (ulong)_shardCount) / 16)}"); + + var response = await _client.GetAsync($"http://{cluster}{endpoint}"); if (response.StatusCode == HttpStatusCode.NotFound) return default; @@ -81,14 +80,13 @@ public class HttpDiscordCache: IDiscordCache if (_eventTarget == null) throw new Exception("missing event target for remote await event"); - var cluster = _cacheEndpoint.Authority; - // todo: there should not be infra-specific code here - if (cluster.Contains(".service.consul") || cluster.Contains("process.pluralkit-gateway.internal")) - // int(((guild_id >> 22) % shard_count) / 16) - cluster = $"cluster{shardId / 16}.{cluster}"; + var cluster = _cacheEndpoint; + + if (cluster.Contains("{clusterid}")) + cluster = cluster.Replace("{clusterid}", $"{(int)(shardId / 16)}"); var response = await _client.PostAsync( - $"{_cacheEndpoint.Scheme}://{cluster}/await_event", + $"http://{cluster}/await_event", new StringContent(JsonSerializer.Serialize(data), Encoding.UTF8) ); From 5fcee4eb297221fb78814e58e0b3db580e9e3a6c Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 19 Apr 2025 18:21:36 +0000 Subject: [PATCH 16/80] feat(gateway): add option to use source address in gateway awaiter --- crates/gateway/src/cache_api.rs | 13 +++++++------ crates/gateway/src/event_awaiter.rs | 27 +++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/crates/gateway/src/cache_api.rs b/crates/gateway/src/cache_api.rs index 2e91f465..4eb23ef1 100644 --- a/crates/gateway/src/cache_api.rs +++ b/crates/gateway/src/cache_api.rs @@ -1,5 +1,5 @@ use axum::{ - extract::{Path, State}, + extract::{ConnectInfo, Path, State}, http::StatusCode, response::{IntoResponse, Response}, routing::{delete, get, post}, @@ -17,7 +17,7 @@ use crate::{ }, event_awaiter::{AwaitEventRequest, EventAwaiter}, }; -use std::sync::Arc; +use std::{net::SocketAddr, sync::Arc}; fn status_code(code: StatusCode, body: String) -> Response { (code, body).into_response() @@ -197,12 +197,13 @@ pub async fn run_server(cache: Arc, runtime_config: Arc, body: String| async move { + info!("got request: {body} from: {addr}"); let Ok(req) = serde_json::from_str::(&body) else { return status_code(StatusCode::BAD_REQUEST, "".to_string()); }; - awaiter.handle_request(req).await; + + awaiter.handle_request(req, addr).await; status_code(StatusCode::NO_CONTENT, "".to_string()) })) .route("/clear_awaiter", post(|| async move { @@ -216,7 +217,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc()).await?; Ok(()) } diff --git a/crates/gateway/src/event_awaiter.rs b/crates/gateway/src/event_awaiter.rs index 10f1888d..765ad8e5 100644 --- a/crates/gateway/src/event_awaiter.rs +++ b/crates/gateway/src/event_awaiter.rs @@ -4,6 +4,7 @@ use std::{ collections::{hash_map::Entry, HashMap}, + net::{IpAddr, SocketAddr}, time::Duration, }; @@ -149,7 +150,7 @@ impl EventAwaiter { } } - pub async fn handle_request(&self, req: AwaitEventRequest) { + pub async fn handle_request(&self, req: AwaitEventRequest, addr: SocketAddr) { match req { AwaitEventRequest::Reaction { message_id, @@ -167,7 +168,7 @@ impl EventAwaiter { .unwrap_or(DEFAULT_TIMEOUT), ) .expect("invalid time"), - target, + target_or_addr(target, addr), ), ); } @@ -188,7 +189,7 @@ impl EventAwaiter { .unwrap_or(DEFAULT_TIMEOUT), ) .expect("invalid time"), - target, + target_or_addr(target, addr), options, ), ); @@ -208,7 +209,7 @@ impl EventAwaiter { .unwrap_or(DEFAULT_TIMEOUT), ) .expect("invalid time"), - target, + target_or_addr(target, addr), ), ); } @@ -221,3 +222,21 @@ impl EventAwaiter { self.interactions.write().await.clear(); } } + +fn target_or_addr(target: String, addr: SocketAddr) -> String { + if target == "source-addr" { + let ip_str = match addr.ip() { + IpAddr::V4(v4) => v4.to_string(), + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + v4.to_string() + } else { + format!("[{v6}]") + } + } + }; + format!("http://{ip_str}:5002/events") + } else { + target + } +} From 7ba16c13477b4dd09c2323f4756b07b6e4abfc48 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 19 Apr 2025 17:23:38 +0000 Subject: [PATCH 17/80] fix(gateway): throw error on non-200 from bot --- crates/gateway/src/main.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 7a2a0721..3864d482 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -7,14 +7,14 @@ use discord::gateway::cluster_config; use event_awaiter::EventAwaiter; use fred::{clients::RedisPool, interfaces::*}; use libpk::runtime_config::RuntimeConfig; -use reqwest::ClientBuilder; +use reqwest::{ClientBuilder, StatusCode}; use signal_hook::{ consts::{SIGINT, SIGTERM}, iterator::Signals, }; use std::{sync::Arc, time::Duration, vec::Vec}; use tokio::{sync::mpsc::channel, task::JoinSet}; -use tracing::{debug, error, info, warn}; +use tracing::{error, info, warn}; use twilight_gateway::{MessageSender, ShardId}; use twilight_model::gateway::payload::outgoing::UpdatePresence; @@ -97,7 +97,7 @@ async fn real_main() -> anyhow::Result<()> { while let Some((shard_id, parsed_event, raw_event)) = event_rx.recv().await { let target = if let Some(target) = awaiter.target_for_event(parsed_event).await { - debug!("sending event to awaiter"); + info!(target = ?target, "sending event to awaiter"); Some(target) } else if let Some(target) = runtime_config.get(RUNTIME_CONFIG_KEY_EVENT_TARGET).await @@ -111,13 +111,24 @@ async fn real_main() -> anyhow::Result<()> { tokio::spawn({ let client = client.clone(); async move { - if let Err(error) = client + match client .post(format!("{target}/{}", shard_id.number())) .body(raw_event) .send() .await { - error!(error = ?error, "failed to request event target") + Ok(res) => { + if res.status() != StatusCode::OK { + error!( + status = ?res.status(), + target = ?target, + "got non-200 from bot while sending event", + ); + } + } + Err(error) => { + error!(error = ?error, "failed to request event target"); + } } } }); From bfa0071f906041714f89a89db29a47b75c6da225 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 20 Apr 2025 18:47:29 +0000 Subject: [PATCH 18/80] feat(gateway): runtime_config_key in config --- crates/gateway/src/main.rs | 6 +++++- crates/libpk/src/_config.rs | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 3864d482..85082c83 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -35,7 +35,11 @@ async fn real_main() -> anyhow::Result<()> { let runtime_config = Arc::new( RuntimeConfig::new( redis.clone(), - format!("gateway:{}", cluster_config().node_id), + format!( + "{}:{}", + libpk::config.runtime_config_key.as_ref().unwrap(), + cluster_config().node_id + ), ) .await?, ); diff --git a/crates/libpk/src/_config.rs b/crates/libpk/src/_config.rs index aa6a1f3b..c0f0b65a 100644 --- a/crates/libpk/src/_config.rs +++ b/crates/libpk/src/_config.rs @@ -116,6 +116,9 @@ pub struct PKConfig { #[serde(default = "_json_log_default")] pub(crate) json_log: bool, + #[serde(default)] + pub runtime_config_key: Option, + #[serde(default)] pub sentry_url: Option, } From 44c5a2d1062e353b1fb73531155a53e8da0f011c Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 20 Apr 2025 19:25:11 +0000 Subject: [PATCH 19/80] feat(gateway): get node id from kubernetes --- crates/gateway/src/discord/gateway.rs | 7 +++++++ crates/libpk/src/_config.rs | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index 4c30e6a9..5ca24185 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use futures::StreamExt; use libpk::{_config::ClusterSettings, runtime_config::RuntimeConfig}; use metrics::counter; @@ -48,6 +49,12 @@ pub fn create_shards(redis: fred::clients::RedisPool) -> anyhow::Result = { + // hacks if let Ok(var) = std::env::var("NOMAD_ALLOC_INDEX") && std::env::var("pluralkit__discord__cluster__total_nodes").is_ok() { std::env::set_var("pluralkit__discord__cluster__node_id", var); } + if let Ok(var) = std::env::var("STATEFULSET_NAME_FOR_INDEX") + && std::env::var("pluralkit__discord__cluster__total_nodes").is_ok() { + std::env::set_var("pluralkit__discord__cluster__node_id", var.split("-").last().unwrap()); + } Arc::new(Config::builder() .add_source(config::Environment::with_prefix("pluralkit").separator("__")) From 094ddbcea215a69eca49160a2563423829f4c7d6 Mon Sep 17 00:00:00 2001 From: rladenson Date: Thu, 24 Apr 2025 01:53:56 -0600 Subject: [PATCH 20/80] fix: string interpolation error --- PluralKit.Bot/Commands/SystemEdit.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PluralKit.Bot/Commands/SystemEdit.cs b/PluralKit.Bot/Commands/SystemEdit.cs index 7df7efff..42d6ab5c 100644 --- a/PluralKit.Bot/Commands/SystemEdit.cs +++ b/PluralKit.Bot/Commands/SystemEdit.cs @@ -473,7 +473,7 @@ public class SystemEdit else str += " Member names will now use the global system tag when proxied in the current server, if there is one set." - + "\n\nTo check or change where your tag appears in your name use the command `{ctx.DefaultPrefix}cfg name format`."; + + $"\n\nTo check or change where your tag appears in your name use the command `{ctx.DefaultPrefix}cfg name format`."; } } From 63d9b411aea89c7b4dcfe24bd7cbbc41fb08ed28 Mon Sep 17 00:00:00 2001 From: alyssa Date: Thu, 24 Apr 2025 10:28:45 +0000 Subject: [PATCH 21/80] fix(bot): only update discord status when gateway is enabled --- PluralKit.Bot/Bot.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PluralKit.Bot/Bot.cs b/PluralKit.Bot/Bot.cs index 46e4e4a1..c0b33c99 100644 --- a/PluralKit.Bot/Bot.cs +++ b/PluralKit.Bot/Bot.cs @@ -282,7 +282,7 @@ public class Bot _logger.Debug("Running once-per-minute scheduled tasks"); // Check from a new custom status from Redis and update Discord accordingly - if (true) + if (!_config.DisableGateway) { var newStatus = await _redis.Connection.GetDatabase().StringGetAsync("pluralkit:botstatus"); if (newStatus != CustomStatusMessage) From 6c0c7a5c9918c106bef1ccc09b4ee434ca830c1b Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 26 Apr 2025 12:03:00 +0000 Subject: [PATCH 22/80] feat(api): pull SP avatars --- PluralKit.API/ApiConfig.cs | 1 + .../AuthorizationTokenHandlerMiddleware.cs | 6 ++ .../Controllers/v2/MemberControllerV2.cs | 34 ++++++++ crates/api/src/main.rs | 2 + crates/api/src/middleware/authnz.rs | 87 ++++++++++++++----- crates/api/src/middleware/logger.rs | 36 +++++--- crates/api/src/middleware/ratelimit.rs | 34 +++----- crates/api/src/util.rs | 1 + crates/avatars/src/main.rs | 2 +- crates/avatars/src/pull.rs | 8 ++ 10 files changed, 153 insertions(+), 58 deletions(-) diff --git a/PluralKit.API/ApiConfig.cs b/PluralKit.API/ApiConfig.cs index fc34d515..46556a79 100644 --- a/PluralKit.API/ApiConfig.cs +++ b/PluralKit.API/ApiConfig.cs @@ -6,4 +6,5 @@ public class ApiConfig public string? ClientId { get; set; } public string? ClientSecret { get; set; } public bool TrustAuth { get; set; } = false; + public string? AvatarServiceUrl { get; set; } } \ No newline at end of file diff --git a/PluralKit.API/AuthorizationTokenHandlerMiddleware.cs b/PluralKit.API/AuthorizationTokenHandlerMiddleware.cs index a09c869e..5f1c4011 100644 --- a/PluralKit.API/AuthorizationTokenHandlerMiddleware.cs +++ b/PluralKit.API/AuthorizationTokenHandlerMiddleware.cs @@ -21,6 +21,12 @@ public class AuthorizationTokenHandlerMiddleware && int.TryParse(sidHeaders[0], out var systemId)) ctx.Items.Add("SystemId", new SystemId(systemId)); + if (cfg.TrustAuth + && ctx.Request.Headers.TryGetValue("X-PluralKit-AppId", out var aidHeaders) + && aidHeaders.Count > 0 + && int.TryParse(aidHeaders[0], out var appId)) + ctx.Items.Add("AppId", appId); + await _next.Invoke(ctx); } } \ No newline at end of file diff --git a/PluralKit.API/Controllers/v2/MemberControllerV2.cs b/PluralKit.API/Controllers/v2/MemberControllerV2.cs index 25163dff..6c37fa81 100644 --- a/PluralKit.API/Controllers/v2/MemberControllerV2.cs +++ b/PluralKit.API/Controllers/v2/MemberControllerV2.cs @@ -1,3 +1,7 @@ +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; + using Microsoft.AspNetCore.Mvc; using Newtonsoft.Json.Linq; @@ -50,6 +54,9 @@ public class MemberControllerV2: PKControllerBase if (patch.Errors.Count > 0) throw new ModelParseError(patch.Errors); + if (patch.AvatarUrl.Value != null) + patch.AvatarUrl = await TryUploadAvatar(patch.AvatarUrl.Value, system); + using var conn = await _db.Obtain(); using var tx = await conn.BeginTransactionAsync(); @@ -110,6 +117,9 @@ public class MemberControllerV2: PKControllerBase if (patch.Errors.Count > 0) throw new ModelParseError(patch.Errors); + if (patch.AvatarUrl.Value != null) + patch.AvatarUrl = await TryUploadAvatar(patch.AvatarUrl.Value, system); + var newMember = await _repo.UpdateMember(member.Id, patch); return Ok(newMember.ToJson(LookupContext.ByOwner, systemStr: system.Hid)); } @@ -129,4 +139,28 @@ public class MemberControllerV2: PKControllerBase return NoContent(); } + + private async Task TryUploadAvatar(string avatarUrl, PKSystem system) + { + if (!avatarUrl.StartsWith("https://serve.apparyllis.com/")) return avatarUrl; + if (_config.AvatarServiceUrl == null) return avatarUrl; + if (!HttpContext.Items.TryGetValue("AppId", out var appId) || (int)appId != 1) return avatarUrl; + + using var client = new HttpClient(); + var response = await client.PostAsJsonAsync(_config.AvatarServiceUrl + "/pull", + new { url = avatarUrl, kind = "avatar", uploaded_by = (string)null, system_id = system.Uuid.ToString() }); + + if (response.StatusCode != HttpStatusCode.OK) + { + var error = await response.Content.ReadFromJsonAsync(); + throw new PKError(500, 0, $"Error uploading image to CDN: {error.Error}"); + } + + var success = await response.Content.ReadFromJsonAsync(); + return success.Url; + } + + public record ErrorResponse(string Error); + + public record SuccessResponse(string Url, bool New); } \ No newline at end of file diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index bf07ff8f..7e23a22d 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -1,3 +1,5 @@ +#![feature(let_chains)] + use axum::{ body::Body, extract::{Request as ExtractRequest, State}, diff --git a/crates/api/src/middleware/authnz.rs b/crates/api/src/middleware/authnz.rs index 4544e6bf..47140767 100644 --- a/crates/api/src/middleware/authnz.rs +++ b/crates/api/src/middleware/authnz.rs @@ -1,45 +1,90 @@ use axum::{ extract::{Request, State}, - http::HeaderValue, + http::StatusCode, middleware::Next, response::Response, }; use tracing::error; -use crate::ApiContext; +use crate::{util::json_err, ApiContext}; -use super::logger::DID_AUTHENTICATE_HEADER; +pub const INTERNAL_SYSTEMID_HEADER: &'static str = "x-pluralkit-systemid"; +pub const INTERNAL_APPID_HEADER: &'static str = "x-pluralkit-appid"; + +// todo: auth should pass down models in request context +// not numerical ids in headers pub async fn authnz(State(ctx): State, mut request: Request, next: Next) -> Response { let headers = request.headers_mut(); - headers.remove("x-pluralkit-systemid"); - let auth_header = headers + + headers.remove(INTERNAL_SYSTEMID_HEADER); + headers.remove(INTERNAL_APPID_HEADER); + + let mut authed_system_id: Option = None; + let mut authed_app_id: Option = None; + + // fetch user authorization + if let Some(system_auth_header) = headers .get("authorization") .map(|h| h.to_str().ok()) - .flatten(); - let mut authenticated = false; - if let Some(auth_header) = auth_header { - if let Some(system_id) = - match libpk::db::repository::legacy_token_auth(&ctx.db, auth_header).await { + .flatten() + && let Some(system_id) = + match libpk::db::repository::legacy_token_auth(&ctx.db, system_auth_header).await { Ok(val) => val, Err(err) => { error!(?err, "failed to query authorization token in postgres"); - None + return json_err( + StatusCode::INTERNAL_SERVER_ERROR, + r#"{"message": "500: Internal Server Error", "code": 0}"#.to_string(), + ); } } - { - headers.append( - "x-pluralkit-systemid", - HeaderValue::from_str(format!("{system_id}").as_str()).unwrap(), - ); - authenticated = true; + { + authed_system_id = Some(system_id); + } + + // fetch app authorization + // todo: actually fetch it from db + if let Some(app_auth_header) = headers + .get("x-pluralkit-app") + .map(|h| h.to_str().ok()) + .flatten() + && let Some(config_token2) = libpk::config + .api + .as_ref() + .expect("missing api config") + .temp_token2 + .as_ref() + // this is NOT how you validate tokens + // but this is low abuse risk so we're keeping it for now + && app_auth_header == config_token2 + { + authed_app_id = Some(1); + } + + // add headers for ratelimiter / dotnet-api + { + let headers = request.headers_mut(); + if let Some(sid) = authed_system_id { + headers.append(INTERNAL_SYSTEMID_HEADER, sid.into()); + } + if let Some(aid) = authed_app_id { + headers.append(INTERNAL_APPID_HEADER, aid.into()); } } + let mut response = next.run(request).await; - if authenticated { - response - .headers_mut() - .insert(DID_AUTHENTICATE_HEADER, HeaderValue::from_static("1")); + + // add headers for logger module (ugh) + { + let headers = response.headers_mut(); + if let Some(sid) = authed_system_id { + headers.append(INTERNAL_SYSTEMID_HEADER, sid.into()); + } + if let Some(aid) = authed_app_id { + headers.append(INTERNAL_APPID_HEADER, aid.into()); + } } + response } diff --git a/crates/api/src/middleware/logger.rs b/crates/api/src/middleware/logger.rs index 020de2e2..8f239042 100644 --- a/crates/api/src/middleware/logger.rs +++ b/crates/api/src/middleware/logger.rs @@ -4,14 +4,15 @@ use axum::{extract::MatchedPath, extract::Request, middleware::Next, response::R use metrics::{counter, histogram}; use tracing::{info, span, warn, Instrument, Level}; -use crate::util::header_or_unknown; +use crate::{ + middleware::authnz::{INTERNAL_APPID_HEADER, INTERNAL_SYSTEMID_HEADER}, + util::header_or_unknown, +}; // log any requests that take longer than 2 seconds // todo: change as necessary const MIN_LOG_TIME: u128 = 2_000; -pub const DID_AUTHENTICATE_HEADER: &'static str = "x-pluralkit-didauthenticate"; - pub async fn logger(request: Request, next: Next) -> Response { let method = request.method().clone(); @@ -40,14 +41,20 @@ pub async fn logger(request: Request, next: Next) -> Response { let mut response = next.run(request).instrument(request_span).await; let elapsed = start.elapsed().as_millis(); - let authenticated = { + let (system_id, app_id) = { let headers = response.headers_mut(); - if headers.contains_key(DID_AUTHENTICATE_HEADER) { - headers.remove(DID_AUTHENTICATE_HEADER); - true - } else { - false - } + ( + headers + .remove(INTERNAL_SYSTEMID_HEADER) + .map(|h| h.to_str().ok().map(|v| v.to_string())) + .flatten() + .unwrap_or("none".to_string()), + headers + .remove(INTERNAL_APPID_HEADER) + .map(|h| h.to_str().ok().map(|v| v.to_string())) + .flatten() + .unwrap_or("none".to_string()), + ) }; counter!( @@ -55,7 +62,8 @@ pub async fn logger(request: Request, next: Next) -> Response { "method" => method.to_string(), "endpoint" => endpoint.clone(), "status" => response.status().to_string(), - "authenticated" => authenticated.to_string(), + "system_id" => system_id.to_string(), + "app_id" => app_id.to_string(), ) .increment(1); histogram!( @@ -63,7 +71,8 @@ pub async fn logger(request: Request, next: Next) -> Response { "method" => method.to_string(), "endpoint" => endpoint.clone(), "status" => response.status().to_string(), - "authenticated" => authenticated.to_string(), + "system_id" => system_id.to_string(), + "app_id" => app_id.to_string(), ) .record(elapsed as f64 / 1_000_f64); @@ -81,7 +90,8 @@ pub async fn logger(request: Request, next: Next) -> Response { "method" => method.to_string(), "endpoint" => endpoint.clone(), "status" => response.status().to_string(), - "authenticated" => authenticated.to_string(), + "system_id" => system_id.to_string(), + "app_id" => app_id.to_string(), ) .increment(1); diff --git a/crates/api/src/middleware/ratelimit.rs b/crates/api/src/middleware/ratelimit.rs index e7bb1dd0..3c4a6be4 100644 --- a/crates/api/src/middleware/ratelimit.rs +++ b/crates/api/src/middleware/ratelimit.rs @@ -12,6 +12,8 @@ use tracing::{debug, error, info, warn}; use crate::util::{header_or_unknown, json_err}; +use super::authnz::{INTERNAL_APPID_HEADER, INTERNAL_SYSTEMID_HEADER}; + const LUA_SCRIPT: &str = include_str!("ratelimit.lua"); lazy_static::lazy_static! { @@ -103,28 +105,8 @@ pub async fn do_request_ratelimited( if let Some(redis) = redis { let headers = request.headers().clone(); let source_ip = header_or_unknown(headers.get("X-PluralKit-Client-IP")); - let authenticated_system_id = header_or_unknown(headers.get("x-pluralkit-systemid")); - - // https://github.com/rust-lang/rust/issues/53667 - let is_temp_token2 = if let Some(header) = request.headers().clone().get("X-PluralKit-App") - { - if let Some(token2) = &libpk::config - .api - .as_ref() - .expect("missing api config") - .temp_token2 - { - if header.to_str().unwrap_or("invalid") == token2 { - true - } else { - false - } - } else { - false - } - } else { - false - }; + let authenticated_system_id = header_or_unknown(headers.get(INTERNAL_SYSTEMID_HEADER)); + let authenticated_app_id = header_or_unknown(headers.get(INTERNAL_APPID_HEADER)); let endpoint = request .extensions() @@ -133,7 +115,13 @@ pub async fn do_request_ratelimited( .map(|v| v.as_str().to_string()) .unwrap_or("unknown".to_string()); - let rlimit = if is_temp_token2 { + // looks like this chooses the tokens/sec by app_id or endpoint + // then chooses the key by system_id or source_ip + // todo: key should probably be chosen by app_id when it's present + // todo: make x-ratelimit-scope actually meaningful + + // hack: for now, we only have one "registered app", so we hardcode the app id + let rlimit = if authenticated_app_id == "1" { RatelimitType::TempCustom } else if endpoint == "/v2/messages/:message_id" { RatelimitType::Message diff --git a/crates/api/src/util.rs b/crates/api/src/util.rs index 03121659..0abb1337 100644 --- a/crates/api/src/util.rs +++ b/crates/api/src/util.rs @@ -56,6 +56,7 @@ pub fn handle_panic(err: Box) -> axum::respo ) } +// todo: make 500 not duplicated pub fn json_err(code: StatusCode, text: String) -> axum::response::Response { let mut response = (code, text).into_response(); let headers = response.headers_mut(); diff --git a/crates/avatars/src/main.rs b/crates/avatars/src/main.rs index 3b621f52..b6f7a2c7 100644 --- a/crates/avatars/src/main.rs +++ b/crates/avatars/src/main.rs @@ -93,7 +93,7 @@ async fn pull( ) -> Result, PKAvatarError> { let parsed = pull::parse_url(&req.url) // parsing beforehand to "normalize" .map_err(|_| PKAvatarError::InvalidCdnUrl)?; - if !req.force { + if !(req.force || req.url.contains("https://serve.apparyllis.com/")) { if let Some(existing) = db::get_by_attachment_id(&state.pool, parsed.attachment_id).await? { // remove any pending image cleanup db::remove_deletion_queue(&state.pool, parsed.attachment_id).await?; diff --git a/crates/avatars/src/pull.rs b/crates/avatars/src/pull.rs index 8b9064d0..3ffb24dc 100644 --- a/crates/avatars/src/pull.rs +++ b/crates/avatars/src/pull.rs @@ -137,6 +137,14 @@ pub fn parse_url(url: &str) -> anyhow::Result { match (url.scheme(), url.domain()) { ("https", Some("media.discordapp.net" | "cdn.discordapp.com")) => {} + ("https", Some("serve.apparyllis.com")) => { + return Ok(ParsedUrl { + channel_id: 0, + attachment_id: 0, + filename: "".to_string(), + full_url: url.to_string(), + }) + } _ => anyhow::bail!("not a discord cdn url"), } From 4a098e45338914636a21495b86b0163f748cccd3 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 26 Apr 2025 17:00:48 +0000 Subject: [PATCH 23/80] fix(gateway): improve shutdown flow --- Cargo.lock | 11 -------- Cargo.toml | 1 - crates/gateway/Cargo.toml | 1 - crates/gateway/src/main.rs | 58 +++++++++++++++++--------------------- 4 files changed, 26 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab8a5c1a..0b9666c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1222,7 +1222,6 @@ dependencies = [ "serde", "serde_json", "serde_variant", - "signal-hook", "tokio", "tracing", "twilight-cache-inmemory", @@ -3634,16 +3633,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "signal-hook" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" -dependencies = [ - "libc", - "signal-hook-registry", -] - [[package]] name = "signal-hook-registry" version = "1.4.2" diff --git a/Cargo.toml b/Cargo.toml index cb9c4dfa..1641ddad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,6 @@ reqwest = { version = "0.12.7" , default-features = false, features = ["rustls-t sentry = { version = "0.36.0", default-features = false, features = ["backtrace", "contexts", "panic", "debug-images", "reqwest", "rustls"] } # replace native-tls with rustls serde = { version = "1.0.196", features = ["derive"] } serde_json = "1.0.117" -signal-hook = "0.3.17" sqlx = { version = "0.8.2", features = ["runtime-tokio", "postgres", "time", "macros", "uuid"] } tokio = { version = "1.36.0", features = ["full"] } tracing = "0.1" diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 420aef1c..c707b29b 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -16,7 +16,6 @@ metrics = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -signal-hook = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 85082c83..7cab2e76 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -8,12 +8,12 @@ use event_awaiter::EventAwaiter; use fred::{clients::RedisPool, interfaces::*}; use libpk::runtime_config::RuntimeConfig; use reqwest::{ClientBuilder, StatusCode}; -use signal_hook::{ - consts::{SIGINT, SIGTERM}, - iterator::Signals, -}; use std::{sync::Arc, time::Duration, vec::Vec}; -use tokio::{sync::mpsc::channel, task::JoinSet}; +use tokio::{ + signal::unix::{signal, SignalKind}, + sync::mpsc::channel, + task::JoinSet, +}; use tracing::{error, info, warn}; use twilight_gateway::{MessageSender, ShardId}; use twilight_model::gateway::payload::outgoing::UpdatePresence; @@ -27,9 +27,6 @@ const RUNTIME_CONFIG_KEY_EVENT_TARGET: &'static str = "event_target"; libpk::main!("gateway"); async fn real_main() -> anyhow::Result<()> { - let (shutdown_tx, mut shutdown_rx) = channel::<()>(1); - let shutdown_tx = Arc::new(shutdown_tx); - let redis = libpk::db::init_redis().await?; let runtime_config = Arc::new( @@ -68,7 +65,8 @@ async fn real_main() -> anyhow::Result<()> { let shards = discord::gateway::create_shards(redis.clone())?; // arbitrary - let (event_tx, mut event_rx) = channel(1000); + // todo: make sure this doesn't fill up + let (event_tx, mut event_rx) = channel::<(ShardId, twilight_gateway::Event, String)>(1000); let mut senders = Vec::new(); let mut signal_senders = Vec::new(); @@ -145,43 +143,39 @@ async fn real_main() -> anyhow::Result<()> { async move { scheduled_task(redis, senders).await }, )); - // todo: probably don't do it this way - let api_shutdown_tx = shutdown_tx.clone(); set.spawn(tokio::spawn(async move { match cache_api::run_server(cache, runtime_config, awaiter.clone()).await { Err(error) => { - tracing::error!(?error, "failed to serve cache api"); - let _ = api_shutdown_tx.send(()); + error!(?error, "failed to serve cache api"); } _ => unreachable!(), } })); - let mut signals = Signals::new(&[SIGINT, SIGTERM])?; - set.spawn(tokio::spawn(async move { - for sig in signals.forever() { - info!("received signal {:?}", sig); - - let presence = UpdatePresence { - op: twilight_model::gateway::OpCode::PresenceUpdate, - d: discord::gateway::presence("Restarting... (please wait)", true), - }; - - for sender in signal_senders.iter() { - let presence = presence.clone(); - let _ = sender.command(&presence); - } - - let _ = shutdown_tx.send(()).await; - break; - } + signal(SignalKind::interrupt()).unwrap().recv().await; + info!("got SIGINT"); })); - let _ = shutdown_rx.recv().await; + set.spawn(tokio::spawn(async move { + signal(SignalKind::terminate()).unwrap().recv().await; + info!("got SIGTERM"); + })); + + set.join_next().await; info!("gateway exiting, have a nice day!"); + let presence = UpdatePresence { + op: twilight_model::gateway::OpCode::PresenceUpdate, + d: discord::gateway::presence("Restarting... (please wait)", true), + }; + + for sender in signal_senders.iter() { + let presence = presence.clone(); + let _ = sender.command(&presence); + } + set.abort_all(); // sleep 500ms to allow everything to clean up properly From dfd9ea553a8fa8177150a6231c86ea5c3cf11070 Mon Sep 17 00:00:00 2001 From: rladenson Date: Sun, 27 Apr 2025 20:43:27 -0600 Subject: [PATCH 24/80] feat: show notice of ToS on system creation --- PluralKit.Bot/Commands/System.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/PluralKit.Bot/Commands/System.cs b/PluralKit.Bot/Commands/System.cs index 8d4026d2..efeeb0f1 100644 --- a/PluralKit.Bot/Commands/System.cs +++ b/PluralKit.Bot/Commands/System.cs @@ -38,10 +38,13 @@ public class System .Field(new Embed.Field("Getting Started", "New to PK? Check out our Getting Started guide on setting up members and proxies: https://pluralkit.me/start\n" + $"Otherwise, type `{ctx.DefaultPrefix}system` to view your system and `{ctx.DefaultPrefix}system help` for more information about commands you can use.")) - .Field(new Embed.Field($"{Emojis.Warn} Notice {Emojis.Warn}", "PluralKit is a bot meant to help you share information about your system. " + + .Field(new Embed.Field($"{Emojis.Warn} Notice: Public By Default {Emojis.Warn}", "PluralKit is a bot meant to help you share information about your system. " + "Member descriptions are meant to be the equivalent to a Discord About Me. Because of this, any info you put in PK is **public by default**.\n" + "Note that this does **not** include message content, only member fields. For more information, check out " + "[the privacy section of the user guide](https://pluralkit.me/guide/#privacy). ")) + .Field(new Embed.Field($"{Emojis.Warn} Notice: Implicit Acceptance of ToS {Emojis.Warn}", "By using the PluralKit bot you implicitly agree to our " + + "[Terms of Service](https://pluralkit.me/terms-of-service/). For questions please ask in our [support server]() or " + + "email legal@pluralkit.me")) .Field(new Embed.Field("System Recovery", "In the case of your Discord account getting lost or deleted, the PluralKit staff can help you recover your system. " + "In order to do so, we will need your **PluralKit token**. This is the *only* way you can prove ownership so we can help you recover your system. " + $"To get it, run `{ctx.DefaultPrefix}token` and then store it in a safe place.\n\n" + From 35090babd4999f228c0855c38d9998f04ef15bbf Mon Sep 17 00:00:00 2001 From: rladenson Date: Sun, 27 Apr 2025 20:48:22 -0600 Subject: [PATCH 25/80] docs: clarify redis address in pk.conf needs changed to match .env --- dev-docs/RUNNING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-docs/RUNNING.md b/dev-docs/RUNNING.md index 9dc5de7e..25e6b97b 100644 --- a/dev-docs/RUNNING.md +++ b/dev-docs/RUNNING.md @@ -47,13 +47,15 @@ pluralkit__discord__bot_token="BOT_TOKEN_GOES_HERE" pluralkit__discord__client_id="BOT_CLIENT_ID_GOES_HERE" pluralkit__discord__client_secret=1 pluralkit__discord__max_concurrency=1 +pluralkit__runtime_config_key="gateway" ``` -**(This should match the username/password/database specified in the flake file and the configuration file)** +**(This should match the username/password/database specified in the flake file and the configuration file AND the redis address in the configuration file should be changed to match this)** *(assuming you already have Git installed, if not, you can start a shell with git by running `nix-shell -p git`)* 1. Clone the repository: `git clone https://github.com/PluralKit/PluralKit` 2. Create a `pluralkit.conf` configuration file in the `PluralKit` directory - Again, the DB URI parameters must match what's in the `flake.nix` file + - Also again, the Redis address must match what is in the .env file created next 3. Create a `.env` configuration file in the `PluralKit` directory *(see above)* 4. Build and run: `nix run .#dev` - This will download the dependencies, build, and run PluralKit From 94a3276979c0ccf98dc68c6d10f4c0ed9cbad7fd Mon Sep 17 00:00:00 2001 From: rladenson Date: Mon, 28 Apr 2025 05:56:21 -0600 Subject: [PATCH 26/80] fix(bot): check message type in message update events --- Myriad/Gateway/Events/MessageUpdateEvent.cs | 2 ++ PluralKit.Bot/Handlers/MessageEdited.cs | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Myriad/Gateway/Events/MessageUpdateEvent.cs b/Myriad/Gateway/Events/MessageUpdateEvent.cs index 2e918fb8..45925549 100644 --- a/Myriad/Gateway/Events/MessageUpdateEvent.cs +++ b/Myriad/Gateway/Events/MessageUpdateEvent.cs @@ -10,6 +10,8 @@ public record MessageUpdateEvent(ulong Id, ulong ChannelId): IGatewayEvent public Optional Member { get; init; } public Optional Attachments { get; init; } + public Message.MessageType Type { get; init; } + public Optional GuildId { get; init; } // TODO: lots of partials } \ No newline at end of file diff --git a/PluralKit.Bot/Handlers/MessageEdited.cs b/PluralKit.Bot/Handlers/MessageEdited.cs index 6ffcd00e..a732fa5b 100644 --- a/PluralKit.Bot/Handlers/MessageEdited.cs +++ b/PluralKit.Bot/Handlers/MessageEdited.cs @@ -107,10 +107,6 @@ public class MessageEdited: IEventHandler ? new Message.Reference(channel.GuildId, evt.ChannelId, lastMessage.ReferencedMessage.Value) : null; - var messageType = lastMessage.ReferencedMessage != null - ? Message.MessageType.Reply - : Message.MessageType.Default; - // TODO: is this missing anything? var equivalentEvt = new MessageCreateEvent { @@ -123,7 +119,7 @@ public class MessageEdited: IEventHandler Attachments = evt.Attachments.Value ?? Array.Empty(), MessageReference = messageReference, ReferencedMessage = referencedMessage, - Type = messageType, + Type = evt.Type, }; return equivalentEvt; } From 7cff9abee36a3425b60510150ee3e2c34ea57ca2 Mon Sep 17 00:00:00 2001 From: skye Date: Sat, 3 May 2025 06:21:39 -0400 Subject: [PATCH 27/80] feat(scheduled_tasks): update for k8s (#734) --- crates/libpk/src/_config.rs | 1 + crates/scheduled_tasks/src/tasks.rs | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/libpk/src/_config.rs b/crates/libpk/src/_config.rs index 5af40a94..6a489c93 100644 --- a/crates/libpk/src/_config.rs +++ b/crates/libpk/src/_config.rs @@ -88,6 +88,7 @@ pub struct ScheduledTasksConfig { pub set_guild_count: bool, pub expected_gateway_count: usize, pub gateway_url: String, + pub prometheus_url: String, } fn _metrics_default() -> bool { diff --git a/crates/scheduled_tasks/src/tasks.rs b/crates/scheduled_tasks/src/tasks.rs index 98396a3f..c7308ed3 100644 --- a/crates/scheduled_tasks/src/tasks.rs +++ b/crates/scheduled_tasks/src/tasks.rs @@ -92,12 +92,14 @@ pub async fn update_discord_stats(ctx: AppCtx) -> anyhow::Result<()> { let mut guild_count = 0; let mut channel_count = 0; + let mut url = cfg.gateway_url.clone(); for idx in 0..cfg.expected_gateway_count { - let res = client - .get(format!("http://cluster{idx}.{}/stats", cfg.gateway_url)) - .send() - .await?; + if url.contains("{clusterid}") { + url = url.replace("{clusterid}", &idx.to_string()); + } + + let res = client.get(&url).send().await?; let stat: GatewayStatus = res.json().await?; @@ -163,6 +165,11 @@ pub async fn update_stats_api(ctx: AppCtx) -> anyhow::Result<()> { .build() .expect("error making client"); + let cfg = config + .scheduled_tasks + .as_ref() + .expect("missing scheduled_tasks config"); + #[derive(serde::Deserialize, Debug)] struct PrometheusResult { data: PrometheusResultData, @@ -178,11 +185,9 @@ pub async fn update_stats_api(ctx: AppCtx) -> anyhow::Result<()> { macro_rules! prom_instant_query { ($t:ty, $q:expr) => {{ + tracing::info!("Query: {}", $q); let resp = client - .get(format!( - "http://vm.svc.pluralkit.net/select/0/prometheus/api/v1/query?query={}", - $q - )) + .get(format!("{}/api/v1/query?query={}", cfg.prometheus_url, $q)) .send() .await?; From f7b594e415c0cb25fdd28e186f148829a7b4fbc5 Mon Sep 17 00:00:00 2001 From: skye Date: Sat, 3 May 2025 12:21:57 -0400 Subject: [PATCH 28/80] chore: add config options and update dev-docs (#736) --- dev-docs/README.md | 50 +++++++++++++++++++++++++++++-- dev-docs/RUNNING.md | 71 --------------------------------------------- dev-docs/dotnet.md | 40 +++++++++++++++++++++++++ dev-docs/rust.md | 48 ++++++++++++++++++++++++++++++ docker-compose.yml | 3 +- flake.nix | 8 ++--- 6 files changed, 140 insertions(+), 80 deletions(-) delete mode 100644 dev-docs/RUNNING.md create mode 100644 dev-docs/dotnet.md create mode 100644 dev-docs/rust.md diff --git a/dev-docs/README.md b/dev-docs/README.md index e712685f..02c18d46 100644 --- a/dev-docs/README.md +++ b/dev-docs/README.md @@ -2,11 +2,19 @@ Most of PluralKit's code is written in C#, and is split into 3 projects: `PluralKit.Core` (supporting libraries), `PluralKit.Bot` (the Discord bot), and `PluralKit.API` (ASP.NET webserver with controllers for most API endpoints). -There is an ongoing effort to port this code to Rust, and we have a handful of crates already. Currently, the main Rust services are `gateway` (connects to Discord to receive events), `api` (handles authentication and rate-limiting for the public API, as well as a couple of private endpoints), and `scheduled_tasks` (background cron job runner, for statistics and miscellaneous cleanup). +There is an ongoing effort to port this code to Rust, and we have a handful of crates already. +Currently, the main Rust services are: +- `gateway` - connects to Discord to receive events +- `api` - handles authentication and rate-limiting for the public API, as well as a couple of private endpoints +- `scheduled_tasks` - background cron job runner, for statistics and miscellaneous cleanup. +- `avatars` - handles avatar storage and cleanup +- `dispatch` - dispatches webhook events + +Additionally, `libpk` handles runtime configuration and database functions. At the very least, `PluralKit.Bot` and `gateway` are required for the bot to run. While code still exists to connect to the Discord gateway directly from the C# bot, this is no longer a supported configuration and may break in the future. -Service-specific documentation can be found for the C# services in [dotnet.md](./dotnet.md), and for the Rust services in [rust.md](./rust.md) (todo; there is a temporary mostly-accurate document at [RUNNING.md](./RUNNING.md)). +Service-specific documentation can be found for the C# services in [dotnet.md](./dotnet.md), and for the Rust services in [rust.md](./rust.md). ## Building/running @@ -16,6 +24,44 @@ The production instance of PluralKit uses Docker images built in CI. These take PluralKit services are configured with environment variables; see service-specific documentation for details. Generally, the configuration from the self-host `docker-compose.yml` should get you started. +### process-compose basic steps + +Your .env should contain at least the following for the bot to run (see the C#/Rust service specific docs for more on configuration): +``` +pluralkit__discord__bot_token="" +PluralKit__Bot__Token="" +pluralkit__discord__client_id="" +PluralKit__Bot__Client="" + +RUST_LOG="info" +pluralkit__db__db_password="postgres" +pluralkit__db__data_db_uri="postgresql://postgres@localhost:5432/pluralkit" +pluralkit__db__data_redis_addr="redis://localhost:6379" +pluralkit__discord__client_secret=1 +pluralkit__discord__max_concurrency=1 +pluralkit__discord__gateway_target="http://localhost:5002/events" +pluralkit__runtime_config_key=gateway +PluralKit__Database="Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=pluralkit" +PluralKit__RedisAddr="localhost:6379" +PluralKit__Bot__DisableGateway="true" +PluralKit__Bot__EventAwaiterTarget="http://localhost:5002/events" +PluralKit__Bot__HttpListenerAddr="127.0.0.1" +PluralKit__Bot__HttpCacheUrl="localhost:5000" +``` + +1. Clone the repository: `git clone https://github.com/PluralKit/PluralKit` +2. Create a `.env` configuration file in the `PluralKit` directory *(see above)* +3. Build and run: `nix run .#dev` + - This will download the dependencies, build, and run PluralKit + - If Nix is not setup to allow flakes, you may need to add `--extra-experimental-features nix-command --extra-experimental-features flakes` to the command + - If the `pluralkit-bot` process fails to run, you can restart it by selecting it and pressing `Ctrl-R` +``` +[nix-shell:~]$ git clone https://github.com/PluralKit/PluralKit +[nix-shell:~]$ cd PluralKit +[nix-shell:~/PluralKit]$ nano .env +[nix-shell:~/PluralKit]$ nix run .#dev +``` + ## Upgrading database from legacy version If you have an instance of the Python version of the bot (from the `legacy` branch), you may need to take extra database migration steps. For more information, see [LEGACYMIGRATE.md](./LEGACYMIGRATE.md). diff --git a/dev-docs/RUNNING.md b/dev-docs/RUNNING.md deleted file mode 100644 index 25e6b97b..00000000 --- a/dev-docs/RUNNING.md +++ /dev/null @@ -1,71 +0,0 @@ -# Technical Overview: - -PluralKit is composed of several different parts, some of which optional and not needed for the bot to function in a testing environment. -##### Required: -- Bot (*PluralKit.Bot*) -- Gateway -- PostgreSQL Database -- Redis Database -##### Optional: -- API (*PluralKit.API*) -- Scheduled Tasks (*scheduled_tasks*) - -*Optionally, it can also integrate with Sentry for error reporting, and InfluxDB for aggregate statistics. In production, we use [VictoriaMetrics](https://victoriametrics.com/) InfluxDB competible endpoint, to query metrics in a Prometheus-compatible format. - -The bot and API are built using .NET 8, with other optional components used for scaling (ex. scheduled_tasks) built using Rust. -# Building + Running - -**The below procedures are intended for development and testing use only! - -Newer versions of the bot have moved to using [Nix](https://nixos.org/) for simplified builds. A docker compose file is also provided, but not recommended as it is not actively maintained. - -See [Configuration](./CONFIGURATION.md) for full details on configuration as only the basics will be covered here. -Configuration is done through a JSON configuration file `pluralkit.conf` placed in the bot's working directory. An example of the configuration format can be seen in [pluralkit.conf.example](pluralkit.conf.example). -The minimum configuration needed for the bot to function must include the following: -- **`PluralKit.Bot.Token`**: the Discord bot token to connect with -- **`PluralKit.Bot.ClientId`**: the ID of the bot's user account, used for calculating the bot's own permissions and for the link in `pk;invite` -- **`PluralKit.Database`**: the URI of the PostgreSQL database to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) -- **`PluralKit.RedisAddr`**: the `host:port` of the Redis database to connect to - -**When running using Docker, you do not need to specify the Postgres or Redis URLs as these will be overwritten by environment variables in the compose file.** - -**When using Nix, the Database URI Username, Password, and Database fields must match what the database was setup with in the `flake.nix` file!** - -The bot can also take configuration from environment variables, which will override the values read from the file. Here, use `:` (colon) or `__` (double underscore) as a level separator (eg. `export PluralKit__Bot__Token=foobar123`) as per [ASP.NET config](https://docs.microsoft.com/en-us/aspnet/core/fundamentals/configuration/?view=aspnetcore-3.1#environment-variables). -## Nix (recommended) -The bot, databases, and services are available to run all in one as a Nix flake (using process-compose-flake). -As of the writing of this document, there are a few caveats with the current flake file. -- The database URI in the config must match the username, password, and database specified during database creation (currently `postgres`, `postgres`, and `pluralkit` respectively). -- Not all services are added to the flake file yet. - -**Additionally, as of the writing of this document, the `pluralkit-gateway` service reads environment variables to get the bot token and database URLs, so you also need to create a `.env` file in the `PluralKit` directory with the following variables:** -``` -pluralkit__db__db_password="postgres" -pluralkit__db__data_db_uri="postgresql://postgres@localhost:5432/pluralkit" -pluralkit__db__data_redis_addr="redis://localhost:6379" -pluralkit__discord__bot_token="BOT_TOKEN_GOES_HERE" -pluralkit__discord__client_id="BOT_CLIENT_ID_GOES_HERE" -pluralkit__discord__client_secret=1 -pluralkit__discord__max_concurrency=1 -pluralkit__runtime_config_key="gateway" -``` -**(This should match the username/password/database specified in the flake file and the configuration file AND the redis address in the configuration file should be changed to match this)** - -*(assuming you already have Git installed, if not, you can start a shell with git by running `nix-shell -p git`)* -1. Clone the repository: `git clone https://github.com/PluralKit/PluralKit` -2. Create a `pluralkit.conf` configuration file in the `PluralKit` directory - - Again, the DB URI parameters must match what's in the `flake.nix` file - - Also again, the Redis address must match what is in the .env file created next -3. Create a `.env` configuration file in the `PluralKit` directory *(see above)* -4. Build and run: `nix run .#dev` - - This will download the dependencies, build, and run PluralKit - - If Nix is not setup to allow flakes, you may need to add `--extra-experimental-features nix-command --extra-experimental-features flakes` to the command - - If the `pluralkit-bot` process fails to run, you can restart it by selecting it and pressing `Ctrl-R` -``` -[nix-shell:~]$ git clone https://github.com/PluralKit/PluralKit -[nix-shell:~]$ cd PluralKit -[nix-shell:~/PluralKit]$ cp pluralkit.conf.example pluralkit.conf -[nix-shell:~/PluralKit]$ nano pluralkit.conf -[nix-shell:~/PluralKit]$ nano .env -[nix-shell:~/PluralKit]$ nix run .#dev -``` diff --git a/dev-docs/dotnet.md b/dev-docs/dotnet.md new file mode 100644 index 00000000..694bc1a8 --- /dev/null +++ b/dev-docs/dotnet.md @@ -0,0 +1,40 @@ +## Configuration +Configuration was previously done through a JSON configuration file `pluralkit.conf` placed in the bot's working directory. To simplify things however and maintain consistency with the Rust services, it is now recommended to use environment variables. + +The minimum configuration needed for the dotnet part of the bot to function include the following: +- **`PluralKit__Bot__Token`**: the Discord bot token to connect with +- **`PluralKit__Bot__ClientId`**: the ID of the bot's user account, used for calculating the bot's own permissions and for the link in `pk;invite` +- **`PluralKit__Database`**: the URI of the PostgreSQL database to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) +- **`PluralKit__RedisAddr`**: the `host:port` of the Redis database to connect to + +**When using Nix, the Database URI Username, Password, and Database fields must match what the database was setup with in the `flake.nix` file!** + + +### Available Configuration Values: + +| Name | Description | Rust Equivalent (if applicable) | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | +| **`PluralKit__Api__ClientId`** | the ID of the bot's user account, used for OAuth with Discord | *`pluralkit__discord__client_id`* | +| **`PluralKit__Api__ClientSecret`** | the client secret of the application, used for OAuth with Discord | *`pluralkit__discord__client_secret`* | +| **`PluralKit__Api__TrustAuth`** | boolean used to determine if the API should trust upstream to provide it the system id of the authenticated user | | +| **`PluralKit__Bot__AdminRole`** | Discord role ID used to determine if a user can use `pk;admin` commands | | +| **`PluralKit__Bot__AvatarServiceUrl`** | the URL of the avatar service | | +| **`PluralKit__Bot__ClientId`** | the ID of the bot's user account, used for calculating the bot's own permissions and for the link in `pk;invite`. | *`pluralkit__discord__client_id`* | +| **`PluralKit__Bot__Cluster__TotalShards`** | the total number of shards | *`pluralkit__discord__cluster__total_shards`* | +| **`PluralKit__Bot__DisableGateway`** | (deprecated) boolean used to enable or disable the inbuilt gateway functions, should be true if using Rust `gateway` | | +| **`PluralKit__Bot__DiscordBaseUrl`** | the base Discord API url used for HTTP API requests | *`pluralkit__discord__api_base_url`* | +| **`PluralKit__Bot__EventAwaiterTarget`** | the target bind address used to receive bot-instance specific events (such as interactive prompts/menus) from `gateway` over http -- value should generally be `source-addr`. | | +| **`PluralKit__Bot__HttpCacheUrl`** | the URL of the http cache to use, as of now, the `gateway` service | | +| **`PluralKit__Bot__HttpListenerAddr`** | the base bind address (use `allv4v6` instead of `::` if you want to also bind to `0.0.0.0`) | | +| **`PluralKit__Bot__Token`** | the Discord bot token to connect with | *`pluralkit__discord__bot_token`* | +| **`PluralKit__Database`** | the URI of the database to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) | *`pluralkit__db__data_db_uri`* (diff formatting) | +| **`PluralKit__MessagesDatabase`** | the URI of the database for message storage to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) | *`pluralkit__db__messages_db_uri`* (diff formatting) | +| **`PluralKit__RedisAddr`** | the `host:port` of a Redis database to connect to | *`pluralkit__db__data_redis_addr`* (diff formatting) | +| **`PluralKit__DispatchProxyToken`** | the token used to authenticate with the dispatch proxy service | | +| **`PluralKit__DispatchProxyUrl`** | the URL of the dispatch proxy service | | +| **`PluralKit__ConsoleLogLevel`** | the minimum [log level](https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.logging.loglevel?view=net-9.0-pp) used for logging | | +| **`PluralKit__InfluxUrl`** | the URL to an [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) server to report aggregate statistics to. An example of these stats can be seen on [the public stats page](https://stats.pluralkit.me). | | +| **`PluralKit__InfluxDb`** | the name of an InfluxDB database to report statistics to. If either this field or `PluralKit.InfluxUrl` are absent, InfluxDB reporting will be disabled. | | +| **`PluralKit__SentryUrl`** | the [Sentry](https://sentry.io/welcome/) client key/DSN to report runtime errors to. If absent, disables Sentry integration. | | + + diff --git a/dev-docs/rust.md b/dev-docs/rust.md new file mode 100644 index 00000000..dec9bd06 --- /dev/null +++ b/dev-docs/rust.md @@ -0,0 +1,48 @@ +## Services Overview +TODO: write more about what each service does (and quirks) + +## Configuration +Configuration is done through environment variables. A potentially uncompleted and/or outdated list is as follows: + +#### Key: +- G - gateway +- A - api +- ST - scheduled_tasks +- AV - avatars +- D - dispatch + +| Used by: | Name | Description | +| --------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| G, ST | **`pluralkit__discord__bot_token`** | the Discord bot token to connect with | +| G, A | **`pluralkit__discord__client_id`** | the ID of the bot's user account, used for calculating the bot's own permissions and for the link in `pk;invite`. | +| A | **`pluralkit__discord__client_secret`** | the client secret of the application, used for OAuth with Discord | +| G | **`pluralkit__discord__cluster__total_shards`** | the total number of shards | +| G | **`pluralkit__discord__cluster__total_nodes`** | the total number of clusters | +| G | **`pluralkit__discord__cluster__node_id`** | the ID of the cluster (overwritten at runtime when operating under managers that can't template the node id into this variable, such as kubernetes) | +| G | **`pluralkit__discord__max_concurrency`** | number of identify requests per 5 seconds -- see Discord docs | +| G | **`pluralkit__discord__gateway_target`** | the URL of a dotnet bot instance to send events to | +| G, ST | **`pluralkit__discord__api_base_url`** | the base Discord API url used for HTTP API requests | +| G, A, ST, AV | **`pluralkit__db__data_db_uri`** | the URI of the PostgreSQL data database in [libpq format](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) | +| G, A, ST, AV | **`pluralkit__db__data_redis_addr`** | the address of the Redis instance, in [standard Redis format](https://redis.io/docs/latest/develop/clients/nodejs/connect/) | +| G, A, ST, AV | **`pluralkit__db__db_password`** | the password to use for PostgreSQL database connections | +| G, A, ST, AV | **`pluralkit__db__messages_db_uri`** | the URI of the PostgreSQL messages database in [libpq format](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) | +| G, A, ST, AV | **`pluralkit__db__stats_db_uri`** | the URI of the PostgreSQL statistics database in [libpq format](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) | +| ST | **`pluralkit__scheduled_tasks__expected_gateway_count`** | the total number of expected running gateway instances | +| ST | **`pluralkit__scheduled_tasks__gateway_url`** | the base URL used for querying statistics from gateway instances | +| ST | **`pluralkit__scheduled_tasks__set_guild_count`** | boolean used to determine if the guild count should be updated in Redis for the bot status | +| A | **`pluralkit__api__addr`** | the bind address used for the Rust API | +| A | **`pluralkit__api__ratelimit_redis_addr`** | the address of a Redis instance to use for request ratelimiting | +| A | **`pluralkit__api__remote_url`** | the remote url of the dotnet API instance | +| A | **`pluralkit__api__temp_token2`** | the token used in the API for fetching app authorization | +| AV | **`pluralkit__avatars__cdn_url`** | the CDN address used for avatar storage | +| AV | **`pluralkit__avatars__cloudflare_token`** | the Cloudflare token to use for avatar cache cleanup | +| AV | **`pluralkit__avatars__cloudflare_zone_id`** | the Cloudflare zone id to use for avatar cache cleanup | +| AV | **`pluralkit__avatars__s3__application_id`** | the application id of the s3 instance to use for avatar storage | +| AV | **`pluralkit__avatars__s3__application_key`** | the application key of the s3 instance to use for avatar storage | +| AV | **`pluralkit__avatars__s3__bucket`** | the bucket to use for avatar storage | +| AV | **`pluralkit__avatars__s3__endpoint`** | the endpoint URL of the s3 instance to use for avatar storage | +| G, A, ST, AV, D | **`pluralkit__json_log`** | boolean used to enable or disable JSON log formatting | +| G | **`pluralkit__runtime_config_key`** | the instance identifier key used when fetching configuration from Redis at runtime to differentiate gateway instances (ex. 'gateway') | +| G, A, ST, AV, D | **`pluralkit__run_metrics_server`** | boolean used to enable or disable the inbuilt Prometheus format metrics server | +| G, A, ST, AV, D | **`pluralkit__sentry_url`** | the URL of a sentry instance to publish errors to | + diff --git a/docker-compose.yml b/docker-compose.yml index 41468461..d58f4c5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - "PluralKit__Bot__Token=${BOT_TOKEN}" - "PluralKit__Bot__ClientId=${CLIENT_ID}" - "PluralKit__Bot__AdminRole=${ADMIN_ROLE}" - - "PluralKit__Bot__HttpCacheUrl=http://gateway:5000" + - "PluralKit__Bot__HttpCacheUrl=gateway:5000" - "PluralKit__Bot__HttpListenerAddr=0.0.0.0" - "PluralKit__Bot__EventAwaiterTarget=http://bot:5002/events" - "PluralKit__Bot__DisableGateway=true" @@ -33,6 +33,7 @@ services: - pluralkit__api__remote_url=1 - pluralkit__api__ratelimit_redis_addr=1 - pluralkit__discord__client_secret=1 + - pluralkit__runtime_config_key=gateway depends_on: - redis restart: unless-stopped diff --git a/flake.nix b/flake.nix index 8cb2c505..cbe6b013 100644 --- a/flake.nix +++ b/flake.nix @@ -103,11 +103,9 @@ process-compose."dev" = let dataDir = ".nix-process-compose"; - pluralkitConfCheck = '' - [[ -f "pluralkit.conf" ]] || (echo "pluralkit config not found, please copy pluralkit.conf.example to pluralkit.conf and edit it" && exit 1) - ''; sourceDotenv = '' - [[ -f ".env" ]] && echo "sourcing .env file..." && export "$(xargs < .env)" + # shellcheck disable=SC2046 + [[ -f ".env" ]] && echo "sourcing .env file..." && export $(xargs < .env) ''; in { imports = [ inp.services.processComposeModules.default ]; @@ -159,7 +157,6 @@ text = '' ${sourceDotenv} set -x - ${pluralkitConfCheck} exec cargo build --bin ${name} ''; }; @@ -177,7 +174,6 @@ text = '' ${sourceDotenv} set -x - ${pluralkitConfCheck} exec ${mkBotEnv "dotnet build -c Release -o obj/"}/bin/env ''; }; From 3d0eb562bfcbdfd8b755794ccbc40ede0ceb48dd Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 27 Apr 2025 19:09:44 +0000 Subject: [PATCH 29/80] feat(bot): listen on multiple addresses, for v4/v6 --- PluralKit.Bot/Services/HttpListenerService.cs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/PluralKit.Bot/Services/HttpListenerService.cs b/PluralKit.Bot/Services/HttpListenerService.cs index 79573625..d2bfc4ce 100644 --- a/PluralKit.Bot/Services/HttpListenerService.cs +++ b/PluralKit.Bot/Services/HttpListenerService.cs @@ -25,15 +25,23 @@ public class HttpListenerService public void Start(string host) { - var server = new WebserverLite(new WebserverSettings(host, 5002), DefaultRoute); + var hosts = new[] { host }; + if (host == "allv4v6") + { + hosts = new[] { "[::]", "0.0.0.0" }; + } + foreach (var h in hosts) + { + var server = new WebserverLite(new WebserverSettings(h, 5002), DefaultRoute); - server.Routes.PreAuthentication.Static.Add(WatsonWebserver.Core.HttpMethod.GET, "/runtime_config", RuntimeConfigGet); - server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.POST, "/runtime_config/{key}", RuntimeConfigSet); - server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.DELETE, "/runtime_config/{key}", RuntimeConfigDelete); + server.Routes.PreAuthentication.Static.Add(WatsonWebserver.Core.HttpMethod.GET, "/runtime_config", RuntimeConfigGet); + server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.POST, "/runtime_config/{key}", RuntimeConfigSet); + server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.DELETE, "/runtime_config/{key}", RuntimeConfigDelete); - server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.POST, "/events/{shard_id}", GatewayEvent); + server.Routes.PreAuthentication.Parameter.Add(WatsonWebserver.Core.HttpMethod.POST, "/events/{shard_id}", GatewayEvent); - server.Start(); + server.Start(); + } } private async Task DefaultRoute(HttpContextBase ctx) From 4dd80805aedb2a70e03b8372aa2c46c8b2a2252d Mon Sep 17 00:00:00 2001 From: Petal Ladenson Date: Tue, 13 May 2025 17:56:29 -0600 Subject: [PATCH 30/80] feat(bot): add system aliases "account" and "acc" --- PluralKit.Bot/CommandMeta/CommandTree.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/PluralKit.Bot/CommandMeta/CommandTree.cs b/PluralKit.Bot/CommandMeta/CommandTree.cs index 3c620e63..8d111ef4 100644 --- a/PluralKit.Bot/CommandMeta/CommandTree.cs +++ b/PluralKit.Bot/CommandMeta/CommandTree.cs @@ -6,7 +6,7 @@ public partial class CommandTree { public Task ExecuteCommand(Context ctx) { - if (ctx.Match("system", "s")) + if (ctx.Match("system", "s", "account", "acc")) return HandleSystemCommand(ctx); if (ctx.Match("member", "m")) return HandleMemberCommand(ctx); @@ -503,6 +503,8 @@ public partial class CommandTree case "system": case "systems": case "s": + case "account": + case "acc": await PrintCommandList(ctx, "systems", SystemCommands); break; case "member": From ddf1e21f25646e1b03fe601ad14833b482aa71f1 Mon Sep 17 00:00:00 2001 From: alyssa Date: Fri, 9 May 2025 12:24:49 +0000 Subject: [PATCH 31/80] feat(avatars): check if images are used before deleting --- crates/avatars/src/cleanup.rs | 4 ++- crates/scheduled_tasks/src/tasks.rs | 54 ++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/crates/avatars/src/cleanup.rs b/crates/avatars/src/cleanup.rs index 0fb815e3..83327cf9 100644 --- a/crates/avatars/src/cleanup.rs +++ b/crates/avatars/src/cleanup.rs @@ -55,9 +55,10 @@ async fn cleanup_job(pool: sqlx::PgPool, bucket: Arc) -> anyhow::Res let mut tx = pool.begin().await?; let image_id: Option = sqlx::query_as( + // no timestamp checking here + // images are only added to the table after 24h r#" select id from image_cleanup_jobs - where ts < now() - interval '1 day' for update skip locked limit 1;"#, ) .fetch_optional(&mut *tx) @@ -72,6 +73,7 @@ async fn cleanup_job(pool: sqlx::PgPool, bucket: Arc) -> anyhow::Res let image_data = libpk::db::repository::avatars::get_by_id(&pool, image_id.clone()).await?; if image_data.is_none() { + // unsure how this can happen? there is a FK reference info!("image {image_id} was already deleted, skipping"); sqlx::query("delete from image_cleanup_jobs where id = $1") .bind(image_id) diff --git a/crates/scheduled_tasks/src/tasks.rs b/crates/scheduled_tasks/src/tasks.rs index c7308ed3..64246fc9 100644 --- a/crates/scheduled_tasks/src/tasks.rs +++ b/crates/scheduled_tasks/src/tasks.rs @@ -18,11 +18,18 @@ pub async fn update_prometheus(ctx: AppCtx) -> anyhow::Result<()> { struct Count { count: i64, } + + let pending_count: Count = sqlx::query_as("select count(*) from image_cleanup_pending_jobs") + .fetch_one(&ctx.data) + .await?; + let count: Count = sqlx::query_as("select count(*) from image_cleanup_jobs") .fetch_one(&ctx.data) .await?; - gauge!("pluralkit_image_cleanup_queue_length").set(count.count as f64); + gauge!("pluralkit_image_cleanup_queue_length", "pending" => "true") + .set(pending_count.count as f64); + gauge!("pluralkit_image_cleanup_queue_length", "pending" => "false").set(count.count as f64); let gateway = ctx.discord.gateway().authed().await?.model().await?; @@ -133,14 +140,10 @@ pub async fn update_discord_stats(ctx: AppCtx) -> anyhow::Result<()> { } pub async fn queue_deleted_image_cleanup(ctx: AppCtx) -> anyhow::Result<()> { - // todo: we want to delete immediately when system is deleted, but after a - // delay if member is deleted - ctx.data - .execute( - r#" -insert into image_cleanup_jobs -select id, now() from images where - not exists (select from image_cleanup_jobs j where j.id = images.id) + // if an image is present on no member, add it to the pending deletion queue + // if it is still present on no member after 24h, actually delete it + + let usage_query = r#" and not exists (select from systems where avatar_url = images.url) and not exists (select from systems where banner_image = images.url) and not exists (select from system_guild where avatar_url = images.url) @@ -152,9 +155,42 @@ select id, now() from images where and not exists (select from groups where icon = images.url) and not exists (select from groups where banner_image = images.url); + "#; + + ctx.data + .execute( + format!( + r#" + insert into image_cleanup_pending_jobs + select id, now() from images where + not exists (select from image_cleanup_pending_jobs j where j.id = images.id) + and not exists (select from image_cleanup_jobs j where j.id = images.id) + {} "#, + usage_query + ) + .as_str(), ) .await?; + + ctx.data + .execute( + format!( + r#" + insert into image_cleanup_jobs + select image_cleanup_pending_jobs.id from image_cleanup_pending_jobs + left join images on images.id = image_cleanup_pending_jobs.id + where + ts < now() - '24 hours'::interval + and not exists (select from image_cleanup_jobs j where j.id = images.id) + {} + "#, + usage_query + ) + .as_str(), + ) + .await?; + Ok(()) } From 8bf738fbd301ab4b2eab8aca6bbf65f194c5e0f4 Mon Sep 17 00:00:00 2001 From: alyssa Date: Thu, 15 May 2025 21:02:57 +0000 Subject: [PATCH 32/80] feat(bot): :bell_pepper: --- PluralKit.Bot/Handlers/ReactionAdded.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/PluralKit.Bot/Handlers/ReactionAdded.cs b/PluralKit.Bot/Handlers/ReactionAdded.cs index 5caa7d6f..c073119d 100644 --- a/PluralKit.Bot/Handlers/ReactionAdded.cs +++ b/PluralKit.Bot/Handlers/ReactionAdded.cs @@ -111,6 +111,7 @@ public class ReactionAdded: IEventHandler case "\U0001F514": // Bell case "\U0001F6CE": // Bellhop bell case "\U0001F3D3": // Ping pong paddle (lol) + case "\U0001FAD1": // Bell pepper case "\u23F0": // Alarm clock case "\u2757": // Exclamation mark { From 1013cf3755e00a5d74b6501f5c05b55e39485a8c Mon Sep 17 00:00:00 2001 From: leo60228 Date: Sun, 18 May 2025 18:11:02 -0400 Subject: [PATCH 33/80] feat(bot): add Zeppelin logclean support (#737) --- PluralKit.Bot/Services/LoggerCleanService.cs | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/PluralKit.Bot/Services/LoggerCleanService.cs b/PluralKit.Bot/Services/LoggerCleanService.cs index 178ce95b..84cc8a1f 100644 --- a/PluralKit.Bot/Services/LoggerCleanService.cs +++ b/PluralKit.Bot/Services/LoggerCleanService.cs @@ -45,6 +45,7 @@ public class LoggerCleanService private static readonly Regex _AnnabelleRegex = new("```\n(\\d{17,19})\n```"); private static readonly Regex _AnnabelleRegexFuzzy = new("\\ A message from \\*\\*[\\w.]{2,32}\\*\\* \\(`(\\d{17,19})`\\) was deleted in <#\\d{17,19}>"); private static readonly Regex _koiraRegex = new("ID:\\*\\* (\\d{17,19})"); + private static readonly Regex _zeppelinRegex = new("🗑 Message \\(`(\\d{17,19})`\\)"); private static readonly Regex _VortexRegex = new("`\\[(\\d\\d:\\d\\d:\\d\\d)\\]` .* \\(ID:(\\d{17,19})\\).* <#\\d{17,19}>:"); @@ -83,7 +84,8 @@ public class LoggerCleanService new LoggerBot("Dozer", 356535250932858885, ExtractDozer), new LoggerBot("Skyra", 266624760782258186, ExtractSkyra), new LoggerBot("Annabelle", 231241068383961088, ExtractAnnabelle, fuzzyExtractFunc: ExtractAnnabelleFuzzy), - new LoggerBot("Koira", 1247013404569239624, ExtractKoira) + new LoggerBot("Koira", 1247013404569239624, ExtractKoira), + new LoggerBot("Zeppelin", 473868086773153793, ExtractZeppelin) // webhook }.ToDictionary(b => b.Id); private static Dictionary _botsByApplicationId @@ -441,6 +443,23 @@ public class LoggerCleanService return match.Success ? ulong.Parse(match.Groups[1].Value) : null; } + private static ulong? ExtractZeppelin(Message msg) + { + // zeppelin uses a non-embed format by default but can be configured to use a customizable embed + // if it's an embed, assume the footer contains the message ID + var embed = msg.Embeds?.FirstOrDefault(); + if (embed == null) + { + var match = _zeppelinRegex.Match(msg.Content ?? ""); + return match.Success ? ulong.Parse(match.Groups[1].Value) : null; + } + else + { + var match = _basicRegex.Match(embed.Footer?.Text ?? ""); + return match.Success ? ulong.Parse(match.Groups[1].Value) : null; + } + } + public class LoggerBot { public ulong Id; From 727f378577d17c2b80522f51765d70e493cacd85 Mon Sep 17 00:00:00 2001 From: leo60228 Date: Sun, 18 May 2025 18:26:06 -0400 Subject: [PATCH 34/80] feat(docs): add Zeppelin to logclean docs (#738) --- docs/content/staff/compatibility.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/staff/compatibility.md b/docs/content/staff/compatibility.md index 77273daa..8d0c6fc5 100644 --- a/docs/content/staff/compatibility.md +++ b/docs/content/staff/compatibility.md @@ -42,6 +42,7 @@ At the moment, log cleanup works with the following bots: - [UnbelievaBoat](https://unbelievaboat.com/) (precise) - Vanessa (fuzzy) - [Vortex](https://github.com/jagrosh/Vortex/wiki) (fuzzy) +- [Zeppelin](https://zeppelin.gg/) (precise) ::: warning In most cases, PluralKit will match log messages by the ID of the deleted message itself. However, some bots (marked with *(fuzzy)* above) don't include this in their logs. In this case, PluralKit will attempt to match based on other parameters, but there may be false positives. From 347add89989311801c69c950aef86cf7f8751745 Mon Sep 17 00:00:00 2001 From: skye Date: Fri, 23 May 2025 13:08:41 -0400 Subject: [PATCH 35/80] fix(dash): fix shard lookup (#742) --- dashboard/src/routes/Status/status.svelte | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboard/src/routes/Status/status.svelte b/dashboard/src/routes/Status/status.svelte index 9f93ecf2..c31c88a1 100644 --- a/dashboard/src/routes/Status/status.svelte +++ b/dashboard/src/routes/Status/status.svelte @@ -26,7 +26,7 @@ const get = async () => { const pkdata = await api().private.discord.shard_state.get(); - let data = pkdata.shards.sort((x, y) => (x.id > y.id) ? 1 : -1); + let data = pkdata.shards.sort((x, y) => (x.shard_id < y.shard_id) ? 1 : -1); let latencies = 0; data = data.map(shard => { latencies += shard.latency; @@ -71,7 +71,7 @@ var match = findShardInput.match(/https:\/\/(?:[\w]*\.)?discord(?:app)?\.com\/channels\/(\d+)\/\d+\/\d+/); if (match != null) { console.log("match", match) - foundShard = shards[Number(getShardID(match[1], shards.length))]; + foundShard = shards[(shards.length - 1) - (Number(getShardID(match[1], shards.length)))]; valid = true; shardInfoMsg = ""; return; @@ -84,7 +84,7 @@ shardInfoMsg = "Invalid server ID"; return; } - foundShard = shards[Number(shard)]; + foundShard = shards[(shards.length - 1) - Number(shard)]; valid = true; shardInfoMsg = ""; } catch(e) { From 7737850afbdf05b6882c95e5e6e5780526f74d0b Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 17 May 2025 15:05:37 +0000 Subject: [PATCH 36/80] chore(rust): correctly format values in errors --- crates/api/src/main.rs | 4 ++-- crates/api/src/middleware/ratelimit.rs | 6 ++--- crates/api/src/util.rs | 12 ++++------ crates/avatars/src/cleanup.rs | 4 ++-- crates/avatars/src/main.rs | 24 -------------------- crates/avatars/src/migrate.rs | 6 ++--- crates/avatars/src/process.rs | 2 +- crates/avatars/src/pull.rs | 23 +++++++++++++++++-- crates/dispatch/src/main.rs | 4 ++-- crates/gateway/src/discord/gateway.rs | 4 ++-- crates/gateway/src/discord/identify_queue.rs | 4 ++-- crates/gateway/src/main.rs | 2 +- crates/gdpr_worker/src/main.rs | 10 ++++---- crates/libpk/src/lib.rs | 8 +++---- crates/scheduled_tasks/src/main.rs | 3 +-- 15 files changed, 54 insertions(+), 62 deletions(-) diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 7e23a22d..24ed58c5 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -47,8 +47,8 @@ async fn rproxy( .rproxy_client .request(req) .await - .map_err(|err| { - error!("failed to serve reverse proxy to dotnet-api: {:?}", err); + .map_err(|error| { + error!(?error, "failed to serve reverse proxy to dotnet-api"); StatusCode::BAD_GATEWAY })? .into_response()) diff --git a/crates/api/src/middleware/ratelimit.rs b/crates/api/src/middleware/ratelimit.rs index 3c4a6be4..29cb205b 100644 --- a/crates/api/src/middleware/ratelimit.rs +++ b/crates/api/src/middleware/ratelimit.rs @@ -52,7 +52,7 @@ pub fn ratelimiter(f: F) -> FromFnLayer, T> { .await { Ok(_) => info!("connected to redis for request rate limiting"), - Err(err) => error!("could not load redis script: {}", err), + Err(error) => error!(?error, "could not load redis script"), } } else { error!("could not wait for connection to load redis script!"); @@ -212,8 +212,8 @@ pub async fn do_request_ratelimited( return response; } - Err(err) => { - tracing::error!("error getting ratelimit info: {}", err); + Err(error) => { + tracing::error!(?error, "error getting ratelimit info"); return json_err( StatusCode::INTERNAL_SERVER_ERROR, r#"{"message": "500: internal server error", "code": 0}"#.to_string(), diff --git a/crates/api/src/util.rs b/crates/api/src/util.rs index 0abb1337..35a5bf0d 100644 --- a/crates/api/src/util.rs +++ b/crates/api/src/util.rs @@ -11,7 +11,7 @@ pub fn header_or_unknown(header: Option<&HeaderValue>) -> &str { match value.to_str() { Ok(v) => v, Err(err) => { - error!("failed to parse header value {:#?}: {:#?}", value, err); + error!(?err, ?value, "failed to parse header value"); "failed to parse" } } @@ -34,11 +34,7 @@ where .unwrap(), ), None => { - error!( - "error in handler {}: {:#?}", - std::any::type_name::(), - error - ); + error!(?error, "error in handler {}", std::any::type_name::(),); json_err( StatusCode::INTERNAL_SERVER_ERROR, r#"{"message": "500: Internal Server Error", "code": 0}"#.to_string(), @@ -48,8 +44,8 @@ where } } -pub fn handle_panic(err: Box) -> axum::response::Response { - error!("caught panic from handler: {:#?}", err); +pub fn handle_panic(error: Box) -> axum::response::Response { + error!(?error, "caught panic from handler"); json_err( StatusCode::INTERNAL_SERVER_ERROR, r#"{"message": "500: Internal Server Error", "code": 0}"#.to_string(), diff --git a/crates/avatars/src/cleanup.rs b/crates/avatars/src/cleanup.rs index 83327cf9..9b5ed249 100644 --- a/crates/avatars/src/cleanup.rs +++ b/crates/avatars/src/cleanup.rs @@ -38,8 +38,8 @@ async fn real_main() -> anyhow::Result<()> { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; match cleanup_job(pool.clone(), bucket.clone()).await { Ok(()) => {} - Err(err) => { - error!("failed to run avatar cleanup job: {}", err); + Err(error) => { + error!(?error, "failed to run avatar cleanup job"); // sentry } } diff --git a/crates/avatars/src/main.rs b/crates/avatars/src/main.rs index b6f7a2c7..b551cc41 100644 --- a/crates/avatars/src/main.rs +++ b/crates/avatars/src/main.rs @@ -232,26 +232,11 @@ async fn real_main() -> anyhow::Result<()> { Ok(()) } -struct AppError(anyhow::Error); - #[derive(Serialize)] struct ErrorResponse { error: String, } -impl IntoResponse for AppError { - fn into_response(self) -> Response { - error!("error handling request: {}", self.0); - ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ErrorResponse { - error: self.0.to_string(), - }), - ) - .into_response() - } -} - impl IntoResponse for PKAvatarError { fn into_response(self) -> Response { let status_code = match self { @@ -278,12 +263,3 @@ impl IntoResponse for PKAvatarError { .into_response() } } - -impl From for AppError -where - E: Into, -{ - fn from(err: E) -> Self { - Self(err.into()) - } -} diff --git a/crates/avatars/src/migrate.rs b/crates/avatars/src/migrate.rs index 87a5747d..8c72267c 100644 --- a/crates/avatars/src/migrate.rs +++ b/crates/avatars/src/migrate.rs @@ -129,9 +129,9 @@ pub async fn worker(worker_id: u32, state: Arc) { Ok(()) => {} Err(e) => { error!( - "error in migrate worker {}: {}", - worker_id, - e.source().unwrap_or(&e) + error = e.source().unwrap_or(&e) + ?worker_id, + "error in migrate worker", ); tokio::time::sleep(Duration::from_secs(5)).await; } diff --git a/crates/avatars/src/process.rs b/crates/avatars/src/process.rs index 61eaaef2..024f40de 100644 --- a/crates/avatars/src/process.rs +++ b/crates/avatars/src/process.rs @@ -84,7 +84,7 @@ pub fn process(data: &[u8], kind: ImageKind) -> Result() { Ok(v) if v.scheme_str() == Some("https") && v.host().is_some() => v, Err(error) => { - error!(?error, "failed to parse uri {}", req.url); + error!(?error, uri = req.url, "failed to parse uri"); return DispatchResponse::BadData.to_string(); } _ => { - error!("uri {} is invalid", req.url); + error!(uri = req.url, "uri is invalid"); return DispatchResponse::BadData.to_string(); } }; diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index 5ca24185..1c0239f5 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -124,7 +124,7 @@ pub async fn runner( .increment(1); if let Err(error) = shard_state.socket_closed(shard_id).await { - error!("failed to update shard state for socket closure: {error}"); + error!(?error, "failed to update shard state for socket closure"); } continue; @@ -145,7 +145,7 @@ pub async fn runner( continue; } Err(error) => { - error!("shard {shard_id} failed to parse gateway event: {error}"); + error!(?error, ?shard_id, "failed to parse gateway event"); continue; } }; diff --git a/crates/gateway/src/discord/identify_queue.rs b/crates/gateway/src/discord/identify_queue.rs index 2d523dfa..4df8dc4f 100644 --- a/crates/gateway/src/discord/identify_queue.rs +++ b/crates/gateway/src/discord/identify_queue.rs @@ -78,8 +78,8 @@ async fn request_inner(redis: RedisPool, concurrency: u32, shard_id: u32, tx: on Ok(None) => { // not allowed yet, waiting } - Err(e) => { - error!(shard_id, bucket, "error getting shard allowance: {}", e) + Err(error) => { + error!(?error, ?shard_id, ?bucket, "error getting shard allowance") } } diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 7cab2e76..5cf08f46 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -129,7 +129,7 @@ async fn real_main() -> anyhow::Result<()> { } } Err(error) => { - error!(error = ?error, "failed to request event target"); + error!(?error, "failed to request event target"); } } } diff --git a/crates/gdpr_worker/src/main.rs b/crates/gdpr_worker/src/main.rs index 7409bdff..e801b9fe 100644 --- a/crates/gdpr_worker/src/main.rs +++ b/crates/gdpr_worker/src/main.rs @@ -42,8 +42,8 @@ async fn real_main() -> anyhow::Result<()> { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; match run_job(db.clone(), client.clone()).await { Ok(()) => {} - Err(err) => { - error!("failed to run messages gdpr job: {:?}", err); + Err(error) => { + error!(?error, "failed to run messages gdpr job"); } } } @@ -131,8 +131,10 @@ async fn run_job(pool: sqlx::PgPool, discord: Arc) -> any } _ => { error!( - "got unknown error deleting message {}: status={status}, code={code}", - message.mid + ?status, + ?code, + message_id = message.mid, + "got unknown error deleting message", ); } } diff --git a/crates/libpk/src/lib.rs b/crates/libpk/src/lib.rs index 10d174d0..8de69d92 100644 --- a/crates/libpk/src/lib.rs +++ b/crates/libpk/src/lib.rs @@ -81,12 +81,12 @@ macro_rules! main { .build() .unwrap() .block_on(async { - if let Err(err) = libpk::init_metrics() { - tracing::error!("failed to init metrics collector: {err}"); + if let Err(error) = libpk::init_metrics() { + tracing::error!(?error, "failed to init metrics collector"); }; tracing::info!("hello world"); - if let Err(err) = real_main().await { - tracing::error!("failed to run service: {err}"); + if let Err(error) = real_main().await { + tracing::error!(?error, "failed to run service"); }; }); Ok(()) diff --git a/crates/scheduled_tasks/src/main.rs b/crates/scheduled_tasks/src/main.rs index cd0a182b..56d58c7c 100644 --- a/crates/scheduled_tasks/src/main.rs +++ b/crates/scheduled_tasks/src/main.rs @@ -74,8 +74,7 @@ async fn real_main() -> anyhow::Result<()> { info!("running {}", $desc); let before = std::time::Instant::now(); if let Err(error) = $fn(ctx).await { - error!("failed to run {}: {}", $desc, error); - // sentry + error!(?error, "failed to run {}", $desc); } let duration = before.elapsed(); info!("ran {} in {duration:?}", $desc); From f9a6c2cefe5545a2584275d45d40f3b97f9d9302 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 17 May 2025 16:19:30 +0000 Subject: [PATCH 37/80] chore(rust): tidier libpk::main macro impl --- Cargo.lock | 22 +++++----- crates/api/src/main.rs | 4 +- crates/avatars/src/cleanup.rs | 4 +- crates/avatars/src/main.rs | 4 +- crates/dispatch/Cargo.toml | 1 + crates/dispatch/src/main.rs | 11 +---- crates/gateway/src/main.rs | 4 +- crates/gdpr_worker/src/main.rs | 4 +- crates/libpk/Cargo.toml | 1 + crates/libpk/src/lib.rs | 29 ++----------- crates/{model_macros => macros}/Cargo.toml | 2 +- crates/macros/src/entrypoint.rs | 41 +++++++++++++++++++ crates/macros/src/lib.rs | 14 +++++++ .../src/lib.rs => macros/src/model.rs} | 3 +- crates/models/Cargo.toml | 2 +- crates/models/src/system.rs | 2 +- crates/models/src/system_config.rs | 2 +- crates/scheduled_tasks/src/main.rs | 4 +- 18 files changed, 90 insertions(+), 64 deletions(-) rename crates/{model_macros => macros}/Cargo.toml (85%) create mode 100644 crates/macros/src/entrypoint.rs create mode 100644 crates/macros/src/lib.rs rename crates/{model_macros/src/lib.rs => macros/src/model.rs} (99%) diff --git a/Cargo.lock b/Cargo.lock index 0b9666c6..2228d94b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -861,6 +861,7 @@ dependencies = [ "anyhow", "axum 0.7.9", "hickory-client", + "libpk", "reqwest 0.12.15", "serde", "serde_json", @@ -2011,6 +2012,7 @@ dependencies = [ "lazy_static", "metrics", "metrics-exporter-prometheus", + "pk_macros", "sentry", "sentry-tracing", "serde", @@ -2213,15 +2215,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "model_macros" -version = "0.1.0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "nibble_vec" version = "0.1.0" @@ -2518,6 +2511,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pk_macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -2550,7 +2552,7 @@ name = "pluralkit_models" version = "0.1.0" dependencies = [ "chrono", - "model_macros", + "pk_macros", "sea-query", "serde", "serde_json", diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 24ed58c5..590064eb 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -130,8 +130,8 @@ fn router(ctx: ApiContext) -> Router { .route("/", get(|| async { axum::response::Redirect::to("https://pluralkit.me/api") })) } -libpk::main!("api"); -async fn real_main() -> anyhow::Result<()> { +#[libpk::main] +async fn main() -> anyhow::Result<()> { let db = libpk::db::init_data_db().await?; let redis = libpk::db::init_redis().await?; diff --git a/crates/avatars/src/cleanup.rs b/crates/avatars/src/cleanup.rs index 9b5ed249..cbf5ef1e 100644 --- a/crates/avatars/src/cleanup.rs +++ b/crates/avatars/src/cleanup.rs @@ -4,8 +4,8 @@ use sqlx::prelude::FromRow; use std::{sync::Arc, time::Duration}; use tracing::{error, info}; -libpk::main!("avatar_cleanup"); -async fn real_main() -> anyhow::Result<()> { +#[libpk::main] +async fn main() -> anyhow::Result<()> { let config = libpk::config .avatars .as_ref() diff --git a/crates/avatars/src/main.rs b/crates/avatars/src/main.rs index b551cc41..863619d6 100644 --- a/crates/avatars/src/main.rs +++ b/crates/avatars/src/main.rs @@ -170,8 +170,8 @@ pub struct AppState { config: Arc, } -libpk::main!("avatars"); -async fn real_main() -> anyhow::Result<()> { +#[libpk::main] +async fn main() -> anyhow::Result<()> { let config = libpk::config .avatars .as_ref() diff --git a/crates/dispatch/Cargo.toml b/crates/dispatch/Cargo.toml index 81e35811..c76856d1 100644 --- a/crates/dispatch/Cargo.toml +++ b/crates/dispatch/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] anyhow = { workspace = true } axum = { workspace = true } +libpk = { path = "../libpk" } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/dispatch/src/main.rs b/crates/dispatch/src/main.rs index 5c355d71..6570cf19 100644 --- a/crates/dispatch/src/main.rs +++ b/crates/dispatch/src/main.rs @@ -19,17 +19,8 @@ use axum::{extract::State, http::Uri, routing::post, Json, Router}; mod logger; -// this package does not currently use libpk - -#[tokio::main] +#[libpk::main] async fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt() - .json() - .with_env_filter(EnvFilter::from_default_env()) - .init(); - - info!("hello world"); - let address = std::env::var("DNS_UPSTREAM").unwrap().parse().unwrap(); let stream = UdpClientStream::::with_timeout(address, Duration::from_secs(3)); let (client, bg) = AsyncClient::connect(stream).await?; diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 5cf08f46..ed9ccca1 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -25,8 +25,8 @@ mod logger; const RUNTIME_CONFIG_KEY_EVENT_TARGET: &'static str = "event_target"; -libpk::main!("gateway"); -async fn real_main() -> anyhow::Result<()> { +#[libpk::main] +async fn main() -> anyhow::Result<()> { let redis = libpk::db::init_redis().await?; let runtime_config = Arc::new( diff --git a/crates/gdpr_worker/src/main.rs b/crates/gdpr_worker/src/main.rs index e801b9fe..b40557c0 100644 --- a/crates/gdpr_worker/src/main.rs +++ b/crates/gdpr_worker/src/main.rs @@ -11,8 +11,8 @@ use twilight_model::id::{ // create table messages_gdpr_jobs (mid bigint not null references messages(mid) on delete cascade, channel bigint not null); -libpk::main!("messages_gdpr_worker"); -async fn real_main() -> anyhow::Result<()> { +#[libpk::main] +async fn main() -> anyhow::Result<()> { let db = libpk::db::init_messages_db().await?; let mut client_builder = twilight_http::Client::builder() diff --git a/crates/libpk/Cargo.toml b/crates/libpk/Cargo.toml index 40d430c8..30d77ae0 100644 --- a/crates/libpk/Cargo.toml +++ b/crates/libpk/Cargo.toml @@ -8,6 +8,7 @@ anyhow = { workspace = true } fred = { workspace = true } lazy_static = { workspace = true } metrics = { workspace = true } +pk_macros = { path = "../macros" } sentry = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/libpk/src/lib.rs b/crates/libpk/src/lib.rs index 8de69d92..55031bf3 100644 --- a/crates/libpk/src/lib.rs +++ b/crates/libpk/src/lib.rs @@ -14,7 +14,9 @@ pub mod state; pub mod _config; pub use crate::_config::CONFIG as config; -// functions in this file are only used by the main function below +// functions in this file are only used by the main function in macros/entrypoint.rs + +pub use pk_macros::main; pub fn init_logging(component: &str) { let sentry_layer = @@ -68,28 +70,3 @@ pub fn init_sentry() -> sentry::ClientInitGuard { ..Default::default() }) } - -#[macro_export] -macro_rules! main { - ($component:expr) => { - fn main() -> anyhow::Result<()> { - let _sentry_guard = libpk::init_sentry(); - // we might also be able to use env!("CARGO_CRATE_NAME") here - libpk::init_logging($component); - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap() - .block_on(async { - if let Err(error) = libpk::init_metrics() { - tracing::error!(?error, "failed to init metrics collector"); - }; - tracing::info!("hello world"); - if let Err(error) = real_main().await { - tracing::error!(?error, "failed to run service"); - }; - }); - Ok(()) - } - }; -} diff --git a/crates/model_macros/Cargo.toml b/crates/macros/Cargo.toml similarity index 85% rename from crates/model_macros/Cargo.toml rename to crates/macros/Cargo.toml index d2d0e009..8090798f 100644 --- a/crates/model_macros/Cargo.toml +++ b/crates/macros/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "model_macros" +name = "pk_macros" version = "0.1.0" edition = "2021" diff --git a/crates/macros/src/entrypoint.rs b/crates/macros/src/entrypoint.rs new file mode 100644 index 00000000..1e10012a --- /dev/null +++ b/crates/macros/src/entrypoint.rs @@ -0,0 +1,41 @@ +use proc_macro::{Delimiter, TokenTree}; +use quote::quote; + +pub fn macro_impl( + _args: proc_macro::TokenStream, + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + // yes, this ignores everything except the codeblock + // it's fine. + let body = match input.into_iter().last().expect("empty") { + TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => group.stream(), + _ => panic!("invalid function"), + }; + + let body = proc_macro2::TokenStream::from(body); + + return quote! { + fn main() { + let _sentry_guard = libpk::init_sentry(); + libpk::init_logging(env!("CARGO_CRATE_NAME")); + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + if let Err(error) = libpk::init_metrics() { + tracing::error!(?error, "failed to init metrics collector"); + }; + + tracing::info!("hello world"); + + let result: anyhow::Result<()> = async { #body }.await; + + if let Err(error) = result { + tracing::error!(?error, "failed to run service"); + }; + }); + } + } + .into(); +} diff --git a/crates/macros/src/lib.rs b/crates/macros/src/lib.rs new file mode 100644 index 00000000..db5a55b7 --- /dev/null +++ b/crates/macros/src/lib.rs @@ -0,0 +1,14 @@ +use proc_macro::TokenStream; + +mod entrypoint; +mod model; + +#[proc_macro_attribute] +pub fn main(args: TokenStream, input: TokenStream) -> TokenStream { + entrypoint::macro_impl(args, input) +} + +#[proc_macro_attribute] +pub fn pk_model(args: TokenStream, input: TokenStream) -> TokenStream { + model::macro_impl(args, input) +} diff --git a/crates/model_macros/src/lib.rs b/crates/macros/src/model.rs similarity index 99% rename from crates/model_macros/src/lib.rs rename to crates/macros/src/model.rs index 77f286e2..bfd219a7 100644 --- a/crates/model_macros/src/lib.rs +++ b/crates/macros/src/model.rs @@ -84,8 +84,7 @@ fn parse_field(field: syn::Field) -> ModelField { f } -#[proc_macro_attribute] -pub fn pk_model( +pub fn macro_impl( _args: proc_macro::TokenStream, input: proc_macro::TokenStream, ) -> proc_macro::TokenStream { diff --git a/crates/models/Cargo.toml b/crates/models/Cargo.toml index da90d79a..0fbc358c 100644 --- a/crates/models/Cargo.toml +++ b/crates/models/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] chrono = { workspace = true, features = ["serde"] } -model_macros = { path = "../model_macros" } +pk_macros = { path = "../macros" } sea-query = "0.32.1" serde = { workspace = true } serde_json = { workspace = true, features = ["preserve_order"] } diff --git a/crates/models/src/system.rs b/crates/models/src/system.rs index d59d5957..42b61fe4 100644 --- a/crates/models/src/system.rs +++ b/crates/models/src/system.rs @@ -1,6 +1,6 @@ use std::error::Error; -use model_macros::pk_model; +use pk_macros::pk_model; use chrono::NaiveDateTime; use sqlx::{postgres::PgTypeInfo, Database, Decode, Postgres, Type}; diff --git a/crates/models/src/system_config.rs b/crates/models/src/system_config.rs index d6b58a58..04b1995b 100644 --- a/crates/models/src/system_config.rs +++ b/crates/models/src/system_config.rs @@ -1,4 +1,4 @@ -use model_macros::pk_model; +use pk_macros::pk_model; use sqlx::{postgres::PgTypeInfo, Database, Decode, Postgres, Type}; use std::error::Error; diff --git a/crates/scheduled_tasks/src/main.rs b/crates/scheduled_tasks/src/main.rs index 56d58c7c..c689a99e 100644 --- a/crates/scheduled_tasks/src/main.rs +++ b/crates/scheduled_tasks/src/main.rs @@ -20,8 +20,8 @@ pub struct AppCtx { pub discord: Arc, } -libpk::main!("scheduled_tasks"); -async fn real_main() -> anyhow::Result<()> { +#[libpk::main] +async fn main() -> anyhow::Result<()> { let mut client_builder = twilight_http::Client::builder().token( libpk::config .discord From 50900ee640de37cb2ee299badfef18d8604eeef5 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 17 May 2025 18:57:48 +0000 Subject: [PATCH 38/80] chore: bump axum version --- Cargo.lock | 39 +++++++++-------- Cargo.toml | 2 +- crates/api/src/main.rs | 74 ++++++++++++++++----------------- crates/gateway/src/cache_api.rs | 24 +++++------ 4 files changed, 72 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2228d94b..6b44de61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,7 +81,7 @@ name = "api" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "fred", "hyper 1.6.0", "hyper-util", @@ -173,7 +173,7 @@ name = "avatars" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "data-encoding", "form_urlencoded", "futures", @@ -256,7 +256,7 @@ dependencies = [ "http-body 0.4.6", "hyper 0.14.32", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -275,13 +275,13 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.9" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ - "async-trait", - "axum-core 0.4.5", + "axum-core 0.5.2", "bytes", + "form_urlencoded", "futures-util", "http 1.3.1", "http-body 1.0.1", @@ -289,7 +289,7 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -326,13 +326,12 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.5" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ - "async-trait", "bytes", - "futures-util", + "futures-core", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -859,7 +858,7 @@ name = "dispatch" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "hickory-client", "libpk", "reqwest 0.12.15", @@ -1211,7 +1210,7 @@ name = "gateway" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "bytes", "chrono", "fred", @@ -1237,7 +1236,7 @@ name = "gdpr_worker" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.9", + "axum 0.8.4", "futures", "libpk", "sqlx", @@ -1992,7 +1991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -2104,6 +2103,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "maybe-async" version = "0.2.10" @@ -3115,7 +3120,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1641ddad..5f1272bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ resolver = "2" [workspace.dependencies] anyhow = "1" -axum = "0.7.5" +axum = "0.8.4" axum-macros = "0.4.1" bytes = "1.6.0" chrono = "0.4" diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 590064eb..083bc89c 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -59,52 +59,52 @@ async fn rproxy( fn router(ctx: ApiContext) -> Router { // processed upside down (???) so we have to put middleware at the end Router::new() - .route("/v2/systems/:system_id", get(rproxy)) - .route("/v2/systems/:system_id", patch(rproxy)) - .route("/v2/systems/:system_id/settings", get(rproxy)) - .route("/v2/systems/:system_id/settings", patch(rproxy)) + .route("/v2/systems/{system_id}", get(rproxy)) + .route("/v2/systems/{system_id}", patch(rproxy)) + .route("/v2/systems/{system_id}/settings", get(rproxy)) + .route("/v2/systems/{system_id}/settings", patch(rproxy)) - .route("/v2/systems/:system_id/members", get(rproxy)) + .route("/v2/systems/{system_id}/members", get(rproxy)) .route("/v2/members", post(rproxy)) - .route("/v2/members/:member_id", get(rproxy)) - .route("/v2/members/:member_id", patch(rproxy)) - .route("/v2/members/:member_id", delete(rproxy)) + .route("/v2/members/{member_id}", get(rproxy)) + .route("/v2/members/{member_id}", patch(rproxy)) + .route("/v2/members/{member_id}", delete(rproxy)) - .route("/v2/systems/:system_id/groups", get(rproxy)) + .route("/v2/systems/{system_id}/groups", get(rproxy)) .route("/v2/groups", post(rproxy)) - .route("/v2/groups/:group_id", get(rproxy)) - .route("/v2/groups/:group_id", patch(rproxy)) - .route("/v2/groups/:group_id", delete(rproxy)) + .route("/v2/groups/{group_id}", get(rproxy)) + .route("/v2/groups/{group_id}", patch(rproxy)) + .route("/v2/groups/{group_id}", delete(rproxy)) - .route("/v2/groups/:group_id/members", get(rproxy)) - .route("/v2/groups/:group_id/members/add", post(rproxy)) - .route("/v2/groups/:group_id/members/remove", post(rproxy)) - .route("/v2/groups/:group_id/members/overwrite", post(rproxy)) + .route("/v2/groups/{group_id}/members", get(rproxy)) + .route("/v2/groups/{group_id}/members/add", post(rproxy)) + .route("/v2/groups/{group_id}/members/remove", post(rproxy)) + .route("/v2/groups/{group_id}/members/overwrite", post(rproxy)) - .route("/v2/members/:member_id/groups", get(rproxy)) - .route("/v2/members/:member_id/groups/add", post(rproxy)) - .route("/v2/members/:member_id/groups/remove", post(rproxy)) - .route("/v2/members/:member_id/groups/overwrite", post(rproxy)) + .route("/v2/members/{member_id}/groups", get(rproxy)) + .route("/v2/members/{member_id}/groups/add", post(rproxy)) + .route("/v2/members/{member_id}/groups/remove", post(rproxy)) + .route("/v2/members/{member_id}/groups/overwrite", post(rproxy)) - .route("/v2/systems/:system_id/switches", get(rproxy)) - .route("/v2/systems/:system_id/switches", post(rproxy)) - .route("/v2/systems/:system_id/fronters", get(rproxy)) + .route("/v2/systems/{system_id}/switches", get(rproxy)) + .route("/v2/systems/{system_id}/switches", post(rproxy)) + .route("/v2/systems/{system_id}/fronters", get(rproxy)) - .route("/v2/systems/:system_id/switches/:switch_id", get(rproxy)) - .route("/v2/systems/:system_id/switches/:switch_id", patch(rproxy)) - .route("/v2/systems/:system_id/switches/:switch_id/members", patch(rproxy)) - .route("/v2/systems/:system_id/switches/:switch_id", delete(rproxy)) + .route("/v2/systems/{system_id}/switches/{switch_id}", get(rproxy)) + .route("/v2/systems/{system_id}/switches/{switch_id}", patch(rproxy)) + .route("/v2/systems/{system_id}/switches/{switch_id}/members", patch(rproxy)) + .route("/v2/systems/{system_id}/switches/{switch_id}", delete(rproxy)) - .route("/v2/systems/:system_id/guilds/:guild_id", get(rproxy)) - .route("/v2/systems/:system_id/guilds/:guild_id", patch(rproxy)) + .route("/v2/systems/{system_id}/guilds/{guild_id}", get(rproxy)) + .route("/v2/systems/{system_id}/guilds/{guild_id}", patch(rproxy)) - .route("/v2/members/:member_id/guilds/:guild_id", get(rproxy)) - .route("/v2/members/:member_id/guilds/:guild_id", patch(rproxy)) + .route("/v2/members/{member_id}/guilds/{guild_id}", get(rproxy)) + .route("/v2/members/{member_id}/guilds/{guild_id}", patch(rproxy)) - .route("/v2/systems/:system_id/autoproxy", get(rproxy)) - .route("/v2/systems/:system_id/autoproxy", patch(rproxy)) + .route("/v2/systems/{system_id}/autoproxy", get(rproxy)) + .route("/v2/systems/{system_id}/autoproxy", patch(rproxy)) - .route("/v2/messages/:message_id", get(rproxy)) + .route("/v2/messages/{message_id}", get(rproxy)) .route("/private/bulk_privacy/member", post(rproxy)) .route("/private/bulk_privacy/group", post(rproxy)) @@ -113,9 +113,9 @@ fn router(ctx: ApiContext) -> Router { .route("/private/discord/shard_state", get(endpoints::private::discord_state)) .route("/private/stats", get(endpoints::private::meta)) - .route("/v2/systems/:system_id/oembed.json", get(rproxy)) - .route("/v2/members/:member_id/oembed.json", get(rproxy)) - .route("/v2/groups/:group_id/oembed.json", get(rproxy)) + .route("/v2/systems/{system_id}/oembed.json", get(rproxy)) + .route("/v2/members/{member_id}/oembed.json", get(rproxy)) + .route("/v2/groups/{group_id}/oembed.json", get(rproxy)) .layer(middleware::ratelimit::ratelimiter(middleware::ratelimit::do_request_ratelimited)) // this sucks .layer(axum::middleware::from_fn_with_state(ctx.clone(), middleware::authnz)) diff --git a/crates/gateway/src/cache_api.rs b/crates/gateway/src/cache_api.rs index 4eb23ef1..375ab86e 100644 --- a/crates/gateway/src/cache_api.rs +++ b/crates/gateway/src/cache_api.rs @@ -33,7 +33,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path(guild_id): Path| async move { match cache.guild(Id::new(guild_id)) { Some(guild) => status_code(StatusCode::FOUND, to_string(&guild).unwrap()), @@ -42,7 +42,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path(guild_id): Path| async move { match cache.0.member(Id::new(guild_id), libpk::config.discord.as_ref().expect("missing discord config").client_id) { Some(member) => status_code(StatusCode::FOUND, to_string(member.value()).unwrap()), @@ -51,7 +51,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path(guild_id): Path| async move { match cache.guild_permissions(Id::new(guild_id), libpk::config.discord.as_ref().expect("missing discord config").client_id).await { Ok(val) => { @@ -65,7 +65,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path((guild_id, user_id)): Path<(u64, u64)>| async move { match cache.guild_permissions(Id::new(guild_id), Id::new(user_id)).await { Ok(val) => status_code(StatusCode::FOUND, to_string(&val.bits()).unwrap()), @@ -78,7 +78,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path(guild_id): Path| async move { let channel_ids = match cache.0.guild_channels(Id::new(guild_id)) { Some(channels) => channels.to_owned(), @@ -104,7 +104,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path((guild_id, channel_id)): Path<(u64, u64)>| async move { if guild_id == 0 { return status_code(StatusCode::FOUND, to_string(&dm_channel(Id::new(channel_id))).unwrap()); @@ -116,7 +116,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path((guild_id, channel_id)): Path<(u64, u64)>| async move { if guild_id == 0 { return status_code(StatusCode::FOUND, to_string(&*DM_PERMISSIONS).unwrap()); @@ -131,11 +131,11 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path((_guild_id, channel_id)): Path<(u64, Id)>| async move { let lm = cache.get_last_message(channel_id).await; status_code(StatusCode::FOUND, to_string(&lm).unwrap()) @@ -143,7 +143,7 @@ pub async fn run_server(cache: Arc, runtime_config: Arc>, Path(guild_id): Path| async move { let role_ids = match cache.0.guild_roles(Id::new(guild_id)) { Some(roles) => roles.to_owned(), @@ -186,12 +186,12 @@ pub async fn run_server(cache: Arc, runtime_config: Arc, body: String| async move { + .route("/runtime_config/{key}", post(|Path(key): Path, body: String| async move { let runtime_config = runtime_config_for_post; runtime_config.set(key, body).await.expect("failed to update runtime config"); status_code(StatusCode::FOUND, to_string(&runtime_config.get_all().await).unwrap()) })) - .route("/runtime_config/:key", delete(|Path(key): Path| async move { + .route("/runtime_config/{key}", delete(|Path(key): Path| async move { let runtime_config = runtime_config_for_delete; runtime_config.delete(key).await.expect("failed to update runtime config"); status_code(StatusCode::FOUND, to_string(&runtime_config.get_all().await).unwrap()) From c56fd36023c78c48b76d833eced8174770693ede Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 17 May 2025 20:39:29 +0000 Subject: [PATCH 39/80] feat(api): improve auth middleware --- crates/api/src/auth.rs | 22 +++++++++ crates/api/src/main.rs | 21 +++++++- .../api/src/middleware/{authnz.rs => auth.rs} | 48 ++++--------------- crates/api/src/middleware/logger.rs | 40 +++++++--------- crates/api/src/middleware/mod.rs | 3 +- crates/api/src/middleware/ratelimit.rs | 28 ++++++----- 6 files changed, 87 insertions(+), 75 deletions(-) create mode 100644 crates/api/src/auth.rs rename crates/api/src/middleware/{authnz.rs => auth.rs} (53%) diff --git a/crates/api/src/auth.rs b/crates/api/src/auth.rs new file mode 100644 index 00000000..49752a4b --- /dev/null +++ b/crates/api/src/auth.rs @@ -0,0 +1,22 @@ +pub const INTERNAL_SYSTEMID_HEADER: &'static str = "x-pluralkit-systemid"; +pub const INTERNAL_APPID_HEADER: &'static str = "x-pluralkit-appid"; + +#[derive(Clone)] +pub struct AuthState { + system_id: Option, + app_id: Option, +} + +impl AuthState { + pub fn new(system_id: Option, app_id: Option) -> Self { + Self { system_id, app_id } + } + + pub fn system_id(&self) -> Option { + self.system_id + } + + pub fn app_id(&self) -> Option { + self.app_id + } +} diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 083bc89c..48666d6f 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -1,12 +1,13 @@ #![feature(let_chains)] +use auth::{AuthState, INTERNAL_APPID_HEADER, INTERNAL_SYSTEMID_HEADER}; use axum::{ body::Body, extract::{Request as ExtractRequest, State}, http::{Response, StatusCode, Uri}, response::IntoResponse, routing::{delete, get, patch, post}, - Router, + Extension, Router, }; use hyper_util::{ client::legacy::{connect::HttpConnector, Client}, @@ -14,6 +15,7 @@ use hyper_util::{ }; use tracing::{error, info}; +mod auth; mod endpoints; mod error; mod middleware; @@ -29,6 +31,7 @@ pub struct ApiContext { } async fn rproxy( + Extension(auth): Extension, State(ctx): State, mut req: ExtractRequest, ) -> Result, StatusCode> { @@ -43,6 +46,19 @@ async fn rproxy( *req.uri_mut() = Uri::try_from(uri).unwrap(); + let headers = req.headers_mut(); + + headers.remove(INTERNAL_SYSTEMID_HEADER); + headers.remove(INTERNAL_APPID_HEADER); + + if let Some(sid) = auth.system_id() { + headers.append(INTERNAL_SYSTEMID_HEADER, sid.into()); + } + + if let Some(aid) = auth.app_id() { + headers.append(INTERNAL_APPID_HEADER, aid.into()); + } + Ok(ctx .rproxy_client .request(req) @@ -118,11 +134,12 @@ fn router(ctx: ApiContext) -> Router { .route("/v2/groups/{group_id}/oembed.json", get(rproxy)) .layer(middleware::ratelimit::ratelimiter(middleware::ratelimit::do_request_ratelimited)) // this sucks - .layer(axum::middleware::from_fn_with_state(ctx.clone(), middleware::authnz)) .layer(axum::middleware::from_fn(middleware::ignore_invalid_routes)) .layer(axum::middleware::from_fn(middleware::cors)) .layer(axum::middleware::from_fn(middleware::logger)) + .layer(axum::middleware::from_fn_with_state(ctx.clone(), middleware::auth::auth)) + .layer(tower_http::catch_panic::CatchPanicLayer::custom(util::handle_panic)) .with_state(ctx) diff --git a/crates/api/src/middleware/authnz.rs b/crates/api/src/middleware/auth.rs similarity index 53% rename from crates/api/src/middleware/authnz.rs rename to crates/api/src/middleware/auth.rs index 47140767..08981c3a 100644 --- a/crates/api/src/middleware/authnz.rs +++ b/crates/api/src/middleware/auth.rs @@ -4,27 +4,19 @@ use axum::{ middleware::Next, response::Response, }; + use tracing::error; +use crate::auth::AuthState; use crate::{util::json_err, ApiContext}; -pub const INTERNAL_SYSTEMID_HEADER: &'static str = "x-pluralkit-systemid"; -pub const INTERNAL_APPID_HEADER: &'static str = "x-pluralkit-appid"; - -// todo: auth should pass down models in request context -// not numerical ids in headers - -pub async fn authnz(State(ctx): State, mut request: Request, next: Next) -> Response { - let headers = request.headers_mut(); - - headers.remove(INTERNAL_SYSTEMID_HEADER); - headers.remove(INTERNAL_APPID_HEADER); - +pub async fn auth(State(ctx): State, mut req: Request, next: Next) -> Response { let mut authed_system_id: Option = None; let mut authed_app_id: Option = None; // fetch user authorization - if let Some(system_auth_header) = headers + if let Some(system_auth_header) = req + .headers() .get("authorization") .map(|h| h.to_str().ok()) .flatten() @@ -45,7 +37,8 @@ pub async fn authnz(State(ctx): State, mut request: Request, next: N // fetch app authorization // todo: actually fetch it from db - if let Some(app_auth_header) = headers + if let Some(app_auth_header) = req + .headers() .get("x-pluralkit-app") .map(|h| h.to_str().ok()) .flatten() @@ -62,29 +55,8 @@ pub async fn authnz(State(ctx): State, mut request: Request, next: N authed_app_id = Some(1); } - // add headers for ratelimiter / dotnet-api - { - let headers = request.headers_mut(); - if let Some(sid) = authed_system_id { - headers.append(INTERNAL_SYSTEMID_HEADER, sid.into()); - } - if let Some(aid) = authed_app_id { - headers.append(INTERNAL_APPID_HEADER, aid.into()); - } - } + req.extensions_mut() + .insert(AuthState::new(authed_system_id, authed_app_id)); - let mut response = next.run(request).await; - - // add headers for logger module (ugh) - { - let headers = response.headers_mut(); - if let Some(sid) = authed_system_id { - headers.append(INTERNAL_SYSTEMID_HEADER, sid.into()); - } - if let Some(aid) = authed_app_id { - headers.append(INTERNAL_APPID_HEADER, aid.into()); - } - } - - response + next.run(req).await } diff --git a/crates/api/src/middleware/logger.rs b/crates/api/src/middleware/logger.rs index 8f239042..38e45e2c 100644 --- a/crates/api/src/middleware/logger.rs +++ b/crates/api/src/middleware/logger.rs @@ -4,10 +4,7 @@ use axum::{extract::MatchedPath, extract::Request, middleware::Next, response::R use metrics::{counter, histogram}; use tracing::{info, span, warn, Instrument, Level}; -use crate::{ - middleware::authnz::{INTERNAL_APPID_HEADER, INTERNAL_SYSTEMID_HEADER}, - util::header_or_unknown, -}; +use crate::{auth::AuthState, util::header_or_unknown}; // log any requests that take longer than 2 seconds // todo: change as necessary @@ -19,13 +16,18 @@ pub async fn logger(request: Request, next: Next) -> Response { let remote_ip = header_or_unknown(request.headers().get("X-PluralKit-Client-IP")); let user_agent = header_or_unknown(request.headers().get("User-Agent")); - let endpoint = request - .extensions() + let extensions = request.extensions().clone(); + + let endpoint = extensions .get::() .cloned() .map(|v| v.as_str().to_string()) .unwrap_or("unknown".to_string()); + let auth = extensions + .get::() + .expect("should always have AuthState"); + let uri = request.uri().clone(); let request_span = span!( @@ -38,24 +40,18 @@ pub async fn logger(request: Request, next: Next) -> Response { ); let start = Instant::now(); - let mut response = next.run(request).instrument(request_span).await; + let response = next.run(request).instrument(request_span).await; let elapsed = start.elapsed().as_millis(); - let (system_id, app_id) = { - let headers = response.headers_mut(); - ( - headers - .remove(INTERNAL_SYSTEMID_HEADER) - .map(|h| h.to_str().ok().map(|v| v.to_string())) - .flatten() - .unwrap_or("none".to_string()), - headers - .remove(INTERNAL_APPID_HEADER) - .map(|h| h.to_str().ok().map(|v| v.to_string())) - .flatten() - .unwrap_or("none".to_string()), - ) - }; + let system_id = auth + .system_id() + .map(|v| v.to_string()) + .unwrap_or("none".to_string()); + + let app_id = auth + .app_id() + .map(|v| v.to_string()) + .unwrap_or("none".to_string()); counter!( "pluralkit_api_requests", diff --git a/crates/api/src/middleware/mod.rs b/crates/api/src/middleware/mod.rs index dbdfba13..ad67be11 100644 --- a/crates/api/src/middleware/mod.rs +++ b/crates/api/src/middleware/mod.rs @@ -9,5 +9,4 @@ pub use ignore_invalid_routes::ignore_invalid_routes; pub mod ratelimit; -mod authnz; -pub use authnz::authnz; +pub mod auth; diff --git a/crates/api/src/middleware/ratelimit.rs b/crates/api/src/middleware/ratelimit.rs index 29cb205b..f4a63f7e 100644 --- a/crates/api/src/middleware/ratelimit.rs +++ b/crates/api/src/middleware/ratelimit.rs @@ -10,9 +10,10 @@ use fred::{clients::RedisPool, interfaces::ClientLike, prelude::LuaInterface, ut use metrics::counter; use tracing::{debug, error, info, warn}; -use crate::util::{header_or_unknown, json_err}; - -use super::authnz::{INTERNAL_APPID_HEADER, INTERNAL_SYSTEMID_HEADER}; +use crate::{ + auth::AuthState, + util::{header_or_unknown, json_err}, +}; const LUA_SCRIPT: &str = include_str!("ratelimit.lua"); @@ -105,23 +106,28 @@ pub async fn do_request_ratelimited( if let Some(redis) = redis { let headers = request.headers().clone(); let source_ip = header_or_unknown(headers.get("X-PluralKit-Client-IP")); - let authenticated_system_id = header_or_unknown(headers.get(INTERNAL_SYSTEMID_HEADER)); - let authenticated_app_id = header_or_unknown(headers.get(INTERNAL_APPID_HEADER)); - let endpoint = request - .extensions() + let extensions = request.extensions().clone(); + + let endpoint = extensions .get::() .cloned() .map(|v| v.as_str().to_string()) .unwrap_or("unknown".to_string()); + let auth = extensions + .get::() + .expect("should always have AuthState"); + // looks like this chooses the tokens/sec by app_id or endpoint // then chooses the key by system_id or source_ip // todo: key should probably be chosen by app_id when it's present // todo: make x-ratelimit-scope actually meaningful // hack: for now, we only have one "registered app", so we hardcode the app id - let rlimit = if authenticated_app_id == "1" { + let rlimit = if let Some(app_id) = auth.app_id() + && app_id == 1 + { RatelimitType::TempCustom } else if endpoint == "/v2/messages/:message_id" { RatelimitType::Message @@ -133,12 +139,12 @@ pub async fn do_request_ratelimited( let rl_key = format!( "{}:{}", - if authenticated_system_id != "unknown" + if let Some(system_id) = auth.system_id() && matches!(rlimit, RatelimitType::GenericUpdate) { - authenticated_system_id + system_id.to_string() } else { - source_ip + source_ip.to_string() }, rlimit.key() ); From 0fa0070d41c0bed76cb88f00e7c8c046bf73c2b6 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 18 May 2025 13:14:29 +0000 Subject: [PATCH 40/80] feat(dotnet): json stdout logging --- .gitmodules | 4 + PluralKit.API/PluralKit.API.csproj | 2 +- PluralKit.API/Startup.cs | 2 +- PluralKit.API/packages.lock.json | 105 ++++----- PluralKit.Bot/Init.cs | 2 +- PluralKit.Bot/packages.lock.json | 12 +- PluralKit.Core/Modules/LoggingModule.cs | 22 +- PluralKit.Core/PluralKit.Core.csproj | 7 +- PluralKit.Core/Utils/SerilogJsonFormatter.cs | 215 +++++++++++++++++++ PluralKit.Core/packages.lock.json | 19 +- PluralKit.Tests/packages.lock.json | 97 +++++---- Serilog | 1 + 12 files changed, 348 insertions(+), 140 deletions(-) create mode 100644 PluralKit.Core/Utils/SerilogJsonFormatter.cs create mode 160000 Serilog diff --git a/.gitmodules b/.gitmodules index e69de29b..96fb8310 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "Serilog"] + path = Serilog + url = https://github.com/pluralkit/serilog + branch = f5eb991cb4c4a0c1e2407de7504c543536786598 diff --git a/PluralKit.API/PluralKit.API.csproj b/PluralKit.API/PluralKit.API.csproj index e35518dd..b1d88098 100644 --- a/PluralKit.API/PluralKit.API.csproj +++ b/PluralKit.API/PluralKit.API.csproj @@ -32,6 +32,6 @@ - + diff --git a/PluralKit.API/Startup.cs b/PluralKit.API/Startup.cs index ecb91710..06af238a 100644 --- a/PluralKit.API/Startup.cs +++ b/PluralKit.API/Startup.cs @@ -35,7 +35,7 @@ public class Startup builder.RegisterInstance(InitUtils.BuildConfiguration(Environment.GetCommandLineArgs()).Build()) .As(); builder.RegisterModule(new ConfigModule("API")); - builder.RegisterModule(new LoggingModule("api", + builder.RegisterModule(new LoggingModule("dotnet-api", cfg: new LoggerConfiguration().Filter.ByExcluding( exc => exc.Exception is PKError || exc.Exception.IsUserError() ))); diff --git a/PluralKit.API/packages.lock.json b/PluralKit.API/packages.lock.json index adc4279b..0107414c 100644 --- a/PluralKit.API/packages.lock.json +++ b/PluralKit.API/packages.lock.json @@ -36,17 +36,20 @@ }, "Serilog.AspNetCore": { "type": "Direct", - "requested": "[9.0.0, )", - "resolved": "9.0.0", - "contentHash": "JslDajPlBsn3Pww1554flJFTqROvK9zz9jONNQgn0D8Lx2Trw8L0A8/n6zEQK1DAZWXrJwiVLw8cnTR3YFuYsg==", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "FAjtKPZ4IzqFQBqZKPv6evcXK/F0ls7RoXI/62Pnx2igkDZ6nZ/jn/C/FxVATqQbEQvtqP+KViWYIe4NZIHa2w==", "dependencies": { - "Serilog": "4.2.0", - "Serilog.Extensions.Hosting": "9.0.0", - "Serilog.Formatting.Compact": "3.0.0", - "Serilog.Settings.Configuration": "9.0.0", - "Serilog.Sinks.Console": "6.0.0", - "Serilog.Sinks.Debug": "3.0.0", - "Serilog.Sinks.File": "6.0.0" + "Microsoft.Extensions.DependencyInjection": "8.0.0", + "Microsoft.Extensions.Logging": "8.0.0", + "Serilog": "3.1.1", + "Serilog.Extensions.Hosting": "8.0.0", + "Serilog.Extensions.Logging": "8.0.0", + "Serilog.Formatting.Compact": "2.0.0", + "Serilog.Settings.Configuration": "8.0.0", + "Serilog.Sinks.Console": "5.0.0", + "Serilog.Sinks.Debug": "2.0.0", + "Serilog.Sinks.File": "5.0.0" } }, "App.Metrics": { @@ -296,21 +299,21 @@ }, "Microsoft.Extensions.DependencyModel": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "saxr2XzwgDU77LaQfYFXmddEDRUKHF4DaGMZkNB3qjdVSZlax3//dGJagJkKrGMIPNZs2jVFXITyCCR6UHJNdA==", + "resolved": "8.0.0", + "contentHash": "NSmDw3K0ozNDgShSIpsZcbFIzBX4w28nDag+TfaQujkXGazBm+lid5onlWoCBy4VsLxqnnKjEBbGSJVWJMf43g==", "dependencies": { - "System.Text.Encodings.Web": "9.0.0", - "System.Text.Json": "9.0.0" + "System.Text.Encodings.Web": "8.0.0", + "System.Text.Json": "8.0.0" } }, "Microsoft.Extensions.Diagnostics.Abstractions": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "1K8P7XzuzX8W8pmXcZjcrqS6x5eSSdvhQohmcpgiQNY/HlDAlnrhR9dvlURfFz428A+RTCJpUyB+aKTA6AgVcQ==", + "resolved": "8.0.0", + "contentHash": "JHYCQG7HmugNYUhOl368g+NMxYE/N/AiclCYRNlgCY9eVyiBkOHMwK4x60RYMxv9EL3+rmj1mqHvdCiPpC+D4Q==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "Microsoft.Extensions.Options": "9.0.0", - "System.Diagnostics.DiagnosticSource": "9.0.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Options": "8.0.0", + "System.Diagnostics.DiagnosticSource": "8.0.0" } }, "Microsoft.Extensions.FileProviders.Abstractions": { @@ -338,14 +341,14 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "yUKJgu81ExjvqbNWqZKshBbLntZMbMVz/P7Way2SBx7bMqA08Mfdc9O7hWDKAiSp+zPUGT6LKcSCQIPeDK+CCw==", + "resolved": "8.0.0", + "contentHash": "AG7HWwVRdCHlaA++1oKDxLsXIBxmDpMPb3VoyOoAghEWnkUvEAdYQUwnV4jJbAaa/nMYNiEh5ByoLauZBEiovg==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "9.0.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "Microsoft.Extensions.Diagnostics.Abstractions": "9.0.0", - "Microsoft.Extensions.FileProviders.Abstractions": "9.0.0", - "Microsoft.Extensions.Logging.Abstractions": "9.0.0" + "Microsoft.Extensions.Configuration.Abstractions": "8.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Diagnostics.Abstractions": "8.0.0", + "Microsoft.Extensions.FileProviders.Abstractions": "8.0.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0" } }, "Microsoft.Extensions.Logging": { @@ -461,30 +464,25 @@ "System.IO.Pipelines": "5.0.1" } }, - "Serilog": { - "type": "Transitive", - "resolved": "4.2.0", - "contentHash": "gmoWVOvKgbME8TYR+gwMf7osROiWAURterc6Rt2dQyX7wtjZYpqFiA/pY6ztjGQKKV62GGCyOcmtP1UKMHgSmA==" - }, "Serilog.Extensions.Hosting": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "u2TRxuxbjvTAldQn7uaAwePkWxTHIqlgjelekBtilAGL5sYyF3+65NWctN4UrwwGLsDC7c3Vz3HnOlu+PcoxXg==", + "resolved": "8.0.0", + "contentHash": "db0OcbWeSCvYQkHWu6n0v40N4kKaTAXNjlM3BKvcbwvNzYphQFcBR+36eQ/7hMMwOkJvAyLC2a9/jNdUL5NjtQ==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "Microsoft.Extensions.Hosting.Abstractions": "9.0.0", - "Microsoft.Extensions.Logging.Abstractions": "9.0.0", - "Serilog": "4.2.0", - "Serilog.Extensions.Logging": "9.0.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Hosting.Abstractions": "8.0.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0", + "Serilog": "3.1.1", + "Serilog.Extensions.Logging": "8.0.0" } }, "Serilog.Extensions.Logging": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "NwSSYqPJeKNzl5AuXVHpGbr6PkZJFlNa14CdIebVjK3k/76kYj/mz5kiTRNVSsSaxM8kAIa1kpy/qyT9E4npRQ==", + "resolved": "8.0.0", + "contentHash": "YEAMWu1UnWgf1c1KP85l1SgXGfiVo0Rz6x08pCiPOIBt2Qe18tcZLvdBUuV5o1QHvrs8FAry9wTIhgBRtjIlEg==", "dependencies": { - "Microsoft.Extensions.Logging": "9.0.0", - "Serilog": "4.2.0" + "Microsoft.Extensions.Logging": "8.0.0", + "Serilog": "3.1.1" } }, "Serilog.Formatting.Compact": { @@ -514,12 +512,12 @@ }, "Serilog.Settings.Configuration": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "4/Et4Cqwa+F88l5SeFeNZ4c4Z6dEAIKbu3MaQb2Zz9F/g27T5a3wvfMcmCOaAiACjfUb4A6wrlTVfyYUZk3RRQ==", + "resolved": "8.0.0", + "contentHash": "nR0iL5HwKj5v6ULo3/zpP8NMcq9E2pxYA6XKTSWCbugVs4YqPyvaqaKOY+OMpPivKp7zMEpax2UKHnDodbRB0Q==", "dependencies": { - "Microsoft.Extensions.Configuration.Binder": "9.0.0", - "Microsoft.Extensions.DependencyModel": "9.0.0", - "Serilog": "4.2.0" + "Microsoft.Extensions.Configuration.Binder": "8.0.0", + "Microsoft.Extensions.DependencyModel": "8.0.0", + "Serilog": "3.1.1" } }, "Serilog.Sinks.Async": { @@ -540,10 +538,10 @@ }, "Serilog.Sinks.Debug": { "type": "Transitive", - "resolved": "3.0.0", - "contentHash": "4BzXcdrgRX7wde9PmHuYd9U6YqycCC28hhpKonK7hx0wb19eiuRj16fPcPSVp0o/Y1ipJuNLYQ00R3q2Zs8FDA==", + "resolved": "2.0.0", + "contentHash": "Y6g3OBJ4JzTyyw16fDqtFcQ41qQAydnEvEqmXjhwhgjsnG/FaJ8GUqF5ldsC/bVkK8KYmqrPhDO+tm4dF6xx4A==", "dependencies": { - "Serilog": "4.0.0" + "Serilog": "2.10.0" } }, "Serilog.Sinks.Elasticsearch": { @@ -837,8 +835,8 @@ "NodaTime.Serialization.JsonNet": "[3.1.0, )", "Npgsql": "[9.0.2, )", "Npgsql.NodaTime": "[9.0.2, )", - "Serilog": "[4.2.0, )", - "Serilog.Extensions.Logging": "[9.0.0, )", + "Serilog": "[4.1.0, )", + "Serilog.Extensions.Logging": "[8.0.0, )", "Serilog.Formatting.Compact": "[3.0.0, )", "Serilog.NodaTime": "[3.0.0, )", "Serilog.Sinks.Async": "[2.1.0, )", @@ -852,6 +850,9 @@ "System.Interactive.Async": "[6.0.1, )", "ipnetwork2": "[3.0.667, )" } + }, + "serilog": { + "type": "Project" } } } diff --git a/PluralKit.Bot/Init.cs b/PluralKit.Bot/Init.cs index c3046aa9..bf48d6c4 100644 --- a/PluralKit.Bot/Init.cs +++ b/PluralKit.Bot/Init.cs @@ -158,7 +158,7 @@ public class Init var builder = new ContainerBuilder(); builder.RegisterInstance(config); builder.RegisterModule(new ConfigModule("Bot")); - builder.RegisterModule(new LoggingModule("bot")); + builder.RegisterModule(new LoggingModule("dotnet-bot")); builder.RegisterModule(new MetricsModule()); builder.RegisterModule(); builder.RegisterModule(); diff --git a/PluralKit.Bot/packages.lock.json b/PluralKit.Bot/packages.lock.json index 79da3d52..3d718fc6 100644 --- a/PluralKit.Bot/packages.lock.json +++ b/PluralKit.Bot/packages.lock.json @@ -423,11 +423,11 @@ }, "Serilog.Extensions.Logging": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "NwSSYqPJeKNzl5AuXVHpGbr6PkZJFlNa14CdIebVjK3k/76kYj/mz5kiTRNVSsSaxM8kAIa1kpy/qyT9E4npRQ==", + "resolved": "8.0.0", + "contentHash": "YEAMWu1UnWgf1c1KP85l1SgXGfiVo0Rz6x08pCiPOIBt2Qe18tcZLvdBUuV5o1QHvrs8FAry9wTIhgBRtjIlEg==", "dependencies": { - "Microsoft.Extensions.Logging": "9.0.0", - "Serilog": "4.2.0" + "Microsoft.Extensions.Logging": "8.0.0", + "Serilog": "3.1.1" } }, "Serilog.Formatting.Compact": { @@ -796,8 +796,8 @@ "NodaTime.Serialization.JsonNet": "[3.1.0, )", "Npgsql": "[9.0.2, )", "Npgsql.NodaTime": "[9.0.2, )", - "Serilog": "[4.2.0, )", - "Serilog.Extensions.Logging": "[9.0.0, )", + "Serilog": "[4.1.0, )", + "Serilog.Extensions.Logging": "[8.0.0, )", "Serilog.Formatting.Compact": "[3.0.0, )", "Serilog.NodaTime": "[3.0.0, )", "Serilog.Sinks.Async": "[2.1.0, )", diff --git a/PluralKit.Core/Modules/LoggingModule.cs b/PluralKit.Core/Modules/LoggingModule.cs index f02bfe69..4be07b50 100644 --- a/PluralKit.Core/Modules/LoggingModule.cs +++ b/PluralKit.Core/Modules/LoggingModule.cs @@ -10,8 +10,7 @@ using NodaTime; using Serilog; using Serilog.Events; -using Serilog.Formatting.Compact; -using Serilog.Sinks.Seq; +using Serilog.Formatting.Json; using Serilog.Sinks.SystemConsole.Themes; using ILogger = Serilog.ILogger; @@ -50,14 +49,9 @@ public class LoggingModule: Module private ILogger InitLogger(CoreConfig config) { - var consoleTemplate = "[{Timestamp:HH:mm:ss.fff}] {Level:u3} {Message:lj}{NewLine}{Exception}"; - var outputTemplate = "[{Timestamp:yyyy-MM-dd HH:mm:ss.ffffff}] {Level:u3} {Message:lj}{NewLine}{Exception}"; - var logCfg = _cfg .Enrich.FromLogContext() - .Enrich.WithProperty("GitCommitHash", BuildInfoService.FullVersion) .ConfigureForNodaTime(DateTimeZoneProviders.Tzdb) - .Enrich.WithProperty("Component", _component) .MinimumLevel.Is(config.ConsoleLogLevel) // Don't want App.Metrics/D#+ spam @@ -75,9 +69,8 @@ public class LoggingModule: Module .Destructure.With() .WriteTo.Async(a => a.Console( - theme: AnsiConsoleTheme.Code, - outputTemplate: consoleTemplate, - restrictedToMinimumLevel: config.ConsoleLogLevel)); + new CustomJsonFormatter(_component), + config.ConsoleLogLevel)); if (config.ElasticUrl != null) { @@ -89,15 +82,6 @@ public class LoggingModule: Module ); } - if (config.SeqLogUrl != null) - { - logCfg.WriteTo.Seq( - config.SeqLogUrl, - restrictedToMinimumLevel: LogEventLevel.Verbose - ); - } - - _fn.Invoke(logCfg); return Log.Logger = logCfg.CreateLogger(); } diff --git a/PluralKit.Core/PluralKit.Core.csproj b/PluralKit.Core/PluralKit.Core.csproj index e4312988..a13bcefb 100644 --- a/PluralKit.Core/PluralKit.Core.csproj +++ b/PluralKit.Core/PluralKit.Core.csproj @@ -15,6 +15,10 @@ true + + + + @@ -37,8 +41,7 @@ - - + diff --git a/PluralKit.Core/Utils/SerilogJsonFormatter.cs b/PluralKit.Core/Utils/SerilogJsonFormatter.cs new file mode 100644 index 00000000..8bcc5731 --- /dev/null +++ b/PluralKit.Core/Utils/SerilogJsonFormatter.cs @@ -0,0 +1,215 @@ +using System.Runtime.CompilerServices; + +using Serilog.Events; +using Serilog.Formatting; +using Serilog.Formatting.Json; +using Serilog.Parsing; +using Serilog.Rendering; + +// Customized Serilog JSON output for PluralKit + +// Copyright 2013-2015 Serilog Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace PluralKit.Core; + +static class Guard +{ + public static T AgainstNull( + T? argument, + [CallerArgumentExpression("argument")] string? paramName = null) + where T : class + { + if (argument is null) + { + throw new ArgumentNullException(paramName); + } + + return argument; + } +} + +/// +/// Formats log events in a simple JSON structure. Instances of this class +/// are safe for concurrent access by multiple threads. +/// +/// New code should prefer formatters from Serilog.Formatting.Compact, or ExpressionTemplate from +/// Serilog.Expressions. +public sealed class CustomJsonFormatter: ITextFormatter +{ + readonly JsonValueFormatter _jsonValueFormatter = new(); + readonly string _component; + + /// + /// Construct a . + /// + /// A string that will be written after each log event is formatted. + /// If null, will be used. + /// If , the message will be rendered and written to the output as a + /// property named RenderedMessage. + /// Supplies culture-specific formatting information, or null. + public CustomJsonFormatter(string component) + { + _component = component; + } + + private string CustomLevelString(LogEventLevel level) + { + switch (level) + { + case LogEventLevel.Verbose: + return "TRACE"; + case LogEventLevel.Debug: + return "DEBUG"; + case LogEventLevel.Information: + return "INFO"; + case LogEventLevel.Warning: + return "WARN"; + case LogEventLevel.Error: + return "ERROR"; + case LogEventLevel.Fatal: + return "FATAL"; + }; + + return "UNKNOWN"; + } + + /// + /// Format the log event into the output. + /// + /// The event to format. + /// The output. + /// When is null + /// When is null + public void Format(LogEvent logEvent, TextWriter output) + { + Guard.AgainstNull(logEvent); + Guard.AgainstNull(output); + + output.Write("{\"component\":\""); + output.Write(_component); + output.Write("\",\"timestamp\":\""); + output.Write(logEvent.Timestamp.ToString("O").Replace("+00:00", "Z")); + output.Write("\",\"level\":\""); + output.Write(CustomLevelString(logEvent.Level)); + + output.Write("\",\"message\":"); + var message = logEvent.MessageTemplate.Render(logEvent.Properties); + JsonValueFormatter.WriteQuotedJsonString(message, output); + + if (logEvent.TraceId != null) + { + output.Write(",\"TraceId\":"); + JsonValueFormatter.WriteQuotedJsonString(logEvent.TraceId.ToString()!, output); + } + + if (logEvent.SpanId != null) + { + output.Write(",\"SpanId\":"); + JsonValueFormatter.WriteQuotedJsonString(logEvent.SpanId.ToString()!, output); + } + + if (logEvent.Exception != null) + { + output.Write(",\"Exception\":"); + JsonValueFormatter.WriteQuotedJsonString(logEvent.Exception.ToString(), output); + } + + if (logEvent.Properties.Count != 0) + { + output.Write(",\"Properties\":{"); + + char? propertyDelimiter = null; + foreach (var property in logEvent.Properties) + { + if (propertyDelimiter != null) + output.Write(propertyDelimiter.Value); + else + propertyDelimiter = ','; + + JsonValueFormatter.WriteQuotedJsonString(property.Key, output); + output.Write(':'); + _jsonValueFormatter.Format(property.Value, output); + } + + output.Write('}'); + } + + var tokensWithFormat = logEvent.MessageTemplate.Tokens + .OfType() + .Where(pt => pt.Format != null) + .GroupBy(pt => pt.PropertyName) + .ToArray(); + + if (tokensWithFormat.Length != 0) + { + output.Write(",\"Renderings\":{"); + WriteRenderingsValues(tokensWithFormat, logEvent.Properties, output); + output.Write('}'); + } + + output.Write('}'); + output.Write("\n"); + } + + void WriteRenderingsValues(IEnumerable> tokensWithFormat, IReadOnlyDictionary properties, TextWriter output) + { + static void WriteNameValuePair(string name, string value, ref char? precedingDelimiter, TextWriter output) + { + if (precedingDelimiter != null) + output.Write(precedingDelimiter.Value); + + JsonValueFormatter.WriteQuotedJsonString(name, output); + output.Write(':'); + JsonValueFormatter.WriteQuotedJsonString(value, output); + precedingDelimiter = ','; + } + + char? propertyDelimiter = null; + foreach (var propertyFormats in tokensWithFormat) + { + if (propertyDelimiter != null) + output.Write(propertyDelimiter.Value); + else + propertyDelimiter = ','; + + output.Write('"'); + output.Write(propertyFormats.Key); + output.Write("\":["); + + char? formatDelimiter = null; + foreach (var format in propertyFormats) + { + if (formatDelimiter != null) + output.Write(formatDelimiter.Value); + + formatDelimiter = ','; + + output.Write('{'); + char? elementDelimiter = null; + + // Caller ensures that `tokensWithFormat` contains only property tokens that have non-null `Format`s. + WriteNameValuePair("Format", format.Format!, ref elementDelimiter, output); + + using var sw = ReusableStringWriter.GetOrCreate(); + MessageTemplateRenderer.RenderPropertyToken(format, properties, sw, null, isLiteral: true, isJson: false); + WriteNameValuePair("Rendering", sw.ToString(), ref elementDelimiter, output); + + output.Write('}'); + } + + output.Write(']'); + } + } +} \ No newline at end of file diff --git a/PluralKit.Core/packages.lock.json b/PluralKit.Core/packages.lock.json index a4768566..4f9ec5d0 100644 --- a/PluralKit.Core/packages.lock.json +++ b/PluralKit.Core/packages.lock.json @@ -198,20 +198,14 @@ "Npgsql": "9.0.2" } }, - "Serilog": { - "type": "Direct", - "requested": "[4.2.0, )", - "resolved": "4.2.0", - "contentHash": "gmoWVOvKgbME8TYR+gwMf7osROiWAURterc6Rt2dQyX7wtjZYpqFiA/pY6ztjGQKKV62GGCyOcmtP1UKMHgSmA==" - }, "Serilog.Extensions.Logging": { "type": "Direct", - "requested": "[9.0.0, )", - "resolved": "9.0.0", - "contentHash": "NwSSYqPJeKNzl5AuXVHpGbr6PkZJFlNa14CdIebVjK3k/76kYj/mz5kiTRNVSsSaxM8kAIa1kpy/qyT9E4npRQ==", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "YEAMWu1UnWgf1c1KP85l1SgXGfiVo0Rz6x08pCiPOIBt2Qe18tcZLvdBUuV5o1QHvrs8FAry9wTIhgBRtjIlEg==", "dependencies": { - "Microsoft.Extensions.Logging": "9.0.0", - "Serilog": "4.2.0" + "Microsoft.Extensions.Logging": "8.0.0", + "Serilog": "3.1.1" } }, "Serilog.Formatting.Compact": { @@ -722,6 +716,9 @@ "Microsoft.NETCore.Targets": "1.1.0", "System.Runtime": "4.3.0" } + }, + "serilog": { + "type": "Project" } } } diff --git a/PluralKit.Tests/packages.lock.json b/PluralKit.Tests/packages.lock.json index b9e49ba9..79926c2a 100644 --- a/PluralKit.Tests/packages.lock.json +++ b/PluralKit.Tests/packages.lock.json @@ -320,21 +320,21 @@ }, "Microsoft.Extensions.DependencyModel": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "saxr2XzwgDU77LaQfYFXmddEDRUKHF4DaGMZkNB3qjdVSZlax3//dGJagJkKrGMIPNZs2jVFXITyCCR6UHJNdA==", + "resolved": "8.0.0", + "contentHash": "NSmDw3K0ozNDgShSIpsZcbFIzBX4w28nDag+TfaQujkXGazBm+lid5onlWoCBy4VsLxqnnKjEBbGSJVWJMf43g==", "dependencies": { - "System.Text.Encodings.Web": "9.0.0", - "System.Text.Json": "9.0.0" + "System.Text.Encodings.Web": "8.0.0", + "System.Text.Json": "8.0.0" } }, "Microsoft.Extensions.Diagnostics.Abstractions": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "1K8P7XzuzX8W8pmXcZjcrqS6x5eSSdvhQohmcpgiQNY/HlDAlnrhR9dvlURfFz428A+RTCJpUyB+aKTA6AgVcQ==", + "resolved": "8.0.0", + "contentHash": "JHYCQG7HmugNYUhOl368g+NMxYE/N/AiclCYRNlgCY9eVyiBkOHMwK4x60RYMxv9EL3+rmj1mqHvdCiPpC+D4Q==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "Microsoft.Extensions.Options": "9.0.0", - "System.Diagnostics.DiagnosticSource": "9.0.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Options": "8.0.0", + "System.Diagnostics.DiagnosticSource": "8.0.0" } }, "Microsoft.Extensions.FileProviders.Abstractions": { @@ -362,14 +362,14 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "yUKJgu81ExjvqbNWqZKshBbLntZMbMVz/P7Way2SBx7bMqA08Mfdc9O7hWDKAiSp+zPUGT6LKcSCQIPeDK+CCw==", + "resolved": "8.0.0", + "contentHash": "AG7HWwVRdCHlaA++1oKDxLsXIBxmDpMPb3VoyOoAghEWnkUvEAdYQUwnV4jJbAaa/nMYNiEh5ByoLauZBEiovg==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "9.0.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "Microsoft.Extensions.Diagnostics.Abstractions": "9.0.0", - "Microsoft.Extensions.FileProviders.Abstractions": "9.0.0", - "Microsoft.Extensions.Logging.Abstractions": "9.0.0" + "Microsoft.Extensions.Configuration.Abstractions": "8.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Diagnostics.Abstractions": "8.0.0", + "Microsoft.Extensions.FileProviders.Abstractions": "8.0.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0" } }, "Microsoft.Extensions.Logging": { @@ -537,37 +537,40 @@ }, "Serilog.AspNetCore": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "JslDajPlBsn3Pww1554flJFTqROvK9zz9jONNQgn0D8Lx2Trw8L0A8/n6zEQK1DAZWXrJwiVLw8cnTR3YFuYsg==", + "resolved": "8.0.0", + "contentHash": "FAjtKPZ4IzqFQBqZKPv6evcXK/F0ls7RoXI/62Pnx2igkDZ6nZ/jn/C/FxVATqQbEQvtqP+KViWYIe4NZIHa2w==", "dependencies": { - "Serilog": "4.2.0", - "Serilog.Extensions.Hosting": "9.0.0", - "Serilog.Formatting.Compact": "3.0.0", - "Serilog.Settings.Configuration": "9.0.0", - "Serilog.Sinks.Console": "6.0.0", - "Serilog.Sinks.Debug": "3.0.0", - "Serilog.Sinks.File": "6.0.0" + "Microsoft.Extensions.DependencyInjection": "8.0.0", + "Microsoft.Extensions.Logging": "8.0.0", + "Serilog": "3.1.1", + "Serilog.Extensions.Hosting": "8.0.0", + "Serilog.Extensions.Logging": "8.0.0", + "Serilog.Formatting.Compact": "2.0.0", + "Serilog.Settings.Configuration": "8.0.0", + "Serilog.Sinks.Console": "5.0.0", + "Serilog.Sinks.Debug": "2.0.0", + "Serilog.Sinks.File": "5.0.0" } }, "Serilog.Extensions.Hosting": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "u2TRxuxbjvTAldQn7uaAwePkWxTHIqlgjelekBtilAGL5sYyF3+65NWctN4UrwwGLsDC7c3Vz3HnOlu+PcoxXg==", + "resolved": "8.0.0", + "contentHash": "db0OcbWeSCvYQkHWu6n0v40N4kKaTAXNjlM3BKvcbwvNzYphQFcBR+36eQ/7hMMwOkJvAyLC2a9/jNdUL5NjtQ==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "Microsoft.Extensions.Hosting.Abstractions": "9.0.0", - "Microsoft.Extensions.Logging.Abstractions": "9.0.0", - "Serilog": "4.2.0", - "Serilog.Extensions.Logging": "9.0.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Hosting.Abstractions": "8.0.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0", + "Serilog": "3.1.1", + "Serilog.Extensions.Logging": "8.0.0" } }, "Serilog.Extensions.Logging": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "NwSSYqPJeKNzl5AuXVHpGbr6PkZJFlNa14CdIebVjK3k/76kYj/mz5kiTRNVSsSaxM8kAIa1kpy/qyT9E4npRQ==", + "resolved": "8.0.0", + "contentHash": "YEAMWu1UnWgf1c1KP85l1SgXGfiVo0Rz6x08pCiPOIBt2Qe18tcZLvdBUuV5o1QHvrs8FAry9wTIhgBRtjIlEg==", "dependencies": { - "Microsoft.Extensions.Logging": "9.0.0", - "Serilog": "4.2.0" + "Microsoft.Extensions.Logging": "8.0.0", + "Serilog": "3.1.1" } }, "Serilog.Formatting.Compact": { @@ -597,12 +600,12 @@ }, "Serilog.Settings.Configuration": { "type": "Transitive", - "resolved": "9.0.0", - "contentHash": "4/Et4Cqwa+F88l5SeFeNZ4c4Z6dEAIKbu3MaQb2Zz9F/g27T5a3wvfMcmCOaAiACjfUb4A6wrlTVfyYUZk3RRQ==", + "resolved": "8.0.0", + "contentHash": "nR0iL5HwKj5v6ULo3/zpP8NMcq9E2pxYA6XKTSWCbugVs4YqPyvaqaKOY+OMpPivKp7zMEpax2UKHnDodbRB0Q==", "dependencies": { - "Microsoft.Extensions.Configuration.Binder": "9.0.0", - "Microsoft.Extensions.DependencyModel": "9.0.0", - "Serilog": "4.2.0" + "Microsoft.Extensions.Configuration.Binder": "8.0.0", + "Microsoft.Extensions.DependencyModel": "8.0.0", + "Serilog": "3.1.1" } }, "Serilog.Sinks.Async": { @@ -623,10 +626,10 @@ }, "Serilog.Sinks.Debug": { "type": "Transitive", - "resolved": "3.0.0", - "contentHash": "4BzXcdrgRX7wde9PmHuYd9U6YqycCC28hhpKonK7hx0wb19eiuRj16fPcPSVp0o/Y1ipJuNLYQ00R3q2Zs8FDA==", + "resolved": "2.0.0", + "contentHash": "Y6g3OBJ4JzTyyw16fDqtFcQ41qQAydnEvEqmXjhwhgjsnG/FaJ8GUqF5ldsC/bVkK8KYmqrPhDO+tm4dF6xx4A==", "dependencies": { - "Serilog": "4.0.0" + "Serilog": "2.10.0" } }, "Serilog.Sinks.Elasticsearch": { @@ -993,7 +996,7 @@ "Microsoft.AspNetCore.Mvc.Versioning.ApiExplorer": "[5.1.0, )", "PluralKit.Core": "[1.0.0, )", "Sentry": "[4.13.0, )", - "Serilog.AspNetCore": "[9.0.0, )" + "Serilog.AspNetCore": "[8.0.0, )" } }, "pluralkit.bot": { @@ -1029,8 +1032,8 @@ "NodaTime.Serialization.JsonNet": "[3.1.0, )", "Npgsql": "[9.0.2, )", "Npgsql.NodaTime": "[9.0.2, )", - "Serilog": "[4.2.0, )", - "Serilog.Extensions.Logging": "[9.0.0, )", + "Serilog": "[4.1.0, )", + "Serilog.Extensions.Logging": "[8.0.0, )", "Serilog.Formatting.Compact": "[3.0.0, )", "Serilog.NodaTime": "[3.0.0, )", "Serilog.Sinks.Async": "[2.1.0, )", diff --git a/Serilog b/Serilog new file mode 160000 index 00000000..f5eb991c --- /dev/null +++ b/Serilog @@ -0,0 +1 @@ +Subproject commit f5eb991cb4c4a0c1e2407de7504c543536786598 From e573e978da4ef7b9557f36c32b843b6dca051f9e Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 18 May 2025 13:19:20 +0000 Subject: [PATCH 41/80] fix .net docker build --- .dockerignore | 1 + .github/workflows/dotnet-docker.yml | 8 +++++++- .github/workflows/rust-docker.yml | 1 + Dockerfile => ci/Dockerfile.dotnet | 1 + docker-compose.yml | 4 +++- 5 files changed, 13 insertions(+), 2 deletions(-) rename Dockerfile => ci/Dockerfile.dotnet (97%) diff --git a/.dockerignore b/.dockerignore index c4f2c30e..ee5756d6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ # Include project code and build files !PluralKit.*/ !Myriad/ +!Serilog/ !.git !dashboard !crates/ diff --git a/.github/workflows/dotnet-docker.yml b/.github/workflows/dotnet-docker.yml index af20524a..9939320b 100644 --- a/.github/workflows/dotnet-docker.yml +++ b/.github/workflows/dotnet-docker.yml @@ -2,7 +2,9 @@ name: Build and push Docker image on: push: paths: - - '.github/workflows/docker.yml' + - '.dockerignore' + - '.github/workflows/dotnet-docker.yml' + - 'ci/Dockerfile.dotnet' - 'ci/dotnet-version.sh' - 'Myriad/**' - 'PluralKit.API/**' @@ -23,6 +25,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.CR_PAT }} - uses: actions/checkout@v2 + with: + submodules: true + - run: echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" | sed 's|/|-|g' >> $GITHUB_ENV - name: Extract Docker metadata @@ -41,6 +46,7 @@ jobs: with: # https://github.com/docker/build-push-action/issues/378 context: . + file: ci/Dockerfile.dotnet push: true tags: ${{ steps.meta.outputs.tags }} cache-from: type=registry,ref=ghcr.io/pluralkit/pluralkit:${{ env.BRANCH_NAME }} diff --git a/.github/workflows/rust-docker.yml b/.github/workflows/rust-docker.yml index 293a2106..a46adf67 100644 --- a/.github/workflows/rust-docker.yml +++ b/.github/workflows/rust-docker.yml @@ -3,6 +3,7 @@ on: push: paths: - 'crates/**' + - '.dockerignore' - '.github/workflows/rust.yml' - 'ci/Dockerfile.rust' - 'ci/rust-docker-target.sh' diff --git a/Dockerfile b/ci/Dockerfile.dotnet similarity index 97% rename from Dockerfile rename to ci/Dockerfile.dotnet index dedc085d..c10952b3 100644 --- a/Dockerfile +++ b/ci/Dockerfile.dotnet @@ -10,6 +10,7 @@ COPY PluralKit.Bot/PluralKit.Bot.csproj /app/PluralKit.Bot/ COPY PluralKit.Core/PluralKit.Core.csproj /app/PluralKit.Core/ COPY PluralKit.Tests/PluralKit.Tests.csproj /app/PluralKit.Tests/ COPY .git/ /app/.git +COPY Serilog/ /app/Serilog/ RUN dotnet restore PluralKit.sln # Copy the rest of the code and build diff --git a/docker-compose.yml b/docker-compose.yml index d58f4c5c..18ddaa03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,9 @@ version: "3" services: bot: - build: . + build: + context: . + dockerfile: ci/Dockerfile.dotnet command: ["bin/PluralKit.Bot.dll"] environment: - "PluralKit__Database=Host=db;Username=postgres;Password=postgres;Database=postgres" From eb9994b2155f3e98a104846d52456272ad955d87 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 18 May 2025 21:27:41 +0000 Subject: [PATCH 42/80] fix(gateway): set custom prefix in bot status --- crates/gateway/src/discord/gateway.rs | 9 ++++++++- crates/gateway/src/main.rs | 13 +++++++++++-- crates/libpk/src/_config.rs | 6 ++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index 1c0239f5..563a9b78 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -63,6 +63,13 @@ pub fn create_shards(redis: fred::clients::RedisPool) -> anyhow::Result anyhow::Result anyhow::Result<()> { } async fn scheduled_task(redis: RedisPool, senders: Vec<(ShardId, MessageSender)>) { + let prefix = libpk::config + .discord + .as_ref() + .expect("missing discord config") + .bot_prefix_for_gateway + .clone(); + + println!("{prefix}"); + loop { tokio::time::sleep(Duration::from_secs( (60 - chrono::offset::Utc::now().second()).into(), @@ -204,9 +213,9 @@ async fn scheduled_task(redis: RedisPool, senders: Vec<(ShardId, MessageSender)> op: twilight_model::gateway::OpCode::PresenceUpdate, d: discord::gateway::presence( if let Some(status) = status { - format!("pk;help | {}", status) + format!("{prefix}help | {status}") } else { - "pk;help".to_string() + format!("{prefix}help") } .as_str(), false, diff --git a/crates/libpk/src/_config.rs b/crates/libpk/src/_config.rs index 6a489c93..8358440b 100644 --- a/crates/libpk/src/_config.rs +++ b/crates/libpk/src/_config.rs @@ -12,10 +12,16 @@ pub struct ClusterSettings { pub total_nodes: u32, } +fn _default_bot_prefix() -> String { + "pk;".to_string() +} + #[derive(Deserialize, Debug)] pub struct DiscordConfig { pub client_id: Id, pub bot_token: String, + #[serde(default = "_default_bot_prefix")] + pub bot_prefix_for_gateway: String, pub client_secret: String, pub max_concurrency: u32, #[serde(default)] From 87558fb0cf7c5fc36e2c7364828e135368552015 Mon Sep 17 00:00:00 2001 From: rladenson Date: Mon, 19 May 2025 22:42:23 -0600 Subject: [PATCH 43/80] feat(docs): add .env entries for custom prefixes to dev docs --- dev-docs/dotnet.md | 51 +++++++++++++++++++++++----------------------- dev-docs/rust.md | 2 +- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/dev-docs/dotnet.md b/dev-docs/dotnet.md index 694bc1a8..b4f99276 100644 --- a/dev-docs/dotnet.md +++ b/dev-docs/dotnet.md @@ -12,29 +12,28 @@ The minimum configuration needed for the dotnet part of the bot to function incl ### Available Configuration Values: -| Name | Description | Rust Equivalent (if applicable) | -| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | -| **`PluralKit__Api__ClientId`** | the ID of the bot's user account, used for OAuth with Discord | *`pluralkit__discord__client_id`* | -| **`PluralKit__Api__ClientSecret`** | the client secret of the application, used for OAuth with Discord | *`pluralkit__discord__client_secret`* | -| **`PluralKit__Api__TrustAuth`** | boolean used to determine if the API should trust upstream to provide it the system id of the authenticated user | | -| **`PluralKit__Bot__AdminRole`** | Discord role ID used to determine if a user can use `pk;admin` commands | | -| **`PluralKit__Bot__AvatarServiceUrl`** | the URL of the avatar service | | -| **`PluralKit__Bot__ClientId`** | the ID of the bot's user account, used for calculating the bot's own permissions and for the link in `pk;invite`. | *`pluralkit__discord__client_id`* | -| **`PluralKit__Bot__Cluster__TotalShards`** | the total number of shards | *`pluralkit__discord__cluster__total_shards`* | -| **`PluralKit__Bot__DisableGateway`** | (deprecated) boolean used to enable or disable the inbuilt gateway functions, should be true if using Rust `gateway` | | -| **`PluralKit__Bot__DiscordBaseUrl`** | the base Discord API url used for HTTP API requests | *`pluralkit__discord__api_base_url`* | -| **`PluralKit__Bot__EventAwaiterTarget`** | the target bind address used to receive bot-instance specific events (such as interactive prompts/menus) from `gateway` over http -- value should generally be `source-addr`. | | -| **`PluralKit__Bot__HttpCacheUrl`** | the URL of the http cache to use, as of now, the `gateway` service | | -| **`PluralKit__Bot__HttpListenerAddr`** | the base bind address (use `allv4v6` instead of `::` if you want to also bind to `0.0.0.0`) | | -| **`PluralKit__Bot__Token`** | the Discord bot token to connect with | *`pluralkit__discord__bot_token`* | -| **`PluralKit__Database`** | the URI of the database to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) | *`pluralkit__db__data_db_uri`* (diff formatting) | -| **`PluralKit__MessagesDatabase`** | the URI of the database for message storage to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) | *`pluralkit__db__messages_db_uri`* (diff formatting) | -| **`PluralKit__RedisAddr`** | the `host:port` of a Redis database to connect to | *`pluralkit__db__data_redis_addr`* (diff formatting) | -| **`PluralKit__DispatchProxyToken`** | the token used to authenticate with the dispatch proxy service | | -| **`PluralKit__DispatchProxyUrl`** | the URL of the dispatch proxy service | | -| **`PluralKit__ConsoleLogLevel`** | the minimum [log level](https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.logging.loglevel?view=net-9.0-pp) used for logging | | -| **`PluralKit__InfluxUrl`** | the URL to an [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) server to report aggregate statistics to. An example of these stats can be seen on [the public stats page](https://stats.pluralkit.me). | | -| **`PluralKit__InfluxDb`** | the name of an InfluxDB database to report statistics to. If either this field or `PluralKit.InfluxUrl` are absent, InfluxDB reporting will be disabled. | | -| **`PluralKit__SentryUrl`** | the [Sentry](https://sentry.io/welcome/) client key/DSN to report runtime errors to. If absent, disables Sentry integration. | | - - +| Name | Description | Rust Equivalent (if applicable) | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------- | +| **`PluralKit__Api__ClientId`** | the ID of the bot's user account, used for OAuth with Discord | *`pluralkit__discord__client_id`* | +| **`PluralKit__Api__ClientSecret`** | the client secret of the application, used for OAuth with Discord | *`pluralkit__discord__client_secret`* | +| **`PluralKit__Api__TrustAuth`** | boolean used to determine if the API should trust upstream to provide it the system id of the authenticated user | | +| **`PluralKit__Bot__AdminRole`** | Discord role ID used to determine if a user can use `pk;admin` commands | | +| **`PluralKit__Bot__AvatarServiceUrl`** | the URL of the avatar service | | +| **`PluralKit__Bot__ClientId`** | the ID of the bot's user account, used for calculating the bot's own permissions and for the link in `pk;invite`. | *`pluralkit__discord__client_id`* | +| **`PluralKit__Bot__Cluster__TotalShards`** | the total number of shards | *`pluralkit__discord__cluster__total_shards`* | +| **`PluralKit__Bot__DisableGateway`** | (deprecated) boolean used to enable or disable the inbuilt gateway functions, should be true if using Rust `gateway` | | +| **`PluralKit__Bot__DiscordBaseUrl`** | the base Discord API url used for HTTP API requests | *`pluralkit__discord__api_base_url`* | +| **`PluralKit__Bot__EventAwaiterTarget`** | the target bind address used to receive bot-instance specific events (such as interactive prompts/menus) from `gateway` over http -- value should generally be `source-addr`. | | +| **`PluralKit__Bot__HttpCacheUrl`** | the URL of the http cache to use, as of now, the `gateway` service | | +| **`PluralKit__Bot__HttpListenerAddr`** | the base bind address (use `allv4v6` instead of `::` if you want to also bind to `0.0.0.0`) | | +| **`PluralKit__Bot__Token`** | the Discord bot token to connect with | *`pluralkit__discord__bot_token`* | +| **`PluralKit__Database`** | the URI of the database to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) | *`pluralkit__db__data_db_uri`* (diff formatting) | +| **`PluralKit__MessagesDatabase`** | the URI of the database for message storage to connect to (in [ADO.NET Npgsql format](https://www.connectionstrings.com/npgsql/)) | *`pluralkit__db__messages_db_uri`* (diff formatting) | +| **`PluralKit__RedisAddr`** | the `host:port` of a Redis database to connect to | *`pluralkit__db__data_redis_addr`* (diff formatting) | +| **`PluralKit__DispatchProxyToken`** | the token used to authenticate with the dispatch proxy service | | +| **`PluralKit__DispatchProxyUrl`** | the URL of the dispatch proxy service | | +| **`PluralKit__ConsoleLogLevel`** | the minimum [log level](https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.logging.loglevel?view=net-9.0-pp) used for logging | | +| **`PluralKit__InfluxUrl`** | the URL to an [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) server to report aggregate statistics to. An example of these stats can be seen on [the public stats page](https://stats.pluralkit.me). | | +| **`PluralKit__InfluxDb`** | the name of an InfluxDB database to report statistics to. If either this field or `PluralKit.InfluxUrl` are absent, InfluxDB reporting will be disabled. | | +| **`PluralKit__SentryUrl`** | the [Sentry](https://sentry.io/welcome/) client key/DSN to report runtime errors to. If absent, disables Sentry integration. | | +| **`PluralKit__Bot__Prefixes__0`** | A custom prefix to use instead of `pk;`, add additional entries replacing 0 with n for more prefixes | *`pluralkit__discord__bot_prefix_for_gateway`* (only first) | diff --git a/dev-docs/rust.md b/dev-docs/rust.md index dec9bd06..4c504101 100644 --- a/dev-docs/rust.md +++ b/dev-docs/rust.md @@ -21,6 +21,7 @@ Configuration is done through environment variables. A potentially uncompleted a | G | **`pluralkit__discord__cluster__node_id`** | the ID of the cluster (overwritten at runtime when operating under managers that can't template the node id into this variable, such as kubernetes) | | G | **`pluralkit__discord__max_concurrency`** | number of identify requests per 5 seconds -- see Discord docs | | G | **`pluralkit__discord__gateway_target`** | the URL of a dotnet bot instance to send events to | +| G | **`pluralkit__discord__bot_prefix_for_gateway`** | the prefix to show in the bot's activity status. If not specified will use `pk;` | | G, ST | **`pluralkit__discord__api_base_url`** | the base Discord API url used for HTTP API requests | | G, A, ST, AV | **`pluralkit__db__data_db_uri`** | the URI of the PostgreSQL data database in [libpq format](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) | | G, A, ST, AV | **`pluralkit__db__data_redis_addr`** | the address of the Redis instance, in [standard Redis format](https://redis.io/docs/latest/develop/clients/nodejs/connect/) | @@ -45,4 +46,3 @@ Configuration is done through environment variables. A potentially uncompleted a | G | **`pluralkit__runtime_config_key`** | the instance identifier key used when fetching configuration from Redis at runtime to differentiate gateway instances (ex. 'gateway') | | G, A, ST, AV, D | **`pluralkit__run_metrics_server`** | boolean used to enable or disable the inbuilt Prometheus format metrics server | | G, A, ST, AV, D | **`pluralkit__sentry_url`** | the URL of a sentry instance to publish errors to | - From 01675198045018b86ab05cda5e732cf89cf4cc3a Mon Sep 17 00:00:00 2001 From: alyssa Date: Thu, 22 May 2025 19:19:33 +0000 Subject: [PATCH 44/80] feat(gateway): instrument shard threads with shard id --- crates/gateway/src/discord/gateway.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index 563a9b78..33c9e12e 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -94,6 +94,7 @@ pub fn create_shards(redis: fred::clients::RedisPool) -> anyhow::Result, tx: Sender<(ShardId, Event, String)>, From e2acaf93be45171ef5dd32dbaafe27853845ced8 Mon Sep 17 00:00:00 2001 From: asleepyskye Date: Thu, 22 May 2025 17:31:29 -0400 Subject: [PATCH 45/80] fix(gateway): move shard state updates, store in hashmap --- crates/gateway/src/discord/gateway.rs | 35 +++++++++---- crates/gateway/src/discord/shard_state.rs | 63 +++++++++++++---------- crates/gateway/src/main.rs | 49 ++++++++++++++++-- crates/libpk/src/state.rs | 8 ++- 4 files changed, 115 insertions(+), 40 deletions(-) diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index 33c9e12e..76643965 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use futures::StreamExt; -use libpk::{_config::ClusterSettings, runtime_config::RuntimeConfig}; +use libpk::{_config::ClusterSettings, runtime_config::RuntimeConfig, state::ShardStateEvent}; use metrics::counter; use std::sync::Arc; use tokio::sync::mpsc::Sender; @@ -19,7 +19,7 @@ use crate::{ RUNTIME_CONFIG_KEY_EVENT_TARGET, }; -use super::{cache::DiscordCache, shard_state::ShardStateManager}; +use super::cache::DiscordCache; pub fn cluster_config() -> ClusterSettings { libpk::config @@ -98,7 +98,7 @@ pub fn create_shards(redis: fred::clients::RedisPool) -> anyhow::Result, tx: Sender<(ShardId, Event, String)>, - shard_state: ShardStateManager, + tx_state: Sender<(ShardId, ShardStateEvent, Option, Option)>, cache: Arc, runtime_config: Arc, ) { @@ -131,8 +131,10 @@ pub async fn runner( ) .increment(1); - if let Err(error) = shard_state.socket_closed(shard_id).await { - error!(?error, "failed to update shard state for socket closure"); + if let Err(error) = + tx_state.try_send((shard.id(), ShardStateEvent::Closed, None, None)) + { + error!("failed to update shard state for socket closure: {error}"); } continue; @@ -173,14 +175,29 @@ pub async fn runner( .increment(1); // update shard state and discord cache - if let Err(error) = shard_state.handle_event(shard_id, event.clone()).await { - tracing::error!(?error, "error updating redis state"); + if let Err(error) = tx_state.try_send(( + shard.id(), + ShardStateEvent::Other, + Some(event.clone()), + None, + )) { + tracing::error!(?error, "error updating shard state"); } // need to do heartbeat separately, to get the latency + let latency_num = shard + .latency() + .recent() + .first() + .map_or_else(|| 0, |d| d.as_millis()) as i32; if let Event::GatewayHeartbeatAck = event - && let Err(error) = shard_state.heartbeated(shard_id, shard.latency()).await + && let Err(error) = tx_state.try_send(( + shard.id(), + ShardStateEvent::Heartbeat, + Some(event.clone()), + Some(latency_num), + )) { - tracing::error!(?error, "error updating redis state for latency"); + tracing::error!(?error, "error updating shard state for latency"); } if let Event::Ready(_) = event { diff --git a/crates/gateway/src/discord/shard_state.rs b/crates/gateway/src/discord/shard_state.rs index a7579583..d063fb5b 100644 --- a/crates/gateway/src/discord/shard_state.rs +++ b/crates/gateway/src/discord/shard_state.rs @@ -1,21 +1,29 @@ use fred::{clients::RedisPool, interfaces::HashesInterface}; use metrics::{counter, gauge}; use tracing::info; -use twilight_gateway::{Event, Latency}; +use twilight_gateway::Event; + +use std::collections::HashMap; use libpk::state::ShardState; +use super::gateway::cluster_config; + #[derive(Clone)] pub struct ShardStateManager { redis: RedisPool, + shards: HashMap, } pub fn new(redis: RedisPool) -> ShardStateManager { - ShardStateManager { redis } + ShardStateManager { + redis: redis, + shards: HashMap::new(), + } } impl ShardStateManager { - pub async fn handle_event(&self, shard_id: u32, event: Event) -> anyhow::Result<()> { + pub async fn handle_event(&mut self, shard_id: u32, event: Event) -> anyhow::Result<()> { match event { Event::Ready(_) => self.ready_or_resumed(shard_id, false).await, Event::Resumed => self.ready_or_resumed(shard_id, true).await, @@ -23,15 +31,8 @@ impl ShardStateManager { } } - async fn get_shard(&self, shard_id: u32) -> anyhow::Result { - let data: Option = self.redis.hget("pluralkit:shardstatus", shard_id).await?; - match data { - Some(buf) => Ok(serde_json::from_str(&buf).expect("could not decode shard data!")), - None => Ok(ShardState::default()), - } - } - - async fn save_shard(&self, shard_id: u32, info: ShardState) -> anyhow::Result<()> { + async fn save_shard(&mut self, shard_id: u32) -> anyhow::Result<()> { + let info = self.shards.get(&shard_id); self.redis .hset::<(), &str, (String, String)>( "pluralkit:shardstatus", @@ -44,7 +45,7 @@ impl ShardStateManager { Ok(()) } - async fn ready_or_resumed(&self, shard_id: u32, resumed: bool) -> anyhow::Result<()> { + async fn ready_or_resumed(&mut self, shard_id: u32, resumed: bool) -> anyhow::Result<()> { info!( "shard {} {}", shard_id, @@ -57,33 +58,41 @@ impl ShardStateManager { ) .increment(1); gauge!("pluralkit_gateway_shard_up").increment(1); - let mut info = self.get_shard(shard_id).await?; + + let info = self.shards.entry(shard_id).or_insert(ShardState::default()); + info.shard_id = shard_id as i32; + info.cluster_id = Some(cluster_config().node_id as i32); info.last_connection = chrono::offset::Utc::now().timestamp() as i32; info.up = true; - self.save_shard(shard_id, info).await?; + + self.save_shard(shard_id).await?; Ok(()) } - pub async fn socket_closed(&self, shard_id: u32) -> anyhow::Result<()> { + pub async fn socket_closed(&mut self, shard_id: u32) -> anyhow::Result<()> { gauge!("pluralkit_gateway_shard_up").decrement(1); - let mut info = self.get_shard(shard_id).await?; + + let info = self.shards.entry(shard_id).or_insert(ShardState::default()); + info.shard_id = shard_id as i32; + info.cluster_id = Some(cluster_config().node_id as i32); info.up = false; info.disconnection_count += 1; - self.save_shard(shard_id, info).await?; + + self.save_shard(shard_id).await?; Ok(()) } - pub async fn heartbeated(&self, shard_id: u32, latency: &Latency) -> anyhow::Result<()> { - let mut info = self.get_shard(shard_id).await?; + pub async fn heartbeated(&mut self, shard_id: u32, latency: i32) -> anyhow::Result<()> { + gauge!("pluralkit_gateway_shard_latency", "shard_id" => shard_id.to_string()).set(latency); + + let info = self.shards.entry(shard_id).or_insert(ShardState::default()); + info.shard_id = shard_id as i32; + info.cluster_id = Some(cluster_config().node_id as i32); info.up = true; info.last_heartbeat = chrono::offset::Utc::now().timestamp() as i32; - info.latency = latency - .recent() - .first() - .map_or_else(|| 0, |d| d.as_millis()) as i32; - gauge!("pluralkit_gateway_shard_latency", "shard_id" => shard_id.to_string()) - .set(info.latency); - self.save_shard(shard_id, info).await?; + info.latency = latency; + + self.save_shard(shard_id).await?; Ok(()) } } diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 6d54eb41..f7c88cb2 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -6,7 +6,7 @@ use chrono::Timelike; use discord::gateway::cluster_config; use event_awaiter::EventAwaiter; use fred::{clients::RedisPool, interfaces::*}; -use libpk::runtime_config::RuntimeConfig; +use libpk::{runtime_config::RuntimeConfig, state::ShardStateEvent}; use reqwest::{ClientBuilder, StatusCode}; use std::{sync::Arc, time::Duration, vec::Vec}; use tokio::{ @@ -54,7 +54,6 @@ async fn main() -> anyhow::Result<()> { .await?; } - let shard_state = discord::shard_state::new(redis.clone()); let cache = Arc::new(discord::cache::new()); let awaiter = Arc::new(EventAwaiter::new()); tokio::spawn({ @@ -68,6 +67,14 @@ async fn main() -> anyhow::Result<()> { // todo: make sure this doesn't fill up let (event_tx, mut event_rx) = channel::<(ShardId, twilight_gateway::Event, String)>(1000); + // todo: make sure this doesn't fill up + let (state_tx, mut state_rx) = channel::<( + ShardId, + ShardStateEvent, + Option, + Option, + )>(1000); + let mut senders = Vec::new(); let mut signal_senders = Vec::new(); @@ -78,12 +85,48 @@ async fn main() -> anyhow::Result<()> { set.spawn(tokio::spawn(discord::gateway::runner( shard, event_tx.clone(), - shard_state.clone(), + state_tx.clone(), cache.clone(), runtime_config.clone(), ))); } + set.spawn(tokio::spawn({ + let mut shard_state = discord::shard_state::new(redis.clone()); + + async move { + while let Some((shard_id, state_event, parsed_event, latency)) = state_rx.recv().await { + match state_event { + ShardStateEvent::Heartbeat => { + if !latency.is_none() + && let Err(error) = shard_state + .heartbeated(shard_id.number(), latency.unwrap()) + .await + { + error!("failed to update shard state for heartbeat: {error}") + }; + } + ShardStateEvent::Closed => { + if let Err(error) = shard_state.socket_closed(shard_id.number()).await { + error!("failed to update shard state for heartbeat: {error}") + }; + } + ShardStateEvent::Other => { + if let Err(error) = shard_state + .handle_event( + shard_id.number(), + parsed_event.expect("shard state event not provided!"), + ) + .await + { + error!("failed to update shard state for heartbeat: {error}") + }; + } + } + } + } + })); + set.spawn(tokio::spawn({ let runtime_config = runtime_config.clone(); let awaiter = awaiter.clone(); diff --git a/crates/libpk/src/state.rs b/crates/libpk/src/state.rs index 90a77c21..df44ea1d 100644 --- a/crates/libpk/src/state.rs +++ b/crates/libpk/src/state.rs @@ -1,4 +1,4 @@ -#[derive(serde::Serialize, serde::Deserialize, Clone, Default)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Default, Debug)] pub struct ShardState { pub shard_id: i32, pub up: bool, @@ -10,3 +10,9 @@ pub struct ShardState { pub last_connection: i32, pub cluster_id: Option, } + +pub enum ShardStateEvent { + Closed, + Heartbeat, + Other, +} From b5f18106e172a0dcd7109e416b0498a70b8a4251 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 24 May 2025 12:23:19 +0000 Subject: [PATCH 46/80] feat(gateway): add /shard_status endpoint --- crates/gateway/src/cache_api.rs | 7 ++- crates/gateway/src/discord/shard_state.rs | 58 ++++++++++++++++------- crates/gateway/src/main.rs | 6 ++- 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/crates/gateway/src/cache_api.rs b/crates/gateway/src/cache_api.rs index 375ab86e..f8c3f556 100644 --- a/crates/gateway/src/cache_api.rs +++ b/crates/gateway/src/cache_api.rs @@ -14,6 +14,7 @@ use crate::{ discord::{ cache::{dm_channel, DiscordCache, DM_PERMISSIONS}, gateway::cluster_config, + shard_state::ShardStateManager, }, event_awaiter::{AwaitEventRequest, EventAwaiter}, }; @@ -25,7 +26,7 @@ fn status_code(code: StatusCode, body: String) -> Response { // this function is manually formatted for easier legibility of route_services #[rustfmt::skip] -pub async fn run_server(cache: Arc, runtime_config: Arc, awaiter: Arc) -> anyhow::Result<()> { +pub async fn run_server(cache: Arc, shard_state: Arc, runtime_config: Arc, awaiter: Arc) -> anyhow::Result<()> { // hacky fix for `move` let runtime_config_for_post = runtime_config.clone(); let runtime_config_for_delete = runtime_config.clone(); @@ -211,6 +212,10 @@ pub async fn run_server(cache: Arc, runtime_config: Arc, + shards: RwLock>, } pub fn new(redis: RedisPool) -> ShardStateManager { ShardStateManager { redis: redis, - shards: HashMap::new(), + shards: RwLock::new(HashMap::new()), } } impl ShardStateManager { - pub async fn handle_event(&mut self, shard_id: u32, event: Event) -> anyhow::Result<()> { + pub async fn handle_event(&self, shard_id: u32, event: Event) -> anyhow::Result<()> { match event { Event::Ready(_) => self.ready_or_resumed(shard_id, false).await, Event::Resumed => self.ready_or_resumed(shard_id, true).await, @@ -31,21 +31,33 @@ impl ShardStateManager { } } - async fn save_shard(&mut self, shard_id: u32) -> anyhow::Result<()> { - let info = self.shards.get(&shard_id); + async fn save_shard(&self, id: u32, state: ShardState) -> anyhow::Result<()> { + { + let mut shards = self.shards.write().await; + shards.insert(id, state.clone()); + } self.redis .hset::<(), &str, (String, String)>( "pluralkit:shardstatus", ( - shard_id.to_string(), - serde_json::to_string(&info).expect("could not serialize shard"), + id.to_string(), + serde_json::to_string(&state).expect("could not serialize shard"), ), ) .await?; Ok(()) } - async fn ready_or_resumed(&mut self, shard_id: u32, resumed: bool) -> anyhow::Result<()> { + async fn get_shard(&self, id: u32) -> Option { + let shards = self.shards.read().await; + shards.get(&id).cloned() + } + + pub async fn get(&self) -> Vec { + self.shards.read().await.values().cloned().collect() + } + + async fn ready_or_resumed(&self, shard_id: u32, resumed: bool) -> anyhow::Result<()> { info!( "shard {} {}", shard_id, @@ -59,40 +71,52 @@ impl ShardStateManager { .increment(1); gauge!("pluralkit_gateway_shard_up").increment(1); - let info = self.shards.entry(shard_id).or_insert(ShardState::default()); + let mut info = self + .get_shard(shard_id) + .await + .unwrap_or(ShardState::default()); + info.shard_id = shard_id as i32; info.cluster_id = Some(cluster_config().node_id as i32); info.last_connection = chrono::offset::Utc::now().timestamp() as i32; info.up = true; - self.save_shard(shard_id).await?; + self.save_shard(shard_id, info).await?; Ok(()) } - pub async fn socket_closed(&mut self, shard_id: u32) -> anyhow::Result<()> { + pub async fn socket_closed(&self, shard_id: u32) -> anyhow::Result<()> { gauge!("pluralkit_gateway_shard_up").decrement(1); - let info = self.shards.entry(shard_id).or_insert(ShardState::default()); + let mut info = self + .get_shard(shard_id) + .await + .unwrap_or(ShardState::default()); + info.shard_id = shard_id as i32; info.cluster_id = Some(cluster_config().node_id as i32); info.up = false; info.disconnection_count += 1; - self.save_shard(shard_id).await?; + self.save_shard(shard_id, info).await?; Ok(()) } - pub async fn heartbeated(&mut self, shard_id: u32, latency: i32) -> anyhow::Result<()> { + pub async fn heartbeated(&self, shard_id: u32, latency: i32) -> anyhow::Result<()> { gauge!("pluralkit_gateway_shard_latency", "shard_id" => shard_id.to_string()).set(latency); - let info = self.shards.entry(shard_id).or_insert(ShardState::default()); + let mut info = self + .get_shard(shard_id) + .await + .unwrap_or(ShardState::default()); + info.shard_id = shard_id as i32; info.cluster_id = Some(cluster_config().node_id as i32); info.up = true; info.last_heartbeat = chrono::offset::Utc::now().timestamp() as i32; info.latency = latency; - self.save_shard(shard_id).await?; + self.save_shard(shard_id, info).await?; Ok(()) } } diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index f7c88cb2..3b45e1b8 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -91,8 +91,10 @@ async fn main() -> anyhow::Result<()> { ))); } + let shard_state = Arc::new(discord::shard_state::new(redis.clone())); + set.spawn(tokio::spawn({ - let mut shard_state = discord::shard_state::new(redis.clone()); + let shard_state = shard_state.clone(); async move { while let Some((shard_id, state_event, parsed_event, latency)) = state_rx.recv().await { @@ -187,7 +189,7 @@ async fn main() -> anyhow::Result<()> { )); set.spawn(tokio::spawn(async move { - match cache_api::run_server(cache, runtime_config, awaiter.clone()).await { + match cache_api::run_server(cache, shard_state, runtime_config, awaiter.clone()).await { Err(error) => { error!(?error, "failed to serve cache api"); } From 05817afdba4221c2c58afc5c251a4ff7c915ea7b Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 24 May 2025 12:24:59 +0000 Subject: [PATCH 47/80] chore(gateway): promote cache_api to just api --- crates/gateway/src/{cache_api.rs => api.rs} | 0 crates/gateway/src/main.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename crates/gateway/src/{cache_api.rs => api.rs} (100%) diff --git a/crates/gateway/src/cache_api.rs b/crates/gateway/src/api.rs similarity index 100% rename from crates/gateway/src/cache_api.rs rename to crates/gateway/src/api.rs diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 3b45e1b8..e61c3445 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -18,7 +18,7 @@ use tracing::{error, info, warn}; use twilight_gateway::{MessageSender, ShardId}; use twilight_model::gateway::payload::outgoing::UpdatePresence; -mod cache_api; +mod api; mod discord; mod event_awaiter; mod logger; @@ -189,7 +189,7 @@ async fn main() -> anyhow::Result<()> { )); set.spawn(tokio::spawn(async move { - match cache_api::run_server(cache, shard_state, runtime_config, awaiter.clone()).await { + match api::run_server(cache, shard_state, runtime_config, awaiter.clone()).await { Err(error) => { error!(?error, "failed to serve cache api"); } From 73103dc2b5bc4075d8edbaa14290b85fdd2a5828 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 24 May 2025 13:19:40 +0000 Subject: [PATCH 48/80] fix(gateway): only send relevant events to shard state manager --- crates/gateway/src/discord/gateway.rs | 16 +++++++++------- crates/gateway/src/discord/shard_state.rs | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/gateway/src/discord/gateway.rs b/crates/gateway/src/discord/gateway.rs index 76643965..92acdeaa 100644 --- a/crates/gateway/src/discord/gateway.rs +++ b/crates/gateway/src/discord/gateway.rs @@ -175,13 +175,15 @@ pub async fn runner( .increment(1); // update shard state and discord cache - if let Err(error) = tx_state.try_send(( - shard.id(), - ShardStateEvent::Other, - Some(event.clone()), - None, - )) { - tracing::error!(?error, "error updating shard state"); + if matches!(event, Event::Ready(_)) || matches!(event, Event::Resumed) { + if let Err(error) = tx_state.try_send(( + shard.id(), + ShardStateEvent::Other, + Some(event.clone()), + None, + )) { + tracing::error!(?error, "error updating shard state"); + } } // need to do heartbeat separately, to get the latency let latency_num = shard diff --git a/crates/gateway/src/discord/shard_state.rs b/crates/gateway/src/discord/shard_state.rs index 437560be..c85e02c8 100644 --- a/crates/gateway/src/discord/shard_state.rs +++ b/crates/gateway/src/discord/shard_state.rs @@ -25,6 +25,7 @@ pub fn new(redis: RedisPool) -> ShardStateManager { impl ShardStateManager { pub async fn handle_event(&self, shard_id: u32, event: Event) -> anyhow::Result<()> { match event { + // also update gateway.rs with event types Event::Ready(_) => self.ready_or_resumed(shard_id, false).await, Event::Resumed => self.ready_or_resumed(shard_id, true).await, _ => Ok(()), From 0c342472b11458745666ebab36e1ecc6e5968bb1 Mon Sep 17 00:00:00 2001 From: leo60228 Date: Mon, 26 May 2025 13:09:11 -0400 Subject: [PATCH 49/80] fix: use Serilog submodule in Myriad --- Myriad/Myriad.csproj | 5 ++++- Myriad/packages.lock.json | 9 +++------ PluralKit.Bot/packages.lock.json | 10 ++++------ PluralKit.Tests/packages.lock.json | 10 ++++------ 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/Myriad/Myriad.csproj b/Myriad/Myriad.csproj index 0a6ed588..fb7ed9bb 100644 --- a/Myriad/Myriad.csproj +++ b/Myriad/Myriad.csproj @@ -20,12 +20,15 @@ true + + + + - diff --git a/Myriad/packages.lock.json b/Myriad/packages.lock.json index 7680637b..7a4bc2c0 100644 --- a/Myriad/packages.lock.json +++ b/Myriad/packages.lock.json @@ -33,12 +33,6 @@ "resolved": "1.1.1", "contentHash": "1MUQLiSo4KDkQe6nzQRhIU05lm9jlexX5BVsbuw0SL82ynZ+GzAHQxJVDPVBboxV37Po3SG077aX8DuSy8TkaA==" }, - "Serilog": { - "type": "Direct", - "requested": "[4.2.0, )", - "resolved": "4.2.0", - "contentHash": "gmoWVOvKgbME8TYR+gwMf7osROiWAURterc6Rt2dQyX7wtjZYpqFiA/pY6ztjGQKKV62GGCyOcmtP1UKMHgSmA==" - }, "StackExchange.Redis": { "type": "Direct", "requested": "[2.8.22, )", @@ -90,6 +84,9 @@ "type": "Transitive", "resolved": "5.0.1", "contentHash": "qEePWsaq9LoEEIqhbGe6D5J8c9IqQOUuTzzV6wn1POlfdLkJliZY3OlB0j0f17uMWlqZYjH7txj+2YbyrIA8Yg==" + }, + "serilog": { + "type": "Project" } } } diff --git a/PluralKit.Bot/packages.lock.json b/PluralKit.Bot/packages.lock.json index 3d718fc6..449923d9 100644 --- a/PluralKit.Bot/packages.lock.json +++ b/PluralKit.Bot/packages.lock.json @@ -416,11 +416,6 @@ "resolved": "1.0.9", "contentHash": "RkQGXIrqHjD5h1mqefhgCbkaSdRYNRG5rrbzyw5zeLWiS0K1wq9xR3cNhQdzYR2MsKZ3GN523yRUsEQIMPxh3Q==" }, - "Serilog": { - "type": "Transitive", - "resolved": "4.2.0", - "contentHash": "gmoWVOvKgbME8TYR+gwMf7osROiWAURterc6Rt2dQyX7wtjZYpqFiA/pY6ztjGQKKV62GGCyOcmtP1UKMHgSmA==" - }, "Serilog.Extensions.Logging": { "type": "Transitive", "resolved": "8.0.0", @@ -768,7 +763,7 @@ "NodaTime.Serialization.JsonNet": "[3.1.0, )", "Polly": "[8.5.0, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", - "Serilog": "[4.2.0, )", + "Serilog": "[4.1.0, )", "StackExchange.Redis": "[2.8.22, )", "System.Linq.Async": "[6.0.1, )" } @@ -811,6 +806,9 @@ "System.Interactive.Async": "[6.0.1, )", "ipnetwork2": "[3.0.667, )" } + }, + "serilog": { + "type": "Project" } } } diff --git a/PluralKit.Tests/packages.lock.json b/PluralKit.Tests/packages.lock.json index 79926c2a..ce0dbc15 100644 --- a/PluralKit.Tests/packages.lock.json +++ b/PluralKit.Tests/packages.lock.json @@ -530,11 +530,6 @@ "resolved": "4.13.0", "contentHash": "Wfw3M1WpFcrYaGzPm7QyUTfIOYkVXQ1ry6p4WYjhbLz9fPwV23SGQZTFDpdox67NHM0V0g1aoQ4YKLm4ANtEEg==" }, - "Serilog": { - "type": "Transitive", - "resolved": "4.2.0", - "contentHash": "gmoWVOvKgbME8TYR+gwMf7osROiWAURterc6Rt2dQyX7wtjZYpqFiA/pY6ztjGQKKV62GGCyOcmtP1UKMHgSmA==" - }, "Serilog.AspNetCore": { "type": "Transitive", "resolved": "8.0.0", @@ -983,7 +978,7 @@ "NodaTime.Serialization.JsonNet": "[3.1.0, )", "Polly": "[8.5.0, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", - "Serilog": "[4.2.0, )", + "Serilog": "[4.1.0, )", "StackExchange.Redis": "[2.8.22, )", "System.Linq.Async": "[6.0.1, )" } @@ -1047,6 +1042,9 @@ "System.Interactive.Async": "[6.0.1, )", "ipnetwork2": "[3.0.667, )" } + }, + "serilog": { + "type": "Project" } } } From 0406c32f6b6df0ac7dbec0adc827267021e8f1bd Mon Sep 17 00:00:00 2001 From: asleepyskye Date: Mon, 26 May 2025 20:20:37 +0000 Subject: [PATCH 50/80] chore: update dev-docs for Serilog submodule --- README.md | 3 ++- dev-docs/README.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 84cc3f56..26b96c6d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ PluralKit is a Discord bot meant for plural communities. It has features like me PluralKit has a Discord server for support, feedback, and discussion: https://discord.gg/PczBt78 # Running -In production, we run PluralKit using Kubernetes (soon). The configuration can be found in the infra repo. +In production, we run PluralKit using Kubernetes. The configuration can be found in the infra repo. For self-hosting, it's simpler to use Docker, with the provided [docker-compose](./docker-compose.yml) file. @@ -22,6 +22,7 @@ If you want to use `pk;admin` commands (to raise member limits and such), set ` ADMIN_ROLE=682632767057428509 ``` +*If you didn't clone the repository with submodules, run `git submodule update --init` first to pull the required submodules.* Run `docker compose build`, then `docker compose up -d`. To view logs, use `docker compose logs`. diff --git a/dev-docs/README.md b/dev-docs/README.md index 02c18d46..76199e09 100644 --- a/dev-docs/README.md +++ b/dev-docs/README.md @@ -49,14 +49,14 @@ PluralKit__Bot__HttpListenerAddr="127.0.0.1" PluralKit__Bot__HttpCacheUrl="localhost:5000" ``` -1. Clone the repository: `git clone https://github.com/PluralKit/PluralKit` +1. Clone the repository: `git clone --recurse-submodules https://github.com/PluralKit/PluralKit` 2. Create a `.env` configuration file in the `PluralKit` directory *(see above)* 3. Build and run: `nix run .#dev` - This will download the dependencies, build, and run PluralKit - If Nix is not setup to allow flakes, you may need to add `--extra-experimental-features nix-command --extra-experimental-features flakes` to the command - If the `pluralkit-bot` process fails to run, you can restart it by selecting it and pressing `Ctrl-R` ``` -[nix-shell:~]$ git clone https://github.com/PluralKit/PluralKit +[nix-shell:~]$ git clone --recurse-submodules https://github.com/PluralKit/PluralKit [nix-shell:~]$ cd PluralKit [nix-shell:~/PluralKit]$ nano .env [nix-shell:~/PluralKit]$ nix run .#dev From 0610701252d1d0a7cf202e20da0c35384d1f7120 Mon Sep 17 00:00:00 2001 From: alyssa Date: Wed, 28 May 2025 23:00:39 +0000 Subject: [PATCH 51/80] feat(api): allow unauthed requests to /systems/:id/settings --- Cargo.lock | 6 +- Cargo.toml | 3 +- crates/api/src/auth.rs | 26 ++++++ crates/api/src/endpoints/mod.rs | 1 + crates/api/src/endpoints/private.rs | 10 +-- crates/api/src/endpoints/system.rs | 68 +++++++++++++++ crates/api/src/main.rs | 10 ++- crates/api/src/middleware/mod.rs | 16 ++-- crates/api/src/middleware/params.rs | 123 ++++++++++++++++++++++++++++ crates/macros/src/model.rs | 65 +++++++++++---- crates/models/src/_util.rs | 14 ++++ crates/models/src/lib.rs | 22 +++++ crates/models/src/system.rs | 33 ++------ crates/models/src/system_config.rs | 7 +- 14 files changed, 334 insertions(+), 70 deletions(-) create mode 100644 crates/api/src/endpoints/system.rs create mode 100644 crates/api/src/middleware/params.rs diff --git a/Cargo.lock b/Cargo.lock index 6b44de61..341ba381 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -276,8 +276,7 @@ dependencies = [ [[package]] name = "axum" version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" +source = "git+https://github.com/pluralkit/axum?branch=v0.8.4-pluralkit#3b45806719f27e69aed912bac724056b910f4aa6" dependencies = [ "axum-core 0.5.2", "bytes", @@ -327,8 +326,7 @@ dependencies = [ [[package]] name = "axum-core" version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" +source = "git+https://github.com/pluralkit/axum?branch=v0.8.4-pluralkit#3b45806719f27e69aed912bac724056b910f4aa6" dependencies = [ "bytes", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index 5f1272bc..48c73439 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,6 @@ resolver = "2" [workspace.dependencies] anyhow = "1" -axum = "0.8.4" axum-macros = "0.4.1" bytes = "1.6.0" chrono = "0.4" @@ -24,6 +23,8 @@ tracing = "0.1" tracing-subscriber = { version = "0.3.16", features = ["env-filter", "json"] } uuid = { version = "1.7.0", features = ["serde"] } +axum = { git = "https://github.com/pluralkit/axum", branch = "v0.8.4-pluralkit" } + twilight-gateway = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590" } twilight-cache-inmemory = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590", features = ["permission-calculator"] } twilight-util = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590", features = ["permission-calculator"] } diff --git a/crates/api/src/auth.rs b/crates/api/src/auth.rs index 49752a4b..c084eafe 100644 --- a/crates/api/src/auth.rs +++ b/crates/api/src/auth.rs @@ -1,3 +1,5 @@ +use pluralkit_models::{PKSystem, PrivacyLevel, SystemId}; + pub const INTERNAL_SYSTEMID_HEADER: &'static str = "x-pluralkit-systemid"; pub const INTERNAL_APPID_HEADER: &'static str = "x-pluralkit-appid"; @@ -19,4 +21,28 @@ impl AuthState { pub fn app_id(&self) -> Option { self.app_id } + + pub fn access_level_for(&self, a: &impl Authable) -> PrivacyLevel { + if self + .system_id + .map(|id| id == a.authable_system_id()) + .unwrap_or(false) + { + PrivacyLevel::Private + } else { + PrivacyLevel::Public + } + } +} + +// authable trait/impls + +pub trait Authable { + fn authable_system_id(&self) -> SystemId; +} + +impl Authable for PKSystem { + fn authable_system_id(&self) -> SystemId { + self.id + } } diff --git a/crates/api/src/endpoints/mod.rs b/crates/api/src/endpoints/mod.rs index f19f44d8..c311367c 100644 --- a/crates/api/src/endpoints/mod.rs +++ b/crates/api/src/endpoints/mod.rs @@ -1 +1,2 @@ pub mod private; +pub mod system; diff --git a/crates/api/src/endpoints/private.rs b/crates/api/src/endpoints/private.rs index ad76e275..df67421c 100644 --- a/crates/api/src/endpoints/private.rs +++ b/crates/api/src/endpoints/private.rs @@ -52,7 +52,7 @@ use axum::{ }; use hyper::StatusCode; use libpk::config; -use pluralkit_models::{PKSystem, PKSystemConfig}; +use pluralkit_models::{PKSystem, PKSystemConfig, PrivacyLevel}; use reqwest::ClientBuilder; #[derive(serde::Deserialize, Debug)] @@ -151,14 +151,12 @@ pub async fn discord_callback( .await .expect("failed to query"); - if system.is_none() { + let Some(system) = system else { return json_err( StatusCode::BAD_REQUEST, "user does not have a system registered".to_string(), ); - } - - let system = system.unwrap(); + }; let system_config: Option = sqlx::query_as( r#" @@ -179,7 +177,7 @@ pub async fn discord_callback( ( StatusCode::OK, serde_json::to_string(&serde_json::json!({ - "system": system.to_json(), + "system": system.to_json(PrivacyLevel::Private), "config": system_config.to_json(), "user": user, "token": token, diff --git a/crates/api/src/endpoints/system.rs b/crates/api/src/endpoints/system.rs new file mode 100644 index 00000000..21672db2 --- /dev/null +++ b/crates/api/src/endpoints/system.rs @@ -0,0 +1,68 @@ +use axum::{ + extract::State, + http::StatusCode, + response::{IntoResponse, Response}, + Extension, +}; +use serde_json::json; +use sqlx::Postgres; +use tracing::error; + +use pluralkit_models::{PKSystem, PKSystemConfig, PrivacyLevel}; + +use crate::{auth::AuthState, util::json_err, ApiContext}; + +pub async fn get_system_settings( + Extension(auth): Extension, + Extension(system): Extension, + State(ctx): State, +) -> Response { + let access_level = auth.access_level_for(&system); + + let config = match sqlx::query_as::( + "select * from system_config where system = $1", + ) + .bind(system.id) + .fetch_optional(&ctx.db) + .await + { + Ok(Some(config)) => config, + Ok(None) => { + error!( + system = system.id, + "failed to find system config for existing system" + ); + return json_err( + StatusCode::INTERNAL_SERVER_ERROR, + r#"{"message": "500: Internal Server Error", "code": 0}"#.to_string(), + ); + } + Err(err) => { + error!(?err, "failed to query system config"); + return json_err( + StatusCode::INTERNAL_SERVER_ERROR, + r#"{"message": "500: Internal Server Error", "code": 0}"#.to_string(), + ); + } + }; + + ( + StatusCode::OK, + serde_json::to_string(&match access_level { + PrivacyLevel::Private => config.to_json(), + PrivacyLevel::Public => json!({ + "pings_enabled": config.pings_enabled, + "latch_timeout": config.latch_timeout, + "case_sensitive_proxy_tags": config.case_sensitive_proxy_tags, + "proxy_error_message_enabled": config.proxy_error_message_enabled, + "hid_display_split": config.hid_display_split, + "hid_display_caps": config.hid_display_caps, + "hid_list_padding": config.hid_list_padding, + "proxy_switch": config.proxy_switch, + "name_format": config.name_format, + }), + }) + .unwrap(), + ) + .into_response() +} diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 48666d6f..57ecbb0a 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -77,7 +77,7 @@ fn router(ctx: ApiContext) -> Router { Router::new() .route("/v2/systems/{system_id}", get(rproxy)) .route("/v2/systems/{system_id}", patch(rproxy)) - .route("/v2/systems/{system_id}/settings", get(rproxy)) + .route("/v2/systems/{system_id}/settings", get(endpoints::system::get_system_settings)) .route("/v2/systems/{system_id}/settings", patch(rproxy)) .route("/v2/systems/{system_id}/members", get(rproxy)) @@ -134,10 +134,12 @@ fn router(ctx: ApiContext) -> Router { .route("/v2/groups/{group_id}/oembed.json", get(rproxy)) .layer(middleware::ratelimit::ratelimiter(middleware::ratelimit::do_request_ratelimited)) // this sucks - .layer(axum::middleware::from_fn(middleware::ignore_invalid_routes)) - .layer(axum::middleware::from_fn(middleware::cors)) - .layer(axum::middleware::from_fn(middleware::logger)) + .layer(axum::middleware::from_fn(middleware::ignore_invalid_routes::ignore_invalid_routes)) + .layer(axum::middleware::from_fn(middleware::cors::cors)) + .layer(axum::middleware::from_fn(middleware::logger::logger)) + + .layer(axum::middleware::from_fn_with_state(ctx.clone(), middleware::params::params)) .layer(axum::middleware::from_fn_with_state(ctx.clone(), middleware::auth::auth)) .layer(tower_http::catch_panic::CatchPanicLayer::custom(util::handle_panic)) diff --git a/crates/api/src/middleware/mod.rs b/crates/api/src/middleware/mod.rs index ad67be11..5c2b04dc 100644 --- a/crates/api/src/middleware/mod.rs +++ b/crates/api/src/middleware/mod.rs @@ -1,12 +1,6 @@ -mod cors; -pub use cors::cors; - -mod logger; -pub use logger::logger; - -mod ignore_invalid_routes; -pub use ignore_invalid_routes::ignore_invalid_routes; - -pub mod ratelimit; - pub mod auth; +pub mod cors; +pub mod ignore_invalid_routes; +pub mod logger; +pub mod params; +pub mod ratelimit; diff --git a/crates/api/src/middleware/params.rs b/crates/api/src/middleware/params.rs new file mode 100644 index 00000000..f1219614 --- /dev/null +++ b/crates/api/src/middleware/params.rs @@ -0,0 +1,123 @@ +use axum::{ + extract::{Request, State}, + http::StatusCode, + middleware::Next, + response::Response, + routing::url_params::UrlParams, +}; + +use sqlx::{types::Uuid, Postgres}; +use tracing::error; + +use crate::auth::AuthState; +use crate::{util::json_err, ApiContext}; +use pluralkit_models::PKSystem; + +pub async fn params(State(ctx): State, mut req: Request, next: Next) -> Response { + let pms = match req.extensions().get::() { + None => Vec::new(), + Some(UrlParams::Params(pms)) => pms.clone(), + _ => { + return json_err( + StatusCode::BAD_REQUEST, + r#"{"error": "400: Bad Request", "code": 0}"#.to_string(), + ) + .into() + } + }; + + for (key, value) in pms { + match key.as_ref() { + "system_id" => match value.as_str() { + "@me" => { + let Some(system_id) = req + .extensions() + .get::() + .expect("missing auth state") + .system_id() + else { + return json_err( + StatusCode::UNAUTHORIZED, + r#"{"error": "401: Missing or invalid Authorization header", "code": 0}"#.to_string(), + ) + .into(); + }; + + match sqlx::query_as::( + "select * from systems where id = $1", + ) + .bind(system_id) + .fetch_optional(&ctx.db) + .await + { + Ok(Some(system)) => { + req.extensions_mut().insert(system); + } + Ok(None) => { + error!( + ?system_id, + "could not find previously authenticated system in db" + ); + return json_err( + StatusCode::INTERNAL_SERVER_ERROR, + r#"{"message": "500: Internal Server Error", "code": 0}"# + .to_string(), + ); + } + Err(err) => { + error!( + ?err, + "failed to query previously authenticated system in db" + ); + return json_err( + StatusCode::INTERNAL_SERVER_ERROR, + r#"{"message": "500: Internal Server Error", "code": 0}"# + .to_string(), + ); + } + } + } + id => { + match match Uuid::parse_str(id) { + Ok(uuid) => sqlx::query_as::( + "select * from systems where uuid = $1", + ) + .bind(uuid), + Err(_) => sqlx::query_as::( + "select * from systems where hid = $1", + ) + .bind(id), + } + .fetch_optional(&ctx.db) + .await + { + Ok(Some(system)) => { + req.extensions_mut().insert(system); + } + Ok(None) => { + return json_err( + StatusCode::NOT_FOUND, + r#"{"message":"System not found.","code":20001}"#.to_string(), + ) + } + Err(err) => { + error!(?err, ?id, "failed to query system from path in db"); + return json_err( + StatusCode::INTERNAL_SERVER_ERROR, + r#"{"message": "500: Internal Server Error", "code": 0}"# + .to_string(), + ); + } + } + } + }, + "member_id" => {} + "group_id" => {} + "switch_id" => {} + "guild_id" => {} + _ => {} + } + } + + next.run(req).await +} diff --git a/crates/macros/src/model.rs b/crates/macros/src/model.rs index bfd219a7..924b5bcd 100644 --- a/crates/macros/src/model.rs +++ b/crates/macros/src/model.rs @@ -16,6 +16,7 @@ struct ModelField { patch: ElemPatchability, json: Option, is_privacy: bool, + privacy: Option, default: Option, } @@ -26,6 +27,7 @@ fn parse_field(field: syn::Field) -> ModelField { patch: ElemPatchability::None, json: None, is_privacy: false, + privacy: None, default: None, }; @@ -61,6 +63,12 @@ fn parse_field(field: syn::Field) -> ModelField { } f.json = Some(nv.value.clone()); } + "privacy" => { + if f.privacy.is_some() { + panic!("cannot set privacy multiple times for same field"); + } + f.privacy = Some(nv.value.clone()); + } "default" => { if f.default.is_some() { panic!("cannot set default multiple times for same field"); @@ -107,8 +115,6 @@ pub fn macro_impl( panic!("fields of a struct must be named"); }; - // println!("{}: {:#?}", tname, fields); - let tfields = mk_tfields(fields.clone()); let from_json = mk_tfrom_json(fields.clone()); let _from_sql = mk_tfrom_sql(fields.clone()); @@ -137,9 +143,7 @@ pub fn macro_impl( #from_json } - pub fn to_json(self) -> serde_json::Value { - #to_json - } + #to_json } #[derive(Debug, Clone)] @@ -188,19 +192,28 @@ fn mk_tfrom_sql(_fields: Vec) -> TokenStream { quote! { unimplemented!(); } } fn mk_tto_json(fields: Vec) -> TokenStream { - // todo: check privacy access + let has_privacy = fields.iter().any(|f| f.privacy.is_some()); let fielddefs: TokenStream = fields .iter() .filter_map(|f| { f.json.as_ref().map(|v| { let tname = f.name.clone(); - if let Some(default) = f.default.as_ref() { + let maybepriv = if let Some(privacy) = f.privacy.as_ref() { quote! { - #v: self.#tname.unwrap_or(#default), + #v: crate::_util::privacy_lookup!(self.#tname, self.#privacy, lookup_level) } } else { quote! { - #v: self.#tname, + #v: self.#tname + } + }; + if let Some(default) = f.default.as_ref() { + quote! { + #maybepriv.unwrap_or(#default), + } + } else { + quote! { + #maybepriv, } } }) @@ -222,13 +235,35 @@ fn mk_tto_json(fields: Vec) -> TokenStream { }) .collect(); - quote! { - serde_json::json!({ - #fielddefs - "privacy": { - #privacyfielddefs + let privdef = if has_privacy { + quote! { + , lookup_level: crate::PrivacyLevel + } + } else { + quote! {} + }; + + let privacy_fielddefs = if has_privacy { + quote! { + "privacy": if matches!(lookup_level, crate::PrivacyLevel::Private) { + Some(serde_json::json!({ + #privacyfielddefs + })) + } else { + None } - }) + } + } else { + quote! {} + }; + + quote! { + pub fn to_json(self #privdef) -> serde_json::Value { + serde_json::json!({ + #fielddefs + #privacy_fielddefs + }) + } } } diff --git a/crates/models/src/_util.rs b/crates/models/src/_util.rs index c13a0bf6..2d4921ad 100644 --- a/crates/models/src/_util.rs +++ b/crates/models/src/_util.rs @@ -33,3 +33,17 @@ macro_rules! fake_enum_impls { } pub(crate) use fake_enum_impls; + +macro_rules! privacy_lookup { + ($v:expr, $vprivacy:expr, $lookup_level:expr) => { + if matches!($vprivacy, crate::PrivacyLevel::Public) + || matches!($lookup_level, crate::PrivacyLevel::Private) + { + Some($v) + } else { + None + } + }; +} + +pub(crate) use privacy_lookup; diff --git a/crates/models/src/lib.rs b/crates/models/src/lib.rs index bb7bb08d..0bd1f92b 100644 --- a/crates/models/src/lib.rs +++ b/crates/models/src/lib.rs @@ -9,3 +9,25 @@ macro_rules! model { model!(system); model!(system_config); + +#[derive(serde::Serialize, Debug, Clone)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyLevel { + Public, + Private, +} + +// this sucks, put it somewhere else +use sqlx::{postgres::PgTypeInfo, Database, Decode, Postgres, Type}; +use std::error::Error; +_util::fake_enum_impls!(PrivacyLevel); + +impl From for PrivacyLevel { + fn from(value: i32) -> Self { + match value { + 1 => PrivacyLevel::Public, + 2 => PrivacyLevel::Private, + _ => unreachable!(), + } + } +} diff --git a/crates/models/src/system.rs b/crates/models/src/system.rs index 42b61fe4..ddaf1980 100644 --- a/crates/models/src/system.rs +++ b/crates/models/src/system.rs @@ -1,36 +1,13 @@ -use std::error::Error; - use pk_macros::pk_model; use chrono::NaiveDateTime; -use sqlx::{postgres::PgTypeInfo, Database, Decode, Postgres, Type}; use uuid::Uuid; -use crate::_util::fake_enum_impls; +use crate::PrivacyLevel; // todo: fix this pub type SystemId = i32; -// todo: move this -#[derive(serde::Serialize, Debug, Clone)] -#[serde(rename_all = "snake_case")] -pub enum PrivacyLevel { - Public, - Private, -} - -fake_enum_impls!(PrivacyLevel); - -impl From for PrivacyLevel { - fn from(value: i32) -> Self { - match value { - 1 => PrivacyLevel::Public, - 2 => PrivacyLevel::Private, - _ => unreachable!(), - } - } -} - #[pk_model] struct System { id: SystemId, @@ -40,21 +17,25 @@ struct System { #[json = "uuid"] uuid: Uuid, #[json = "name"] + #[privacy = name_privacy] name: Option, #[json = "description"] + #[privacy = description_privacy] description: Option, #[json = "tag"] tag: Option, #[json = "pronouns"] + #[privacy = pronoun_privacy] pronouns: Option, #[json = "avatar_url"] + #[privacy = avatar_privacy] avatar_url: Option, - #[json = "banner_image"] + #[json = "banner"] + #[privacy = banner_privacy] banner_image: Option, #[json = "color"] color: Option, token: Option, - #[json = "webhook_url"] webhook_url: Option, webhook_token: Option, #[json = "created"] diff --git a/crates/models/src/system_config.rs b/crates/models/src/system_config.rs index 04b1995b..772d231d 100644 --- a/crates/models/src/system_config.rs +++ b/crates/models/src/system_config.rs @@ -10,7 +10,7 @@ pub const DEFAULT_GROUP_LIMIT: i32 = 250; #[derive(serde::Serialize, Debug, Clone)] #[serde(rename_all = "snake_case")] -enum HidPadFormat { +pub enum HidPadFormat { #[serde(rename = "off")] None, Left, @@ -31,7 +31,7 @@ impl From for HidPadFormat { #[derive(serde::Serialize, Debug, Clone)] #[serde(rename_all = "snake_case")] -enum ProxySwitchAction { +pub enum ProxySwitchAction { Off, New, Add, @@ -83,7 +83,8 @@ struct SystemConfig { #[json = "proxy_switch"] proxy_switch: ProxySwitchAction, #[json = "name_format"] - name_format: String, + #[default = "{name} {tag}".to_string()] + name_format: Option, #[json = "description_templates"] description_templates: Vec, } From 301bf25d44cd831900b16c6dc623fb3ccba88f53 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 8 Jun 2025 11:02:09 +0000 Subject: [PATCH 52/80] feat(docs): update system settings model docs --- docs/content/api/endpoints.md | 2 ++ docs/content/api/models.md | 43 +++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/content/api/endpoints.md b/docs/content/api/endpoints.md index 908af0bb..fa249fff 100644 --- a/docs/content/api/endpoints.md +++ b/docs/content/api/endpoints.md @@ -34,6 +34,8 @@ GET `/systems/{systemRef}/settings` Returns a [system settings object](/api/models#system-settings-model). +If not authenticated, or authenticated as a different system, returns a [public system settings object](/api/models#public-system-settings-model). + ### Update System Settings PATCH `/systems/{systemRef}/settings` diff --git a/docs/content/api/models.md b/docs/content/api/models.md index ad81f837..23ecdc3c 100644 --- a/docs/content/api/models.md +++ b/docs/content/api/models.md @@ -113,16 +113,55 @@ Every PluralKit entity has two IDs: a short (5 or 6 character) ID and a longer U |key|type|notes| |---|---|---| |timezone|string|defaults to `UTC`| -|pings_enabled|boolean| -|latch_timeout|int?| +|pings_enabled|boolean|whether proxied messages can be pinged using the :bell: reaction| +|latch_timeout|int?|seconds after which latch autoproxy will timeout (null/default is 6 hours, 0 is "never")| |member_default_private*|boolean|whether members created through the bot have privacy settings set to private by default| |group_default_private*|boolean|whether groups created through the bot have privacy settings set to private by default| |show_private_info|boolean|whether the bot shows the system's own private information without a `-private` flag| |member_limit|int|read-only, defaults to 1000| |group_limit|int|read-only, defaults to 250| +|case_sensitive_proxy_tags|bool|whether the bot will match proxy tags matching only the case used in the trigger message| +|proxy_error_message_enabled|bool|whether the bot will show errors when proxying fails| +|hid_display_split|bool|whether 6-character ids will be shown by the bot as two 3-character parts separated by a `-`| +|hid_display_caps|bool|whether ids will be shown by the bot in uppercase| +|hid_list_padding|[ID padding format](#id-padding-format-enum)|whether the bot will pad 5-character ids in lists| +|proxy_switch|[proxy switch action](#proxy-switch-action-enum)|switch action the bot will take when proxying| +|name_format|string|format used for webhook names during proxying (defaults to `{name} {tag}`)| \* this *does not* affect members/groups created through the API - please specify privacy keys in the JSON payload instead +#### ID padding format enum + +|key|description| +|---|---| +|none|do not pad 5-character ids| +|left|add a padding space to the left of 5-character ids in lists| +|right|add a padding space to the right of 5-character ids in lists| + +#### Proxy switch action enum + +|key|description| +|---|---| +|off|do nothing| +|new|if the currently proxied member is not present in the current switch, log a new switch with this member| +|add|if the current switch has 0 members, log a new switch with the currently proxied member; otherwise, add the member to the current switch| + +### Public system settings model + +This model is used when querying system settings without authenticating, or for a system other than the one you are authenticated as. + +|key|type|notes| +|---|---|---| +|pings_enabled|boolean|whether proxied messages can be pinged using the :bell: reaction| +|latch_timeout|int?|seconds after which latch autoproxy will timeout (null/default is 6 hours, 0 is "never")| +|case_sensitive_proxy_tags|bool|whether the bot will match proxy tags matching only the case used in the trigger message| +|proxy_error_message_enabled|bool|whether the bot will show errors when proxying fails| +|hid_display_split|bool|whether 6-character ids will be shown by the bot as two 3-character parts separated by a `-`| +|hid_display_caps|bool|whether ids will be shown by the bot in uppercase| +|hid_list_padding|[ID padding format](#id-padding-format-enum)|whether the bot will pad 5-character ids in lists| +|proxy_switch|[proxy switch action](#proxy-switch-action-enum)|switch action the bot will take when proxying| +|name_format|string|format used for webhook names during proxying (defaults to `{name} {tag}`)| + ### System guild settings model |key|type|notes| From 539fc82342ae283517019e8040467aa67be0ffea Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 8 Jun 2025 19:52:37 +0000 Subject: [PATCH 53/80] feat(bot): lookup command messages --- PluralKit.Bot/ApplicationCommands/Message.cs | 19 +++++++- .../CommandSystem/Context/Context.cs | 17 ++++--- PluralKit.Bot/Commands/Message.cs | 44 ++++++++++++------- .../Services/CommandMessageService.cs | 20 +++------ PluralKit.Bot/Services/EmbedService.cs | 18 ++++++++ .../Repository/ModelRepository.Message.cs | 39 ++++++++++++++++ 6 files changed, 120 insertions(+), 37 deletions(-) diff --git a/PluralKit.Bot/ApplicationCommands/Message.cs b/PluralKit.Bot/ApplicationCommands/Message.cs index a425b31c..2f7be692 100644 --- a/PluralKit.Bot/ApplicationCommands/Message.cs +++ b/PluralKit.Bot/ApplicationCommands/Message.cs @@ -33,7 +33,10 @@ public class ApplicationCommandProxiedMessage var messageId = ctx.Event.Data!.TargetId!.Value; var msg = await ctx.Repository.GetFullMessage(messageId); if (msg == null) - throw Errors.MessageNotFound(messageId); + { + await QueryCommandMessage(ctx); + return; + } var showContent = true; var channel = await _rest.GetChannelOrNull(msg.Message.Channel); @@ -58,6 +61,20 @@ public class ApplicationCommandProxiedMessage await ctx.Reply(embeds: embeds.ToArray()); } + private async Task QueryCommandMessage(InteractionContext ctx) + { + var messageId = ctx.Event.Data!.TargetId!.Value; + var msg = await ctx.Repository.GetCommandMessage(messageId); + if (msg == null) + throw Errors.MessageNotFound(messageId); + + var embeds = new List(); + + embeds.Add(await _embeds.CreateCommandMessageInfoEmbed(msg, true)); + + await ctx.Reply(embeds: embeds.ToArray()); + } + public async Task DeleteMessage(InteractionContext ctx) { var messageId = ctx.Event.Data!.TargetId!.Value; diff --git a/PluralKit.Bot/CommandSystem/Context/Context.cs b/PluralKit.Bot/CommandSystem/Context/Context.cs index b18dd9a8..2dd2dce9 100644 --- a/PluralKit.Bot/CommandSystem/Context/Context.cs +++ b/PluralKit.Bot/CommandSystem/Context/Context.cs @@ -101,12 +101,17 @@ public class Context AllowedMentions = mentions ?? new AllowedMentions() }); - // if (embed != null) - // { - // Sensitive information that might want to be deleted by :x: reaction is typically in an embed format (member cards, for example) - // but since we can, we just store all sent messages for possible deletion - await _commandMessageService.RegisterMessage(msg.Id, Guild?.Id ?? 0, msg.ChannelId, Author.Id); - // } + // store log of sent message, so it can be queried or deleted later + // skip DMs as DM messages can always be deleted + if (Guild != null) + await Repository.AddCommandMessage(new Core.CommandMessage + { + Mid = msg.Id, + Guild = Guild!.Id, + Channel = Channel.Id, + Sender = Author.Id, + OriginalMid = Message.Id, + }); return msg; } diff --git a/PluralKit.Bot/Commands/Message.cs b/PluralKit.Bot/Commands/Message.cs index 94602a06..ec34cea9 100644 --- a/PluralKit.Bot/Commands/Message.cs +++ b/PluralKit.Bot/Commands/Message.cs @@ -347,13 +347,8 @@ public class ProxiedMessage var message = await ctx.Repository.GetFullMessage(messageId.Value); if (message == null) { - if (isDelete) - { - await DeleteCommandMessage(ctx, messageId.Value); - return; - } - - throw Errors.MessageNotFound(messageId.Value); + await GetCommandMessage(ctx, messageId.Value, isDelete); + return; } var showContent = true; @@ -448,20 +443,35 @@ public class ProxiedMessage await ctx.Reply(embed: await _embeds.CreateMessageInfoEmbed(message, showContent, ctx.Config)); } - private async Task DeleteCommandMessage(Context ctx, ulong messageId) + private async Task GetCommandMessage(Context ctx, ulong messageId, bool isDelete) { - var cmessage = await ctx.Services.Resolve().GetCommandMessage(messageId); - if (cmessage == null) + var msg = await _repo.GetCommandMessage(messageId); + if (msg == null) throw Errors.MessageNotFound(messageId); - if (cmessage!.AuthorId != ctx.Author.Id) - throw new PKError("You can only delete command messages queried by this account."); + if (isDelete) + { + if (msg.Sender != ctx.Author.Id) + throw new PKError("You can only delete command messages queried by this account."); - await ctx.Rest.DeleteMessage(cmessage.ChannelId, messageId); + await ctx.Rest.DeleteMessage(msg.Channel, messageId); - if (ctx.Guild != null) - await ctx.Rest.DeleteMessage(ctx.Message); - else - await ctx.Rest.CreateReaction(ctx.Message.ChannelId, ctx.Message.Id, new Emoji { Name = Emojis.Success }); + if (ctx.Guild != null) + await ctx.Rest.DeleteMessage(ctx.Message); + else + await ctx.Rest.CreateReaction(ctx.Message.ChannelId, ctx.Message.Id, new Emoji { Name = Emojis.Success }); + + return; + } + + var showContent = true; + + var channel = await _rest.GetChannelOrNull(msg.Channel); + if (channel == null) + showContent = false; + else if (!await ctx.CheckPermissionsInGuildChannel(channel, PermissionSet.ViewChannel)) + showContent = false; + + await ctx.Reply(embed: await _embeds.CreateCommandMessageInfoEmbed(msg, showContent)); } } \ No newline at end of file diff --git a/PluralKit.Bot/Services/CommandMessageService.cs b/PluralKit.Bot/Services/CommandMessageService.cs index 440670ab..684daf86 100644 --- a/PluralKit.Bot/Services/CommandMessageService.cs +++ b/PluralKit.Bot/Services/CommandMessageService.cs @@ -9,29 +9,23 @@ namespace PluralKit.Bot; public class CommandMessageService { private readonly RedisService _redis; + private readonly ModelRepository _repo; private readonly ILogger _logger; private static readonly TimeSpan CommandMessageRetention = TimeSpan.FromHours(24); - public CommandMessageService(RedisService redis, IClock clock, ILogger logger) + public CommandMessageService(RedisService redis, ModelRepository repo, IClock clock, ILogger logger) { _redis = redis; + _repo = repo; _logger = logger.ForContext(); } - public async Task RegisterMessage(ulong messageId, ulong guildId, ulong channelId, ulong authorId) - { - if (_redis.Connection == null) return; - - _logger.Debug( - "Registering command response {MessageId} from author {AuthorId} in {ChannelId}", - messageId, authorId, channelId - ); - - await _redis.Connection.GetDatabase().StringSetAsync("command_message:" + messageId.ToString(), $"{authorId}-{channelId}-{guildId}", expiry: CommandMessageRetention); - } - public async Task GetCommandMessage(ulong messageId) { + var repoMsg = await _repo.GetCommandMessage(messageId); + if (repoMsg != null) + return new CommandMessage(repoMsg.Sender, repoMsg.Channel, repoMsg.Guild); + var str = await _redis.Connection.GetDatabase().StringGetAsync(messageId.ToString()); if (str.HasValue) { diff --git a/PluralKit.Bot/Services/EmbedService.cs b/PluralKit.Bot/Services/EmbedService.cs index 178d1339..186e3a45 100644 --- a/PluralKit.Bot/Services/EmbedService.cs +++ b/PluralKit.Bot/Services/EmbedService.cs @@ -420,6 +420,24 @@ public class EmbedService return eb.Build(); } + public async Task CreateCommandMessageInfoEmbed(Core.CommandMessage msg, bool showContent) + { + var content = "*(command message deleted or inaccessible)*"; + if (showContent) + { + var discordMessage = await _rest.GetMessageOrNull(msg.Channel, msg.OriginalMid); + if (discordMessage != null) + content = discordMessage.Content; + } + + return new EmbedBuilder() + .Title("Command response message") + .Description(content) + .Field(new("Original message", $"https://discord.com/channels/{msg.Guild}/{msg.Channel}/{msg.OriginalMid}", true)) + .Field(new("Sent by", $"<@{msg.Sender}>", true)) + .Build(); + } + public Task CreateFrontPercentEmbed(FrontBreakdown breakdown, PKSystem system, PKGroup group, DateTimeZone tz, LookupContext ctx, string embedTitle, bool ignoreNoFronters, bool showFlat) diff --git a/PluralKit.Core/Database/Repository/ModelRepository.Message.cs b/PluralKit.Core/Database/Repository/ModelRepository.Message.cs index 706e3a13..05313c89 100644 --- a/PluralKit.Core/Database/Repository/ModelRepository.Message.cs +++ b/PluralKit.Core/Database/Repository/ModelRepository.Message.cs @@ -42,12 +42,37 @@ public partial class ModelRepository }; } + public async Task AddCommandMessage(CommandMessage msg) + { + var query = new Query("command_messages").AsInsert(new + { + mid = msg.Mid, + guild = msg.Guild, + channel = msg.Channel, + sender = msg.Sender, + original_mid = msg.OriginalMid + }); + await _db.ExecuteQuery(query, messages: true); + + _logger.Debug("Stored command message {@StoredMessage} in channel {Channel}", msg, msg.Channel); + } + + public Task GetCommandMessage(ulong id) + => _db.QueryFirst(new Query("command_messages").Where("mid", id), messages: true); + public async Task DeleteMessage(ulong id) { var query = new Query("messages").AsDelete().Where("mid", id); var rowCount = await _db.ExecuteQuery(query, messages: true); if (rowCount > 0) _logger.Information("Deleted message {MessageId} from database", id); + else + { + var cquery = new Query("command_messages").AsDelete().Where("mid", id); + var crowCount = await _db.ExecuteQuery(query, messages: true); + if (crowCount > 0) + _logger.Information("Deleted command message {MessageId} from database", id); + } } public async Task DeleteMessagesBulk(IReadOnlyCollection ids) @@ -59,5 +84,19 @@ public partial class ModelRepository if (rowCount > 0) _logger.Information("Bulk deleted messages ({FoundCount} found) from database: {MessageIds}", rowCount, ids); + var cquery = new Query("command_messages").AsDelete().WhereIn("mid", ids.Select(id => (long)id).ToArray()); + var crowCount = await _db.ExecuteQuery(query, messages: true); + if (crowCount > 0) + _logger.Information("Bulk deleted command messages ({FoundCount} found) from database: {MessageIds}", rowCount, + ids); } +} + +public class CommandMessage +{ + public ulong Mid { get; set; } + public ulong Guild { get; set; } + public ulong Channel { get; set; } + public ulong Sender { get; set; } + public ulong OriginalMid { get; set; } } \ No newline at end of file From 6fea94f990df5c537daacfdb0c2e35e3a0d944a8 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 8 Jun 2025 20:27:35 +0000 Subject: [PATCH 54/80] feat(api): try harder at checking whether user is in guild --- PluralKit.API/ApiConfig.cs | 1 + .../Controllers/v2/DiscordControllerV2.cs | 8 ++-- .../Repository/ModelRepository.Guild.cs | 45 ++++++++++++++++--- .../Database/Utils/DatabaseMigrator.cs | 2 +- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/PluralKit.API/ApiConfig.cs b/PluralKit.API/ApiConfig.cs index 46556a79..8da70196 100644 --- a/PluralKit.API/ApiConfig.cs +++ b/PluralKit.API/ApiConfig.cs @@ -7,4 +7,5 @@ public class ApiConfig public string? ClientSecret { get; set; } public bool TrustAuth { get; set; } = false; public string? AvatarServiceUrl { get; set; } + public bool SearchGuildSettings = false; } \ No newline at end of file diff --git a/PluralKit.API/Controllers/v2/DiscordControllerV2.cs b/PluralKit.API/Controllers/v2/DiscordControllerV2.cs index 340c8698..7547a751 100644 --- a/PluralKit.API/Controllers/v2/DiscordControllerV2.cs +++ b/PluralKit.API/Controllers/v2/DiscordControllerV2.cs @@ -20,7 +20,7 @@ public class DiscordControllerV2: PKControllerBase if (ContextFor(system) != LookupContext.ByOwner) throw Errors.GenericMissingPermissions; - var settings = await _repo.GetSystemGuild(guild_id, system.Id, false); + var settings = await _repo.GetSystemGuild(guild_id, system.Id, false, _config.SearchGuildSettings); if (settings == null) throw Errors.SystemGuildNotFound; @@ -34,7 +34,7 @@ public class DiscordControllerV2: PKControllerBase if (ContextFor(system) != LookupContext.ByOwner) throw Errors.GenericMissingPermissions; - var settings = await _repo.GetSystemGuild(guild_id, system.Id, false); + var settings = await _repo.GetSystemGuild(guild_id, system.Id, false, _config.SearchGuildSettings); if (settings == null) throw Errors.SystemGuildNotFound; @@ -58,7 +58,7 @@ public class DiscordControllerV2: PKControllerBase if (member.System != system.Id) throw Errors.NotOwnMemberError; - var settings = await _repo.GetMemberGuild(guild_id, member.Id, false); + var settings = await _repo.GetMemberGuild(guild_id, member.Id, false, _config.SearchGuildSettings ? system.Id : null); if (settings == null) throw Errors.MemberGuildNotFound; @@ -75,7 +75,7 @@ public class DiscordControllerV2: PKControllerBase if (member.System != system.Id) throw Errors.NotOwnMemberError; - var settings = await _repo.GetMemberGuild(guild_id, member.Id, false); + var settings = await _repo.GetMemberGuild(guild_id, member.Id, false, _config.SearchGuildSettings ? system.Id : null); if (settings == null) throw Errors.MemberGuildNotFound; diff --git a/PluralKit.Core/Database/Repository/ModelRepository.Guild.cs b/PluralKit.Core/Database/Repository/ModelRepository.Guild.cs index f0d70900..361a2bf3 100644 --- a/PluralKit.Core/Database/Repository/ModelRepository.Guild.cs +++ b/PluralKit.Core/Database/Repository/ModelRepository.Guild.cs @@ -19,16 +19,40 @@ public partial class ModelRepository } - public Task GetSystemGuild(ulong guild, SystemId system, bool defaultInsert = true) + public async Task GetSystemGuild(ulong guild, SystemId system, bool defaultInsert = true, bool search = false) { if (!defaultInsert) - return _db.QueryFirst(new Query("system_guild") + { + var simpleRes = await _db.QueryFirst(new Query("system_guild") .Where("guild", guild) .Where("system", system) ); + if (simpleRes != null || !search) + return simpleRes; + + var accounts = await GetSystemAccounts(system); + + var searchRes = await _db.QueryFirst( + "select exists(select 1 from command_messages where guild = @guild and sender = any(@accounts))", + new { guild = guild, accounts = accounts.Select(u => (long)u).ToArray() }, + queryName: "find_system_from_commands", + messages: true + ); + + if (!searchRes) + searchRes = await _db.QueryFirst( + "select exists(select 1 from command_messages where guild = @guild and sender = any(@accounts))", + new { guild = guild, accounts = accounts.Select(u => (long)u).ToArray() }, + queryName: "find_system_from_messages", + messages: true + ); + + if (!searchRes) + return null; + } var query = new Query("system_guild").AsInsert(new { guild, system }); - return _db.QueryFirst(query, + return await _db.QueryFirst(query, "on conflict (guild, system) do update set guild = $1, system = $2 returning *" ); } @@ -42,16 +66,25 @@ public partial class ModelRepository return settings; } - public Task GetMemberGuild(ulong guild, MemberId member, bool defaultInsert = true) + public async Task GetMemberGuild(ulong guild, MemberId member, bool defaultInsert = true, SystemId? search = null) { if (!defaultInsert) - return _db.QueryFirst(new Query("member_guild") + { + var simpleRes = await _db.QueryFirst(new Query("member_guild") .Where("guild", guild) .Where("member", member) ); + if (simpleRes != null || !search.HasValue) + return simpleRes; + + var systemConfig = await GetSystemGuild(guild, search.Value, defaultInsert: false, search: true); + + if (systemConfig == null) + return null; + } var query = new Query("member_guild").AsInsert(new { guild, member }); - return _db.QueryFirst(query, + return await _db.QueryFirst(query, "on conflict (guild, member) do update set guild = $1, member = $2 returning *" ); } diff --git a/PluralKit.Core/Database/Utils/DatabaseMigrator.cs b/PluralKit.Core/Database/Utils/DatabaseMigrator.cs index 11c6e2a9..d4d58093 100644 --- a/PluralKit.Core/Database/Utils/DatabaseMigrator.cs +++ b/PluralKit.Core/Database/Utils/DatabaseMigrator.cs @@ -9,7 +9,7 @@ namespace PluralKit.Core; internal class DatabaseMigrator { private const string RootPath = "PluralKit.Core.Database"; // "resource path" root for SQL files - private const int TargetSchemaVersion = 51; + private const int TargetSchemaVersion = 52; private readonly ILogger _logger; public DatabaseMigrator(ILogger logger) From 2f334ec8d7975c9bde1d210925929aecb638a649 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 8 Jun 2025 20:46:57 +0000 Subject: [PATCH 55/80] chore: remove website banner --- docs/content/.vuepress/config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/content/.vuepress/config.js b/docs/content/.vuepress/config.js index 25e14ac0..5bdc945e 100644 --- a/docs/content/.vuepress/config.js +++ b/docs/content/.vuepress/config.js @@ -82,7 +82,6 @@ module.exports = { }, ["https://discord.gg/PczBt78", "Join the support server"], ], - pkBannerContent: "PluralKit needs your help! Check out our latest funding update for details.", }, plugins: [ From 3cefa364b12aef9a226f58ac7e9ec0d537bb83d3 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 8 Jun 2025 22:38:19 +0000 Subject: [PATCH 56/80] fix(api): run cors logic before auth --- crates/api/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 57ecbb0a..e3a201cb 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -136,12 +136,12 @@ fn router(ctx: ApiContext) -> Router { .layer(middleware::ratelimit::ratelimiter(middleware::ratelimit::do_request_ratelimited)) // this sucks .layer(axum::middleware::from_fn(middleware::ignore_invalid_routes::ignore_invalid_routes)) - .layer(axum::middleware::from_fn(middleware::cors::cors)) .layer(axum::middleware::from_fn(middleware::logger::logger)) .layer(axum::middleware::from_fn_with_state(ctx.clone(), middleware::params::params)) .layer(axum::middleware::from_fn_with_state(ctx.clone(), middleware::auth::auth)) + .layer(axum::middleware::from_fn(middleware::cors::cors)) .layer(tower_http::catch_panic::CatchPanicLayer::custom(util::handle_panic)) .with_state(ctx) From bea41ab8a0ade19d94c5e111b20794ad8488e374 Mon Sep 17 00:00:00 2001 From: alyssa Date: Mon, 9 Jun 2025 05:39:44 +0000 Subject: [PATCH 57/80] fix(api): allow hids in uppercase and with dashes; document this --- crates/api/src/middleware/params.rs | 11 ++++++++++- docs/content/api/models.md | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/api/src/middleware/params.rs b/crates/api/src/middleware/params.rs index f1219614..ce4850bc 100644 --- a/crates/api/src/middleware/params.rs +++ b/crates/api/src/middleware/params.rs @@ -13,6 +13,15 @@ use crate::auth::AuthState; use crate::{util::json_err, ApiContext}; use pluralkit_models::PKSystem; +// move this somewhere else +fn parse_hid(hid: &str) -> String { + if hid.len() > 7 || hid.len() < 5 { + hid.to_string() + } else { + hid.to_lowercase().replace("-", "") + } +} + pub async fn params(State(ctx): State, mut req: Request, next: Next) -> Response { let pms = match req.extensions().get::() { None => Vec::new(), @@ -86,7 +95,7 @@ pub async fn params(State(ctx): State, mut req: Request, next: Next) Err(_) => sqlx::query_as::( "select * from systems where hid = $1", ) - .bind(id), + .bind(parse_hid(id)), } .fetch_optional(&ctx.db) .await diff --git a/docs/content/api/models.md b/docs/content/api/models.md index 23ecdc3c..558cb622 100644 --- a/docs/content/api/models.md +++ b/docs/content/api/models.md @@ -16,6 +16,8 @@ Privacy objects (`privacy` key in models) contain values "private" or "public". Every PluralKit entity has two IDs: a short (5 or 6 character) ID and a longer UUID. The short ID is unique across the resource (a member can have the same short ID as a system, for example), while the UUID is consistent for the lifetime of the entity and globally unique across the bot. +The PluralKit Discord bot can be configured to display short IDs in uppercase, or (in the case of 6-character IDs) as two parts of 3 characters separated by a dash (for example, `EXA-MPL`). For convenience, IDs are accepted by the API in any format displayable by the bot. + ### System model |key|type|notes| From 94b90470d5fd1de0fab6004ed74457de5c21e645 Mon Sep 17 00:00:00 2001 From: alyssa Date: Mon, 9 Jun 2025 20:14:38 +0000 Subject: [PATCH 58/80] fix(api): forgot about discord id lookup --- crates/api/src/middleware/params.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/api/src/middleware/params.rs b/crates/api/src/middleware/params.rs index ce4850bc..42a41122 100644 --- a/crates/api/src/middleware/params.rs +++ b/crates/api/src/middleware/params.rs @@ -87,15 +87,22 @@ pub async fn params(State(ctx): State, mut req: Request, next: Next) } } id => { + println!("a {id}"); match match Uuid::parse_str(id) { Ok(uuid) => sqlx::query_as::( "select * from systems where uuid = $1", ) .bind(uuid), - Err(_) => sqlx::query_as::( - "select * from systems where hid = $1", - ) - .bind(parse_hid(id)), + Err(_) => match id.parse::() { + Ok(parsed) => sqlx::query_as::( + "select * from systems where id = (select system from accounts where uid = $1)" + ) + .bind(parsed), + Err(_) => sqlx::query_as::( + "select * from systems where hid = $1", + ) + .bind(parse_hid(id)) + }, } .fetch_optional(&ctx.db) .await From 320c711c13d89b404013243e26f5b79ba1f6c7c7 Mon Sep 17 00:00:00 2001 From: alyssa Date: Fri, 13 Jun 2025 21:49:08 +0000 Subject: [PATCH 59/80] feat(docs): add note about 204 on /systems/:id/switches endpoint --- docs/content/api/endpoints.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/api/endpoints.md b/docs/content/api/endpoints.md index fa249fff..a40b2dd8 100644 --- a/docs/content/api/endpoints.md +++ b/docs/content/api/endpoints.md @@ -279,6 +279,8 @@ GET `/systems/{systemRef}/fronters` Returns a [switch object](/api/models#switch-model) containing a list of member objects. +If the target system has no registered switches, returns 204 status code with no content. + ### Create Switch POST `/systems/{systemRef}/switches` From f8b5e56fdce10e6eef67921a2fe156d7a576b555 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 21 Jun 2025 21:14:23 +0000 Subject: [PATCH 60/80] fix: add missing migration --- PluralKit.Core/Database/Migrations/52.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 PluralKit.Core/Database/Migrations/52.sql diff --git a/PluralKit.Core/Database/Migrations/52.sql b/PluralKit.Core/Database/Migrations/52.sql new file mode 100644 index 00000000..db6e7541 --- /dev/null +++ b/PluralKit.Core/Database/Migrations/52.sql @@ -0,0 +1,18 @@ +-- database version 52 +-- messages db updates + +create index concurrently messages_by_original on messages(original_mid); +create index concurrently messages_by_sender on messages(sender); + +create table command_messages ( + mid bigint primary key, + channel bigint not null, + guild bigint not null, + sender bigint not null, + original_mid bigint not null +); + +create index concurrently command_messages_by_original on command_messages(original_mid); +create index concurrently command_messages_by_sender on command_messages(sender); + +update info set schema_version = 52; From a81841f9f59e12adb10d68ad0c4ed2c21d853a1e Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 21 Jun 2025 21:31:47 +0000 Subject: [PATCH 61/80] fix: remove "concurrently" from migration --- PluralKit.Core/Database/Migrations/52.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/PluralKit.Core/Database/Migrations/52.sql b/PluralKit.Core/Database/Migrations/52.sql index db6e7541..396ee838 100644 --- a/PluralKit.Core/Database/Migrations/52.sql +++ b/PluralKit.Core/Database/Migrations/52.sql @@ -1,8 +1,8 @@ -- database version 52 -- messages db updates -create index concurrently messages_by_original on messages(original_mid); -create index concurrently messages_by_sender on messages(sender); +create index messages_by_original on messages(original_mid); +create index messages_by_sender on messages(sender); create table command_messages ( mid bigint primary key, @@ -12,7 +12,7 @@ create table command_messages ( original_mid bigint not null ); -create index concurrently command_messages_by_original on command_messages(original_mid); -create index concurrently command_messages_by_sender on command_messages(sender); +create index command_messages_by_original on command_messages(original_mid); +create index command_messages_by_sender on command_messages(sender); update info set schema_version = 52; From 0f840914d737e015973052723238febd337d4b6e Mon Sep 17 00:00:00 2001 From: Kat <65649991+00-kat@users.noreply.github.com> Date: Sat, 28 Jun 2025 12:41:33 +0000 Subject: [PATCH 62/80] fix(docs): mark member guild settings model's `guild_id` as optional --- docs/content/api/models.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/api/models.md b/docs/content/api/models.md index 558cb622..5d5938d0 100644 --- a/docs/content/api/models.md +++ b/docs/content/api/models.md @@ -196,7 +196,7 @@ This model is used when querying system settings without authenticating, or for |key|type|notes| |---|---|---| -|guild_id|snowflake|only sent if the guild ID isn't already known (in dispatch payloads)| +|?guild_id|snowflake|only sent if the guild ID isn't already known (in dispatch payloads)| |display_name|?string|100-character limit| |avatar_url|?string|256-character limit, must be a publicly-accessible URL| |keep_proxy|?boolean|| From 91d5ae6dd7f98ba768f731160a0713d4b280726d Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 5 Jul 2025 17:28:57 +0000 Subject: [PATCH 63/80] fix(api): default value for hid_pad_format, content-type for /systems/:id/settings endpoint --- crates/api/src/endpoints/system.rs | 43 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/crates/api/src/endpoints/system.rs b/crates/api/src/endpoints/system.rs index 21672db2..e510f7c5 100644 --- a/crates/api/src/endpoints/system.rs +++ b/crates/api/src/endpoints/system.rs @@ -2,7 +2,7 @@ use axum::{ extract::State, http::StatusCode, response::{IntoResponse, Response}, - Extension, + Extension, Json, }; use serde_json::json; use sqlx::Postgres; @@ -19,7 +19,7 @@ pub async fn get_system_settings( ) -> Response { let access_level = auth.access_level_for(&system); - let config = match sqlx::query_as::( + let mut config = match sqlx::query_as::( "select * from system_config where system = $1", ) .bind(system.id) @@ -46,23 +46,24 @@ pub async fn get_system_settings( } }; - ( - StatusCode::OK, - serde_json::to_string(&match access_level { - PrivacyLevel::Private => config.to_json(), - PrivacyLevel::Public => json!({ - "pings_enabled": config.pings_enabled, - "latch_timeout": config.latch_timeout, - "case_sensitive_proxy_tags": config.case_sensitive_proxy_tags, - "proxy_error_message_enabled": config.proxy_error_message_enabled, - "hid_display_split": config.hid_display_split, - "hid_display_caps": config.hid_display_caps, - "hid_list_padding": config.hid_list_padding, - "proxy_switch": config.proxy_switch, - "name_format": config.name_format, - }), - }) - .unwrap(), - ) - .into_response() + // fix this + if config.name_format.is_none() { + config.name_format = Some("{name} {tag}".to_string()); + } + + Json(&match access_level { + PrivacyLevel::Private => config.to_json(), + PrivacyLevel::Public => json!({ + "pings_enabled": config.pings_enabled, + "latch_timeout": config.latch_timeout, + "case_sensitive_proxy_tags": config.case_sensitive_proxy_tags, + "proxy_error_message_enabled": config.proxy_error_message_enabled, + "hid_display_split": config.hid_display_split, + "hid_display_caps": config.hid_display_caps, + "hid_list_padding": config.hid_list_padding, + "proxy_switch": config.proxy_switch, + "name_format": config.name_format, + }), + }) + .into_response() } From 5f08a3ed132bcfdc731372caf64a3d95bc55b835 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 5 Jul 2025 17:31:02 +0000 Subject: [PATCH 64/80] fix(api): use correct json key for errors --- crates/api/src/middleware/params.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/api/src/middleware/params.rs b/crates/api/src/middleware/params.rs index 42a41122..06a76f64 100644 --- a/crates/api/src/middleware/params.rs +++ b/crates/api/src/middleware/params.rs @@ -29,7 +29,7 @@ pub async fn params(State(ctx): State, mut req: Request, next: Next) _ => { return json_err( StatusCode::BAD_REQUEST, - r#"{"error": "400: Bad Request", "code": 0}"#.to_string(), + r#"{"message":"400: Bad Request","code": 0}"#.to_string(), ) .into() } @@ -47,7 +47,7 @@ pub async fn params(State(ctx): State, mut req: Request, next: Next) else { return json_err( StatusCode::UNAUTHORIZED, - r#"{"error": "401: Missing or invalid Authorization header", "code": 0}"#.to_string(), + r#"{"message":"401: Missing or invalid Authorization header","code": 0}"#.to_string(), ) .into(); }; From fdfa2baaef12b66e428a6d432fefe3616a7dd646 Mon Sep 17 00:00:00 2001 From: alyssa Date: Sun, 6 Jul 2025 12:58:52 +0000 Subject: [PATCH 65/80] chore: bump max switch limit in import --- PluralKit.Core/Utils/BulkImporter/PluralKitImport.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PluralKit.Core/Utils/BulkImporter/PluralKitImport.cs b/PluralKit.Core/Utils/BulkImporter/PluralKitImport.cs index 816dd618..f972c728 100644 --- a/PluralKit.Core/Utils/BulkImporter/PluralKitImport.cs +++ b/PluralKit.Core/Utils/BulkImporter/PluralKitImport.cs @@ -205,7 +205,7 @@ public partial class BulkImporter ? existingSwitches.Select(sw => sw.Id).Max() : (SwitchId?)null; - if (switches.Count > 10000) + if (switches.Count > 100000) throw new ImportException("Too many switches present in import file."); // Import switch definitions From 497bee9487e53a1c521665a617ff22d61e0faff8 Mon Sep 17 00:00:00 2001 From: alyssa Date: Mon, 14 Jul 2025 13:10:33 +0000 Subject: [PATCH 66/80] fix: move old command_messages table in migration 52 --- PluralKit.Core/Database/Migrations/52.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/PluralKit.Core/Database/Migrations/52.sql b/PluralKit.Core/Database/Migrations/52.sql index 396ee838..7c603021 100644 --- a/PluralKit.Core/Database/Migrations/52.sql +++ b/PluralKit.Core/Database/Migrations/52.sql @@ -4,6 +4,9 @@ create index messages_by_original on messages(original_mid); create index messages_by_sender on messages(sender); +-- remove old table from database version 11 +alter table command_messages rename to command_messages_old; + create table command_messages ( mid bigint primary key, channel bigint not null, From 3ad28eb15f5b01d5227cd6f8f1219e61f892128f Mon Sep 17 00:00:00 2001 From: asleepyskye Date: Wed, 16 Jul 2025 11:48:48 -0400 Subject: [PATCH 67/80] fix(bot): generate and attach color previews --- .../CommandSystem/Context/Context.cs | 7 ++++-- PluralKit.Bot/Commands/Groups.cs | 10 +++++--- PluralKit.Bot/Commands/MemberEdit.cs | 10 +++++--- PluralKit.Bot/Commands/SystemEdit.cs | 10 +++++--- PluralKit.Bot/Utils/MiscUtils.cs | 25 ++++++++++++++++++- 5 files changed, 47 insertions(+), 15 deletions(-) diff --git a/PluralKit.Bot/CommandSystem/Context/Context.cs b/PluralKit.Bot/CommandSystem/Context/Context.cs index 2dd2dce9..d70ee7d2 100644 --- a/PluralKit.Bot/CommandSystem/Context/Context.cs +++ b/PluralKit.Bot/CommandSystem/Context/Context.cs @@ -82,7 +82,7 @@ public class Context internal readonly ModelRepository Repository; internal readonly RedisService Redis; - public async Task Reply(string text = null, Embed embed = null, AllowedMentions? mentions = null) + public async Task Reply(string text = null, Embed embed = null, AllowedMentions? mentions = null, MultipartFile[]? files = null) { var botPerms = await BotPermissions; @@ -92,6 +92,9 @@ public class Context if (embed != null && !botPerms.HasFlag(PermissionSet.EmbedLinks)) throw new PKError("PluralKit does not have permission to send embeds in this channel. Please ensure I have the **Embed Links** permission enabled."); + + if (files != null && !botPerms.HasFlag(PermissionSet.AttachFiles)) + throw new PKError("PluralKit does not have permission to attach files in this channel. Please ensure I have the **Attach Files** permission enabled."); var msg = await Rest.CreateMessage(Channel.Id, new MessageRequest { @@ -99,7 +102,7 @@ public class Context Embeds = embed != null ? new[] { embed } : null, // Default to an empty allowed mentions object instead of null (which means no mentions allowed) AllowedMentions = mentions ?? new AllowedMentions() - }); + }, files: files); // store log of sent message, so it can be queried or deleted later // skip DMs as DM messages can always be deleted diff --git a/PluralKit.Bot/Commands/Groups.cs b/PluralKit.Bot/Commands/Groups.cs index a314b248..45c5f8b4 100644 --- a/PluralKit.Bot/Commands/Groups.cs +++ b/PluralKit.Bot/Commands/Groups.cs @@ -443,10 +443,11 @@ public class Groups await ctx.Reply(embed: new EmbedBuilder() .Title("Group color") .Color(target.Color.ToDiscordColor()) - .Thumbnail(new Embed.EmbedThumbnail($"https://fakeimg.pl/256x256/{target.Color}/?text=%20")) + .Thumbnail(new Embed.EmbedThumbnail($"attachment://color.gif")) .Description($"This group's color is **#{target.Color}**." + (isOwnSystem ? $" To clear it, type `{ctx.DefaultPrefix}group {target.Reference(ctx)} color -clear`." : "")) - .Build()); + .Build(), + files: [MiscUtils.GenerateColorPreview(target.Color)]); return; } @@ -471,8 +472,9 @@ public class Groups await ctx.Reply(embed: new EmbedBuilder() .Title($"{Emojis.Success} Group color changed.") .Color(color.ToDiscordColor()) - .Thumbnail(new Embed.EmbedThumbnail($"https://fakeimg.pl/256x256/{color}/?text=%20")) - .Build()); + .Thumbnail(new Embed.EmbedThumbnail($"attachment://color.gif")) + .Build(), + files: [MiscUtils.GenerateColorPreview(color)]); } } diff --git a/PluralKit.Bot/Commands/MemberEdit.cs b/PluralKit.Bot/Commands/MemberEdit.cs index 0585abd7..525d60ab 100644 --- a/PluralKit.Bot/Commands/MemberEdit.cs +++ b/PluralKit.Bot/Commands/MemberEdit.cs @@ -308,10 +308,11 @@ public class MemberEdit await ctx.Reply(embed: new EmbedBuilder() .Title("Member color") .Color(target.Color.ToDiscordColor()) - .Thumbnail(new Embed.EmbedThumbnail($"https://fakeimg.pl/256x256/{target.Color}/?text=%20")) + .Thumbnail(new Embed.EmbedThumbnail($"attachment://color.gif")) .Description($"This member's color is **#{target.Color}**." + (isOwnSystem ? $" To clear it, type `{ctx.DefaultPrefix}member {target.Reference(ctx)} color -clear`." : "")) - .Build()); + .Build(), + files: [MiscUtils.GenerateColorPreview(target.Color)]); return; } @@ -336,8 +337,9 @@ public class MemberEdit await ctx.Reply(embed: new EmbedBuilder() .Title($"{Emojis.Success} Member color changed.") .Color(color.ToDiscordColor()) - .Thumbnail(new Embed.EmbedThumbnail($"https://fakeimg.pl/256x256/{color}/?text=%20")) - .Build()); + .Thumbnail(new Embed.EmbedThumbnail($"attachment://color.gif")) + .Build(), + files: [MiscUtils.GenerateColorPreview(color)]); } } diff --git a/PluralKit.Bot/Commands/SystemEdit.cs b/PluralKit.Bot/Commands/SystemEdit.cs index 42d6ab5c..3af4a639 100644 --- a/PluralKit.Bot/Commands/SystemEdit.cs +++ b/PluralKit.Bot/Commands/SystemEdit.cs @@ -241,10 +241,11 @@ public class SystemEdit await ctx.Reply(embed: new EmbedBuilder() .Title("System color") .Color(target.Color.ToDiscordColor()) - .Thumbnail(new Embed.EmbedThumbnail($"https://fakeimg.pl/256x256/{target.Color}/?text=%20")) + .Thumbnail(new Embed.EmbedThumbnail($"attachment://color.gif")) .Description( $"This system's color is **#{target.Color}**." + (isOwnSystem ? $" To clear it, type `{ctx.DefaultPrefix}s color -clear`." : "")) - .Build()); + .Build(), + files: [MiscUtils.GenerateColorPreview(target.Color)]); return; } @@ -269,8 +270,9 @@ public class SystemEdit await ctx.Reply(embed: new EmbedBuilder() .Title($"{Emojis.Success} System color changed.") .Color(color.ToDiscordColor()) - .Thumbnail(new Embed.EmbedThumbnail($"https://fakeimg.pl/256x256/{color}/?text=%20")) - .Build()); + .Thumbnail(new Embed.EmbedThumbnail($"attachment://color.gif")) + .Build(), + files: [MiscUtils.GenerateColorPreview(color)]); } } diff --git a/PluralKit.Bot/Utils/MiscUtils.cs b/PluralKit.Bot/Utils/MiscUtils.cs index 5ee99901..8cc24029 100644 --- a/PluralKit.Bot/Utils/MiscUtils.cs +++ b/PluralKit.Bot/Utils/MiscUtils.cs @@ -1,7 +1,8 @@ using System.Net; using System.Net.Sockets; - +using System.Globalization; using Myriad.Rest.Exceptions; +using Myriad.Rest.Types; using Newtonsoft.Json; @@ -102,4 +103,26 @@ public static class MiscUtils return true; } + + public static MultipartFile GenerateColorPreview(string color) + { + //generate a 128x128 solid color gif from bytes + //image data is a 1x1 pixel, using the background color to fill the rest of the canvas + var imgBytes = new byte[] + { + 0x47, 0x49, 0x46, 0x38, 0x39, 0x61, // Header + 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x00, // Logical Screen Descriptor + 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, // Global Color Table + 0x21, 0xF9, 0x04, 0x08, 0x00, 0x00, 0x00, 0x00, // Graphics Control Extension + 0x2C, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, // Image Descriptor + 0x02, 0x02, 0x4C, 0x01, 0x00, // Image Data + 0x3B // Trailer + }; //indices 13, 14 and 15 are the R, G, and B values respectively + + imgBytes[13] = byte.Parse(color.Substring(0, 2), NumberStyles.HexNumber); + imgBytes[14] = byte.Parse(color.Substring(2, 2), NumberStyles.HexNumber); + imgBytes[15] = byte.Parse(color.Substring(4, 2), NumberStyles.HexNumber); + + return new MultipartFile("color.gif", new MemoryStream(imgBytes), null, null, null); + } } \ No newline at end of file From c5e5c5a1f192117e9afc1a055324136608ffed42 Mon Sep 17 00:00:00 2001 From: asleepyskye Date: Wed, 16 Jul 2025 12:01:01 -0400 Subject: [PATCH 68/80] chore(bot): format --- PluralKit.Bot/CommandSystem/Context/Context.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PluralKit.Bot/CommandSystem/Context/Context.cs b/PluralKit.Bot/CommandSystem/Context/Context.cs index d70ee7d2..9bab740e 100644 --- a/PluralKit.Bot/CommandSystem/Context/Context.cs +++ b/PluralKit.Bot/CommandSystem/Context/Context.cs @@ -92,7 +92,7 @@ public class Context if (embed != null && !botPerms.HasFlag(PermissionSet.EmbedLinks)) throw new PKError("PluralKit does not have permission to send embeds in this channel. Please ensure I have the **Embed Links** permission enabled."); - + if (files != null && !botPerms.HasFlag(PermissionSet.AttachFiles)) throw new PKError("PluralKit does not have permission to attach files in this channel. Please ensure I have the **Attach Files** permission enabled."); From 407e0ced1b75d1230b5241f6ed3f955cc09174ae Mon Sep 17 00:00:00 2001 From: asleepyskye Date: Sun, 20 Jul 2025 18:40:16 -0400 Subject: [PATCH 69/80] fix(bot): add read func to handle requests w/ closed connections --- PluralKit.Bot/Services/HttpListenerService.cs | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/PluralKit.Bot/Services/HttpListenerService.cs b/PluralKit.Bot/Services/HttpListenerService.cs index d2bfc4ce..bafeae41 100644 --- a/PluralKit.Bot/Services/HttpListenerService.cs +++ b/PluralKit.Bot/Services/HttpListenerService.cs @@ -1,3 +1,4 @@ +using System.Text; using System.Text.Json; using Serilog; @@ -57,7 +58,7 @@ public class HttpListenerService private async Task RuntimeConfigSet(HttpContextBase ctx) { var key = ctx.Request.Url.Parameters["key"]; - var value = ctx.Request.DataAsString; + var value = ReadStream(ctx.Request.Data, ctx.Request.ContentLength); await _runtimeConfig.Set(key, value); await RuntimeConfigGet(ctx); } @@ -76,7 +77,7 @@ public class HttpListenerService var shardIdString = ctx.Request.Url.Parameters["shard_id"]; if (!int.TryParse(shardIdString, out var shardId)) return; - var packet = JsonSerializer.Deserialize(ctx.Request.DataAsString, _jsonSerializerOptions); + var packet = JsonSerializer.Deserialize(ReadStream(ctx.Request.Data, ctx.Request.ContentLength), _jsonSerializerOptions); var evt = DeserializeEvent(shardId, packet.EventType!, (JsonElement)packet.Payload!); if (evt != null) { @@ -107,4 +108,39 @@ public class HttpListenerService return null; } } + + //temporary re-implementation of the ReadStream function found in WatsonWebserver.Lite, but with handling for closed connections + //https://github.com/dotnet/WatsonWebserver/issues/171 + private static string ReadStream(Stream input, long contentLength) + { + if (input == null) throw new ArgumentNullException(nameof(input)); + if (!input.CanRead) throw new InvalidOperationException("Input stream is not readable"); + if (contentLength < 1) return ""; + + byte[] buffer = new byte[65536]; + long bytesRemaining = contentLength; + + using (MemoryStream ms = new MemoryStream()) + { + int read; + + while (bytesRemaining > 0) + { + read = input.Read(buffer, 0, buffer.Length); + if (read > 0) + { + ms.Write(buffer, 0, read); + bytesRemaining -= read; + } + else + { + throw new IOException("Connection closed before reading end of stream."); + } + } + + if (ms.Length < 1) return null; + var str = Encoding.Default.GetString(ms.ToArray()); + return str; + } + } } \ No newline at end of file From 277bfebb330c9ed4716e48319bf6f2bfb2ad0d3f Mon Sep 17 00:00:00 2001 From: asleepyskye Date: Wed, 23 Jul 2025 19:44:59 -0400 Subject: [PATCH 70/80] chore: bump twilight --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 341ba381..e50bf992 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4433,7 +4433,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twilight-cache-inmemory" version = "0.16.0" -source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-70105ef#941604f888bb8d25239f4f3d4dfb1a9ecdb9851b" dependencies = [ "bitflags 2.9.0", "dashmap", @@ -4445,7 +4445,7 @@ dependencies = [ [[package]] name = "twilight-gateway" version = "0.16.0" -source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-70105ef#941604f888bb8d25239f4f3d4dfb1a9ecdb9851b" dependencies = [ "bitflags 2.9.0", "fastrand", @@ -4465,7 +4465,7 @@ dependencies = [ [[package]] name = "twilight-gateway-queue" version = "0.16.0" -source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-70105ef#941604f888bb8d25239f4f3d4dfb1a9ecdb9851b" dependencies = [ "tokio", "tracing", @@ -4474,7 +4474,7 @@ dependencies = [ [[package]] name = "twilight-http" version = "0.16.0" -source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-70105ef#941604f888bb8d25239f4f3d4dfb1a9ecdb9851b" dependencies = [ "fastrand", "http 1.3.1", @@ -4496,7 +4496,7 @@ dependencies = [ [[package]] name = "twilight-http-ratelimiting" version = "0.16.0" -source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-70105ef#941604f888bb8d25239f4f3d4dfb1a9ecdb9851b" dependencies = [ "tokio", "tracing", @@ -4505,7 +4505,7 @@ dependencies = [ [[package]] name = "twilight-model" version = "0.16.0" -source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-70105ef#941604f888bb8d25239f4f3d4dfb1a9ecdb9851b" dependencies = [ "bitflags 2.9.0", "serde", @@ -4517,7 +4517,7 @@ dependencies = [ [[package]] name = "twilight-util" version = "0.16.0" -source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-70105ef#941604f888bb8d25239f4f3d4dfb1a9ecdb9851b" dependencies = [ "twilight-model", ] @@ -4525,7 +4525,7 @@ dependencies = [ [[package]] name = "twilight-validate" version = "0.16.0" -source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-fb4f590#82072d0600d5bb527cf1ef700641fcf86a985b17" +source = "git+https://github.com/pluralkit/twilight?branch=pluralkit-70105ef#941604f888bb8d25239f4f3d4dfb1a9ecdb9851b" dependencies = [ "twilight-model", ] diff --git a/Cargo.toml b/Cargo.toml index 48c73439..23ab960f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,11 +25,11 @@ uuid = { version = "1.7.0", features = ["serde"] } axum = { git = "https://github.com/pluralkit/axum", branch = "v0.8.4-pluralkit" } -twilight-gateway = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590" } -twilight-cache-inmemory = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590", features = ["permission-calculator"] } -twilight-util = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590", features = ["permission-calculator"] } -twilight-model = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590" } -twilight-http = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-fb4f590", default-features = false, features = ["rustls-aws_lc_rs", "rustls-native-roots"] } +twilight-gateway = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-70105ef" } +twilight-cache-inmemory = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-70105ef", features = ["permission-calculator"] } +twilight-util = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-70105ef", features = ["permission-calculator"] } +twilight-model = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-70105ef" } +twilight-http = { git = "https://github.com/pluralkit/twilight", branch = "pluralkit-70105ef", default-features = false, features = ["rustls-aws_lc_rs", "rustls-native-roots"] } # twilight-gateway = { path = "../twilight/twilight-gateway" } # twilight-cache-inmemory = { path = "../twilight/twilight-cache-inmemory", features = ["permission-calculator"] } From 47c59902181b6f81b046f9c471780efd85fd708f Mon Sep 17 00:00:00 2001 From: alyssa Date: Thu, 24 Jul 2025 01:36:04 +0000 Subject: [PATCH 71/80] chore: move migrations to rust also adds some basic test seed data --- Cargo.lock | 11 +++ PluralKit.Bot/Init.cs | 10 --- ci/Dockerfile.rust | 2 + ci/rust-docker-target.sh | 1 + crates/h | 0 crates/migrate/Cargo.toml | 12 ++++ crates/migrate/build.rs | 55 +++++++++++++++ .../migrate/data}/clean.sql | 0 .../migrate/data}/functions.sql | 0 .../migrate/data/migrations}/0.sql | 0 .../migrate/data/migrations}/1.sql | 0 .../migrate/data/migrations}/10.sql | 0 .../migrate/data/migrations}/11.sql | 0 .../migrate/data/migrations}/12.sql | 0 .../migrate/data/migrations}/13.sql | 0 .../migrate/data/migrations}/14.sql | 0 .../migrate/data/migrations}/15.sql | 0 .../migrate/data/migrations}/16.sql | 0 .../migrate/data/migrations}/17.sql | 0 .../migrate/data/migrations}/18.sql | 0 .../migrate/data/migrations}/19.sql | 0 .../migrate/data/migrations}/2.sql | 0 .../migrate/data/migrations}/20.sql | 0 .../migrate/data/migrations}/21.sql | 0 .../migrate/data/migrations}/22.sql | 0 .../migrate/data/migrations}/23.sql | 0 .../migrate/data/migrations}/24.sql | 0 .../migrate/data/migrations}/25.sql | 0 .../migrate/data/migrations}/26.sql | 0 .../migrate/data/migrations}/27.sql | 0 .../migrate/data/migrations}/28.sql | 0 .../migrate/data/migrations}/29.sql | 0 .../migrate/data/migrations}/3.sql | 0 .../migrate/data/migrations}/30.sql | 0 .../migrate/data/migrations}/31.sql | 0 .../migrate/data/migrations}/32.sql | 0 .../migrate/data/migrations}/33.sql | 0 .../migrate/data/migrations}/34.sql | 0 .../migrate/data/migrations}/35.sql | 0 .../migrate/data/migrations}/36.sql | 0 .../migrate/data/migrations}/37.sql | 0 .../migrate/data/migrations}/38.sql | 0 .../migrate/data/migrations}/39.sql | 0 .../migrate/data/migrations}/4.sql | 0 .../migrate/data/migrations}/40.sql | 0 .../migrate/data/migrations}/41.sql | 0 .../migrate/data/migrations}/42.sql | 0 .../migrate/data/migrations}/43.sql | 0 .../migrate/data/migrations}/44.sql | 0 .../migrate/data/migrations}/45.sql | 0 .../migrate/data/migrations}/46.sql | 0 .../migrate/data/migrations}/47.sql | 0 .../migrate/data/migrations}/48.sql | 0 .../migrate/data/migrations}/49.sql | 0 .../migrate/data/migrations}/5.sql | 0 .../migrate/data/migrations}/50.sql | 0 .../migrate/data/migrations}/51.sql | 0 .../migrate/data/migrations}/52.sql | 0 .../migrate/data/migrations}/6.sql | 0 .../migrate/data/migrations}/7.sql | 0 .../migrate/data/migrations}/8.sql | 0 .../migrate/data/migrations}/9.sql | 0 crates/migrate/data/seed.sql | 8 +++ .../Views => crates/migrate/data}/views.sql | 0 crates/migrate/src/main.rs | 70 +++++++++++++++++++ docker-compose.yml | 14 ++++ 66 files changed, 173 insertions(+), 10 deletions(-) create mode 100644 crates/h create mode 100644 crates/migrate/Cargo.toml create mode 100644 crates/migrate/build.rs rename {PluralKit.Core/Database => crates/migrate/data}/clean.sql (100%) rename {PluralKit.Core/Database/Functions => crates/migrate/data}/functions.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/0.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/1.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/10.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/11.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/12.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/13.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/14.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/15.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/16.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/17.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/18.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/19.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/2.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/20.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/21.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/22.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/23.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/24.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/25.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/26.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/27.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/28.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/29.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/3.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/30.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/31.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/32.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/33.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/34.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/35.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/36.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/37.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/38.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/39.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/4.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/40.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/41.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/42.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/43.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/44.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/45.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/46.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/47.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/48.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/49.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/5.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/50.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/51.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/52.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/6.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/7.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/8.sql (100%) rename {PluralKit.Core/Database/Migrations => crates/migrate/data/migrations}/9.sql (100%) create mode 100644 crates/migrate/data/seed.sql rename {PluralKit.Core/Database/Views => crates/migrate/data}/views.sql (100%) create mode 100644 crates/migrate/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index e50bf992..96bb2e24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2185,6 +2185,17 @@ dependencies = [ "sketches-ddsketch", ] +[[package]] +name = "migrate" +version = "0.1.0" +dependencies = [ + "anyhow", + "libpk", + "sqlx", + "tokio", + "tracing", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/PluralKit.Bot/Init.cs b/PluralKit.Bot/Init.cs index bf48d6c4..25b98f1b 100644 --- a/PluralKit.Bot/Init.cs +++ b/PluralKit.Bot/Init.cs @@ -57,16 +57,6 @@ public class Init var cache = services.Resolve(); - if (config.Cluster == null) - { - // "Connect to the database" (ie. set off database migrations and ensure state) - logger.Information("Connecting to database"); - await services.Resolve().ApplyMigrations(); - - // Clear shard status from Redis - await redis.Connection.GetDatabase().KeyDeleteAsync("pluralkit:shardstatus"); - } - logger.Information("Initializing bot"); var bot = services.Resolve(); diff --git a/ci/Dockerfile.rust b/ci/Dockerfile.rust index b035cfcf..e320fb00 100644 --- a/ci/Dockerfile.rust +++ b/ci/Dockerfile.rust @@ -25,6 +25,7 @@ COPY Cargo.lock /build/ COPY crates/ /build/crates +RUN cargo build --bin migrate --release --target x86_64-unknown-linux-musl RUN cargo build --bin api --release --target x86_64-unknown-linux-musl RUN cargo build --bin dispatch --release --target x86_64-unknown-linux-musl RUN cargo build --bin gateway --release --target x86_64-unknown-linux-musl @@ -35,6 +36,7 @@ RUN cargo build --bin gdpr_worker --release --target x86_64-unknown-linux-musl FROM alpine:latest +COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/migrate /migrate COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/api /api COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/dispatch /dispatch COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/gateway /gateway diff --git a/ci/rust-docker-target.sh b/ci/rust-docker-target.sh index 501f65c2..ba6df5e9 100755 --- a/ci/rust-docker-target.sh +++ b/ci/rust-docker-target.sh @@ -37,6 +37,7 @@ EOF } # add rust binaries here to build +build migrate build api build dispatch build gateway diff --git a/crates/h b/crates/h new file mode 100644 index 00000000..e69de29b diff --git a/crates/migrate/Cargo.toml b/crates/migrate/Cargo.toml new file mode 100644 index 00000000..cf4eff2d --- /dev/null +++ b/crates/migrate/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "migrate" +version = "0.1.0" +edition = "2021" + +[dependencies] +libpk = { path = "../libpk" } + +anyhow = { workspace = true } +sqlx = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } diff --git a/crates/migrate/build.rs b/crates/migrate/build.rs new file mode 100644 index 00000000..80829b0d --- /dev/null +++ b/crates/migrate/build.rs @@ -0,0 +1,55 @@ +use std::{ + env, + error::Error, + fs::{self, File}, + io::Write, + path::Path, +}; + +fn main() -> Result<(), Box> { + let out_dir = env::var("OUT_DIR")?; + let dest_path = Path::new(&out_dir).join("data.rs"); + let mut datafile = File::create(&dest_path)?; + + let prefix = "../../../../../../crates/migrate/data"; + + let ct = fs::read_dir("data/migrations")? + .filter(|p| { + p.as_ref() + .unwrap() + .file_name() + .into_string() + .unwrap() + .contains(".sql") + }) + .count(); + + writeln!(&mut datafile, "const MIGRATIONS: [&'static str; {ct}] = [")?; + for idx in 0..ct { + writeln!( + &mut datafile, + "\tinclude_str!(\"{prefix}/migrations/{idx}.sql\")," + )?; + } + writeln!(&mut datafile, "];\n")?; + + writeln!( + &mut datafile, + "const CLEAN: &'static str = include_str!(\"{prefix}/clean.sql\");" + )?; + writeln!( + &mut datafile, + "const VIEWS: &'static str = include_str!(\"{prefix}/views.sql\");" + )?; + writeln!( + &mut datafile, + "const FUNCTIONS: &'static str = include_str!(\"{prefix}/functions.sql\");" + )?; + + writeln!( + &mut datafile, + "const SEED: &'static str = include_str!(\"{prefix}/seed.sql\");" + )?; + + Ok(()) +} diff --git a/PluralKit.Core/Database/clean.sql b/crates/migrate/data/clean.sql similarity index 100% rename from PluralKit.Core/Database/clean.sql rename to crates/migrate/data/clean.sql diff --git a/PluralKit.Core/Database/Functions/functions.sql b/crates/migrate/data/functions.sql similarity index 100% rename from PluralKit.Core/Database/Functions/functions.sql rename to crates/migrate/data/functions.sql diff --git a/PluralKit.Core/Database/Migrations/0.sql b/crates/migrate/data/migrations/0.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/0.sql rename to crates/migrate/data/migrations/0.sql diff --git a/PluralKit.Core/Database/Migrations/1.sql b/crates/migrate/data/migrations/1.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/1.sql rename to crates/migrate/data/migrations/1.sql diff --git a/PluralKit.Core/Database/Migrations/10.sql b/crates/migrate/data/migrations/10.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/10.sql rename to crates/migrate/data/migrations/10.sql diff --git a/PluralKit.Core/Database/Migrations/11.sql b/crates/migrate/data/migrations/11.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/11.sql rename to crates/migrate/data/migrations/11.sql diff --git a/PluralKit.Core/Database/Migrations/12.sql b/crates/migrate/data/migrations/12.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/12.sql rename to crates/migrate/data/migrations/12.sql diff --git a/PluralKit.Core/Database/Migrations/13.sql b/crates/migrate/data/migrations/13.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/13.sql rename to crates/migrate/data/migrations/13.sql diff --git a/PluralKit.Core/Database/Migrations/14.sql b/crates/migrate/data/migrations/14.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/14.sql rename to crates/migrate/data/migrations/14.sql diff --git a/PluralKit.Core/Database/Migrations/15.sql b/crates/migrate/data/migrations/15.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/15.sql rename to crates/migrate/data/migrations/15.sql diff --git a/PluralKit.Core/Database/Migrations/16.sql b/crates/migrate/data/migrations/16.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/16.sql rename to crates/migrate/data/migrations/16.sql diff --git a/PluralKit.Core/Database/Migrations/17.sql b/crates/migrate/data/migrations/17.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/17.sql rename to crates/migrate/data/migrations/17.sql diff --git a/PluralKit.Core/Database/Migrations/18.sql b/crates/migrate/data/migrations/18.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/18.sql rename to crates/migrate/data/migrations/18.sql diff --git a/PluralKit.Core/Database/Migrations/19.sql b/crates/migrate/data/migrations/19.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/19.sql rename to crates/migrate/data/migrations/19.sql diff --git a/PluralKit.Core/Database/Migrations/2.sql b/crates/migrate/data/migrations/2.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/2.sql rename to crates/migrate/data/migrations/2.sql diff --git a/PluralKit.Core/Database/Migrations/20.sql b/crates/migrate/data/migrations/20.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/20.sql rename to crates/migrate/data/migrations/20.sql diff --git a/PluralKit.Core/Database/Migrations/21.sql b/crates/migrate/data/migrations/21.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/21.sql rename to crates/migrate/data/migrations/21.sql diff --git a/PluralKit.Core/Database/Migrations/22.sql b/crates/migrate/data/migrations/22.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/22.sql rename to crates/migrate/data/migrations/22.sql diff --git a/PluralKit.Core/Database/Migrations/23.sql b/crates/migrate/data/migrations/23.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/23.sql rename to crates/migrate/data/migrations/23.sql diff --git a/PluralKit.Core/Database/Migrations/24.sql b/crates/migrate/data/migrations/24.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/24.sql rename to crates/migrate/data/migrations/24.sql diff --git a/PluralKit.Core/Database/Migrations/25.sql b/crates/migrate/data/migrations/25.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/25.sql rename to crates/migrate/data/migrations/25.sql diff --git a/PluralKit.Core/Database/Migrations/26.sql b/crates/migrate/data/migrations/26.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/26.sql rename to crates/migrate/data/migrations/26.sql diff --git a/PluralKit.Core/Database/Migrations/27.sql b/crates/migrate/data/migrations/27.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/27.sql rename to crates/migrate/data/migrations/27.sql diff --git a/PluralKit.Core/Database/Migrations/28.sql b/crates/migrate/data/migrations/28.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/28.sql rename to crates/migrate/data/migrations/28.sql diff --git a/PluralKit.Core/Database/Migrations/29.sql b/crates/migrate/data/migrations/29.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/29.sql rename to crates/migrate/data/migrations/29.sql diff --git a/PluralKit.Core/Database/Migrations/3.sql b/crates/migrate/data/migrations/3.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/3.sql rename to crates/migrate/data/migrations/3.sql diff --git a/PluralKit.Core/Database/Migrations/30.sql b/crates/migrate/data/migrations/30.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/30.sql rename to crates/migrate/data/migrations/30.sql diff --git a/PluralKit.Core/Database/Migrations/31.sql b/crates/migrate/data/migrations/31.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/31.sql rename to crates/migrate/data/migrations/31.sql diff --git a/PluralKit.Core/Database/Migrations/32.sql b/crates/migrate/data/migrations/32.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/32.sql rename to crates/migrate/data/migrations/32.sql diff --git a/PluralKit.Core/Database/Migrations/33.sql b/crates/migrate/data/migrations/33.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/33.sql rename to crates/migrate/data/migrations/33.sql diff --git a/PluralKit.Core/Database/Migrations/34.sql b/crates/migrate/data/migrations/34.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/34.sql rename to crates/migrate/data/migrations/34.sql diff --git a/PluralKit.Core/Database/Migrations/35.sql b/crates/migrate/data/migrations/35.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/35.sql rename to crates/migrate/data/migrations/35.sql diff --git a/PluralKit.Core/Database/Migrations/36.sql b/crates/migrate/data/migrations/36.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/36.sql rename to crates/migrate/data/migrations/36.sql diff --git a/PluralKit.Core/Database/Migrations/37.sql b/crates/migrate/data/migrations/37.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/37.sql rename to crates/migrate/data/migrations/37.sql diff --git a/PluralKit.Core/Database/Migrations/38.sql b/crates/migrate/data/migrations/38.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/38.sql rename to crates/migrate/data/migrations/38.sql diff --git a/PluralKit.Core/Database/Migrations/39.sql b/crates/migrate/data/migrations/39.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/39.sql rename to crates/migrate/data/migrations/39.sql diff --git a/PluralKit.Core/Database/Migrations/4.sql b/crates/migrate/data/migrations/4.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/4.sql rename to crates/migrate/data/migrations/4.sql diff --git a/PluralKit.Core/Database/Migrations/40.sql b/crates/migrate/data/migrations/40.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/40.sql rename to crates/migrate/data/migrations/40.sql diff --git a/PluralKit.Core/Database/Migrations/41.sql b/crates/migrate/data/migrations/41.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/41.sql rename to crates/migrate/data/migrations/41.sql diff --git a/PluralKit.Core/Database/Migrations/42.sql b/crates/migrate/data/migrations/42.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/42.sql rename to crates/migrate/data/migrations/42.sql diff --git a/PluralKit.Core/Database/Migrations/43.sql b/crates/migrate/data/migrations/43.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/43.sql rename to crates/migrate/data/migrations/43.sql diff --git a/PluralKit.Core/Database/Migrations/44.sql b/crates/migrate/data/migrations/44.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/44.sql rename to crates/migrate/data/migrations/44.sql diff --git a/PluralKit.Core/Database/Migrations/45.sql b/crates/migrate/data/migrations/45.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/45.sql rename to crates/migrate/data/migrations/45.sql diff --git a/PluralKit.Core/Database/Migrations/46.sql b/crates/migrate/data/migrations/46.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/46.sql rename to crates/migrate/data/migrations/46.sql diff --git a/PluralKit.Core/Database/Migrations/47.sql b/crates/migrate/data/migrations/47.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/47.sql rename to crates/migrate/data/migrations/47.sql diff --git a/PluralKit.Core/Database/Migrations/48.sql b/crates/migrate/data/migrations/48.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/48.sql rename to crates/migrate/data/migrations/48.sql diff --git a/PluralKit.Core/Database/Migrations/49.sql b/crates/migrate/data/migrations/49.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/49.sql rename to crates/migrate/data/migrations/49.sql diff --git a/PluralKit.Core/Database/Migrations/5.sql b/crates/migrate/data/migrations/5.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/5.sql rename to crates/migrate/data/migrations/5.sql diff --git a/PluralKit.Core/Database/Migrations/50.sql b/crates/migrate/data/migrations/50.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/50.sql rename to crates/migrate/data/migrations/50.sql diff --git a/PluralKit.Core/Database/Migrations/51.sql b/crates/migrate/data/migrations/51.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/51.sql rename to crates/migrate/data/migrations/51.sql diff --git a/PluralKit.Core/Database/Migrations/52.sql b/crates/migrate/data/migrations/52.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/52.sql rename to crates/migrate/data/migrations/52.sql diff --git a/PluralKit.Core/Database/Migrations/6.sql b/crates/migrate/data/migrations/6.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/6.sql rename to crates/migrate/data/migrations/6.sql diff --git a/PluralKit.Core/Database/Migrations/7.sql b/crates/migrate/data/migrations/7.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/7.sql rename to crates/migrate/data/migrations/7.sql diff --git a/PluralKit.Core/Database/Migrations/8.sql b/crates/migrate/data/migrations/8.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/8.sql rename to crates/migrate/data/migrations/8.sql diff --git a/PluralKit.Core/Database/Migrations/9.sql b/crates/migrate/data/migrations/9.sql similarity index 100% rename from PluralKit.Core/Database/Migrations/9.sql rename to crates/migrate/data/migrations/9.sql diff --git a/crates/migrate/data/seed.sql b/crates/migrate/data/seed.sql new file mode 100644 index 00000000..27bca359 --- /dev/null +++ b/crates/migrate/data/seed.sql @@ -0,0 +1,8 @@ +-- example data (for integration tests or such) + +insert into systems (hid, token) values ( + 'exmpl', + 'vlPitT0tEgT++a450w1/afODy5NXdALcHDwryX6dOIZdGUGbZg+5IH3nrUsQihsw' +); +insert into system_config (system) values (1); +insert into system_guild (system, guild) values (1, 466707357099884544); diff --git a/PluralKit.Core/Database/Views/views.sql b/crates/migrate/data/views.sql similarity index 100% rename from PluralKit.Core/Database/Views/views.sql rename to crates/migrate/data/views.sql diff --git a/crates/migrate/src/main.rs b/crates/migrate/src/main.rs new file mode 100644 index 00000000..85b15e33 --- /dev/null +++ b/crates/migrate/src/main.rs @@ -0,0 +1,70 @@ +#![feature(let_chains)] + +use tracing::info; + +include!(concat!(env!("OUT_DIR"), "/data.rs")); + +#[libpk::main] +async fn main() -> anyhow::Result<()> { + let db = libpk::db::init_data_db().await?; + + // clean + // get current migration + // migrate to latest + // run views + // run functions + + #[derive(sqlx::FromRow)] + struct CurrentMigration { + schema_version: i32, + } + + let info = match sqlx::query_as("select schema_version from info") + .fetch_optional(&db) + .await + { + Ok(Some(result)) => result, + Ok(None) => CurrentMigration { schema_version: -1 }, + Err(e) if format!("{e}").contains("relation \"info\" does not exist") => { + CurrentMigration { schema_version: -1 } + } + Err(e) => return Err(e.into()), + }; + + info!("current migration: {}", info.schema_version); + + info!("running clean.sql"); + sqlx::raw_sql(fix_feff(CLEAN)).execute(&db).await?; + + for idx in (info.schema_version + 1) as usize..MIGRATIONS.len() { + info!("running migration {idx}"); + sqlx::raw_sql(fix_feff(MIGRATIONS[idx as usize])) + .execute(&db) + .await?; + } + + info!("running views.sql"); + sqlx::raw_sql(fix_feff(VIEWS)).execute(&db).await?; + + info!("running functions.sql"); + sqlx::raw_sql(fix_feff(FUNCTIONS)).execute(&db).await?; + + if let Ok(var) = std::env::var("SEED") + && var == "true" + { + info!("running seed.sql"); + sqlx::raw_sql(fix_feff(SEED)).execute(&db).await?; + info!( + "example system created with hid 'exmpl', token 'vlPitT0tEgT++a450w1/afODy5NXdALcHDwryX6dOIZdGUGbZg+5IH3nrUsQihsw', guild_id 466707357099884544" + ); + } + + info!("all done!"); + + Ok(()) +} + +// some migration scripts have \u{feff} at the start +fn fix_feff(sql: &str) -> &str { + sql.trim_start_matches("\u{feff}") +} diff --git a/docker-compose.yml b/docker-compose.yml index 18ddaa03..e2caa46a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,17 @@ version: "3" services: + migrate: + build: + context: . + dockerfile: ci/Dockerfile.rust + environment: + - RUST_LOG=info + - pluralkit__db__data_db_uri=postgresql://postgres:postgres@db:5432/postgres + - pluralkit__db__data_redis_addr=1 + command: ["/migrate"] + depends_on: ["db"] + bot: build: context: . @@ -17,6 +28,9 @@ services: - "PluralKit__Bot__EventAwaiterTarget=http://bot:5002/events" - "PluralKit__Bot__DisableGateway=true" restart: unless-stopped + depends_on: + migrate: + condition: service_completed_successfully gateway: build: From 049f06d9b1f75d4ab73323120287703607d3c86a Mon Sep 17 00:00:00 2001 From: alyssa Date: Sat, 26 Jul 2025 05:30:23 +0000 Subject: [PATCH 72/80] fix(avatars): use auto region instead of s3 --- crates/avatars/src/cleanup.rs | 2 +- crates/avatars/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/avatars/src/cleanup.rs b/crates/avatars/src/cleanup.rs index cbf5ef1e..48b25f98 100644 --- a/crates/avatars/src/cleanup.rs +++ b/crates/avatars/src/cleanup.rs @@ -13,7 +13,7 @@ async fn main() -> anyhow::Result<()> { let bucket = { let region = s3::Region::Custom { - region: "s3".to_string(), + region: "auto".to_string(), endpoint: config.s3.endpoint.to_string(), }; diff --git a/crates/avatars/src/main.rs b/crates/avatars/src/main.rs index 863619d6..c8399086 100644 --- a/crates/avatars/src/main.rs +++ b/crates/avatars/src/main.rs @@ -179,7 +179,7 @@ async fn main() -> anyhow::Result<()> { let bucket = { let region = s3::Region::Custom { - region: "s3".to_string(), + region: "auto".to_string(), endpoint: config.s3.endpoint.to_string(), }; From 101c585d44efc5d6002be54550f63db6c110503a Mon Sep 17 00:00:00 2001 From: Zowie <50455229+z0w13@users.noreply.github.com> Date: Mon, 4 Aug 2025 19:59:08 +0200 Subject: [PATCH 73/80] fix(docs): add missing `created` property to group model (#761) --- docs/content/api/models.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/api/models.md b/docs/content/api/models.md index 5d5938d0..dd8e1fc8 100644 --- a/docs/content/api/models.md +++ b/docs/content/api/models.md @@ -82,6 +82,7 @@ The PluralKit Discord bot can be configured to display short IDs in uppercase, o |name|string|100-character limit| |display_name|?string|100-character limit| |description|?string|1000-character limit| +|created|?datetime|| |icon|?string|256-character limit, must be a publicly-accessible URL| |banner|?string|256-character limit, must be a publicly-accessible URL| |color|?string|6-character hex code, no `#` at the beginning| From abc614a2c0dcb448143d9af32f3217c0f1fc3f41 Mon Sep 17 00:00:00 2001 From: Zowie <50455229+z0w13@users.noreply.github.com> Date: Mon, 4 Aug 2025 20:00:14 +0200 Subject: [PATCH 74/80] fix(docs): add return type for 'Create Switch' (#756) --- docs/content/api/endpoints.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/api/endpoints.md b/docs/content/api/endpoints.md index a40b2dd8..ee692db6 100644 --- a/docs/content/api/endpoints.md +++ b/docs/content/api/endpoints.md @@ -296,6 +296,8 @@ JSON Body Parameters ** Can be short IDs or UUIDs. +Returns a [switch object](/api/models#switch-model) containing a list of member objects. + ### Get Switch GET `/systems/{systemRef}/switches/{switchRef}` From 0225275b1ce527a78cc170f64c1b047e8ef34bad Mon Sep 17 00:00:00 2001 From: Zowie <50455229+z0w13@users.noreply.github.com> Date: Mon, 4 Aug 2025 20:01:24 +0200 Subject: [PATCH 75/80] fix(docs): mark system model's `created` as not nullable (#751) --- docs/content/api/models.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/api/models.md b/docs/content/api/models.md index dd8e1fc8..abf8c7db 100644 --- a/docs/content/api/models.md +++ b/docs/content/api/models.md @@ -31,7 +31,7 @@ The PluralKit Discord bot can be configured to display short IDs in uppercase, o |avatar_url|?string|256-character limit, must be a publicly-accessible URL| |banner|?string|256-character limit, must be a publicly-accessible URL| |color|?string|6-character hex code, no `#` at the beginning| -|created|?datetime|| +|created|datetime|| |privacy|?system privacy object|| * System privacy keys: `description_privacy`, `pronoun_privacy`, `member_list_privacy`, `group_list_privacy`, `front_privacy`, `front_history_privacy` From afbd510bdbdfa7d8ce249677650eb94aafaa65ef Mon Sep 17 00:00:00 2001 From: Zowie <50455229+z0w13@users.noreply.github.com> Date: Mon, 4 Aug 2025 20:03:55 +0200 Subject: [PATCH 76/80] fix(docs): `id_padding_format` uses `off` instead of `none` in the API (#757) --- docs/content/api/models.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/api/models.md b/docs/content/api/models.md index abf8c7db..47dc41af 100644 --- a/docs/content/api/models.md +++ b/docs/content/api/models.md @@ -137,7 +137,7 @@ The PluralKit Discord bot can be configured to display short IDs in uppercase, o |key|description| |---|---| -|none|do not pad 5-character ids| +|off|do not pad 5-character ids| |left|add a padding space to the left of 5-character ids in lists| |right|add a padding space to the right of 5-character ids in lists| From 19d2d0f37592fc41144769be80c849e8e4298c3b Mon Sep 17 00:00:00 2001 From: Zowie <50455229+z0w13@users.noreply.github.com> Date: Mon, 4 Aug 2025 20:05:45 +0200 Subject: [PATCH 77/80] fix(docs): add return type for `/members/{memberRef}/groups` (#755) --- docs/content/api/endpoints.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/api/endpoints.md b/docs/content/api/endpoints.md index ee692db6..a9e48dbe 100644 --- a/docs/content/api/endpoints.md +++ b/docs/content/api/endpoints.md @@ -139,6 +139,8 @@ Returns 204 No Content on success. GET `/members/{memberRef}/groups` +Returns an array of [group objects](/api/models/#group-model). + ### Add Member To Groups POST `/members/{memberRef}/groups/add` From e89b31045bd1b6098eefce5c26ca5163c4a761aa Mon Sep 17 00:00:00 2001 From: Zowie <50455229+z0w13@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:38:35 +0200 Subject: [PATCH 78/80] fix(docs): correct docs for clearing display name (#762) --- docs/content/user-guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/user-guide.md b/docs/content/user-guide.md index a4b7d5c8..51654841 100644 --- a/docs/content/user-guide.md +++ b/docs/content/user-guide.md @@ -212,9 +212,9 @@ a display name using the `pk;member displayname` command, like so: pk;member John displayname Jonathan pk;member Robert displayname Bob (he/him) -To remove a display name, just use the same command with no last parameter, eg: +To remove a display name, use the same command with `-clear` as the parameter, eg: - pk;member John displayname + pk;member John displayname -clear This will remove the display name, and thus the member will be proxied with their canonical name. From b75bd489b64503e49981daf2750e98ffb9644895 Mon Sep 17 00:00:00 2001 From: Zowie <50455229+z0w13@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:39:11 +0200 Subject: [PATCH 79/80] fix(docs): add missing privacy keys (#763) --- docs/content/api/models.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/api/models.md b/docs/content/api/models.md index 47dc41af..64bc475c 100644 --- a/docs/content/api/models.md +++ b/docs/content/api/models.md @@ -34,7 +34,7 @@ The PluralKit Discord bot can be configured to display short IDs in uppercase, o |created|datetime|| |privacy|?system privacy object|| -* System privacy keys: `description_privacy`, `pronoun_privacy`, `member_list_privacy`, `group_list_privacy`, `front_privacy`, `front_history_privacy` +* System privacy keys: `name_privacy`, `description_privacy`, `avatar_privacy`, `banner_privacy`, `pronoun_privacy`, `member_list_privacy`, `group_list_privacy`, `front_privacy`, `front_history_privacy` ### Member model @@ -61,7 +61,7 @@ The PluralKit Discord bot can be configured to display short IDs in uppercase, o |last_message_timestamp|?datetime|| |privacy|?member privacy object|| -* Member privacy keys: `visibility`, `name_privacy`, `description_privacy`, `birthday_privacy`, `pronoun_privacy`, `avatar_privacy`, `metadata_privacy`, `proxy_privacy` +* Member privacy keys: `visibility`, `name_privacy`, `description_privacy`, `birthday_privacy`, `pronoun_privacy`, `avatar_privacy`, `banner_privacy`, `metadata_privacy`, `proxy_privacy` #### ProxyTag object @@ -88,7 +88,7 @@ The PluralKit Discord bot can be configured to display short IDs in uppercase, o |color|?string|6-character hex code, no `#` at the beginning| |privacy|?group privacy object|| -* Group privacy keys: `name_privacy`, `description_privacy`, `icon_privacy`, `list_privacy`, `metadata_privacy`, `visibility` +* Group privacy keys: `name_privacy`, `description_privacy`, `banner_privacy`, `icon_privacy`, `list_privacy`, `metadata_privacy`, `visibility` ### Switch model From aa103f85e79acc1e227fa28b6d87a9fc3293cdf5 Mon Sep 17 00:00:00 2001 From: Zowie <50455229+z0w13@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:39:23 +0200 Subject: [PATCH 80/80] fix(docs): add missing `avatar_url` and `display_name` properties to `system guild settings` model (#765) --- docs/content/api/models.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/api/models.md b/docs/content/api/models.md index 64bc475c..65fba699 100644 --- a/docs/content/api/models.md +++ b/docs/content/api/models.md @@ -173,6 +173,8 @@ This model is used when querying system settings without authenticating, or for |proxying_enabled|boolean|| |tag|?string|79-character limit| |tag_enabled|boolean|| +|avatar_url|?string|256-character limit| +|display_name|?string|100-character limit| ### Autoproxy settings model