Merge branch 'main' into rust-command-parser

.cargo/config.toml

@@ -0,0 +1,3 @@
+[build]
+rustflags = ["-C", "target-cpu=native"]
+

.github/workflows/rust.yml

@@ -3,16 +3,19 @@
 # todo: don't use docker/build-push-action
 # todo: run builds on pull request
 
-name: Build and push API Docker image
+name: Build and push Rust service Docker images
 on:
   push:
-    branches:
-      - main
     paths:
     - 'lib/libpk/**'
     - 'services/api/**'
+    - 'services/gateway/**'
+    - 'services/avatars/**'
     - '.github/workflows/rust.yml'
     - 'Dockerfile.rust'
+    - 'Dockerfile.bin'
+    - 'Cargo.toml'
+    - 'Cargo.lock'
 
 jobs:
   deploy:
@@ -45,7 +48,7 @@ jobs:
 
       # add more binaries here
       - run: |
-          for binary in "api"; do
+          for binary in "api" "gateway" "avatars"; do
             for tag in latest ${{ env.BRANCH_NAME }} ${{ github.sha }}; do
               cat Dockerfile.bin | sed "s/__BINARY__/$binary/g" | docker build -t ghcr.io/pluralkit/$binary:$tag -f - .
             done

.github/workflows/rustfmt.yml

@@ -0,0 +1,16 @@
+name: "Check Rust formatting"
+on:
+  push:
+  pull_request:
+
+jobs:
+  rustfmt:
+    name: cargo fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
+      - name: Rustfmt Check
+        uses: actions-rust-lang/rustfmt@v1

.github/workflows/scheduled_tasks.yml

@@ -2,8 +2,9 @@ name: Build scheduled tasks runner Docker image
 
 on:
   push:
-    branches: [main]
+    branches: [main, gateway-service]
     paths:
+      - .github/workflows/scheduled_tasks.yml
       - 'services/scheduled_tasks/**'
 
 jobs:

Cargo.toml

@@ -4,21 +4,43 @@ members = [
     "./lib/libpk",
     "./lib/commands",
     "./services/api",
-    "./services/dispatch"
+    "./services/dispatch",
+    "./services/gateway",
+    "./services/avatars"
 ]
 
 [workspace.dependencies]
 anyhow = "1"
 axum = "0.7.5"
+axum-macros = "0.4.1"
+bytes = "1.6.0"
+chrono = "0.4"
 fred = { version = "5.2.0", default-features = false, features = ["tracing", "pool-prefer-active"] }
+futures = "0.3.30"
 lazy_static = "1.4.0"
-metrics = "0.20.1"
+metrics = "0.23.0"
-serde = "1.0.152"
+reqwest = { version = "0.12.7" , default-features = false, features = ["rustls-tls", "trust-dns"]}
+serde = { version = "1.0.196", features = ["derive"] }
 serde_json = "1.0.117"
-sqlx = { version = "0.7.4", features = ["runtime-tokio", "postgres", "chrono", "macros"] }
+signal-hook = "0.3.17"
-tokio = { version = "1.25.0", features = ["full"] }
+sqlx = { version = "0.7.4", features = ["runtime-tokio", "postgres", "time", "macros", "uuid"] }
-tracing = "0.1.37"
+time = "0.3.34"
+tokio = { version = "1.36.0", features = ["full"] }
+tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter", "json"] }
+uuid = { version = "1.7.0", features = ["serde"] }
+
+twilight-gateway = { git = "https://github.com/pluralkit/twilight" }
+twilight-cache-inmemory = { git = "https://github.com/pluralkit/twilight", features = ["permission-calculator"] }
+twilight-util = { git = "https://github.com/pluralkit/twilight", features = ["permission-calculator"] }
+twilight-model = { git = "https://github.com/pluralkit/twilight" }
+twilight-http = { git = "https://github.com/pluralkit/twilight", default-features = false, features = ["rustls-native-roots"] }
+
+#twilight-gateway = { path = "../twilight/twilight-gateway" }
+#twilight-cache-inmemory = { path = "../twilight/twilight-cache-inmemory", features = ["permission-calculator"] }
+#twilight-util = { path = "../twilight/twilight-util", features = ["permission-calculator"] }
+#twilight-model = { path = "../twilight/twilight-model" }
+#twilight-http = { path = "../twilight/twilight-http", default-features = false, features = ["rustls-native-roots"] }
 
 prost = "0.12"
 prost-types = "0.12"

Dockerfile.rust

@@ -4,7 +4,7 @@ WORKDIR /build
 
 RUN apk add rustup build-base protoc
 # todo: arm64 target
-RUN rustup-init --default-host x86_64-unknown-linux-musl --default-toolchain stable --profile default -y
+RUN rustup-init --default-host x86_64-unknown-linux-musl --default-toolchain nightly-2024-08-20 --profile default -y
 
 ENV PATH=/root/.cargo/bin:$PATH
 ENV RUSTFLAGS='-C link-arg=-s'
@@ -27,9 +27,15 @@ COPY proto/ /build/proto
 # this needs to match workspaces in Cargo.toml
 COPY lib/libpk /build/lib/libpk
 COPY services/api/ /build/services/api
+COPY services/gateway/ /build/services/gateway
+COPY services/avatars/ /build/services/avatars
 
 RUN cargo build --bin api --release --target x86_64-unknown-linux-musl
+RUN cargo build --bin gateway --release --target x86_64-unknown-linux-musl
+RUN cargo build --bin avatars --release --target x86_64-unknown-linux-musl
 
 FROM scratch 
 
 COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/api /api
+COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/gateway /gateway
+COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/avatars /avatars

Myriad/Cache/DiscordCacheExtensions.cs

@@ -100,15 +100,19 @@ public static class DiscordCacheExtensions
             await cache.SaveChannel(thread);
     }
 
-    public static async Task<PermissionSet> BotPermissionsIn(this IDiscordCache cache, ulong channelId)
+    public static async Task<PermissionSet> BotPermissionsIn(this IDiscordCache cache, ulong guildId, ulong channelId)
     {
-        var channel = await cache.GetRootChannel(channelId);
+        // disable this for now
+        //if (cache is HttpDiscordCache)
+        //    return await ((HttpDiscordCache)cache).BotChannelPermissions(guildId, channelId);
+
+        var channel = await cache.GetRootChannel(guildId, channelId);
 
         if (channel.GuildId != null)
         {
             var userId = cache.GetOwnUser();
             var member = await cache.TryGetSelfMember(channel.GuildId.Value);
-            return await cache.PermissionsFor2(channelId, userId, member);
+            return await cache.PermissionsFor2(guildId, channelId, userId, member);
         }
 
         return PermissionSet.Dm;

Myriad/Cache/HTTPDiscordCache.cs

@@ -0,0 +1,187 @@
+using Serilog;
+using System.Net;
+using System.Text.Json;
+
+using Myriad.Serialization;
+using Myriad.Types;
+
+namespace Myriad.Cache;
+
+public class HttpDiscordCache: IDiscordCache
+{
+    private readonly ILogger _logger;
+    private readonly HttpClient _client;
+    private readonly Uri _cacheEndpoint;
+    private readonly int _shardCount;
+    private readonly ulong _ownUserId;
+
+    private readonly MemoryDiscordCache _innerCache;
+
+    private readonly JsonSerializerOptions _jsonSerializerOptions;
+
+    public EventHandler<(bool?, string)> OnDebug;
+
+    public HttpDiscordCache(ILogger logger, HttpClient client, string cacheEndpoint, int shardCount, ulong ownUserId, bool useInnerCache)
+    {
+        _logger = logger;
+        _client = client;
+        _cacheEndpoint = new Uri(cacheEndpoint);
+        _shardCount = shardCount;
+        _ownUserId = ownUserId;
+        _jsonSerializerOptions = new JsonSerializerOptions().ConfigureForMyriad();
+        if (useInnerCache) _innerCache = new MemoryDiscordCache(ownUserId);
+    }
+
+    public ValueTask SaveGuild(Guild guild) => _innerCache?.SaveGuild(guild) ?? default;
+    public ValueTask SaveChannel(Channel channel) => _innerCache?.SaveChannel(channel) ?? default;
+    public ValueTask SaveUser(User user) => default;
+    public ValueTask SaveSelfMember(ulong guildId, GuildMemberPartial member) => _innerCache?.SaveSelfMember(guildId, member) ?? default;
+    public ValueTask SaveRole(ulong guildId, Myriad.Types.Role role) => _innerCache?.SaveRole(guildId, role) ?? default;
+    public ValueTask SaveDmChannelStub(ulong channelId) => _innerCache?.SaveDmChannelStub(channelId) ?? default;
+    public ValueTask RemoveGuild(ulong guildId) => _innerCache?.RemoveGuild(guildId) ?? default;
+    public ValueTask RemoveChannel(ulong channelId) => _innerCache?.RemoveChannel(channelId) ?? default;
+    public ValueTask RemoveUser(ulong userId) => _innerCache?.RemoveUser(userId) ?? default;
+    public ValueTask RemoveRole(ulong guildId, ulong roleId) => _innerCache?.RemoveRole(guildId, roleId) ?? default;
+
+    public ulong GetOwnUser() => _ownUserId;
+
+    private async Task<T?> QueryCache<T>(string endpoint, ulong guildId)
+    {
+        var cluster = _cacheEndpoint.Authority;
+        if (cluster.Contains(".service.consul"))
+            // int(((guild_id >> 22) % shard_count) / 16)
+            cluster = $"cluster{(int)(((guildId >> 22) % (ulong)_shardCount) / 16)}.{cluster}";
+
+        var response = await _client.GetAsync($"{_cacheEndpoint.Scheme}://{cluster}{endpoint}");
+
+        if (response.StatusCode == HttpStatusCode.NotFound)
+            return default;
+
+        if (response.StatusCode != HttpStatusCode.Found)
+            throw new Exception($"failed to query http cache: {response.StatusCode}");
+
+        var plaintext = await response.Content.ReadAsStringAsync();
+        return JsonSerializer.Deserialize<T>(plaintext, _jsonSerializerOptions);
+    }
+
+    public async Task<Guild?> TryGetGuild(ulong guildId)
+    {
+        var hres = await QueryCache<Guild?>($"/guilds/{guildId}", guildId);
+        if (_innerCache == null) return hres;
+        var lres = await _innerCache.TryGetGuild(guildId);
+
+        if (lres == null && hres == null) return null;
+        if (lres == null)
+        {
+            _logger.Warning($"TryGetGuild({guildId}) was only successful on remote cache");
+            OnDebug(null, (true, "guild"));
+            return hres;
+        }
+        if (hres == null)
+        {
+            _logger.Warning($"TryGetGuild({guildId}) was only successful on local cache");
+            OnDebug(null, (false, "guild"));
+            return lres;
+        }
+        return hres;
+    }
+
+    public async Task<Channel?> TryGetChannel(ulong guildId, ulong channelId)
+    {
+        var hres = await QueryCache<Channel?>($"/guilds/{guildId}/channels/{channelId}", guildId);
+        if (_innerCache == null) return hres;
+        var lres = await _innerCache.TryGetChannel(guildId, channelId);
+        if (lres == null && hres == null) return null;
+        if (lres == null)
+        {
+            _logger.Warning($"TryGetChannel({guildId}, {channelId}) was only successful on remote cache");
+            OnDebug(null, (true, "channel"));
+            return hres;
+        }
+        if (hres == null)
+        {
+            _logger.Warning($"TryGetChannel({guildId}, {channelId}) was only successful on local cache");
+            OnDebug(null, (false, "channel"));
+            return lres;
+        }
+        return hres;
+    }
+
+    // this should be a GetUserCached method on nirn-proxy (it's always called as GetOrFetchUser)
+    // so just return nothing
+    public Task<User?> TryGetUser(ulong userId)
+        => Task.FromResult<User?>(null);
+
+    public async Task<GuildMemberPartial?> TryGetSelfMember(ulong guildId)
+    {
+        var hres = await QueryCache<GuildMemberPartial?>($"/guilds/{guildId}/members/@me", guildId);
+        if (_innerCache == null) return hres;
+        var lres = await _innerCache.TryGetSelfMember(guildId);
+        if (lres == null && hres == null) return null;
+        if (lres == null)
+        {
+            _logger.Warning($"TryGetSelfMember({guildId}) was only successful on remote cache");
+            OnDebug(null, (true, "self_member"));
+            return hres;
+        }
+        if (hres == null)
+        {
+            _logger.Warning($"TryGetSelfMember({guildId}) was only successful on local cache");
+            OnDebug(null, (false, "self_member"));
+            return lres;
+        }
+        return hres;
+    }
+
+    //    public async Task<PermissionSet> BotChannelPermissions(ulong guildId, ulong channelId)
+    //    {
+    //        // todo: local cache throws rather than returning null
+    //        // we need to throw too, and try/catch local cache here
+    //        var lres = await _innerCache.BotPermissionsIn(guildId, channelId);
+    //        var hres = await QueryCache<PermissionSet?>($"/guilds/{guildId}/channels/{channelId}/permissions/@me", guildId);
+    //        if (lres == null && hres == null) return null;
+    //        if (lres == null)
+    //        {
+    //            _logger.Warning($"TryGetChannel({guildId}, {channelId}) was only successful on remote cache");
+    //           OnDebug(null, (true, "botchannelperms"));
+    //            return hres;
+    //        }
+    //        if (hres == null)
+    //        {
+    //            _logger.Warning($"TryGetChannel({guildId}, {channelId}) was only successful on local cache");
+    //            OnDebug(null, (false, "botchannelperms"));
+    //            return lres;
+    //        }
+    //
+    //        // this one is easy to check, so let's check it
+    //        if ((int)lres != (int)hres)
+    //        {
+    //            // trust local
+    //            _logger.Warning($"got different permissions for {channelId} (local {(int)lres}, remote {(int)hres})");
+    //            OnDebug(null, (null, "botchannelperms"));
+    //            return lres;
+    //        }
+    //        return hres;
+    //    }
+
+    public async Task<IEnumerable<Channel>> GetGuildChannels(ulong guildId)
+    {
+        var hres = await QueryCache<IEnumerable<Channel>>($"/guilds/{guildId}/channels", guildId);
+        if (_innerCache == null) return hres;
+        var lres = await _innerCache.GetGuildChannels(guildId);
+        if (lres == null && hres == null) return null;
+        if (lres == null)
+        {
+            _logger.Warning($"GetGuildChannels({guildId}) was only successful on remote cache");
+            OnDebug(null, (true, "guild_channels"));
+            return hres;
+        }
+        if (hres == null)
+        {
+            _logger.Warning($"GetGuildChannels({guildId}) was only successful on local cache");
+            OnDebug(null, (false, "guild_channels"));
+            return lres;
+        }
+        return hres;
+    }
+}

Myriad/Cache/IDiscordCache.cs

@@ -18,11 +18,9 @@ public interface IDiscordCache
 
     internal ulong GetOwnUser();
     public Task<Guild?> TryGetGuild(ulong guildId);
-    public Task<Channel?> TryGetChannel(ulong channelId);
+    public Task<Channel?> TryGetChannel(ulong guildId, ulong channelId);
     public Task<User?> TryGetUser(ulong userId);
     public Task<GuildMemberPartial?> TryGetSelfMember(ulong guildId);
-    public Task<Role?> TryGetRole(ulong roleId);
 
-    public IAsyncEnumerable<Guild> GetAllGuilds();
     public Task<IEnumerable<Channel>> GetGuildChannels(ulong guildId);
 }

Myriad/Cache/MemoryDiscordCache.cs

@@ -137,7 +137,7 @@ public class MemoryDiscordCache: IDiscordCache
         return Task.FromResult(cg?.Guild);
     }
 
-    public Task<Channel?> TryGetChannel(ulong channelId)
+    public Task<Channel?> TryGetChannel(ulong _, ulong channelId)
     {
         _channels.TryGetValue(channelId, out var channel);
         return Task.FromResult(channel);
@@ -155,19 +155,6 @@ public class MemoryDiscordCache: IDiscordCache
         return Task.FromResult(guildMember);
     }
 
-    public Task<Role?> TryGetRole(ulong roleId)
-    {
-        _roles.TryGetValue(roleId, out var role);
-        return Task.FromResult(role);
-    }
-
-    public IAsyncEnumerable<Guild> GetAllGuilds()
-    {
-        return _guilds.Values
-            .Select(g => g.Guild)
-            .ToAsyncEnumerable();
-    }
-
     public Task<IEnumerable<Channel>> GetGuildChannels(ulong guildId)
     {
         if (!_guilds.TryGetValue(guildId, out var guild))

Myriad/Cache/RedisDiscordCache.cs

@@ -1,340 +0,0 @@
-using Google.Protobuf;
-
-using StackExchange.Redis;
-using StackExchange.Redis.KeyspaceIsolation;
-
-using Serilog;
-
-using Myriad.Types;
-
-namespace Myriad.Cache;
-
-#pragma warning disable 4014
-public class RedisDiscordCache: IDiscordCache
-{
-    private readonly ILogger _logger;
-    private readonly ulong _ownUserId;
-    public RedisDiscordCache(ILogger logger, ulong ownUserId)
-    {
-        _logger = logger;
-        _ownUserId = ownUserId;
-    }
-
-    private ConnectionMultiplexer _redis { get; set; }
-
-    public async Task InitAsync(string addr)
-    {
-        _redis = await ConnectionMultiplexer.ConnectAsync(addr);
-    }
-
-    private IDatabase db => _redis.GetDatabase().WithKeyPrefix("discord:");
-
-    public async ValueTask SaveGuild(Guild guild)
-    {
-        _logger.Verbose("Saving guild {GuildId} to redis", guild.Id);
-
-        var g = new CachedGuild();
-        g.Id = guild.Id;
-        g.Name = guild.Name;
-        g.OwnerId = guild.OwnerId;
-        g.PremiumTier = (int)guild.PremiumTier;
-
-        var tr = db.CreateTransaction();
-
-        tr.HashSetAsync("guilds", guild.Id.HashWrapper(g));
-
-        foreach (var role in guild.Roles)
-        {
-            // Don't call SaveRole because that updates guild state
-            // and we just got a brand new one :)
-            // actually with redis it doesn't update guild state, but we're still doing it here because transaction
-            tr.HashSetAsync("roles", role.Id.HashWrapper(new CachedRole()
-            {
-                Id = role.Id,
-                Name = role.Name,
-                Position = role.Position,
-                Permissions = (ulong)role.Permissions,
-                Mentionable = role.Mentionable,
-            }));
-
-            tr.HashSetAsync($"guild_roles:{guild.Id}", role.Id, true, When.NotExists);
-        }
-
-        await tr.ExecuteAsync();
-    }
-
-    public async ValueTask SaveChannel(Channel channel)
-    {
-        _logger.Verbose("Saving channel {ChannelId} to redis", channel.Id);
-
-        await db.HashSetAsync("channels", channel.Id.HashWrapper(channel.ToProtobuf()));
-
-        if (channel.GuildId != null)
-            await db.HashSetAsync($"guild_channels:{channel.GuildId.Value}", channel.Id, true, When.NotExists);
-
-        // todo: use a transaction for this?
-        if (channel.Recipients != null)
-            foreach (var recipient in channel.Recipients)
-                await SaveUser(recipient);
-    }
-
-    public async ValueTask SaveUser(User user)
-    {
-        _logger.Verbose("Saving user {UserId} to redis", user.Id);
-
-        var u = new CachedUser()
-        {
-            Id = user.Id,
-            Username = user.Username,
-            Discriminator = user.Discriminator,
-            Bot = user.Bot,
-        };
-
-        if (user.Avatar != null)
-            u.Avatar = user.Avatar;
-
-        await db.HashSetAsync("users", user.Id.HashWrapper(u));
-    }
-
-    public async ValueTask SaveSelfMember(ulong guildId, GuildMemberPartial member)
-    {
-        _logger.Verbose("Saving self member for guild {GuildId} to redis", guildId);
-
-        var gm = new CachedGuildMember();
-        foreach (var role in member.Roles)
-            gm.Roles.Add(role);
-
-        await db.HashSetAsync("members", guildId.HashWrapper(gm));
-    }
-
-    public async ValueTask SaveRole(ulong guildId, Myriad.Types.Role role)
-    {
-        _logger.Verbose("Saving role {RoleId} in {GuildId} to redis", role.Id, guildId);
-
-        await db.HashSetAsync("roles", role.Id.HashWrapper(new CachedRole()
-        {
-            Id = role.Id,
-            Mentionable = role.Mentionable,
-            Name = role.Name,
-            Permissions = (ulong)role.Permissions,
-            Position = role.Position,
-        }));
-
-        await db.HashSetAsync($"guild_roles:{guildId}", role.Id, true, When.NotExists);
-    }
-
-    public async ValueTask SaveDmChannelStub(ulong channelId)
-    {
-        // Use existing channel object if present, otherwise add a stub
-        // We may get a message create before channel create and we want to have it saved
-
-        if (await TryGetChannel(channelId) == null)
-            await db.HashSetAsync("channels", channelId.HashWrapper(new CachedChannel()
-            {
-                Id = channelId,
-                Type = (int)Channel.ChannelType.Dm,
-            }));
-    }
-
-    public async ValueTask RemoveGuild(ulong guildId)
-        => await db.HashDeleteAsync("guilds", guildId);
-
-    public async ValueTask RemoveChannel(ulong channelId)
-    {
-        var oldChannel = await TryGetChannel(channelId);
-
-        if (oldChannel == null)
-            return;
-
-        await db.HashDeleteAsync("channels", channelId);
-
-        if (oldChannel.GuildId != null)
-            await db.HashDeleteAsync($"guild_channels:{oldChannel.GuildId.Value}", oldChannel.Id);
-    }
-
-    public async ValueTask RemoveUser(ulong userId)
-        => await db.HashDeleteAsync("users", userId);
-
-    public ulong GetOwnUser() => _ownUserId;
-
-    public async ValueTask RemoveRole(ulong guildId, ulong roleId)
-    {
-        await db.HashDeleteAsync("roles", roleId);
-        await db.HashDeleteAsync($"guild_roles:{guildId}", roleId);
-    }
-
-    public async Task<Guild?> TryGetGuild(ulong guildId)
-    {
-        var redisGuild = await db.HashGetAsync("guilds", guildId);
-        if (redisGuild.IsNullOrEmpty)
-            return null;
-
-        var guild = ((byte[])redisGuild).Unmarshal<CachedGuild>();
-
-        var redisRoles = await db.HashGetAllAsync($"guild_roles:{guildId}");
-
-        // todo: put this in a transaction or something
-        var roles = await Task.WhenAll(redisRoles.Select(r => TryGetRole((ulong)r.Name)));
-
-#pragma warning disable 8619
-        return guild.FromProtobuf() with { Roles = roles };
-#pragma warning restore 8619
-    }
-
-    public async Task<Channel?> TryGetChannel(ulong channelId)
-    {
-        var redisChannel = await db.HashGetAsync("channels", channelId);
-        if (redisChannel.IsNullOrEmpty)
-            return null;
-
-        return ((byte[])redisChannel).Unmarshal<CachedChannel>().FromProtobuf();
-    }
-
-    public async Task<User?> TryGetUser(ulong userId)
-    {
-        var redisUser = await db.HashGetAsync("users", userId);
-        if (redisUser.IsNullOrEmpty)
-            return null;
-
-        return ((byte[])redisUser).Unmarshal<CachedUser>().FromProtobuf();
-    }
-
-    public async Task<GuildMemberPartial?> TryGetSelfMember(ulong guildId)
-    {
-        var redisMember = await db.HashGetAsync("members", guildId);
-        if (redisMember.IsNullOrEmpty)
-            return null;
-
-        return new GuildMemberPartial()
-        {
-            Roles = ((byte[])redisMember).Unmarshal<CachedGuildMember>().Roles.ToArray()
-        };
-    }
-
-    public async Task<Myriad.Types.Role?> TryGetRole(ulong roleId)
-    {
-        var redisRole = await db.HashGetAsync("roles", roleId);
-        if (redisRole.IsNullOrEmpty)
-            return null;
-
-        var role = ((byte[])redisRole).Unmarshal<CachedRole>();
-
-        return new Myriad.Types.Role()
-        {
-            Id = role.Id,
-            Name = role.Name,
-            Position = role.Position,
-            Permissions = (PermissionSet)role.Permissions,
-            Mentionable = role.Mentionable,
-        };
-    }
-
-    public IAsyncEnumerable<Guild> GetAllGuilds()
-    {
-        // return _guilds.Values
-        //     .Select(g => g.Guild)
-        //     .ToAsyncEnumerable();
-        return new Guild[] { }.ToAsyncEnumerable();
-    }
-
-    public async Task<IEnumerable<Channel>> GetGuildChannels(ulong guildId)
-    {
-        var redisChannels = await db.HashGetAllAsync($"guild_channels:{guildId}");
-        if (redisChannels.Length == 0)
-            throw new ArgumentException("Guild not found", nameof(guildId));
-
-#pragma warning disable 8619
-        return await Task.WhenAll(redisChannels.Select(c => TryGetChannel((ulong)c.Name)));
-#pragma warning restore 8619
-    }
-}
-
-internal static class CacheProtoExt
-{
-    public static Guild FromProtobuf(this CachedGuild guild)
-        => new Guild()
-        {
-            Id = guild.Id,
-            Name = guild.Name,
-            OwnerId = guild.OwnerId,
-            PremiumTier = (PremiumTier)guild.PremiumTier,
-        };
-
-    public static CachedChannel ToProtobuf(this Channel channel)
-    {
-        var c = new CachedChannel();
-        c.Id = channel.Id;
-        c.Type = (int)channel.Type;
-        if (channel.Position != null)
-            c.Position = channel.Position.Value;
-        c.Name = channel.Name;
-        if (channel.PermissionOverwrites != null)
-            foreach (var overwrite in channel.PermissionOverwrites)
-                c.PermissionOverwrites.Add(new Overwrite()
-                {
-                    Id = overwrite.Id,
-                    Type = (int)overwrite.Type,
-                    Allow = (ulong)overwrite.Allow,
-                    Deny = (ulong)overwrite.Deny,
-                });
-        if (channel.GuildId != null)
-            c.GuildId = channel.GuildId.Value;
-
-        return c;
-    }
-
-    public static Channel FromProtobuf(this CachedChannel channel)
-        => new Channel()
-        {
-            Id = channel.Id,
-            Type = (Channel.ChannelType)channel.Type,
-            Position = channel.Position,
-            Name = channel.Name,
-            PermissionOverwrites = channel.PermissionOverwrites
-                .Select(x => new Channel.Overwrite()
-                {
-                    Id = x.Id,
-                    Type = (Channel.OverwriteType)x.Type,
-                    Allow = (PermissionSet)x.Allow,
-                    Deny = (PermissionSet)x.Deny,
-                }).ToArray(),
-            GuildId = channel.HasGuildId ? channel.GuildId : null,
-            ParentId = channel.HasParentId ? channel.ParentId : null,
-        };
-
-    public static User FromProtobuf(this CachedUser user)
-        => new User()
-        {
-            Id = user.Id,
-            Username = user.Username,
-            Discriminator = user.Discriminator,
-            Avatar = user.HasAvatar ? user.Avatar : null,
-            Bot = user.Bot,
-        };
-}
-
-internal static class RedisExt
-{
-    // convenience method
-    public static HashEntry[] HashWrapper<T>(this ulong key, T value) where T : IMessage
-        => new[] { new HashEntry(key, value.ToByteArray()) };
-}
-
-public static class ProtobufExt
-{
-    private static Dictionary<string, MessageParser> _parser = new();
-
-    public static byte[] Marshal(this IMessage message) => message.ToByteArray();
-
-    public static T Unmarshal<T>(this byte[] message) where T : IMessage<T>, new()
-    {
-        var type = typeof(T).ToString();
-        if (_parser.ContainsKey(type))
-            return (T)_parser[type].ParseFrom(message);
-        else
-        {
-            _parser.Add(type, new MessageParser<T>(() => new T()));
-            return Unmarshal<T>(message);
-        }
-    }
-}

Myriad/Extensions/CacheExtensions.cs

@@ -13,27 +13,13 @@ public static class CacheExtensions
         return guild;
     }
 
-    public static async Task<Channel> GetChannel(this IDiscordCache cache, ulong channelId)
+    public static async Task<Channel> GetChannel(this IDiscordCache cache, ulong guildId, ulong channelId)
     {
-        if (!(await cache.TryGetChannel(channelId) is Channel channel))
+        if (!(await cache.TryGetChannel(guildId, channelId) is Channel channel))
             throw new KeyNotFoundException($"Channel {channelId} not found in cache");
         return channel;
     }
 
-    public static async Task<User> GetUser(this IDiscordCache cache, ulong userId)
-    {
-        if (!(await cache.TryGetUser(userId) is User user))
-            throw new KeyNotFoundException($"User {userId} not found in cache");
-        return user;
-    }
-
-    public static async Task<Role> GetRole(this IDiscordCache cache, ulong roleId)
-    {
-        if (!(await cache.TryGetRole(roleId) is Role role))
-            throw new KeyNotFoundException($"Role {roleId} not found in cache");
-        return role;
-    }
-
     public static async ValueTask<User?> GetOrFetchUser(this IDiscordCache cache, DiscordApiClient rest,
                                                         ulong userId)
     {
@@ -47,9 +33,9 @@ public static class CacheExtensions
     }
 
     public static async ValueTask<Channel?> GetOrFetchChannel(this IDiscordCache cache, DiscordApiClient rest,
-                                                              ulong channelId)
+                                                              ulong guildId, ulong channelId)
     {
-        if (await cache.TryGetChannel(channelId) is { } cacheChannel)
+        if (await cache.TryGetChannel(guildId, channelId) is { } cacheChannel)
             return cacheChannel;
 
         var restChannel = await rest.GetChannel(channelId);
@@ -58,13 +44,13 @@ public static class CacheExtensions
         return restChannel;
     }
 
-    public static async Task<Channel> GetRootChannel(this IDiscordCache cache, ulong channelOrThread)
+    public static async Task<Channel> GetRootChannel(this IDiscordCache cache, ulong guildId, ulong channelOrThread)
     {
-        var channel = await cache.GetChannel(channelOrThread);
+        var channel = await cache.GetChannel(guildId, channelOrThread);
         if (!channel.IsThread())
             return channel;
 
-        var parent = await cache.GetChannel(channel.ParentId!.Value);
+        var parent = await cache.GetChannel(guildId, channel.ParentId!.Value);
         return parent;
     }
 }

Myriad/Extensions/PermissionExtensions.cs

@@ -32,23 +32,23 @@ public static class PermissionExtensions
         PermissionSet.EmbedLinks;
 
     public static Task<PermissionSet> PermissionsForMCE(this IDiscordCache cache, MessageCreateEvent message) =>
-        PermissionsFor2(cache, message.ChannelId, message.Author.Id, message.Member, message.WebhookId != null);
+        PermissionsFor2(cache, message.GuildId ?? 0, message.ChannelId, message.Author.Id, message.Member, message.WebhookId != null);
 
     public static Task<PermissionSet>
-        PermissionsForMemberInChannel(this IDiscordCache cache, ulong channelId, GuildMember member) =>
+        PermissionsForMemberInChannel(this IDiscordCache cache, ulong guildId, ulong channelId, GuildMember member) =>
-        PermissionsFor2(cache, channelId, member.User.Id, member);
+        PermissionsFor2(cache, guildId, channelId, member.User.Id, member);
 
-    public static async Task<PermissionSet> PermissionsFor2(this IDiscordCache cache, ulong channelId, ulong userId,
+    public static async Task<PermissionSet> PermissionsFor2(this IDiscordCache cache, ulong guildId, ulong channelId, ulong userId,
                                                            GuildMemberPartial? member, bool isThread = false)
     {
-        if (!(await cache.TryGetChannel(channelId) is Channel channel))
+        if (!(await cache.TryGetChannel(guildId, channelId) is Channel channel))
             // todo: handle channel not found better
             return PermissionSet.Dm;
 
         if (channel.GuildId == null)
             return PermissionSet.Dm;
 
-        var rootChannel = await cache.GetRootChannel(channelId);
+        var rootChannel = await cache.GetRootChannel(guildId, channelId);
 
         var guild = await cache.GetGuild(channel.GuildId.Value);
 

Myriad/Gateway/ShardConnection.cs

@@ -72,7 +72,8 @@ public class ShardConnection: IAsyncDisposable
         }
         catch (Exception e)
         {
-            _logger.Error(e, "Shard {ShardId}: Error reading from WebSocket");
+            // these are never useful
+            // _logger.Error(e, "Shard {ShardId}: Error reading from WebSocket");
             // force close so we can "reset"
             await CloseInner(WebSocketCloseStatus.NormalClosure, null);
         }

Myriad/Rest/Types/Requests/ExecuteWebhookRequest.cs

@@ -13,4 +13,14 @@ public record ExecuteWebhookRequest
     public AllowedMentions? AllowedMentions { get; init; }
     public bool? Tts { get; init; }
     public Message.MessageFlags? Flags { get; set; }
+    public WebhookPoll? Poll { get; set; }
+
+    public record WebhookPoll
+    {
+        public Message.PollMedia Question { get; init; }
+        public Message.PollAnswer[] Answers { get; init; }
+        public int? Duration { get; init; }
+        public bool AllowMultiselect { get; init; }
+        public int LayoutType { get; init; }
+    }
 }

Myriad/Types/Message.cs

@@ -70,6 +70,8 @@ public record Message
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
     public Optional<Message?> ReferencedMessage { get; init; }
 
+    public MessagePoll? Poll { get; init; }
+
     // public MessageComponent[]? Components { get; init; }
 
     public record Reference(ulong? GuildId, ulong? ChannelId, ulong? MessageId);
@@ -96,4 +98,17 @@ public record Message
         public bool Me { get; init; }
         public Emoji Emoji { get; init; }
     }
+
+    public record PollMedia(string? Text, Emoji? Emoji);
+
+    public record PollAnswer(PollMedia PollMedia);
+
+    public record MessagePoll
+    {
+        public PollMedia Question { get; init; }
+        public PollAnswer[] Answers { get; init; }
+        public string? Expiry { get; init; }
+        public bool AllowMultiselect { get; init; }
+        public int LayoutType { get; init; }
+    }
 }

PluralKit.API/Controllers/v2/GroupMemberControllerV2.cs

@@ -147,6 +147,8 @@ public class GroupMemberControllerV2: PKControllerBase
     public async Task<IActionResult> GetMemberGroups(string memberRef)
     {
         var member = await ResolveMember(memberRef);
+        if (member == null)
+            throw Errors.MemberNotFoundWithRef(memberRef);
         var ctx = ContextFor(member);
 
         var system = await _repo.GetSystem(member.System);

PluralKit.Bot/ApplicationCommands/Message.cs

@@ -63,14 +63,14 @@ public class ApplicationCommandProxiedMessage
         var messageId = ctx.Event.Data!.TargetId!.Value;
 
         // check for command messages
-        var (authorId, channelId) = await ctx.Services.Resolve<CommandMessageService>().GetCommandMessage(messageId);
+        var cmessage = await ctx.Services.Resolve<CommandMessageService>().GetCommandMessage(messageId);
-        if (authorId != null)
+        if (cmessage != null)
         {
-            if (authorId != ctx.User.Id)
+            if (cmessage.AuthorId != ctx.User.Id)
                 throw new PKError("You can only delete command messages queried by this account.");
 
-            var isDM = (await _repo.GetDmChannel(ctx.User!.Id)) == channelId;
+            var isDM = (await _repo.GetDmChannel(ctx.User!.Id)) == cmessage.ChannelId;
-            await DeleteMessageInner(ctx, channelId!.Value, messageId, isDM);
+            await DeleteMessageInner(ctx, cmessage.GuildId, cmessage.ChannelId, messageId, isDM);
             return;
         }
 
@@ -78,10 +78,10 @@ public class ApplicationCommandProxiedMessage
         var message = await ctx.Repository.GetFullMessage(messageId);
         if (message != null)
         {
-            if (message.System?.Id != ctx.System.Id && message.Message.Sender != ctx.User.Id)
+            if (message.Message.Sender != ctx.User.Id && (ctx.System != null && message.System?.Id != ctx.System.Id))
                 throw new PKError("You can only delete your own messages.");
 
-            await DeleteMessageInner(ctx, message.Message.Channel, message.Message.Mid, false);
+            await DeleteMessageInner(ctx, message.Message.Guild ?? 0, message.Message.Channel, message.Message.Mid, false);
             return;
         }
 
@@ -89,9 +89,9 @@ public class ApplicationCommandProxiedMessage
         throw Errors.MessageNotFound(messageId);
     }
 
-    internal async Task DeleteMessageInner(InteractionContext ctx, ulong channelId, ulong messageId, bool isDM = false)
+    internal async Task DeleteMessageInner(InteractionContext ctx, ulong guildId, ulong channelId, ulong messageId, bool isDM = false)
     {
-        if (!((await _cache.BotPermissionsIn(channelId)).HasFlag(PermissionSet.ManageMessages) || isDM))
+        if (!((await _cache.BotPermissionsIn(guildId, channelId)).HasFlag(PermissionSet.ManageMessages) || isDM))
             throw new PKError("PluralKit does not have the *Manage Messages* permission in this channel, and thus cannot delete the message."
                 + " Please contact a server administrator to remedy this.");
 
@@ -110,7 +110,7 @@ public class ApplicationCommandProxiedMessage
         // (if not, PK shouldn't send messages on their behalf)
         var member = await _rest.GetGuildMember(ctx.GuildId, ctx.User.Id);
         var requiredPerms = PermissionSet.ViewChannel | PermissionSet.SendMessages;
-        if (member == null || !(await _cache.PermissionsForMemberInChannel(ctx.ChannelId, member)).HasFlag(requiredPerms))
+        if (member == null || !(await _cache.PermissionsForMemberInChannel(ctx.GuildId, ctx.ChannelId, member)).HasFlag(requiredPerms))
         {
             throw new PKError("You do not have permission to send messages in this channel.");
         };

PluralKit.Bot/Bot.cs

@@ -101,9 +101,7 @@ public class Bot
     {
         // we HandleGatewayEvent **before** getting the own user, because the own user is set in HandleGatewayEvent for ReadyEvent
         await _cache.HandleGatewayEvent(evt);
-
         await _cache.TryUpdateSelfMember(_config.ClientId, evt);
-
         await OnEventReceivedInner(shardId, evt);
     }
 
@@ -175,7 +173,16 @@ public class Bot
             }
 
             using var _ = LogContext.PushProperty("EventId", Guid.NewGuid());
-            using var __ = LogContext.Push(await serviceScope.Resolve<SerilogGatewayEnricherFactory>().GetEnricher(shardId, evt));
+            // this fails when cache lookup fails, so put it in a try-catch
+            try
+            {
+                using var __ = LogContext.Push(await serviceScope.Resolve<SerilogGatewayEnricherFactory>().GetEnricher(shardId, evt));
+            }
+            catch (Exception exc)
+            {
+
+                await HandleError(handler, evt, serviceScope, exc);
+            }
             _logger.Verbose("Received gateway event: {@Event}", evt);
 
             try
@@ -243,7 +250,7 @@ public class Bot
             if (!exc.ShowToUser()) return;
 
             // Once we've sent it to Sentry, report it to the user (if we have permission to)
-            var reportChannel = handler.ErrorChannelFor(evt, _config.ClientId);
+            var (guildId, reportChannel) = handler.ErrorChannelFor(evt, _config.ClientId);
             if (reportChannel == null)
             {
                 if (evt is InteractionCreateEvent ice && ice.Type == Interaction.InteractionType.ApplicationCommand)
@@ -251,7 +258,7 @@ public class Bot
                 return;
             }
 
-            var botPerms = await _cache.BotPermissionsIn(reportChannel.Value);
+            var botPerms = await _cache.BotPermissionsIn(guildId ?? 0, reportChannel.Value);
             if (botPerms.HasFlag(PermissionSet.SendMessages | PermissionSet.EmbedLinks))
                 await _errorMessageService.SendErrorMessage(reportChannel.Value, sentryEvent.EventId.ToString());
         }

PluralKit.Bot/BotConfig.cs

@@ -20,7 +20,9 @@ public class BotConfig
 
     public string? GatewayQueueUrl { get; set; }
     public bool UseRedisRatelimiter { get; set; } = false;
-    public bool UseRedisCache { get; set; } = false;
+
+    public string? HttpCacheUrl { get; set; }
+    public bool HttpUseInnerCache { get; set; } = false;
 
     public string? RedisGatewayUrl { get; set; }
 

PluralKit.Bot/BotMetrics.cs

@@ -136,4 +136,11 @@ public static class BotMetrics
         DurationUnit = TimeUnit.Seconds,
         Context = "Bot"
     };
+
+    public static MeterOptions CacheDebug => new()
+    {
+        Name = "Bad responses to cache lookups",
+        Context = "Bot",
+        MeasurementUnit = Unit.Calls
+    };
 }

PluralKit.Bot/CommandMeta/CommandHelp.cs

@@ -82,6 +82,7 @@ public partial class CommandTree
     public static Command SwitchMove = new Command("switch move", "switch move <date/time>", "Moves the latest switch in time");
     public static Command SwitchEdit = new Command("switch edit", "switch edit <member> [member 2] [member 3...]", "Edits the members in the latest switch");
     public static Command SwitchEditOut = new Command("switch edit out", "switch edit out", "Turns the latest switch into a switch-out");
+    public static Command SwitchCopy = new Command("switch copy", "switch copy <member> [member 2] [member 3...]", "Makes a new switch with the listed members added");
     public static Command SwitchDelete = new Command("switch delete", "switch delete", "Deletes the latest switch");
     public static Command SwitchDeleteAll = new Command("switch delete", "switch delete all", "Deletes all logged switches");
     public static Command Link = new Command("link", "link <account>", "Links your system to another account");
@@ -92,6 +93,7 @@ public partial class CommandTree
     public static Command Export = new Command("export", "export", "Exports system information to a data file");
     public static Command Help = new Command("help", "help", "Shows help information about PluralKit");
     public static Command Explain = new Command("explain", "explain", "Explains the basics of systems and proxying");
+    public static Command Dashboard = new Command("dashboard", "dashboard", "Get a link to the PluralKit dashboard");
     public static Command Message = new Command("message", "message <id|link> [delete|author]", "Looks up a proxied message");
     public static Command MessageEdit = new Command("edit", "edit [link] <text>", "Edit a previously proxied message");
     public static Command MessageReproxy = new Command("reproxy", "reproxy [link] <member>", "Reproxy a previously proxied message using a different member");
@@ -137,7 +139,7 @@ public partial class CommandTree
 
     public static Command[] SwitchCommands =
     {
-        Switch, SwitchOut, SwitchMove, SwitchEdit, SwitchEditOut, SwitchDelete, SwitchDeleteAll
+        Switch, SwitchOut, SwitchMove, SwitchEdit, SwitchEditOut, SwitchDelete, SwitchDeleteAll, SwitchCopy
     };
 
     public static Command[] ConfigCommands =

PluralKit.Bot/CommandMeta/CommandTree.cs

@@ -18,7 +18,7 @@ public partial class CommandTree
             return CommandHelpRoot(ctx);
         if (ctx.Match("ap", "autoproxy", "auto"))
             return HandleAutoproxyCommand(ctx);
-        if (ctx.Match("config", "cfg"))
+        if (ctx.Match("config", "cfg", "configure"))
             return HandleConfigCommand(ctx);
         if (ctx.Match("list", "find", "members", "search", "query", "l", "f", "fd", "ls"))
             return ctx.Execute<SystemList>(SystemList, m => m.MemberList(ctx, ctx.System));
@@ -105,6 +105,8 @@ public partial class CommandTree
                 return ctx.Execute<Random>(GroupRandom, r => r.Group(ctx, ctx.System));
             else
                 return ctx.Execute<Random>(MemberRandom, m => m.Member(ctx, ctx.System));
+        if (ctx.Match("dashboard", "dash"))
+            return ctx.Execute<Help>(Dashboard, m => m.Dashboard(ctx));
 
         // remove compiler warning
         return ctx.Reply(
@@ -430,13 +432,15 @@ public partial class CommandTree
                 await ctx.Execute<Switch>(SwitchEdit, m => m.SwitchEdit(ctx));
         else if (ctx.Match("delete", "remove", "erase", "cancel", "yeet"))
             await ctx.Execute<Switch>(SwitchDelete, m => m.SwitchDelete(ctx));
+        else if (ctx.Match("copy", "add", "duplicate", "dupe"))
+            await ctx.Execute<Switch>(SwitchCopy, m => m.SwitchEdit(ctx, true));
         else if (ctx.Match("commands", "help"))
             await PrintCommandList(ctx, "switching", SwitchCommands);
         else if (ctx.HasNext()) // there are following arguments
             await ctx.Execute<Switch>(Switch, m => m.SwitchDo(ctx));
         else
             await PrintCommandNotFoundError(ctx, Switch, SwitchOut, SwitchMove, SwitchEdit, SwitchEditOut,
-                SwitchDelete, SystemFronter, SystemFrontHistory);
+                SwitchDelete, SwitchCopy, SystemFronter, SystemFrontHistory);
     }
 
     private async Task CommandHelpRoot(Context ctx)
@@ -544,6 +548,8 @@ public partial class CommandTree
             return ctx.Execute<Config>(null, m => m.HidDisplayCaps(ctx));
         if (ctx.MatchMultiple(new[] { "pad" }, new[] { "id", "ids" }) || ctx.MatchMultiple(new[] { "id" }, new[] { "pad", "padding" }) || ctx.Match("idpad", "padid", "padids"))
             return ctx.Execute<Config>(null, m => m.HidListPadding(ctx));
+        if (ctx.MatchMultiple(new[] { "member", "group" }, new[] { "limit" }) || ctx.Match("limit"))
+            return ctx.Execute<Config>(null, m => m.LimitUpdate(ctx));
 
         // todo: maybe add the list of configuration keys here?
         return ctx.Reply($"{Emojis.Error} Could not find a setting with that name. Please see `pk;commands config` for the list of possible config settings.");

PluralKit.Bot/CommandSystem/Context/Context.cs

@@ -72,7 +72,7 @@ public class Context
     public readonly int ShardId;
     public readonly Cluster Cluster;
 
-    public Task<PermissionSet> BotPermissions => Cache.BotPermissionsIn(Channel.Id);
+    public Task<PermissionSet> BotPermissions => Cache.BotPermissionsIn(Guild?.Id ?? 0, Channel.Id);
     public Task<PermissionSet> UserPermissions => Cache.PermissionsForMCE((MessageCreateEvent)Message);
 
 
@@ -110,7 +110,7 @@ public class Context
         // {
         // Sensitive information that might want to be deleted by :x: reaction is typically in an embed format (member cards, for example)
         // but since we can, we just store all sent messages for possible deletion
-        await _commandMessageService.RegisterMessage(msg.Id, msg.ChannelId, Author.Id);
+        await _commandMessageService.RegisterMessage(msg.Id, Guild?.Id ?? 0, msg.ChannelId, Author.Id);
         // }
 
         return msg;

PluralKit.Bot/CommandSystem/Context/ContextArgumentsExt.cs

@@ -91,8 +91,12 @@ public static class ContextArgumentsExt
     public static bool MatchClear(this Context ctx)
         => ctx.Match("clear", "reset", "default") || ctx.MatchFlag("c", "clear");
 
-    public static bool MatchRaw(this Context ctx) =>
+    public static ReplyFormat MatchFormat(this Context ctx)
-        ctx.Match("r", "raw") || ctx.MatchFlag("r", "raw");
+    {
+        if (ctx.Match("r", "raw") || ctx.MatchFlag("r", "raw")) return ReplyFormat.Raw;
+        if (ctx.Match("pt", "plaintext") || ctx.MatchFlag("pt", "plaintext")) return ReplyFormat.Plaintext;
+        return ReplyFormat.Standard;
+    }
 
     public static bool MatchToggle(this Context ctx, bool? defaultValue = null)
     {
@@ -184,4 +188,11 @@ public static class ContextArgumentsExt
 
         return groups;
     }
+}
+
+public enum ReplyFormat
+{
+    Standard,
+    Raw,
+    Plaintext
 }

PluralKit.Bot/CommandSystem/Context/ContextEntityArgumentsExt.cs

@@ -188,7 +188,8 @@ public static class ContextEntityArgumentsExt
         if (!MentionUtils.TryParseChannel(ctx.PeekArgument(), out var id))
             return null;
 
-        var channel = await ctx.Cache.TryGetChannel(id);
+        // todo: match channels in other guilds
+        var channel = await ctx.Cache.TryGetChannel(ctx.Guild!.Id, id);
         if (channel == null)
             channel = await ctx.Rest.GetChannelOrNull(id);
         if (channel == null)

PluralKit.Bot/Commands/Admin.cs

@@ -1,8 +1,12 @@
 using System.Text.RegularExpressions;
 
+using Humanizer;
 using Dapper;
 using SqlKata;
 
+using Myriad.Builders;
+using Myriad.Extensions;
+using Myriad.Cache;
 using Myriad.Rest;
 using Myriad.Types;
 
@@ -14,11 +18,64 @@ public class Admin
 {
     private readonly BotConfig _botConfig;
     private readonly DiscordApiClient _rest;
+    private readonly IDiscordCache _cache;
 
-    public Admin(BotConfig botConfig, DiscordApiClient rest)
+    public Admin(BotConfig botConfig, DiscordApiClient rest, IDiscordCache cache)
     {
         _botConfig = botConfig;
         _rest = rest;
+        _cache = cache;
+    }
+
+    public async Task<Embed> CreateEmbed(Context ctx, PKSystem system)
+    {
+        string UntilLimit(int count, int limit)
+        {
+            var brackets = new List<int> { 10, 25, 50, 100 };
+            if (count == limit)
+                return "(at limit)";
+
+            foreach (var x in brackets)
+            {
+                if (limit - x <= count)
+                    return $"(approx. {x} to limit)";
+            }
+
+            return "";
+        }
+
+        Task<(ulong Id, User? User)[]> GetUsers(IEnumerable<ulong> ids)
+        {
+            async Task<(ulong Id, User? User)> Inner(ulong id)
+            {
+                var user = await _cache.GetOrFetchUser(_rest, id);
+                return (id, user);
+            }
+
+            return Task.WhenAll(ids.Select(Inner));
+        }
+
+        var config = await ctx.Repository.GetSystemConfig(system.Id);
+
+        // Fetch/render info for all accounts simultaneously
+        var accounts = await ctx.Repository.GetSystemAccounts(system.Id);
+        var users = (await GetUsers(accounts)).Select(x => x.User?.NameAndMention() ?? $"(deleted: `{x.Id}`)");
+
+        var eb = new EmbedBuilder()
+            .Title("System info")
+            .Color(DiscordUtils.Green)
+            .Field(new Embed.Field("System ID", $"`{system.Hid}`"))
+            .Field(new Embed.Field("Linked accounts", string.Join("\n", users).Truncate(1000)));
+
+        var memberLimit = config.MemberLimitOverride ?? Limits.MaxMemberCount;
+        var memberCount = await ctx.Repository.GetSystemMemberCount(system.Id);
+        eb.Field(new Embed.Field("Member limit", $"{memberLimit} {UntilLimit(memberCount, memberLimit)}", true));
+
+        var groupLimit = config.GroupLimitOverride ?? Limits.MaxGroupCount;
+        var groupCount = await ctx.Repository.GetSystemGroupCount(system.Id);
+        eb.Field(new Embed.Field("Group limit", $"{groupLimit} {UntilLimit(groupCount, groupLimit)}", true));
+
+        return eb.Build();
     }
 
     public async Task UpdateSystemId(Context ctx)
@@ -37,6 +94,8 @@ public class Admin
         if (existingSystem != null)
             throw new PKError($"Another system already exists with ID `{newHid}`.");
 
+        await ctx.Reply(null, await CreateEmbed(ctx, target));
+
         if (!await ctx.PromptYesNo($"Change system ID of `{target.Hid}` to `{newHid}`?", "Change"))
             throw new PKError("ID change cancelled.");
 
@@ -60,6 +119,9 @@ public class Admin
         if (existingMember != null)
             throw new PKError($"Another member already exists with ID `{newHid}`.");
 
+        var system = await ctx.Repository.GetSystem(target.System);
+        await ctx.Reply(null, await CreateEmbed(ctx, system));
+
         if (!await ctx.PromptYesNo(
             $"Change member ID of **{target.NameFor(LookupContext.ByNonOwner)}** (`{target.Hid}`) to `{newHid}`?",
             "Change"
@@ -86,6 +148,9 @@ public class Admin
         if (existingGroup != null)
             throw new PKError($"Another group already exists with ID `{newHid}`.");
 
+        var system = await ctx.Repository.GetSystem(target.System);
+        await ctx.Reply(null, await CreateEmbed(ctx, system));
+
         if (!await ctx.PromptYesNo($"Change group ID of **{target.Name}** (`{target.Hid}`) to `{newHid}`?",
             "Change"
         ))
@@ -103,6 +168,8 @@ public class Admin
         if (target == null)
             throw new PKError("Unknown system.");
 
+        await ctx.Reply(null, await CreateEmbed(ctx, target));
+
         if (!await ctx.PromptYesNo($"Reroll system ID `{target.Hid}`?", "Reroll"))
             throw new PKError("ID change cancelled.");
 
@@ -124,6 +191,9 @@ public class Admin
         if (target == null)
             throw new PKError("Unknown member.");
 
+        var system = await ctx.Repository.GetSystem(target.System);
+        await ctx.Reply(null, await CreateEmbed(ctx, system));
+
         if (!await ctx.PromptYesNo(
             $"Reroll member ID for **{target.NameFor(LookupContext.ByNonOwner)}** (`{target.Hid}`)?",
             "Reroll"
@@ -148,6 +218,9 @@ public class Admin
         if (target == null)
             throw new PKError("Unknown group.");
 
+        var system = await ctx.Repository.GetSystem(target.System);
+        await ctx.Reply(null, await CreateEmbed(ctx, system));
+
         if (!await ctx.PromptYesNo($"Reroll group ID for **{target.Name}** (`{target.Hid}`)?",
             "Change"
         ))
@@ -176,7 +249,7 @@ public class Admin
         var currentLimit = config.MemberLimitOverride ?? Limits.MaxMemberCount;
         if (!ctx.HasNext())
         {
-            await ctx.Reply($"Current member limit is **{currentLimit}** members.");
+            await ctx.Reply(null, await CreateEmbed(ctx, target));
             return;
         }
 
@@ -184,6 +257,7 @@ public class Admin
         if (!int.TryParse(newLimitStr, out var newLimit))
             throw new PKError($"Couldn't parse `{newLimitStr}` as number.");
 
+        await ctx.Reply(null, await CreateEmbed(ctx, target));
         if (!await ctx.PromptYesNo($"Update member limit from **{currentLimit}** to **{newLimit}**?", "Update"))
             throw new PKError("Member limit change cancelled.");
 
@@ -204,7 +278,7 @@ public class Admin
         var currentLimit = config.GroupLimitOverride ?? Limits.MaxGroupCount;
         if (!ctx.HasNext())
         {
-            await ctx.Reply($"Current group limit is **{currentLimit}** groups.");
+            await ctx.Reply(null, await CreateEmbed(ctx, target));
             return;
         }
 
@@ -212,6 +286,7 @@ public class Admin
         if (!int.TryParse(newLimitStr, out var newLimit))
             throw new PKError($"Couldn't parse `{newLimitStr}` as number.");
 
+        await ctx.Reply(null, await CreateEmbed(ctx, target));
         if (!await ctx.PromptYesNo($"Update group limit from **{currentLimit}** to **{newLimit}**?", "Update"))
             throw new PKError("Group limit change cancelled.");
 
@@ -243,6 +318,7 @@ public class Admin
             throw Errors.AccountInOtherSystem(existingAccount, ctx.Config);
 
         var system = await ctx.Repository.GetSystem(systemId.Value!);
+        await ctx.Reply(null, await CreateEmbed(ctx, system));
 
         if (!await ctx.PromptYesNo($"Associate account {account.NameAndMention()} with system `{system.Hid}`?", "Recover account"))
             throw new PKError("System recovery cancelled.");

PluralKit.Bot/Commands/Checks.cs

@@ -143,6 +143,7 @@ public class Checks
         var error = "Channel not found or you do not have permissions to access it.";
 
         // todo: this breaks if channel is not in cache and bot does not have View Channel permissions
+        // with new cache it breaks if channel is not in current guild
         var channel = await ctx.MatchChannel();
         if (channel == null || channel.GuildId == null)
             throw new PKError(error);
@@ -156,7 +157,8 @@ public class Checks
         if (!await ctx.CheckPermissionsInGuildChannel(channel, PermissionSet.ViewChannel))
             throw new PKError(error);
 
-        var botPermissions = await _cache.BotPermissionsIn(channel.Id);
+        // todo: permcheck channel outside of guild?
+        var botPermissions = await _cache.BotPermissionsIn(ctx.Guild.Id, channel.Id);
 
         // We use a bitfield so we can set individual permission bits
         ulong missingPermissions = 0;
@@ -231,11 +233,11 @@ public class Checks
         var channel = await _rest.GetChannelOrNull(channelId.Value);
         if (channel == null)
             throw new PKError("Unable to get the channel associated with this message.");
-
-        var rootChannel = await _cache.GetRootChannel(channel.Id);
         if (channel.GuildId == null)
             throw new PKError("PluralKit is not able to proxy messages in DMs.");
 
+        var rootChannel = await _cache.GetRootChannel(channel.GuildId!.Value, channel.Id);
+
         // using channel.GuildId here since _rest.GetMessage() doesn't return the GuildId
         var context = await ctx.Repository.GetMessageContext(msg.Author.Id, channel.GuildId.Value, rootChannel.Id, msg.ChannelId);
         var members = (await ctx.Repository.GetProxyMembers(msg.Author.Id, channel.GuildId.Value)).ToList();

PluralKit.Bot/Commands/Config.cs

@@ -536,4 +536,10 @@ public class Config
         }
         else throw new PKError(badInputError);
     }
+
+    public Task LimitUpdate(Context ctx)
+    {
+        throw new PKError("You cannot update your own member or group limits. If you need a limit update, please join the " +
+        "support server and ask in #bot-support: https://discord.gg/PczBt78");
+    }
 }

PluralKit.Bot/Commands/GroupMember.cs

@@ -53,41 +53,55 @@ public class GroupMember
 
     public async Task ListMemberGroups(Context ctx, PKMember target)
     {
-        var pctx = ctx.DirectLookupContextFor(target.System);
+        var targetSystem = await ctx.Repository.GetSystem(target.System);
+        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(target.System), ctx.LookupContextFor(target.System));
+        opts.MemberFilter = target.Id;
 
-        var groups = await ctx.Repository.GetMemberGroups(target.Id)
+        var title = new StringBuilder($"Groups containing {target.Name} (`{target.DisplayHid(ctx.Config)}`) in ");
-            .Where(g => g.Visibility.CanAccess(pctx))
+        if (ctx.Guild != null)
-            .OrderBy(g => (g.DisplayName ?? g.Name), StringComparer.InvariantCultureIgnoreCase)
-            .ToListAsync();
-
-        var description = "";
-        var msg = "";
-
-        if (groups.Count == 0)
-            description = "This member has no groups.";
-        else
-            description = string.Join("\n", groups.Select(g => $"[`{g.DisplayHid(ctx.Config, isList: true)}`] **{g.DisplayName ?? g.Name}**"));
-
-        if (pctx == LookupContext.ByOwner)
         {
-            msg +=
+            var guildSettings = await ctx.Repository.GetSystemGuild(ctx.Guild.Id, targetSystem.Id);
-                $"\n\nTo add this member to one or more groups, use `pk;m {target.Reference(ctx)} group add <group> [group 2] [group 3...]`";
+            if (guildSettings.DisplayName != null)
-            if (groups.Count > 0)
+                title.Append($"{guildSettings.DisplayName} (`{targetSystem.DisplayHid(ctx.Config)}`)");
-                msg +=
+            else if (targetSystem.NameFor(ctx) != null)
-                    $"\nTo remove this member from one or more groups, use `pk;m {target.Reference(ctx)} group remove <group> [group 2] [group 3...]`";
+                title.Append($"{targetSystem.NameFor(ctx)} (`{targetSystem.DisplayHid(ctx.Config)}`)");
+            else
+                title.Append($"`{targetSystem.DisplayHid(ctx.Config)}`");
         }
+        else
+        {
+            if (targetSystem.NameFor(ctx) != null)
+                title.Append($"{targetSystem.NameFor(ctx)} (`{targetSystem.DisplayHid(ctx.Config)}`)");
+            else
+                title.Append($"`{targetSystem.DisplayHid(ctx.Config)}`");
+        }
+        if (opts.Search != null)
+            title.Append($" matching **{opts.Search.Truncate(100)}**");
 
-        await ctx.Reply(msg, new EmbedBuilder().Title($"{target.Name}'s groups").Description(description).Build());
+        await ctx.RenderGroupList(ctx.LookupContextFor(target.System), target.System, title.ToString(),
+            target.Color, opts);
     }
 
     public async Task AddRemoveMembers(Context ctx, PKGroup target, Groups.AddRemoveOperation op)
     {
         ctx.CheckOwnGroup(target);
 
-        var members = (await ctx.ParseMemberList(ctx.System.Id))
+        List<MemberId> members;
+        if (ctx.MatchFlag("all", "a"))
+        {
+            members = (await ctx.Database.Execute(conn => conn.QueryMemberList(target.System,
+                    new DatabaseViewsExt.ListQueryOptions { })))
+                .Select(m => m.Id)
+                .Distinct()
+                .ToList();
+        }
+        else
+        {
+            members = (await ctx.ParseMemberList(ctx.System.Id))
             .Select(m => m.Id)
             .Distinct()
             .ToList();
+        }
 
         var existingMembersInGroup = (await ctx.Database.Execute(conn => conn.QueryMemberList(target.System,
                 new DatabaseViewsExt.ListQueryOptions { GroupFilter = target.Id })))
@@ -127,7 +141,7 @@ public class GroupMember
         var targetSystem = await GetGroupSystem(ctx, target);
         ctx.CheckSystemPrivacy(targetSystem.Id, target.ListPrivacy);
 
-        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(target.System));
+        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(target.System), ctx.LookupContextFor(target.System));
         opts.GroupFilter = target.Id;
 
         var title = new StringBuilder($"Members of {target.DisplayName ?? target.Name} (`{target.DisplayHid(ctx.Config)}`) in ");

PluralKit.Bot/Commands/Groups.cs

@@ -132,40 +132,47 @@ public class Groups
 
         // No perms check, display name isn't covered by member privacy
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
             if (target.DisplayName == null)
+            {
                 await ctx.Reply(noDisplayNameSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{target.DisplayName}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"```\n{target.DisplayName}\n```");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing displayname for group {target.Reference(ctx)}");
+            await ctx.Reply(target.DisplayName, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (target.DisplayName == null)
+            var eb = new EmbedBuilder()
-            {
+                .Field(new Embed.Field("Name", target.Name))
-                await ctx.Reply(noDisplayNameSetMessage);
+                .Field(new Embed.Field("Display Name", target.DisplayName));
-            }
-            else
-            {
-                var eb = new EmbedBuilder()
-                    .Field(new Embed.Field("Name", target.Name))
-                    .Field(new Embed.Field("Display Name", target.DisplayName));
 
-                var reference = target.Reference(ctx);
+            var reference = target.Reference(ctx);
 
-                if (ctx.System?.Id == target.System)
+            if (ctx.System?.Id == target.System)
-                    eb.Description(
+                eb.Description(
-                        $"To change display name, type `pk;group {reference} displayname <display name>`.\n"
+                    $"To change display name, type `pk;group {reference} displayname <display name>`.\n"
-                        + $"To clear it, type `pk;group {reference} displayname -clear`.\n"
+                    + $"To clear it, type `pk;group {reference} displayname -clear`.\n"
-                        + $"To print the raw display name, type `pk;group {reference} displayname -raw`.");
+                    + $"To print the raw display name, type `pk;group {reference} displayname -raw`.");
 
-                if (ctx.System?.Id == target.System)
+            if (ctx.System?.Id == target.System)
-                    eb.Footer(new Embed.EmbedFooter($"Using {target.DisplayName.Length}/{Limits.MaxGroupNameLength} characters."));
+                eb.Footer(new Embed.EmbedFooter($"Using {target.DisplayName.Length}/{Limits.MaxGroupNameLength} characters."));
 
-                await ctx.Reply(embed: eb.Build());
+            await ctx.Reply(embed: eb.Build());
-            }
 
             return;
         }
@@ -184,6 +191,8 @@ public class Groups
         else
         {
             var newDisplayName = ctx.RemainderOrNull(false).NormalizeLineEndSpacing();
+            if (newDisplayName.Length > Limits.MaxGroupNameLength)
+                throw new PKError($"Group name too long ({newDisplayName.Length}/{Limits.MaxGroupNameLength} characters).");
 
             var patch = new GroupPatch { DisplayName = Partial<string>.Present(newDisplayName) };
             await ctx.Repository.UpdateGroup(target.Id, patch);
@@ -201,30 +210,41 @@ public class Groups
             noDescriptionSetMessage +=
                 $" To set one, type `pk;group {target.Reference(ctx)} description <description>`.";
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
             if (target.Description == null)
+            {
                 await ctx.Reply(noDescriptionSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{target.Description}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"```\n{target.Description}\n```");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing description for group {target.Reference(ctx)}");
+            await ctx.Reply(target.Description, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (target.Description == null)
+            await ctx.Reply(embed: new EmbedBuilder()
-                await ctx.Reply(noDescriptionSetMessage);
+                .Title("Group description")
-            else
+                .Description(target.Description)
-                await ctx.Reply(embed: new EmbedBuilder()
+                .Field(new Embed.Field("\u200B",
-                    .Title("Group description")
+                    $"To print the description with formatting, type `pk;group {target.Reference(ctx)} description -raw`."
-                    .Description(target.Description)
+                    + (ctx.System?.Id == target.System
-                    .Field(new Embed.Field("\u200B",
+                        ? $" To clear it, type `pk;group {target.Reference(ctx)} description -clear`."
-                        $"To print the description with formatting, type `pk;group {target.Reference(ctx)} description -raw`."
+                        : "")
-                        + (ctx.System?.Id == target.System
+                        + $" Using {target.Description.Length}/{Limits.MaxDescriptionLength} characters."))
-                            ? $" To clear it, type `pk;group {target.Reference(ctx)} description -clear`."
+                .Build());
-                            : "")
-                            + $" Using {target.Description.Length}/{Limits.MaxDescriptionLength} characters."))
-                    .Build());
             return;
         }
 
@@ -385,7 +405,7 @@ public class Groups
     public async Task GroupColor(Context ctx, PKGroup target)
     {
         var isOwnSystem = ctx.System?.Id == target.System;
-        var matchedRaw = ctx.MatchRaw();
+        var matchedFormat = ctx.MatchFormat();
         var matchedClear = ctx.MatchClear();
 
         if (!isOwnSystem || !(ctx.HasNext() || matchedClear))
@@ -393,8 +413,10 @@ public class Groups
             if (target.Color == null)
                 await ctx.Reply(
                     "This group does not have a color set." + (isOwnSystem ? $" To set one, type `pk;group {target.Reference(ctx)} color <color>`." : ""));
-            else if (matchedRaw)
+            else if (matchedFormat == ReplyFormat.Raw)
                 await ctx.Reply("```\n#" + target.Color + "\n```");
+            else if (matchedFormat == ReplyFormat.Plaintext)
+                await ctx.Reply(target.Color);
             else
                 await ctx.Reply(embed: new EmbedBuilder()
                     .Title("Group color")
@@ -446,7 +468,7 @@ public class Groups
         // - ParseListOptions checks list access privacy and sets the privacy filter (which members show up in list)
         // - RenderGroupList checks the indivual privacy for each member (NameFor, etc)
         // the own system is always allowed to look up their list
-        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(system.Id));
+        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(system.Id), ctx.LookupContextFor(system.Id));
         await ctx.RenderGroupList(
             ctx.LookupContextFor(system.Id),
             system.Id,
@@ -460,8 +482,8 @@ public class Groups
     {
         var title = new StringBuilder("Groups of ");
 
-        if (target.Name != null)
+        if (target.NameFor(ctx) != null)
-            title.Append($"{target.Name} (`{target.DisplayHid(ctx.Config)}`)");
+            title.Append($"{target.NameFor(ctx)} (`{target.DisplayHid(ctx.Config)}`)");
         else
             title.Append($"`{target.DisplayHid(ctx.Config)}`");
 

PluralKit.Bot/Commands/Help.cs

@@ -17,6 +17,26 @@ public class Help
 
     private static Dictionary<string, Embed.Field[]> helpEmbedPages = new Dictionary<string, Embed.Field[]>
     {
+        {
+            "default",
+            new Embed.Field[]
+            {
+                new
+                (
+                    "System Recovery",
+                    "In the case of your Discord account getting lost or deleted, the PluralKit staff can help you recover your system. "
+                    + "In order to do so, we will need your **PluralKit token**. This is the *only* way you can prove ownership so we can help you recover your system. "
+                    + "To get it, run `pk;token` and then store it in a safe place.\n\n"
+                    + "Keep your token safe, if other people get access to it they can also use it to access your system. "
+                    + "If your token is ever compromised run `pk;token refresh` to invalidate the old token and get a new one."
+                ),
+                new
+                (
+                    "Use the buttons below to see more info!",
+                    ""
+                )
+            }
+        },
         {
             "basicinfo",
             new Embed.Field[]
@@ -31,7 +51,7 @@ public class Help
                 (
                     "Why are people's names saying [APP] or [BOT] next to them?",
                     "These people are not actually apps or bots, this is just a Discord limitation. See [the documentation](https://pluralkit.me/guide#proxying) for an in-depth explanation."
-                ),
+                )
             }
         },
         {
@@ -137,7 +157,9 @@ public class Help
     public Task HelpRoot(Context ctx)
         => ctx.Rest.CreateMessage(ctx.Channel.Id, new MessageRequest
         {
-            Embeds = new[] { helpEmbed with { Description = helpEmbed.Description + "\n\n**Use the buttons below to see more info!**" } },
+            Content = $"{Emojis.Warn} If you cannot see the rest of this message see [the FAQ](<https://pluralkit.me/faq/#why-do-most-of-pluralkit-s-messages-look-blank-or-empty>)",
+            Embeds = new[] { helpEmbed with { Description = helpEmbed.Description,
+                                              Fields = helpEmbedPages.GetValueOrDefault("default") } },
             Components = new[] { helpPageButtons(ctx.Author.Id) },
         });
 
@@ -151,7 +173,7 @@ public class Help
         if (ctx.Event.Message.Components.First().Components.Where(x => x.CustomId == ctx.CustomId).First().Style == ButtonStyle.Primary)
             return ctx.Respond(InteractionResponse.ResponseType.UpdateMessage, new()
             {
-                Embeds = new[] { helpEmbed with { Description = helpEmbed.Description + "\n\n**Use the buttons below to see more info!**" } },
+                Embeds = new[] { helpEmbed with { Fields = helpEmbedPages.GetValueOrDefault("default") } },
                 Components = new[] { buttons }
             });
 
@@ -172,4 +194,6 @@ public class Help
     });
 
     public Task Explain(Context ctx) => ctx.Reply(explanation);
+
+    public Task Dashboard(Context ctx) => ctx.Reply("The PluralKit dashboard is at <https://dash.pluralkit.me>");
 }

PluralKit.Bot/Commands/Lists/ContextListExt.cs

@@ -11,7 +11,7 @@ namespace PluralKit.Bot;
 
 public static class ContextListExt
 {
-    public static ListOptions ParseListOptions(this Context ctx, LookupContext lookupCtx)
+    public static ListOptions ParseListOptions(this Context ctx, LookupContext directLookupCtx, LookupContext lookupContext)
     {
         var p = new ListOptions();
 
@@ -55,10 +55,13 @@ public static class ContextListExt
         if (ctx.MatchFlag("private-only", "po")) p.PrivacyFilter = PrivacyLevel.Private;
 
         // PERM CHECK: If we're trying to access non-public members of another system, error
-        if (p.PrivacyFilter != PrivacyLevel.Public && lookupCtx != LookupContext.ByOwner)
+        if (p.PrivacyFilter != PrivacyLevel.Public && directLookupCtx != LookupContext.ByOwner)
             // TODO: should this just return null instead of throwing or something? >.>
             throw Errors.NotOwnInfo;
 
+        //this is for searching
+        p.Context = lookupContext;
+
         // Additional fields to include in the search results
         if (ctx.MatchFlag("with-last-switch", "with-last-fronted", "with-last-front", "wls", "wlf"))
             p.IncludeLastSwitch = true;
@@ -124,11 +127,14 @@ public static class ContextListExt
 
         void ShortRenderer(EmbedBuilder eb, IEnumerable<ListedMember> page)
         {
+            // if there are both 5 and 6 character Hids they should be padded to align correctly.
+            var shouldPad = page.Any(x => x.Hid.Length > 5);
+
             // We may end up over the description character limit
             // so run it through a helper that "makes it work" :)
             eb.WithSimpleLineContent(page.Select(m =>
             {
-                var ret = $"[`{m.DisplayHid(ctx.Config, isList: true)}`] **{m.NameFor(ctx)}** ";
+                var ret = $"[`{m.DisplayHid(ctx.Config, isList: true, shouldPad: shouldPad)}`] **{m.NameFor(ctx)}** ";
 
                 if (opts.IncludeMessageCount && m.MessageCountFor(lookupCtx) is { } count)
                     ret += $"({count} messages)";
@@ -234,11 +240,14 @@ public static class ContextListExt
 
         void ShortRenderer(EmbedBuilder eb, IEnumerable<ListedGroup> page)
         {
+            // if there are both 5 and 6 character Hids they should be padded to align correctly.
+            var shouldPad = page.Any(x => x.Hid.Length > 5);
+
             // We may end up over the description character limit
             // so run it through a helper that "makes it work" :)
             eb.WithSimpleLineContent(page.Select(g =>
             {
-                var ret = $"[`{g.DisplayHid(ctx.Config, isList: true)}`] **{g.NameFor(ctx)}** ";
+                var ret = $"[`{g.DisplayHid(ctx.Config, isList: true, shouldPad: shouldPad)}`] **{g.NameFor(ctx)}** ";
 
                 switch (opts.SortProperty)
                 {

PluralKit.Bot/Commands/Lists/ListOptions.cs

@@ -28,7 +28,9 @@ public class ListOptions
     public bool Reverse { get; set; }
 
     public PrivacyLevel? PrivacyFilter { get; set; } = PrivacyLevel.Public;
+    public LookupContext Context { get; set; } = LookupContext.ByNonOwner;
     public GroupId? GroupFilter { get; set; }
+    public MemberId? MemberFilter { get; set; }
     public string? Search { get; set; }
     public bool SearchDescription { get; set; }
 
@@ -96,8 +98,10 @@ public class ListOptions
         {
             PrivacyFilter = PrivacyFilter,
             GroupFilter = GroupFilter,
+            MemberFilter = MemberFilter,
             Search = Search,
-            SearchDescription = SearchDescription
+            SearchDescription = SearchDescription,
+            Context = Context
         };
 }
 

PluralKit.Bot/Commands/MemberEdit.cs

@@ -70,30 +70,41 @@ public class MemberEdit
             noDescriptionSetMessage +=
                 $" To set one, type `pk;member {target.Reference(ctx)} description <description>`.";
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
             if (target.Description == null)
+            {
                 await ctx.Reply(noDescriptionSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{target.Description}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"```\n{target.Description}\n```");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing description for member {target.Reference(ctx)}");
+            await ctx.Reply(target.Description, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (target.Description == null)
+            await ctx.Reply(embed: new EmbedBuilder()
-                await ctx.Reply(noDescriptionSetMessage);
+                .Title("Member description")
-            else
+                .Description(target.Description)
-                await ctx.Reply(embed: new EmbedBuilder()
+                .Field(new Embed.Field("\u200B",
-                    .Title("Member description")
+                    $"To print the description with formatting, type `pk;member {target.Reference(ctx)} description -raw`."
-                    .Description(target.Description)
+                    + (ctx.System?.Id == target.System
-                    .Field(new Embed.Field("\u200B",
+                        ? $" To clear it, type `pk;member {target.Reference(ctx)} description -clear`."
-                        $"To print the description with formatting, type `pk;member {target.Reference(ctx)} description -raw`."
+                        : "")
-                        + (ctx.System?.Id == target.System
+                    + $" Using {target.Description.Length}/{Limits.MaxDescriptionLength} characters."))
-                            ? $" To clear it, type `pk;member {target.Reference(ctx)} description -clear`."
+                .Build());
-                            : "")
-                        + $" Using {target.Description.Length}/{Limits.MaxDescriptionLength} characters."))
-                    .Build());
             return;
         }
 
@@ -126,26 +137,37 @@ public class MemberEdit
 
         ctx.CheckSystemPrivacy(target.System, target.PronounPrivacy);
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
             if (target.Pronouns == null)
+            {
                 await ctx.Reply(noPronounsSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{target.Pronouns}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"```\n{target.Pronouns}\n```");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing pronouns for member {target.Reference(ctx)}");
+            await ctx.Reply(target.Pronouns, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (target.Pronouns == null)
+            await ctx.Reply(
-                await ctx.Reply(noPronounsSetMessage);
+                $"**{target.NameFor(ctx)}**'s pronouns are **{target.Pronouns}**.\nTo print the pronouns with formatting, type `pk;member {target.Reference(ctx)} pronouns -raw`."
-            else
+                + (ctx.System?.Id == target.System
-                await ctx.Reply(
+                    ? $" To clear them, type `pk;member {target.Reference(ctx)} pronouns -clear`."
-                    $"**{target.NameFor(ctx)}**'s pronouns are **{target.Pronouns}**.\nTo print the pronouns with formatting, type `pk;member {target.Reference(ctx)} pronouns -raw`."
+                    : "")
-                    + (ctx.System?.Id == target.System
+                + $" Using {target.Pronouns.Length}/{Limits.MaxPronounsLength} characters.");
-                        ? $" To clear them, type `pk;member {target.Reference(ctx)} pronouns -clear`."
-                        : "")
-                    + $" Using {target.Pronouns.Length}/{Limits.MaxPronounsLength} characters.");
             return;
         }
 
@@ -232,7 +254,7 @@ public class MemberEdit
     public async Task Color(Context ctx, PKMember target)
     {
         var isOwnSystem = ctx.System?.Id == target.System;
-        var matchedRaw = ctx.MatchRaw();
+        var matchedFormat = ctx.MatchFormat();
         var matchedClear = ctx.MatchClear();
 
         if (!isOwnSystem || !(ctx.HasNext() || matchedClear))
@@ -240,8 +262,10 @@ public class MemberEdit
             if (target.Color == null)
                 await ctx.Reply(
                     "This member does not have a color set." + (isOwnSystem ? $" To set one, type `pk;member {target.Reference(ctx)} color <color>`." : ""));
-            else if (matchedRaw)
+            else if (matchedFormat == ReplyFormat.Raw)
                 await ctx.Reply("```\n#" + target.Color + "\n```");
+            else if (matchedFormat == ReplyFormat.Plaintext)
+                await ctx.Reply(target.Color);
             else
                 await ctx.Reply(embed: new EmbedBuilder()
                     .Title("Member color")
@@ -388,12 +412,26 @@ public class MemberEdit
 
         // No perms check, display name isn't covered by member privacy
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if what's next is "raw"/"plaintext" we need to check for null
+        if (format != ReplyFormat.Standard)
             if (target.DisplayName == null)
+            {
                 await ctx.Reply(noDisplayNameSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{target.DisplayName}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"```\n{target.DisplayName}\n```");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing displayname for member {target.Reference(ctx)}");
+            await ctx.Reply(target.DisplayName, embed: eb.Build());
             return;
         }
 
@@ -450,12 +488,26 @@ public class MemberEdit
         // No perms check, display name isn't covered by member privacy
         var memberGuildConfig = await ctx.Repository.GetMemberGuild(ctx.Guild.Id, target.Id);
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if what's next is "raw"/"plaintext" we need to check for null
+        if (format != ReplyFormat.Standard)
             if (memberGuildConfig.DisplayName == null)
+            {
                 await ctx.Reply(noServerNameSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{memberGuildConfig.DisplayName}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"```\n{memberGuildConfig.DisplayName}\n```");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing servername for member {target.Reference(ctx)}");
+            await ctx.Reply(memberGuildConfig.DisplayName, embed: eb.Build());
             return;
         }
 

PluralKit.Bot/Commands/MemberProxy.cs

@@ -63,7 +63,7 @@ public class MemberProxy
             if (!ctx.HasNext(false))
                 throw new PKSyntaxError("You must pass an example proxy to add (eg. `[text]` or `J:text`).");
 
-            var tagToAdd = ParseProxyTags(ctx.RemainderOrNull(false));
+            var tagToAdd = ParseProxyTags(ctx.RemainderOrNull(false).NormalizeLineEndSpacing());
             if (tagToAdd.IsEmpty) throw Errors.EmptyProxyTags(target, ctx);
             if (target.ProxyTags.Contains(tagToAdd))
                 throw Errors.ProxyTagAlreadyExists(tagToAdd, target);
@@ -87,10 +87,17 @@ public class MemberProxy
             if (!ctx.HasNext(false))
                 throw new PKSyntaxError("You must pass a proxy tag to remove (eg. `[text]` or `J:text`).");
 
-            var tagToRemove = ParseProxyTags(ctx.RemainderOrNull(false));
+            var remainder = ctx.RemainderOrNull(false);
+            var tagToRemove = ParseProxyTags(remainder.NormalizeLineEndSpacing());
             if (tagToRemove.IsEmpty) throw Errors.EmptyProxyTags(target, ctx);
             if (!target.ProxyTags.Contains(tagToRemove))
-                throw Errors.ProxyTagDoesNotExist(tagToRemove, target);
+            {
+                // Legacy support for when line endings weren't normalized
+                tagToRemove = ParseProxyTags(remainder);
+                if (!target.ProxyTags.Contains(tagToRemove))
+                    throw Errors.ProxyTagDoesNotExist(tagToRemove, target);
+            }
+
 
             var newTags = target.ProxyTags.ToList();
             newTags.Remove(tagToRemove);
@@ -102,7 +109,7 @@ public class MemberProxy
         // Subcommand: bare proxy tag given
         else
         {
-            var requestedTag = ParseProxyTags(ctx.RemainderOrNull(false));
+            var requestedTag = ParseProxyTags(ctx.RemainderOrNull(false).NormalizeLineEndSpacing());
             if (requestedTag.IsEmpty) throw Errors.EmptyProxyTags(target, ctx);
 
             // This is mostly a legacy command, so it's gonna warn if there's

PluralKit.Bot/Commands/Message.cs

@@ -218,7 +218,7 @@ public class ProxiedMessage
         try
         {
             var editedMsg =
-                await _webhookExecutor.EditWebhookMessage(msg.Channel, msg.Mid, newContent, clearEmbeds);
+                await _webhookExecutor.EditWebhookMessage(msg.Guild ?? 0, msg.Channel, msg.Mid, newContent, clearEmbeds);
 
             if (ctx.Guild == null)
                 await _rest.CreateReaction(ctx.Channel.Id, ctx.Message.Id, new Emoji { Name = Emojis.Success });
@@ -352,7 +352,9 @@ public class ProxiedMessage
         else if (!await ctx.CheckPermissionsInGuildChannel(channel, PermissionSet.ViewChannel))
             showContent = false;
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
+
+        if (format != ReplyFormat.Standard)
         {
             var discordMessage = await _rest.GetMessageOrNull(message.Message.Channel, message.Message.Mid);
             if (discordMessage == null || !showContent)
@@ -365,21 +367,32 @@ public class ProxiedMessage
                 return;
             }
 
-            await ctx.Reply($"```{content}```");
+            if (format == ReplyFormat.Raw)
-
-            if (Regex.IsMatch(content, "```.*```", RegexOptions.Singleline))
             {
-                var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
+                await ctx.Reply($"```{content}```");
-                await ctx.Rest.CreateMessage(
+
-                    ctx.Channel.Id,
+                if (Regex.IsMatch(content, "```.*```", RegexOptions.Singleline))
-                    new MessageRequest
+                {
-                    {
+                    var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
-                        Content = $"{Emojis.Warn} Message contains codeblocks, raw source sent as an attachment."
+                    await ctx.Rest.CreateMessage(
-                    },
+                        ctx.Channel.Id,
-                    new[] { new MultipartFile("message.txt", stream, null, null, null) });
+                        new MessageRequest
+                        {
+                            Content = $"{Emojis.Warn} Message contains codeblocks, raw source sent as an attachment."
+                        },
+                        new[] { new MultipartFile("message.txt", stream, null, null, null) });
+                }
+                return;
+            }
+
+            if (format == ReplyFormat.Plaintext)
+            {
+                var eb = new EmbedBuilder()
+                .Description($"Showing contents of message {message.Message.Mid}");
+                await ctx.Reply(content, embed: eb.Build());
+                return;
             }
 
-            return;
         }
 
         if (isDelete)
@@ -387,7 +400,7 @@ public class ProxiedMessage
             if (!showContent)
                 throw new PKError(noShowContentError);
 
-            if (message.System?.Id != ctx.System.Id && message.Message.Sender != ctx.Author.Id)
+            if (message.Message.Sender != ctx.Author.Id && (ctx.System != null && message.System?.Id != ctx.System.Id))
                 throw new PKError("You can only delete your own messages.");
 
             await ctx.Rest.DeleteMessage(message.Message.Channel, message.Message.Mid);
@@ -423,14 +436,14 @@ public class ProxiedMessage
 
     private async Task DeleteCommandMessage(Context ctx, ulong messageId)
     {
-        var (authorId, channelId) = await ctx.Services.Resolve<CommandMessageService>().GetCommandMessage(messageId);
+        var cmessage = await ctx.Services.Resolve<CommandMessageService>().GetCommandMessage(messageId);
-        if (authorId == null)
+        if (cmessage == null)
             throw Errors.MessageNotFound(messageId);
 
-        if (authorId != ctx.Author.Id)
+        if (cmessage!.AuthorId != ctx.Author.Id)
             throw new PKError("You can only delete command messages queried by this account.");
 
-        await ctx.Rest.DeleteMessage(channelId!.Value, messageId);
+        await ctx.Rest.DeleteMessage(cmessage.ChannelId, messageId);
 
         if (ctx.Guild != null)
             await ctx.Rest.DeleteMessage(ctx.Message);

PluralKit.Bot/Commands/Random.cs

@@ -67,7 +67,7 @@ public class Random
     {
         ctx.CheckSystemPrivacy(group.System, group.ListPrivacy);
 
-        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(group.System));
+        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(group.System), ctx.LookupContextFor(group.System));
         opts.GroupFilter = group.Id;
 
         var members = await ctx.Database.Execute(conn => conn.QueryMemberList(group.System, opts.ToQueryOptions()));

PluralKit.Bot/Commands/ServerConfig.cs

@@ -49,7 +49,7 @@ public class ServerConfig
         if (channel.Type != Channel.ChannelType.GuildText && channel.Type != Channel.ChannelType.GuildPublicThread && channel.Type != Channel.ChannelType.GuildPrivateThread)
             throw new PKError("PluralKit cannot log messages to this type of channel.");
 
-        var perms = await _cache.BotPermissionsIn(channel.Id);
+        var perms = await _cache.BotPermissionsIn(ctx.Guild.Id, channel.Id);
         if (!perms.HasFlag(PermissionSet.SendMessages))
             throw new PKError("PluralKit is missing **Send Messages** permissions in the new log channel.");
         if (!perms.HasFlag(PermissionSet.EmbedLinks))
@@ -104,7 +104,7 @@ public class ServerConfig
 
         // Resolve all channels from the cache and order by position
         var channels = (await Task.WhenAll(blacklist.Blacklist
-                .Select(id => _cache.TryGetChannel(id))))
+                .Select(id => _cache.TryGetChannel(ctx.Guild.Id, id))))
             .Where(c => c != null)
             .OrderBy(c => c.Position)
             .ToList();
@@ -121,7 +121,7 @@ public class ServerConfig
             async (eb, l) =>
             {
                 async Task<string> CategoryName(ulong? id) =>
-                    id != null ? (await _cache.GetChannel(id.Value)).Name : "(no category)";
+                    id != null ? (await _cache.GetChannel(ctx.Guild.Id, id.Value)).Name : "(no category)";
 
                 ulong? lastCategory = null;
 
@@ -153,8 +153,9 @@ public class ServerConfig
         var config = await ctx.Repository.GetGuild(ctx.Guild.Id);
 
         // Resolve all channels from the cache and order by position
+        // todo: GetAllChannels?
         var channels = (await Task.WhenAll(config.LogBlacklist
-                .Select(id => _cache.TryGetChannel(id))))
+                .Select(id => _cache.TryGetChannel(ctx.Guild.Id, id))))
             .Where(c => c != null)
             .OrderBy(c => c.Position)
             .ToList();
@@ -171,7 +172,7 @@ public class ServerConfig
             async (eb, l) =>
             {
                 async Task<string> CategoryName(ulong? id) =>
-                    id != null ? (await _cache.GetChannel(id.Value)).Name : "(no category)";
+                    id != null ? (await _cache.GetChannel(ctx.Guild.Id, id.Value)).Name : "(no category)";
 
                 ulong? lastCategory = null;
 
@@ -204,7 +205,8 @@ public class ServerConfig
         var affectedChannels = new List<Channel>();
         if (ctx.Match("all"))
             affectedChannels = (await _cache.GetGuildChannels(ctx.Guild.Id))
-                .Where(x => x.Type == Channel.ChannelType.GuildText).ToList();
+                // All the channel types you can proxy in
+                .Where(x => DiscordUtils.IsValidGuildChannel(x)).ToList();
         else if (!ctx.HasNext()) throw new PKSyntaxError("You must pass one or more #channels.");
         else
             while (ctx.HasNext())

PluralKit.Bot/Commands/Switch.cs

@@ -7,6 +7,7 @@ namespace PluralKit.Bot;
 
 public class Switch
 {
+
     public async Task SwitchDo(Context ctx)
     {
         ctx.CheckSystem();
@@ -103,12 +104,69 @@ public class Switch
         await ctx.Reply($"{Emojis.Success} Switch moved to <t:{newSwitchTime}> ({newSwitchDeltaStr} ago).");
     }
 
-    public async Task SwitchEdit(Context ctx)
+    public async Task SwitchEdit(Context ctx, bool newSwitch = false)
     {
         ctx.CheckSystem();
 
-        var members = await ctx.ParseMemberList(ctx.System.Id);
+        var newMembers = await ctx.ParseMemberList(ctx.System.Id);
-        await DoEditCommand(ctx, members);
+
+        await using var conn = await ctx.Database.Obtain();
+        var currentSwitch = await ctx.Repository.GetLatestSwitch(ctx.System.Id);
+        if (currentSwitch == null)
+            throw Errors.NoRegisteredSwitches;
+        var currentSwitchMembers = await ctx.Repository.GetSwitchMembers(conn, currentSwitch.Id).ToListAsync().AsTask();
+
+        if (ctx.MatchFlag("first", "f"))
+            newMembers = FirstInSwitch(newMembers[0], currentSwitchMembers);
+        else if (ctx.MatchFlag("remove", "r"))
+            newMembers = RemoveFromSwitch(newMembers, currentSwitchMembers);
+        else if (ctx.MatchFlag("append", "a"))
+            newMembers = AppendToSwitch(newMembers, currentSwitchMembers);
+        else if (ctx.MatchFlag("prepend", "p"))
+            newMembers = PrependToSwitch(newMembers, currentSwitchMembers);
+
+        if (newSwitch)
+        {
+            // if there's no edit flag, assume we're appending
+            if (!ctx.MatchFlag("first", "f", "remove", "r", "append", "a", "prepend", "p"))
+                newMembers = AppendToSwitch(newMembers, currentSwitchMembers);
+            await DoSwitchCommand(ctx, newMembers);
+        }
+        else
+            await DoEditCommand(ctx, newMembers);
+    }
+
+    public List<PKMember> PrependToSwitch(List<PKMember> members, List<PKMember> currentSwitchMembers)
+    {
+        members.AddRange(currentSwitchMembers);
+
+        return members;
+    }
+
+    public List<PKMember> AppendToSwitch(List<PKMember> members, List<PKMember> currentSwitchMembers)
+    {
+        currentSwitchMembers.AddRange(members);
+        members = currentSwitchMembers;
+
+        return members;
+    }
+
+    public List<PKMember> RemoveFromSwitch(List<PKMember> members, List<PKMember> currentSwitchMembers)
+    {
+        var memberIds = members.Select(m => m.Id.Value);
+        currentSwitchMembers = currentSwitchMembers.Where(m => !memberIds.Contains(m.Id.Value)).ToList();
+        members = currentSwitchMembers;
+
+        return members;
+    }
+
+    public List<PKMember> FirstInSwitch(PKMember member, List<PKMember> currentSwitchMembers)
+    {
+        currentSwitchMembers = currentSwitchMembers.Where(m => m.Id != member.Id).ToList();
+        var members = new List<PKMember> { member };
+        members.AddRange(currentSwitchMembers);
+
+        return members;
     }
 
     public async Task SwitchEditOut(Context ctx)

PluralKit.Bot/Commands/System.cs

@@ -2,6 +2,9 @@ using PluralKit.Core;
 
 namespace PluralKit.Bot;
 
+using Myriad.Builders;
+using Myriad.Types;
+
 public class System
 {
     private readonly EmbedService _embeds;
@@ -29,9 +32,25 @@ public class System
         var system = await ctx.Repository.CreateSystem(systemName);
         await ctx.Repository.AddAccount(system.Id, ctx.Author.Id);
 
-        // TODO: better message, perhaps embed like in groups?
+        var eb = new EmbedBuilder()
-        await ctx.Reply(
+            .Title(
-            $"{Emojis.Success} Your system has been created. Type `pk;system` to view it, and type `pk;system help` for more information about commands you can use now. Now that you have that set up, check out the getting started guide on setting up members and proxies: <https://pluralkit.me/start>");
+                $"{Emojis.Success} Your system has been created.")
+            .Field(new Embed.Field("Getting Started",
+                "New to PK? Check out our Getting Started guide on setting up members and proxies: https://pluralkit.me/start\n" +
+                "Otherwise, type `pk;system` to view your system and `pk;system help` for more information about commands you can use."))
+            .Field(new Embed.Field($"{Emojis.Warn} Notice {Emojis.Warn}", "PluralKit is a bot meant to help you share information about your system. " +
+                "Member descriptions are meant to be the equivalent to a Discord About Me. Because of this, any info you put in PK is **public by default**.\n" +
+                "Note that this does **not** include message content, only member fields. For more information, check out " +
+                "[the privacy section of the user guide](https://pluralkit.me/guide/#privacy). "))
+            .Field(new Embed.Field("System Recovery", "In the case of your Discord account getting lost or deleted, the PluralKit staff can help you recover your system. " +
+                "In order to do so, we will need your **PluralKit token**. This is the *only* way you can prove ownership so we can help you recover your system. " +
+                "To get it, run `pk;token` and then store it in a safe place.\n\n" +
+                "Keep your token safe, if other people get access to it they can also use it to access your system. " +
+                "If your token is ever compromised run `pk;token refresh` to invalidate the old token and get a new one."))
+            .Field(new Embed.Field("Questions?",
+                "Please join the PK server https://discord.gg/PczBt78 if you have any questions, we're happy to help"));
+        await ctx.Reply($"{Emojis.Warn} If you cannot see the rest of this message see [the FAQ](<https://pluralkit.me/faq/#why-do-most-of-pluralkit-s-messages-look-blank-or-empty>)", eb.Build());
+
     }
 
     public async Task DisplayId(Context ctx, PKSystem target)

PluralKit.Bot/Commands/SystemEdit.cs

@@ -37,24 +37,35 @@ public class SystemEdit
         if (isOwnSystem)
             noNameSetMessage += " Type `pk;system name <name>` to set one.";
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
-            if (target.Name != null)
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
-                await ctx.Reply($"```\n{target.Name}\n```");
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
-            else
+            if (target.Name == null)
+            {
                 await ctx.Reply(noNameSetMessage);
+                return;
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"` ``\n{target.Name}\n` ``");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing name for system {target.DisplayHid()}");
+            await ctx.Reply(target.Name, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (target.Name != null)
+            await ctx.Reply(
-                await ctx.Reply(
+                $"{(isOwnSystem ? "Your" : "This")} system's name is currently **{target.Name}**."
-                    $"{(isOwnSystem ? "Your" : "This")} system's name is currently **{target.Name}**."
+                + (isOwnSystem ? " Type `pk;system name -clear` to clear it." : "")
-                    + (isOwnSystem ? " Type `pk;system name -clear` to clear it." : "")
+                + $" Using {target.Name.Length}/{Limits.MaxSystemNameLength} characters.");
-                    + $" Using {target.Name.Length}/{Limits.MaxSystemNameLength} characters.");
-            else
-                await ctx.Reply(noNameSetMessage);
             return;
         }
 
@@ -91,24 +102,35 @@ public class SystemEdit
 
         var settings = await ctx.Repository.GetSystemGuild(ctx.Guild.Id, target.Id);
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
-            if (settings.DisplayName != null)
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
-                await ctx.Reply($"```\n{settings.DisplayName}\n```");
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
-            else
+            if (settings.DisplayName == null)
+            {
                 await ctx.Reply(noNameSetMessage);
+                return;
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"` ``\n{settings.DisplayName}\n` ``");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing servername for system {target.DisplayHid()}");
+            await ctx.Reply(settings.DisplayName, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (settings.DisplayName != null)
+            await ctx.Reply(
-                await ctx.Reply(
+                $"{(isOwnSystem ? "Your" : "This")} system's name for this server is currently **{settings.DisplayName}**."
-                    $"{(isOwnSystem ? "Your" : "This")} system's name for this server is currently **{settings.DisplayName}**."
+                + (isOwnSystem ? " Type `pk;system servername -clear` to clear it." : "")
-                    + (isOwnSystem ? " Type `pk;system servername -clear` to clear it." : "")
+                + $" Using {settings.DisplayName.Length}/{Limits.MaxSystemNameLength} characters.");
-                    + $" Using {settings.DisplayName.Length}/{Limits.MaxSystemNameLength} characters.");
-            else
-                await ctx.Reply(noNameSetMessage);
             return;
         }
 
@@ -143,28 +165,39 @@ public class SystemEdit
         if (isOwnSystem)
             noDescriptionSetMessage += " To set one, type `pk;s description <description>`.";
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
             if (target.Description == null)
+            {
                 await ctx.Reply(noDescriptionSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{target.Description}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"` ``\n{target.Description}\n` ``");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing description for system {target.DisplayHid()}");
+            await ctx.Reply(target.Description, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (target.Description == null)
+            await ctx.Reply(embed: new EmbedBuilder()
-                await ctx.Reply(noDescriptionSetMessage);
+                .Title("System description")
-            else
+                .Description(target.Description)
-                await ctx.Reply(embed: new EmbedBuilder()
+                .Footer(new Embed.EmbedFooter(
-                    .Title("System description")
+                    "To print the description with formatting, type `pk;s description -raw`."
-                    .Description(target.Description)
+                        + (isOwnSystem ? " To clear it, type `pk;s description -clear`. To change it, type `pk;s description <new description>`." : "")
-                    .Footer(new Embed.EmbedFooter(
+                        + $" Using {target.Description.Length}/{Limits.MaxDescriptionLength} characters."))
-                        "To print the description with formatting, type `pk;s description -raw`."
+                .Build());
-                            + (isOwnSystem ? " To clear it, type `pk;s description -clear`. To change it, type `pk;s description <new description>`." : "")
-                            + $" Using {target.Description.Length}/{Limits.MaxDescriptionLength} characters."))
-                    .Build());
             return;
         }
 
@@ -191,7 +224,7 @@ public class SystemEdit
     public async Task Color(Context ctx, PKSystem target)
     {
         var isOwnSystem = ctx.System?.Id == target.Id;
-        var matchedRaw = ctx.MatchRaw();
+        var matchedFormat = ctx.MatchFormat();
         var matchedClear = ctx.MatchClear();
 
         if (!isOwnSystem || !(ctx.HasNext() || matchedClear))
@@ -199,8 +232,10 @@ public class SystemEdit
             if (target.Color == null)
                 await ctx.Reply(
                     "This system does not have a color set." + (isOwnSystem ? " To set one, type `pk;system color <color>`." : ""));
-            else if (matchedRaw)
+            else if (matchedFormat == ReplyFormat.Raw)
                 await ctx.Reply("```\n#" + target.Color + "\n```");
+            else if (matchedFormat == ReplyFormat.Plaintext)
+                await ctx.Reply(target.Color);
             else
                 await ctx.Reply(embed: new EmbedBuilder()
                     .Title("System color")
@@ -246,22 +281,33 @@ public class SystemEdit
             ? "You currently have no system tag set. To set one, type `pk;s tag <tag>`."
             : "This system currently has no system tag set.";
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
             if (target.Tag == null)
+            {
                 await ctx.Reply(noTagSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{target.Tag}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"```\n{target.Tag}\n```");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing tag for system {target.DisplayHid()}");
+            await ctx.Reply(target.Tag, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (target.Tag == null)
+            await ctx.Reply($"{(isOwnSystem ? "Your" : "This system's")} current system tag is {target.Tag.AsCode()}."
-                await ctx.Reply(noTagSetMessage);
+                + (isOwnSystem ? "To change it, type `pk;s tag <tag>`. To clear it, type `pk;s tag -clear`." : ""));
-            else
-                await ctx.Reply($"{(isOwnSystem ? "Your" : "This system's")} current system tag is {target.Tag.AsCode()}."
-                    + (isOwnSystem ? "To change it, type `pk;s tag <tag>`. To clear it, type `pk;s tag -clear`." : ""));
             return;
         }
 
@@ -296,15 +342,22 @@ public class SystemEdit
 
         var settings = await ctx.Repository.GetSystemGuild(ctx.Guild.Id, target.Id);
 
-        async Task Show(bool raw = false)
+        async Task Show(ReplyFormat format = ReplyFormat.Standard)
         {
             if (settings.Tag != null)
             {
-                if (raw)
+                if (format == ReplyFormat.Raw)
                 {
                     await ctx.Reply($"```{settings.Tag}```");
                     return;
                 }
+                if (format == ReplyFormat.Plaintext)
+                {
+                    var eb = new EmbedBuilder()
+                        .Description($"Showing servertag for system {target.DisplayHid()}");
+                    await ctx.Reply(settings.Tag, embed: eb.Build());
+                    return;
+                }
 
                 var msg = $"Your current system tag in '{ctx.Guild.Name}' is {settings.Tag.AsCode()}";
                 if (!settings.TagEnabled)
@@ -400,8 +453,8 @@ public class SystemEdit
             await EnableDisable(false);
         else if (ctx.Match("enable") || ctx.MatchFlag("enable"))
             await EnableDisable(true);
-        else if (ctx.MatchRaw())
+        else if (ctx.MatchFormat() != ReplyFormat.Standard)
-            await Show(true);
+            await Show(ctx.MatchFormat());
         else if (!ctx.HasNext(false))
             await Show();
         else
@@ -418,24 +471,35 @@ public class SystemEdit
         if (isOwnSystem)
             noPronounsSetMessage += " To set some, type `pk;system pronouns <pronouns>`";
 
-        if (ctx.MatchRaw())
+        var format = ctx.MatchFormat();
-        {
+
+        // if there's nothing next or what's next is "raw"/"plaintext" we're doing a query, so check for null
+        if (!ctx.HasNext(false) || format != ReplyFormat.Standard)
             if (target.Pronouns == null)
+            {
                 await ctx.Reply(noPronounsSetMessage);
-            else
+                return;
-                await ctx.Reply($"```\n{target.Pronouns}\n```");
+            }
+
+        if (format == ReplyFormat.Raw)
+        {
+            await ctx.Reply($"```\n{target.Pronouns}\n```");
+            return;
+        }
+        if (format == ReplyFormat.Plaintext)
+        {
+            var eb = new EmbedBuilder()
+                .Description($"Showing pronouns for system {target.DisplayHid()}");
+            await ctx.Reply(target.Pronouns, embed: eb.Build());
             return;
         }
 
         if (!ctx.HasNext(false))
         {
-            if (target.Pronouns == null)
+            await ctx.Reply($"{(isOwnSystem ? "Your" : "This system's")} current pronouns are **{target.Pronouns}**.\nTo print the pronouns with formatting, type `pk;system pronouns -raw`."
-                await ctx.Reply(noPronounsSetMessage);
+            + (isOwnSystem ? " To clear them, type `pk;system pronouns -clear`."
-            else
+            : "")
-                await ctx.Reply($"{(isOwnSystem ? "Your" : "This system's")} current pronouns are **{target.Pronouns}**.\nTo print the pronouns with formatting, type `pk;system pronouns -raw`."
+            + $" Using {target.Pronouns.Length}/{Limits.MaxPronounsLength} characters.");
-                + (isOwnSystem ? " To clear them, type `pk;system pronouns -clear`."
-                : "")
-                + $" Using {target.Pronouns.Length}/{Limits.MaxPronounsLength} characters.");
             return;
         }
 

PluralKit.Bot/Commands/SystemList.cs

@@ -17,7 +17,7 @@ public class SystemList
         // - ParseListOptions checks list access privacy and sets the privacy filter (which members show up in list)
         // - RenderMemberList checks the indivual privacy for each member (NameFor, etc)
         // the own system is always allowed to look up their list
-        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(target.Id));
+        var opts = ctx.ParseListOptions(ctx.DirectLookupContextFor(target.Id), ctx.LookupContextFor(target.Id));
         await ctx.RenderMemberList(
             ctx.LookupContextFor(target.Id),
             target.Id,

PluralKit.Bot/Handlers/IEventHandler.cs

@@ -6,5 +6,5 @@ public interface IEventHandler<in T> where T : IGatewayEvent
 {
     Task Handle(int shardId, T evt);
 
-    ulong? ErrorChannelFor(T evt, ulong userId) => null;
+    (ulong?, ulong?) ErrorChannelFor(T evt, ulong userId) => (null, null);
 }

PluralKit.Bot/Handlers/MessageCreated.cs

@@ -52,7 +52,7 @@ public class MessageCreated: IEventHandler<MessageCreateEvent>
         _dmCache = dmCache;
     }
 
-    public ulong? ErrorChannelFor(MessageCreateEvent evt, ulong userId) => evt.ChannelId;
+    public (ulong?, ulong?) ErrorChannelFor(MessageCreateEvent evt, ulong userId) => (evt.GuildId, evt.ChannelId);
     private bool IsDuplicateMessage(Message msg) =>
         // We consider a message duplicate if it has the same ID as the previous message that hit the gateway
         _lastMessageCache.GetLastMessage(msg.ChannelId)?.Current.Id == msg.Id;
@@ -63,7 +63,7 @@ public class MessageCreated: IEventHandler<MessageCreateEvent>
         if (evt.Type != Message.MessageType.Default && evt.Type != Message.MessageType.Reply) return;
         if (IsDuplicateMessage(evt)) return;
 
-        var botPermissions = await _cache.BotPermissionsIn(evt.ChannelId);
+        var botPermissions = await _cache.BotPermissionsIn(evt.GuildId ?? 0, evt.ChannelId);
         if (!botPermissions.HasFlag(PermissionSet.SendMessages)) return;
 
         // spawn off saving the private channel into another thread
@@ -71,8 +71,8 @@ public class MessageCreated: IEventHandler<MessageCreateEvent>
         _ = _dmCache.TrySavePrivateChannel(evt);
 
         var guild = evt.GuildId != null ? await _cache.GetGuild(evt.GuildId.Value) : null;
-        var channel = await _cache.GetChannel(evt.ChannelId);
+        var channel = await _cache.GetChannel(evt.GuildId ?? 0, evt.ChannelId);
-        var rootChannel = await _cache.GetRootChannel(evt.ChannelId);
+        var rootChannel = await _cache.GetRootChannel(evt.GuildId ?? 0, evt.ChannelId);
 
         // Log metrics and message info
         _metrics.Measure.Meter.Mark(BotMetrics.MessagesReceived);
@@ -90,7 +90,8 @@ public class MessageCreated: IEventHandler<MessageCreateEvent>
         if (await TryHandleCommand(shardId, evt, guild, channel))
             return;
 
-        await TryHandleProxy(evt, guild, channel, rootChannel.Id, botPermissions);
+        if (evt.GuildId != null)
+            await TryHandleProxy(evt, guild, channel, rootChannel.Id, botPermissions);
     }
 
     private async Task TryHandleLogClean(Channel channel, MessageCreateEvent evt)

PluralKit.Bot/Handlers/MessageEdited.cs

@@ -52,10 +52,12 @@ public class MessageEdited: IEventHandler<MessageUpdateEvent>
         if (!evt.Content.HasValue || !evt.Author.HasValue || !evt.Member.HasValue)
             return;
 
-        var channel = await _cache.GetChannel(evt.ChannelId);
+        var guildIdMaybe = evt.GuildId.HasValue ? evt.GuildId.Value ?? 0 : 0;
+
+        var channel = await _cache.GetChannel(guildIdMaybe, evt.ChannelId); // todo: is this correct for message update?
         if (!DiscordUtils.IsValidGuildChannel(channel))
             return;
-        var rootChannel = await _cache.GetRootChannel(channel.Id);
+        var rootChannel = await _cache.GetRootChannel(guildIdMaybe, channel.Id);
         var guild = await _cache.GetGuild(channel.GuildId!.Value);
         var lastMessage = _lastMessageCache.GetLastMessage(evt.ChannelId)?.Current;
 
@@ -69,7 +71,7 @@ public class MessageEdited: IEventHandler<MessageUpdateEvent>
             ctx = await _repo.GetMessageContext(evt.Author.Value!.Id, channel.GuildId!.Value, rootChannel.Id, evt.ChannelId);
 
         var equivalentEvt = await GetMessageCreateEvent(evt, lastMessage, channel);
-        var botPermissions = await _cache.BotPermissionsIn(channel.Id);
+        var botPermissions = await _cache.BotPermissionsIn(guildIdMaybe, channel.Id);
 
         try
         {
@@ -91,7 +93,7 @@ public class MessageEdited: IEventHandler<MessageUpdateEvent>
     private async Task<MessageCreateEvent> GetMessageCreateEvent(MessageUpdateEvent evt, CachedMessage lastMessage,
                                                                  Channel channel)
     {
-        var referencedMessage = await GetReferencedMessage(evt.ChannelId, lastMessage.ReferencedMessage);
+        var referencedMessage = await GetReferencedMessage(evt.GuildId.HasValue ? evt.GuildId.Value ?? 0 : 0, evt.ChannelId, lastMessage.ReferencedMessage);
 
         var messageReference = lastMessage.ReferencedMessage != null
             ? new Message.Reference(channel.GuildId, evt.ChannelId, lastMessage.ReferencedMessage.Value)
@@ -118,12 +120,12 @@ public class MessageEdited: IEventHandler<MessageUpdateEvent>
         return equivalentEvt;
     }
 
-    private async Task<Message?> GetReferencedMessage(ulong channelId, ulong? referencedMessageId)
+    private async Task<Message?> GetReferencedMessage(ulong guildId, ulong channelId, ulong? referencedMessageId)
     {
         if (referencedMessageId == null)
             return null;
 
-        var botPermissions = await _cache.BotPermissionsIn(channelId);
+        var botPermissions = await _cache.BotPermissionsIn(guildId, channelId);
         if (!botPermissions.HasFlag(PermissionSet.ReadMessageHistory))
         {
             _logger.Warning(

PluralKit.Bot/Handlers/ReactionAdded.cs

@@ -62,7 +62,7 @@ public class ReactionAdded: IEventHandler<MessageReactionAddEvent>
         // but we aren't able to get DMs from bots anyway, so it's not really needed
         if (evt.GuildId != null && (evt.Member?.User?.Bot ?? false)) return;
 
-        var channel = await _cache.GetChannel(evt.ChannelId);
+        var channel = await _cache.GetChannel(evt.GuildId ?? 0, evt.ChannelId);
 
         // check if it's a command message first
         // since this can happen in DMs as well
@@ -75,10 +75,10 @@ public class ReactionAdded: IEventHandler<MessageReactionAddEvent>
                 return;
             }
 
-            var (authorId, _) = await _commandMessageService.GetCommandMessage(evt.MessageId);
+            var cmessage = await _commandMessageService.GetCommandMessage(evt.MessageId);
-            if (authorId != null)
+            if (cmessage != null)
             {
-                await HandleCommandDeleteReaction(evt, authorId.Value, false);
+                await HandleCommandDeleteReaction(evt, cmessage.AuthorId, false);
                 return;
             }
         }
@@ -123,7 +123,7 @@ public class ReactionAdded: IEventHandler<MessageReactionAddEvent>
 
     private async ValueTask HandleProxyDeleteReaction(MessageReactionAddEvent evt, PKMessage msg)
     {
-        if (!(await _cache.BotPermissionsIn(evt.ChannelId)).HasFlag(PermissionSet.ManageMessages))
+        if (!(await _cache.BotPermissionsIn(evt.GuildId ?? 0, evt.ChannelId)).HasFlag(PermissionSet.ManageMessages))
             return;
 
         var isSameSystem = msg.Member != null && await _repo.IsMemberOwnedByAccount(msg.Member.Value, evt.UserId);
@@ -150,7 +150,7 @@ public class ReactionAdded: IEventHandler<MessageReactionAddEvent>
         if (authorId != null && authorId != evt.UserId)
             return;
 
-        if (!((await _cache.BotPermissionsIn(evt.ChannelId)).HasFlag(PermissionSet.ManageMessages) || isDM))
+        if (!((await _cache.BotPermissionsIn(evt.GuildId ?? 0, evt.ChannelId)).HasFlag(PermissionSet.ManageMessages) || isDM))
             return;
 
         // todo: don't try to delete the user's own messages in DMs
@@ -206,14 +206,14 @@ public class ReactionAdded: IEventHandler<MessageReactionAddEvent>
 
     private async ValueTask HandlePingReaction(MessageReactionAddEvent evt, FullMessage msg)
     {
-        if (!(await _cache.BotPermissionsIn(evt.ChannelId)).HasFlag(PermissionSet.ManageMessages))
+        if (!(await _cache.BotPermissionsIn(evt.GuildId ?? 0, evt.ChannelId)).HasFlag(PermissionSet.ManageMessages))
             return;
 
         // Check if the "pinger" has permission to send messages in this channel
         // (if not, PK shouldn't send messages on their behalf)
         var member = await _rest.GetGuildMember(evt.GuildId!.Value, evt.UserId);
         var requiredPerms = PermissionSet.ViewChannel | PermissionSet.SendMessages;
-        if (member == null || !(await _cache.PermissionsForMemberInChannel(evt.ChannelId, member)).HasFlag(requiredPerms)) return;
+        if (member == null || !(await _cache.PermissionsForMemberInChannel(evt.GuildId ?? 0, evt.ChannelId, member)).HasFlag(requiredPerms)) return;
 
         if (msg.Member == null) return;
 
@@ -266,7 +266,7 @@ public class ReactionAdded: IEventHandler<MessageReactionAddEvent>
 
     private async Task TryRemoveOriginalReaction(MessageReactionAddEvent evt)
     {
-        if ((await _cache.BotPermissionsIn(evt.ChannelId)).HasFlag(PermissionSet.ManageMessages))
+        if ((await _cache.BotPermissionsIn(evt.GuildId ?? 0, evt.ChannelId)).HasFlag(PermissionSet.ManageMessages))
             await _rest.DeleteUserReaction(evt.ChannelId, evt.MessageId, evt.Emoji, evt.UserId);
     }
 }

PluralKit.Bot/Init.cs

@@ -42,10 +42,10 @@ public class Init
 
             using var _ = SentrySdk.Init(opts =>
             {
-                opts.Dsn = services.Resolve<CoreConfig>().SentryUrl;
+                opts.Dsn = services.Resolve<CoreConfig>().SentryUrl ?? "";
                 opts.Release = BuildInfoService.FullVersion;
                 opts.AutoSessionTracking = true;
-                opts.DisableTaskUnobservedTaskExceptionCapture();
+                //                opts.DisableTaskUnobservedTaskExceptionCapture();
             });
 
             var config = services.Resolve<BotConfig>();
@@ -56,8 +56,6 @@ public class Init
             await redis.InitAsync(coreConfig);
 
             var cache = services.Resolve<IDiscordCache>();
-            if (cache is RedisDiscordCache)
-                await (cache as RedisDiscordCache).InitAsync(coreConfig.RedisAddr);
 
             if (config.Cluster == null)
             {

PluralKit.Bot/Interactive/YesNoPrompt.cs

@@ -84,20 +84,32 @@ public class YesNoPrompt: BaseInteractive
 
         var queue = _ctx.Services.Resolve<HandlerQueue<MessageCreateEvent>>();
 
-        var messageDispatch = queue.WaitFor(MessagePredicate, Timeout, cts.Token);
+        async Task WaitForMessage()
+        {
+            try
+            {
+                await queue.WaitFor(MessagePredicate, Timeout, cts.Token);
+            }
+            catch (TimeoutException e)
+            {
+                if (e.Message != "HandlerQueue#WaitFor timed out")
+                    throw;
+            }
+        }
 
         await Start();
 
-        cts.Token.Register(() => _tcs.TrySetException(new TimeoutException("Action timed out")));
+        var messageDispatch = WaitForMessage();
+
+        cts.Token.Register(() => _tcs.TrySetException(new TimeoutException("YesNoPrompt timed out")));
 
         try
         {
             var doneTask = await Task.WhenAny(_tcs.Task, messageDispatch);
-            if (doneTask == messageDispatch)
-                await Finish();
         }
         finally
         {
+            await Finish();
             Cleanup();
         }
     }

PluralKit.Bot/Modules.cs

@@ -48,8 +48,25 @@ public class BotModule: Module
         {
             var botConfig = c.Resolve<BotConfig>();
 
-            if (botConfig.UseRedisCache)
+            if (botConfig.HttpCacheUrl != null)
-                return new RedisDiscordCache(c.Resolve<ILogger>(), botConfig.ClientId);
+            {
+                var cache = new HttpDiscordCache(c.Resolve<ILogger>(),
+                    c.Resolve<HttpClient>(), botConfig.HttpCacheUrl, botConfig.Cluster?.TotalShards ?? 1, botConfig.ClientId, botConfig.HttpUseInnerCache);
+
+                var metrics = c.Resolve<IMetrics>();
+
+                cache.OnDebug += (_, ev) =>
+                {
+                    var (remote, key) = ev;
+                    metrics.Measure.Meter.Mark(BotMetrics.CacheDebug, new MetricTags(
+                        new[] { "remote", "key" },
+                        new[] { remote.ToString(), key }
+                    ));
+                };
+
+                return cache;
+            }
+
             return new MemoryDiscordCache(botConfig.ClientId);
         }).AsSelf().SingleInstance();
         builder.RegisterType<PrivateChannelService>().AsSelf().SingleInstance();

PluralKit.Bot/PluralKit.Bot.csproj

@@ -27,7 +27,7 @@
         <PackageReference Include="Grpc.Net.ClientFactory" Version="2.32.0" />
         <PackageReference Include="Grpc.Tools" Version="2.47.0" PrivateAssets="all" />
         <PackageReference Include="Humanizer.Core" Version="2.8.26" />
-        <PackageReference Include="Sentry" Version="3.11.1" />
+        <PackageReference Include="Sentry" Version="4.12.1" />
         <PackageReference Include="SixLabors.ImageSharp" Version="3.1.5" />
     </ItemGroup>
 

PluralKit.Bot/Proxy/ProxyService.cs

@@ -59,7 +59,7 @@ public class ProxyService
     public async Task<bool> HandleIncomingMessage(MessageCreateEvent message, MessageContext ctx,
                                 Guild guild, Channel channel, bool allowAutoproxy, PermissionSet botPermissions)
     {
-        var rootChannel = await _cache.GetRootChannel(message.ChannelId);
+        var rootChannel = await _cache.GetRootChannel(message.GuildId!.Value, message.ChannelId);
 
         if (!ShouldProxy(channel, rootChannel, message, ctx))
             return false;
@@ -111,31 +111,10 @@ public class ProxyService
         return true;
     }
 
-#pragma warning disable CA1822 // Mark members as static
+    // Proxy checks that give user errors
-    internal bool CanProxyInChannel(Channel ch, bool isRootChannel = false)
-#pragma warning restore CA1822 // Mark members as static
-    {
-        // this is explicitly selecting known channel types so that when Discord add new
-        // ones, users don't get flooded with error codes if that new channel type doesn't
-        // support a feature we need for proxying
-        return ch.Type switch
-        {
-            Channel.ChannelType.GuildText => true,
-            Channel.ChannelType.GuildPublicThread => true,
-            Channel.ChannelType.GuildPrivateThread => true,
-            Channel.ChannelType.GuildNews => true,
-            Channel.ChannelType.GuildNewsThread => true,
-            Channel.ChannelType.GuildVoice => true,
-            Channel.ChannelType.GuildStageVoice => true,
-            Channel.ChannelType.GuildForum => isRootChannel,
-            Channel.ChannelType.GuildMedia => isRootChannel,
-            _ => false,
-        };
-    }
-
     public async Task<string> CanProxy(Channel channel, Channel rootChannel, Message msg, MessageContext ctx)
     {
-        if (!(CanProxyInChannel(channel) && CanProxyInChannel(rootChannel, true)))
+        if (!DiscordUtils.IsValidGuildChannel(channel))
             return $"PluralKit cannot proxy messages in this type of channel.";
 
         // Check if the message does not go over any Discord Nitro limits
@@ -159,6 +138,7 @@ public class ProxyService
         return null;
     }
 
+    // Proxy checks that don't give user errors unless `pk;debug proxy` is used
     public bool ShouldProxy(Channel channel, Channel rootChannel, Message msg, MessageContext ctx)
     {
         // Make sure author has a system
@@ -189,9 +169,9 @@ public class ProxyService
             throw new ProxyChecksFailedException(
                 "Your system has proxying disabled in this server. Type `pk;proxy on` to enable it.");
 
-        // Make sure we have either an attachment or message content
+        // Make sure we have an attachment, message content, or poll
         var isMessageBlank = msg.Content == null || msg.Content.Trim().Length == 0;
-        if (isMessageBlank && msg.Attachments.Length == 0)
+        if (isMessageBlank && msg.Attachments.Length == 0 && msg.Poll == null)
             throw new ProxyChecksFailedException("Message cannot be blank.");
 
         if (msg.Activity != null)
@@ -227,8 +207,8 @@ public class ProxyService
         var content = match.ProxyContent;
         if (!allowEmbeds) content = content.BreakLinkEmbeds();
 
-        var messageChannel = await _cache.GetChannel(trigger.ChannelId);
+        var messageChannel = await _cache.GetChannel(trigger.GuildId!.Value, trigger.ChannelId);
-        var rootChannel = await _cache.GetRootChannel(trigger.ChannelId);
+        var rootChannel = await _cache.GetRootChannel(trigger.GuildId!.Value, trigger.ChannelId);
         var threadId = messageChannel.IsThread() ? messageChannel.Id : (ulong?)null;
         var guild = await _cache.GetGuild(trigger.GuildId.Value);
         var guildMember = await _rest.GetGuildMember(trigger.GuildId!.Value, trigger.Author.Id);
@@ -242,6 +222,7 @@ public class ProxyService
             GuildId = trigger.GuildId!.Value,
             ChannelId = rootChannel.Id,
             ThreadId = threadId,
+            MessageId = trigger.Id,
             Name = await FixSameName(messageChannel.Id, ctx, match.Member),
             AvatarUrl = AvatarUtils.TryRewriteCdnUrl(match.Member.ProxyAvatar(ctx)),
             Content = content,
@@ -252,6 +233,7 @@ public class ProxyService
             AllowEveryone = allowEveryone,
             Flags = trigger.Flags.HasFlag(Message.MessageFlags.VoiceMessage) ? Message.MessageFlags.VoiceMessage : null,
             Tts = tts,
+            Poll = trigger.Poll,
         });
         await HandleProxyExecutedActions(ctx, autoproxySettings, trigger, proxyMessage, match);
     }
@@ -310,6 +292,7 @@ public class ProxyService
             GuildId = guild.Id,
             ChannelId = rootChannel.Id,
             ThreadId = threadId,
+            MessageId = originalMsg.Id,
             Name = match.Member.ProxyName(ctx),
             AvatarUrl = AvatarUtils.TryRewriteCdnUrl(match.Member.ProxyAvatar(ctx)),
             Content = match.ProxyContent!,
@@ -320,6 +303,7 @@ public class ProxyService
             AllowEveryone = allowEveryone,
             Flags = originalMsg.Flags.HasFlag(Message.MessageFlags.VoiceMessage) ? Message.MessageFlags.VoiceMessage : null,
             Tts = tts,
+            Poll = originalMsg.Poll,
         });
 
 

PluralKit.Bot/Services/AvatarHostingService.cs

@@ -9,22 +9,35 @@ public class AvatarHostingService
     private readonly BotConfig _config;
     private readonly HttpClient _client;
 
-    public AvatarHostingService(BotConfig config, HttpClient client)
+    public AvatarHostingService(BotConfig config)
     {
         _config = config;
-        _client = client;
+        _client = new HttpClient
+        {
+            Timeout = TimeSpan.FromSeconds(10),
+        };
     }
 
     public async Task<ParsedImage> TryRehostImage(ParsedImage input, RehostedImageType type, ulong userId, PKSystem? system)
     {
-        var uploaded = await TryUploadAvatar(input.Url, type, userId, system);
+        try
-        if (uploaded != null)
         {
-            // todo: make new image type called Cdn?
+            var uploaded = await TryUploadAvatar(input.Url, type, userId, system);
-            return new ParsedImage { Url = uploaded, Source = AvatarSource.HostedCdn };
+            if (uploaded != null)
-        }
+            {
+                // todo: make new image type called Cdn?
+                return new ParsedImage { Url = uploaded, Source = AvatarSource.HostedCdn };
+            }
 
-        return input;
+            return input;
+        }
+        catch (TaskCanceledException e)
+        {
+            // don't show an internal error to users
+            if (e.Message.Contains("HttpClient.Timeout"))
+                throw new PKError("Temporary error setting image, please try again later");
+            throw;
+        }
     }
 
     public async Task<string?> TryUploadAvatar(string? avatarUrl, RehostedImageType type, ulong userId, PKSystem? system)

PluralKit.Bot/Services/CommandMessageService.cs

@@ -18,7 +18,7 @@ public class CommandMessageService
         _logger = logger.ForContext<CommandMessageService>();
     }
 
-    public async Task RegisterMessage(ulong messageId, ulong channelId, ulong authorId)
+    public async Task RegisterMessage(ulong messageId, ulong guildId, ulong channelId, ulong authorId)
     {
         if (_redis.Connection == null) return;
 
@@ -27,17 +27,19 @@ public class CommandMessageService
             messageId, authorId, channelId
         );
 
-        await _redis.Connection.GetDatabase().StringSetAsync(messageId.ToString(), $"{authorId}-{channelId}", expiry: CommandMessageRetention);
+        await _redis.Connection.GetDatabase().StringSetAsync(messageId.ToString(), $"{authorId}-{channelId}-{guildId}", expiry: CommandMessageRetention);
     }
 
-    public async Task<(ulong?, ulong?)> GetCommandMessage(ulong messageId)
+    public async Task<CommandMessage?> GetCommandMessage(ulong messageId)
     {
         var str = await _redis.Connection.GetDatabase().StringGetAsync(messageId.ToString());
         if (str.HasValue)
         {
             var split = ((string)str).Split("-");
-            return (ulong.Parse(split[0]), ulong.Parse(split[1]));
+            return new CommandMessage(ulong.Parse(split[0]), ulong.Parse(split[1]), ulong.Parse(split[2]));
         }
-        return (null, null);
+        return null;
     }
-}
+}
+
+public record CommandMessage(ulong AuthorId, ulong ChannelId, ulong GuildId);

PluralKit.Bot/Services/EmbedService.cs

@@ -263,7 +263,7 @@ public class EmbedService
         else if (system.NameFor(ctx) != null)
             nameField = $"{nameField} ({system.NameFor(ctx)})";
         else
-            nameField = $"{nameField} ({system.Name})";
+            nameField = $"{nameField}";
 
         var eb = new EmbedBuilder()
             .Author(new Embed.EmbedAuthor(nameField, IconUrl: target.IconFor(pctx), Url: $"https://dash.pluralkit.me/profile/g/{target.Hid}"))
@@ -336,7 +336,7 @@ public class EmbedService
 
     public async Task<Embed> CreateMessageInfoEmbed(FullMessage msg, bool showContent, SystemConfig? ccfg = null)
     {
-        var channel = await _cache.GetOrFetchChannel(_rest, msg.Message.Channel);
+        var channel = await _cache.GetOrFetchChannel(_rest, msg.Message.Guild ?? 0, msg.Message.Channel);
         var ctx = LookupContext.ByNonOwner;
 
         var serverMsg = await _rest.GetMessageOrNull(msg.Message.Channel, msg.Message.Mid);
@@ -403,14 +403,15 @@ public class EmbedService
         var roles = memberInfo?.Roles?.ToList();
         if (roles != null && roles.Count > 0 && showContent)
         {
-            var rolesString = string.Join(", ", (await Task.WhenAll(roles
+            var guild = await _cache.GetGuild(channel.GuildId!.Value);
-                    .Select(async id =>
+            var rolesString = string.Join(", ", (roles
+                    .Select(id =>
                     {
-                        var role = await _cache.TryGetRole(id);
+                        var role = Array.Find(guild.Roles, r => r.Id == id);
                         if (role != null)
                             return role;
                         return new Role { Name = "*(unknown role)*", Position = 0 };
-                    })))
+                    }))
                 .OrderByDescending(role => role.Position)
                 .Select(role => role.Name));
             eb.Field(new Embed.Field($"Account roles ({roles.Count})", rolesString.Truncate(1024)));

PluralKit.Bot/Services/LogChannelService.cs

@@ -42,7 +42,7 @@ public class LogChannelService
         if (logChannelId == null)
             return;
 
-        var triggerChannel = await _cache.GetChannel(proxiedMessage.Channel);
+        var triggerChannel = await _cache.GetChannel(proxiedMessage.Guild!.Value, proxiedMessage.Channel);
 
         var member = await _repo.GetMember(proxiedMessage.Member!.Value);
         var system = await _repo.GetSystem(member.System);
@@ -63,7 +63,7 @@ public class LogChannelService
             return null;
 
         var guildId = proxiedMessage.Guild ?? trigger.GuildId.Value;
-        var rootChannel = await _cache.GetRootChannel(trigger.ChannelId);
+        var rootChannel = await _cache.GetRootChannel(guildId, trigger.ChannelId);
 
         // get log channel info from the database
         var guild = await _repo.GetGuild(guildId);
@@ -109,7 +109,7 @@ public class LogChannelService
     private async Task<Channel?> FindLogChannel(ulong guildId, ulong channelId)
     {
         // TODO: fetch it directly on cache miss?
-        if (await _cache.TryGetChannel(channelId) is Channel channel)
+        if (await _cache.TryGetChannel(guildId, channelId) is Channel channel)
             return channel;
 
         if (await _rest.GetChannelOrNull(channelId) is Channel restChannel)

PluralKit.Bot/Services/LoggerCleanService.cs

@@ -23,6 +23,7 @@ public class LoggerCleanService
     private static readonly Regex _basicRegex = new("(\\d{17,19})");
     private static readonly Regex _dynoRegex = new("Message ID: (\\d{17,19})");
     private static readonly Regex _carlRegex = new("Message ID: (\\d{17,19})");
+    private static readonly Regex _makiRegex = new("Message ID: (\\d{17,19})");
     private static readonly Regex _circleRegex = new("\\(`(\\d{17,19})`\\)");
     private static readonly Regex _loggerARegex = new("Message = (\\d{17,19})");
     private static readonly Regex _loggerBRegex = new("MessageID:(\\d{17,19})");
@@ -60,6 +61,7 @@ public class LoggerCleanService
         new LoggerBot("Dyno#8389", 470724017205149701, ExtractDyno), // webhook
         new LoggerBot("Dyno#5714", 470723870270160917, ExtractDyno), // webhook
         new LoggerBot("Dyno#1961", 347378323418251264, ExtractDyno), // webhook
+        new LoggerBot("Maki", 563434444321587202, ExtractMaki), // webhook
         new LoggerBot("Auttaja", 242730576195354624, ExtractAuttaja), // webhook
         new LoggerBot("GenericBot", 295329346590343168, ExtractGenericBot),
         new LoggerBot("blargbot", 134133271750639616, ExtractBlargBot),
@@ -98,10 +100,10 @@ public class LoggerCleanService
 
     public async ValueTask HandleLoggerBotCleanup(Message msg)
     {
-        var channel = await _cache.GetChannel(msg.ChannelId);
+        var channel = await _cache.GetChannel(msg.GuildId!.Value, msg.ChannelId!);
 
         if (channel.Type != Channel.ChannelType.GuildText) return;
-        if (!(await _cache.BotPermissionsIn(channel.Id)).HasFlag(PermissionSet.ManageMessages)) return;
+        if (!(await _cache.BotPermissionsIn(msg.GuildId!.Value, channel.Id)).HasFlag(PermissionSet.ManageMessages)) return;
 
         // If this message is from a *webhook*, check if the application ID matches one of the bots we know
         // If it's from a *bot*, check the bot ID to see if we know it.
@@ -231,6 +233,15 @@ public class LoggerCleanService
         return match.Success ? ulong.Parse(match.Groups[1].Value) : null;
     }
 
+    private static ulong? ExtractMaki(Message msg)
+    {
+        // Embed, Message Author Name field: "Message Deleted", footer is "Message ID: [id]"
+        var embed = msg.Embeds?.FirstOrDefault();
+        if (embed.Author?.Name == null || embed?.Footer == null || !embed.Author.Name.StartsWith("Message Deleted")) return null;
+        var match = _makiRegex.Match(embed.Footer.Text ?? "");
+        return match.Success ? ulong.Parse(match.Groups[1].Value) : null;
+    }
+
     private static FuzzyExtractResult? ExtractCircle(Message msg)
     {
         // Like Auttaja, Circle has both embed and compact modes, but the regex works for both.

PluralKit.Bot/Services/PeriodicStatCollector.cs

@@ -54,33 +54,6 @@ public class PeriodicStatCollector
         var stopwatch = new Stopwatch();
         stopwatch.Start();
 
-        // Aggregate guild/channel stats
-        var guildCount = 0;
-        var channelCount = 0;
-
-        // No LINQ today, sorry
-        await foreach (var guild in _cache.GetAllGuilds())
-        {
-            guildCount++;
-            foreach (var channel in await _cache.GetGuildChannels(guild.Id))
-                if (DiscordUtils.IsValidGuildChannel(channel))
-                    channelCount++;
-        }
-
-        if (_config.UseRedisMetrics)
-        {
-            var db = _redis.Connection.GetDatabase();
-            await db.HashSetAsync("pluralkit:cluster_stats", new StackExchange.Redis.HashEntry[] {
-                new(_botConfig.Cluster.NodeIndex, JsonConvert.SerializeObject(new ClusterMetricInfo
-                {
-                    GuildCount = guildCount,
-                    ChannelCount = channelCount,
-                    DatabaseConnectionCount = _countHolder.ConnectionCount,
-                    WebhookCacheSize = _webhookCache.CacheSize,
-                })),
-            });
-        }
-
         // Process info
         var process = Process.GetCurrentProcess();
         _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPhysicalMemory, process.WorkingSet64);

PluralKit.Bot/Services/WebhookExecutorService.cs

@@ -4,6 +4,8 @@ using App.Metrics;
 
 using Humanizer;
 
+using NodaTime.Text;
+
 using Myriad.Cache;
 using Myriad.Extensions;
 using Myriad.Rest;
@@ -35,6 +37,7 @@ public record ProxyRequest
     public ulong GuildId { get; init; }
     public ulong ChannelId { get; init; }
     public ulong? ThreadId { get; init; }
+    public ulong MessageId { get; init; }
     public string Name { get; init; }
     public string? AvatarUrl { get; init; }
     public string? Content { get; init; }
@@ -45,6 +48,7 @@ public record ProxyRequest
     public bool AllowEveryone { get; init; }
     public Message.MessageFlags? Flags { get; init; }
     public bool Tts { get; init; }
+    public Message.MessagePoll? Poll { get; init; }
 }
 
 public class WebhookExecutorService
@@ -83,7 +87,7 @@ public class WebhookExecutorService
         return webhookMessage;
     }
 
-    public async Task<Message> EditWebhookMessage(ulong channelId, ulong messageId, string newContent, bool clearEmbeds = false)
+    public async Task<Message> EditWebhookMessage(ulong guildId, ulong channelId, ulong messageId, string newContent, bool clearEmbeds = false)
     {
         var allowedMentions = newContent.ParseMentions() with
         {
@@ -92,7 +96,7 @@ public class WebhookExecutorService
         };
 
         ulong? threadId = null;
-        var channel = await _cache.GetOrFetchChannel(_rest, channelId);
+        var channel = await _cache.GetOrFetchChannel(_rest, guildId, channelId);
         if (channel.IsThread())
         {
             threadId = channelId;
@@ -154,6 +158,26 @@ public class WebhookExecutorService
             }).ToArray();
         }
 
+        if (req.Poll is Message.MessagePoll poll)
+        {
+            int? duration = null;
+            if (poll.Expiry is string expiry)
+            {
+                var then = OffsetDateTimePattern.ExtendedIso.Parse(expiry).Value.ToInstant();
+                var now = DiscordUtils.SnowflakeToInstant(req.MessageId);
+                // in theory .TotalHours should be exact, but just in case
+                duration = (int)Math.Round((then - now).TotalMinutes / 60.0);
+            }
+            webhookReq.Poll = new ExecuteWebhookRequest.WebhookPoll
+            {
+                Question = poll.Question,
+                Answers = poll.Answers,
+                Duration = duration,
+                AllowMultiselect = poll.AllowMultiselect,
+                LayoutType = poll.LayoutType
+            };
+        }
+
         Message webhookMessage;
         using (_metrics.Measure.Timer.Time(BotMetrics.WebhookResponseTime))
         {

PluralKit.Bot/Utils/MiscUtils.cs

@@ -49,17 +49,17 @@ public static class MiscUtils
         if (e is WebhookExecutionErrorOnDiscordsEnd) return false;
 
         // Socket errors are *not our problem*
-        if (e.GetBaseException() is SocketException) return false;
+        // if (e.GetBaseException() is SocketException) return false;
 
         // Tasks being cancelled for whatver reason are, you guessed it, also not our problem.
-        if (e is TaskCanceledException) return false;
+        // if (e is TaskCanceledException) return false;
 
         // Sometimes Discord just times everything out.
-        if (e is TimeoutException) return false;
+        // if (e is TimeoutException) return false;
         if (e is UnknownDiscordRequestException tde && tde.Message == "Request Timeout") return false;
 
         // HTTP/2 streams are complicated and break sometimes.
-        if (e is HttpRequestException) return false;
+        // if (e is HttpRequestException) return false;
 
         // This may expanded at some point.
         return true;

PluralKit.Bot/Utils/ModelUtils.cs

@@ -29,14 +29,14 @@ public static class ModelUtils
 
 
     public static string DisplayHid(this PKSystem system, SystemConfig? cfg = null, bool isList = false) => HidTransform(system.Hid, cfg, isList);
-    public static string DisplayHid(this PKGroup group, SystemConfig? cfg = null, bool isList = false) => HidTransform(group.Hid, cfg, isList);
+    public static string DisplayHid(this PKGroup group, SystemConfig? cfg = null, bool isList = false, bool shouldPad = true) => HidTransform(group.Hid, cfg, isList, shouldPad);
-    public static string DisplayHid(this PKMember member, SystemConfig? cfg = null, bool isList = false) => HidTransform(member.Hid, cfg, isList);
+    public static string DisplayHid(this PKMember member, SystemConfig? cfg = null, bool isList = false, bool shouldPad = true) => HidTransform(member.Hid, cfg, isList, shouldPad);
-    private static string HidTransform(string hid, SystemConfig? cfg = null, bool isList = false) =>
+    private static string HidTransform(string hid, SystemConfig? cfg = null, bool isList = false, bool shouldPad = true) =>
         HidUtils.HidTransform(
             hid,
             cfg != null && cfg.HidDisplaySplit,
             cfg != null && cfg.HidDisplayCaps,
-            isList ? (cfg?.HidListPadding ?? SystemConfig.HidPadFormat.None) : SystemConfig.HidPadFormat.None // padding only on lists
+            isList && shouldPad ? (cfg?.HidListPadding ?? SystemConfig.HidPadFormat.None) : SystemConfig.HidPadFormat.None // padding only on lists
         );
 
     private static string EntityReference(string hid, string name)

PluralKit.Bot/Utils/SerilogGatewayEnricherFactory.cs

@@ -38,9 +38,11 @@ public class SerilogGatewayEnricherFactory
         {
             props.Add(new LogEventProperty("ChannelId", new ScalarValue(channel.Value)));
 
-            if (await _cache.TryGetChannel(channel.Value) != null)
+            var guildIdForCache = guild != null ? guild.Value : 0;
+
+            if (await _cache.TryGetChannel(guildIdForCache, channel.Value) != null)
             {
-                var botPermissions = await _cache.BotPermissionsIn(channel.Value);
+                var botPermissions = await _cache.BotPermissionsIn(guildIdForCache, channel.Value);
                 props.Add(new LogEventProperty("BotPermissions", new ScalarValue(botPermissions)));
             }
         }

PluralKit.Bot/packages.lock.json

@@ -36,9 +36,9 @@
       },
       "Sentry": {
         "type": "Direct",
-        "requested": "[3.11.1, )",
+        "requested": "[4.12.1, )",
-        "resolved": "3.11.1",
+        "resolved": "4.12.1",
-        "contentHash": "T/NLfs6MMkUSYsPEDajB9ad0124T18I0uUod5MNOev3iwjvcnIEQBrStEX2olbIxzqfvGXzQ/QFqTfA2ElLPlA=="
+        "contentHash": "OLf7885OKHWLaTLTyw884mwOT4XKCWj2Hz5Wuz/TJemJqXwCIdIljkJBIoeHviRUPvtB7ulDgeYXf/Z7ScToSA=="
       },
       "SixLabors.ImageSharp": {
         "type": "Direct",

PluralKit.Core/CoreConfig.cs

@@ -8,7 +8,6 @@ public class CoreConfig
     public string? MessagesDatabase { get; set; }
     public string? DatabasePassword { get; set; }
     public string RedisAddr { get; set; }
-    public bool UseRedisMetrics { get; set; } = false;
     public string SentryUrl { get; set; }
     public string InfluxUrl { get; set; }
     public string InfluxDb { get; set; }

PluralKit.Core/Database/Views/DatabaseViewsExt.cs

@@ -10,7 +10,11 @@ public static class DatabaseViewsExt
     public static Task<IEnumerable<ListedGroup>> QueryGroupList(this IPKConnection conn, SystemId system,
                                                                   ListQueryOptions opts)
     {
-        StringBuilder query = new StringBuilder("select * from group_list where system = @system");
+        StringBuilder query;
+        if (opts.MemberFilter == null)
+            query = new StringBuilder("select * from group_list where system = @system");
+        else
+            query = new StringBuilder("select group_list.* from group_members inner join group_list on group_list.id = group_members.group_id where member_id = @MemberFilter");
 
         if (opts.PrivacyFilter != null)
             query.Append($" and visibility = {(int)opts.PrivacyFilter}");
@@ -20,7 +24,8 @@ public static class DatabaseViewsExt
             static string Filter(string column) =>
                 $"(position(lower(@filter) in lower(coalesce({column}, ''))) > 0)";
 
-            query.Append($" and ({Filter("name")} or {Filter("display_name")}");
+            var nameColumn = opts.Context == LookupContext.ByOwner ? "name" : "public_name";
+            query.Append($" and ({Filter(nameColumn)} or {Filter("display_name")}");
             if (opts.SearchDescription)
             {
                 // We need to account for the possibility of description privacy when searching
@@ -36,7 +41,7 @@ public static class DatabaseViewsExt
 
         return conn.QueryAsync<ListedGroup>(
             query.ToString(),
-            new { system, filter = opts.Search });
+            new { system, filter = opts.Search, memberFilter = opts.MemberFilter });
     }
     public static Task<IEnumerable<ListedMember>> QueryMemberList(this IPKConnection conn, SystemId system,
                                                                   ListQueryOptions opts)
@@ -56,7 +61,8 @@ public static class DatabaseViewsExt
             static string Filter(string column) =>
                 $"(position(lower(@filter) in lower(coalesce({column}, ''))) > 0)";
 
-            query.Append($" and ({Filter("name")} or {Filter("display_name")}");
+            var nameColumn = opts.Context == LookupContext.ByOwner ? "name" : "public_name";
+            query.Append($" and ({Filter(nameColumn)} or {Filter("display_name")}");
             if (opts.SearchDescription)
             {
                 // We need to account for the possibility of description privacy when searching
@@ -81,5 +87,6 @@ public static class DatabaseViewsExt
         public bool SearchDescription;
         public LookupContext Context;
         public GroupId? GroupFilter;
+        public MemberId? MemberFilter;
     }
 }

PluralKit.Core/Database/Views/views.sql

@@ -30,7 +30,15 @@ select members.*,
            -- Privacy '1' = public; just return description as normal
            when members.description_privacy = 1 then members.description
            -- Any other privacy (rn just '2'), return null description (missing case = null in SQL)
-        end as public_description
+        end as public_description,
+        
+        -- Extract member name as seen by "the public"
+        case
+            -- Privacy '1' = public; just return name as normal
+            when members.name_privacy = 1 then members.name
+            -- Any other privacy (rn just '2'), return display name
+            else coalesce(members.display_name, members.name)
+        end as public_name
 from members;
 
 create view group_list as
@@ -48,5 +56,20 @@ select groups.*,
             inner join members on group_members.member_id = members.id
         where
             group_members.group_id = groups.id
-    ) as total_member_count
+    ) as total_member_count,
-from groups;
+
+    -- Extract group description as seen by "the public"
+    case
+        -- Privacy '1' = public; just return description as normal
+        when groups.description_privacy = 1 then groups.description
+        -- Any other privacy (rn just '2'), return null description (missing case = null in SQL)
+    end as public_description,
+    
+    -- Extract member name as seen by "the public"
+    case
+        -- Privacy '1' = public; just return name as normal
+        when groups.name_privacy = 1 then groups.name
+        -- Any other privacy (rn just '2'), return display name
+        else coalesce(groups.display_name, groups.name)
+    end as public_name
+from groups;

PluralKit.Core/Dispatch/DispatchService.cs

@@ -247,7 +247,7 @@ public class DispatchService
     {
         var repo = _provider.Resolve<ModelRepository>();
         var system = await repo.GetSystemByAccount(accountId);
-        if (system.WebhookUrl == null)
+        if (system?.WebhookUrl == null)
             return;
 
         var data = new UpdateDispatchData();

PluralKit.Core/Utils/HandlerQueue.cs

@@ -29,7 +29,7 @@ public class HandlerQueue<T>
         {
             var theTask = await Task.WhenAny(timeoutTask, tcs.Task);
             if (theTask == timeoutTask)
-                throw new TimeoutException();
+                throw new TimeoutException("HandlerQueue#WaitFor timed out");
         }
         finally
         {

dashboard/package.json

@@ -27,7 +27,7 @@
     "bootstrap": "^5.1.3",
     "bootstrap-dark-5": "^1.1.3",
     "core-js-pure": "^3.23.4",
-    "discord-markdown": "github:draconizations/discord-markdown#1f74a7094777d5bdfd123c0aac59d8b10db89b30",
+    "discord-markdown": "github:draconizations/discord-markdown#9d25e45015766779916baea52c37ae0fe12aac73",
     "gh-pages": "^3.2.3",
     "highlight.js": "^11.7.0",
     "import": "^0.0.6",

dashboard/src/components/common/CardsHeader.svelte

@@ -39,8 +39,6 @@
         else icon_url = item.webhook_avatar_url ?? default_avatar
     }
 
-    $: icon_url_resized = icon_url ? resizeMedia(icon_url) : default_avatar
-
     let avatarOpen = false;
     const toggleAvatarModal = () => (avatarOpen = !avatarOpen);
 
@@ -65,8 +63,8 @@
             <span bind:this={nameElement} style="vertical-align: middle;"><AwaitHtml htmlPromise={htmlNamePromise} /> ({item.id})</span>
         </div>
         <div style="margin-left: auto;">
-        {#if item && (item.avatar_url || item.icon)}
+        {#if item && (item.avatar_url || item.webhook_avatar_url || item.icon)}
-        <img tabindex={0} on:keydown|stopPropagation={(event) => {if (event.key === "Enter") {avatarOpen = true}}} on:click|stopPropagation={toggleAvatarModal} class="rounded-circle avatar" src={icon_url_resized} alt={altText} />
+        <img tabindex={0} on:keydown|stopPropagation={(event) => {if (event.key === "Enter") {avatarOpen = true}}} on:click|stopPropagation={toggleAvatarModal} class="rounded-circle avatar" src={icon_url} alt={altText} />
         {:else}
         <img class="rounded-circle avatar" src={default_avatar} alt="icon (default)" tabindex={0} />
         {/if}

dashboard/src/components/group/Edit.svelte

@@ -25,6 +25,9 @@
         err = [];
         success = false;
 
+        // trim all string fields
+        Object.keys(data).forEach(k => data[k] = typeof data[k] == "string" ? data[k].trim() : data[k]);
+
         if (!data.name) err.push("Group name cannot be empty.")
 
         if (data.color && !/^#?[A-Fa-f0-9]{6}$/.test(input.color)) {
@@ -35,9 +38,6 @@
             }
         }
 
-        // trim all string fields
-        Object.keys(data).forEach(k => data[k] = typeof data[k] == "string" ? data[k].trim() : data[k]);
-
         err = err;
         if (err.length > 0) return;
 

dashboard/src/components/member/Edit.svelte

@@ -24,6 +24,9 @@
         let data = input;
         err = [];
 
+        // trim all string fields
+        Object.keys(data).forEach(k => data[k] = typeof data[k] == "string" ? data[k].trim() : data[k]);
+
         if (!data.name) err.push("Member name cannot be empty.")
 
         if (data.color && !/^#?[A-Fa-f0-9]{6}$/.test(input.color)) {
@@ -58,9 +61,6 @@
             }
         }
 
-        // trim all string fields
-        Object.keys(data).forEach(k => data[k] = typeof data[k] == "string" ? data[k].trim() : data[k]);
-
         err = err;
         if (err.length > 0) return;
 

dashboard/src/main.ts

@@ -7,7 +7,7 @@ if (!String.prototype.replaceAll)
   String.prototype.replaceAll = replaceAll;
 
 Sentry.init({
-  dsn: "https://973beecd91934f9992c72c942770bdd2@sentry.pluralkit.me/3",
+  dsn: "https://79ba4b55fdce475ebc5d37df8b75d72a@gt.pluralkit.me/5",
   integrations: [new Integrations.BrowserTracing()],
 
   enabled: !window.location.origin.includes("localhost"),

dashboard/styles/dark.scss

@@ -52,4 +52,8 @@
     .d-spoiler:active {
         color: $body-color-alt; //overwrite
     }
+
+    small {
+        color: #707070;
+    }
 }

dashboard/styles/generic.scss

@@ -193,6 +193,12 @@ code {
     vertical-align: -0.1125em;
 }
 
+small {
+    display: block;
+    color: #808080;
+    font-size: 0.5rem;
+}
+
 //twemoji
 img.emoji {
     height: 1.125em;

dashboard/yarn.lock

@@ -341,9 +341,9 @@ detect-indent@^6.0.0:
   resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.1.0.tgz#592485ebbbf6b3b1ab2be175c8393d04ca0d57e6"
   integrity sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==
 
-"discord-markdown@github:draconizations/discord-markdown#1f74a7094777d5bdfd123c0aac59d8b10db89b30":
+"discord-markdown@github:draconizations/discord-markdown#9d25e45015766779916baea52c37ae0fe12aac73":
   version "2.5.1"
-  resolved "https://codeload.github.com/draconizations/discord-markdown/tar.gz/1f74a7094777d5bdfd123c0aac59d8b10db89b30"
+  resolved "https://codeload.github.com/draconizations/discord-markdown/tar.gz/9d25e45015766779916baea52c37ae0fe12aac73"
   dependencies:
     js-base64 "^3.7.7"
     simple-markdown "^0.7.3"

docs/content/api/changelog.md

@@ -5,7 +5,7 @@ permalink: /api/changelog
 
 # Version history
 
-* 2024-08-84
+* 2024-08-04
   * Added ratelimit scopes (separate limits for different sets of endpoints)
 * 2024-05-01
   * Short IDs (the `id` field in system / member / group models) can now be either 5 or 6 characters.

docs/content/command-list.md

@@ -123,7 +123,8 @@ You can have a space after `pk;`, e.g. `pk;system` and `pk; system` will do the
 ## Switching commands
 - `pk;switch [member...]` - Registers a switch with the given members.
 - `pk;switch out` - Registers a 'switch-out' - a switch with no associated members.
-- `pk;switch edit <member...|out>` - Edits the members in the latest switch. 
+- `pk;switch edit <member...|out>` - Edits the members in the latest switch.
+- `pk;switch add <member...>` - Makes a new switch based off the current switch with the listed members added or removed.
 - `pk;switch move <time>` - Moves the latest switch backwards in time.
 - `pk;switch delete` - Deletes the latest switch.
 - `pk;switch delete all` - Deletes all logged switches.

docs/content/faq.md

@@ -26,6 +26,13 @@ We also track feature requests through [Github Issues](https://github.com/Plural
 ### How can I support the bot's development?
 I (the bot author, [Ske](https://twitter.com/floofstrid)) have a Patreon. The income from there goes towards server hosting, domains, infrastructure, my Monster Energy addiction, et cetera. There are no benefits. There might never be any. But nevertheless, it can be found here: [https://www.patreon.com/floofstrid](https://www.patreon.com/floofstrid)
 
+### Can I recover my system if I lose access to my Discord account?
+Yes, through one of two methods. Both require you to do preparations **before** you lose the account. 
+
+Option 1: If you have an alternate discord account you can link your PluralKit system to that account using `pk;link <@account>`. Then if you use access to your main discord account, you already have access on your alternate account. 
+
+Option 2: The PluralKit staff can help you recover your system if you have your token (gotten using `pk;token`). This is the *only* way you can prove ownership so we can help you recover your system, so store it in a safe place. Make sure to keep your token safe; if other people get access to it they can also use it to access your system. If your token is ever compromised run `pk;token refresh` to invalidate the old token and get a new one.
+
 ## Privacy / safety
 
 ### Who has access to the bot's data?
@@ -91,3 +98,5 @@ It is not possible to edit messages via ID. Please use the full link, or reply t
 ### How do I reply-ping/reply-@ proxied messages?
 You cannot reply-@ a proxied messages due to their nature as webhooks. If you want to "reply-@" a proxied message, you must react to the message with 🔔, 🛎, or 🏓. This will send a message from PluralKit that reads "Psst, MEMBER (@User), you have been pinged by @You", which will ping the Discord account behind the proxied message.
 
+### Why do most of PluralKit's messages look blank or empty?
+A lot of PluralKit's command responses use Discord embeds. If you can't see them, it's likely you have embeds turned off. To change this, go into your discord settings and find the tab "Chat" under "App Settings". Find the setting "Show embeds and preview website links" and turn it on. If it's already on, try turning it off and then on again. 

docs/content/getting-started.md

@@ -61,7 +61,7 @@ If you don't have a link, you can leave that out entirely, and then **attach** t
 Avatars have some restrictions: 
 - The image must be in **.jpg**, **.png**, or **.webp** format
 - The image must be under **1024 KB** in size
-- The image must be below **1024 x 1024 pixels** in resolution (along the smallest axis).
+- The image must be below **1000 x 1000 pixels** in resolution (along the smallest axis).
 - Animated GIFs are **not** supported (even if you have Nitro).
 :::
 
@@ -73,4 +73,4 @@ You could...
 - [configure privacy settings](/guide/#privacy)
 - or something else!
 
-See the [User Guide](/guide) for a more complete reference of the bot's features.
+See the [User Guide](/guide) for a more complete reference of the bot's features.

docs/content/tips-and-tricks.md

@@ -77,11 +77,16 @@ You cannot look up private members or groups of another system.
 |pk;system frontpercent|-flat||Show "flat" frontpercent - percentages add up to 100%|
 |pk;group \<group> frontpercent|-fronters-only|-fo|Show a group's frontpercent without the "no fronter" entry|
 |pk;group \<group> frontpercent|-flat||Show "flat" frontpercent - percentages add up to 100%|
+|pk;group \<group> \<add/remove>|-all|-a|Add or remove all members from a group.|
 |pk;edit|-append|-a|Append the new content to the old message instead of overwriting it|
 |pk;edit|-prepend|-p|Prepend the new content to the old message instead of overwriting it|
 |pk;edit|-nospace|-ns|Append/prepend without adding a space|
 |pk;edit|-clear-embed|-ce|Remove embeds from a message|
 |pk;edit|-regex|-x|Edit using a C# Regex formatted like s\|X\|Y or s\|X\|Y\|F, where \| is any character, X is a Regex, Y is a substitution string, and F is a set of Regex flags|
+|pk;switch edit and pk;switch add|-append|-a|Append members to the current switch or make a new switch with members appended|
+|pk;switch edit and pk;switch add|-prepend|-p|Prepend members to the current switch or make a new switch with members prepended|
+|pk;switch edit and pk;switch add|-first|-f|Move member to the front of the current switch or make a new switch with the member at the front|
+|pk;switch edit and pk;switch add|-remove|-r|Remove members from the current switch or make a new switch with members removed|
 |Most commands|-all|-a|Show hidden/private information|
 |Most commands|-raw|-r|Show text with formatting, for easier copy-pasting|
 |All commands|-private|-priv|Show private information|

go.work

@@ -2,5 +2,4 @@ go 1.19
 
 use (
 	./services/scheduled_tasks
-	./services/web-proxy
 )

lib/libpk/Cargo.toml

@@ -5,17 +5,21 @@ edition = "2021"
 
 [dependencies]
 anyhow = { workspace = true } 
-config = "0.13.3"
+config = "0.14.0"
 fred = { workspace = true }
 gethostname = "0.4.1"
 lazy_static = { workspace = true }
 metrics = { workspace = true }
-metrics-exporter-prometheus = { version = "0.11.0", default-features = false, features = ["tokio", "http-listener", "tracing"] }
+metrics-exporter-prometheus = { version = "0.15.3", default-features = false, features = ["tokio", "http-listener", "tracing"] }
 serde = { workspace = true }
 sqlx = { workspace = true }
+time = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
+tracing-gelf = "0.7.1"
 tracing-subscriber = { workspace = true}
+twilight-model = { workspace = true }
+uuid = { workspace = true }
 
 prost = { workspace = true }
 prost-types = { workspace = true }

lib/libpk/src/_config.rs

@@ -3,11 +3,27 @@ use lazy_static::lazy_static;
 use serde::Deserialize;
 use std::sync::Arc;
 
+use twilight_model::id::{marker::UserMarker, Id};
+
+#[derive(Clone, Deserialize, Debug)]
+pub struct ClusterSettings {
+    pub node_id: u32,
+    pub total_shards: u32,
+    pub total_nodes: u32,
+}
+
 #[derive(Deserialize, Debug)]
 pub struct DiscordConfig {
-    pub client_id: u32,
+    pub client_id: Id<UserMarker>,
     pub bot_token: String,
     pub client_secret: String,
+    pub max_concurrency: u32,
+    #[serde(default)]
+    pub cluster: Option<ClusterSettings>,
+    pub api_base_url: Option<String>,
+
+    #[serde(default = "_default_api_addr")]
+    pub cache_api_addr: String,
 }
 
 #[derive(Deserialize, Debug)]
@@ -24,7 +40,7 @@ fn _default_api_addr() -> String {
     "0.0.0.0:5000".to_string()
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Clone, Debug)]
 pub struct ApiConfig {
     #[serde(default = "_default_api_addr")]
     pub addr: String,
@@ -38,27 +54,69 @@ pub struct ApiConfig {
     pub temp_token2: Option<String>,
 }
 
+#[derive(Deserialize, Clone, Debug)]
+pub struct AvatarsConfig {
+    pub s3: S3Config,
+    pub cdn_url: String,
+
+    #[serde(default)]
+    pub migrate_worker_count: u32,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct S3Config {
+    pub bucket: String,
+    pub application_id: String,
+    pub application_key: String,
+    pub endpoint: String,
+}
+
 fn _metrics_default() -> bool {
     false
 }
+fn _json_log_default() -> bool {
+    false
+}
 
 #[derive(Deserialize, Debug)]
 pub struct PKConfig {
     pub db: DatabaseConfig,
 
-    pub discord: DiscordConfig,
+    #[serde(default)]
-    pub api: ApiConfig,
+    pub discord: Option<DiscordConfig>,
+    #[serde(default)]
+    pub api: Option<ApiConfig>,
+    #[serde(default)]
+    pub avatars: Option<AvatarsConfig>,
 
     #[serde(default = "_metrics_default")]
     pub run_metrics_server: bool,
 
-    pub(crate) gelf_log_url: Option<String>,
+    #[serde(default = "_json_log_default")]
+    pub(crate) json_log: bool,
+}
+
+impl PKConfig {
+    pub fn api(self) -> ApiConfig {
+        self.api.expect("missing api config")
+    }
+
+    pub fn discord_config(self) -> DiscordConfig {
+        self.discord.expect("missing discord config")
+    }
 }
 
 lazy_static! {
     #[derive(Debug)]
-    pub static ref CONFIG: Arc<PKConfig> = Arc::new(Config::builder()
+    pub static ref CONFIG: Arc<PKConfig> = {
-    .add_source(config::Environment::with_prefix("pluralkit").separator("__"))
+        if let Ok(var) = std::env::var("NOMAD_ALLOC_INDEX")
-    .build().unwrap()
+            && std::env::var("pluralkit__discord__cluster__total_nodes").is_ok() {
-    .try_deserialize::<PKConfig>().unwrap());
+            std::env::set_var("pluralkit__discord__cluster__node_id", var);
+        }
+
+        Arc::new(Config::builder()
+        .add_source(config::Environment::with_prefix("pluralkit").separator("__"))
+        .build().unwrap()
+        .try_deserialize::<PKConfig>().unwrap())
+    };
 }

lib/libpk/src/db/mod.rs

@@ -4,6 +4,7 @@ use std::str::FromStr;
 use tracing::info;
 
 pub mod repository;
+pub mod types;
 
 pub async fn init_redis() -> anyhow::Result<RedisPool> {
     info!("connecting to redis");

lib/libpk/src/db/repository/avatars.rs

@@ -0,0 +1,87 @@
+use sqlx::{PgPool, Postgres, Transaction};
+
+use crate::db::types::avatars::*;
+
+pub async fn get_by_original_url(
+    pool: &PgPool,
+    original_url: &str,
+) -> anyhow::Result<Option<ImageMeta>> {
+    Ok(
+        sqlx::query_as("select * from images where original_url = $1")
+            .bind(original_url)
+            .fetch_optional(pool)
+            .await?,
+    )
+}
+
+pub async fn get_by_attachment_id(
+    pool: &PgPool,
+    attachment_id: u64,
+) -> anyhow::Result<Option<ImageMeta>> {
+    Ok(
+        sqlx::query_as("select * from images where original_attachment_id = $1")
+            .bind(attachment_id as i64)
+            .fetch_optional(pool)
+            .await?,
+    )
+}
+
+pub async fn pop_queue(
+    pool: &PgPool,
+) -> anyhow::Result<Option<(Transaction<Postgres>, ImageQueueEntry)>> {
+    let mut tx = pool.begin().await?;
+    let res: Option<ImageQueueEntry> = sqlx::query_as("delete from image_queue where itemid = (select itemid from image_queue order by itemid for update skip locked limit 1) returning *")
+        .fetch_optional(&mut *tx).await?;
+    Ok(res.map(|x| (tx, x)))
+}
+
+pub async fn get_queue_length(pool: &PgPool) -> anyhow::Result<i64> {
+    Ok(sqlx::query_scalar("select count(*) from image_queue")
+        .fetch_one(pool)
+        .await?)
+}
+
+pub async fn get_stats(pool: &PgPool) -> anyhow::Result<Stats> {
+    Ok(sqlx::query_as(
+        "select count(*) as total_images, sum(file_size) as total_file_size from images",
+    )
+    .fetch_one(pool)
+    .await?)
+}
+
+pub async fn add_image(pool: &PgPool, meta: ImageMeta) -> anyhow::Result<bool> {
+    let kind_str = match meta.kind {
+        ImageKind::Avatar => "avatar",
+        ImageKind::Banner => "banner",
+    };
+
+    let res = sqlx::query("insert into images (id, url, content_type, original_url, file_size, width, height, original_file_size, original_type, original_attachment_id, kind, uploaded_by_account, uploaded_by_system, uploaded_at) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, (now() at time zone 'utc')) on conflict (id) do nothing")
+        .bind(meta.id)
+        .bind(meta.url)
+        .bind(meta.content_type)
+        .bind(meta.original_url)
+        .bind(meta.file_size)
+        .bind(meta.width)
+        .bind(meta.height)
+        .bind(meta.original_file_size)
+        .bind(meta.original_type)
+        .bind(meta.original_attachment_id)
+        .bind(kind_str)
+        .bind(meta.uploaded_by_account)
+        .bind(meta.uploaded_by_system)
+        .execute(pool).await?;
+    Ok(res.rows_affected() > 0)
+}
+
+pub async fn push_queue(
+    conn: &mut sqlx::PgConnection,
+    url: &str,
+    kind: ImageKind,
+) -> anyhow::Result<()> {
+    sqlx::query("insert into image_queue (url, kind) values ($1, $2)")
+        .bind(url)
+        .bind(kind)
+        .execute(conn)
+        .await?;
+    Ok(())
+}

lib/libpk/src/db/repository/mod.rs

@@ -1,5 +1,7 @@
 mod stats;
 pub use stats::*;
 
+pub mod avatars;
+
 mod auth;
 pub use auth::*;

lib/libpk/src/db/types/avatars.rs

@@ -0,0 +1,53 @@
+use serde::{Deserialize, Serialize};
+use sqlx::FromRow;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+#[derive(FromRow)]
+pub struct ImageMeta {
+    pub id: String,
+    pub kind: ImageKind,
+    pub content_type: String,
+    pub url: String,
+    pub file_size: i32,
+    pub width: i32,
+    pub height: i32,
+    pub uploaded_at: Option<OffsetDateTime>,
+
+    pub original_url: Option<String>,
+    pub original_attachment_id: Option<i64>,
+    pub original_file_size: Option<i32>,
+    pub original_type: Option<String>,
+    pub uploaded_by_account: Option<i64>,
+    pub uploaded_by_system: Option<Uuid>,
+}
+
+#[derive(FromRow, Serialize)]
+pub struct Stats {
+    pub total_images: i64,
+    pub total_file_size: i64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, sqlx::Type, PartialEq)]
+#[serde(rename_all = "snake_case")]
+#[sqlx(rename_all = "snake_case", type_name = "text")]
+pub enum ImageKind {
+    Avatar,
+    Banner,
+}
+
+impl ImageKind {
+    pub fn size(&self) -> (u32, u32) {
+        match self {
+            Self::Avatar => (512, 512),
+            Self::Banner => (1024, 1024),
+        }
+    }
+}
+
+#[derive(FromRow)]
+pub struct ImageQueueEntry {
+    pub itemid: i32,
+    pub url: String,
+    pub kind: ImageKind,
+}

lib/libpk/src/db/types/mod.rs

@@ -0,0 +1 @@
+pub mod avatars;

lib/libpk/src/lib.rs

@@ -1,30 +1,25 @@
-use gethostname::gethostname;
+#![feature(let_chains)]
 use metrics_exporter_prometheus::PrometheusBuilder;
-use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, EnvFilter, Registry};
+use tracing_subscriber::EnvFilter;
 
 pub mod db;
 pub mod proto;
+pub mod util;
 
 pub mod _config;
 pub use crate::_config::CONFIG as config;
 
 pub fn init_logging(component: &str) -> anyhow::Result<()> {
-    let subscriber = Registry::default()
+    // todo: fix component
-        .with(EnvFilter::from_default_env())
+    if config.json_log {
-        .with(tracing_subscriber::fmt::layer());
+        tracing_subscriber::fmt()
-
+            .json()
-    if let Some(gelf_url) = &config.gelf_log_url {
+            .with_env_filter(EnvFilter::from_default_env())
-        let gelf_logger = tracing_gelf::Logger::builder()
+            .init();
-            .additional_field("component", component)
-            .additional_field("hostname", gethostname().to_str());
-        let mut conn_handle = gelf_logger
-            .init_udp_with_subscriber(gelf_url, subscriber)
-            .unwrap();
-        tokio::spawn(async move { conn_handle.connect().await });
     } else {
-        // gelf_logger internally sets the global subscriber
+        tracing_subscriber::fmt()
-        tracing::subscriber::set_global_default(subscriber)
+            .with_env_filter(EnvFilter::from_default_env())
-            .expect("unable to set global subscriber");
+            .init();
     }
 
     Ok(())

lib/libpk/src/util/mod.rs

@@ -0,0 +1 @@
+pub mod redis;

lib/libpk/src/util/redis.rs

@@ -0,0 +1,15 @@
+use fred::error::RedisError;
+
+pub trait RedisErrorExt<T> {
+    fn to_option_or_error(self) -> Result<Option<T>, RedisError>;
+}
+
+impl<T> RedisErrorExt<T> for Result<T, RedisError> {
+    fn to_option_or_error(self) -> Result<Option<T>, RedisError> {
+        match self {
+            Ok(v) => Ok(Some(v)),
+            Err(error) if error.is_not_found() => Ok(None),
+            Err(error) => Err(error),
+        }
+    }
+}

services/api/src/main.rs

@@ -63,7 +63,7 @@ async fn main() -> anyhow::Result<()> {
     let db = libpk::db::init_data_db().await?;
     let redis = libpk::db::init_redis().await?;
 
-    let rproxy_uri = Uri::from_static(&libpk::config.api.remote_url).to_string();
+    let rproxy_uri = Uri::from_static(&libpk::config.api.as_ref().expect("missing api config").remote_url).to_string();
     let rproxy_client = hyper_util::client::legacy::Client::<(), ()>::builder(TokioExecutor::new())
             .build(HttpConnector::new());
 
@@ -145,8 +145,9 @@ async fn main() -> anyhow::Result<()> {
 
         .route("/", get(|| async { axum::response::Redirect::to("https://pluralkit.me/api") }));
 
-    let addr: &str = libpk::config.api.addr.as_ref();
+    let addr: &str = libpk::config.api.as_ref().expect("missing api config").addr.as_ref();
     let listener = tokio::net::TcpListener::bind(addr).await?;
+    info!("listening on {}", addr);
     axum::serve(listener, app).await?;
 
     Ok(())

services/api/src/middleware/cors.rs

@@ -11,7 +11,7 @@ fn add_cors_headers(headers: &mut HeaderMap) {
     headers.append("Access-Control-Allow-Methods", HeaderValue::from_static("*"));
     headers.append("Access-Control-Allow-Credentials", HeaderValue::from_static("true"));
     headers.append("Access-Control-Allow-Headers", HeaderValue::from_static("Content-Type, Authorization, sentry-trace, User-Agent"));
-    headers.append("Access-Control-Expose-Headers", HeaderValue::from_static("X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset"));
+    headers.append("Access-Control-Expose-Headers", HeaderValue::from_static("X-PluralKit-Version, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-RateLimit-Scope"));
     headers.append("Access-Control-Max-Age", HeaderValue::from_static("86400"));
 }
 

services/api/src/middleware/logger.rs

@@ -49,11 +49,11 @@ pub async fn logger(request: Request, next: Next) -> Response {
     );
     histogram!(
         "pk_http_requests",
-        (elapsed as f64) / 1_000_f64,
         "method" => method.to_string(),
         "route" => endpoint.clone(),
         "status" => response.status().to_string()
-    );
+    )
+    .record((elapsed as f64) / 1_000_f64);
 
     if elapsed > MIN_LOG_TIME {
         warn!(

services/api/src/middleware/ratelimit.lua

@@ -2,13 +2,11 @@
 -- redis.replicate_commands()
 
 local rate_limit_key = KEYS[1]
-local burst = ARGV[1]
+local rate = ARGV[1]
-local rate = ARGV[2]
+local period = ARGV[2]
-local period = ARGV[3]
+local cost = tonumber(ARGV[3])
 
--- we're only ever asking for 1 request at a time
+local burst = rate
--- todo: this is no longer true
-local cost = 1 --local cost = tonumber(ARGV[4])
 
 local emission_interval = period / rate
 local increment = emission_interval * cost

services/api/src/middleware/ratelimit.rs

@@ -7,7 +7,7 @@ use axum::{
     response::Response,
 };
 use fred::{pool::RedisPool, prelude::LuaInterface, types::ReconnectPolicy, util::sha1_hash};
-use metrics::increment_counter;
+use metrics::counter;
 use tracing::{debug, error, info, warn};
 
 use crate::util::{header_or_unknown, json_err};
@@ -20,32 +20,38 @@ lazy_static::lazy_static! {
 
 // this is awful but it works
 pub fn ratelimiter<F, T>(f: F) -> FromFnLayer<F, Option<RedisPool>, T> {
-    let redis = libpk::config.api.ratelimit_redis_addr.as_ref().map(|val| {
+    let redis = libpk::config
-        let r = fred::pool::RedisPool::new(
+        .api
-            fred::types::RedisConfig::from_url_centralized(val.as_ref())
+        .as_ref()
-                .expect("redis url is invalid"),
+        .expect("missing api config")
-            10,
+        .ratelimit_redis_addr
-        )
+        .as_ref()
-        .expect("failed to connect to redis");
+        .map(|val| {
+            let r = fred::pool::RedisPool::new(
+                fred::types::RedisConfig::from_url_centralized(val.as_ref())
+                    .expect("redis url is invalid"),
+                10,
+            )
+            .expect("failed to connect to redis");
 
-        let handle = r.connect(Some(ReconnectPolicy::default()));
+            let handle = r.connect(Some(ReconnectPolicy::default()));
 
-        tokio::spawn(async move { handle });
+            tokio::spawn(async move { handle });
 
-        let rscript = r.clone();
+            let rscript = r.clone();
-        tokio::spawn(async move {
+            tokio::spawn(async move {
-            if let Ok(()) = rscript.wait_for_connect().await {
+                if let Ok(()) = rscript.wait_for_connect().await {
-                match rscript.script_load(LUA_SCRIPT).await {
+                    match rscript.script_load(LUA_SCRIPT).await {
-                    Ok(_) => info!("connected to redis for request rate limiting"),
+                        Ok(_) => info!("connected to redis for request rate limiting"),
-                    Err(err) => error!("could not load redis script: {}", err),
+                        Err(err) => error!("could not load redis script: {}", err),
+                    }
+                } else {
+                    error!("could not wait for connection to load redis script!");
                 }
-            } else {
+            });
-                error!("could not wait for connection to load redis script!");
-            }
-        });
 
-        r
+            r
-    });
+        });
 
     if redis.is_none() {
         warn!("running without request rate limiting!");
@@ -95,7 +101,12 @@ pub async fn do_request_ratelimited(
         // https://github.com/rust-lang/rust/issues/53667
         let is_temp_token2 = if let Some(header) = request.headers().clone().get("X-PluralKit-App")
         {
-            if let Some(token2) = &libpk::config.api.temp_token2 {
+            if let Some(token2) = &libpk::config
+                .api
+                .as_ref()
+                .expect("missing api config")
+                .temp_token2
+            {
                 if header.to_str().unwrap_or("invalid") == token2 {
                     true
                 } else {
@@ -137,24 +148,23 @@ pub async fn do_request_ratelimited(
             rlimit.key()
         );
 
-        let burst = 5;
         let period = 1; // seconds
+        let cost = 1; // todo: update this for group member endpoints
 
         // local rate_limit_key = KEYS[1]
-        // local burst = ARGV[1]
+        // local rate = ARGV[1]
-        // local rate = ARGV[2]
+        // local period = ARGV[2]
-        // local period = ARGV[3]
         // return {remaining, tostring(retry_after), reset_after}
         let resp = redis
             .evalsha::<(i32, String, u64), String, Vec<String>, Vec<i32>>(
                 LUA_SCRIPT_SHA.to_string(),
                 vec![rl_key.clone()],
-                vec![burst, rlimit.rate(), period],
+                vec![rlimit.rate(), period, cost],
             )
             .await;
 
         match resp {
-            Ok((mut remaining, retry_after, reset_after)) => {
+            Ok((remaining, retry_after, reset_after)) => {
                 // redis's lua doesn't support returning floats
                 let retry_after: f64 = retry_after
                     .parse()
@@ -165,7 +175,7 @@ pub async fn do_request_ratelimited(
                 } else {
                     let retry_after = (retry_after * 1_000_f64).ceil() as u64;
                     debug!("ratelimited request from {rl_key}, retry_after={retry_after}",);
-                    increment_counter!("pk_http_requests_ratelimited");
+                    counter!("pk_http_requests_ratelimited").increment(1);
                     json_err(
                         StatusCode::TOO_MANY_REQUESTS,
                         format!(
@@ -175,9 +185,6 @@ pub async fn do_request_ratelimited(
                     )
                 };
 
-                // the redis script puts burst in remaining for ??? some reason
-                remaining -= burst - rlimit.rate();
-
                 let reset_time = SystemTime::now()
                     .checked_add(Duration::from_secs(reset_after))
                     .expect("invalid timestamp")

services/avatars/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "avatars"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+libpk = { path = "../../lib/libpk" }
+anyhow = { workspace = true }
+axum = { workspace = true }
+data-encoding = "2.5.0"
+form_urlencoded = "1.2.1"
+futures = { workspace = true }
+gif = "0.13.1"
+image = { version = "0.24.8", default-features = false, features = ["gif", "jpeg", "png", "webp", "tiff"] }
+reqwest = { workspace = true }
+rust-s3 = { version = "0.33.0", default-features = false, features = ["tokio-rustls-tls"] }
+sha2 = "0.10.8"
+serde = { workspace = true }
+sqlx = { workspace = true }
+thiserror = "1.0.56"
+time = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+uuid = { workspace = true }
+webp = "0.2.6"

services/avatars/src/hash.rs

@@ -0,0 +1,21 @@
+use std::fmt::Display;
+
+use sha2::{Digest, Sha256};
+
+#[derive(Debug)]
+pub struct Hash([u8; 32]);
+
+impl Hash {
+    pub fn sha256(data: &[u8]) -> Hash {
+        let mut hasher = Sha256::new();
+        hasher.update(data);
+        Hash(hasher.finalize().into())
+    }
+}
+
+impl Display for Hash {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let encoding = data_encoding::BASE32_NOPAD;
+        write!(f, "{}", encoding.encode(&self.0[..16]).to_lowercase())
+    }
+}