refactor: remove protobuf

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "addr2line"
@@ -94,7 +94,6 @@ dependencies = [
  "lazy_static",
  "libpk",
  "metrics",
- "prost",
  "reverse-proxy-service",
  "serde",
  "serde_json",
@@ -837,12 +836,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "fixedbitset"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
-
 [[package]]
 name = "flate2"
 version = "1.0.30"
@@ -1042,7 +1035,6 @@ dependencies = [
  "lazy_static",
  "libpk",
  "metrics",
- "prost",
  "serde_json",
  "serde_variant",
  "signal-hook",
@@ -1547,15 +1539,6 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
 
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.5"
@@ -1644,9 +1627,6 @@ dependencies = [
  "lazy_static",
  "metrics",
  "metrics-exporter-prometheus",
- "prost",
- "prost-build",
- "prost-types",
  "sentry",
  "serde",
  "serde_json",
@@ -1874,12 +1854,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "multimap"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03"
-
 [[package]]
 name = "nibble_vec"
 version = "0.1.0"
@@ -2139,16 +2113,6 @@ dependencies = [
  "sha2",
 ]
 
-[[package]]
-name = "petgraph"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
-dependencies = [
- "fixedbitset",
- "indexmap",
-]
-
 [[package]]
 name = "pin-project"
 version = "1.0.12"
@@ -2239,16 +2203,6 @@ version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
-[[package]]
-name = "prettyplease"
-version = "0.2.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
-dependencies = [
- "proc-macro2",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.84"
@@ -2258,59 +2212,6 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "prost"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29"
-dependencies = [
- "bytes",
- "prost-derive",
-]
-
-[[package]]
-name = "prost-build"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
-dependencies = [
- "bytes",
- "heck 0.5.0",
- "itertools",
- "log",
- "multimap",
- "once_cell",
- "petgraph",
- "prettyplease",
- "prost",
- "prost-types",
- "regex",
- "syn 2.0.66",
- "tempfile",
-]
-
-[[package]]
-name = "prost-derive"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
-dependencies = [
- "anyhow",
- "itertools",
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "prost-types"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0"
-dependencies = [
- "prost",
-]
-
 [[package]]
 name = "quanta"
 version = "0.12.3"

Cargo.toml

@@ -40,7 +40,3 @@ twilight-http = { git = "https://github.com/pluralkit/twilight", default-feature
 #twilight-util = { path = "../twilight/twilight-util", features = ["permission-calculator"] }
 #twilight-model = { path = "../twilight/twilight-model" }
 #twilight-http = { path = "../twilight/twilight-http", default-features = false, features = ["rustls-native-roots"] }
-
-prost = "0.12"
-prost-types = "0.12"
-prost-build = "0.12"

Dockerfile

@@ -10,7 +10,6 @@ COPY PluralKit.Bot/PluralKit.Bot.csproj /app/PluralKit.Bot/
 COPY PluralKit.Core/PluralKit.Core.csproj /app/PluralKit.Core/
 COPY PluralKit.Tests/PluralKit.Tests.csproj /app/PluralKit.Tests/
 COPY .git/ /app/.git
-COPY proto/ /app/proto
 RUN dotnet restore PluralKit.sln
 
 # Copy the rest of the code and build

Myriad/Myriad.csproj

@@ -21,17 +21,10 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Google.Protobuf" Version="3.29.1" />
-        <PackageReference Include="Grpc.Net.ClientFactory" Version="2.67.0" />
-        <PackageReference Include="Grpc.Tools" Version="2.68.1" PrivateAssets="all" />
         <PackageReference Include="Polly" Version="8.5.0" />
         <PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
         <PackageReference Include="Serilog" Version="4.2.0" />
         <PackageReference Include="StackExchange.Redis" Version="2.8.22" />
         <PackageReference Include="System.Linq.Async" Version="6.0.1" />
     </ItemGroup>
-
-    <ItemGroup>
-        <Protobuf Include="../proto/discord_cache.proto" GrpcServices="Client" Link="Protos/discord_cache.proto" />
-    </ItemGroup>
 </Project>

Myriad/packages.lock.json

@@ -2,28 +2,6 @@
   "version": 1,
   "dependencies": {
     "net8.0": {
-      "Google.Protobuf": {
-        "type": "Direct",
-        "requested": "[3.29.1, )",
-        "resolved": "3.29.1",
-        "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
-      },
-      "Grpc.Net.ClientFactory": {
-        "type": "Direct",
-        "requested": "[2.67.0, )",
-        "resolved": "2.67.0",
-        "contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==",
-        "dependencies": {
-          "Grpc.Net.Client": "2.67.0",
-          "Microsoft.Extensions.Http": "6.0.0"
-        }
-      },
-      "Grpc.Tools": {
-        "type": "Direct",
-        "requested": "[2.68.1, )",
-        "resolved": "2.68.1",
-        "contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q=="
-      },
       "Polly": {
         "type": "Direct",
         "requested": "[8.5.0, )",
@@ -64,92 +42,16 @@
           "Microsoft.Bcl.AsyncInterfaces": "6.0.0"
         }
       },
-      "Grpc.Core.Api": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q=="
-      },
-      "Grpc.Net.Client": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==",
-        "dependencies": {
-          "Grpc.Net.Common": "2.67.0",
-          "Microsoft.Extensions.Logging.Abstractions": "6.0.0"
-        }
-      },
-      "Grpc.Net.Common": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==",
-        "dependencies": {
-          "Grpc.Core.Api": "2.67.0"
-        }
-      },
       "Microsoft.Bcl.AsyncInterfaces": {
         "type": "Transitive",
         "resolved": "6.0.0",
         "contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg=="
       },
-      "Microsoft.Extensions.DependencyInjection": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "k6PWQMuoBDGGHOQTtyois2u4AwyVcIwL2LaSLlTZQm2CYcJ1pxbt6jfAnpWmzENA/wfrYRI/X9DTLoUkE4AsLw==",
-        "dependencies": {
-          "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
-          "System.Runtime.CompilerServices.Unsafe": "6.0.0"
-        }
-      },
-      "Microsoft.Extensions.DependencyInjection.Abstractions": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg=="
-      },
-      "Microsoft.Extensions.Http": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==",
-        "dependencies": {
-          "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Logging": "6.0.0",
-          "Microsoft.Extensions.Logging.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Options": "6.0.0"
-        }
-      },
-      "Microsoft.Extensions.Logging": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "eIbyj40QDg1NDz0HBW0S5f3wrLVnKWnDJ/JtZ+yJDFnDj90VoPuoPmFkeaXrtu+0cKm5GRAwoDf+dBWXK0TUdg==",
-        "dependencies": {
-          "Microsoft.Extensions.DependencyInjection": "6.0.0",
-          "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Logging.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Options": "6.0.0",
-          "System.Diagnostics.DiagnosticSource": "6.0.0"
-        }
-      },
       "Microsoft.Extensions.Logging.Abstractions": {
         "type": "Transitive",
         "resolved": "6.0.0",
         "contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA=="
       },
-      "Microsoft.Extensions.Options": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==",
-        "dependencies": {
-          "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Primitives": "6.0.0"
-        }
-      },
-      "Microsoft.Extensions.Primitives": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==",
-        "dependencies": {
-          "System.Runtime.CompilerServices.Unsafe": "6.0.0"
-        }
-      },
       "Pipelines.Sockets.Unofficial": {
         "type": "Transitive",
         "resolved": "2.2.8",
@@ -163,23 +65,10 @@
         "resolved": "8.5.0",
         "contentHash": "VYYMZNitZ85UEhwOKkTQI63WEMvzUqwQc74I2mm8h/DBVAMcBBxqYPni4DmuRtbCwngmuONuK2yBJfWNRKzI+A=="
       },
-      "System.Diagnostics.DiagnosticSource": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "frQDfv0rl209cKm1lnwTgFPzNigy2EKk1BS3uAvHvlBVKe5cymGyHO+Sj+NLv5VF/AhHsqPIUUwya5oV4CHMUw==",
-        "dependencies": {
-          "System.Runtime.CompilerServices.Unsafe": "6.0.0"
-        }
-      },
       "System.IO.Pipelines": {
         "type": "Transitive",
         "resolved": "5.0.1",
         "contentHash": "qEePWsaq9LoEEIqhbGe6D5J8c9IqQOUuTzzV6wn1POlfdLkJliZY3OlB0j0f17uMWlqZYjH7txj+2YbyrIA8Yg=="
-      },
-      "System.Runtime.CompilerServices.Unsafe": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg=="
       }
     }
   }

PluralKit.API/Controllers/PrivateController.cs

@@ -22,28 +22,6 @@ public class PrivateController: PKControllerBase
 {
     public PrivateController(IServiceProvider svc) : base(svc) { }
 
-    [HttpGet("meta")]
-    public async Task<ActionResult<JObject>> Meta()
-    {
-        var db = _redis.Connection.GetDatabase();
-        var redisInfo = await db.HashGetAllAsync("pluralkit:shardstatus");
-        var shards = redisInfo.Select(x => Proto.Unmarshal<ShardState>(x.Value)).OrderBy(x => x.ShardId);
-
-        var redisClusterInfo = await db.HashGetAllAsync("pluralkit:cluster_stats");
-        var clusterInfo = redisClusterInfo.Select(x => JsonConvert.DeserializeObject<ClusterMetricInfo>(x.Value));
-
-        var guildCount = clusterInfo.Sum(x => x.GuildCount);
-        var channelCount = clusterInfo.Sum(x => x.ChannelCount);
-
-        var stats = await _repo.GetStats();
-
-        var o = new JObject();
-        o.Add("shards", shards.ToJson());
-        o.Add("stats", stats.ToJson(guildCount, channelCount));
-
-        return Ok(o);
-    }
-
     [HttpPost("bulk_privacy/member")]
     public async Task<IActionResult> BulkMemberPrivacy([FromBody] JObject inner)
     {
@@ -158,31 +136,4 @@ public static class PrivateJsonExt
         o.Add("error", error);
         return o;
     }
-
-    public static JArray ToJson(this IEnumerable<ShardState> shards)
-    {
-        var o = new JArray();
-
-        foreach (var shard in shards)
-        {
-            var s = new JObject();
-            s.Add("id", shard.ShardId);
-
-            if (!shard.Up)
-                s.Add("status", "down");
-            else
-                s.Add("status", "up");
-
-            s.Add("ping", shard.Latency);
-            s.Add("disconnection_count", shard.DisconnectionCount);
-            s.Add("last_heartbeat", shard.LastHeartbeat.ToString());
-            s.Add("last_connection", shard.LastConnection.ToString());
-            if (shard.HasClusterId)
-                s.Add("cluster_id", shard.ClusterId);
-
-            o.Add(s);
-        }
-
-        return o;
-    }
 }

PluralKit.API/PluralKit.API.csproj

@@ -28,15 +28,9 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Google.Protobuf" Version="3.29.1" />
-        <PackageReference Include="Grpc.Tools" Version="2.68.1" PrivateAssets="all" />
         <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.11" />
         <PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning" Version="5.1.0" />
         <PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning.ApiExplorer" Version="5.1.0" />
         <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
     </ItemGroup>
-
-    <ItemGroup>
-        <Protobuf Include="../proto/state.proto" GrpcServices="Client" Link="Protos/state.proto" />
-    </ItemGroup>
 </Project>

PluralKit.API/packages.lock.json

@@ -2,18 +2,6 @@
   "version": 1,
   "dependencies": {
     "net8.0": {
-      "Google.Protobuf": {
-        "type": "Direct",
-        "requested": "[3.29.1, )",
-        "resolved": "3.29.1",
-        "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
-      },
-      "Grpc.Tools": {
-        "type": "Direct",
-        "requested": "[2.68.1, )",
-        "resolved": "2.68.1",
-        "contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q=="
-      },
       "Microsoft.AspNetCore.Mvc.NewtonsoftJson": {
         "type": "Direct",
         "requested": "[8.0.11, )",
@@ -830,7 +818,6 @@
           "Autofac.Extensions.DependencyInjection": "[10.0.0, )",
           "Dapper": "[2.1.35, )",
           "Dapper.Contrib": "[2.0.78, )",
-          "Google.Protobuf": "[3.29.1, )",
           "Microsoft.Extensions.Caching.Memory": "[9.0.0, )",
           "Microsoft.Extensions.Configuration": "[9.0.0, )",
           "Microsoft.Extensions.Configuration.Binder": "[9.0.0, )",

PluralKit.Bot/Commands/Misc.cs

@@ -94,7 +94,7 @@ public class Misc
         var memoryUsage = process.WorkingSet64;
 
         var now = SystemClock.Instance.GetCurrentInstant().ToUnixTimeSeconds();
-        var shardUptime = Duration.FromSeconds(now - shardInfo?.LastConnection ?? 0);
+        var shardUptime = Duration.FromSeconds(now - shardInfo.LastConnection);
 
         var shardTotal = _botConfig.Cluster?.TotalShards ?? shards.Count();
         int shardClusterTotal = ctx.Cluster.Shards.Count;
@@ -106,11 +106,11 @@ public class Misc
                     + (isCluster ? $" {shardClusterTotal} in this cluster," : "") + $" {shardUpTotal} are up)"
                 , true))
             .Field(new Embed.Field("Shard uptime",
-                $"{shardUptime.FormatDuration()} ({shardInfo?.DisconnectionCount} disconnections)", true))
+                $"{shardUptime.FormatDuration()} ({shardInfo.DisconnectionCount} disconnections)", true))
             .Field(new Embed.Field("CPU usage", $"{_cpu.LastCpuMeasure:P1}", true))
             .Field(new Embed.Field("Memory usage", $"{memoryUsage / 1024 / 1024} MiB", true))
             .Field(new Embed.Field("Latency",
-                $"API: {apiLatency.TotalMilliseconds:F0} ms, shard: {shardInfo?.Latency} ms",
+                $"API: {apiLatency.TotalMilliseconds:F0} ms, shard: {shardInfo.Latency} ms",
                 true));
 
         embed.Field(new("Total numbers", $" {counts.SystemCount:N0} systems,"

PluralKit.Bot/PluralKit.Bot.csproj

@@ -22,15 +22,8 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Google.Protobuf" Version="3.29.1" />
-        <PackageReference Include="Grpc.Net.ClientFactory" Version="2.67.0" />
-        <PackageReference Include="Grpc.Tools" Version="2.68.1" PrivateAssets="all" />
         <PackageReference Include="Humanizer.Core" Version="2.14.1" />
         <PackageReference Include="Sentry" Version="4.13.0" />
         <PackageReference Include="SixLabors.ImageSharp" Version="3.1.6" />
     </ItemGroup>
-
-    <ItemGroup>
-        <Protobuf Include="../proto/state.proto" GrpcServices="Client" Link="Protos/state.proto" />
-    </ItemGroup>
 </Project>

PluralKit.Bot/Services/ShardInfoService.cs

@@ -1,11 +1,11 @@
 using System.Net.WebSockets;
 
-using Google.Protobuf;
-
 using Myriad.Gateway;
 
 using NodaTime;
 
+using Newtonsoft.Json;
+
 using StackExchange.Redis;
 
 using PluralKit.Core;
@@ -14,6 +14,17 @@ using Serilog;
 
 namespace PluralKit.Bot;
 
+public struct ShardState
+{
+    [JsonProperty("shard_id")] public int ShardId;
+    [JsonProperty("up")] public bool Up;
+    [JsonProperty("disconnection_count")] public int DisconnectionCount;
+    [JsonProperty("latency")] public int Latency;
+    [JsonProperty("last_heartbeat")] public int LastHeartbeat;
+    [JsonProperty("last_connection")] public int LastConnection;
+    [JsonProperty("cluster_id")] public int? ClusterId;
+}
+
 public class ShardInfoService
 {
     private readonly int? _clusterId;
@@ -43,7 +54,7 @@ public class ShardInfoService
             return new ShardState[] { };
         var db = _redis.Connection.GetDatabase();
         var redisInfo = await db.HashGetAllAsync("pluralkit:shardstatus");
-        return redisInfo.Select(x => Proto.Unmarshal<ShardState>(x.Value));
+        return redisInfo.Select(x => JsonConvert.DeserializeObject<ShardState>(x.Value));
     }
 
     private void InitializeShard(Shard shard)
@@ -53,13 +64,8 @@ public class ShardInfoService
         async Task Inner()
         {
             var db = _redis.Connection.GetDatabase();
-            var redisInfo = await db.HashGetAsync("pluralkit::shardstatus", shard.ShardId);
 
-            // Skip adding listeners if we've seen this shard & already added listeners to it
+            // latency = 1 because otherwise shard 0 would serialize to an empty array, thanks protobuf
-            if (redisInfo.HasValue)
-                return;
-
-            // latency = 0 because otherwise shard 0 would serialize to an empty array, thanks protobuf
             var state = new ShardState() { ShardId = shard.ShardId, Up = false, Latency = 1 };
             if (_clusterId != null)
                 state.ClusterId = _clusterId.Value;
@@ -75,13 +81,13 @@ public class ShardInfoService
         }
     }
 
-    private async Task<ShardState?> TryGetShard(Shard shard)
+    private async Task<ShardState> TryGetShard(Shard shard)
     {
         var db = _redis.Connection.GetDatabase();
         var redisInfo = await db.HashGetAsync("pluralkit:shardstatus", shard.ShardId);
         if (redisInfo.HasValue)
-            return Proto.Unmarshal<ShardState>(redisInfo);
+            return JsonConvert.DeserializeObject<ShardState>(redisInfo);
-        return null;
+        throw new Exception("expected shard, but didn't get one");
     }
 
     private void ReadyOrResumed(Shard shard)
@@ -145,5 +151,5 @@ public static class RedisExt
 {
     // convenience method
     public static HashEntry[] HashWrapper(this ShardState state)
-        => new[] { new HashEntry(state.ShardId, state.ToByteArray()) };
+        => new[] { new HashEntry(state.ShardId, JsonConvert.SerializeObject(state)) };
 }

PluralKit.Bot/packages.lock.json

@@ -2,28 +2,6 @@
   "version": 1,
   "dependencies": {
     "net8.0": {
-      "Google.Protobuf": {
-        "type": "Direct",
-        "requested": "[3.29.1, )",
-        "resolved": "3.29.1",
-        "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
-      },
-      "Grpc.Net.ClientFactory": {
-        "type": "Direct",
-        "requested": "[2.67.0, )",
-        "resolved": "2.67.0",
-        "contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==",
-        "dependencies": {
-          "Grpc.Net.Client": "2.67.0",
-          "Microsoft.Extensions.Http": "6.0.0"
-        }
-      },
-      "Grpc.Tools": {
-        "type": "Direct",
-        "requested": "[2.68.1, )",
-        "resolved": "2.68.1",
-        "contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q=="
-      },
       "Humanizer.Core": {
         "type": "Direct",
         "requested": "[2.14.1, )",
@@ -158,28 +136,6 @@
           "System.Diagnostics.DiagnosticSource": "5.0.0"
         }
       },
-      "Grpc.Core.Api": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q=="
-      },
-      "Grpc.Net.Client": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==",
-        "dependencies": {
-          "Grpc.Net.Common": "2.67.0",
-          "Microsoft.Extensions.Logging.Abstractions": "6.0.0"
-        }
-      },
-      "Grpc.Net.Common": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==",
-        "dependencies": {
-          "Grpc.Core.Api": "2.67.0"
-        }
-      },
       "IPNetwork2": {
         "type": "Transitive",
         "resolved": "3.0.667",
@@ -318,17 +274,6 @@
         "resolved": "9.0.0",
         "contentHash": "jGFKZiXs2HNseK3NK/rfwHNNovER71jSj4BD1a/649ml9+h6oEtYd0GSALZDNW8jZ2Rh+oAeadOa6sagYW1F2A=="
       },
-      "Microsoft.Extensions.Http": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==",
-        "dependencies": {
-          "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Logging": "6.0.0",
-          "Microsoft.Extensions.Logging.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Options": "6.0.0"
-        }
-      },
       "Microsoft.Extensions.Logging": {
         "type": "Transitive",
         "resolved": "9.0.0",
@@ -778,8 +723,6 @@
       "myriad": {
         "type": "Project",
         "dependencies": {
-          "Google.Protobuf": "[3.29.1, )",
-          "Grpc.Net.ClientFactory": "[2.67.0, )",
           "Polly": "[8.5.0, )",
           "Polly.Contrib.WaitAndRetry": "[1.1.1, )",
           "Serilog": "[4.2.0, )",
@@ -797,7 +740,6 @@
           "Autofac.Extensions.DependencyInjection": "[10.0.0, )",
           "Dapper": "[2.1.35, )",
           "Dapper.Contrib": "[2.0.78, )",
-          "Google.Protobuf": "[3.29.1, )",
           "Microsoft.Extensions.Caching.Memory": "[9.0.0, )",
           "Microsoft.Extensions.Configuration": "[9.0.0, )",
           "Microsoft.Extensions.Configuration.Binder": "[9.0.0, )",

PluralKit.Core/PluralKit.Core.csproj

@@ -23,7 +23,6 @@
         <PackageReference Include="Autofac.Extensions.DependencyInjection" Version="10.0.0" />
         <PackageReference Include="Dapper" Version="2.1.35" />
         <PackageReference Include="Dapper.Contrib" Version="2.0.78" />
-        <PackageReference Include="Google.Protobuf" Version="3.29.1" />
         <PackageReference Include="ipnetwork2" Version="3.0.667" />
         <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
         <PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.0" />

PluralKit.Core/Utils/ProtobufUtils.cs

@@ -1,22 +0,0 @@
-using Google.Protobuf;
-
-namespace PluralKit.Core;
-
-public static class Proto
-{
-    private static readonly Dictionary<string, MessageParser> _parser = new();
-
-    public static byte[] Marshal(this IMessage message) => message.ToByteArray();
-
-    public static T Unmarshal<T>(this byte[] message) where T : IMessage<T>, new()
-    {
-        var type = typeof(T).ToString();
-        if (_parser.ContainsKey(type))
-        {
-            return (T)_parser[type].ParseFrom(message);
-        }
-
-        _parser.Add(type, new MessageParser<T>(() => new T()));
-        return Unmarshal<T>(message);
-    }
-}

PluralKit.Core/packages.lock.json

@@ -66,12 +66,6 @@
           "Dapper": "2.0.78"
         }
       },
-      "Google.Protobuf": {
-        "type": "Direct",
-        "requested": "[3.29.1, )",
-        "resolved": "3.29.1",
-        "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
-      },
       "IPNetwork2": {
         "type": "Direct",
         "requested": "[3.0.667, )",

PluralKit.Tests/packages.lock.json

@@ -151,42 +151,6 @@
           "System.Diagnostics.DiagnosticSource": "5.0.0"
         }
       },
-      "Google.Protobuf": {
-        "type": "Transitive",
-        "resolved": "3.29.1",
-        "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
-      },
-      "Grpc.Core.Api": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q=="
-      },
-      "Grpc.Net.Client": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==",
-        "dependencies": {
-          "Grpc.Net.Common": "2.67.0",
-          "Microsoft.Extensions.Logging.Abstractions": "6.0.0"
-        }
-      },
-      "Grpc.Net.ClientFactory": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==",
-        "dependencies": {
-          "Grpc.Net.Client": "2.67.0",
-          "Microsoft.Extensions.Http": "6.0.0"
-        }
-      },
-      "Grpc.Net.Common": {
-        "type": "Transitive",
-        "resolved": "2.67.0",
-        "contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==",
-        "dependencies": {
-          "Grpc.Core.Api": "2.67.0"
-        }
-      },
       "Humanizer.Core": {
         "type": "Transitive",
         "resolved": "2.14.1",
@@ -398,17 +362,6 @@
           "Microsoft.Extensions.Logging.Abstractions": "9.0.0"
         }
       },
-      "Microsoft.Extensions.Http": {
-        "type": "Transitive",
-        "resolved": "6.0.0",
-        "contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==",
-        "dependencies": {
-          "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Logging": "6.0.0",
-          "Microsoft.Extensions.Logging.Abstractions": "6.0.0",
-          "Microsoft.Extensions.Options": "6.0.0"
-        }
-      },
       "Microsoft.Extensions.Logging": {
         "type": "Transitive",
         "resolved": "9.0.0",
@@ -982,8 +935,6 @@
       "myriad": {
         "type": "Project",
         "dependencies": {
-          "Google.Protobuf": "[3.29.1, )",
-          "Grpc.Net.ClientFactory": "[2.67.0, )",
           "Polly": "[8.5.0, )",
           "Polly.Contrib.WaitAndRetry": "[1.1.1, )",
           "Serilog": "[4.2.0, )",
@@ -994,7 +945,6 @@
       "pluralkit.api": {
         "type": "Project",
         "dependencies": {
-          "Google.Protobuf": "[3.29.1, )",
           "Microsoft.AspNetCore.Mvc.NewtonsoftJson": "[8.0.11, )",
           "Microsoft.AspNetCore.Mvc.Versioning": "[5.1.0, )",
           "Microsoft.AspNetCore.Mvc.Versioning.ApiExplorer": "[5.1.0, )",
@@ -1005,8 +955,6 @@
       "pluralkit.bot": {
         "type": "Project",
         "dependencies": {
-          "Google.Protobuf": "[3.29.1, )",
-          "Grpc.Net.ClientFactory": "[2.67.0, )",
           "Humanizer.Core": "[2.14.1, )",
           "Myriad": "[1.0.0, )",
           "PluralKit.Core": "[1.0.0, )",
@@ -1024,7 +972,6 @@
           "Autofac.Extensions.DependencyInjection": "[10.0.0, )",
           "Dapper": "[2.1.35, )",
           "Dapper.Contrib": "[2.0.78, )",
-          "Google.Protobuf": "[3.29.1, )",
           "Microsoft.Extensions.Caching.Memory": "[9.0.0, )",
           "Microsoft.Extensions.Configuration": "[9.0.0, )",
           "Microsoft.Extensions.Configuration.Binder": "[9.0.0, )",

ci/Dockerfile.rust

@@ -2,7 +2,7 @@ FROM alpine:latest AS builder
 
 WORKDIR /build
 
-RUN apk add rustup build-base protoc
+RUN apk add rustup build-base
 # todo: arm64 target
 RUN rustup-init --default-host x86_64-unknown-linux-musl --default-toolchain nightly-2024-08-20 --profile default -y
 
@@ -22,7 +22,6 @@ RUN cargo chef cook --release --recipe-path recipe.json --target x86_64-unknown-
 
 COPY Cargo.toml /build/
 COPY Cargo.lock /build/
-COPY proto/ /build/proto
 
 # this needs to match workspaces in Cargo.toml
 COPY lib/libpk /build/lib/libpk

dashboard/src/routes/Status/status.svelte

@@ -25,7 +25,7 @@
     let valid = false;
 
     const get = async () => {
-        const pkdata = await api().private.meta.get();
+        const pkdata = await api().private.discord.shard_state.get();
         let data = pkdata.shards.sort((x, y) => (x.id > y.id) ? 1 : -1);
         let latencies = 0;
         data = data.map(shard => {

flake.nix

@@ -52,7 +52,6 @@
                   git
                   dotnet-sdk_8
                   gcc
-                  protobuf
                   omnisharp-roslyn
                   bashInteractive
                 ];
@@ -80,14 +79,14 @@
             path = ./.;
             export = false;
           };
-          nci.crates."gateway" = {
+          # nci.crates."gateway" = {
-            depsDrvConfig.mkDerivation = {
+          #   depsDrvConfig.mkDerivation = {
-              nativeBuildInputs = [ pkgs.protobuf ];
+          #     nativeBuildInputs = [ pkgs.protobuf ];
-            };
+          #   };
-            drvConfig.mkDerivation = {
+          #   drvConfig.mkDerivation = {
-              nativeBuildInputs = [ pkgs.protobuf ];
+          #     nativeBuildInputs = [ pkgs.protobuf ];
-            };
+          #   };
-          };
+          # };
 
           # TODO: expose other rust packages after it's verified they build and work properly
           packages = lib.genAttrs ["gateway"] (name: rustOutputs.${name}.packages.release);

lib/libpk/Cargo.toml

@@ -8,8 +8,6 @@ anyhow = { workspace = true }
 fred = { workspace = true }
 lazy_static = { workspace = true }
 metrics = { workspace = true }
-prost = { workspace = true }
-prost-types = { workspace = true }
 sentry = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -24,6 +22,3 @@ uuid = { workspace = true }
 config = "0.14.0"
 json-subscriber = { version = "0.2.2", features = ["env-filter"] }
 metrics-exporter-prometheus = { version = "0.15.3", default-features = false, features = ["tokio", "http-listener", "tracing"] }
-
-[build-dependencies]
-prost-build = { workspace = true }

lib/libpk/build.rs

@@ -1,9 +0,0 @@
-use std::io::Result;
-
-fn main() -> Result<()> {
-    prost_build::Config::new()
-        .type_attribute(".ShardState", "#[derive(serde::Serialize)]")
-        .field_attribute(".ShardState.shard_id", "#[serde(rename = \"id\")]")
-        .compile_protos(&["../../proto/state.proto"], &["../../proto/"])?;
-    Ok(())
-}

lib/libpk/src/lib.rs

@@ -6,7 +6,7 @@ use sentry::IntoDsn;
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
 pub mod db;
-pub mod proto;
+pub mod state;
 pub mod util;
 
 pub mod _config;

lib/libpk/src/proto.rs

@@ -1 +0,0 @@
-include!(concat!(env!("OUT_DIR"), "/_.rs"));

proto/discord_cache.proto

@@ -1,47 +0,0 @@
-syntax = "proto3";
-
-package myriad.cache;
-
-message Overwrite {
-    uint64 id = 1;
-    int32 type = 2;
-    uint64 allow = 3;
-    uint64 deny = 4;
-}
-
-message CachedChannel {
-    uint64 id = 1;
-    int32 type = 2;
-    int32 position = 3;
-    string name = 4;
-    repeated Overwrite permission_overwrites = 5;
-    optional uint64 guild_id = 6;
-    optional uint64 parent_id = 7;
-}
-
-message CachedGuildMember {
-    repeated uint64 roles = 1;
-}
-
-message CachedGuild {
-    uint64 id = 1;
-    string name = 2;
-    uint64 owner_id = 3;
-    int32 premium_tier = 4;
-}
-
-message CachedRole {
-    uint64 id = 1;
-    string name = 2;
-    int32 position = 3;
-    uint64 permissions = 4;
-    bool mentionable = 5;
-}
-
-message CachedUser {
-    uint64 id = 1;
-    string username = 2;
-    string discriminator = 3;
-    optional string avatar = 4;
-    bool bot = 5;
-}

proto/state.proto

@@ -1,21 +0,0 @@
-syntax = "proto3";
-
-message ShardState {
-    int32 shard_id = 1;
-    bool up = 2;
-    int32 disconnection_count = 3;
-
-    // milliseconds
-    int32 latency = 4;
-
-    // unix timestamp
-    int32 last_heartbeat = 5;
-    int32 last_connection = 6;
-
-    optional int32 cluster_id = 7;
-}
-
-message GatewayEvent {
-    int32 shard_id = 1;
-    bytes payload = 2;
-}

services/api/Cargo.toml

@@ -10,7 +10,6 @@ fred = { workspace = true }
 lazy_static = { workspace = true }
 libpk = { path = "../../lib/libpk" }
 metrics = { workspace = true }
-prost = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 sqlx = { workspace = true }

services/api/src/endpoints/private.rs

@@ -1,8 +1,7 @@
 use crate::ApiContext;
 use axum::{extract::State, response::Json};
 use fred::interfaces::*;
-use libpk::proto::ShardState;
+use libpk::state::ShardState;
-use prost::Message;
 use serde::Deserialize;
 use serde_json::{json, Value};
 use std::collections::HashMap;
@@ -14,16 +13,22 @@ struct ClusterStats {
     pub channel_count: i32,
 }
 
-pub async fn meta(State(ctx): State<ApiContext>) -> Json<Value> {
+pub async fn discord_state(State(ctx): State<ApiContext>) -> Json<Value> {
     let shard_status = ctx
         .redis
-        .hgetall::<HashMap<String, Vec<u8>>, &str>("pluralkit:shardstatus")
+        .hgetall::<HashMap<String, String>, &str>("pluralkit:shardstatus")
         .await
         .unwrap()
         .values()
-        .map(|v| ShardState::decode(v.as_slice()).unwrap())
+        .map(|v| serde_json::from_str(v).expect("could not deserialize shard"))
         .collect::<Vec<ShardState>>();
 
+    Json(json!({
+        "shards": shard_status,
+    }))
+}
+
+pub async fn meta(State(ctx): State<ApiContext>) -> Json<Value> {
     let cluster_stats = ctx
         .redis
         .hgetall::<HashMap<String, String>, &str>("pluralkit:cluster_stats")
@@ -39,15 +44,12 @@ pub async fn meta(State(ctx): State<ApiContext>) -> Json<Value> {
     let channel_count: i32 = cluster_stats.iter().map(|v| v.channel_count).sum();
 
     Json(json!({
-        "shards": shard_status,
+        "system_count": db_stats.system_count,
-        "stats": {
+        "member_count": db_stats.member_count,
-            "system_count": db_stats.system_count,
+        "group_count": db_stats.group_count,
-            "member_count": db_stats.member_count,
+        "switch_count": db_stats.switch_count,
-            "group_count": db_stats.group_count,
+        "message_count": db_stats.message_count,
-            "switch_count": db_stats.switch_count,
+        "guild_count": guild_count,
-            "message_count": db_stats.message_count,
+        "channel_count": channel_count,
-            "guild_count": guild_count,
-            "channel_count": channel_count,
-        }
     }))
 }

services/api/src/main.rs

@@ -104,10 +104,11 @@ fn router(ctx: ApiContext) -> Router {
 
         .route("/v2/messages/:message_id", get(rproxy))
 
-        .route("/private/meta", get(endpoints::private::meta))
         .route("/private/bulk_privacy/member", post(rproxy))
         .route("/private/bulk_privacy/group", post(rproxy))
         .route("/private/discord/callback", post(rproxy))
+        .route("/private/discord/shard_state", get(endpoints::private::discord_state))
+        .route("/private/stats", get(endpoints::private::meta))
 
         .route("/v2/systems/:system_id/oembed.json", get(rproxy))
         .route("/v2/members/:member_id/oembed.json", get(rproxy))

services/api/src/middleware/logger.rs

@@ -42,7 +42,6 @@ pub async fn logger(request: Request, next: Next) -> Response {
 
     let authenticated = {
         let headers = response.headers_mut();
-        println!("{:#?}", headers.keys());
         if headers.contains_key(DID_AUTHENTICATE_HEADER) {
             headers.remove(DID_AUTHENTICATE_HEADER);
             true

services/gateway/Cargo.toml

@@ -13,7 +13,6 @@ futures = { workspace = true }
 lazy_static = { workspace = true }
 libpk = { path = "../../lib/libpk" }
 metrics = { workspace = true }
-prost = { workspace = true }
 serde_json = { workspace = true }
 signal-hook = { workspace = true }
 tokio = { workspace = true }

services/gateway/src/discord/shard_state.rs

@@ -1,11 +1,10 @@
 use bytes::Bytes;
 use fred::{clients::RedisPool, interfaces::HashesInterface};
 use metrics::{counter, gauge};
-use prost::Message;
 use tracing::info;
 use twilight_gateway::Event;
 
-use libpk::{proto::*, util::redis::*};
+use libpk::{state::*, util::redis::*};
 
 #[derive(Clone)]
 pub struct ShardStateManager {
@@ -28,26 +27,24 @@ impl ShardStateManager {
     }
 
     async fn get_shard(&self, shard_id: u32) -> anyhow::Result<ShardState> {
-        let data: Option<Vec<u8>> = self
+        let data: Option<String> = self
             .redis
             .hget("pluralkit:shardstatus", shard_id)
             .await
             .to_option_or_error()?;
         match data {
-            Some(buf) => {
+            Some(buf) => Ok(serde_json::from_str(&buf).expect("could not decode shard data!")),
-                Ok(ShardState::decode(buf.as_slice()).expect("could not decode shard data!"))
-            }
             None => Ok(ShardState::default()),
         }
     }
 
     async fn save_shard(&self, shard_id: u32, info: ShardState) -> anyhow::Result<()> {
         self.redis
-            .hset::<(), &str, (String, Bytes)>(
+            .hset::<(), &str, (String, String)>(
                 "pluralkit:shardstatus",
                 (
                     shard_id.to_string(),
-                    Bytes::copy_from_slice(&info.encode_to_vec()),
+                    serde_json::to_string(&info).expect("could not serialize shard"),
                 ),
             )
             .await?;