diff --git a/Cargo.lock b/Cargo.lock index cb904814..7532dde2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -94,7 +94,6 @@ dependencies = [ "lazy_static", "libpk", "metrics", - "prost", "reverse-proxy-service", "serde", "serde_json", @@ -837,12 +836,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "flate2" version = "1.0.30" @@ -1042,7 +1035,6 @@ dependencies = [ "lazy_static", "libpk", "metrics", - "prost", "serde_json", "serde_variant", "signal-hook", @@ -1547,15 +1539,6 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.5" @@ -1644,9 +1627,6 @@ dependencies = [ "lazy_static", "metrics", "metrics-exporter-prometheus", - "prost", - "prost-build", - "prost-types", "sentry", "serde", "serde_json", @@ -1874,12 +1854,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "multimap" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" - [[package]] name = "nibble_vec" version = "0.1.0" @@ -2139,16 +2113,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset", - "indexmap", -] - [[package]] name = "pin-project" version = "1.0.12" @@ -2239,16 +2203,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "prettyplease" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" -dependencies = [ - "proc-macro2", - "syn 2.0.66", -] - [[package]] name = "proc-macro2" version = "1.0.84" @@ -2258,59 +2212,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" -dependencies = [ - "bytes", - "heck 0.5.0", - "itertools", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.66", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "prost-types" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" -dependencies = [ - "prost", -] - [[package]] name = "quanta" version = "0.12.3" diff --git a/Cargo.toml b/Cargo.toml index 2c7aa9de..366e2c60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,3 @@ twilight-http = { git = "https://github.com/pluralkit/twilight", default-feature #twilight-util = { path = "../twilight/twilight-util", features = ["permission-calculator"] } #twilight-model = { path = "../twilight/twilight-model" } #twilight-http = { path = "../twilight/twilight-http", default-features = false, features = ["rustls-native-roots"] } - -prost = "0.12" -prost-types = "0.12" -prost-build = "0.12" diff --git a/Dockerfile b/Dockerfile index 9cec299d..dedc085d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ COPY PluralKit.Bot/PluralKit.Bot.csproj /app/PluralKit.Bot/ COPY PluralKit.Core/PluralKit.Core.csproj /app/PluralKit.Core/ COPY PluralKit.Tests/PluralKit.Tests.csproj /app/PluralKit.Tests/ COPY .git/ /app/.git -COPY proto/ /app/proto RUN dotnet restore PluralKit.sln # Copy the rest of the code and build diff --git a/Myriad/Myriad.csproj b/Myriad/Myriad.csproj index c899d7b9..3b92ea30 100644 --- a/Myriad/Myriad.csproj +++ b/Myriad/Myriad.csproj @@ -21,17 +21,10 @@ - - - - - - - diff --git a/Myriad/packages.lock.json b/Myriad/packages.lock.json index 781c0c02..945321a9 100644 --- a/Myriad/packages.lock.json +++ b/Myriad/packages.lock.json @@ -2,28 +2,6 @@ "version": 1, "dependencies": { "net8.0": { - "Google.Protobuf": { - "type": "Direct", - "requested": "[3.29.1, )", - "resolved": "3.29.1", - "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ==" - }, - "Grpc.Net.ClientFactory": { - "type": "Direct", - "requested": "[2.67.0, )", - "resolved": "2.67.0", - "contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==", - "dependencies": { - "Grpc.Net.Client": "2.67.0", - "Microsoft.Extensions.Http": "6.0.0" - } - }, - "Grpc.Tools": { - "type": "Direct", - "requested": "[2.68.1, )", - "resolved": "2.68.1", - "contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q==" - }, "Polly": { "type": "Direct", "requested": "[8.5.0, )", @@ -64,92 +42,16 @@ "Microsoft.Bcl.AsyncInterfaces": "6.0.0" } }, - "Grpc.Core.Api": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q==" - }, - "Grpc.Net.Client": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==", - "dependencies": { - "Grpc.Net.Common": "2.67.0", - "Microsoft.Extensions.Logging.Abstractions": "6.0.0" - } - }, - "Grpc.Net.Common": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==", - "dependencies": { - "Grpc.Core.Api": "2.67.0" - } - }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", "resolved": "6.0.0", "contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg==" }, - "Microsoft.Extensions.DependencyInjection": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "k6PWQMuoBDGGHOQTtyois2u4AwyVcIwL2LaSLlTZQm2CYcJ1pxbt6jfAnpWmzENA/wfrYRI/X9DTLoUkE4AsLw==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "System.Runtime.CompilerServices.Unsafe": "6.0.0" - } - }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==" - }, - "Microsoft.Extensions.Http": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.Logging": "6.0.0", - "Microsoft.Extensions.Logging.Abstractions": "6.0.0", - "Microsoft.Extensions.Options": "6.0.0" - } - }, - "Microsoft.Extensions.Logging": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "eIbyj40QDg1NDz0HBW0S5f3wrLVnKWnDJ/JtZ+yJDFnDj90VoPuoPmFkeaXrtu+0cKm5GRAwoDf+dBWXK0TUdg==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "6.0.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.Logging.Abstractions": "6.0.0", - "Microsoft.Extensions.Options": "6.0.0", - "System.Diagnostics.DiagnosticSource": "6.0.0" - } - }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", "resolved": "6.0.0", "contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==" }, - "Microsoft.Extensions.Options": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.Primitives": "6.0.0" - } - }, - "Microsoft.Extensions.Primitives": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==", - "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.0.0" - } - }, "Pipelines.Sockets.Unofficial": { "type": "Transitive", "resolved": "2.2.8", @@ -163,23 +65,10 @@ "resolved": "8.5.0", "contentHash": "VYYMZNitZ85UEhwOKkTQI63WEMvzUqwQc74I2mm8h/DBVAMcBBxqYPni4DmuRtbCwngmuONuK2yBJfWNRKzI+A==" }, - "System.Diagnostics.DiagnosticSource": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "frQDfv0rl209cKm1lnwTgFPzNigy2EKk1BS3uAvHvlBVKe5cymGyHO+Sj+NLv5VF/AhHsqPIUUwya5oV4CHMUw==", - "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.0.0" - } - }, "System.IO.Pipelines": { "type": "Transitive", "resolved": "5.0.1", "contentHash": "qEePWsaq9LoEEIqhbGe6D5J8c9IqQOUuTzzV6wn1POlfdLkJliZY3OlB0j0f17uMWlqZYjH7txj+2YbyrIA8Yg==" - }, - "System.Runtime.CompilerServices.Unsafe": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" } } } diff --git a/PluralKit.API/Controllers/PrivateController.cs b/PluralKit.API/Controllers/PrivateController.cs index 561fd23b..1f37dc8f 100644 --- a/PluralKit.API/Controllers/PrivateController.cs +++ b/PluralKit.API/Controllers/PrivateController.cs @@ -22,28 +22,6 @@ public class PrivateController: PKControllerBase { public PrivateController(IServiceProvider svc) : base(svc) { } - [HttpGet("meta")] - public async Task> Meta() - { - var db = _redis.Connection.GetDatabase(); - var redisInfo = await db.HashGetAllAsync("pluralkit:shardstatus"); - var shards = redisInfo.Select(x => Proto.Unmarshal(x.Value)).OrderBy(x => x.ShardId); - - var redisClusterInfo = await db.HashGetAllAsync("pluralkit:cluster_stats"); - var clusterInfo = redisClusterInfo.Select(x => JsonConvert.DeserializeObject(x.Value)); - - var guildCount = clusterInfo.Sum(x => x.GuildCount); - var channelCount = clusterInfo.Sum(x => x.ChannelCount); - - var stats = await _repo.GetStats(); - - var o = new JObject(); - o.Add("shards", shards.ToJson()); - o.Add("stats", stats.ToJson(guildCount, channelCount)); - - return Ok(o); - } - [HttpPost("bulk_privacy/member")] public async Task BulkMemberPrivacy([FromBody] JObject inner) { @@ -158,31 +136,4 @@ public static class PrivateJsonExt o.Add("error", error); return o; } - - public static JArray ToJson(this IEnumerable shards) - { - var o = new JArray(); - - foreach (var shard in shards) - { - var s = new JObject(); - s.Add("id", shard.ShardId); - - if (!shard.Up) - s.Add("status", "down"); - else - s.Add("status", "up"); - - s.Add("ping", shard.Latency); - s.Add("disconnection_count", shard.DisconnectionCount); - s.Add("last_heartbeat", shard.LastHeartbeat.ToString()); - s.Add("last_connection", shard.LastConnection.ToString()); - if (shard.HasClusterId) - s.Add("cluster_id", shard.ClusterId); - - o.Add(s); - } - - return o; - } } \ No newline at end of file diff --git a/PluralKit.API/PluralKit.API.csproj b/PluralKit.API/PluralKit.API.csproj index f5db2009..ea792b98 100644 --- a/PluralKit.API/PluralKit.API.csproj +++ b/PluralKit.API/PluralKit.API.csproj @@ -28,15 +28,9 @@ - - - - - - diff --git a/PluralKit.API/packages.lock.json b/PluralKit.API/packages.lock.json index a6350e7c..8c8be7c4 100644 --- a/PluralKit.API/packages.lock.json +++ b/PluralKit.API/packages.lock.json @@ -2,18 +2,6 @@ "version": 1, "dependencies": { "net8.0": { - "Google.Protobuf": { - "type": "Direct", - "requested": "[3.29.1, )", - "resolved": "3.29.1", - "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ==" - }, - "Grpc.Tools": { - "type": "Direct", - "requested": "[2.68.1, )", - "resolved": "2.68.1", - "contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q==" - }, "Microsoft.AspNetCore.Mvc.NewtonsoftJson": { "type": "Direct", "requested": "[8.0.11, )", @@ -830,7 +818,6 @@ "Autofac.Extensions.DependencyInjection": "[10.0.0, )", "Dapper": "[2.1.35, )", "Dapper.Contrib": "[2.0.78, )", - "Google.Protobuf": "[3.29.1, )", "Microsoft.Extensions.Caching.Memory": "[9.0.0, )", "Microsoft.Extensions.Configuration": "[9.0.0, )", "Microsoft.Extensions.Configuration.Binder": "[9.0.0, )", diff --git a/PluralKit.Bot/Commands/Misc.cs b/PluralKit.Bot/Commands/Misc.cs index 8ada0ee4..decb574a 100644 --- a/PluralKit.Bot/Commands/Misc.cs +++ b/PluralKit.Bot/Commands/Misc.cs @@ -94,7 +94,7 @@ public class Misc var memoryUsage = process.WorkingSet64; var now = SystemClock.Instance.GetCurrentInstant().ToUnixTimeSeconds(); - var shardUptime = Duration.FromSeconds(now - shardInfo?.LastConnection ?? 0); + var shardUptime = Duration.FromSeconds(now - shardInfo.LastConnection); var shardTotal = _botConfig.Cluster?.TotalShards ?? shards.Count(); int shardClusterTotal = ctx.Cluster.Shards.Count; @@ -106,11 +106,11 @@ public class Misc + (isCluster ? $" {shardClusterTotal} in this cluster," : "") + $" {shardUpTotal} are up)" , true)) .Field(new Embed.Field("Shard uptime", - $"{shardUptime.FormatDuration()} ({shardInfo?.DisconnectionCount} disconnections)", true)) + $"{shardUptime.FormatDuration()} ({shardInfo.DisconnectionCount} disconnections)", true)) .Field(new Embed.Field("CPU usage", $"{_cpu.LastCpuMeasure:P1}", true)) .Field(new Embed.Field("Memory usage", $"{memoryUsage / 1024 / 1024} MiB", true)) .Field(new Embed.Field("Latency", - $"API: {apiLatency.TotalMilliseconds:F0} ms, shard: {shardInfo?.Latency} ms", + $"API: {apiLatency.TotalMilliseconds:F0} ms, shard: {shardInfo.Latency} ms", true)); embed.Field(new("Total numbers", $" {counts.SystemCount:N0} systems," diff --git a/PluralKit.Bot/PluralKit.Bot.csproj b/PluralKit.Bot/PluralKit.Bot.csproj index dd8406a3..f15e7051 100644 --- a/PluralKit.Bot/PluralKit.Bot.csproj +++ b/PluralKit.Bot/PluralKit.Bot.csproj @@ -22,15 +22,8 @@ - - - - - - - diff --git a/PluralKit.Bot/Services/ShardInfoService.cs b/PluralKit.Bot/Services/ShardInfoService.cs index 0defb23b..2382cb8a 100644 --- a/PluralKit.Bot/Services/ShardInfoService.cs +++ b/PluralKit.Bot/Services/ShardInfoService.cs @@ -1,11 +1,11 @@ using System.Net.WebSockets; -using Google.Protobuf; - using Myriad.Gateway; using NodaTime; +using Newtonsoft.Json; + using StackExchange.Redis; using PluralKit.Core; @@ -14,6 +14,17 @@ using Serilog; namespace PluralKit.Bot; +public struct ShardState +{ + [JsonProperty("shard_id")] public int ShardId; + [JsonProperty("up")] public bool Up; + [JsonProperty("disconnection_count")] public int DisconnectionCount; + [JsonProperty("latency")] public int Latency; + [JsonProperty("last_heartbeat")] public int LastHeartbeat; + [JsonProperty("last_connection")] public int LastConnection; + [JsonProperty("cluster_id")] public int? ClusterId; +} + public class ShardInfoService { private readonly int? _clusterId; @@ -43,7 +54,7 @@ public class ShardInfoService return new ShardState[] { }; var db = _redis.Connection.GetDatabase(); var redisInfo = await db.HashGetAllAsync("pluralkit:shardstatus"); - return redisInfo.Select(x => Proto.Unmarshal(x.Value)); + return redisInfo.Select(x => JsonConvert.DeserializeObject(x.Value)); } private void InitializeShard(Shard shard) @@ -53,13 +64,8 @@ public class ShardInfoService async Task Inner() { var db = _redis.Connection.GetDatabase(); - var redisInfo = await db.HashGetAsync("pluralkit::shardstatus", shard.ShardId); - // Skip adding listeners if we've seen this shard & already added listeners to it - if (redisInfo.HasValue) - return; - - // latency = 0 because otherwise shard 0 would serialize to an empty array, thanks protobuf + // latency = 1 because otherwise shard 0 would serialize to an empty array, thanks protobuf var state = new ShardState() { ShardId = shard.ShardId, Up = false, Latency = 1 }; if (_clusterId != null) state.ClusterId = _clusterId.Value; @@ -75,13 +81,13 @@ public class ShardInfoService } } - private async Task TryGetShard(Shard shard) + private async Task TryGetShard(Shard shard) { var db = _redis.Connection.GetDatabase(); var redisInfo = await db.HashGetAsync("pluralkit:shardstatus", shard.ShardId); if (redisInfo.HasValue) - return Proto.Unmarshal(redisInfo); - return null; + return JsonConvert.DeserializeObject(redisInfo); + throw new Exception("expected shard, but didn't get one"); } private void ReadyOrResumed(Shard shard) @@ -145,5 +151,5 @@ public static class RedisExt { // convenience method public static HashEntry[] HashWrapper(this ShardState state) - => new[] { new HashEntry(state.ShardId, state.ToByteArray()) }; + => new[] { new HashEntry(state.ShardId, JsonConvert.SerializeObject(state)) }; } \ No newline at end of file diff --git a/PluralKit.Bot/packages.lock.json b/PluralKit.Bot/packages.lock.json index f5141a33..fbe69392 100644 --- a/PluralKit.Bot/packages.lock.json +++ b/PluralKit.Bot/packages.lock.json @@ -2,28 +2,6 @@ "version": 1, "dependencies": { "net8.0": { - "Google.Protobuf": { - "type": "Direct", - "requested": "[3.29.1, )", - "resolved": "3.29.1", - "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ==" - }, - "Grpc.Net.ClientFactory": { - "type": "Direct", - "requested": "[2.67.0, )", - "resolved": "2.67.0", - "contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==", - "dependencies": { - "Grpc.Net.Client": "2.67.0", - "Microsoft.Extensions.Http": "6.0.0" - } - }, - "Grpc.Tools": { - "type": "Direct", - "requested": "[2.68.1, )", - "resolved": "2.68.1", - "contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q==" - }, "Humanizer.Core": { "type": "Direct", "requested": "[2.14.1, )", @@ -158,28 +136,6 @@ "System.Diagnostics.DiagnosticSource": "5.0.0" } }, - "Grpc.Core.Api": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q==" - }, - "Grpc.Net.Client": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==", - "dependencies": { - "Grpc.Net.Common": "2.67.0", - "Microsoft.Extensions.Logging.Abstractions": "6.0.0" - } - }, - "Grpc.Net.Common": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==", - "dependencies": { - "Grpc.Core.Api": "2.67.0" - } - }, "IPNetwork2": { "type": "Transitive", "resolved": "3.0.667", @@ -318,17 +274,6 @@ "resolved": "9.0.0", "contentHash": "jGFKZiXs2HNseK3NK/rfwHNNovER71jSj4BD1a/649ml9+h6oEtYd0GSALZDNW8jZ2Rh+oAeadOa6sagYW1F2A==" }, - "Microsoft.Extensions.Http": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.Logging": "6.0.0", - "Microsoft.Extensions.Logging.Abstractions": "6.0.0", - "Microsoft.Extensions.Options": "6.0.0" - } - }, "Microsoft.Extensions.Logging": { "type": "Transitive", "resolved": "9.0.0", @@ -778,8 +723,6 @@ "myriad": { "type": "Project", "dependencies": { - "Google.Protobuf": "[3.29.1, )", - "Grpc.Net.ClientFactory": "[2.67.0, )", "Polly": "[8.5.0, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Serilog": "[4.2.0, )", @@ -797,7 +740,6 @@ "Autofac.Extensions.DependencyInjection": "[10.0.0, )", "Dapper": "[2.1.35, )", "Dapper.Contrib": "[2.0.78, )", - "Google.Protobuf": "[3.29.1, )", "Microsoft.Extensions.Caching.Memory": "[9.0.0, )", "Microsoft.Extensions.Configuration": "[9.0.0, )", "Microsoft.Extensions.Configuration.Binder": "[9.0.0, )", diff --git a/PluralKit.Core/PluralKit.Core.csproj b/PluralKit.Core/PluralKit.Core.csproj index 1a0063f1..2e956ec5 100644 --- a/PluralKit.Core/PluralKit.Core.csproj +++ b/PluralKit.Core/PluralKit.Core.csproj @@ -23,7 +23,6 @@ - diff --git a/PluralKit.Core/Utils/ProtobufUtils.cs b/PluralKit.Core/Utils/ProtobufUtils.cs deleted file mode 100644 index 63cd9374..00000000 --- a/PluralKit.Core/Utils/ProtobufUtils.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Google.Protobuf; - -namespace PluralKit.Core; - -public static class Proto -{ - private static readonly Dictionary _parser = new(); - - public static byte[] Marshal(this IMessage message) => message.ToByteArray(); - - public static T Unmarshal(this byte[] message) where T : IMessage, new() - { - var type = typeof(T).ToString(); - if (_parser.ContainsKey(type)) - { - return (T)_parser[type].ParseFrom(message); - } - - _parser.Add(type, new MessageParser(() => new T())); - return Unmarshal(message); - } -} \ No newline at end of file diff --git a/PluralKit.Core/packages.lock.json b/PluralKit.Core/packages.lock.json index 624ba549..a4768566 100644 --- a/PluralKit.Core/packages.lock.json +++ b/PluralKit.Core/packages.lock.json @@ -66,12 +66,6 @@ "Dapper": "2.0.78" } }, - "Google.Protobuf": { - "type": "Direct", - "requested": "[3.29.1, )", - "resolved": "3.29.1", - "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ==" - }, "IPNetwork2": { "type": "Direct", "requested": "[3.0.667, )", diff --git a/PluralKit.Tests/packages.lock.json b/PluralKit.Tests/packages.lock.json index 19fefd0b..419ec053 100644 --- a/PluralKit.Tests/packages.lock.json +++ b/PluralKit.Tests/packages.lock.json @@ -151,42 +151,6 @@ "System.Diagnostics.DiagnosticSource": "5.0.0" } }, - "Google.Protobuf": { - "type": "Transitive", - "resolved": "3.29.1", - "contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ==" - }, - "Grpc.Core.Api": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q==" - }, - "Grpc.Net.Client": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==", - "dependencies": { - "Grpc.Net.Common": "2.67.0", - "Microsoft.Extensions.Logging.Abstractions": "6.0.0" - } - }, - "Grpc.Net.ClientFactory": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==", - "dependencies": { - "Grpc.Net.Client": "2.67.0", - "Microsoft.Extensions.Http": "6.0.0" - } - }, - "Grpc.Net.Common": { - "type": "Transitive", - "resolved": "2.67.0", - "contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==", - "dependencies": { - "Grpc.Core.Api": "2.67.0" - } - }, "Humanizer.Core": { "type": "Transitive", "resolved": "2.14.1", @@ -398,17 +362,6 @@ "Microsoft.Extensions.Logging.Abstractions": "9.0.0" } }, - "Microsoft.Extensions.Http": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.Logging": "6.0.0", - "Microsoft.Extensions.Logging.Abstractions": "6.0.0", - "Microsoft.Extensions.Options": "6.0.0" - } - }, "Microsoft.Extensions.Logging": { "type": "Transitive", "resolved": "9.0.0", @@ -982,8 +935,6 @@ "myriad": { "type": "Project", "dependencies": { - "Google.Protobuf": "[3.29.1, )", - "Grpc.Net.ClientFactory": "[2.67.0, )", "Polly": "[8.5.0, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Serilog": "[4.2.0, )", @@ -994,7 +945,6 @@ "pluralkit.api": { "type": "Project", "dependencies": { - "Google.Protobuf": "[3.29.1, )", "Microsoft.AspNetCore.Mvc.NewtonsoftJson": "[8.0.11, )", "Microsoft.AspNetCore.Mvc.Versioning": "[5.1.0, )", "Microsoft.AspNetCore.Mvc.Versioning.ApiExplorer": "[5.1.0, )", @@ -1005,8 +955,6 @@ "pluralkit.bot": { "type": "Project", "dependencies": { - "Google.Protobuf": "[3.29.1, )", - "Grpc.Net.ClientFactory": "[2.67.0, )", "Humanizer.Core": "[2.14.1, )", "Myriad": "[1.0.0, )", "PluralKit.Core": "[1.0.0, )", @@ -1024,7 +972,6 @@ "Autofac.Extensions.DependencyInjection": "[10.0.0, )", "Dapper": "[2.1.35, )", "Dapper.Contrib": "[2.0.78, )", - "Google.Protobuf": "[3.29.1, )", "Microsoft.Extensions.Caching.Memory": "[9.0.0, )", "Microsoft.Extensions.Configuration": "[9.0.0, )", "Microsoft.Extensions.Configuration.Binder": "[9.0.0, )", diff --git a/ci/Dockerfile.rust b/ci/Dockerfile.rust index 9e6dad46..e0156e5c 100644 --- a/ci/Dockerfile.rust +++ b/ci/Dockerfile.rust @@ -2,7 +2,7 @@ FROM alpine:latest AS builder WORKDIR /build -RUN apk add rustup build-base protoc +RUN apk add rustup build-base # todo: arm64 target RUN rustup-init --default-host x86_64-unknown-linux-musl --default-toolchain nightly-2024-08-20 --profile default -y @@ -22,7 +22,6 @@ RUN cargo chef cook --release --recipe-path recipe.json --target x86_64-unknown- COPY Cargo.toml /build/ COPY Cargo.lock /build/ -COPY proto/ /build/proto # this needs to match workspaces in Cargo.toml COPY lib/libpk /build/lib/libpk diff --git a/dashboard/src/routes/Status/status.svelte b/dashboard/src/routes/Status/status.svelte index 8264057b..57568bb1 100644 --- a/dashboard/src/routes/Status/status.svelte +++ b/dashboard/src/routes/Status/status.svelte @@ -25,7 +25,7 @@ let valid = false; const get = async () => { - const pkdata = await api().private.meta.get(); + const pkdata = await api().private.discord.shard_state.get(); let data = pkdata.shards.sort((x, y) => (x.id > y.id) ? 1 : -1); let latencies = 0; data = data.map(shard => { diff --git a/flake.nix b/flake.nix index 5f3989a7..ac600aa4 100644 --- a/flake.nix +++ b/flake.nix @@ -52,7 +52,6 @@ git dotnet-sdk_8 gcc - protobuf omnisharp-roslyn bashInteractive ]; @@ -80,14 +79,14 @@ path = ./.; export = false; }; - nci.crates."gateway" = { - depsDrvConfig.mkDerivation = { - nativeBuildInputs = [ pkgs.protobuf ]; - }; - drvConfig.mkDerivation = { - nativeBuildInputs = [ pkgs.protobuf ]; - }; - }; + # nci.crates."gateway" = { + # depsDrvConfig.mkDerivation = { + # nativeBuildInputs = [ pkgs.protobuf ]; + # }; + # drvConfig.mkDerivation = { + # nativeBuildInputs = [ pkgs.protobuf ]; + # }; + # }; # TODO: expose other rust packages after it's verified they build and work properly packages = lib.genAttrs ["gateway"] (name: rustOutputs.${name}.packages.release); diff --git a/lib/libpk/Cargo.toml b/lib/libpk/Cargo.toml index a21bf55f..714e7d00 100644 --- a/lib/libpk/Cargo.toml +++ b/lib/libpk/Cargo.toml @@ -8,8 +8,6 @@ anyhow = { workspace = true } fred = { workspace = true } lazy_static = { workspace = true } metrics = { workspace = true } -prost = { workspace = true } -prost-types = { workspace = true } sentry = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -24,6 +22,3 @@ uuid = { workspace = true } config = "0.14.0" json-subscriber = { version = "0.2.2", features = ["env-filter"] } metrics-exporter-prometheus = { version = "0.15.3", default-features = false, features = ["tokio", "http-listener", "tracing"] } - -[build-dependencies] -prost-build = { workspace = true } diff --git a/lib/libpk/build.rs b/lib/libpk/build.rs deleted file mode 100644 index 6569962b..00000000 --- a/lib/libpk/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -use std::io::Result; - -fn main() -> Result<()> { - prost_build::Config::new() - .type_attribute(".ShardState", "#[derive(serde::Serialize)]") - .field_attribute(".ShardState.shard_id", "#[serde(rename = \"id\")]") - .compile_protos(&["../../proto/state.proto"], &["../../proto/"])?; - Ok(()) -} diff --git a/lib/libpk/src/lib.rs b/lib/libpk/src/lib.rs index 19d4e1ee..ee7a6536 100644 --- a/lib/libpk/src/lib.rs +++ b/lib/libpk/src/lib.rs @@ -6,7 +6,7 @@ use sentry::IntoDsn; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; pub mod db; -pub mod proto; +pub mod state; pub mod util; pub mod _config; diff --git a/lib/libpk/src/proto.rs b/lib/libpk/src/proto.rs deleted file mode 100644 index 3e6d84c3..00000000 --- a/lib/libpk/src/proto.rs +++ /dev/null @@ -1 +0,0 @@ -include!(concat!(env!("OUT_DIR"), "/_.rs")); diff --git a/proto/discord_cache.proto b/proto/discord_cache.proto deleted file mode 100644 index fa11fde3..00000000 --- a/proto/discord_cache.proto +++ /dev/null @@ -1,47 +0,0 @@ -syntax = "proto3"; - -package myriad.cache; - -message Overwrite { - uint64 id = 1; - int32 type = 2; - uint64 allow = 3; - uint64 deny = 4; -} - -message CachedChannel { - uint64 id = 1; - int32 type = 2; - int32 position = 3; - string name = 4; - repeated Overwrite permission_overwrites = 5; - optional uint64 guild_id = 6; - optional uint64 parent_id = 7; -} - -message CachedGuildMember { - repeated uint64 roles = 1; -} - -message CachedGuild { - uint64 id = 1; - string name = 2; - uint64 owner_id = 3; - int32 premium_tier = 4; -} - -message CachedRole { - uint64 id = 1; - string name = 2; - int32 position = 3; - uint64 permissions = 4; - bool mentionable = 5; -} - -message CachedUser { - uint64 id = 1; - string username = 2; - string discriminator = 3; - optional string avatar = 4; - bool bot = 5; -} \ No newline at end of file diff --git a/proto/state.proto b/proto/state.proto deleted file mode 100644 index 448d869b..00000000 --- a/proto/state.proto +++ /dev/null @@ -1,21 +0,0 @@ -syntax = "proto3"; - -message ShardState { - int32 shard_id = 1; - bool up = 2; - int32 disconnection_count = 3; - - // milliseconds - int32 latency = 4; - - // unix timestamp - int32 last_heartbeat = 5; - int32 last_connection = 6; - - optional int32 cluster_id = 7; -} - -message GatewayEvent { - int32 shard_id = 1; - bytes payload = 2; -} diff --git a/services/api/Cargo.toml b/services/api/Cargo.toml index e77eaab5..a8de117b 100644 --- a/services/api/Cargo.toml +++ b/services/api/Cargo.toml @@ -10,7 +10,6 @@ fred = { workspace = true } lazy_static = { workspace = true } libpk = { path = "../../lib/libpk" } metrics = { workspace = true } -prost = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } diff --git a/services/api/src/endpoints/private.rs b/services/api/src/endpoints/private.rs index 9db137ee..95da5041 100644 --- a/services/api/src/endpoints/private.rs +++ b/services/api/src/endpoints/private.rs @@ -1,8 +1,7 @@ use crate::ApiContext; use axum::{extract::State, response::Json}; use fred::interfaces::*; -use libpk::proto::ShardState; -use prost::Message; +use libpk::state::ShardState; use serde::Deserialize; use serde_json::{json, Value}; use std::collections::HashMap; @@ -14,16 +13,22 @@ struct ClusterStats { pub channel_count: i32, } -pub async fn meta(State(ctx): State) -> Json { +pub async fn discord_state(State(ctx): State) -> Json { let shard_status = ctx .redis - .hgetall::>, &str>("pluralkit:shardstatus") + .hgetall::, &str>("pluralkit:shardstatus") .await .unwrap() .values() - .map(|v| ShardState::decode(v.as_slice()).unwrap()) + .map(|v| serde_json::from_str(v).expect("could not deserialize shard")) .collect::>(); + Json(json!({ + "shards": shard_status, + })) +} + +pub async fn meta(State(ctx): State) -> Json { let cluster_stats = ctx .redis .hgetall::, &str>("pluralkit:cluster_stats") @@ -39,15 +44,12 @@ pub async fn meta(State(ctx): State) -> Json { let channel_count: i32 = cluster_stats.iter().map(|v| v.channel_count).sum(); Json(json!({ - "shards": shard_status, - "stats": { - "system_count": db_stats.system_count, - "member_count": db_stats.member_count, - "group_count": db_stats.group_count, - "switch_count": db_stats.switch_count, - "message_count": db_stats.message_count, - "guild_count": guild_count, - "channel_count": channel_count, - } + "system_count": db_stats.system_count, + "member_count": db_stats.member_count, + "group_count": db_stats.group_count, + "switch_count": db_stats.switch_count, + "message_count": db_stats.message_count, + "guild_count": guild_count, + "channel_count": channel_count, })) } diff --git a/services/api/src/main.rs b/services/api/src/main.rs index bbf3c6ee..1a00f1a0 100644 --- a/services/api/src/main.rs +++ b/services/api/src/main.rs @@ -104,10 +104,11 @@ fn router(ctx: ApiContext) -> Router { .route("/v2/messages/:message_id", get(rproxy)) - .route("/private/meta", get(endpoints::private::meta)) .route("/private/bulk_privacy/member", post(rproxy)) .route("/private/bulk_privacy/group", post(rproxy)) .route("/private/discord/callback", post(rproxy)) + .route("/private/discord/shard_state", get(endpoints::private::discord_state)) + .route("/private/stats", get(endpoints::private::meta)) .route("/v2/systems/:system_id/oembed.json", get(rproxy)) .route("/v2/members/:member_id/oembed.json", get(rproxy)) diff --git a/services/api/src/middleware/logger.rs b/services/api/src/middleware/logger.rs index cfb84b4a..020de2e2 100644 --- a/services/api/src/middleware/logger.rs +++ b/services/api/src/middleware/logger.rs @@ -42,7 +42,6 @@ pub async fn logger(request: Request, next: Next) -> Response { let authenticated = { let headers = response.headers_mut(); - println!("{:#?}", headers.keys()); if headers.contains_key(DID_AUTHENTICATE_HEADER) { headers.remove(DID_AUTHENTICATE_HEADER); true diff --git a/services/gateway/Cargo.toml b/services/gateway/Cargo.toml index 6b9f3e25..98142a7f 100644 --- a/services/gateway/Cargo.toml +++ b/services/gateway/Cargo.toml @@ -13,7 +13,6 @@ futures = { workspace = true } lazy_static = { workspace = true } libpk = { path = "../../lib/libpk" } metrics = { workspace = true } -prost = { workspace = true } serde_json = { workspace = true } signal-hook = { workspace = true } tokio = { workspace = true } diff --git a/services/gateway/src/discord/shard_state.rs b/services/gateway/src/discord/shard_state.rs index f92ff866..1e7f94fe 100644 --- a/services/gateway/src/discord/shard_state.rs +++ b/services/gateway/src/discord/shard_state.rs @@ -1,11 +1,10 @@ use bytes::Bytes; use fred::{clients::RedisPool, interfaces::HashesInterface}; use metrics::{counter, gauge}; -use prost::Message; use tracing::info; use twilight_gateway::Event; -use libpk::{proto::*, util::redis::*}; +use libpk::{state::*, util::redis::*}; #[derive(Clone)] pub struct ShardStateManager { @@ -28,26 +27,24 @@ impl ShardStateManager { } async fn get_shard(&self, shard_id: u32) -> anyhow::Result { - let data: Option> = self + let data: Option = self .redis .hget("pluralkit:shardstatus", shard_id) .await .to_option_or_error()?; match data { - Some(buf) => { - Ok(ShardState::decode(buf.as_slice()).expect("could not decode shard data!")) - } + Some(buf) => Ok(serde_json::from_str(&buf).expect("could not decode shard data!")), None => Ok(ShardState::default()), } } async fn save_shard(&self, shard_id: u32, info: ShardState) -> anyhow::Result<()> { self.redis - .hset::<(), &str, (String, Bytes)>( + .hset::<(), &str, (String, String)>( "pluralkit:shardstatus", ( shard_id.to_string(), - Bytes::copy_from_slice(&info.encode_to_vec()), + serde_json::to_string(&info).expect("could not serialize shard"), ), ) .await?;