eightball/PluralKit
1
0
Fork 0
mirror of https://github.com/PluralKit/PluralKit.git synced 2026-02-04 04:56:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

refactor: remove protobuf

Browse source
This commit is contained in:
alyssa 2024-12-23 22:31:20 +00:00
parent f1685495eb
commit 4596d98670
31 changed files with 58 additions and 577 deletions
Download patch file Download diff file Expand all files Collapse all files

101
Cargo.lock generated
View file

@ -1,6 +1,6 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
version = 4
[[package]]
name = "addr2line"
@ -94,7 +94,6 @@ dependencies = [
"lazy_static",
"libpk",
"metrics",
"prost",
"reverse-proxy-service",
"serde",
"serde_json",
@ -837,12 +836,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "fixedbitset"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flate2"
version = "1.0.30"
@ -1042,7 +1035,6 @@ dependencies = [
"lazy_static",
"libpk",
"metrics",
"prost",
"serde_json",
"serde_variant",
"signal-hook",
@ -1547,15 +1539,6 @@ version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
[[package]]
name = "itertools"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.5"
@ -1644,9 +1627,6 @@ dependencies = [
"lazy_static",
"metrics",
"metrics-exporter-prometheus",
"prost",
"prost-build",
"prost-types",
"sentry",
"serde",
"serde_json",
@ -1874,12 +1854,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "multimap"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03"
[[package]]
name = "nibble_vec"
version = "0.1.0"
@ -2139,16 +2113,6 @@ dependencies = [
"sha2",
]
[[package]]
name = "petgraph"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
dependencies = [
"fixedbitset",
"indexmap",
]
[[package]]
name = "pin-project"
version = "1.0.12"
@ -2239,16 +2203,6 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "prettyplease"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
dependencies = [
"proc-macro2",
"syn 2.0.66",
]
[[package]]
name = "proc-macro2"
version = "1.0.84"
@ -2258,59 +2212,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prost"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-build"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
dependencies = [
"bytes",
"heck 0.5.0",
"itertools",
"log",
"multimap",
"once_cell",
"petgraph",
"prettyplease",
"prost",
"prost-types",
"regex",
"syn 2.0.66",
"tempfile",
]
[[package]]
name = "prost-derive"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]]
name = "prost-types"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0"
dependencies = [
"prost",
]
[[package]]
name = "quanta"
version = "0.12.3"

4
Cargo.toml
View file

@ -40,7 +40,3 @@ twilight-http = { git = "https://github.com/pluralkit/twilight", default-feature
#twilight-util = { path = "../twilight/twilight-util", features = ["permission-calculator"] }
#twilight-model = { path = "../twilight/twilight-model" }
#twilight-http = { path = "../twilight/twilight-http", default-features = false, features = ["rustls-native-roots"] }
prost = "0.12"
prost-types = "0.12"
prost-build = "0.12"

1
Dockerfile
View file

@ -10,7 +10,6 @@ COPY PluralKit.Bot/PluralKit.Bot.csproj /app/PluralKit.Bot/
COPY PluralKit.Core/PluralKit.Core.csproj /app/PluralKit.Core/
COPY PluralKit.Tests/PluralKit.Tests.csproj /app/PluralKit.Tests/
COPY .git/ /app/.git
COPY proto/ /app/proto
RUN dotnet restore PluralKit.sln
# Copy the rest of the code and build

7
Myriad/Myriad.csproj
View file

@ -21,17 +21,10 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.29.1" />
<PackageReference Include="Grpc.Net.ClientFactory" Version="2.67.0" />
<PackageReference Include="Grpc.Tools" Version="2.68.1" PrivateAssets="all" />
<PackageReference Include="Polly" Version="8.5.0" />
<PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
<PackageReference Include="Serilog" Version="4.2.0" />
<PackageReference Include="StackExchange.Redis" Version="2.8.22" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
<Protobuf Include="../proto/discord_cache.proto" GrpcServices="Client" Link="Protos/discord_cache.proto" />
</ItemGroup>
</Project>

111
Myriad/packages.lock.json
View file

@ -2,28 +2,6 @@
"version": 1,
"dependencies": {
"net8.0": {
"Google.Protobuf": {
"type": "Direct",
"requested": "[3.29.1, )",
"resolved": "3.29.1",
"contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
},
"Grpc.Net.ClientFactory": {
"type": "Direct",
"requested": "[2.67.0, )",
"resolved": "2.67.0",
"contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==",
"dependencies": {
"Grpc.Net.Client": "2.67.0",
"Microsoft.Extensions.Http": "6.0.0"
}
},
"Grpc.Tools": {
"type": "Direct",
"requested": "[2.68.1, )",
"resolved": "2.68.1",
"contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q=="
},
"Polly": {
"type": "Direct",
"requested": "[8.5.0, )",
@ -64,92 +42,16 @@
"Microsoft.Bcl.AsyncInterfaces": "6.0.0"
}
},
"Grpc.Core.Api": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q=="
},
"Grpc.Net.Client": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==",
"dependencies": {
"Grpc.Net.Common": "2.67.0",
"Microsoft.Extensions.Logging.Abstractions": "6.0.0"
}
},
"Grpc.Net.Common": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==",
"dependencies": {
"Grpc.Core.Api": "2.67.0"
}
},
"Microsoft.Bcl.AsyncInterfaces": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg=="
},
"Microsoft.Extensions.DependencyInjection": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "k6PWQMuoBDGGHOQTtyois2u4AwyVcIwL2LaSLlTZQm2CYcJ1pxbt6jfAnpWmzENA/wfrYRI/X9DTLoUkE4AsLw==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
"System.Runtime.CompilerServices.Unsafe": "6.0.0"
}
},
"Microsoft.Extensions.DependencyInjection.Abstractions": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg=="
},
"Microsoft.Extensions.Http": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
"Microsoft.Extensions.Logging": "6.0.0",
"Microsoft.Extensions.Logging.Abstractions": "6.0.0",
"Microsoft.Extensions.Options": "6.0.0"
}
},
"Microsoft.Extensions.Logging": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "eIbyj40QDg1NDz0HBW0S5f3wrLVnKWnDJ/JtZ+yJDFnDj90VoPuoPmFkeaXrtu+0cKm5GRAwoDf+dBWXK0TUdg==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection": "6.0.0",
"Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
"Microsoft.Extensions.Logging.Abstractions": "6.0.0",
"Microsoft.Extensions.Options": "6.0.0",
"System.Diagnostics.DiagnosticSource": "6.0.0"
}
},
"Microsoft.Extensions.Logging.Abstractions": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA=="
},
"Microsoft.Extensions.Options": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
"Microsoft.Extensions.Primitives": "6.0.0"
}
},
"Microsoft.Extensions.Primitives": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==",
"dependencies": {
"System.Runtime.CompilerServices.Unsafe": "6.0.0"
}
},
"Pipelines.Sockets.Unofficial": {
"type": "Transitive",
"resolved": "2.2.8",
@ -163,23 +65,10 @@
"resolved": "8.5.0",
"contentHash": "VYYMZNitZ85UEhwOKkTQI63WEMvzUqwQc74I2mm8h/DBVAMcBBxqYPni4DmuRtbCwngmuONuK2yBJfWNRKzI+A=="
},
"System.Diagnostics.DiagnosticSource": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "frQDfv0rl209cKm1lnwTgFPzNigy2EKk1BS3uAvHvlBVKe5cymGyHO+Sj+NLv5VF/AhHsqPIUUwya5oV4CHMUw==",
"dependencies": {
"System.Runtime.CompilerServices.Unsafe": "6.0.0"
}
},
"System.IO.Pipelines": {
"type": "Transitive",
"resolved": "5.0.1",
"contentHash": "qEePWsaq9LoEEIqhbGe6D5J8c9IqQOUuTzzV6wn1POlfdLkJliZY3OlB0j0f17uMWlqZYjH7txj+2YbyrIA8Yg=="
},
"System.Runtime.CompilerServices.Unsafe": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg=="
}
}
}

49
PluralKit.API/Controllers/PrivateController.cs
View file

@ -22,28 +22,6 @@ public class PrivateController: PKControllerBase
{
public PrivateController(IServiceProvider svc) : base(svc) { }
[HttpGet("meta")]
public async Task<ActionResult<JObject>> Meta()
{
var db = _redis.Connection.GetDatabase();
var redisInfo = await db.HashGetAllAsync("pluralkit:shardstatus");
var shards = redisInfo.Select(x => Proto.Unmarshal<ShardState>(x.Value)).OrderBy(x => x.ShardId);
var redisClusterInfo = await db.HashGetAllAsync("pluralkit:cluster_stats");
var clusterInfo = redisClusterInfo.Select(x => JsonConvert.DeserializeObject<ClusterMetricInfo>(x.Value));
var guildCount = clusterInfo.Sum(x => x.GuildCount);
var channelCount = clusterInfo.Sum(x => x.ChannelCount);
var stats = await _repo.GetStats();
var o = new JObject();
o.Add("shards", shards.ToJson());
o.Add("stats", stats.ToJson(guildCount, channelCount));
return Ok(o);
}
[HttpPost("bulk_privacy/member")]
public async Task<IActionResult> BulkMemberPrivacy([FromBody] JObject inner)
{
@ -158,31 +136,4 @@ public static class PrivateJsonExt
o.Add("error", error);
return o;
}
public static JArray ToJson(this IEnumerable<ShardState> shards)
{
var o = new JArray();
foreach (var shard in shards)
{
var s = new JObject();
s.Add("id", shard.ShardId);
if (!shard.Up)
s.Add("status", "down");
else
s.Add("status", "up");
s.Add("ping", shard.Latency);
s.Add("disconnection_count", shard.DisconnectionCount);
s.Add("last_heartbeat", shard.LastHeartbeat.ToString());
s.Add("last_connection", shard.LastConnection.ToString());
if (shard.HasClusterId)
s.Add("cluster_id", shard.ClusterId);
o.Add(s);
}
return o;
}
}

6
PluralKit.API/PluralKit.API.csproj
View file

@ -28,15 +28,9 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.29.1" />
<PackageReference Include="Grpc.Tools" Version="2.68.1" PrivateAssets="all" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.11" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning" Version="5.1.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning.ApiExplorer" Version="5.1.0" />
<PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
</ItemGroup>
<ItemGroup>
<Protobuf Include="../proto/state.proto" GrpcServices="Client" Link="Protos/state.proto" />
</ItemGroup>
</Project>

13
PluralKit.API/packages.lock.json
View file

@ -2,18 +2,6 @@
"version": 1,
"dependencies": {
"net8.0": {
"Google.Protobuf": {
"type": "Direct",
"requested": "[3.29.1, )",
"resolved": "3.29.1",
"contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
},
"Grpc.Tools": {
"type": "Direct",
"requested": "[2.68.1, )",
"resolved": "2.68.1",
"contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q=="
},
"Microsoft.AspNetCore.Mvc.NewtonsoftJson": {
"type": "Direct",
"requested": "[8.0.11, )",
@ -830,7 +818,6 @@
"Autofac.Extensions.DependencyInjection": "[10.0.0, )",
"Dapper": "[2.1.35, )",
"Dapper.Contrib": "[2.0.78, )",
"Google.Protobuf": "[3.29.1, )",
"Microsoft.Extensions.Caching.Memory": "[9.0.0, )",
"Microsoft.Extensions.Configuration": "[9.0.0, )",
"Microsoft.Extensions.Configuration.Binder": "[9.0.0, )",

6
PluralKit.Bot/Commands/Misc.cs
View file

@ -94,7 +94,7 @@ public class Misc
var memoryUsage = process.WorkingSet64;
var now = SystemClock.Instance.GetCurrentInstant().ToUnixTimeSeconds();
var shardUptime = Duration.FromSeconds(now - shardInfo?.LastConnection ?? 0);
var shardUptime = Duration.FromSeconds(now - shardInfo.LastConnection);
var shardTotal = _botConfig.Cluster?.TotalShards ?? shards.Count();
int shardClusterTotal = ctx.Cluster.Shards.Count;
@ -106,11 +106,11 @@ public class Misc
+ (isCluster ? $" {shardClusterTotal} in this cluster," : "") + $" {shardUpTotal} are up)"
, true))
.Field(new Embed.Field("Shard uptime",
$"{shardUptime.FormatDuration()} ({shardInfo?.DisconnectionCount} disconnections)", true))
$"{shardUptime.FormatDuration()} ({shardInfo.DisconnectionCount} disconnections)", true))
.Field(new Embed.Field("CPU usage", $"{_cpu.LastCpuMeasure:P1}", true))
.Field(new Embed.Field("Memory usage", $"{memoryUsage / 1024 / 1024} MiB", true))
.Field(new Embed.Field("Latency",
$"API: {apiLatency.TotalMilliseconds:F0} ms, shard: {shardInfo?.Latency} ms",
$"API: {apiLatency.TotalMilliseconds:F0} ms, shard: {shardInfo.Latency} ms",
true));
embed.Field(new("Total numbers", $" {counts.SystemCount:N0} systems,"

7
PluralKit.Bot/PluralKit.Bot.csproj
View file

@ -22,15 +22,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.29.1" />
<PackageReference Include="Grpc.Net.ClientFactory" Version="2.67.0" />
<PackageReference Include="Grpc.Tools" Version="2.68.1" PrivateAssets="all" />
<PackageReference Include="Humanizer.Core" Version="2.14.1" />
<PackageReference Include="Sentry" Version="4.13.0" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.6" />
</ItemGroup>
<ItemGroup>
<Protobuf Include="../proto/state.proto" GrpcServices="Client" Link="Protos/state.proto" />
</ItemGroup>
</Project>

32
PluralKit.Bot/Services/ShardInfoService.cs
View file

@ -1,11 +1,11 @@
using System.Net.WebSockets;
using Google.Protobuf;
using Myriad.Gateway;
using NodaTime;
using Newtonsoft.Json;
using StackExchange.Redis;
using PluralKit.Core;
@ -14,6 +14,17 @@ using Serilog;
namespace PluralKit.Bot;
public struct ShardState
{
[JsonProperty("shard_id")] public int ShardId;
[JsonProperty("up")] public bool Up;
[JsonProperty("disconnection_count")] public int DisconnectionCount;
[JsonProperty("latency")] public int Latency;
[JsonProperty("last_heartbeat")] public int LastHeartbeat;
[JsonProperty("last_connection")] public int LastConnection;
[JsonProperty("cluster_id")] public int? ClusterId;
}
public class ShardInfoService
{
private readonly int? _clusterId;
@ -43,7 +54,7 @@ public class ShardInfoService
return new ShardState[] { };
var db = _redis.Connection.GetDatabase();
var redisInfo = await db.HashGetAllAsync("pluralkit:shardstatus");
return redisInfo.Select(x => Proto.Unmarshal<ShardState>(x.Value));
return redisInfo.Select(x => JsonConvert.DeserializeObject<ShardState>(x.Value));
}
private void InitializeShard(Shard shard)
@ -53,13 +64,8 @@ public class ShardInfoService
async Task Inner()
{
var db = _redis.Connection.GetDatabase();
var redisInfo = await db.HashGetAsync("pluralkit::shardstatus", shard.ShardId);
// Skip adding listeners if we've seen this shard & already added listeners to it
if (redisInfo.HasValue)
return;
// latency = 0 because otherwise shard 0 would serialize to an empty array, thanks protobuf
// latency = 1 because otherwise shard 0 would serialize to an empty array, thanks protobuf
var state = new ShardState() { ShardId = shard.ShardId, Up = false, Latency = 1 };
if (_clusterId != null)
state.ClusterId = _clusterId.Value;
@ -75,13 +81,13 @@ public class ShardInfoService
}
}
private async Task<ShardState?> TryGetShard(Shard shard)
private async Task<ShardState> TryGetShard(Shard shard)
{
var db = _redis.Connection.GetDatabase();
var redisInfo = await db.HashGetAsync("pluralkit:shardstatus", shard.ShardId);
if (redisInfo.HasValue)
return Proto.Unmarshal<ShardState>(redisInfo);
return null;
return JsonConvert.DeserializeObject<ShardState>(redisInfo);
throw new Exception("expected shard, but didn't get one");
}
private void ReadyOrResumed(Shard shard)
@ -145,5 +151,5 @@ public static class RedisExt
{
// convenience method
public static HashEntry[] HashWrapper(this ShardState state)
=> new[] { new HashEntry(state.ShardId, state.ToByteArray()) };
=> new[] { new HashEntry(state.ShardId, JsonConvert.SerializeObject(state)) };
}

58
PluralKit.Bot/packages.lock.json
View file

@ -2,28 +2,6 @@
"version": 1,
"dependencies": {
"net8.0": {
"Google.Protobuf": {
"type": "Direct",
"requested": "[3.29.1, )",
"resolved": "3.29.1",
"contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
},
"Grpc.Net.ClientFactory": {
"type": "Direct",
"requested": "[2.67.0, )",
"resolved": "2.67.0",
"contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==",
"dependencies": {
"Grpc.Net.Client": "2.67.0",
"Microsoft.Extensions.Http": "6.0.0"
}
},
"Grpc.Tools": {
"type": "Direct",
"requested": "[2.68.1, )",
"resolved": "2.68.1",
"contentHash": "BZ96s7ijKAhJoRpIK+pqCeLaGaSwyc5/CAZFwgCcBuAnkU2naYvH0P6qnYCkl0pWDY/JBOnE2RvX9IvRX1Yc5Q=="
},
"Humanizer.Core": {
"type": "Direct",
"requested": "[2.14.1, )",
@ -158,28 +136,6 @@
"System.Diagnostics.DiagnosticSource": "5.0.0"
}
},
"Grpc.Core.Api": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q=="
},
"Grpc.Net.Client": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==",
"dependencies": {
"Grpc.Net.Common": "2.67.0",
"Microsoft.Extensions.Logging.Abstractions": "6.0.0"
}
},
"Grpc.Net.Common": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==",
"dependencies": {
"Grpc.Core.Api": "2.67.0"
}
},
"IPNetwork2": {
"type": "Transitive",
"resolved": "3.0.667",
@ -318,17 +274,6 @@
"resolved": "9.0.0",
"contentHash": "jGFKZiXs2HNseK3NK/rfwHNNovER71jSj4BD1a/649ml9+h6oEtYd0GSALZDNW8jZ2Rh+oAeadOa6sagYW1F2A=="
},
"Microsoft.Extensions.Http": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
"Microsoft.Extensions.Logging": "6.0.0",
"Microsoft.Extensions.Logging.Abstractions": "6.0.0",
"Microsoft.Extensions.Options": "6.0.0"
}
},
"Microsoft.Extensions.Logging": {
"type": "Transitive",
"resolved": "9.0.0",
@ -778,8 +723,6 @@
"myriad": {
"type": "Project",
"dependencies": {
"Google.Protobuf": "[3.29.1, )",
"Grpc.Net.ClientFactory": "[2.67.0, )",
"Polly": "[8.5.0, )",
"Polly.Contrib.WaitAndRetry": "[1.1.1, )",
"Serilog": "[4.2.0, )",
@ -797,7 +740,6 @@
"Autofac.Extensions.DependencyInjection": "[10.0.0, )",
"Dapper": "[2.1.35, )",
"Dapper.Contrib": "[2.0.78, )",
"Google.Protobuf": "[3.29.1, )",
"Microsoft.Extensions.Caching.Memory": "[9.0.0, )",
"Microsoft.Extensions.Configuration": "[9.0.0, )",
"Microsoft.Extensions.Configuration.Binder": "[9.0.0, )",

1
PluralKit.Core/PluralKit.Core.csproj
View file

@ -23,7 +23,6 @@
<PackageReference Include="Autofac.Extensions.DependencyInjection" Version="10.0.0" />
<PackageReference Include="Dapper" Version="2.1.35" />
<PackageReference Include="Dapper.Contrib" Version="2.0.78" />
<PackageReference Include="Google.Protobuf" Version="3.29.1" />
<PackageReference Include="ipnetwork2" Version="3.0.667" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.0" />

22
PluralKit.Core/Utils/ProtobufUtils.cs
View file

@ -1,22 +0,0 @@
using Google.Protobuf;
namespace PluralKit.Core;
public static class Proto
{
private static readonly Dictionary<string, MessageParser> _parser = new();
public static byte[] Marshal(this IMessage message) => message.ToByteArray();
public static T Unmarshal<T>(this byte[] message) where T : IMessage<T>, new()
{
var type = typeof(T).ToString();
if (_parser.ContainsKey(type))
{
return (T)_parser[type].ParseFrom(message);
}
_parser.Add(type, new MessageParser<T>(() => new T()));
return Unmarshal<T>(message);
}
}

6
PluralKit.Core/packages.lock.json
View file

@ -66,12 +66,6 @@
"Dapper": "2.0.78"
}
},
"Google.Protobuf": {
"type": "Direct",
"requested": "[3.29.1, )",
"resolved": "3.29.1",
"contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
},
"IPNetwork2": {
"type": "Direct",
"requested": "[3.0.667, )",

53
PluralKit.Tests/packages.lock.json
View file

@ -151,42 +151,6 @@
"System.Diagnostics.DiagnosticSource": "5.0.0"
}
},
"Google.Protobuf": {
"type": "Transitive",
"resolved": "3.29.1",
"contentHash": "kDFLP72bPu4GwquVN7HtFnfqxhQs4CLbUEyOc/0yV48HB0Pxf7tDK7zIx6ifaQwGp+RSoLY1sz1CXqoAEAu+AQ=="
},
"Grpc.Core.Api": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "cL1/2f8kc8lsAGNdfCU25deedXVehhLA6GXKLLN4hAWx16XN7BmjYn3gFU+FBpir5yJynvDTHEypr3Tl0j7x/Q=="
},
"Grpc.Net.Client": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "ofTjJQfegWkVlk5R4k/LlwpcucpsBzntygd4iAeuKd/eLMkmBWoXN+xcjYJ5IibAahRpIJU461jABZvT6E9dwA==",
"dependencies": {
"Grpc.Net.Common": "2.67.0",
"Microsoft.Extensions.Logging.Abstractions": "6.0.0"
}
},
"Grpc.Net.ClientFactory": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "owkRL6j8ZXvLnn8SsFGCRiWoe/2wGv+u+TASAylzUtGbHLnw7IiFRoeTs0IgpCNTETvlMLVb0gHDiLGo+mJZ6g==",
"dependencies": {
"Grpc.Net.Client": "2.67.0",
"Microsoft.Extensions.Http": "6.0.0"
}
},
"Grpc.Net.Common": {
"type": "Transitive",
"resolved": "2.67.0",
"contentHash": "gazn1cD2Eol0/W5ZJRV4PYbNrxJ9oMs8pGYux5S9E4MymClvl7aqYSmpqgmWAUWvziRqK9K+yt3cjCMfQ3x/5A==",
"dependencies": {
"Grpc.Core.Api": "2.67.0"
}
},
"Humanizer.Core": {
"type": "Transitive",
"resolved": "2.14.1",
@ -398,17 +362,6 @@
"Microsoft.Extensions.Logging.Abstractions": "9.0.0"
}
},
"Microsoft.Extensions.Http": {
"type": "Transitive",
"resolved": "6.0.0",
"contentHash": "15+pa2G0bAMHbHewaQIdr/y6ag2H3yh4rd9hTXavtWDzQBkvpe2RMqFg8BxDpcQWssmjmBApGPcw93QRz6YcMg==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0",
"Microsoft.Extensions.Logging": "6.0.0",
"Microsoft.Extensions.Logging.Abstractions": "6.0.0",
"Microsoft.Extensions.Options": "6.0.0"
}
},
"Microsoft.Extensions.Logging": {
"type": "Transitive",
"resolved": "9.0.0",
@ -982,8 +935,6 @@
"myriad": {
"type": "Project",
"dependencies": {
"Google.Protobuf": "[3.29.1, )",
"Grpc.Net.ClientFactory": "[2.67.0, )",
"Polly": "[8.5.0, )",
"Polly.Contrib.WaitAndRetry": "[1.1.1, )",
"Serilog": "[4.2.0, )",
@ -994,7 +945,6 @@
"pluralkit.api": {
"type": "Project",
"dependencies": {
"Google.Protobuf": "[3.29.1, )",
"Microsoft.AspNetCore.Mvc.NewtonsoftJson": "[8.0.11, )",
"Microsoft.AspNetCore.Mvc.Versioning": "[5.1.0, )",
"Microsoft.AspNetCore.Mvc.Versioning.ApiExplorer": "[5.1.0, )",
@ -1005,8 +955,6 @@
"pluralkit.bot": {
"type": "Project",
"dependencies": {
"Google.Protobuf": "[3.29.1, )",
"Grpc.Net.ClientFactory": "[2.67.0, )",
"Humanizer.Core": "[2.14.1, )",
"Myriad": "[1.0.0, )",
"PluralKit.Core": "[1.0.0, )",
@ -1024,7 +972,6 @@
"Autofac.Extensions.DependencyInjection": "[10.0.0, )",
"Dapper": "[2.1.35, )",
"Dapper.Contrib": "[2.0.78, )",
"Google.Protobuf": "[3.29.1, )",
"Microsoft.Extensions.Caching.Memory": "[9.0.0, )",
"Microsoft.Extensions.Configuration": "[9.0.0, )",
"Microsoft.Extensions.Configuration.Binder": "[9.0.0, )",

3
ci/Dockerfile.rust
View file

@ -2,7 +2,7 @@ FROM alpine:latest AS builder
WORKDIR /build
RUN apk add rustup build-base protoc
RUN apk add rustup build-base
# todo: arm64 target
RUN rustup-init --default-host x86_64-unknown-linux-musl --default-toolchain nightly-2024-08-20 --profile default -y
@ -22,7 +22,6 @@ RUN cargo chef cook --release --recipe-path recipe.json --target x86_64-unknown-
COPY Cargo.toml /build/
COPY Cargo.lock /build/
COPY proto/ /build/proto
# this needs to match workspaces in Cargo.toml
COPY lib/libpk /build/lib/libpk

2
dashboard/src/routes/Status/status.svelte
View file

@ -25,7 +25,7 @@
let valid = false;
const get = async () => {
const pkdata = await api().private.meta.get();
const pkdata = await api().private.discord.shard_state.get();
let data = pkdata.shards.sort((x, y) => (x.id > y.id) ? 1 : -1);
let latencies = 0;
data = data.map(shard => {

17
flake.nix
View file

@ -52,7 +52,6 @@
git
dotnet-sdk_8
gcc
protobuf
omnisharp-roslyn
bashInteractive
];
@ -80,14 +79,14 @@
path = ./.;
export = false;
};
nci.crates."gateway" = {
depsDrvConfig.mkDerivation = {
nativeBuildInputs = [ pkgs.protobuf ];
};
drvConfig.mkDerivation = {
nativeBuildInputs = [ pkgs.protobuf ];
};
};
# nci.crates."gateway" = {
# depsDrvConfig.mkDerivation = {
# nativeBuildInputs = [ pkgs.protobuf ];
# };
# drvConfig.mkDerivation = {
# nativeBuildInputs = [ pkgs.protobuf ];
# };
# };
# TODO: expose other rust packages after it's verified they build and work properly
packages = lib.genAttrs ["gateway"] (name: rustOutputs.${name}.packages.release);

5
lib/libpk/Cargo.toml
View file

@ -8,8 +8,6 @@ anyhow = { workspace = true }
fred = { workspace = true }
lazy_static = { workspace = true }
metrics = { workspace = true }
prost = { workspace = true }
prost-types = { workspace = true }
sentry = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
@ -24,6 +22,3 @@ uuid = { workspace = true }
config = "0.14.0"
json-subscriber = { version = "0.2.2", features = ["env-filter"] }
metrics-exporter-prometheus = { version = "0.15.3", default-features = false, features = ["tokio", "http-listener", "tracing"] }
[build-dependencies]
prost-build = { workspace = true }

9
lib/libpk/build.rs
View file

@ -1,9 +0,0 @@
use std::io::Result;
fn main() -> Result<()> {
prost_build::Config::new()
.type_attribute(".ShardState", "#[derive(serde::Serialize)]")
.field_attribute(".ShardState.shard_id", "#[serde(rename = \"id\")]")
.compile_protos(&["../../proto/state.proto"], &["../../proto/"])?;
Ok(())
}

2
lib/libpk/src/lib.rs
View file

@ -6,7 +6,7 @@ use sentry::IntoDsn;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
pub mod db;
pub mod proto;
pub mod state;
pub mod util;
pub mod _config;

1
lib/libpk/src/proto.rs
View file

@ -1 +0,0 @@
include!(concat!(env!("OUT_DIR"), "/_.rs"));

47
proto/discord_cache.proto
View file

@ -1,47 +0,0 @@
syntax = "proto3";
package myriad.cache;
message Overwrite {
uint64 id = 1;
int32 type = 2;
uint64 allow = 3;
uint64 deny = 4;
}
message CachedChannel {
uint64 id = 1;
int32 type = 2;
int32 position = 3;
string name = 4;
repeated Overwrite permission_overwrites = 5;
optional uint64 guild_id = 6;
optional uint64 parent_id = 7;
}
message CachedGuildMember {
repeated uint64 roles = 1;
}
message CachedGuild {
uint64 id = 1;
string name = 2;
uint64 owner_id = 3;
int32 premium_tier = 4;
}
message CachedRole {
uint64 id = 1;
string name = 2;
int32 position = 3;
uint64 permissions = 4;
bool mentionable = 5;
}
message CachedUser {
uint64 id = 1;
string username = 2;
string discriminator = 3;
optional string avatar = 4;
bool bot = 5;
}

21
proto/state.proto
View file

@ -1,21 +0,0 @@
syntax = "proto3";
message ShardState {
int32 shard_id = 1;
bool up = 2;
int32 disconnection_count = 3;
// milliseconds
int32 latency = 4;
// unix timestamp
int32 last_heartbeat = 5;
int32 last_connection = 6;
optional int32 cluster_id = 7;
}
message GatewayEvent {
int32 shard_id = 1;
bytes payload = 2;
}

1
services/api/Cargo.toml
View file

@ -10,7 +10,6 @@ fred = { workspace = true }
lazy_static = { workspace = true }
libpk = { path = "../../lib/libpk" }
metrics = { workspace = true }
prost = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true }

18
services/api/src/endpoints/private.rs
View file

@ -1,8 +1,7 @@
use crate::ApiContext;
use axum::{extract::State, response::Json};
use fred::interfaces::*;
use libpk::proto::ShardState;
use prost::Message;
use libpk::state::ShardState;
use serde::Deserialize;
use serde_json::{json, Value};
use std::collections::HashMap;
@ -14,16 +13,22 @@ struct ClusterStats {
pub channel_count: i32,
}
pub async fn meta(State(ctx): State<ApiContext>) -> Json<Value> {
pub async fn discord_state(State(ctx): State<ApiContext>) -> Json<Value> {
let shard_status = ctx
.redis
.hgetall::<HashMap<String, Vec<u8>>, &str>("pluralkit:shardstatus")
.hgetall::<HashMap<String, String>, &str>("pluralkit:shardstatus")
.await
.unwrap()
.values()
.map(|v| ShardState::decode(v.as_slice()).unwrap())
.map(|v| serde_json::from_str(v).expect("could not deserialize shard"))
.collect::<Vec<ShardState>>();
Json(json!({
"shards": shard_status,
}))
}
pub async fn meta(State(ctx): State<ApiContext>) -> Json<Value> {
let cluster_stats = ctx
.redis
.hgetall::<HashMap<String, String>, &str>("pluralkit:cluster_stats")
@ -39,8 +44,6 @@ pub async fn meta(State(ctx): State<ApiContext>) -> Json<Value> {
let channel_count: i32 = cluster_stats.iter().map(|v| v.channel_count).sum();
Json(json!({
"shards": shard_status,
"stats": {
"system_count": db_stats.system_count,
"member_count": db_stats.member_count,
"group_count": db_stats.group_count,
@ -48,6 +51,5 @@ pub async fn meta(State(ctx): State<ApiContext>) -> Json<Value> {
"message_count": db_stats.message_count,
"guild_count": guild_count,
"channel_count": channel_count,
}
}))
}

3
services/api/src/main.rs
View file

@ -104,10 +104,11 @@ fn router(ctx: ApiContext) -> Router {
.route("/v2/messages/:message_id", get(rproxy))
.route("/private/meta", get(endpoints::private::meta))
.route("/private/bulk_privacy/member", post(rproxy))
.route("/private/bulk_privacy/group", post(rproxy))
.route("/private/discord/callback", post(rproxy))
.route("/private/discord/shard_state", get(endpoints::private::discord_state))
.route("/private/stats", get(endpoints::private::meta))
.route("/v2/systems/:system_id/oembed.json", get(rproxy))
.route("/v2/members/:member_id/oembed.json", get(rproxy))

1
services/api/src/middleware/logger.rs
View file

@ -42,7 +42,6 @@ pub async fn logger(request: Request, next: Next) -> Response {
let authenticated = {
let headers = response.headers_mut();
println!("{:#?}", headers.keys());
if headers.contains_key(DID_AUTHENTICATE_HEADER) {
headers.remove(DID_AUTHENTICATE_HEADER);
true

1
services/gateway/Cargo.toml
View file

@ -13,7 +13,6 @@ futures = { workspace = true }
lazy_static = { workspace = true }
libpk = { path = "../../lib/libpk" }
metrics = { workspace = true }
prost = { workspace = true }
serde_json = { workspace = true }
signal-hook = { workspace = true }
tokio = { workspace = true }

13
services/gateway/src/discord/shard_state.rs
View file

@ -1,11 +1,10 @@
use bytes::Bytes;
use fred::{clients::RedisPool, interfaces::HashesInterface};
use metrics::{counter, gauge};
use prost::Message;
use tracing::info;
use twilight_gateway::Event;
use libpk::{proto::*, util::redis::*};
use libpk::{state::*, util::redis::*};
#[derive(Clone)]
pub struct ShardStateManager {
@ -28,26 +27,24 @@ impl ShardStateManager {
}
async fn get_shard(&self, shard_id: u32) -> anyhow::Result<ShardState> {
let data: Option<Vec<u8>> = self
let data: Option<String> = self
.redis
.hget("pluralkit:shardstatus", shard_id)
.await
.to_option_or_error()?;
match data {
Some(buf) => {
Ok(ShardState::decode(buf.as_slice()).expect("could not decode shard data!"))
}
Some(buf) => Ok(serde_json::from_str(&buf).expect("could not decode shard data!")),
None => Ok(ShardState::default()),
}
}
async fn save_shard(&self, shard_id: u32, info: ShardState) -> anyhow::Result<()> {
self.redis
.hset::<(), &str, (String, Bytes)>(
.hset::<(), &str, (String, String)>(
"pluralkit:shardstatus",
(
shard_id.to_string(),
Bytes::copy_from_slice(&info.encode_to_vec()),
serde_json::to_string(&info).expect("could not serialize shard"),
),
)
.await?;