feat: add global component tag to rust json logs, add sentry, some cleanup

Cargo.lock

@@ -637,6 +637,16 @@ version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
 
+[[package]]
+name = "debugid"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
+dependencies = [
+ "serde",
+ "uuid",
+]
+
 [[package]]
 name = "der"
 version = "0.7.9"
@@ -815,6 +825,18 @@ dependencies = [
  "simd-adler32",
 ]
 
+[[package]]
+name = "findshlibs"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64"
+dependencies = [
+ "cc",
+ "lazy_static",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "fixedbitset"
 version = "0.4.2"
@@ -1251,6 +1273,17 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "hostname"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "windows",
+]
+
 [[package]]
 name = "http"
 version = "0.2.8"
@@ -1553,6 +1586,21 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "json-subscriber"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91d0a86fd2fba3a8721e7086b2c9fceb0994f71cdbd64ad2dfc1b202a5c062b4"
+dependencies = [
+ "serde",
+ "serde_json",
+ "tracing",
+ "tracing-core",
+ "tracing-serde",
+ "tracing-subscriber",
+ "uuid",
+]
+
 [[package]]
 name = "json5"
 version = "0.4.1"
@@ -1592,13 +1640,16 @@ dependencies = [
  "anyhow",
  "config",
  "fred",
+ "json-subscriber",
  "lazy_static",
  "metrics",
  "metrics-exporter-prometheus",
  "prost",
  "prost-build",
  "prost-types",
+ "sentry",
  "serde",
+ "serde_json",
  "sqlx",
  "time",
  "tokio",
@@ -1971,6 +2022,17 @@ dependencies = [
  "hashbrown 0.13.2",
 ]
 
+[[package]]
+name = "os_info"
+version = "3.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae99c7fa6dd38c7cafe1ec085e804f8f555a2f8659b0dbe03f1f9963a9b51092"
+dependencies = [
+ "log",
+ "serde",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -2518,6 +2580,7 @@ checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
 dependencies = [
  "base64 0.22.1",
  "bytes",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "http 1.1.0",
@@ -2692,6 +2755,15 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "0.38.34"
@@ -2735,6 +2807,7 @@ version = "0.22.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432"
 dependencies = [
+ "log",
  "ring 0.17.8",
  "rustls-pki-types",
  "rustls-webpki 0.102.4",
@@ -2748,6 +2821,7 @@ version = "0.23.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402"
 dependencies = [
+ "log",
  "once_cell",
  "ring 0.17.8",
  "rustls-pki-types",
@@ -2881,6 +2955,115 @@ version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
 
+[[package]]
+name = "sentry"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5484316556650182f03b43d4c746ce0e3e48074a21e2f51244b648b6542e1066"
+dependencies = [
+ "httpdate",
+ "reqwest 0.12.8",
+ "rustls 0.22.4",
+ "sentry-backtrace",
+ "sentry-contexts",
+ "sentry-core",
+ "sentry-debug-images",
+ "sentry-panic",
+ "sentry-tracing",
+ "tokio",
+ "ureq",
+ "webpki-roots 0.26.6",
+]
+
+[[package]]
+name = "sentry-backtrace"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40aa225bb41e2ec9d7c90886834367f560efc1af028f1c5478a6cce6a59c463a"
+dependencies = [
+ "backtrace",
+ "once_cell",
+ "regex",
+ "sentry-core",
+]
+
+[[package]]
+name = "sentry-contexts"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a8dd746da3d16cb8c39751619cefd4fcdbd6df9610f3310fd646b55f6e39910"
+dependencies = [
+ "hostname",
+ "libc",
+ "os_info",
+ "rustc_version",
+ "sentry-core",
+ "uname",
+]
+
+[[package]]
+name = "sentry-core"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "161283cfe8e99c8f6f236a402b9ccf726b201f365988b5bb637ebca0abbd4a30"
+dependencies = [
+ "once_cell",
+ "rand",
+ "sentry-types",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "sentry-debug-images"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc6b25e945fcaa5e97c43faee0267eebda9f18d4b09a251775d8fef1086238a"
+dependencies = [
+ "findshlibs",
+ "once_cell",
+ "sentry-core",
+]
+
+[[package]]
+name = "sentry-panic"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc74f229c7186dd971a9491ffcbe7883544aa064d1589bd30b83fb856cd22d63"
+dependencies = [
+ "sentry-backtrace",
+ "sentry-core",
+]
+
+[[package]]
+name = "sentry-tracing"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd3c5faf2103cd01eeda779ea439b68c4ee15adcdb16600836e97feafab362ec"
+dependencies = [
+ "sentry-backtrace",
+ "sentry-core",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
+[[package]]
+name = "sentry-types"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d68cdf6bc41b8ff3ae2a9c4671e97426dcdd154cc1d4b6b72813f285d6b163f"
+dependencies = [
+ "debugid",
+ "hex",
+ "rand",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "time",
+ "url",
+ "uuid",
+]
+
 [[package]]
 name = "serde"
 version = "1.0.203"
@@ -3931,6 +4114,15 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
 
+[[package]]
+name = "uname"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.10"
@@ -3982,6 +4174,21 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
+[[package]]
+name = "ureq"
+version = "2.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a"
+dependencies = [
+ "base64 0.22.1",
+ "log",
+ "once_cell",
+ "rustls 0.23.10",
+ "rustls-pki-types",
+ "url",
+ "webpki-roots 0.26.6",
+]
+
 [[package]]
 name = "url"
 version = "2.5.2"
@@ -3991,6 +4198,7 @@ dependencies = [
  "form_urlencoded",
  "idna 0.5.0",
  "percent-encoding",
+ "serde",
 ]
 
 [[package]]
@@ -4005,6 +4213,7 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
 dependencies = [
+ "getrandom",
  "serde",
 ]
 
@@ -4219,6 +4428,16 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
+dependencies = [
+ "windows-core",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-core"
 version = "0.52.0"

Cargo.toml

@@ -18,6 +18,7 @@ futures = "0.3.30"
 lazy_static = "1.4.0"
 metrics = "0.23.0"
 reqwest = { version = "0.12.7" , default-features = false, features = ["rustls-tls", "trust-dns"]}
+sentry = { version = "0.34.0", default-features = false, features =  ["backtrace", "contexts", "panic", "debug-images", "reqwest", "rustls"] } # replace native-tls with rustls
 serde = { version = "1.0.196", features = ["derive"] }
 serde_json = "1.0.117"
 signal-hook = "0.3.17"

lib/libpk/Cargo.toml

@@ -10,7 +10,9 @@ lazy_static = { workspace = true }
 metrics = { workspace = true }
 prost = { workspace = true }
 prost-types = { workspace = true }
+sentry = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 sqlx = { workspace = true }
 time = { workspace = true }
 tokio = { workspace = true }
@@ -20,6 +22,7 @@ twilight-model = { workspace = true }
 uuid = { workspace = true }
 
 config = "0.14.0"
+json-subscriber = { version = "0.2.2", features = ["env-filter"] }
 metrics-exporter-prometheus = { version = "0.15.3", default-features = false, features = ["tokio", "http-listener", "tracing"] }
 
 [build-dependencies]

lib/libpk/src/_config.rs

@@ -102,6 +102,9 @@ pub struct PKConfig {
 
     #[serde(default = "_json_log_default")]
     pub(crate) json_log: bool,
+
+    #[serde(default)]
+    pub sentry_url: Option<String>,
 }
 
 impl PKConfig {

lib/libpk/src/lib.rs

@@ -1,6 +1,7 @@
 #![feature(let_chains)]
 use metrics_exporter_prometheus::PrometheusBuilder;
-use tracing_subscriber::EnvFilter;
+use sentry::IntoDsn;
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
 pub mod db;
 pub mod proto;
@@ -9,12 +10,18 @@ pub mod util;
 pub mod _config;
 pub use crate::_config::CONFIG as config;
 
+// functions in this file are only used by the main function below
+
 pub fn init_logging(component: &str) -> anyhow::Result<()> {
-    // todo: fix component
     if config.json_log {
-        tracing_subscriber::fmt()
+        let mut layer = json_subscriber::layer();
-            .json()
+        layer.inner_layer_mut().add_static_field(
-            .with_env_filter(EnvFilter::from_default_env())
+            "component",
+            serde_json::Value::String(component.to_string()),
+        );
+        tracing_subscriber::registry()
+            .with(layer)
+            .with(EnvFilter::from_default_env())
             .init();
     } else {
         tracing_subscriber::fmt()
@@ -33,3 +40,40 @@ pub fn init_metrics() -> anyhow::Result<()> {
     }
     Ok(())
 }
+
+pub fn init_sentry() -> sentry::ClientInitGuard {
+    sentry::init(sentry::ClientOptions {
+        dsn: config
+            .sentry_url
+            .clone()
+            .map(|u| u.into_dsn().unwrap())
+            .flatten(),
+        release: sentry::release_name!(),
+        ..Default::default()
+    })
+}
+
+#[macro_export]
+macro_rules! main {
+    ($component:expr) => {
+        fn main() -> anyhow::Result<()> {
+            let _sentry_guard = libpk::init_sentry();
+            // we might also be able to use env!("CARGO_CRATE_NAME") here
+            libpk::init_logging($component)?;
+            tokio::runtime::Builder::new_multi_thread()
+                .enable_all()
+                .build()
+                .unwrap()
+                .block_on(async {
+                    if let Err(err) = libpk::init_metrics() {
+                        tracing::error!("failed to init metrics collector: {err}");
+                    };
+                    tracing::info!("hello world");
+                    if let Err(err) = real_main().await {
+                        tracing::error!("failed to run service: {err}");
+                    };
+                });
+            Ok(())
+        }
+    };
+}

services/api/src/main.rs

@@ -54,29 +54,9 @@ async fn rproxy(
 
 // this function is manually formatted for easier legibility of route_services
 #[rustfmt::skip]
-#[tokio::main]
+fn router(ctx: ApiContext) -> Router {
-async fn main() -> anyhow::Result<()> {
-    libpk::init_logging("api")?;
-    libpk::init_metrics()?;
-    info!("hello world");
-
-    let db = libpk::db::init_data_db().await?;
-    let redis = libpk::db::init_redis().await?;
-
-    let rproxy_uri = Uri::from_static(&libpk::config.api.as_ref().expect("missing api config").remote_url).to_string();
-    let rproxy_client = hyper_util::client::legacy::Client::<(), ()>::builder(TokioExecutor::new())
-            .build(HttpConnector::new());
-
-    let ctx = ApiContext {
-        db,
-        redis,
-
-        rproxy_uri: rproxy_uri[..rproxy_uri.len() - 1].to_string(),
-        rproxy_client,
-     };
-
     // processed upside down (???) so we have to put middleware at the end
-    let app = Router::new()
+    Router::new()
         .route("/v2/systems/:system_id", get(rproxy))
         .route("/v2/systems/:system_id", patch(rproxy))
         .route("/v2/systems/:system_id/settings", get(rproxy))
@@ -143,9 +123,42 @@ async fn main() -> anyhow::Result<()> {
 
         .with_state(ctx)
 
-        .route("/", get(|| async { axum::response::Redirect::to("https://pluralkit.me/api") }));
+        .route("/", get(|| async { axum::response::Redirect::to("https://pluralkit.me/api") }))
+}
+
+libpk::main!("api");
+async fn real_main() -> anyhow::Result<()> {
+    let db = libpk::db::init_data_db().await?;
+    let redis = libpk::db::init_redis().await?;
+
+    let rproxy_uri = Uri::from_static(
+        &libpk::config
+            .api
+            .as_ref()
+            .expect("missing api config")
+            .remote_url,
+    )
+    .to_string();
+    let rproxy_client = hyper_util::client::legacy::Client::<(), ()>::builder(TokioExecutor::new())
+        .build(HttpConnector::new());
+
+    let ctx = ApiContext {
+        db,
+        redis,
+
+        rproxy_uri: rproxy_uri[..rproxy_uri.len() - 1].to_string(),
+        rproxy_client,
+    };
+
+    let app = router(ctx);
+
+    let addr: &str = libpk::config
+        .api
+        .as_ref()
+        .expect("missing api config")
+        .addr
+        .as_ref();
 
-    let addr: &str = libpk::config.api.as_ref().expect("missing api config").addr.as_ref();
     let listener = tokio::net::TcpListener::bind(addr).await?;
     info!("listening on {}", addr);
     axum::serve(listener, app).await?;

services/avatars/src/cleanup.rs

@@ -4,12 +4,8 @@ use sqlx::prelude::FromRow;
 use std::{sync::Arc, time::Duration};
 use tracing::{error, info};
 
-#[tokio::main]
+libpk::main!("avatar_cleanup");
-async fn main() -> anyhow::Result<()> {
+async fn real_main() -> anyhow::Result<()> {
-    libpk::init_logging("avatar_cleanup")?;
-    libpk::init_metrics()?;
-    info!("hello world");
-
     let config = libpk::config
         .avatars
         .as_ref()
@@ -129,7 +125,7 @@ async fn cleanup_job(pool: sqlx::PgPool, bucket: Arc<s3::Bucket>) -> anyhow::Res
             }
             _ => {
                 let status = cf_resp.status();
-                println!("{:#?}", cf_resp.text().await?);
+                tracing::info!("raw response from cloudflare: {:#?}", cf_resp.text().await?);
                 anyhow::bail!("cloudflare returned bad error code {}", status);
             }
         }

services/avatars/src/main.rs

@@ -142,12 +142,8 @@ pub struct AppState {
     config: Arc<AvatarsConfig>,
 }
 
-#[tokio::main]
+libpk::main!("avatars");
-async fn main() -> anyhow::Result<()> {
+async fn real_main() -> anyhow::Result<()> {
-    libpk::init_logging("avatars")?;
-    libpk::init_metrics()?;
-    info!("hello world");
-
     let config = libpk::config
         .avatars
         .as_ref()

services/dispatch/src/main.rs

@@ -19,6 +19,8 @@ use axum::{extract::State, http::Uri, routing::post, Json, Router};
 
 mod logger;
 
+// this package does not currently use libpk
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     tracing_subscriber::fmt()

services/gateway/src/cache_api.rs

@@ -47,7 +47,6 @@ pub async fn run_server(cache: Arc<DiscordCache>) -> anyhow::Result<()> {
             get(|State(cache): State<Arc<DiscordCache>>, Path(guild_id): Path<u64>| async move {
                 match cache.guild_permissions(Id::new(guild_id), libpk::config.discord.as_ref().expect("missing discord config").client_id).await {
                     Ok(val) => {
-                        println!("hh {}", Permissions::all().bits());
                         status_code(StatusCode::FOUND, to_string(&val.bits()).unwrap())
                     },
                     Err(err) => {

services/gateway/src/main.rs

@@ -18,12 +18,8 @@ mod cache_api;
 mod discord;
 mod logger;
 
-#[tokio::main]
+libpk::main!("gateway");
-async fn main() -> anyhow::Result<()> {
+async fn real_main() -> anyhow::Result<()> {
-    libpk::init_logging("gateway")?;
-    libpk::init_metrics()?;
-    info!("hello world");
-
     let (shutdown_tx, shutdown_rx) = channel::<()>();
     let shutdown_tx = Arc::new(shutdown_tx);