chore: merge avatars service into monorepo

lib/libpk/Cargo.toml

@@ -13,11 +13,13 @@ metrics = { workspace = true }
 metrics-exporter-prometheus = { version = "0.15.3", default-features = false, features = ["tokio", "http-listener", "tracing"] }
 serde = { workspace = true }
 sqlx = { workspace = true }
+time = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
 tracing-gelf = "0.7.1"
 tracing-subscriber = { workspace = true}
 twilight-model = { workspace = true }
+uuid = { workspace = true }
 
 prost = { workspace = true }
 prost-types = { workspace = true }

lib/libpk/src/_config.rs

@@ -20,6 +20,9 @@ pub struct DiscordConfig {
     pub max_concurrency: u32,
     pub cluster: Option<ClusterSettings>,
     pub api_base_url: Option<String>,
+
+    #[serde(default = "_default_api_addr")]
+    pub cache_api_addr: String,
 }
 
 #[derive(Deserialize, Debug)]
@@ -36,7 +39,7 @@ fn _default_api_addr() -> String {
     "0.0.0.0:5000".to_string()
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Clone, Debug)]
 pub struct ApiConfig {
     #[serde(default = "_default_api_addr")]
     pub addr: String,
@@ -50,6 +53,23 @@ pub struct ApiConfig {
     pub temp_token2: Option<String>,
 }
 
+#[derive(Deserialize, Clone, Debug)]
+pub struct AvatarsConfig {
+    pub s3: S3Config,
+    pub cdn_url: String,
+
+    #[serde(default)]
+    pub migrate_worker_count: u32,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct S3Config {
+    pub bucket: String,
+    pub application_id: String,
+    pub application_key: String,
+    pub endpoint: String,
+}
+
 fn _metrics_default() -> bool {
     false
 }
@@ -61,8 +81,9 @@ fn _json_log_default() -> bool {
 pub struct PKConfig {
     pub db: DatabaseConfig,
 
-    pub discord: DiscordConfig,
-    pub api: ApiConfig,
+    pub discord: Option<DiscordConfig>,
+    pub api: Option<ApiConfig>,
+    pub avatars: Option<AvatarsConfig>,
 
     #[serde(default = "_metrics_default")]
     pub run_metrics_server: bool,
@@ -71,6 +92,16 @@ pub struct PKConfig {
     pub(crate) json_log: bool,
 }
 
+impl PKConfig {
+    pub fn api(self) -> ApiConfig {
+        self.api.expect("missing api config")
+    }
+
+    pub fn discord_config(self) -> DiscordConfig {
+        self.discord.expect("missing discord config")
+    }
+}
+
 lazy_static! {
     #[derive(Debug)]
     pub static ref CONFIG: Arc<PKConfig> = {

lib/libpk/src/db/mod.rs

@@ -4,6 +4,7 @@ use std::str::FromStr;
 use tracing::info;
 
 pub mod repository;
+pub mod types;
 
 pub async fn init_redis() -> anyhow::Result<RedisPool> {
     info!("connecting to redis");

lib/libpk/src/db/repository/avatars.rs

@@ -0,0 +1,87 @@
+use sqlx::{PgPool, Postgres, Transaction};
+
+use crate::db::types::avatars::*;
+
+pub async fn get_by_original_url(
+    pool: &PgPool,
+    original_url: &str,
+) -> anyhow::Result<Option<ImageMeta>> {
+    Ok(
+        sqlx::query_as("select * from images where original_url = $1")
+            .bind(original_url)
+            .fetch_optional(pool)
+            .await?,
+    )
+}
+
+pub async fn get_by_attachment_id(
+    pool: &PgPool,
+    attachment_id: u64,
+) -> anyhow::Result<Option<ImageMeta>> {
+    Ok(
+        sqlx::query_as("select * from images where original_attachment_id = $1")
+            .bind(attachment_id as i64)
+            .fetch_optional(pool)
+            .await?,
+    )
+}
+
+pub async fn pop_queue(
+    pool: &PgPool,
+) -> anyhow::Result<Option<(Transaction<Postgres>, ImageQueueEntry)>> {
+    let mut tx = pool.begin().await?;
+    let res: Option<ImageQueueEntry> = sqlx::query_as("delete from image_queue where itemid = (select itemid from image_queue order by itemid for update skip locked limit 1) returning *")
+        .fetch_optional(&mut *tx).await?;
+    Ok(res.map(|x| (tx, x)))
+}
+
+pub async fn get_queue_length(pool: &PgPool) -> anyhow::Result<i64> {
+    Ok(sqlx::query_scalar("select count(*) from image_queue")
+        .fetch_one(pool)
+        .await?)
+}
+
+pub async fn get_stats(pool: &PgPool) -> anyhow::Result<Stats> {
+    Ok(sqlx::query_as(
+        "select count(*) as total_images, sum(file_size) as total_file_size from images",
+    )
+    .fetch_one(pool)
+    .await?)
+}
+
+pub async fn add_image(pool: &PgPool, meta: ImageMeta) -> anyhow::Result<bool> {
+    let kind_str = match meta.kind {
+        ImageKind::Avatar => "avatar",
+        ImageKind::Banner => "banner",
+    };
+
+    let res = sqlx::query("insert into images (id, url, content_type, original_url, file_size, width, height, original_file_size, original_type, original_attachment_id, kind, uploaded_by_account, uploaded_by_system, uploaded_at) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, (now() at time zone 'utc')) on conflict (id) do nothing")
+        .bind(meta.id)
+        .bind(meta.url)
+        .bind(meta.content_type)
+        .bind(meta.original_url)
+        .bind(meta.file_size)
+        .bind(meta.width)
+        .bind(meta.height)
+        .bind(meta.original_file_size)
+        .bind(meta.original_type)
+        .bind(meta.original_attachment_id)
+        .bind(kind_str)
+        .bind(meta.uploaded_by_account)
+        .bind(meta.uploaded_by_system)
+        .execute(pool).await?;
+    Ok(res.rows_affected() > 0)
+}
+
+pub async fn push_queue(
+    conn: &mut sqlx::PgConnection,
+    url: &str,
+    kind: ImageKind,
+) -> anyhow::Result<()> {
+    sqlx::query("insert into image_queue (url, kind) values ($1, $2)")
+        .bind(url)
+        .bind(kind)
+        .execute(conn)
+        .await?;
+    Ok(())
+}

lib/libpk/src/db/repository/mod.rs

@@ -1,5 +1,7 @@
 mod stats;
 pub use stats::*;
 
+pub mod avatars;
+
 mod auth;
 pub use auth::*;

lib/libpk/src/db/types/avatars.rs

@@ -0,0 +1,53 @@
+use serde::{Deserialize, Serialize};
+use sqlx::FromRow;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+#[derive(FromRow)]
+pub struct ImageMeta {
+    pub id: String,
+    pub kind: ImageKind,
+    pub content_type: String,
+    pub url: String,
+    pub file_size: i32,
+    pub width: i32,
+    pub height: i32,
+    pub uploaded_at: Option<OffsetDateTime>,
+
+    pub original_url: Option<String>,
+    pub original_attachment_id: Option<i64>,
+    pub original_file_size: Option<i32>,
+    pub original_type: Option<String>,
+    pub uploaded_by_account: Option<i64>,
+    pub uploaded_by_system: Option<Uuid>,
+}
+
+#[derive(FromRow, Serialize)]
+pub struct Stats {
+    pub total_images: i64,
+    pub total_file_size: i64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, sqlx::Type, PartialEq)]
+#[serde(rename_all = "snake_case")]
+#[sqlx(rename_all = "snake_case", type_name = "text")]
+pub enum ImageKind {
+    Avatar,
+    Banner,
+}
+
+impl ImageKind {
+    pub fn size(&self) -> (u32, u32) {
+        match self {
+            Self::Avatar => (512, 512),
+            Self::Banner => (1024, 1024),
+        }
+    }
+}
+
+#[derive(FromRow)]
+pub struct ImageQueueEntry {
+    pub itemid: i32,
+    pub url: String,
+    pub kind: ImageKind,
+}

lib/libpk/src/db/types/mod.rs

@@ -0,0 +1 @@
+pub mod avatars;

lib/libpk/src/lib.rs

@@ -1,5 +1,5 @@
 use metrics_exporter_prometheus::PrometheusBuilder;
-use tracing_subscriber::{EnvFilter, Registry};
+use tracing_subscriber::EnvFilter;
 
 pub mod db;
 pub mod proto;