chore: move migrations to rust

also adds some basic test seed data

Cargo.lock

@@ -2185,6 +2185,17 @@ dependencies = [
  "sketches-ddsketch",
 ]
 
+[[package]]
+name = "migrate"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "libpk",
+ "sqlx",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"

PluralKit.Bot/Init.cs

@@ -57,16 +57,6 @@ public class Init
 
             var cache = services.Resolve<IDiscordCache>();
 
-            if (config.Cluster == null)
-            {
-                // "Connect to the database" (ie. set off database migrations and ensure state)
-                logger.Information("Connecting to database");
-                await services.Resolve<IDatabase>().ApplyMigrations();
-
-                // Clear shard status from Redis
-                await redis.Connection.GetDatabase().KeyDeleteAsync("pluralkit:shardstatus");
-            }
-
             logger.Information("Initializing bot");
             var bot = services.Resolve<Bot>();
 

ci/Dockerfile.rust

@@ -25,6 +25,7 @@ COPY Cargo.lock /build/
 
 COPY crates/ /build/crates
 
+RUN cargo build --bin migrate --release --target x86_64-unknown-linux-musl
 RUN cargo build --bin api --release --target x86_64-unknown-linux-musl
 RUN cargo build --bin dispatch --release --target x86_64-unknown-linux-musl
 RUN cargo build --bin gateway --release --target x86_64-unknown-linux-musl
@@ -35,6 +36,7 @@ RUN cargo build --bin gdpr_worker --release --target x86_64-unknown-linux-musl
 
 FROM alpine:latest
 
+COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/migrate /migrate
 COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/api /api
 COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/dispatch /dispatch
 COPY --from=binary-builder /build/target/x86_64-unknown-linux-musl/release/gateway /gateway

ci/rust-docker-target.sh

@@ -37,6 +37,7 @@ EOF
 }
 
 # add rust binaries here to build
+build migrate
 build api
 build dispatch
 build gateway

crates/migrate/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "migrate"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+libpk = { path = "../libpk" }
+
+anyhow = { workspace = true }
+sqlx = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }

crates/migrate/build.rs

@@ -0,0 +1,55 @@
+use std::{
+    env,
+    error::Error,
+    fs::{self, File},
+    io::Write,
+    path::Path,
+};
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let out_dir = env::var("OUT_DIR")?;
+    let dest_path = Path::new(&out_dir).join("data.rs");
+    let mut datafile = File::create(&dest_path)?;
+
+    let prefix = "../../../../../../crates/migrate/data";
+
+    let ct = fs::read_dir("data/migrations")?
+        .filter(|p| {
+            p.as_ref()
+                .unwrap()
+                .file_name()
+                .into_string()
+                .unwrap()
+                .contains(".sql")
+        })
+        .count();
+
+    writeln!(&mut datafile, "const MIGRATIONS: [&'static str; {ct}] = [")?;
+    for idx in 0..ct {
+        writeln!(
+            &mut datafile,
+            "\tinclude_str!(\"{prefix}/migrations/{idx}.sql\"),"
+        )?;
+    }
+    writeln!(&mut datafile, "];\n")?;
+
+    writeln!(
+        &mut datafile,
+        "const CLEAN: &'static str = include_str!(\"{prefix}/clean.sql\");"
+    )?;
+    writeln!(
+        &mut datafile,
+        "const VIEWS: &'static str = include_str!(\"{prefix}/views.sql\");"
+    )?;
+    writeln!(
+        &mut datafile,
+        "const FUNCTIONS: &'static str = include_str!(\"{prefix}/functions.sql\");"
+    )?;
+
+    writeln!(
+        &mut datafile,
+        "const SEED: &'static str = include_str!(\"{prefix}/seed.sql\");"
+    )?;
+
+    Ok(())
+}

crates/migrate/data/seed.sql

@@ -0,0 +1,8 @@
+-- example data (for integration tests or such)
+
+insert into systems (hid, token) values (
+	'exmpl',
+	'vlPitT0tEgT++a450w1/afODy5NXdALcHDwryX6dOIZdGUGbZg+5IH3nrUsQihsw'
+);
+insert into system_config (system) values (1);
+insert into system_guild (system, guild) values (1, 466707357099884544);

crates/migrate/src/main.rs

@@ -0,0 +1,70 @@
+#![feature(let_chains)]
+
+use tracing::info;
+
+include!(concat!(env!("OUT_DIR"), "/data.rs"));
+
+#[libpk::main]
+async fn main() -> anyhow::Result<()> {
+    let db = libpk::db::init_data_db().await?;
+
+    // clean
+    // get current migration
+    // migrate to latest
+    // run views
+    // run functions
+
+    #[derive(sqlx::FromRow)]
+    struct CurrentMigration {
+        schema_version: i32,
+    }
+
+    let info = match sqlx::query_as("select schema_version from info")
+        .fetch_optional(&db)
+        .await
+    {
+        Ok(Some(result)) => result,
+        Ok(None) => CurrentMigration { schema_version: -1 },
+        Err(e) if format!("{e}").contains("relation \"info\" does not exist") => {
+            CurrentMigration { schema_version: -1 }
+        }
+        Err(e) => return Err(e.into()),
+    };
+
+    info!("current migration: {}", info.schema_version);
+
+    info!("running clean.sql");
+    sqlx::raw_sql(fix_feff(CLEAN)).execute(&db).await?;
+
+    for idx in (info.schema_version + 1) as usize..MIGRATIONS.len() {
+        info!("running migration {idx}");
+        sqlx::raw_sql(fix_feff(MIGRATIONS[idx as usize]))
+            .execute(&db)
+            .await?;
+    }
+
+    info!("running views.sql");
+    sqlx::raw_sql(fix_feff(VIEWS)).execute(&db).await?;
+
+    info!("running functions.sql");
+    sqlx::raw_sql(fix_feff(FUNCTIONS)).execute(&db).await?;
+
+    if let Ok(var) = std::env::var("SEED")
+        && var == "true"
+    {
+        info!("running seed.sql");
+        sqlx::raw_sql(fix_feff(SEED)).execute(&db).await?;
+        info!(
+            "example system created with hid 'exmpl', token 'vlPitT0tEgT++a450w1/afODy5NXdALcHDwryX6dOIZdGUGbZg+5IH3nrUsQihsw', guild_id 466707357099884544"
+        );
+    }
+
+    info!("all done!");
+
+    Ok(())
+}
+
+// some migration scripts have \u{feff} at the start
+fn fix_feff(sql: &str) -> &str {
+    sql.trim_start_matches("\u{feff}")
+}

docker-compose.yml

@@ -1,6 +1,17 @@
 version: "3"
 
 services:
+  migrate:
+    build:
+      context: .
+      dockerfile: ci/Dockerfile.rust
+    environment:
+      - RUST_LOG=info
+      - pluralkit__db__data_db_uri=postgresql://postgres:postgres@db:5432/postgres
+      - pluralkit__db__data_redis_addr=1
+    command: ["/migrate"]
+    depends_on: ["db"]
+
   bot:
     build:
       context: .
@@ -17,6 +28,9 @@ services:
       - "PluralKit__Bot__EventAwaiterTarget=http://bot:5002/events"
       - "PluralKit__Bot__DisableGateway=true"
     restart: unless-stopped
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
 
   gateway:
     build: