eightball/PluralKit
1
0
Fork 0
mirror of https://github.com/PluralKit/PluralKit.git synced 2026-02-15 10:10:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

WIP: revise avatars service
Some checks failed
Build and push Docker image / .net docker build (push) Has been cancelled
Details
.net checks / run .net tests (push) Has been cancelled
Details
.net checks / dotnet-format (push) Has been cancelled
Details
Build and push Rust service Docker images / rust docker build (push) Has been cancelled
Details
rust checks / cargo fmt (push) Has been cancelled
Details

Browse source
This commit is contained in:
asleepyskye 2026-01-24 11:43:05 -05:00
parent 0a474c43eb
commit f69587ceaf
26 changed files with 912 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crates/scheduled_tasks/src/main.rs
View file

@ -105,6 +105,12 @@ async fn main() -> anyhow::Result<()> {
"queue deleted image cleanup job",
queue_deleted_image_cleanup
);
// on hh:15 and hh:45
doforever!(
"15,45 * * * *",
"queue orphaned hash cleanup job",
queue_orphaned_hash_cleanup
);
// non-standard cron: at hh:mm:00, hh:mm:30
doforever!("0,30 * * * * *", "stats api updater", update_stats_api);
// every hour (could probably even be less frequent, basebackups are taken rarely)

114
crates/scheduled_tasks/src/tasks.rs
View file

@ -228,32 +228,44 @@ pub async fn update_discord_stats(ctx: AppCtx) -> anyhow::Result<()> {
Ok(())
}
const IMAGE_CHECK_COLUMNS: &[(&str, &str)] = &[
("systems", "avatar_url"),
("systems", "banner_image"),
("system_guild", "avatar_url"),
("members", "avatar_url"),
("members", "banner_image"),
("members", "webhook_avatar_url"),
("member_guild", "avatar_url"),
("groups", "icon"),
("groups", "banner_image"),
];
pub async fn queue_deleted_image_cleanup(ctx: AppCtx) -> anyhow::Result<()> {
// if an image is present on no member, add it to the pending deletion queue
// if it is still present on no member after 24h, actually delete it
let usage_query = r#"
and not exists (select from systems where avatar_url = images.url)
and not exists (select from systems where banner_image = images.url)
and not exists (select from system_guild where avatar_url = images.url)
and not exists (select from members where avatar_url = images.url)
and not exists (select from members where banner_image = images.url)
and not exists (select from members where webhook_avatar_url = images.url)
and not exists (select from member_guild where avatar_url = images.url)
and not exists (select from groups where icon = images.url)
and not exists (select from groups where banner_image = images.url);
"#;
let mut usage_query = String::new();
for (table, col) in IMAGE_CHECK_COLUMNS {
usage_query.push_str(&format!(
r#"
and not exists (
select 1 from {table}
where {col} = h.url
or {col} like '%/' || a.system_uuid::text || '/' || a.id::text || '.%'
)
"#
));
}
ctx.data
.execute(
format!(
r#"
insert into image_cleanup_pending_jobs
select id, now() from images where
not exists (select from image_cleanup_pending_jobs j where j.id = images.id)
and not exists (select from image_cleanup_jobs j where j.id = images.id)
select a.id, a.system_uuid, now() from images_assets a
join images_hashes h on a.image = h.hash where
a.kind not in ('premium_banner', 'premium_avatar')
and not exists (select from image_cleanup_pending_jobs j where j.id = a.id)
and not exists (select from image_cleanup_jobs j where j.id = a.id)
{}
"#,
usage_query
@ -266,12 +278,14 @@ pub async fn queue_deleted_image_cleanup(ctx: AppCtx) -> anyhow::Result<()> {
.execute(
format!(
r#"
insert into image_cleanup_jobs
select image_cleanup_pending_jobs.id from image_cleanup_pending_jobs
left join images on images.id = image_cleanup_pending_jobs.id
insert into image_cleanup_jobs (id, system_uuid)
select p.id, p.system_uuid from image_cleanup_pending_jobs p
join images_assets a on a.id = p.id
join images_hashes h on a.image = h.hash
where
ts < now() - '24 hours'::interval
and not exists (select from image_cleanup_jobs j where j.id = images.id)
a.kind not in ('premium_banner', 'premium_avatar')
and ts < now() - '24 hours'::interval
and not exists (select from image_cleanup_jobs j where j.id = p.id)
{}
"#,
usage_query
@ -283,6 +297,62 @@ pub async fn queue_deleted_image_cleanup(ctx: AppCtx) -> anyhow::Result<()> {
Ok(())
}
pub async fn queue_orphaned_hash_cleanup(ctx: AppCtx) -> anyhow::Result<()> {
let mut usage_checks = String::new();
for (table, col) in IMAGE_CHECK_COLUMNS {
usage_checks.push_str(&format!(
"and not exists (select 1 from {table} where {col} = h.url) "
));
}
ctx.data
.execute(
format!(
r#"
insert into image_hash_cleanup_pending_jobs (hash, ts)
select h.hash, now()
from images_hashes h
where
not exists (
select 1 from images_assets a
where a.image = h.hash
or a.proxy_image = h.hash
)
{usage_checks}
and not exists (select 1 from image_hash_cleanup_pending_jobs p where p.hash = h.hash)
and not exists (select 1 from image_hash_cleanup_jobs j where j.hash = h.hash)
"#
)
.as_str(),
)
.await?;
ctx.data
.execute(
format!(
r#"
insert into image_hash_cleanup_jobs (hash)
select p.hash
from image_hash_cleanup_pending_jobs p
join images_hashes h ON h.hash = p.hash
where
p.ts < now() - '24 hours'::interval
and not exists (
select 1 from images_assets a
where a.image = h.hash
or a.proxy_image = h.hash
)
{usage_checks}
and not exists (select 1 from image_hash_cleanup_jobs j where j.hash = p.hash)
"#
)
.as_str(),
)
.await?;
Ok(())
}
pub async fn update_stats_api(ctx: AppCtx) -> anyhow::Result<()> {
let client = ClientBuilder::new()
.connect_timeout(Duration::from_secs(3))