From 1f7b77873c6c724048d1d4cc0543f3e7c22607f6 Mon Sep 17 00:00:00 2001 From: yusufipk Date: Sun, 26 Jul 2026 19:08:03 +0700 Subject: [PATCH] fix(scripts): make the orphan cleanups reviewable before they delete Two problems with running these unattended, both found while wiring the Bunny cleanup up to a Coolify scheduled task against production. A dry run reported a count and nothing else. "Orphaned: 31" is not something anyone can approve: it says how many objects would go, never which. Both scripts now list every orphan they would delete, and print the same list when deleting, so a real run is auditable afterwards too. Each line carries who the object belongs to, as far as each provider can answer: - R2 reads the owner out of `videoUploadSession`, which keeps `objectKey` alongside the initiating and billed user and survives an upload that never became a video. That is the case producing orphans, so this is an answer rather than a guess. - Bunny has no equivalent. `bunny-init` sends the provider a title and nothing else, and an orphan by definition has no row pointing at it, so there is nothing authoritative to look up. The title is matched against titles still in the database instead, which catches the common shape (a version upload that failed and was retried successfully leaves a live row with the same title). A hit prints as "possibly", because it is a hint. The grace periods were also too short to be safe: - Bunny counted a video abandoned after 24 hours. - R2 counted an object abandoned after 15 minutes, which is shorter than a slow multipart upload of a large file. An object still being written, or written but not yet finalised into a row, looked abandoned and could be deleted out from under the upload creating it. Both are seven days now: long enough that no upload, retry or delayed finalisation can still be in flight. --- scripts/bunny-orphan-cleanup.ts | 112 +++++++++++++++++++++++++++++++- scripts/r2-orphan-cleanup.ts | 76 +++++++++++++++++++--- 2 files changed, 178 insertions(+), 10 deletions(-) diff --git a/scripts/bunny-orphan-cleanup.ts b/scripts/bunny-orphan-cleanup.ts index eb1a39c..7f7c5aa 100644 --- a/scripts/bunny-orphan-cleanup.ts +++ b/scripts/bunny-orphan-cleanup.ts @@ -7,7 +7,10 @@ const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; const ITEMS_PER_PAGE = 100; const MAX_PAGES = 200; const CHUNK_SIZE = 500; -const DEFAULT_GRACE_HOURS = 24; +// Seven days, not one. A video only counts as abandoned once nothing has claimed it for +// long enough that no upload, retry or delayed finalisation could still be in flight, and +// a day is short enough that an upload interrupted overnight looks abandoned by morning. +const DEFAULT_GRACE_HOURS = 7 * 24; type BunnyConfig = { apiKey: string; @@ -17,6 +20,7 @@ type BunnyConfig = { type BunnyVideo = { id: string; uploadedAt: Date; + title: string | null; }; function getBunnyConfig(): BunnyConfig { @@ -51,6 +55,17 @@ function parseVideoId(item: unknown): string | null { return null; } +function parseTitle(item: unknown): string | null { + const record = toRecord(item); + if (!record) return null; + + for (const candidate of [record.title, record.Title]) { + if (typeof candidate === 'string' && candidate.trim()) return candidate.trim(); + } + + return null; +} + function parseUploadedAt(item: unknown): Date | null { const record = toRecord(item); if (!record) return null; @@ -133,7 +148,7 @@ async function listBunnyVideos( skippedInvalid += 1; continue; } - videos.push({ id, uploadedAt }); + videos.push({ id, uploadedAt, title: parseTitle(item) }); } if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break; @@ -182,6 +197,73 @@ async function findReferencedVideoIds(videoIds: string[]): Promise> return referenced; } +/** + * Best-effort owner for an orphan, by matching its Bunny title against titles still in the + * database. + * + * An orphan is by definition a video nothing in the database points at, so there is no + * authoritative owner to look up: `bunny-init` sends Bunny a title and nothing else, no + * user id and no email. What is left is the title, and it is worth matching because the + * common way an orphan appears is a version upload that failed and was then retried + * successfully, which leaves a live row carrying the same title. + * + * A hit is therefore a hint, not a fact, and the output says so. A miss means the video + * cannot be attributed at all from what Bunny and the database hold today. + */ +async function findOwnerHintsByTitle(titles: string[]): Promise> { + const hints = new Map>(); + const unique = [...new Set(titles.filter((title): title is string => Boolean(title)))]; + + const remember = (title: string | null, emails: Array) => { + if (!title) return; + const existing = hints.get(title) ?? new Set(); + for (const email of emails) { + if (email) existing.add(email); + } + if (existing.size > 0) hints.set(title, existing); + }; + + const ownerSelect = { + project: { + select: { + owner: { select: { email: true } }, + workspace: { select: { owner: { select: { email: true } } } }, + }, + }, + } as const; + + for (const group of chunk(unique, CHUNK_SIZE)) { + const [videos, versions, assets] = await Promise.all([ + db.video.findMany({ + where: { title: { in: group } }, + select: { title: true, ...ownerSelect }, + }), + db.videoVersion.findMany({ + where: { title: { in: group } }, + select: { title: true, video: { select: ownerSelect } }, + }), + db.videoAsset.findMany({ + where: { displayName: { in: group } }, + select: { displayName: true, video: { select: ownerSelect } }, + }), + ]); + + for (const row of videos) { + remember(row.title, [row.project.owner?.email, row.project.workspace.owner?.email]); + } + for (const row of versions) { + const project = row.video.project; + remember(row.title, [project.owner?.email, project.workspace.owner?.email]); + } + for (const row of assets) { + const project = row.video.project; + remember(row.displayName, [project.owner?.email, project.workspace.owner?.email]); + } + } + + return new Map([...hints].map(([title, emails]) => [title, [...emails].sort()])); +} + async function deleteBunnyVideo( config: BunnyConfig, videoId: string @@ -238,6 +320,32 @@ async function main() { const referenced = await findReferencedVideoIds(eligibleIds); const orphanIds = eligibleIds.filter((id) => !referenced.has(id)); + + // A dry run that only reports a count cannot be acted on: the point of it is to see + // what would go before anything does. Deleting prints the same list, so a real run is + // auditable after the fact too. + if (orphanIds.length > 0) { + const byId = new Map(eligible.map((video) => [video.id, video])); + const orphanTitles = orphanIds + .map((orphanId) => byId.get(orphanId)?.title) + .filter((title): title is string => Boolean(title)); + const ownerHints = await findOwnerHintsByTitle(orphanTitles); + + console.log( + `[bunny-orphan-cleanup] Orphans ${dryRun ? 'that would be deleted' : 'to delete'}:` + ); + for (const orphanId of orphanIds) { + const video = byId.get(orphanId); + const uploadedAt = video?.uploadedAt.toISOString() ?? 'unknown date'; + const title = video?.title ?? 'untitled'; + const emails = video?.title ? (ownerHints.get(video.title) ?? []) : []; + // "possibly" is load-bearing: this is a title match, not a stored owner. + const owner = + emails.length > 0 ? `possibly ${emails.join(', ')}` : 'owner unknown (no title match)'; + console.log(`[bunny-orphan-cleanup] ${orphanId} ${uploadedAt} ${title} ${owner}`); + } + } + let deleted = 0; let alreadyMissing = 0; let failed = 0; diff --git a/scripts/r2-orphan-cleanup.ts b/scripts/r2-orphan-cleanup.ts index c92bf0b..99dde3b 100644 --- a/scripts/r2-orphan-cleanup.ts +++ b/scripts/r2-orphan-cleanup.ts @@ -8,7 +8,12 @@ import { r2Client, R2_BUCKET_NAME } from '../lib/r2'; import { cleanupExpiredBillingWorkspaces } from './expired-billing-cleanup'; import { logError } from '@/lib/logger'; -const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000; +// Seven days. This was fifteen minutes, which is shorter than a slow multipart upload of +// a large file: an object still being written, or written but not yet finalised into a +// row, looked abandoned and could be deleted out from under the upload that was creating +// it. An object only counts as abandoned once nothing has claimed it for long enough that +// no upload, retry or delayed finalisation could still be in flight. +const UNATTACHED_UPLOAD_TTL_MS = 7 * 24 * 60 * 60 * 1000; const CHUNK_SIZE = 500; const PREFIXES = ['images/', 'voice/', 'videos/'] as const; @@ -40,6 +45,50 @@ function keyToProxyUrl(key: string): string | null { return null; } +/** + * The owner of each orphaned object key, from the upload session that created it. + * + * Unlike the Bunny side, this is an answer rather than a guess: `videoUploadSession` keeps + * `objectKey` alongside the user who initiated the upload, and the row survives even when + * the upload never became a video, which is exactly the case that produces an orphan. + * Both the initiating user and the billed user are reported when they differ, because the + * billed one is who paid for the bytes. + */ +async function findUploadSessionOwners(keys: string[]): Promise> { + const owners = new Map(); + + for (const group of chunk(keys, CHUNK_SIZE)) { + // VideoUploadSession holds the ids but declares no relation to User, so the addresses + // are resolved in a second query rather than by widening the schema for a script. + const sessions = await db.videoUploadSession.findMany({ + where: { objectKey: { in: group } }, + select: { objectKey: true, status: true, userId: true, billedUserId: true }, + }); + if (sessions.length === 0) continue; + + const userIds = [ + ...new Set(sessions.flatMap((session) => [session.userId, session.billedUserId])), + ]; + const users = await db.user.findMany({ + where: { id: { in: userIds } }, + select: { id: true, email: true }, + }); + const emailById = new Map(users.map((user) => [user.id, user.email])); + + for (const session of sessions) { + const initiator = emailById.get(session.userId) ?? null; + const billed = emailById.get(session.billedUserId) ?? null; + const who = + billed && billed !== initiator + ? `${initiator ?? 'unknown'} (billed: ${billed})` + : (initiator ?? billed ?? 'unknown'); + owners.set(session.objectKey, `${who} [session ${session.status.toLowerCase()}]`); + } + } + + return owners; +} + function chunk(items: T[], size: number): T[][] { const out: T[][] = []; for (let i = 0; i < items.length; i += size) { @@ -181,16 +230,27 @@ async function main() { let deleted = 0; let failed = 0; - let orphaned = 0; - let referencedCount = 0; - for (const candidate of candidates) { - if (referenced.has(candidate.url)) { - referencedCount += 1; - continue; + // A dry run that only reports a count cannot be acted on: the point of it is to see + // what would go before anything does. Deleting prints the same list, so a real run is + // auditable after the fact too. + const orphanCandidates = candidates.filter((candidate) => !referenced.has(candidate.url)); + const referencedCount = candidates.length - orphanCandidates.length; + const orphaned = orphanCandidates.length; + + if (orphanCandidates.length > 0) { + const owners = await findUploadSessionOwners(orphanCandidates.map((c) => c.key)); + + console.log(`[r2-orphan-cleanup] Orphans ${dryRun ? 'that would be deleted' : 'to delete'}:`); + for (const candidate of orphanCandidates) { + const email = owners.get(candidate.key); + console.log( + `[r2-orphan-cleanup] ${candidate.key} ${email ?? 'owner unknown (no upload session)'}` + ); } + } - orphaned += 1; + for (const candidate of orphanCandidates) { if (dryRun) continue; try {