diff --git a/scripts/bunny-orphan-cleanup.ts b/scripts/bunny-orphan-cleanup.ts index eb1a39c..7f7c5aa 100644 --- a/scripts/bunny-orphan-cleanup.ts +++ b/scripts/bunny-orphan-cleanup.ts @@ -7,7 +7,10 @@ const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; const ITEMS_PER_PAGE = 100; const MAX_PAGES = 200; const CHUNK_SIZE = 500; -const DEFAULT_GRACE_HOURS = 24; +// Seven days, not one. A video only counts as abandoned once nothing has claimed it for +// long enough that no upload, retry or delayed finalisation could still be in flight, and +// a day is short enough that an upload interrupted overnight looks abandoned by morning. +const DEFAULT_GRACE_HOURS = 7 * 24; type BunnyConfig = { apiKey: string; @@ -17,6 +20,7 @@ type BunnyConfig = { type BunnyVideo = { id: string; uploadedAt: Date; + title: string | null; }; function getBunnyConfig(): BunnyConfig { @@ -51,6 +55,17 @@ function parseVideoId(item: unknown): string | null { return null; } +function parseTitle(item: unknown): string | null { + const record = toRecord(item); + if (!record) return null; + + for (const candidate of [record.title, record.Title]) { + if (typeof candidate === 'string' && candidate.trim()) return candidate.trim(); + } + + return null; +} + function parseUploadedAt(item: unknown): Date | null { const record = toRecord(item); if (!record) return null; @@ -133,7 +148,7 @@ async function listBunnyVideos( skippedInvalid += 1; continue; } - videos.push({ id, uploadedAt }); + videos.push({ id, uploadedAt, title: parseTitle(item) }); } if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break; @@ -182,6 +197,73 @@ async function findReferencedVideoIds(videoIds: string[]): Promise> return referenced; } +/** + * Best-effort owner for an orphan, by matching its Bunny title against titles still in the + * database. + * + * An orphan is by definition a video nothing in the database points at, so there is no + * authoritative owner to look up: `bunny-init` sends Bunny a title and nothing else, no + * user id and no email. What is left is the title, and it is worth matching because the + * common way an orphan appears is a version upload that failed and was then retried + * successfully, which leaves a live row carrying the same title. + * + * A hit is therefore a hint, not a fact, and the output says so. A miss means the video + * cannot be attributed at all from what Bunny and the database hold today. + */ +async function findOwnerHintsByTitle(titles: string[]): Promise> { + const hints = new Map>(); + const unique = [...new Set(titles.filter((title): title is string => Boolean(title)))]; + + const remember = (title: string | null, emails: Array) => { + if (!title) return; + const existing = hints.get(title) ?? new Set(); + for (const email of emails) { + if (email) existing.add(email); + } + if (existing.size > 0) hints.set(title, existing); + }; + + const ownerSelect = { + project: { + select: { + owner: { select: { email: true } }, + workspace: { select: { owner: { select: { email: true } } } }, + }, + }, + } as const; + + for (const group of chunk(unique, CHUNK_SIZE)) { + const [videos, versions, assets] = await Promise.all([ + db.video.findMany({ + where: { title: { in: group } }, + select: { title: true, ...ownerSelect }, + }), + db.videoVersion.findMany({ + where: { title: { in: group } }, + select: { title: true, video: { select: ownerSelect } }, + }), + db.videoAsset.findMany({ + where: { displayName: { in: group } }, + select: { displayName: true, video: { select: ownerSelect } }, + }), + ]); + + for (const row of videos) { + remember(row.title, [row.project.owner?.email, row.project.workspace.owner?.email]); + } + for (const row of versions) { + const project = row.video.project; + remember(row.title, [project.owner?.email, project.workspace.owner?.email]); + } + for (const row of assets) { + const project = row.video.project; + remember(row.displayName, [project.owner?.email, project.workspace.owner?.email]); + } + } + + return new Map([...hints].map(([title, emails]) => [title, [...emails].sort()])); +} + async function deleteBunnyVideo( config: BunnyConfig, videoId: string @@ -238,6 +320,32 @@ async function main() { const referenced = await findReferencedVideoIds(eligibleIds); const orphanIds = eligibleIds.filter((id) => !referenced.has(id)); + + // A dry run that only reports a count cannot be acted on: the point of it is to see + // what would go before anything does. Deleting prints the same list, so a real run is + // auditable after the fact too. + if (orphanIds.length > 0) { + const byId = new Map(eligible.map((video) => [video.id, video])); + const orphanTitles = orphanIds + .map((orphanId) => byId.get(orphanId)?.title) + .filter((title): title is string => Boolean(title)); + const ownerHints = await findOwnerHintsByTitle(orphanTitles); + + console.log( + `[bunny-orphan-cleanup] Orphans ${dryRun ? 'that would be deleted' : 'to delete'}:` + ); + for (const orphanId of orphanIds) { + const video = byId.get(orphanId); + const uploadedAt = video?.uploadedAt.toISOString() ?? 'unknown date'; + const title = video?.title ?? 'untitled'; + const emails = video?.title ? (ownerHints.get(video.title) ?? []) : []; + // "possibly" is load-bearing: this is a title match, not a stored owner. + const owner = + emails.length > 0 ? `possibly ${emails.join(', ')}` : 'owner unknown (no title match)'; + console.log(`[bunny-orphan-cleanup] ${orphanId} ${uploadedAt} ${title} ${owner}`); + } + } + let deleted = 0; let alreadyMissing = 0; let failed = 0; diff --git a/scripts/r2-orphan-cleanup.ts b/scripts/r2-orphan-cleanup.ts index c92bf0b..99dde3b 100644 --- a/scripts/r2-orphan-cleanup.ts +++ b/scripts/r2-orphan-cleanup.ts @@ -8,7 +8,12 @@ import { r2Client, R2_BUCKET_NAME } from '../lib/r2'; import { cleanupExpiredBillingWorkspaces } from './expired-billing-cleanup'; import { logError } from '@/lib/logger'; -const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000; +// Seven days. This was fifteen minutes, which is shorter than a slow multipart upload of +// a large file: an object still being written, or written but not yet finalised into a +// row, looked abandoned and could be deleted out from under the upload that was creating +// it. An object only counts as abandoned once nothing has claimed it for long enough that +// no upload, retry or delayed finalisation could still be in flight. +const UNATTACHED_UPLOAD_TTL_MS = 7 * 24 * 60 * 60 * 1000; const CHUNK_SIZE = 500; const PREFIXES = ['images/', 'voice/', 'videos/'] as const; @@ -40,6 +45,50 @@ function keyToProxyUrl(key: string): string | null { return null; } +/** + * The owner of each orphaned object key, from the upload session that created it. + * + * Unlike the Bunny side, this is an answer rather than a guess: `videoUploadSession` keeps + * `objectKey` alongside the user who initiated the upload, and the row survives even when + * the upload never became a video, which is exactly the case that produces an orphan. + * Both the initiating user and the billed user are reported when they differ, because the + * billed one is who paid for the bytes. + */ +async function findUploadSessionOwners(keys: string[]): Promise> { + const owners = new Map(); + + for (const group of chunk(keys, CHUNK_SIZE)) { + // VideoUploadSession holds the ids but declares no relation to User, so the addresses + // are resolved in a second query rather than by widening the schema for a script. + const sessions = await db.videoUploadSession.findMany({ + where: { objectKey: { in: group } }, + select: { objectKey: true, status: true, userId: true, billedUserId: true }, + }); + if (sessions.length === 0) continue; + + const userIds = [ + ...new Set(sessions.flatMap((session) => [session.userId, session.billedUserId])), + ]; + const users = await db.user.findMany({ + where: { id: { in: userIds } }, + select: { id: true, email: true }, + }); + const emailById = new Map(users.map((user) => [user.id, user.email])); + + for (const session of sessions) { + const initiator = emailById.get(session.userId) ?? null; + const billed = emailById.get(session.billedUserId) ?? null; + const who = + billed && billed !== initiator + ? `${initiator ?? 'unknown'} (billed: ${billed})` + : (initiator ?? billed ?? 'unknown'); + owners.set(session.objectKey, `${who} [session ${session.status.toLowerCase()}]`); + } + } + + return owners; +} + function chunk(items: T[], size: number): T[][] { const out: T[][] = []; for (let i = 0; i < items.length; i += size) { @@ -181,16 +230,27 @@ async function main() { let deleted = 0; let failed = 0; - let orphaned = 0; - let referencedCount = 0; - for (const candidate of candidates) { - if (referenced.has(candidate.url)) { - referencedCount += 1; - continue; + // A dry run that only reports a count cannot be acted on: the point of it is to see + // what would go before anything does. Deleting prints the same list, so a real run is + // auditable after the fact too. + const orphanCandidates = candidates.filter((candidate) => !referenced.has(candidate.url)); + const referencedCount = candidates.length - orphanCandidates.length; + const orphaned = orphanCandidates.length; + + if (orphanCandidates.length > 0) { + const owners = await findUploadSessionOwners(orphanCandidates.map((c) => c.key)); + + console.log(`[r2-orphan-cleanup] Orphans ${dryRun ? 'that would be deleted' : 'to delete'}:`); + for (const candidate of orphanCandidates) { + const email = owners.get(candidate.key); + console.log( + `[r2-orphan-cleanup] ${candidate.key} ${email ?? 'owner unknown (no upload session)'}` + ); } + } - orphaned += 1; + for (const candidate of orphanCandidates) { if (dryRun) continue; try {