fix(scripts): make the orphan cleanups reviewable before they delete

Two problems with running these unattended, both found while wiring the Bunny
cleanup up to a Coolify scheduled task against production.

A dry run reported a count and nothing else. "Orphaned: 31" is not something
anyone can approve: it says how many objects would go, never which. Both scripts
now list every orphan they would delete, and print the same list when deleting,
so a real run is auditable afterwards too.

Each line carries who the object belongs to, as far as each provider can answer:

- R2 reads the owner out of `videoUploadSession`, which keeps `objectKey`
  alongside the initiating and billed user and survives an upload that never
  became a video. That is the case producing orphans, so this is an answer
  rather than a guess.
- Bunny has no equivalent. `bunny-init` sends the provider a title and nothing
  else, and an orphan by definition has no row pointing at it, so there is
  nothing authoritative to look up. The title is matched against titles still in
  the database instead, which catches the common shape (a version upload that
  failed and was retried successfully leaves a live row with the same title).
  A hit prints as "possibly", because it is a hint.

The grace periods were also too short to be safe:

- Bunny counted a video abandoned after 24 hours.
- R2 counted an object abandoned after 15 minutes, which is shorter than a slow
  multipart upload of a large file. An object still being written, or written
  but not yet finalised into a row, looked abandoned and could be deleted out
  from under the upload creating it.

Both are seven days now: long enough that no upload, retry or delayed
finalisation can still be in flight.
This commit is contained in:
yusufipk
2026-07-26 19:08:03 +07:00
parent 0ceba72d5b
commit 1f7b77873c
2 changed files with 178 additions and 10 deletions
+110 -2
View File
@@ -7,7 +7,10 @@ const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
const ITEMS_PER_PAGE = 100; const ITEMS_PER_PAGE = 100;
const MAX_PAGES = 200; const MAX_PAGES = 200;
const CHUNK_SIZE = 500; const CHUNK_SIZE = 500;
const DEFAULT_GRACE_HOURS = 24; // Seven days, not one. A video only counts as abandoned once nothing has claimed it for
// long enough that no upload, retry or delayed finalisation could still be in flight, and
// a day is short enough that an upload interrupted overnight looks abandoned by morning.
const DEFAULT_GRACE_HOURS = 7 * 24;
type BunnyConfig = { type BunnyConfig = {
apiKey: string; apiKey: string;
@@ -17,6 +20,7 @@ type BunnyConfig = {
type BunnyVideo = { type BunnyVideo = {
id: string; id: string;
uploadedAt: Date; uploadedAt: Date;
title: string | null;
}; };
function getBunnyConfig(): BunnyConfig { function getBunnyConfig(): BunnyConfig {
@@ -51,6 +55,17 @@ function parseVideoId(item: unknown): string | null {
return null; return null;
} }
function parseTitle(item: unknown): string | null {
const record = toRecord(item);
if (!record) return null;
for (const candidate of [record.title, record.Title]) {
if (typeof candidate === 'string' && candidate.trim()) return candidate.trim();
}
return null;
}
function parseUploadedAt(item: unknown): Date | null { function parseUploadedAt(item: unknown): Date | null {
const record = toRecord(item); const record = toRecord(item);
if (!record) return null; if (!record) return null;
@@ -133,7 +148,7 @@ async function listBunnyVideos(
skippedInvalid += 1; skippedInvalid += 1;
continue; continue;
} }
videos.push({ id, uploadedAt }); videos.push({ id, uploadedAt, title: parseTitle(item) });
} }
if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break; if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break;
@@ -182,6 +197,73 @@ async function findReferencedVideoIds(videoIds: string[]): Promise<Set<string>>
return referenced; return referenced;
} }
/**
* Best-effort owner for an orphan, by matching its Bunny title against titles still in the
* database.
*
* An orphan is by definition a video nothing in the database points at, so there is no
* authoritative owner to look up: `bunny-init` sends Bunny a title and nothing else, no
* user id and no email. What is left is the title, and it is worth matching because the
* common way an orphan appears is a version upload that failed and was then retried
* successfully, which leaves a live row carrying the same title.
*
* A hit is therefore a hint, not a fact, and the output says so. A miss means the video
* cannot be attributed at all from what Bunny and the database hold today.
*/
async function findOwnerHintsByTitle(titles: string[]): Promise<Map<string, string[]>> {
const hints = new Map<string, Set<string>>();
const unique = [...new Set(titles.filter((title): title is string => Boolean(title)))];
const remember = (title: string | null, emails: Array<string | null | undefined>) => {
if (!title) return;
const existing = hints.get(title) ?? new Set<string>();
for (const email of emails) {
if (email) existing.add(email);
}
if (existing.size > 0) hints.set(title, existing);
};
const ownerSelect = {
project: {
select: {
owner: { select: { email: true } },
workspace: { select: { owner: { select: { email: true } } } },
},
},
} as const;
for (const group of chunk(unique, CHUNK_SIZE)) {
const [videos, versions, assets] = await Promise.all([
db.video.findMany({
where: { title: { in: group } },
select: { title: true, ...ownerSelect },
}),
db.videoVersion.findMany({
where: { title: { in: group } },
select: { title: true, video: { select: ownerSelect } },
}),
db.videoAsset.findMany({
where: { displayName: { in: group } },
select: { displayName: true, video: { select: ownerSelect } },
}),
]);
for (const row of videos) {
remember(row.title, [row.project.owner?.email, row.project.workspace.owner?.email]);
}
for (const row of versions) {
const project = row.video.project;
remember(row.title, [project.owner?.email, project.workspace.owner?.email]);
}
for (const row of assets) {
const project = row.video.project;
remember(row.displayName, [project.owner?.email, project.workspace.owner?.email]);
}
}
return new Map([...hints].map(([title, emails]) => [title, [...emails].sort()]));
}
async function deleteBunnyVideo( async function deleteBunnyVideo(
config: BunnyConfig, config: BunnyConfig,
videoId: string videoId: string
@@ -238,6 +320,32 @@ async function main() {
const referenced = await findReferencedVideoIds(eligibleIds); const referenced = await findReferencedVideoIds(eligibleIds);
const orphanIds = eligibleIds.filter((id) => !referenced.has(id)); const orphanIds = eligibleIds.filter((id) => !referenced.has(id));
// A dry run that only reports a count cannot be acted on: the point of it is to see
// what would go before anything does. Deleting prints the same list, so a real run is
// auditable after the fact too.
if (orphanIds.length > 0) {
const byId = new Map(eligible.map((video) => [video.id, video]));
const orphanTitles = orphanIds
.map((orphanId) => byId.get(orphanId)?.title)
.filter((title): title is string => Boolean(title));
const ownerHints = await findOwnerHintsByTitle(orphanTitles);
console.log(
`[bunny-orphan-cleanup] Orphans ${dryRun ? 'that would be deleted' : 'to delete'}:`
);
for (const orphanId of orphanIds) {
const video = byId.get(orphanId);
const uploadedAt = video?.uploadedAt.toISOString() ?? 'unknown date';
const title = video?.title ?? 'untitled';
const emails = video?.title ? (ownerHints.get(video.title) ?? []) : [];
// "possibly" is load-bearing: this is a title match, not a stored owner.
const owner =
emails.length > 0 ? `possibly ${emails.join(', ')}` : 'owner unknown (no title match)';
console.log(`[bunny-orphan-cleanup] ${orphanId} ${uploadedAt} ${title} ${owner}`);
}
}
let deleted = 0; let deleted = 0;
let alreadyMissing = 0; let alreadyMissing = 0;
let failed = 0; let failed = 0;
+68 -8
View File
@@ -8,7 +8,12 @@ import { r2Client, R2_BUCKET_NAME } from '../lib/r2';
import { cleanupExpiredBillingWorkspaces } from './expired-billing-cleanup'; import { cleanupExpiredBillingWorkspaces } from './expired-billing-cleanup';
import { logError } from '@/lib/logger'; import { logError } from '@/lib/logger';
const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000; // Seven days. This was fifteen minutes, which is shorter than a slow multipart upload of
// a large file: an object still being written, or written but not yet finalised into a
// row, looked abandoned and could be deleted out from under the upload that was creating
// it. An object only counts as abandoned once nothing has claimed it for long enough that
// no upload, retry or delayed finalisation could still be in flight.
const UNATTACHED_UPLOAD_TTL_MS = 7 * 24 * 60 * 60 * 1000;
const CHUNK_SIZE = 500; const CHUNK_SIZE = 500;
const PREFIXES = ['images/', 'voice/', 'videos/'] as const; const PREFIXES = ['images/', 'voice/', 'videos/'] as const;
@@ -40,6 +45,50 @@ function keyToProxyUrl(key: string): string | null {
return null; return null;
} }
/**
* The owner of each orphaned object key, from the upload session that created it.
*
* Unlike the Bunny side, this is an answer rather than a guess: `videoUploadSession` keeps
* `objectKey` alongside the user who initiated the upload, and the row survives even when
* the upload never became a video, which is exactly the case that produces an orphan.
* Both the initiating user and the billed user are reported when they differ, because the
* billed one is who paid for the bytes.
*/
async function findUploadSessionOwners(keys: string[]): Promise<Map<string, string>> {
const owners = new Map<string, string>();
for (const group of chunk(keys, CHUNK_SIZE)) {
// VideoUploadSession holds the ids but declares no relation to User, so the addresses
// are resolved in a second query rather than by widening the schema for a script.
const sessions = await db.videoUploadSession.findMany({
where: { objectKey: { in: group } },
select: { objectKey: true, status: true, userId: true, billedUserId: true },
});
if (sessions.length === 0) continue;
const userIds = [
...new Set(sessions.flatMap((session) => [session.userId, session.billedUserId])),
];
const users = await db.user.findMany({
where: { id: { in: userIds } },
select: { id: true, email: true },
});
const emailById = new Map(users.map((user) => [user.id, user.email]));
for (const session of sessions) {
const initiator = emailById.get(session.userId) ?? null;
const billed = emailById.get(session.billedUserId) ?? null;
const who =
billed && billed !== initiator
? `${initiator ?? 'unknown'} (billed: ${billed})`
: (initiator ?? billed ?? 'unknown');
owners.set(session.objectKey, `${who} [session ${session.status.toLowerCase()}]`);
}
}
return owners;
}
function chunk<T>(items: T[], size: number): T[][] { function chunk<T>(items: T[], size: number): T[][] {
const out: T[][] = []; const out: T[][] = [];
for (let i = 0; i < items.length; i += size) { for (let i = 0; i < items.length; i += size) {
@@ -181,16 +230,27 @@ async function main() {
let deleted = 0; let deleted = 0;
let failed = 0; let failed = 0;
let orphaned = 0;
let referencedCount = 0;
for (const candidate of candidates) { // A dry run that only reports a count cannot be acted on: the point of it is to see
if (referenced.has(candidate.url)) { // what would go before anything does. Deleting prints the same list, so a real run is
referencedCount += 1; // auditable after the fact too.
continue; const orphanCandidates = candidates.filter((candidate) => !referenced.has(candidate.url));
const referencedCount = candidates.length - orphanCandidates.length;
const orphaned = orphanCandidates.length;
if (orphanCandidates.length > 0) {
const owners = await findUploadSessionOwners(orphanCandidates.map((c) => c.key));
console.log(`[r2-orphan-cleanup] Orphans ${dryRun ? 'that would be deleted' : 'to delete'}:`);
for (const candidate of orphanCandidates) {
const email = owners.get(candidate.key);
console.log(
`[r2-orphan-cleanup] ${candidate.key} ${email ?? 'owner unknown (no upload session)'}`
);
} }
}
orphaned += 1; for (const candidate of orphanCandidates) {
if (dryRun) continue; if (dryRun) continue;
try { try {