Merge pull request #46 from yusufipk/fix/orphan-cleanup-review

fix(scripts): make the orphan cleanups reviewable before they delete

scripts/bunny-orphan-cleanup.ts

@@ -7,7 +7,10 @@ const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
 const ITEMS_PER_PAGE = 100;
 const MAX_PAGES = 200;
 const CHUNK_SIZE = 500;
-const DEFAULT_GRACE_HOURS = 24;
+// Seven days, not one. A video only counts as abandoned once nothing has claimed it for
+// long enough that no upload, retry or delayed finalisation could still be in flight, and
+// a day is short enough that an upload interrupted overnight looks abandoned by morning.
+const DEFAULT_GRACE_HOURS = 7 * 24;
 
 type BunnyConfig = {
   apiKey: string;
@@ -17,6 +20,7 @@ type BunnyConfig = {
 type BunnyVideo = {
   id: string;
   uploadedAt: Date;
+  title: string | null;
 };
 
 function getBunnyConfig(): BunnyConfig {
@@ -51,6 +55,17 @@ function parseVideoId(item: unknown): string | null {
   return null;
 }
 
+function parseTitle(item: unknown): string | null {
+  const record = toRecord(item);
+  if (!record) return null;
+
+  for (const candidate of [record.title, record.Title]) {
+    if (typeof candidate === 'string' && candidate.trim()) return candidate.trim();
+  }
+
+  return null;
+}
+
 function parseUploadedAt(item: unknown): Date | null {
   const record = toRecord(item);
   if (!record) return null;
@@ -133,7 +148,7 @@ async function listBunnyVideos(
         skippedInvalid += 1;
         continue;
       }
-      videos.push({ id, uploadedAt });
+      videos.push({ id, uploadedAt, title: parseTitle(item) });
     }
 
     if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break;
@@ -182,6 +197,73 @@ async function findReferencedVideoIds(videoIds: string[]): Promise<Set<string>>
   return referenced;
 }
 
+/**
+ * Best-effort owner for an orphan, by matching its Bunny title against titles still in the
+ * database.
+ *
+ * An orphan is by definition a video nothing in the database points at, so there is no
+ * authoritative owner to look up: `bunny-init` sends Bunny a title and nothing else, no
+ * user id and no email. What is left is the title, and it is worth matching because the
+ * common way an orphan appears is a version upload that failed and was then retried
+ * successfully, which leaves a live row carrying the same title.
+ *
+ * A hit is therefore a hint, not a fact, and the output says so. A miss means the video
+ * cannot be attributed at all from what Bunny and the database hold today.
+ */
+async function findOwnerHintsByTitle(titles: string[]): Promise<Map<string, string[]>> {
+  const hints = new Map<string, Set<string>>();
+  const unique = [...new Set(titles.filter((title): title is string => Boolean(title)))];
+
+  const remember = (title: string | null, emails: Array<string | null | undefined>) => {
+    if (!title) return;
+    const existing = hints.get(title) ?? new Set<string>();
+    for (const email of emails) {
+      if (email) existing.add(email);
+    }
+    if (existing.size > 0) hints.set(title, existing);
+  };
+
+  const ownerSelect = {
+    project: {
+      select: {
+        owner: { select: { email: true } },
+        workspace: { select: { owner: { select: { email: true } } } },
+      },
+    },
+  } as const;
+
+  for (const group of chunk(unique, CHUNK_SIZE)) {
+    const [videos, versions, assets] = await Promise.all([
+      db.video.findMany({
+        where: { title: { in: group } },
+        select: { title: true, ...ownerSelect },
+      }),
+      db.videoVersion.findMany({
+        where: { title: { in: group } },
+        select: { title: true, video: { select: ownerSelect } },
+      }),
+      db.videoAsset.findMany({
+        where: { displayName: { in: group } },
+        select: { displayName: true, video: { select: ownerSelect } },
+      }),
+    ]);
+
+    for (const row of videos) {
+      remember(row.title, [row.project.owner?.email, row.project.workspace.owner?.email]);
+    }
+    for (const row of versions) {
+      const project = row.video.project;
+      remember(row.title, [project.owner?.email, project.workspace.owner?.email]);
+    }
+    for (const row of assets) {
+      const project = row.video.project;
+      remember(row.displayName, [project.owner?.email, project.workspace.owner?.email]);
+    }
+  }
+
+  return new Map([...hints].map(([title, emails]) => [title, [...emails].sort()]));
+}
+
 async function deleteBunnyVideo(
   config: BunnyConfig,
   videoId: string
@@ -238,6 +320,32 @@ async function main() {
   const referenced = await findReferencedVideoIds(eligibleIds);
 
   const orphanIds = eligibleIds.filter((id) => !referenced.has(id));
+
+  // A dry run that only reports a count cannot be acted on: the point of it is to see
+  // what would go before anything does. Deleting prints the same list, so a real run is
+  // auditable after the fact too.
+  if (orphanIds.length > 0) {
+    const byId = new Map(eligible.map((video) => [video.id, video]));
+    const orphanTitles = orphanIds
+      .map((orphanId) => byId.get(orphanId)?.title)
+      .filter((title): title is string => Boolean(title));
+    const ownerHints = await findOwnerHintsByTitle(orphanTitles);
+
+    console.log(
+      `[bunny-orphan-cleanup] Orphans ${dryRun ? 'that would be deleted' : 'to delete'}:`
+    );
+    for (const orphanId of orphanIds) {
+      const video = byId.get(orphanId);
+      const uploadedAt = video?.uploadedAt.toISOString() ?? 'unknown date';
+      const title = video?.title ?? 'untitled';
+      const emails = video?.title ? (ownerHints.get(video.title) ?? []) : [];
+      // "possibly" is load-bearing: this is a title match, not a stored owner.
+      const owner =
+        emails.length > 0 ? `possibly ${emails.join(', ')}` : 'owner unknown (no title match)';
+      console.log(`[bunny-orphan-cleanup]   ${orphanId}  ${uploadedAt}  ${title}  ${owner}`);
+    }
+  }
+
   let deleted = 0;
   let alreadyMissing = 0;
   let failed = 0;

scripts/r2-orphan-cleanup.ts

@@ -8,7 +8,12 @@ import { r2Client, R2_BUCKET_NAME } from '../lib/r2';
 import { cleanupExpiredBillingWorkspaces } from './expired-billing-cleanup';
 import { logError } from '@/lib/logger';
 
-const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000;
+// Seven days. This was fifteen minutes, which is shorter than a slow multipart upload of
+// a large file: an object still being written, or written but not yet finalised into a
+// row, looked abandoned and could be deleted out from under the upload that was creating
+// it. An object only counts as abandoned once nothing has claimed it for long enough that
+// no upload, retry or delayed finalisation could still be in flight.
+const UNATTACHED_UPLOAD_TTL_MS = 7 * 24 * 60 * 60 * 1000;
 const CHUNK_SIZE = 500;
 const PREFIXES = ['images/', 'voice/', 'videos/'] as const;
 
@@ -40,6 +45,50 @@ function keyToProxyUrl(key: string): string | null {
   return null;
 }
 
+/**
+ * The owner of each orphaned object key, from the upload session that created it.
+ *
+ * Unlike the Bunny side, this is an answer rather than a guess: `videoUploadSession` keeps
+ * `objectKey` alongside the user who initiated the upload, and the row survives even when
+ * the upload never became a video, which is exactly the case that produces an orphan.
+ * Both the initiating user and the billed user are reported when they differ, because the
+ * billed one is who paid for the bytes.
+ */
+async function findUploadSessionOwners(keys: string[]): Promise<Map<string, string>> {
+  const owners = new Map<string, string>();
+
+  for (const group of chunk(keys, CHUNK_SIZE)) {
+    // VideoUploadSession holds the ids but declares no relation to User, so the addresses
+    // are resolved in a second query rather than by widening the schema for a script.
+    const sessions = await db.videoUploadSession.findMany({
+      where: { objectKey: { in: group } },
+      select: { objectKey: true, status: true, userId: true, billedUserId: true },
+    });
+    if (sessions.length === 0) continue;
+
+    const userIds = [
+      ...new Set(sessions.flatMap((session) => [session.userId, session.billedUserId])),
+    ];
+    const users = await db.user.findMany({
+      where: { id: { in: userIds } },
+      select: { id: true, email: true },
+    });
+    const emailById = new Map(users.map((user) => [user.id, user.email]));
+
+    for (const session of sessions) {
+      const initiator = emailById.get(session.userId) ?? null;
+      const billed = emailById.get(session.billedUserId) ?? null;
+      const who =
+        billed && billed !== initiator
+          ? `${initiator ?? 'unknown'} (billed: ${billed})`
+          : (initiator ?? billed ?? 'unknown');
+      owners.set(session.objectKey, `${who} [session ${session.status.toLowerCase()}]`);
+    }
+  }
+
+  return owners;
+}
+
 function chunk<T>(items: T[], size: number): T[][] {
   const out: T[][] = [];
   for (let i = 0; i < items.length; i += size) {
@@ -181,16 +230,27 @@ async function main() {
 
   let deleted = 0;
   let failed = 0;
-  let orphaned = 0;
-  let referencedCount = 0;
 
-  for (const candidate of candidates) {
+  // A dry run that only reports a count cannot be acted on: the point of it is to see
-    if (referenced.has(candidate.url)) {
+  // what would go before anything does. Deleting prints the same list, so a real run is
-      referencedCount += 1;
+  // auditable after the fact too.
-      continue;
+  const orphanCandidates = candidates.filter((candidate) => !referenced.has(candidate.url));
+  const referencedCount = candidates.length - orphanCandidates.length;
+  const orphaned = orphanCandidates.length;
+
+  if (orphanCandidates.length > 0) {
+    const owners = await findUploadSessionOwners(orphanCandidates.map((c) => c.key));
+
+    console.log(`[r2-orphan-cleanup] Orphans ${dryRun ? 'that would be deleted' : 'to delete'}:`);
+    for (const candidate of orphanCandidates) {
+      const email = owners.get(candidate.key);
+      console.log(
+        `[r2-orphan-cleanup]   ${candidate.key}  ${email ?? 'owner unknown (no upload session)'}`
+      );
+    }
   }
 
-    orphaned += 1;
+  for (const candidate of orphanCandidates) {
     if (dryRun) continue;
 
     try {