feat(storage): add shared R2 media proxy and orphan cleanup tooling for R2/Bunny

2026-09-11 09:36:08 +00:00 · 2026-02-24 17:02:54 +03:00
parent ffa55d7dcc
commit fa51847aa4
9 changed files with 703 additions and 172 deletions
@@ -0,0 +1,238 @@
+import { db, disconnectDb } from '../lib/db';
+
+const BUNNY_API_BASE = 'https://video.bunnycdn.com';
+const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
+const ITEMS_PER_PAGE = 100;
+const MAX_PAGES = 200;
+const CHUNK_SIZE = 500;
+const DEFAULT_GRACE_HOURS = 24;
+
+type BunnyConfig = {
+  apiKey: string;
+  libraryId: string;
+};
+
+type BunnyVideo = {
+  id: string;
+  uploadedAt: Date;
+};
+
+function getBunnyConfig(): BunnyConfig {
+  const apiKey = process.env.BUNNY_STREAM_API_KEY;
+  const libraryId = process.env.BUNNY_STREAM_LIBRARY_ID || process.env.NEXT_PUBLIC_BUNNY_STREAM_LIBRARY_ID;
+
+  if (!apiKey || !libraryId) {
+    throw new Error('Missing BUNNY_STREAM_API_KEY or BUNNY_STREAM_LIBRARY_ID.');
+  }
+
+  return { apiKey, libraryId };
+}
+
+function toRecord(value: unknown): Record<string, unknown> | null {
+  if (!value || typeof value !== 'object') return null;
+  return value as Record<string, unknown>;
+}
+
+function parseVideoId(item: unknown): string | null {
+  const record = toRecord(item);
+  if (!record) return null;
+
+  const candidates = [record.guid, record.videoId, record.id];
+  for (const candidate of candidates) {
+    if (typeof candidate === 'string') {
+      const trimmed = candidate.trim();
+      if (BUNNY_VIDEO_ID_PATTERN.test(trimmed)) return trimmed;
+    }
+  }
+
+  return null;
+}
+
+function parseUploadedAt(item: unknown): Date | null {
+  const record = toRecord(item);
+  if (!record) return null;
+
+  const candidates = [
+    record.dateUploaded,
+    record.DateUploaded,
+    record.dateCreated,
+    record.DateCreated,
+    record.createdAt,
+    record.CreatedAt,
+  ];
+
+  for (const candidate of candidates) {
+    if (typeof candidate !== 'string') continue;
+    const parsed = new Date(candidate);
+    if (!Number.isNaN(parsed.getTime())) return parsed;
+  }
+
+  return null;
+}
+
+async function fetchBunnyPage(config: BunnyConfig, page: number): Promise<{ items: unknown[]; totalItems: number | null }> {
+  const response = await fetch(
+    `${BUNNY_API_BASE}/library/${config.libraryId}/videos?page=${page}&itemsPerPage=${ITEMS_PER_PAGE}`,
+    {
+      headers: {
+        AccessKey: config.apiKey,
+        Accept: 'application/json',
+      },
+      cache: 'no-store',
+    }
+  );
+
+  if (!response.ok) {
+    const body = await response.text().catch(() => '');
+    throw new Error(`Bunny list API failed (${response.status}): ${body.slice(0, 300)}`);
+  }
+
+  const payload = await response.json();
+  const record = toRecord(payload);
+  if (!record) return { items: [], totalItems: null };
+
+  const items = Array.isArray(record.items)
+    ? record.items
+    : (Array.isArray(record.Items) ? record.Items : []);
+
+  const totalItems = typeof record.totalItems === 'number'
+    ? record.totalItems
+    : (typeof record.TotalItems === 'number' ? record.TotalItems : null);
+
+  return { items, totalItems };
+}
+
+async function listBunnyVideos(config: BunnyConfig): Promise<{ videos: BunnyVideo[]; scanned: number; skippedInvalid: number }> {
+  const videos: BunnyVideo[] = [];
+  let scanned = 0;
+  let skippedInvalid = 0;
+  let page = 1;
+
+  while (page <= MAX_PAGES) {
+    const { items, totalItems } = await fetchBunnyPage(config, page);
+    if (items.length === 0) break;
+
+    scanned += items.length;
+    for (const item of items) {
+      const id = parseVideoId(item);
+      const uploadedAt = parseUploadedAt(item);
+      if (!id || !uploadedAt) {
+        skippedInvalid += 1;
+        continue;
+      }
+      videos.push({ id, uploadedAt });
+    }
+
+    if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break;
+    page += 1;
+  }
+
+  return { videos, scanned, skippedInvalid };
+}
+
+function chunk<T>(items: T[], size: number): T[][] {
+  const out: T[][] = [];
+  for (let i = 0; i < items.length; i += size) {
+    out.push(items.slice(i, i + size));
+  }
+  return out;
+}
+
+async function findReferencedVideoIds(videoIds: string[]): Promise<Set<string>> {
+  const referenced = new Set<string>();
+
+  for (const group of chunk(videoIds, CHUNK_SIZE)) {
+    const rows = await db.videoVersion.findMany({
+      where: {
+        providerId: 'bunny',
+        videoId: { in: group },
+      },
+      select: { videoId: true },
+    });
+    rows.forEach((row) => {
+      if (row.videoId) referenced.add(row.videoId);
+    });
+  }
+
+  return referenced;
+}
+
+async function deleteBunnyVideo(config: BunnyConfig, videoId: string): Promise<'deleted' | 'already_missing'> {
+  const response = await fetch(
+    `${BUNNY_API_BASE}/library/${config.libraryId}/videos/${encodeURIComponent(videoId)}`,
+    {
+      method: 'DELETE',
+      headers: {
+        AccessKey: config.apiKey,
+      },
+    }
+  );
+
+  if (response.status === 404) return 'already_missing';
+  if (response.ok) return 'deleted';
+
+  const body = await response.text().catch(() => '');
+  throw new Error(`Bunny delete API failed for ${videoId} (${response.status}): ${body.slice(0, 300)}`);
+}
+
+async function main() {
+  const dryRun = process.argv.includes('--dry-run');
+  const graceHours = DEFAULT_GRACE_HOURS;
+  const graceMs = graceHours * 60 * 60 * 1000;
+  const cutoff = Date.now() - graceMs;
+
+  const config = getBunnyConfig();
+  console.log(`[bunny-orphan-cleanup] Starting (${dryRun ? 'dry-run' : 'delete mode'})`);
+  console.log(`[bunny-orphan-cleanup] Grace period: ${graceHours}h`);
+
+  const { videos, scanned, skippedInvalid } = await listBunnyVideos(config);
+  const eligible = videos.filter((video) => video.uploadedAt.getTime() <= cutoff);
+  console.log(`[bunny-orphan-cleanup] Scanned: ${scanned}`);
+  console.log(`[bunny-orphan-cleanup] Skipped invalid metadata: ${skippedInvalid}`);
+  console.log(`[bunny-orphan-cleanup] Eligible (old enough): ${eligible.length}`);
+
+  if (eligible.length === 0) {
+    console.log('[bunny-orphan-cleanup] No eligible Bunny videos found');
+    return;
+  }
+
+  const eligibleIds = eligible.map((video) => video.id);
+  const referenced = await findReferencedVideoIds(eligibleIds);
+
+  const orphanIds = eligibleIds.filter((id) => !referenced.has(id));
+  let deleted = 0;
+  let alreadyMissing = 0;
+  let failed = 0;
+
+  for (const orphanId of orphanIds) {
+    if (dryRun) continue;
+
+    try {
+      const result = await deleteBunnyVideo(config, orphanId);
+      if (result === 'already_missing') {
+        alreadyMissing += 1;
+      } else {
+        deleted += 1;
+      }
+    } catch (error) {
+      failed += 1;
+      console.error(`[bunny-orphan-cleanup] Failed deleting ${orphanId}:`, error);
+    }
+  }
+
+  console.log('[bunny-orphan-cleanup] Summary');
+  console.log(`[bunny-orphan-cleanup] Referenced: ${referenced.size}`);
+  console.log(`[bunny-orphan-cleanup] Orphaned: ${orphanIds.length}`);
+  console.log(`[bunny-orphan-cleanup] Deleted: ${deleted}`);
+  console.log(`[bunny-orphan-cleanup] Already missing: ${alreadyMissing}`);
+  console.log(`[bunny-orphan-cleanup] Failed: ${failed}`);
+}
+
+main()
+  .catch((error) => {
+    console.error('[bunny-orphan-cleanup] Fatal error:', error);
+    process.exitCode = 1;
+  })
+  .finally(async () => {
+    await disconnectDb();
+  });
@@ -0,0 +1,176 @@
+import { DeleteObjectCommand, ListObjectsV2Command, type ListObjectsV2CommandInput } from '@aws-sdk/client-s3';
+import { db, disconnectDb } from '../lib/db';
+import { r2Client, R2_BUCKET_NAME } from '../lib/r2';
+
+const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000;
+const CHUNK_SIZE = 500;
+const PREFIXES = ['images/', 'voice/'] as const;
+
+type CleanupCandidate = {
+  key: string;
+  url: string;
+};
+
+type UserFeedbackScreenshotDelegate = {
+  findMany: (args: { where: { url: { in: string[] } }; select: { url: true } }) => Promise<Array<{ url: string }>>;
+};
+
+function keyToProxyUrl(key: string): string | null {
+  if (key.startsWith('images/')) {
+    const filename = key.slice('images/'.length);
+    return filename ? `/api/upload/image/${filename}` : null;
+  }
+  if (key.startsWith('voice/')) {
+    const filename = key.slice('voice/'.length);
+    return filename ? `/api/upload/audio/${filename}` : null;
+  }
+  return null;
+}
+
+function chunk<T>(items: T[], size: number): T[][] {
+  const out: T[][] = [];
+  for (let i = 0; i < items.length; i += size) {
+    out.push(items.slice(i, i + size));
+  }
+  return out;
+}
+
+async function listCleanupCandidates(): Promise<{ candidates: CleanupCandidate[]; scanned: number }> {
+  const candidates: CleanupCandidate[] = [];
+  const cutoff = Date.now() - UNATTACHED_UPLOAD_TTL_MS;
+  let scanned = 0;
+
+  for (const prefix of PREFIXES) {
+    let continuationToken: string | undefined;
+    let isTruncated = true;
+
+    while (isTruncated) {
+      const input: ListObjectsV2CommandInput = {
+        Bucket: R2_BUCKET_NAME,
+        Prefix: prefix,
+      };
+      if (continuationToken) input.ContinuationToken = continuationToken;
+
+      const response = await r2Client.send(new ListObjectsV2Command(input));
+      const contents = response.Contents ?? [];
+      scanned += contents.length;
+
+      for (const item of contents) {
+        if (!item.Key || !item.LastModified) continue;
+        if (item.LastModified.getTime() > cutoff) continue;
+
+        const url = keyToProxyUrl(item.Key);
+        if (!url) continue;
+        candidates.push({ key: item.Key, url });
+      }
+
+      isTruncated = response.IsTruncated ?? false;
+      continuationToken = response.NextContinuationToken;
+    }
+  }
+
+  return { candidates, scanned };
+}
+
+async function findReferencedUrls(urls: string[]): Promise<Set<string>> {
+  const referenced = new Set<string>();
+  const userFeedbackScreenshotDelegate = (db as unknown as {
+    userFeedbackScreenshot?: UserFeedbackScreenshotDelegate;
+  }).userFeedbackScreenshot;
+
+  for (const group of chunk(urls, CHUNK_SIZE)) {
+    const [commentRows, feedbackRows, feedbackAttachmentRows] = await Promise.all([
+      db.comment.findMany({
+        where: {
+          OR: [{ voiceUrl: { in: group } }, { imageUrl: { in: group } }],
+        },
+        select: {
+          voiceUrl: true,
+          imageUrl: true,
+        },
+      }),
+      db.userFeedback.findMany({
+        where: { screenshotUrl: { in: group } },
+        select: { screenshotUrl: true },
+      }),
+      userFeedbackScreenshotDelegate
+        ? userFeedbackScreenshotDelegate.findMany({
+          where: { url: { in: group } },
+          select: { url: true },
+        })
+        : Promise.resolve([] as Array<{ url: string }>),
+    ]);
+
+    for (const row of commentRows) {
+      if (row.voiceUrl) referenced.add(row.voiceUrl);
+      if (row.imageUrl) referenced.add(row.imageUrl);
+    }
+    for (const row of feedbackRows) {
+      if (row.screenshotUrl) referenced.add(row.screenshotUrl);
+    }
+    for (const row of feedbackAttachmentRows) {
+      if (row.url) referenced.add(row.url);
+    }
+  }
+
+  return referenced;
+}
+
+async function main() {
+  const dryRun = process.argv.includes('--dry-run');
+  console.log(`[r2-orphan-cleanup] Starting (${dryRun ? 'dry-run' : 'delete mode'})`);
+
+  const { candidates, scanned } = await listCleanupCandidates();
+  console.log(`[r2-orphan-cleanup] Scanned: ${scanned}, eligible (old enough): ${candidates.length}`);
+
+  if (candidates.length === 0) {
+    console.log('[r2-orphan-cleanup] No eligible objects found');
+    return;
+  }
+
+  const referenced = await findReferencedUrls(candidates.map((candidate) => candidate.url));
+
+  let deleted = 0;
+  let failed = 0;
+  let orphaned = 0;
+  let referencedCount = 0;
+
+  for (const candidate of candidates) {
+    if (referenced.has(candidate.url)) {
+      referencedCount += 1;
+      continue;
+    }
+
+    orphaned += 1;
+    if (dryRun) continue;
+
+    try {
+      await r2Client.send(
+        new DeleteObjectCommand({
+          Bucket: R2_BUCKET_NAME,
+          Key: candidate.key,
+        })
+      );
+      deleted += 1;
+    } catch (error) {
+      failed += 1;
+      console.error(`[r2-orphan-cleanup] Failed deleting ${candidate.key}:`, error);
+    }
+  }
+
+  console.log('[r2-orphan-cleanup] Summary');
+  console.log(`[r2-orphan-cleanup] Referenced: ${referencedCount}`);
+  console.log(`[r2-orphan-cleanup] Orphaned: ${orphaned}`);
+  console.log(`[r2-orphan-cleanup] Deleted: ${deleted}`);
+  console.log(`[r2-orphan-cleanup] Failed: ${failed}`);
+}
+
+main()
+  .catch((error) => {
+    console.error('[r2-orphan-cleanup] Fatal error:', error);
+    process.exitCode = 1;
+  })
+  .finally(async () => {
+    await disconnectDb();
+    r2Client.destroy();
+  });