From fa51847aa43ae32b0534a8842666501ad0476611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yusuf=20=C4=B0pek?= Date: Tue, 24 Feb 2026 17:02:54 +0300 Subject: [PATCH] feat(storage): add shared R2 media proxy and orphan cleanup tooling for R2/Bunny --- .env.example | 78 ++++++++ .gitignore | 1 + app/api/upload/audio/[filename]/route.ts | 84 +------- app/api/upload/image/[filename]/route.ts | 96 +-------- lib/db.ts | 18 +- lib/r2-media-proxy.ts | 178 +++++++++++++++++ package.json | 6 +- scripts/bunny-orphan-cleanup.ts | 238 +++++++++++++++++++++++ scripts/r2-orphan-cleanup.ts | 176 +++++++++++++++++ 9 files changed, 703 insertions(+), 172 deletions(-) create mode 100644 .env.example create mode 100644 lib/r2-media-proxy.ts create mode 100644 scripts/bunny-orphan-cleanup.ts create mode 100644 scripts/r2-orphan-cleanup.ts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d0c8b82 --- /dev/null +++ b/.env.example @@ -0,0 +1,78 @@ +# Copy this file to .env and fill in your values + +# ============================================================================ +# DATABASE +# ============================================================================ +# PostgreSQL connection string +DATABASE_URL="postgresql://user:password@localhost:5432/openframe?schema=public" + +# ============================================================================ +# AUTHENTICATION - NextAuth.js +# ============================================================================ +NEXTAUTH_URL="http://localhost:3000" +NEXTAUTH_SECRET="your-secret-key-here-generate-with-openssl-rand-base64-32" + +# ============================================================================ +# OAUTH PROVIDERS +# ============================================================================ +# Google OAuth +# Create credentials at: https://console.cloud.google.com/apis/credentials +GOOGLE_CLIENT_ID="your-google-client-id.apps.googleusercontent.com" +GOOGLE_CLIENT_SECRET="your-google-client-secret" + +# GitHub OAuth +# Create credentials at: https://github.com/settings/developers +GITHUB_CLIENT_ID="your-github-client-id" +GITHUB_CLIENT_SECRET="your-github-client-secret" + +# ============================================================================ +# FILE STORAGE +# ============================================================================ +# Cloudflare R2 (S3-compatible) +R2_ACCOUNT_ID="your-account-id" +R2_ACCESS_KEY_ID="your-access-key" +R2_SECRET_ACCESS_KEY="your-secret-key" +R2_BUCKET_NAME="openframe" +# R2 orphan cleanup configuration (script + external cron; app runtime does not schedule this) +# R2_ORPHAN_CLEANUP_CRON="*/15 * * * *" +# Scheduled delete mode: +# */15 * * * * cd /home/yusuf/Programming/OpenFrame && bun run r2:cleanup-orphans + +# ============================================================================ +# EMAIL & NOTIFICATIONS +# ============================================================================ + +SMTP_HOST="smtp.gmail.com" +SMTP_PORT="587" +SMTP_USER="your-email@gmail.com" +SMTP_PASSWORD="your-app-specific-password" +SMTP_FROM="noreply@openframe.dev" + +# ============================================================================ +# APPLICATION +# ============================================================================ +# development | production | test +NODE_ENV="development" +# Admin emails for accessing the /admin panel (comma separated list) +# e.g., "yusuf@example.com,admin@example.com" +ADMIN_EMAILS="" +# Invite code for internal registration (required to sign up) +# Generate a secure code for your team +INVITE_CODE="your-secret-invite-code" +# Enable debug logging +# DEBUG="openframe:*" + +# ============================================================================ +# VIDEO PROCESSING +# ============================================================================ + +# Bunny Stream for direct video uploads +BUNNY_STREAM_API_KEY="your-stream-library-api-key" +BUNNY_STREAM_LIBRARY_ID="your-library-id" +# Bunny Core API key (account-level) used to enforce KeepOriginalFiles/ExposeOriginals on the library +BUNNY_API_KEY="your-account-api-key" +# Bunny Stream CDN base URL (for HLS streaming) +BUNNY_CDN_URL="https://vz-965f4f4a-fc1.b-cdn.net" +# Bunny orphan cleanup configuration (script + external cron; app runtime does not schedule this) +# Grace period is fixed at 24 hours in the script. +# */15 * * * * cd /home/yusuf/Programming/OpenFrame && bun run bunny:cleanup-orphans diff --git a/.gitignore b/.gitignore index 3142145..c41959f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ yarn-error.log* # env files (can opt-in for committing if needed) .env* +!.env.example # vercel .vercel diff --git a/app/api/upload/audio/[filename]/route.ts b/app/api/upload/audio/[filename]/route.ts index 074906c..b10df65 100644 --- a/app/api/upload/audio/[filename]/route.ts +++ b/app/api/upload/audio/[filename]/route.ts @@ -1,8 +1,5 @@ -import { NextResponse } from 'next/server'; -import { r2Client, R2_BUCKET_NAME } from '@/lib/r2'; -import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'; import { apiErrors } from '@/lib/api-response'; -import { db } from '@/lib/db'; +import { proxyR2MediaObject } from '@/lib/r2-media-proxy'; // Only allow UUID filenames with safe extensions const SAFE_FILENAME = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.[a-z0-9]+$/i; @@ -16,15 +13,13 @@ const CONTENT_TYPE_MAP: Record = { ogg: 'audio/ogg', wav: 'audio/wav', }; -const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000; - function getContentType(filename: string): string { const ext = filename.split('.').pop()?.toLowerCase() || ''; return CONTENT_TYPE_MAP[ext] || 'audio/webm'; } export async function GET( - _request: Request, + request: Request, { params }: { params: Promise<{ filename: string }> } ) { try { @@ -36,77 +31,14 @@ export async function GET( } const key = `voice/${filename}`; - const mediaUrl = `/api/upload/audio/${filename}`; - - // Get file metadata to determine content type - const headResponse = await r2Client.send( - new HeadObjectCommand({ - Bucket: R2_BUCKET_NAME, - Key: key, - }) - ); - - const lastModified = headResponse.LastModified; - if (lastModified && Date.now() - lastModified.getTime() > UNATTACHED_UPLOAD_TTL_MS) { - const referenced = await db.comment.findFirst({ - where: { voiceUrl: mediaUrl }, - select: { id: true }, - }); - if (!referenced) { - await r2Client.send( - new DeleteObjectCommand({ - Bucket: R2_BUCKET_NAME, - Key: key, - }) - ).catch(() => undefined); - return apiErrors.notFound('File'); - } - } - - // Use the stored content-type or infer from filename extension - const contentType = headResponse.ContentType || getContentType(filename); - - // Get the object - const objectResponse = await r2Client.send( - new GetObjectCommand({ - Bucket: R2_BUCKET_NAME, - Key: key, - }) - ); - - // Handle the body properly - AWS SDK returns a stream - const body = objectResponse.Body; - if (!body) { - return apiErrors.internalError('Empty file'); - } - - // Convert stream to Uint8Array - const chunks: Uint8Array[] = []; - const asyncIterable = body as AsyncIterable; - for await (const chunk of asyncIterable) { - chunks.push(chunk); - } - const uint8Array = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0)); - let offset = 0; - for (const chunk of chunks) { - uint8Array.set(chunk, offset); - offset += chunk.length; - } - - // Create response with proper content-type - return new NextResponse(uint8Array, { - status: 200, - headers: { - 'Content-Type': contentType, - 'Cache-Control': 'private, no-store', - 'Accept-Ranges': 'bytes', - }, + return proxyR2MediaObject({ + request, + key, + fallbackContentType: getContentType(filename), + cacheControl: 'private, no-store', + internalErrorMessage: 'Failed to retrieve audio', }); } catch (error: unknown) { - const errorName = error instanceof Error ? error.name : ''; - if (errorName === 'NoSuchKey') { - return apiErrors.notFound('File'); - } console.error('Error serving audio:', error); return apiErrors.internalError('Failed to retrieve audio'); } diff --git a/app/api/upload/image/[filename]/route.ts b/app/api/upload/image/[filename]/route.ts index 3688ac7..1f3d7b8 100644 --- a/app/api/upload/image/[filename]/route.ts +++ b/app/api/upload/image/[filename]/route.ts @@ -1,8 +1,5 @@ -import { NextResponse } from 'next/server'; -import { r2Client, R2_BUCKET_NAME } from '@/lib/r2'; -import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'; import { apiErrors } from '@/lib/api-response'; -import { db } from '@/lib/db'; +import { proxyR2MediaObject } from '@/lib/r2-media-proxy'; // Only allow UUID filenames with safe extensions const SAFE_FILENAME = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.[a-z0-9]+$/i; @@ -14,15 +11,13 @@ const CONTENT_TYPE_MAP: Record = { webp: 'image/webp', gif: 'image/gif', }; -const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000; - function getContentType(filename: string): string { const ext = filename.split('.').pop()?.toLowerCase() || ''; return CONTENT_TYPE_MAP[ext] || 'application/octet-stream'; } export async function GET( - _request: Request, + request: Request, { params }: { params: Promise<{ filename: string }> } ) { try { @@ -34,91 +29,18 @@ export async function GET( } const key = `images/${filename}`; - const mediaUrl = `/api/upload/image/${filename}`; - - // Get file metadata to determine content type - const headResponse = await r2Client.send( - new HeadObjectCommand({ - Bucket: R2_BUCKET_NAME, - Key: key, - }) - ); - - const lastModified = headResponse.LastModified; - if (lastModified && Date.now() - lastModified.getTime() > UNATTACHED_UPLOAD_TTL_MS) { - const userFeedbackScreenshotDelegate = (db as unknown as { - userFeedbackScreenshot?: { - findFirst: (args?: unknown) => Promise<{ id: string } | null>; - }; - }).userFeedbackScreenshot; - const [commentReferenced, feedbackReferenced, feedbackAttachmentReferenced] = await Promise.all([ - db.comment.findFirst({ - where: { imageUrl: mediaUrl }, - select: { id: true }, - }), - db.userFeedback.findFirst({ - where: { screenshotUrl: mediaUrl }, - select: { id: true }, - }), - userFeedbackScreenshotDelegate - ? userFeedbackScreenshotDelegate.findFirst({ - where: { url: mediaUrl }, - select: { id: true }, - }) - : Promise.resolve(null), - ]); - if (!commentReferenced && !feedbackReferenced && !feedbackAttachmentReferenced) { - await r2Client.send( - new DeleteObjectCommand({ - Bucket: R2_BUCKET_NAME, - Key: key, - }) - ).catch(() => undefined); - return apiErrors.notFound('File'); - } - } - - const contentType = getContentType(filename); - - const objectResponse = await r2Client.send( - new GetObjectCommand({ - Bucket: R2_BUCKET_NAME, - Key: key, - }) - ); - - const body = objectResponse.Body; - if (!body) { - return apiErrors.internalError('Empty file'); - } - - const chunks: Uint8Array[] = []; - // @ts-expect-error - body is an iterable - for await (const chunk of body) { - chunks.push(chunk); - } - const uint8Array = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0)); - let offset = 0; - for (const chunk of chunks) { - uint8Array.set(chunk, offset); - offset += chunk.length; - } - - return new NextResponse(uint8Array, { - status: 200, - headers: { - 'Content-Type': contentType, - 'Cache-Control': 'private, no-store', - 'Accept-Ranges': 'bytes', + return proxyR2MediaObject({ + request, + key, + fallbackContentType: getContentType(filename), + cacheControl: 'private, no-store', + extraHeaders: { 'X-Content-Type-Options': 'nosniff', 'Content-Security-Policy': "default-src 'none'; sandbox", }, + internalErrorMessage: 'Failed to retrieve image', }); } catch (error: unknown) { - const errorName = error instanceof Error ? error.name : ''; - if (errorName === 'NoSuchKey') { - return apiErrors.notFound('File'); - } console.error('Error serving image:', error); return apiErrors.internalError('Failed to retrieve image'); } diff --git a/lib/db.ts b/lib/db.ts index cb1bb3b..fdabba0 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -77,17 +77,19 @@ if (process.env.NODE_ENV !== 'production') { globalForPrisma.prisma = db; } +export async function disconnectDb(): Promise { + if (globalForPool.pgPool) { + await globalForPool.pgPool.end(); + globalForPool.pgPool = undefined; + } + await db.$disconnect(); +} + // Graceful shutdown handler async function shutdown() { console.log('Shutting down database connections...'); - - if (globalForPool.pgPool) { - await globalForPool.pgPool.end(); - console.log('Database pool closed'); - } - - await db.$disconnect(); - console.log('Prisma client disconnected'); + await disconnectDb(); + console.log('Database connections closed'); } // Register shutdown handlers diff --git a/lib/r2-media-proxy.ts b/lib/r2-media-proxy.ts new file mode 100644 index 0000000..39cbf62 --- /dev/null +++ b/lib/r2-media-proxy.ts @@ -0,0 +1,178 @@ +import { GetObjectCommand, type GetObjectCommandInput, type GetObjectCommandOutput } from '@aws-sdk/client-s3'; +import { Readable } from 'node:stream'; +import { NextResponse } from 'next/server'; +import { apiErrors } from '@/lib/api-response'; +import { r2Client, R2_BUCKET_NAME } from '@/lib/r2'; + +type ProxyR2MediaOptions = { + request: Request; + key: string; + fallbackContentType: string; + cacheControl: string; + extraHeaders?: Record; + notFoundLabel?: string; + internalErrorMessage: string; +}; + +type R2LikeError = { + name?: string; + Code?: string; + $metadata?: { httpStatusCode?: number }; +}; + +function isStrongEtag(value: string): boolean { + return /^"[^"]+"$/.test(value); +} + +function parseHttpDate(value: string): Date | null { + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +function getErrorStatus(error: unknown): number | null { + const status = (error as R2LikeError | null | undefined)?.$metadata?.httpStatusCode; + return typeof status === 'number' ? status : null; +} + +function isNotFoundError(error: unknown): boolean { + const err = error as R2LikeError | null | undefined; + return err?.name === 'NoSuchKey' || err?.Code === 'NoSuchKey' || getErrorStatus(error) === 404; +} + +function isInvalidRangeError(error: unknown): boolean { + const err = error as R2LikeError | null | undefined; + return err?.name === 'InvalidRange' || err?.Code === 'InvalidRange' || getErrorStatus(error) === 416; +} + +function isPreconditionFailed(error: unknown): boolean { + return getErrorStatus(error) === 412; +} + +function toWebStream(body: unknown): ReadableStream | null { + if (!body) return null; + + const withTransform = body as { transformToWebStream?: () => ReadableStream }; + if (typeof withTransform.transformToWebStream === 'function') { + return withTransform.transformToWebStream(); + } + + if (body instanceof Readable) { + return Readable.toWeb(body) as ReadableStream; + } + + if (body instanceof ReadableStream) { + return body; + } + + return null; +} + +function setIfPresent(headers: Headers, key: string, value: string | number | null | undefined): void { + if (value === undefined || value === null) return; + headers.set(key, String(value)); +} + +export async function proxyR2MediaObject({ + request, + key, + fallbackContentType, + cacheControl, + extraHeaders, + notFoundLabel = 'File', + internalErrorMessage, +}: ProxyR2MediaOptions): Promise { + const range = request.headers.get('range'); + const ifRange = request.headers.get('if-range'); + const commandInput: GetObjectCommandInput = { + Bucket: R2_BUCKET_NAME, + Key: key, + }; + + let usedConditionalIfRange = false; + if (range) { + commandInput.Range = range; + + if (ifRange) { + const token = ifRange.trim(); + if (isStrongEtag(token)) { + commandInput.IfMatch = token; + usedConditionalIfRange = true; + } else { + const asDate = parseHttpDate(token); + if (asDate) { + commandInput.IfUnmodifiedSince = asDate; + usedConditionalIfRange = true; + } + } + } + } + + let objectResponse: GetObjectCommandOutput; + try { + objectResponse = await r2Client.send(new GetObjectCommand(commandInput)); + } catch (error) { + if (usedConditionalIfRange && range && isPreconditionFailed(error)) { + try { + objectResponse = await r2Client.send( + new GetObjectCommand({ + Bucket: R2_BUCKET_NAME, + Key: key, + }) + ); + } catch (retryError) { + if (isNotFoundError(retryError)) { + return apiErrors.notFound(notFoundLabel); + } + if (isInvalidRangeError(retryError)) { + return new NextResponse(null, { + status: 416, + headers: { + 'Cache-Control': cacheControl, + 'Accept-Ranges': 'bytes', + }, + }); + } + console.error('Error proxying R2 object:', retryError); + return apiErrors.internalError(internalErrorMessage); + } + } else if (isNotFoundError(error)) { + return apiErrors.notFound(notFoundLabel); + } else if (isInvalidRangeError(error)) { + return new NextResponse(null, { + status: 416, + headers: { + 'Cache-Control': cacheControl, + 'Accept-Ranges': 'bytes', + }, + }); + } else { + console.error('Error proxying R2 object:', error); + return apiErrors.internalError(internalErrorMessage); + } + } + + const stream = toWebStream(objectResponse.Body); + if (!stream) { + return apiErrors.internalError('Empty file'); + } + + const headers = new Headers(); + setIfPresent(headers, 'Content-Type', objectResponse.ContentType || fallbackContentType); + setIfPresent(headers, 'Content-Length', objectResponse.ContentLength); + setIfPresent(headers, 'Content-Range', objectResponse.ContentRange); + setIfPresent(headers, 'ETag', objectResponse.ETag); + setIfPresent(headers, 'Last-Modified', objectResponse.LastModified?.toUTCString()); + setIfPresent(headers, 'Accept-Ranges', objectResponse.AcceptRanges || 'bytes'); + headers.set('Cache-Control', cacheControl); + + if (extraHeaders) { + for (const [name, value] of Object.entries(extraHeaders)) { + headers.set(name, value); + } + } + + return new NextResponse(stream, { + status: objectResponse.ContentRange ? 206 : 200, + headers, + }); +} diff --git a/package.json b/package.json index f13cc6f..b8d96ef 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,11 @@ "db:migrate": "prisma migrate deploy", "db:seed": "prisma db seed", "db:setup": "bun run db:generate && bun run db:push && bun run db:extras", - "db:extras": "bun run scripts/db-extras.ts" + "db:extras": "bun run scripts/db-extras.ts", + "r2:cleanup-orphans:dry": "bun run scripts/r2-orphan-cleanup.ts --dry-run", + "r2:cleanup-orphans": "bun run scripts/r2-orphan-cleanup.ts", + "bunny:cleanup-orphans:dry": "bun run scripts/bunny-orphan-cleanup.ts --dry-run", + "bunny:cleanup-orphans": "bun run scripts/bunny-orphan-cleanup.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.985.0", diff --git a/scripts/bunny-orphan-cleanup.ts b/scripts/bunny-orphan-cleanup.ts new file mode 100644 index 0000000..1301fae --- /dev/null +++ b/scripts/bunny-orphan-cleanup.ts @@ -0,0 +1,238 @@ +import { db, disconnectDb } from '../lib/db'; + +const BUNNY_API_BASE = 'https://video.bunnycdn.com'; +const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; +const ITEMS_PER_PAGE = 100; +const MAX_PAGES = 200; +const CHUNK_SIZE = 500; +const DEFAULT_GRACE_HOURS = 24; + +type BunnyConfig = { + apiKey: string; + libraryId: string; +}; + +type BunnyVideo = { + id: string; + uploadedAt: Date; +}; + +function getBunnyConfig(): BunnyConfig { + const apiKey = process.env.BUNNY_STREAM_API_KEY; + const libraryId = process.env.BUNNY_STREAM_LIBRARY_ID || process.env.NEXT_PUBLIC_BUNNY_STREAM_LIBRARY_ID; + + if (!apiKey || !libraryId) { + throw new Error('Missing BUNNY_STREAM_API_KEY or BUNNY_STREAM_LIBRARY_ID.'); + } + + return { apiKey, libraryId }; +} + +function toRecord(value: unknown): Record | null { + if (!value || typeof value !== 'object') return null; + return value as Record; +} + +function parseVideoId(item: unknown): string | null { + const record = toRecord(item); + if (!record) return null; + + const candidates = [record.guid, record.videoId, record.id]; + for (const candidate of candidates) { + if (typeof candidate === 'string') { + const trimmed = candidate.trim(); + if (BUNNY_VIDEO_ID_PATTERN.test(trimmed)) return trimmed; + } + } + + return null; +} + +function parseUploadedAt(item: unknown): Date | null { + const record = toRecord(item); + if (!record) return null; + + const candidates = [ + record.dateUploaded, + record.DateUploaded, + record.dateCreated, + record.DateCreated, + record.createdAt, + record.CreatedAt, + ]; + + for (const candidate of candidates) { + if (typeof candidate !== 'string') continue; + const parsed = new Date(candidate); + if (!Number.isNaN(parsed.getTime())) return parsed; + } + + return null; +} + +async function fetchBunnyPage(config: BunnyConfig, page: number): Promise<{ items: unknown[]; totalItems: number | null }> { + const response = await fetch( + `${BUNNY_API_BASE}/library/${config.libraryId}/videos?page=${page}&itemsPerPage=${ITEMS_PER_PAGE}`, + { + headers: { + AccessKey: config.apiKey, + Accept: 'application/json', + }, + cache: 'no-store', + } + ); + + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`Bunny list API failed (${response.status}): ${body.slice(0, 300)}`); + } + + const payload = await response.json(); + const record = toRecord(payload); + if (!record) return { items: [], totalItems: null }; + + const items = Array.isArray(record.items) + ? record.items + : (Array.isArray(record.Items) ? record.Items : []); + + const totalItems = typeof record.totalItems === 'number' + ? record.totalItems + : (typeof record.TotalItems === 'number' ? record.TotalItems : null); + + return { items, totalItems }; +} + +async function listBunnyVideos(config: BunnyConfig): Promise<{ videos: BunnyVideo[]; scanned: number; skippedInvalid: number }> { + const videos: BunnyVideo[] = []; + let scanned = 0; + let skippedInvalid = 0; + let page = 1; + + while (page <= MAX_PAGES) { + const { items, totalItems } = await fetchBunnyPage(config, page); + if (items.length === 0) break; + + scanned += items.length; + for (const item of items) { + const id = parseVideoId(item); + const uploadedAt = parseUploadedAt(item); + if (!id || !uploadedAt) { + skippedInvalid += 1; + continue; + } + videos.push({ id, uploadedAt }); + } + + if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break; + page += 1; + } + + return { videos, scanned, skippedInvalid }; +} + +function chunk(items: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) { + out.push(items.slice(i, i + size)); + } + return out; +} + +async function findReferencedVideoIds(videoIds: string[]): Promise> { + const referenced = new Set(); + + for (const group of chunk(videoIds, CHUNK_SIZE)) { + const rows = await db.videoVersion.findMany({ + where: { + providerId: 'bunny', + videoId: { in: group }, + }, + select: { videoId: true }, + }); + rows.forEach((row) => { + if (row.videoId) referenced.add(row.videoId); + }); + } + + return referenced; +} + +async function deleteBunnyVideo(config: BunnyConfig, videoId: string): Promise<'deleted' | 'already_missing'> { + const response = await fetch( + `${BUNNY_API_BASE}/library/${config.libraryId}/videos/${encodeURIComponent(videoId)}`, + { + method: 'DELETE', + headers: { + AccessKey: config.apiKey, + }, + } + ); + + if (response.status === 404) return 'already_missing'; + if (response.ok) return 'deleted'; + + const body = await response.text().catch(() => ''); + throw new Error(`Bunny delete API failed for ${videoId} (${response.status}): ${body.slice(0, 300)}`); +} + +async function main() { + const dryRun = process.argv.includes('--dry-run'); + const graceHours = DEFAULT_GRACE_HOURS; + const graceMs = graceHours * 60 * 60 * 1000; + const cutoff = Date.now() - graceMs; + + const config = getBunnyConfig(); + console.log(`[bunny-orphan-cleanup] Starting (${dryRun ? 'dry-run' : 'delete mode'})`); + console.log(`[bunny-orphan-cleanup] Grace period: ${graceHours}h`); + + const { videos, scanned, skippedInvalid } = await listBunnyVideos(config); + const eligible = videos.filter((video) => video.uploadedAt.getTime() <= cutoff); + console.log(`[bunny-orphan-cleanup] Scanned: ${scanned}`); + console.log(`[bunny-orphan-cleanup] Skipped invalid metadata: ${skippedInvalid}`); + console.log(`[bunny-orphan-cleanup] Eligible (old enough): ${eligible.length}`); + + if (eligible.length === 0) { + console.log('[bunny-orphan-cleanup] No eligible Bunny videos found'); + return; + } + + const eligibleIds = eligible.map((video) => video.id); + const referenced = await findReferencedVideoIds(eligibleIds); + + const orphanIds = eligibleIds.filter((id) => !referenced.has(id)); + let deleted = 0; + let alreadyMissing = 0; + let failed = 0; + + for (const orphanId of orphanIds) { + if (dryRun) continue; + + try { + const result = await deleteBunnyVideo(config, orphanId); + if (result === 'already_missing') { + alreadyMissing += 1; + } else { + deleted += 1; + } + } catch (error) { + failed += 1; + console.error(`[bunny-orphan-cleanup] Failed deleting ${orphanId}:`, error); + } + } + + console.log('[bunny-orphan-cleanup] Summary'); + console.log(`[bunny-orphan-cleanup] Referenced: ${referenced.size}`); + console.log(`[bunny-orphan-cleanup] Orphaned: ${orphanIds.length}`); + console.log(`[bunny-orphan-cleanup] Deleted: ${deleted}`); + console.log(`[bunny-orphan-cleanup] Already missing: ${alreadyMissing}`); + console.log(`[bunny-orphan-cleanup] Failed: ${failed}`); +} + +main() + .catch((error) => { + console.error('[bunny-orphan-cleanup] Fatal error:', error); + process.exitCode = 1; + }) + .finally(async () => { + await disconnectDb(); + }); diff --git a/scripts/r2-orphan-cleanup.ts b/scripts/r2-orphan-cleanup.ts new file mode 100644 index 0000000..c9559e9 --- /dev/null +++ b/scripts/r2-orphan-cleanup.ts @@ -0,0 +1,176 @@ +import { DeleteObjectCommand, ListObjectsV2Command, type ListObjectsV2CommandInput } from '@aws-sdk/client-s3'; +import { db, disconnectDb } from '../lib/db'; +import { r2Client, R2_BUCKET_NAME } from '../lib/r2'; + +const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000; +const CHUNK_SIZE = 500; +const PREFIXES = ['images/', 'voice/'] as const; + +type CleanupCandidate = { + key: string; + url: string; +}; + +type UserFeedbackScreenshotDelegate = { + findMany: (args: { where: { url: { in: string[] } }; select: { url: true } }) => Promise>; +}; + +function keyToProxyUrl(key: string): string | null { + if (key.startsWith('images/')) { + const filename = key.slice('images/'.length); + return filename ? `/api/upload/image/${filename}` : null; + } + if (key.startsWith('voice/')) { + const filename = key.slice('voice/'.length); + return filename ? `/api/upload/audio/${filename}` : null; + } + return null; +} + +function chunk(items: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) { + out.push(items.slice(i, i + size)); + } + return out; +} + +async function listCleanupCandidates(): Promise<{ candidates: CleanupCandidate[]; scanned: number }> { + const candidates: CleanupCandidate[] = []; + const cutoff = Date.now() - UNATTACHED_UPLOAD_TTL_MS; + let scanned = 0; + + for (const prefix of PREFIXES) { + let continuationToken: string | undefined; + let isTruncated = true; + + while (isTruncated) { + const input: ListObjectsV2CommandInput = { + Bucket: R2_BUCKET_NAME, + Prefix: prefix, + }; + if (continuationToken) input.ContinuationToken = continuationToken; + + const response = await r2Client.send(new ListObjectsV2Command(input)); + const contents = response.Contents ?? []; + scanned += contents.length; + + for (const item of contents) { + if (!item.Key || !item.LastModified) continue; + if (item.LastModified.getTime() > cutoff) continue; + + const url = keyToProxyUrl(item.Key); + if (!url) continue; + candidates.push({ key: item.Key, url }); + } + + isTruncated = response.IsTruncated ?? false; + continuationToken = response.NextContinuationToken; + } + } + + return { candidates, scanned }; +} + +async function findReferencedUrls(urls: string[]): Promise> { + const referenced = new Set(); + const userFeedbackScreenshotDelegate = (db as unknown as { + userFeedbackScreenshot?: UserFeedbackScreenshotDelegate; + }).userFeedbackScreenshot; + + for (const group of chunk(urls, CHUNK_SIZE)) { + const [commentRows, feedbackRows, feedbackAttachmentRows] = await Promise.all([ + db.comment.findMany({ + where: { + OR: [{ voiceUrl: { in: group } }, { imageUrl: { in: group } }], + }, + select: { + voiceUrl: true, + imageUrl: true, + }, + }), + db.userFeedback.findMany({ + where: { screenshotUrl: { in: group } }, + select: { screenshotUrl: true }, + }), + userFeedbackScreenshotDelegate + ? userFeedbackScreenshotDelegate.findMany({ + where: { url: { in: group } }, + select: { url: true }, + }) + : Promise.resolve([] as Array<{ url: string }>), + ]); + + for (const row of commentRows) { + if (row.voiceUrl) referenced.add(row.voiceUrl); + if (row.imageUrl) referenced.add(row.imageUrl); + } + for (const row of feedbackRows) { + if (row.screenshotUrl) referenced.add(row.screenshotUrl); + } + for (const row of feedbackAttachmentRows) { + if (row.url) referenced.add(row.url); + } + } + + return referenced; +} + +async function main() { + const dryRun = process.argv.includes('--dry-run'); + console.log(`[r2-orphan-cleanup] Starting (${dryRun ? 'dry-run' : 'delete mode'})`); + + const { candidates, scanned } = await listCleanupCandidates(); + console.log(`[r2-orphan-cleanup] Scanned: ${scanned}, eligible (old enough): ${candidates.length}`); + + if (candidates.length === 0) { + console.log('[r2-orphan-cleanup] No eligible objects found'); + return; + } + + const referenced = await findReferencedUrls(candidates.map((candidate) => candidate.url)); + + let deleted = 0; + let failed = 0; + let orphaned = 0; + let referencedCount = 0; + + for (const candidate of candidates) { + if (referenced.has(candidate.url)) { + referencedCount += 1; + continue; + } + + orphaned += 1; + if (dryRun) continue; + + try { + await r2Client.send( + new DeleteObjectCommand({ + Bucket: R2_BUCKET_NAME, + Key: candidate.key, + }) + ); + deleted += 1; + } catch (error) { + failed += 1; + console.error(`[r2-orphan-cleanup] Failed deleting ${candidate.key}:`, error); + } + } + + console.log('[r2-orphan-cleanup] Summary'); + console.log(`[r2-orphan-cleanup] Referenced: ${referencedCount}`); + console.log(`[r2-orphan-cleanup] Orphaned: ${orphaned}`); + console.log(`[r2-orphan-cleanup] Deleted: ${deleted}`); + console.log(`[r2-orphan-cleanup] Failed: ${failed}`); +} + +main() + .catch((error) => { + console.error('[r2-orphan-cleanup] Fatal error:', error); + process.exitCode = 1; + }) + .finally(async () => { + await disconnectDb(); + r2Client.destroy(); + });