feat(storage): add shared R2 media proxy and orphan cleanup tooling for R2/Bunny

This commit is contained in:
Yusuf İpek
2026-02-24 17:02:54 +03:00
parent ffa55d7dcc
commit fa51847aa4
9 changed files with 703 additions and 172 deletions
+78
View File
@@ -0,0 +1,78 @@
# Copy this file to .env and fill in your values
# ============================================================================
# DATABASE
# ============================================================================
# PostgreSQL connection string
DATABASE_URL="postgresql://user:password@localhost:5432/openframe?schema=public"
# ============================================================================
# AUTHENTICATION - NextAuth.js
# ============================================================================
NEXTAUTH_URL="http://localhost:3000"
NEXTAUTH_SECRET="your-secret-key-here-generate-with-openssl-rand-base64-32"
# ============================================================================
# OAUTH PROVIDERS
# ============================================================================
# Google OAuth
# Create credentials at: https://console.cloud.google.com/apis/credentials
GOOGLE_CLIENT_ID="your-google-client-id.apps.googleusercontent.com"
GOOGLE_CLIENT_SECRET="your-google-client-secret"
# GitHub OAuth
# Create credentials at: https://github.com/settings/developers
GITHUB_CLIENT_ID="your-github-client-id"
GITHUB_CLIENT_SECRET="your-github-client-secret"
# ============================================================================
# FILE STORAGE
# ============================================================================
# Cloudflare R2 (S3-compatible)
R2_ACCOUNT_ID="your-account-id"
R2_ACCESS_KEY_ID="your-access-key"
R2_SECRET_ACCESS_KEY="your-secret-key"
R2_BUCKET_NAME="openframe"
# R2 orphan cleanup configuration (script + external cron; app runtime does not schedule this)
# R2_ORPHAN_CLEANUP_CRON="*/15 * * * *"
# Scheduled delete mode:
# */15 * * * * cd /home/yusuf/Programming/OpenFrame && bun run r2:cleanup-orphans
# ============================================================================
# EMAIL & NOTIFICATIONS
# ============================================================================
SMTP_HOST="smtp.gmail.com"
SMTP_PORT="587"
SMTP_USER="[email protected]"
SMTP_PASSWORD="your-app-specific-password"
SMTP_FROM="[email protected]"
# ============================================================================
# APPLICATION
# ============================================================================
# development | production | test
NODE_ENV="development"
# Admin emails for accessing the /admin panel (comma separated list)
# e.g., "[email protected],[email protected]"
ADMIN_EMAILS=""
# Invite code for internal registration (required to sign up)
# Generate a secure code for your team
INVITE_CODE="your-secret-invite-code"
# Enable debug logging
# DEBUG="openframe:*"
# ============================================================================
# VIDEO PROCESSING
# ============================================================================
# Bunny Stream for direct video uploads
BUNNY_STREAM_API_KEY="your-stream-library-api-key"
BUNNY_STREAM_LIBRARY_ID="your-library-id"
# Bunny Core API key (account-level) used to enforce KeepOriginalFiles/ExposeOriginals on the library
BUNNY_API_KEY="your-account-api-key"
# Bunny Stream CDN base URL (for HLS streaming)
BUNNY_CDN_URL="https://vz-965f4f4a-fc1.b-cdn.net"
# Bunny orphan cleanup configuration (script + external cron; app runtime does not schedule this)
# Grace period is fixed at 24 hours in the script.
# */15 * * * * cd /home/yusuf/Programming/OpenFrame && bun run bunny:cleanup-orphans
+1
View File
@@ -32,6 +32,7 @@ yarn-error.log*
# env files (can opt-in for committing if needed)
.env*
!.env.example
# vercel
.vercel
+8 -76
View File
@@ -1,8 +1,5 @@
import { NextResponse } from 'next/server';
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
import { apiErrors } from '@/lib/api-response';
import { db } from '@/lib/db';
import { proxyR2MediaObject } from '@/lib/r2-media-proxy';
// Only allow UUID filenames with safe extensions
const SAFE_FILENAME = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.[a-z0-9]+$/i;
@@ -16,15 +13,13 @@ const CONTENT_TYPE_MAP: Record<string, string> = {
ogg: 'audio/ogg',
wav: 'audio/wav',
};
const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000;
function getContentType(filename: string): string {
const ext = filename.split('.').pop()?.toLowerCase() || '';
return CONTENT_TYPE_MAP[ext] || 'audio/webm';
}
export async function GET(
_request: Request,
request: Request,
{ params }: { params: Promise<{ filename: string }> }
) {
try {
@@ -36,77 +31,14 @@ export async function GET(
}
const key = `voice/${filename}`;
const mediaUrl = `/api/upload/audio/${filename}`;
// Get file metadata to determine content type
const headResponse = await r2Client.send(
new HeadObjectCommand({
Bucket: R2_BUCKET_NAME,
Key: key,
})
);
const lastModified = headResponse.LastModified;
if (lastModified && Date.now() - lastModified.getTime() > UNATTACHED_UPLOAD_TTL_MS) {
const referenced = await db.comment.findFirst({
where: { voiceUrl: mediaUrl },
select: { id: true },
});
if (!referenced) {
await r2Client.send(
new DeleteObjectCommand({
Bucket: R2_BUCKET_NAME,
Key: key,
})
).catch(() => undefined);
return apiErrors.notFound('File');
}
}
// Use the stored content-type or infer from filename extension
const contentType = headResponse.ContentType || getContentType(filename);
// Get the object
const objectResponse = await r2Client.send(
new GetObjectCommand({
Bucket: R2_BUCKET_NAME,
Key: key,
})
);
// Handle the body properly - AWS SDK returns a stream
const body = objectResponse.Body;
if (!body) {
return apiErrors.internalError('Empty file');
}
// Convert stream to Uint8Array
const chunks: Uint8Array[] = [];
const asyncIterable = body as AsyncIterable<Uint8Array>;
for await (const chunk of asyncIterable) {
chunks.push(chunk);
}
const uint8Array = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0));
let offset = 0;
for (const chunk of chunks) {
uint8Array.set(chunk, offset);
offset += chunk.length;
}
// Create response with proper content-type
return new NextResponse(uint8Array, {
status: 200,
headers: {
'Content-Type': contentType,
'Cache-Control': 'private, no-store',
'Accept-Ranges': 'bytes',
},
return proxyR2MediaObject({
request,
key,
fallbackContentType: getContentType(filename),
cacheControl: 'private, no-store',
internalErrorMessage: 'Failed to retrieve audio',
});
} catch (error: unknown) {
const errorName = error instanceof Error ? error.name : '';
if (errorName === 'NoSuchKey') {
return apiErrors.notFound('File');
}
console.error('Error serving audio:', error);
return apiErrors.internalError('Failed to retrieve audio');
}
+9 -87
View File
@@ -1,8 +1,5 @@
import { NextResponse } from 'next/server';
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
import { apiErrors } from '@/lib/api-response';
import { db } from '@/lib/db';
import { proxyR2MediaObject } from '@/lib/r2-media-proxy';
// Only allow UUID filenames with safe extensions
const SAFE_FILENAME = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.[a-z0-9]+$/i;
@@ -14,15 +11,13 @@ const CONTENT_TYPE_MAP: Record<string, string> = {
webp: 'image/webp',
gif: 'image/gif',
};
const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000;
function getContentType(filename: string): string {
const ext = filename.split('.').pop()?.toLowerCase() || '';
return CONTENT_TYPE_MAP[ext] || 'application/octet-stream';
}
export async function GET(
_request: Request,
request: Request,
{ params }: { params: Promise<{ filename: string }> }
) {
try {
@@ -34,91 +29,18 @@ export async function GET(
}
const key = `images/${filename}`;
const mediaUrl = `/api/upload/image/${filename}`;
// Get file metadata to determine content type
const headResponse = await r2Client.send(
new HeadObjectCommand({
Bucket: R2_BUCKET_NAME,
Key: key,
})
);
const lastModified = headResponse.LastModified;
if (lastModified && Date.now() - lastModified.getTime() > UNATTACHED_UPLOAD_TTL_MS) {
const userFeedbackScreenshotDelegate = (db as unknown as {
userFeedbackScreenshot?: {
findFirst: (args?: unknown) => Promise<{ id: string } | null>;
};
}).userFeedbackScreenshot;
const [commentReferenced, feedbackReferenced, feedbackAttachmentReferenced] = await Promise.all([
db.comment.findFirst({
where: { imageUrl: mediaUrl },
select: { id: true },
}),
db.userFeedback.findFirst({
where: { screenshotUrl: mediaUrl },
select: { id: true },
}),
userFeedbackScreenshotDelegate
? userFeedbackScreenshotDelegate.findFirst({
where: { url: mediaUrl },
select: { id: true },
})
: Promise.resolve(null),
]);
if (!commentReferenced && !feedbackReferenced && !feedbackAttachmentReferenced) {
await r2Client.send(
new DeleteObjectCommand({
Bucket: R2_BUCKET_NAME,
Key: key,
})
).catch(() => undefined);
return apiErrors.notFound('File');
}
}
const contentType = getContentType(filename);
const objectResponse = await r2Client.send(
new GetObjectCommand({
Bucket: R2_BUCKET_NAME,
Key: key,
})
);
const body = objectResponse.Body;
if (!body) {
return apiErrors.internalError('Empty file');
}
const chunks: Uint8Array[] = [];
// @ts-expect-error - body is an iterable
for await (const chunk of body) {
chunks.push(chunk);
}
const uint8Array = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0));
let offset = 0;
for (const chunk of chunks) {
uint8Array.set(chunk, offset);
offset += chunk.length;
}
return new NextResponse(uint8Array, {
status: 200,
headers: {
'Content-Type': contentType,
'Cache-Control': 'private, no-store',
'Accept-Ranges': 'bytes',
return proxyR2MediaObject({
request,
key,
fallbackContentType: getContentType(filename),
cacheControl: 'private, no-store',
extraHeaders: {
'X-Content-Type-Options': 'nosniff',
'Content-Security-Policy': "default-src 'none'; sandbox",
},
internalErrorMessage: 'Failed to retrieve image',
});
} catch (error: unknown) {
const errorName = error instanceof Error ? error.name : '';
if (errorName === 'NoSuchKey') {
return apiErrors.notFound('File');
}
console.error('Error serving image:', error);
return apiErrors.internalError('Failed to retrieve image');
}
+10 -8
View File
@@ -77,17 +77,19 @@ if (process.env.NODE_ENV !== 'production') {
globalForPrisma.prisma = db;
}
export async function disconnectDb(): Promise<void> {
if (globalForPool.pgPool) {
await globalForPool.pgPool.end();
globalForPool.pgPool = undefined;
}
await db.$disconnect();
}
// Graceful shutdown handler
async function shutdown() {
console.log('Shutting down database connections...');
if (globalForPool.pgPool) {
await globalForPool.pgPool.end();
console.log('Database pool closed');
}
await db.$disconnect();
console.log('Prisma client disconnected');
await disconnectDb();
console.log('Database connections closed');
}
// Register shutdown handlers
+178
View File
@@ -0,0 +1,178 @@
import { GetObjectCommand, type GetObjectCommandInput, type GetObjectCommandOutput } from '@aws-sdk/client-s3';
import { Readable } from 'node:stream';
import { NextResponse } from 'next/server';
import { apiErrors } from '@/lib/api-response';
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
type ProxyR2MediaOptions = {
request: Request;
key: string;
fallbackContentType: string;
cacheControl: string;
extraHeaders?: Record<string, string>;
notFoundLabel?: string;
internalErrorMessage: string;
};
type R2LikeError = {
name?: string;
Code?: string;
$metadata?: { httpStatusCode?: number };
};
function isStrongEtag(value: string): boolean {
return /^"[^"]+"$/.test(value);
}
function parseHttpDate(value: string): Date | null {
const parsed = new Date(value);
return Number.isNaN(parsed.getTime()) ? null : parsed;
}
function getErrorStatus(error: unknown): number | null {
const status = (error as R2LikeError | null | undefined)?.$metadata?.httpStatusCode;
return typeof status === 'number' ? status : null;
}
function isNotFoundError(error: unknown): boolean {
const err = error as R2LikeError | null | undefined;
return err?.name === 'NoSuchKey' || err?.Code === 'NoSuchKey' || getErrorStatus(error) === 404;
}
function isInvalidRangeError(error: unknown): boolean {
const err = error as R2LikeError | null | undefined;
return err?.name === 'InvalidRange' || err?.Code === 'InvalidRange' || getErrorStatus(error) === 416;
}
function isPreconditionFailed(error: unknown): boolean {
return getErrorStatus(error) === 412;
}
function toWebStream(body: unknown): ReadableStream<Uint8Array> | null {
if (!body) return null;
const withTransform = body as { transformToWebStream?: () => ReadableStream<Uint8Array> };
if (typeof withTransform.transformToWebStream === 'function') {
return withTransform.transformToWebStream();
}
if (body instanceof Readable) {
return Readable.toWeb(body) as ReadableStream<Uint8Array>;
}
if (body instanceof ReadableStream) {
return body;
}
return null;
}
function setIfPresent(headers: Headers, key: string, value: string | number | null | undefined): void {
if (value === undefined || value === null) return;
headers.set(key, String(value));
}
export async function proxyR2MediaObject({
request,
key,
fallbackContentType,
cacheControl,
extraHeaders,
notFoundLabel = 'File',
internalErrorMessage,
}: ProxyR2MediaOptions): Promise<NextResponse> {
const range = request.headers.get('range');
const ifRange = request.headers.get('if-range');
const commandInput: GetObjectCommandInput = {
Bucket: R2_BUCKET_NAME,
Key: key,
};
let usedConditionalIfRange = false;
if (range) {
commandInput.Range = range;
if (ifRange) {
const token = ifRange.trim();
if (isStrongEtag(token)) {
commandInput.IfMatch = token;
usedConditionalIfRange = true;
} else {
const asDate = parseHttpDate(token);
if (asDate) {
commandInput.IfUnmodifiedSince = asDate;
usedConditionalIfRange = true;
}
}
}
}
let objectResponse: GetObjectCommandOutput;
try {
objectResponse = await r2Client.send(new GetObjectCommand(commandInput));
} catch (error) {
if (usedConditionalIfRange && range && isPreconditionFailed(error)) {
try {
objectResponse = await r2Client.send(
new GetObjectCommand({
Bucket: R2_BUCKET_NAME,
Key: key,
})
);
} catch (retryError) {
if (isNotFoundError(retryError)) {
return apiErrors.notFound(notFoundLabel);
}
if (isInvalidRangeError(retryError)) {
return new NextResponse(null, {
status: 416,
headers: {
'Cache-Control': cacheControl,
'Accept-Ranges': 'bytes',
},
});
}
console.error('Error proxying R2 object:', retryError);
return apiErrors.internalError(internalErrorMessage);
}
} else if (isNotFoundError(error)) {
return apiErrors.notFound(notFoundLabel);
} else if (isInvalidRangeError(error)) {
return new NextResponse(null, {
status: 416,
headers: {
'Cache-Control': cacheControl,
'Accept-Ranges': 'bytes',
},
});
} else {
console.error('Error proxying R2 object:', error);
return apiErrors.internalError(internalErrorMessage);
}
}
const stream = toWebStream(objectResponse.Body);
if (!stream) {
return apiErrors.internalError('Empty file');
}
const headers = new Headers();
setIfPresent(headers, 'Content-Type', objectResponse.ContentType || fallbackContentType);
setIfPresent(headers, 'Content-Length', objectResponse.ContentLength);
setIfPresent(headers, 'Content-Range', objectResponse.ContentRange);
setIfPresent(headers, 'ETag', objectResponse.ETag);
setIfPresent(headers, 'Last-Modified', objectResponse.LastModified?.toUTCString());
setIfPresent(headers, 'Accept-Ranges', objectResponse.AcceptRanges || 'bytes');
headers.set('Cache-Control', cacheControl);
if (extraHeaders) {
for (const [name, value] of Object.entries(extraHeaders)) {
headers.set(name, value);
}
}
return new NextResponse(stream, {
status: objectResponse.ContentRange ? 206 : 200,
headers,
});
}
+5 -1
View File
@@ -16,7 +16,11 @@
"db:migrate": "prisma migrate deploy",
"db:seed": "prisma db seed",
"db:setup": "bun run db:generate && bun run db:push && bun run db:extras",
"db:extras": "bun run scripts/db-extras.ts"
"db:extras": "bun run scripts/db-extras.ts",
"r2:cleanup-orphans:dry": "bun run scripts/r2-orphan-cleanup.ts --dry-run",
"r2:cleanup-orphans": "bun run scripts/r2-orphan-cleanup.ts",
"bunny:cleanup-orphans:dry": "bun run scripts/bunny-orphan-cleanup.ts --dry-run",
"bunny:cleanup-orphans": "bun run scripts/bunny-orphan-cleanup.ts"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.985.0",
+238
View File
@@ -0,0 +1,238 @@
import { db, disconnectDb } from '../lib/db';
const BUNNY_API_BASE = 'https://video.bunnycdn.com';
const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
const ITEMS_PER_PAGE = 100;
const MAX_PAGES = 200;
const CHUNK_SIZE = 500;
const DEFAULT_GRACE_HOURS = 24;
type BunnyConfig = {
apiKey: string;
libraryId: string;
};
type BunnyVideo = {
id: string;
uploadedAt: Date;
};
function getBunnyConfig(): BunnyConfig {
const apiKey = process.env.BUNNY_STREAM_API_KEY;
const libraryId = process.env.BUNNY_STREAM_LIBRARY_ID || process.env.NEXT_PUBLIC_BUNNY_STREAM_LIBRARY_ID;
if (!apiKey || !libraryId) {
throw new Error('Missing BUNNY_STREAM_API_KEY or BUNNY_STREAM_LIBRARY_ID.');
}
return { apiKey, libraryId };
}
function toRecord(value: unknown): Record<string, unknown> | null {
if (!value || typeof value !== 'object') return null;
return value as Record<string, unknown>;
}
function parseVideoId(item: unknown): string | null {
const record = toRecord(item);
if (!record) return null;
const candidates = [record.guid, record.videoId, record.id];
for (const candidate of candidates) {
if (typeof candidate === 'string') {
const trimmed = candidate.trim();
if (BUNNY_VIDEO_ID_PATTERN.test(trimmed)) return trimmed;
}
}
return null;
}
function parseUploadedAt(item: unknown): Date | null {
const record = toRecord(item);
if (!record) return null;
const candidates = [
record.dateUploaded,
record.DateUploaded,
record.dateCreated,
record.DateCreated,
record.createdAt,
record.CreatedAt,
];
for (const candidate of candidates) {
if (typeof candidate !== 'string') continue;
const parsed = new Date(candidate);
if (!Number.isNaN(parsed.getTime())) return parsed;
}
return null;
}
async function fetchBunnyPage(config: BunnyConfig, page: number): Promise<{ items: unknown[]; totalItems: number | null }> {
const response = await fetch(
`${BUNNY_API_BASE}/library/${config.libraryId}/videos?page=${page}&itemsPerPage=${ITEMS_PER_PAGE}`,
{
headers: {
AccessKey: config.apiKey,
Accept: 'application/json',
},
cache: 'no-store',
}
);
if (!response.ok) {
const body = await response.text().catch(() => '');
throw new Error(`Bunny list API failed (${response.status}): ${body.slice(0, 300)}`);
}
const payload = await response.json();
const record = toRecord(payload);
if (!record) return { items: [], totalItems: null };
const items = Array.isArray(record.items)
? record.items
: (Array.isArray(record.Items) ? record.Items : []);
const totalItems = typeof record.totalItems === 'number'
? record.totalItems
: (typeof record.TotalItems === 'number' ? record.TotalItems : null);
return { items, totalItems };
}
async function listBunnyVideos(config: BunnyConfig): Promise<{ videos: BunnyVideo[]; scanned: number; skippedInvalid: number }> {
const videos: BunnyVideo[] = [];
let scanned = 0;
let skippedInvalid = 0;
let page = 1;
while (page <= MAX_PAGES) {
const { items, totalItems } = await fetchBunnyPage(config, page);
if (items.length === 0) break;
scanned += items.length;
for (const item of items) {
const id = parseVideoId(item);
const uploadedAt = parseUploadedAt(item);
if (!id || !uploadedAt) {
skippedInvalid += 1;
continue;
}
videos.push({ id, uploadedAt });
}
if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break;
page += 1;
}
return { videos, scanned, skippedInvalid };
}
function chunk<T>(items: T[], size: number): T[][] {
const out: T[][] = [];
for (let i = 0; i < items.length; i += size) {
out.push(items.slice(i, i + size));
}
return out;
}
async function findReferencedVideoIds(videoIds: string[]): Promise<Set<string>> {
const referenced = new Set<string>();
for (const group of chunk(videoIds, CHUNK_SIZE)) {
const rows = await db.videoVersion.findMany({
where: {
providerId: 'bunny',
videoId: { in: group },
},
select: { videoId: true },
});
rows.forEach((row) => {
if (row.videoId) referenced.add(row.videoId);
});
}
return referenced;
}
async function deleteBunnyVideo(config: BunnyConfig, videoId: string): Promise<'deleted' | 'already_missing'> {
const response = await fetch(
`${BUNNY_API_BASE}/library/${config.libraryId}/videos/${encodeURIComponent(videoId)}`,
{
method: 'DELETE',
headers: {
AccessKey: config.apiKey,
},
}
);
if (response.status === 404) return 'already_missing';
if (response.ok) return 'deleted';
const body = await response.text().catch(() => '');
throw new Error(`Bunny delete API failed for ${videoId} (${response.status}): ${body.slice(0, 300)}`);
}
async function main() {
const dryRun = process.argv.includes('--dry-run');
const graceHours = DEFAULT_GRACE_HOURS;
const graceMs = graceHours * 60 * 60 * 1000;
const cutoff = Date.now() - graceMs;
const config = getBunnyConfig();
console.log(`[bunny-orphan-cleanup] Starting (${dryRun ? 'dry-run' : 'delete mode'})`);
console.log(`[bunny-orphan-cleanup] Grace period: ${graceHours}h`);
const { videos, scanned, skippedInvalid } = await listBunnyVideos(config);
const eligible = videos.filter((video) => video.uploadedAt.getTime() <= cutoff);
console.log(`[bunny-orphan-cleanup] Scanned: ${scanned}`);
console.log(`[bunny-orphan-cleanup] Skipped invalid metadata: ${skippedInvalid}`);
console.log(`[bunny-orphan-cleanup] Eligible (old enough): ${eligible.length}`);
if (eligible.length === 0) {
console.log('[bunny-orphan-cleanup] No eligible Bunny videos found');
return;
}
const eligibleIds = eligible.map((video) => video.id);
const referenced = await findReferencedVideoIds(eligibleIds);
const orphanIds = eligibleIds.filter((id) => !referenced.has(id));
let deleted = 0;
let alreadyMissing = 0;
let failed = 0;
for (const orphanId of orphanIds) {
if (dryRun) continue;
try {
const result = await deleteBunnyVideo(config, orphanId);
if (result === 'already_missing') {
alreadyMissing += 1;
} else {
deleted += 1;
}
} catch (error) {
failed += 1;
console.error(`[bunny-orphan-cleanup] Failed deleting ${orphanId}:`, error);
}
}
console.log('[bunny-orphan-cleanup] Summary');
console.log(`[bunny-orphan-cleanup] Referenced: ${referenced.size}`);
console.log(`[bunny-orphan-cleanup] Orphaned: ${orphanIds.length}`);
console.log(`[bunny-orphan-cleanup] Deleted: ${deleted}`);
console.log(`[bunny-orphan-cleanup] Already missing: ${alreadyMissing}`);
console.log(`[bunny-orphan-cleanup] Failed: ${failed}`);
}
main()
.catch((error) => {
console.error('[bunny-orphan-cleanup] Fatal error:', error);
process.exitCode = 1;
})
.finally(async () => {
await disconnectDb();
});
+176
View File
@@ -0,0 +1,176 @@
import { DeleteObjectCommand, ListObjectsV2Command, type ListObjectsV2CommandInput } from '@aws-sdk/client-s3';
import { db, disconnectDb } from '../lib/db';
import { r2Client, R2_BUCKET_NAME } from '../lib/r2';
const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000;
const CHUNK_SIZE = 500;
const PREFIXES = ['images/', 'voice/'] as const;
type CleanupCandidate = {
key: string;
url: string;
};
type UserFeedbackScreenshotDelegate = {
findMany: (args: { where: { url: { in: string[] } }; select: { url: true } }) => Promise<Array<{ url: string }>>;
};
function keyToProxyUrl(key: string): string | null {
if (key.startsWith('images/')) {
const filename = key.slice('images/'.length);
return filename ? `/api/upload/image/${filename}` : null;
}
if (key.startsWith('voice/')) {
const filename = key.slice('voice/'.length);
return filename ? `/api/upload/audio/${filename}` : null;
}
return null;
}
function chunk<T>(items: T[], size: number): T[][] {
const out: T[][] = [];
for (let i = 0; i < items.length; i += size) {
out.push(items.slice(i, i + size));
}
return out;
}
async function listCleanupCandidates(): Promise<{ candidates: CleanupCandidate[]; scanned: number }> {
const candidates: CleanupCandidate[] = [];
const cutoff = Date.now() - UNATTACHED_UPLOAD_TTL_MS;
let scanned = 0;
for (const prefix of PREFIXES) {
let continuationToken: string | undefined;
let isTruncated = true;
while (isTruncated) {
const input: ListObjectsV2CommandInput = {
Bucket: R2_BUCKET_NAME,
Prefix: prefix,
};
if (continuationToken) input.ContinuationToken = continuationToken;
const response = await r2Client.send(new ListObjectsV2Command(input));
const contents = response.Contents ?? [];
scanned += contents.length;
for (const item of contents) {
if (!item.Key || !item.LastModified) continue;
if (item.LastModified.getTime() > cutoff) continue;
const url = keyToProxyUrl(item.Key);
if (!url) continue;
candidates.push({ key: item.Key, url });
}
isTruncated = response.IsTruncated ?? false;
continuationToken = response.NextContinuationToken;
}
}
return { candidates, scanned };
}
async function findReferencedUrls(urls: string[]): Promise<Set<string>> {
const referenced = new Set<string>();
const userFeedbackScreenshotDelegate = (db as unknown as {
userFeedbackScreenshot?: UserFeedbackScreenshotDelegate;
}).userFeedbackScreenshot;
for (const group of chunk(urls, CHUNK_SIZE)) {
const [commentRows, feedbackRows, feedbackAttachmentRows] = await Promise.all([
db.comment.findMany({
where: {
OR: [{ voiceUrl: { in: group } }, { imageUrl: { in: group } }],
},
select: {
voiceUrl: true,
imageUrl: true,
},
}),
db.userFeedback.findMany({
where: { screenshotUrl: { in: group } },
select: { screenshotUrl: true },
}),
userFeedbackScreenshotDelegate
? userFeedbackScreenshotDelegate.findMany({
where: { url: { in: group } },
select: { url: true },
})
: Promise.resolve([] as Array<{ url: string }>),
]);
for (const row of commentRows) {
if (row.voiceUrl) referenced.add(row.voiceUrl);
if (row.imageUrl) referenced.add(row.imageUrl);
}
for (const row of feedbackRows) {
if (row.screenshotUrl) referenced.add(row.screenshotUrl);
}
for (const row of feedbackAttachmentRows) {
if (row.url) referenced.add(row.url);
}
}
return referenced;
}
async function main() {
const dryRun = process.argv.includes('--dry-run');
console.log(`[r2-orphan-cleanup] Starting (${dryRun ? 'dry-run' : 'delete mode'})`);
const { candidates, scanned } = await listCleanupCandidates();
console.log(`[r2-orphan-cleanup] Scanned: ${scanned}, eligible (old enough): ${candidates.length}`);
if (candidates.length === 0) {
console.log('[r2-orphan-cleanup] No eligible objects found');
return;
}
const referenced = await findReferencedUrls(candidates.map((candidate) => candidate.url));
let deleted = 0;
let failed = 0;
let orphaned = 0;
let referencedCount = 0;
for (const candidate of candidates) {
if (referenced.has(candidate.url)) {
referencedCount += 1;
continue;
}
orphaned += 1;
if (dryRun) continue;
try {
await r2Client.send(
new DeleteObjectCommand({
Bucket: R2_BUCKET_NAME,
Key: candidate.key,
})
);
deleted += 1;
} catch (error) {
failed += 1;
console.error(`[r2-orphan-cleanup] Failed deleting ${candidate.key}:`, error);
}
}
console.log('[r2-orphan-cleanup] Summary');
console.log(`[r2-orphan-cleanup] Referenced: ${referencedCount}`);
console.log(`[r2-orphan-cleanup] Orphaned: ${orphaned}`);
console.log(`[r2-orphan-cleanup] Deleted: ${deleted}`);
console.log(`[r2-orphan-cleanup] Failed: ${failed}`);
}
main()
.catch((error) => {
console.error('[r2-orphan-cleanup] Fatal error:', error);
process.exitCode = 1;
})
.finally(async () => {
await disconnectDb();
r2Client.destroy();
});