mirror of
https://github.com/yusufipk/OpenFrame.git
synced 2026-09-11 17:46:06 +00:00
feat(storage): add shared R2 media proxy and orphan cleanup tooling for R2/Bunny
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
# Copy this file to .env and fill in your values
|
||||
|
||||
# ============================================================================
|
||||
# DATABASE
|
||||
# ============================================================================
|
||||
# PostgreSQL connection string
|
||||
DATABASE_URL="postgresql://user:password@localhost:5432/openframe?schema=public"
|
||||
|
||||
# ============================================================================
|
||||
# AUTHENTICATION - NextAuth.js
|
||||
# ============================================================================
|
||||
NEXTAUTH_URL="http://localhost:3000"
|
||||
NEXTAUTH_SECRET="your-secret-key-here-generate-with-openssl-rand-base64-32"
|
||||
|
||||
# ============================================================================
|
||||
# OAUTH PROVIDERS
|
||||
# ============================================================================
|
||||
# Google OAuth
|
||||
# Create credentials at: https://console.cloud.google.com/apis/credentials
|
||||
GOOGLE_CLIENT_ID="your-google-client-id.apps.googleusercontent.com"
|
||||
GOOGLE_CLIENT_SECRET="your-google-client-secret"
|
||||
|
||||
# GitHub OAuth
|
||||
# Create credentials at: https://github.com/settings/developers
|
||||
GITHUB_CLIENT_ID="your-github-client-id"
|
||||
GITHUB_CLIENT_SECRET="your-github-client-secret"
|
||||
|
||||
# ============================================================================
|
||||
# FILE STORAGE
|
||||
# ============================================================================
|
||||
# Cloudflare R2 (S3-compatible)
|
||||
R2_ACCOUNT_ID="your-account-id"
|
||||
R2_ACCESS_KEY_ID="your-access-key"
|
||||
R2_SECRET_ACCESS_KEY="your-secret-key"
|
||||
R2_BUCKET_NAME="openframe"
|
||||
# R2 orphan cleanup configuration (script + external cron; app runtime does not schedule this)
|
||||
# R2_ORPHAN_CLEANUP_CRON="*/15 * * * *"
|
||||
# Scheduled delete mode:
|
||||
# */15 * * * * cd /home/yusuf/Programming/OpenFrame && bun run r2:cleanup-orphans
|
||||
|
||||
# ============================================================================
|
||||
# EMAIL & NOTIFICATIONS
|
||||
# ============================================================================
|
||||
|
||||
SMTP_HOST="smtp.gmail.com"
|
||||
SMTP_PORT="587"
|
||||
SMTP_USER="[email protected]"
|
||||
SMTP_PASSWORD="your-app-specific-password"
|
||||
SMTP_FROM="[email protected]"
|
||||
|
||||
# ============================================================================
|
||||
# APPLICATION
|
||||
# ============================================================================
|
||||
# development | production | test
|
||||
NODE_ENV="development"
|
||||
# Admin emails for accessing the /admin panel (comma separated list)
|
||||
# e.g., "[email protected],[email protected]"
|
||||
ADMIN_EMAILS=""
|
||||
# Invite code for internal registration (required to sign up)
|
||||
# Generate a secure code for your team
|
||||
INVITE_CODE="your-secret-invite-code"
|
||||
# Enable debug logging
|
||||
# DEBUG="openframe:*"
|
||||
|
||||
# ============================================================================
|
||||
# VIDEO PROCESSING
|
||||
# ============================================================================
|
||||
|
||||
# Bunny Stream for direct video uploads
|
||||
BUNNY_STREAM_API_KEY="your-stream-library-api-key"
|
||||
BUNNY_STREAM_LIBRARY_ID="your-library-id"
|
||||
# Bunny Core API key (account-level) used to enforce KeepOriginalFiles/ExposeOriginals on the library
|
||||
BUNNY_API_KEY="your-account-api-key"
|
||||
# Bunny Stream CDN base URL (for HLS streaming)
|
||||
BUNNY_CDN_URL="https://vz-965f4f4a-fc1.b-cdn.net"
|
||||
# Bunny orphan cleanup configuration (script + external cron; app runtime does not schedule this)
|
||||
# Grace period is fixed at 24 hours in the script.
|
||||
# */15 * * * * cd /home/yusuf/Programming/OpenFrame && bun run bunny:cleanup-orphans
|
||||
@@ -32,6 +32,7 @@ yarn-error.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
!.env.example
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
|
||||
import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { apiErrors } from '@/lib/api-response';
|
||||
import { db } from '@/lib/db';
|
||||
import { proxyR2MediaObject } from '@/lib/r2-media-proxy';
|
||||
|
||||
// Only allow UUID filenames with safe extensions
|
||||
const SAFE_FILENAME = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.[a-z0-9]+$/i;
|
||||
@@ -16,15 +13,13 @@ const CONTENT_TYPE_MAP: Record<string, string> = {
|
||||
ogg: 'audio/ogg',
|
||||
wav: 'audio/wav',
|
||||
};
|
||||
const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000;
|
||||
|
||||
function getContentType(filename: string): string {
|
||||
const ext = filename.split('.').pop()?.toLowerCase() || '';
|
||||
return CONTENT_TYPE_MAP[ext] || 'audio/webm';
|
||||
}
|
||||
|
||||
export async function GET(
|
||||
_request: Request,
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ filename: string }> }
|
||||
) {
|
||||
try {
|
||||
@@ -36,77 +31,14 @@ export async function GET(
|
||||
}
|
||||
|
||||
const key = `voice/${filename}`;
|
||||
const mediaUrl = `/api/upload/audio/${filename}`;
|
||||
|
||||
// Get file metadata to determine content type
|
||||
const headResponse = await r2Client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
})
|
||||
);
|
||||
|
||||
const lastModified = headResponse.LastModified;
|
||||
if (lastModified && Date.now() - lastModified.getTime() > UNATTACHED_UPLOAD_TTL_MS) {
|
||||
const referenced = await db.comment.findFirst({
|
||||
where: { voiceUrl: mediaUrl },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!referenced) {
|
||||
await r2Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
})
|
||||
).catch(() => undefined);
|
||||
return apiErrors.notFound('File');
|
||||
}
|
||||
}
|
||||
|
||||
// Use the stored content-type or infer from filename extension
|
||||
const contentType = headResponse.ContentType || getContentType(filename);
|
||||
|
||||
// Get the object
|
||||
const objectResponse = await r2Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
})
|
||||
);
|
||||
|
||||
// Handle the body properly - AWS SDK returns a stream
|
||||
const body = objectResponse.Body;
|
||||
if (!body) {
|
||||
return apiErrors.internalError('Empty file');
|
||||
}
|
||||
|
||||
// Convert stream to Uint8Array
|
||||
const chunks: Uint8Array[] = [];
|
||||
const asyncIterable = body as AsyncIterable<Uint8Array>;
|
||||
for await (const chunk of asyncIterable) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const uint8Array = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0));
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
uint8Array.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
// Create response with proper content-type
|
||||
return new NextResponse(uint8Array, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'Cache-Control': 'private, no-store',
|
||||
'Accept-Ranges': 'bytes',
|
||||
},
|
||||
return proxyR2MediaObject({
|
||||
request,
|
||||
key,
|
||||
fallbackContentType: getContentType(filename),
|
||||
cacheControl: 'private, no-store',
|
||||
internalErrorMessage: 'Failed to retrieve audio',
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
const errorName = error instanceof Error ? error.name : '';
|
||||
if (errorName === 'NoSuchKey') {
|
||||
return apiErrors.notFound('File');
|
||||
}
|
||||
console.error('Error serving audio:', error);
|
||||
return apiErrors.internalError('Failed to retrieve audio');
|
||||
}
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
|
||||
import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { apiErrors } from '@/lib/api-response';
|
||||
import { db } from '@/lib/db';
|
||||
import { proxyR2MediaObject } from '@/lib/r2-media-proxy';
|
||||
|
||||
// Only allow UUID filenames with safe extensions
|
||||
const SAFE_FILENAME = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.[a-z0-9]+$/i;
|
||||
@@ -14,15 +11,13 @@ const CONTENT_TYPE_MAP: Record<string, string> = {
|
||||
webp: 'image/webp',
|
||||
gif: 'image/gif',
|
||||
};
|
||||
const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000;
|
||||
|
||||
function getContentType(filename: string): string {
|
||||
const ext = filename.split('.').pop()?.toLowerCase() || '';
|
||||
return CONTENT_TYPE_MAP[ext] || 'application/octet-stream';
|
||||
}
|
||||
|
||||
export async function GET(
|
||||
_request: Request,
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ filename: string }> }
|
||||
) {
|
||||
try {
|
||||
@@ -34,91 +29,18 @@ export async function GET(
|
||||
}
|
||||
|
||||
const key = `images/${filename}`;
|
||||
const mediaUrl = `/api/upload/image/${filename}`;
|
||||
|
||||
// Get file metadata to determine content type
|
||||
const headResponse = await r2Client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
})
|
||||
);
|
||||
|
||||
const lastModified = headResponse.LastModified;
|
||||
if (lastModified && Date.now() - lastModified.getTime() > UNATTACHED_UPLOAD_TTL_MS) {
|
||||
const userFeedbackScreenshotDelegate = (db as unknown as {
|
||||
userFeedbackScreenshot?: {
|
||||
findFirst: (args?: unknown) => Promise<{ id: string } | null>;
|
||||
};
|
||||
}).userFeedbackScreenshot;
|
||||
const [commentReferenced, feedbackReferenced, feedbackAttachmentReferenced] = await Promise.all([
|
||||
db.comment.findFirst({
|
||||
where: { imageUrl: mediaUrl },
|
||||
select: { id: true },
|
||||
}),
|
||||
db.userFeedback.findFirst({
|
||||
where: { screenshotUrl: mediaUrl },
|
||||
select: { id: true },
|
||||
}),
|
||||
userFeedbackScreenshotDelegate
|
||||
? userFeedbackScreenshotDelegate.findFirst({
|
||||
where: { url: mediaUrl },
|
||||
select: { id: true },
|
||||
})
|
||||
: Promise.resolve(null),
|
||||
]);
|
||||
if (!commentReferenced && !feedbackReferenced && !feedbackAttachmentReferenced) {
|
||||
await r2Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
})
|
||||
).catch(() => undefined);
|
||||
return apiErrors.notFound('File');
|
||||
}
|
||||
}
|
||||
|
||||
const contentType = getContentType(filename);
|
||||
|
||||
const objectResponse = await r2Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
})
|
||||
);
|
||||
|
||||
const body = objectResponse.Body;
|
||||
if (!body) {
|
||||
return apiErrors.internalError('Empty file');
|
||||
}
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
// @ts-expect-error - body is an iterable
|
||||
for await (const chunk of body) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const uint8Array = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0));
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
uint8Array.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
return new NextResponse(uint8Array, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'Cache-Control': 'private, no-store',
|
||||
'Accept-Ranges': 'bytes',
|
||||
return proxyR2MediaObject({
|
||||
request,
|
||||
key,
|
||||
fallbackContentType: getContentType(filename),
|
||||
cacheControl: 'private, no-store',
|
||||
extraHeaders: {
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
'Content-Security-Policy': "default-src 'none'; sandbox",
|
||||
},
|
||||
internalErrorMessage: 'Failed to retrieve image',
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
const errorName = error instanceof Error ? error.name : '';
|
||||
if (errorName === 'NoSuchKey') {
|
||||
return apiErrors.notFound('File');
|
||||
}
|
||||
console.error('Error serving image:', error);
|
||||
return apiErrors.internalError('Failed to retrieve image');
|
||||
}
|
||||
|
||||
@@ -77,17 +77,19 @@ if (process.env.NODE_ENV !== 'production') {
|
||||
globalForPrisma.prisma = db;
|
||||
}
|
||||
|
||||
export async function disconnectDb(): Promise<void> {
|
||||
if (globalForPool.pgPool) {
|
||||
await globalForPool.pgPool.end();
|
||||
globalForPool.pgPool = undefined;
|
||||
}
|
||||
await db.$disconnect();
|
||||
}
|
||||
|
||||
// Graceful shutdown handler
|
||||
async function shutdown() {
|
||||
console.log('Shutting down database connections...');
|
||||
|
||||
if (globalForPool.pgPool) {
|
||||
await globalForPool.pgPool.end();
|
||||
console.log('Database pool closed');
|
||||
}
|
||||
|
||||
await db.$disconnect();
|
||||
console.log('Prisma client disconnected');
|
||||
await disconnectDb();
|
||||
console.log('Database connections closed');
|
||||
}
|
||||
|
||||
// Register shutdown handlers
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
import { GetObjectCommand, type GetObjectCommandInput, type GetObjectCommandOutput } from '@aws-sdk/client-s3';
|
||||
import { Readable } from 'node:stream';
|
||||
import { NextResponse } from 'next/server';
|
||||
import { apiErrors } from '@/lib/api-response';
|
||||
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
|
||||
|
||||
type ProxyR2MediaOptions = {
|
||||
request: Request;
|
||||
key: string;
|
||||
fallbackContentType: string;
|
||||
cacheControl: string;
|
||||
extraHeaders?: Record<string, string>;
|
||||
notFoundLabel?: string;
|
||||
internalErrorMessage: string;
|
||||
};
|
||||
|
||||
type R2LikeError = {
|
||||
name?: string;
|
||||
Code?: string;
|
||||
$metadata?: { httpStatusCode?: number };
|
||||
};
|
||||
|
||||
function isStrongEtag(value: string): boolean {
|
||||
return /^"[^"]+"$/.test(value);
|
||||
}
|
||||
|
||||
function parseHttpDate(value: string): Date | null {
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||
}
|
||||
|
||||
function getErrorStatus(error: unknown): number | null {
|
||||
const status = (error as R2LikeError | null | undefined)?.$metadata?.httpStatusCode;
|
||||
return typeof status === 'number' ? status : null;
|
||||
}
|
||||
|
||||
function isNotFoundError(error: unknown): boolean {
|
||||
const err = error as R2LikeError | null | undefined;
|
||||
return err?.name === 'NoSuchKey' || err?.Code === 'NoSuchKey' || getErrorStatus(error) === 404;
|
||||
}
|
||||
|
||||
function isInvalidRangeError(error: unknown): boolean {
|
||||
const err = error as R2LikeError | null | undefined;
|
||||
return err?.name === 'InvalidRange' || err?.Code === 'InvalidRange' || getErrorStatus(error) === 416;
|
||||
}
|
||||
|
||||
function isPreconditionFailed(error: unknown): boolean {
|
||||
return getErrorStatus(error) === 412;
|
||||
}
|
||||
|
||||
function toWebStream(body: unknown): ReadableStream<Uint8Array> | null {
|
||||
if (!body) return null;
|
||||
|
||||
const withTransform = body as { transformToWebStream?: () => ReadableStream<Uint8Array> };
|
||||
if (typeof withTransform.transformToWebStream === 'function') {
|
||||
return withTransform.transformToWebStream();
|
||||
}
|
||||
|
||||
if (body instanceof Readable) {
|
||||
return Readable.toWeb(body) as ReadableStream<Uint8Array>;
|
||||
}
|
||||
|
||||
if (body instanceof ReadableStream) {
|
||||
return body;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function setIfPresent(headers: Headers, key: string, value: string | number | null | undefined): void {
|
||||
if (value === undefined || value === null) return;
|
||||
headers.set(key, String(value));
|
||||
}
|
||||
|
||||
export async function proxyR2MediaObject({
|
||||
request,
|
||||
key,
|
||||
fallbackContentType,
|
||||
cacheControl,
|
||||
extraHeaders,
|
||||
notFoundLabel = 'File',
|
||||
internalErrorMessage,
|
||||
}: ProxyR2MediaOptions): Promise<NextResponse> {
|
||||
const range = request.headers.get('range');
|
||||
const ifRange = request.headers.get('if-range');
|
||||
const commandInput: GetObjectCommandInput = {
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
};
|
||||
|
||||
let usedConditionalIfRange = false;
|
||||
if (range) {
|
||||
commandInput.Range = range;
|
||||
|
||||
if (ifRange) {
|
||||
const token = ifRange.trim();
|
||||
if (isStrongEtag(token)) {
|
||||
commandInput.IfMatch = token;
|
||||
usedConditionalIfRange = true;
|
||||
} else {
|
||||
const asDate = parseHttpDate(token);
|
||||
if (asDate) {
|
||||
commandInput.IfUnmodifiedSince = asDate;
|
||||
usedConditionalIfRange = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let objectResponse: GetObjectCommandOutput;
|
||||
try {
|
||||
objectResponse = await r2Client.send(new GetObjectCommand(commandInput));
|
||||
} catch (error) {
|
||||
if (usedConditionalIfRange && range && isPreconditionFailed(error)) {
|
||||
try {
|
||||
objectResponse = await r2Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
})
|
||||
);
|
||||
} catch (retryError) {
|
||||
if (isNotFoundError(retryError)) {
|
||||
return apiErrors.notFound(notFoundLabel);
|
||||
}
|
||||
if (isInvalidRangeError(retryError)) {
|
||||
return new NextResponse(null, {
|
||||
status: 416,
|
||||
headers: {
|
||||
'Cache-Control': cacheControl,
|
||||
'Accept-Ranges': 'bytes',
|
||||
},
|
||||
});
|
||||
}
|
||||
console.error('Error proxying R2 object:', retryError);
|
||||
return apiErrors.internalError(internalErrorMessage);
|
||||
}
|
||||
} else if (isNotFoundError(error)) {
|
||||
return apiErrors.notFound(notFoundLabel);
|
||||
} else if (isInvalidRangeError(error)) {
|
||||
return new NextResponse(null, {
|
||||
status: 416,
|
||||
headers: {
|
||||
'Cache-Control': cacheControl,
|
||||
'Accept-Ranges': 'bytes',
|
||||
},
|
||||
});
|
||||
} else {
|
||||
console.error('Error proxying R2 object:', error);
|
||||
return apiErrors.internalError(internalErrorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
const stream = toWebStream(objectResponse.Body);
|
||||
if (!stream) {
|
||||
return apiErrors.internalError('Empty file');
|
||||
}
|
||||
|
||||
const headers = new Headers();
|
||||
setIfPresent(headers, 'Content-Type', objectResponse.ContentType || fallbackContentType);
|
||||
setIfPresent(headers, 'Content-Length', objectResponse.ContentLength);
|
||||
setIfPresent(headers, 'Content-Range', objectResponse.ContentRange);
|
||||
setIfPresent(headers, 'ETag', objectResponse.ETag);
|
||||
setIfPresent(headers, 'Last-Modified', objectResponse.LastModified?.toUTCString());
|
||||
setIfPresent(headers, 'Accept-Ranges', objectResponse.AcceptRanges || 'bytes');
|
||||
headers.set('Cache-Control', cacheControl);
|
||||
|
||||
if (extraHeaders) {
|
||||
for (const [name, value] of Object.entries(extraHeaders)) {
|
||||
headers.set(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
return new NextResponse(stream, {
|
||||
status: objectResponse.ContentRange ? 206 : 200,
|
||||
headers,
|
||||
});
|
||||
}
|
||||
+5
-1
@@ -16,7 +16,11 @@
|
||||
"db:migrate": "prisma migrate deploy",
|
||||
"db:seed": "prisma db seed",
|
||||
"db:setup": "bun run db:generate && bun run db:push && bun run db:extras",
|
||||
"db:extras": "bun run scripts/db-extras.ts"
|
||||
"db:extras": "bun run scripts/db-extras.ts",
|
||||
"r2:cleanup-orphans:dry": "bun run scripts/r2-orphan-cleanup.ts --dry-run",
|
||||
"r2:cleanup-orphans": "bun run scripts/r2-orphan-cleanup.ts",
|
||||
"bunny:cleanup-orphans:dry": "bun run scripts/bunny-orphan-cleanup.ts --dry-run",
|
||||
"bunny:cleanup-orphans": "bun run scripts/bunny-orphan-cleanup.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.985.0",
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import { db, disconnectDb } from '../lib/db';
|
||||
|
||||
const BUNNY_API_BASE = 'https://video.bunnycdn.com';
|
||||
const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
|
||||
const ITEMS_PER_PAGE = 100;
|
||||
const MAX_PAGES = 200;
|
||||
const CHUNK_SIZE = 500;
|
||||
const DEFAULT_GRACE_HOURS = 24;
|
||||
|
||||
type BunnyConfig = {
|
||||
apiKey: string;
|
||||
libraryId: string;
|
||||
};
|
||||
|
||||
type BunnyVideo = {
|
||||
id: string;
|
||||
uploadedAt: Date;
|
||||
};
|
||||
|
||||
function getBunnyConfig(): BunnyConfig {
|
||||
const apiKey = process.env.BUNNY_STREAM_API_KEY;
|
||||
const libraryId = process.env.BUNNY_STREAM_LIBRARY_ID || process.env.NEXT_PUBLIC_BUNNY_STREAM_LIBRARY_ID;
|
||||
|
||||
if (!apiKey || !libraryId) {
|
||||
throw new Error('Missing BUNNY_STREAM_API_KEY or BUNNY_STREAM_LIBRARY_ID.');
|
||||
}
|
||||
|
||||
return { apiKey, libraryId };
|
||||
}
|
||||
|
||||
function toRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== 'object') return null;
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function parseVideoId(item: unknown): string | null {
|
||||
const record = toRecord(item);
|
||||
if (!record) return null;
|
||||
|
||||
const candidates = [record.guid, record.videoId, record.id];
|
||||
for (const candidate of candidates) {
|
||||
if (typeof candidate === 'string') {
|
||||
const trimmed = candidate.trim();
|
||||
if (BUNNY_VIDEO_ID_PATTERN.test(trimmed)) return trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseUploadedAt(item: unknown): Date | null {
|
||||
const record = toRecord(item);
|
||||
if (!record) return null;
|
||||
|
||||
const candidates = [
|
||||
record.dateUploaded,
|
||||
record.DateUploaded,
|
||||
record.dateCreated,
|
||||
record.DateCreated,
|
||||
record.createdAt,
|
||||
record.CreatedAt,
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (typeof candidate !== 'string') continue;
|
||||
const parsed = new Date(candidate);
|
||||
if (!Number.isNaN(parsed.getTime())) return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchBunnyPage(config: BunnyConfig, page: number): Promise<{ items: unknown[]; totalItems: number | null }> {
|
||||
const response = await fetch(
|
||||
`${BUNNY_API_BASE}/library/${config.libraryId}/videos?page=${page}&itemsPerPage=${ITEMS_PER_PAGE}`,
|
||||
{
|
||||
headers: {
|
||||
AccessKey: config.apiKey,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
cache: 'no-store',
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new Error(`Bunny list API failed (${response.status}): ${body.slice(0, 300)}`);
|
||||
}
|
||||
|
||||
const payload = await response.json();
|
||||
const record = toRecord(payload);
|
||||
if (!record) return { items: [], totalItems: null };
|
||||
|
||||
const items = Array.isArray(record.items)
|
||||
? record.items
|
||||
: (Array.isArray(record.Items) ? record.Items : []);
|
||||
|
||||
const totalItems = typeof record.totalItems === 'number'
|
||||
? record.totalItems
|
||||
: (typeof record.TotalItems === 'number' ? record.TotalItems : null);
|
||||
|
||||
return { items, totalItems };
|
||||
}
|
||||
|
||||
async function listBunnyVideos(config: BunnyConfig): Promise<{ videos: BunnyVideo[]; scanned: number; skippedInvalid: number }> {
|
||||
const videos: BunnyVideo[] = [];
|
||||
let scanned = 0;
|
||||
let skippedInvalid = 0;
|
||||
let page = 1;
|
||||
|
||||
while (page <= MAX_PAGES) {
|
||||
const { items, totalItems } = await fetchBunnyPage(config, page);
|
||||
if (items.length === 0) break;
|
||||
|
||||
scanned += items.length;
|
||||
for (const item of items) {
|
||||
const id = parseVideoId(item);
|
||||
const uploadedAt = parseUploadedAt(item);
|
||||
if (!id || !uploadedAt) {
|
||||
skippedInvalid += 1;
|
||||
continue;
|
||||
}
|
||||
videos.push({ id, uploadedAt });
|
||||
}
|
||||
|
||||
if (totalItems !== null && page * ITEMS_PER_PAGE >= totalItems) break;
|
||||
page += 1;
|
||||
}
|
||||
|
||||
return { videos, scanned, skippedInvalid };
|
||||
}
|
||||
|
||||
function chunk<T>(items: T[], size: number): T[][] {
|
||||
const out: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += size) {
|
||||
out.push(items.slice(i, i + size));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function findReferencedVideoIds(videoIds: string[]): Promise<Set<string>> {
|
||||
const referenced = new Set<string>();
|
||||
|
||||
for (const group of chunk(videoIds, CHUNK_SIZE)) {
|
||||
const rows = await db.videoVersion.findMany({
|
||||
where: {
|
||||
providerId: 'bunny',
|
||||
videoId: { in: group },
|
||||
},
|
||||
select: { videoId: true },
|
||||
});
|
||||
rows.forEach((row) => {
|
||||
if (row.videoId) referenced.add(row.videoId);
|
||||
});
|
||||
}
|
||||
|
||||
return referenced;
|
||||
}
|
||||
|
||||
async function deleteBunnyVideo(config: BunnyConfig, videoId: string): Promise<'deleted' | 'already_missing'> {
|
||||
const response = await fetch(
|
||||
`${BUNNY_API_BASE}/library/${config.libraryId}/videos/${encodeURIComponent(videoId)}`,
|
||||
{
|
||||
method: 'DELETE',
|
||||
headers: {
|
||||
AccessKey: config.apiKey,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
if (response.status === 404) return 'already_missing';
|
||||
if (response.ok) return 'deleted';
|
||||
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new Error(`Bunny delete API failed for ${videoId} (${response.status}): ${body.slice(0, 300)}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
const graceHours = DEFAULT_GRACE_HOURS;
|
||||
const graceMs = graceHours * 60 * 60 * 1000;
|
||||
const cutoff = Date.now() - graceMs;
|
||||
|
||||
const config = getBunnyConfig();
|
||||
console.log(`[bunny-orphan-cleanup] Starting (${dryRun ? 'dry-run' : 'delete mode'})`);
|
||||
console.log(`[bunny-orphan-cleanup] Grace period: ${graceHours}h`);
|
||||
|
||||
const { videos, scanned, skippedInvalid } = await listBunnyVideos(config);
|
||||
const eligible = videos.filter((video) => video.uploadedAt.getTime() <= cutoff);
|
||||
console.log(`[bunny-orphan-cleanup] Scanned: ${scanned}`);
|
||||
console.log(`[bunny-orphan-cleanup] Skipped invalid metadata: ${skippedInvalid}`);
|
||||
console.log(`[bunny-orphan-cleanup] Eligible (old enough): ${eligible.length}`);
|
||||
|
||||
if (eligible.length === 0) {
|
||||
console.log('[bunny-orphan-cleanup] No eligible Bunny videos found');
|
||||
return;
|
||||
}
|
||||
|
||||
const eligibleIds = eligible.map((video) => video.id);
|
||||
const referenced = await findReferencedVideoIds(eligibleIds);
|
||||
|
||||
const orphanIds = eligibleIds.filter((id) => !referenced.has(id));
|
||||
let deleted = 0;
|
||||
let alreadyMissing = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const orphanId of orphanIds) {
|
||||
if (dryRun) continue;
|
||||
|
||||
try {
|
||||
const result = await deleteBunnyVideo(config, orphanId);
|
||||
if (result === 'already_missing') {
|
||||
alreadyMissing += 1;
|
||||
} else {
|
||||
deleted += 1;
|
||||
}
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
console.error(`[bunny-orphan-cleanup] Failed deleting ${orphanId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[bunny-orphan-cleanup] Summary');
|
||||
console.log(`[bunny-orphan-cleanup] Referenced: ${referenced.size}`);
|
||||
console.log(`[bunny-orphan-cleanup] Orphaned: ${orphanIds.length}`);
|
||||
console.log(`[bunny-orphan-cleanup] Deleted: ${deleted}`);
|
||||
console.log(`[bunny-orphan-cleanup] Already missing: ${alreadyMissing}`);
|
||||
console.log(`[bunny-orphan-cleanup] Failed: ${failed}`);
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((error) => {
|
||||
console.error('[bunny-orphan-cleanup] Fatal error:', error);
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(async () => {
|
||||
await disconnectDb();
|
||||
});
|
||||
@@ -0,0 +1,176 @@
|
||||
import { DeleteObjectCommand, ListObjectsV2Command, type ListObjectsV2CommandInput } from '@aws-sdk/client-s3';
|
||||
import { db, disconnectDb } from '../lib/db';
|
||||
import { r2Client, R2_BUCKET_NAME } from '../lib/r2';
|
||||
|
||||
const UNATTACHED_UPLOAD_TTL_MS = 15 * 60 * 1000;
|
||||
const CHUNK_SIZE = 500;
|
||||
const PREFIXES = ['images/', 'voice/'] as const;
|
||||
|
||||
type CleanupCandidate = {
|
||||
key: string;
|
||||
url: string;
|
||||
};
|
||||
|
||||
type UserFeedbackScreenshotDelegate = {
|
||||
findMany: (args: { where: { url: { in: string[] } }; select: { url: true } }) => Promise<Array<{ url: string }>>;
|
||||
};
|
||||
|
||||
function keyToProxyUrl(key: string): string | null {
|
||||
if (key.startsWith('images/')) {
|
||||
const filename = key.slice('images/'.length);
|
||||
return filename ? `/api/upload/image/${filename}` : null;
|
||||
}
|
||||
if (key.startsWith('voice/')) {
|
||||
const filename = key.slice('voice/'.length);
|
||||
return filename ? `/api/upload/audio/${filename}` : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function chunk<T>(items: T[], size: number): T[][] {
|
||||
const out: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += size) {
|
||||
out.push(items.slice(i, i + size));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function listCleanupCandidates(): Promise<{ candidates: CleanupCandidate[]; scanned: number }> {
|
||||
const candidates: CleanupCandidate[] = [];
|
||||
const cutoff = Date.now() - UNATTACHED_UPLOAD_TTL_MS;
|
||||
let scanned = 0;
|
||||
|
||||
for (const prefix of PREFIXES) {
|
||||
let continuationToken: string | undefined;
|
||||
let isTruncated = true;
|
||||
|
||||
while (isTruncated) {
|
||||
const input: ListObjectsV2CommandInput = {
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Prefix: prefix,
|
||||
};
|
||||
if (continuationToken) input.ContinuationToken = continuationToken;
|
||||
|
||||
const response = await r2Client.send(new ListObjectsV2Command(input));
|
||||
const contents = response.Contents ?? [];
|
||||
scanned += contents.length;
|
||||
|
||||
for (const item of contents) {
|
||||
if (!item.Key || !item.LastModified) continue;
|
||||
if (item.LastModified.getTime() > cutoff) continue;
|
||||
|
||||
const url = keyToProxyUrl(item.Key);
|
||||
if (!url) continue;
|
||||
candidates.push({ key: item.Key, url });
|
||||
}
|
||||
|
||||
isTruncated = response.IsTruncated ?? false;
|
||||
continuationToken = response.NextContinuationToken;
|
||||
}
|
||||
}
|
||||
|
||||
return { candidates, scanned };
|
||||
}
|
||||
|
||||
async function findReferencedUrls(urls: string[]): Promise<Set<string>> {
|
||||
const referenced = new Set<string>();
|
||||
const userFeedbackScreenshotDelegate = (db as unknown as {
|
||||
userFeedbackScreenshot?: UserFeedbackScreenshotDelegate;
|
||||
}).userFeedbackScreenshot;
|
||||
|
||||
for (const group of chunk(urls, CHUNK_SIZE)) {
|
||||
const [commentRows, feedbackRows, feedbackAttachmentRows] = await Promise.all([
|
||||
db.comment.findMany({
|
||||
where: {
|
||||
OR: [{ voiceUrl: { in: group } }, { imageUrl: { in: group } }],
|
||||
},
|
||||
select: {
|
||||
voiceUrl: true,
|
||||
imageUrl: true,
|
||||
},
|
||||
}),
|
||||
db.userFeedback.findMany({
|
||||
where: { screenshotUrl: { in: group } },
|
||||
select: { screenshotUrl: true },
|
||||
}),
|
||||
userFeedbackScreenshotDelegate
|
||||
? userFeedbackScreenshotDelegate.findMany({
|
||||
where: { url: { in: group } },
|
||||
select: { url: true },
|
||||
})
|
||||
: Promise.resolve([] as Array<{ url: string }>),
|
||||
]);
|
||||
|
||||
for (const row of commentRows) {
|
||||
if (row.voiceUrl) referenced.add(row.voiceUrl);
|
||||
if (row.imageUrl) referenced.add(row.imageUrl);
|
||||
}
|
||||
for (const row of feedbackRows) {
|
||||
if (row.screenshotUrl) referenced.add(row.screenshotUrl);
|
||||
}
|
||||
for (const row of feedbackAttachmentRows) {
|
||||
if (row.url) referenced.add(row.url);
|
||||
}
|
||||
}
|
||||
|
||||
return referenced;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
console.log(`[r2-orphan-cleanup] Starting (${dryRun ? 'dry-run' : 'delete mode'})`);
|
||||
|
||||
const { candidates, scanned } = await listCleanupCandidates();
|
||||
console.log(`[r2-orphan-cleanup] Scanned: ${scanned}, eligible (old enough): ${candidates.length}`);
|
||||
|
||||
if (candidates.length === 0) {
|
||||
console.log('[r2-orphan-cleanup] No eligible objects found');
|
||||
return;
|
||||
}
|
||||
|
||||
const referenced = await findReferencedUrls(candidates.map((candidate) => candidate.url));
|
||||
|
||||
let deleted = 0;
|
||||
let failed = 0;
|
||||
let orphaned = 0;
|
||||
let referencedCount = 0;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (referenced.has(candidate.url)) {
|
||||
referencedCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
orphaned += 1;
|
||||
if (dryRun) continue;
|
||||
|
||||
try {
|
||||
await r2Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: candidate.key,
|
||||
})
|
||||
);
|
||||
deleted += 1;
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
console.error(`[r2-orphan-cleanup] Failed deleting ${candidate.key}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[r2-orphan-cleanup] Summary');
|
||||
console.log(`[r2-orphan-cleanup] Referenced: ${referencedCount}`);
|
||||
console.log(`[r2-orphan-cleanup] Orphaned: ${orphaned}`);
|
||||
console.log(`[r2-orphan-cleanup] Deleted: ${deleted}`);
|
||||
console.log(`[r2-orphan-cleanup] Failed: ${failed}`);
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((error) => {
|
||||
console.error('[r2-orphan-cleanup] Fatal error:', error);
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(async () => {
|
||||
await disconnectDb();
|
||||
r2Client.destroy();
|
||||
});
|
||||
Reference in New Issue
Block a user