Improve media cleanup performance with bounded concurrency and aggregated failure reporting

This commit is contained in:
Yusuf İpek
2026-02-24 17:44:23 +03:00
parent 6e14d4d19f
commit 7f07a271a4
4 changed files with 85 additions and 37 deletions
+15 -11
View File
@@ -8,6 +8,9 @@ import { validateShareLinkAccess } from '@/lib/share-links';
import { getShareSessionFromRequest } from '@/lib/share-session'; import { getShareSessionFromRequest } from '@/lib/share-session';
import { apiErrors, successResponse, withCacheControl } from '@/lib/api-response'; import { apiErrors, successResponse, withCacheControl } from '@/lib/api-response';
import { getGuestIdentityFromRequest } from '@/lib/guest-identity'; import { getGuestIdentityFromRequest } from '@/lib/guest-identity';
import { runWithConcurrency } from '@/lib/async-pool';
const CLEANUP_DELETE_CONCURRENCY = 5;
type RouteParams = { params: Promise<{ commentId: string }> }; type RouteParams = { params: Promise<{ commentId: string }> };
@@ -298,30 +301,31 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) {
// Clean up media files from R2 (best-effort, don't block on failure) // Clean up media files from R2 (best-effort, don't block on failure)
const AUDIO_PREFIX = '/api/upload/audio/'; const AUDIO_PREFIX = '/api/upload/audio/';
const IMAGE_PREFIX = '/api/upload/image/'; const IMAGE_PREFIX = '/api/upload/image/';
for (const url of mediaUrls) { const mediaKeys = [...new Set(mediaUrls.map((url) => {
try {
// Extract filename using string parsing (safe against ReDoS) // Extract filename using string parsing (safe against ReDoS)
let key: string | null = null;
if (url.includes(AUDIO_PREFIX)) { if (url.includes(AUDIO_PREFIX)) {
const filename = url.slice(url.indexOf(AUDIO_PREFIX) + AUDIO_PREFIX.length); const filename = url.slice(url.indexOf(AUDIO_PREFIX) + AUDIO_PREFIX.length);
if (filename) key = `voice/${filename}`; return filename ? `voice/${filename}` : null;
} else if (url.includes(IMAGE_PREFIX)) {
const filename = url.slice(url.indexOf(IMAGE_PREFIX) + IMAGE_PREFIX.length);
if (filename) key = `images/${filename}`;
} }
if (url.includes(IMAGE_PREFIX)) {
const filename = url.slice(url.indexOf(IMAGE_PREFIX) + IMAGE_PREFIX.length);
return filename ? `images/${filename}` : null;
}
return null;
}).filter((key): key is string => Boolean(key)))];
if (key) { await runWithConcurrency(mediaKeys, CLEANUP_DELETE_CONCURRENCY, async (key) => {
try {
await r2Client.send( await r2Client.send(
new DeleteObjectCommand({ new DeleteObjectCommand({
Bucket: R2_BUCKET_NAME, Bucket: R2_BUCKET_NAME,
Key: key, Key: key,
}) })
); );
}
} catch (err) { } catch (err) {
console.error('Failed to delete audio from R2:', err); console.error(`Failed to delete media from R2 (key: ${key}):`, err);
}
} }
});
const response = successResponse({ message: 'Comment deleted' }); const response = successResponse({ message: 'Comment deleted' });
return withCacheControl(response, 'private, no-store'); return withCacheControl(response, 'private, no-store');
+29
View File
@@ -0,0 +1,29 @@
/**
* Execute async work with a bounded number of in-flight tasks.
*/
export async function runWithConcurrency<T>(
items: T[],
limit: number,
worker: (item: T) => Promise<void>
): Promise<void> {
if (items.length === 0) return;
const concurrency = Math.max(1, Math.floor(limit));
let nextIndex = 0;
async function runWorker(): Promise<void> {
while (true) {
const currentIndex = nextIndex;
nextIndex += 1;
if (currentIndex >= items.length) {
return;
}
await worker(items[currentIndex]);
}
}
const workerCount = Math.min(concurrency, items.length);
await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
}
+19 -5
View File
@@ -1,3 +1,5 @@
import { runWithConcurrency } from '@/lib/async-pool';
interface BunnyVideoRef { interface BunnyVideoRef {
providerId: string; providerId: string;
videoId: string; videoId: string;
@@ -5,6 +7,7 @@ interface BunnyVideoRef {
const BUNNY_API_BASE = 'https://video.bunnycdn.com'; const BUNNY_API_BASE = 'https://video.bunnycdn.com';
const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
const BUNNY_DELETE_CONCURRENCY = 5;
function getBunnyConfig(): { apiKey: string; libraryId: string } { function getBunnyConfig(): { apiKey: string; libraryId: string } {
const apiKey = process.env.BUNNY_STREAM_API_KEY; const apiKey = process.env.BUNNY_STREAM_API_KEY;
@@ -33,8 +36,9 @@ export async function cleanupBunnyStreamVideos(videoRefs: BunnyVideoRef[]): Prom
if (bunnyVideoIds.length === 0) return; if (bunnyVideoIds.length === 0) return;
const { apiKey, libraryId } = getBunnyConfig(); const { apiKey, libraryId } = getBunnyConfig();
const failures: Array<{ videoId: string; status: number; bodySnippet: string }> = [];
for (const bunnyVideoId of bunnyVideoIds) { await runWithConcurrency(bunnyVideoIds, BUNNY_DELETE_CONCURRENCY, async (bunnyVideoId) => {
const response = await fetch(`${BUNNY_API_BASE}/library/${libraryId}/videos/${encodeURIComponent(bunnyVideoId)}`, { const response = await fetch(`${BUNNY_API_BASE}/library/${libraryId}/videos/${encodeURIComponent(bunnyVideoId)}`, {
method: 'DELETE', method: 'DELETE',
headers: { headers: {
@@ -43,13 +47,23 @@ export async function cleanupBunnyStreamVideos(videoRefs: BunnyVideoRef[]): Prom
}); });
// Treat not-found as already deleted. // Treat not-found as already deleted.
if (response.status === 404) continue; if (response.status === 404) return;
if (!response.ok) { if (!response.ok) {
const body = await response.text().catch(() => ''); const body = await response.text().catch(() => '');
throw new Error( failures.push({
`Bunny cleanup failed for video ${bunnyVideoId}: ${response.status} ${body.slice(0, 300)}` videoId: bunnyVideoId,
); status: response.status,
bodySnippet: body.slice(0, 300),
});
} }
});
if (failures.length > 0) {
const preview = failures
.slice(0, 3)
.map((failure) => `${failure.videoId} (${failure.status})`)
.join(', ');
throw new Error(`Bunny cleanup failed for ${failures.length} video(s): ${preview}`);
} }
} }
+7 -6
View File
@@ -1,11 +1,13 @@
import { DeleteObjectCommand } from '@aws-sdk/client-s3'; import { DeleteObjectCommand } from '@aws-sdk/client-s3';
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2'; import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
import { db } from '@/lib/db'; import { db } from '@/lib/db';
import { runWithConcurrency } from '@/lib/async-pool';
/** The path prefix for images served by the upload API. */ /** The path prefix for images served by the upload API. */
const IMAGE_PATH_PREFIX = '/api/upload/image/'; const IMAGE_PATH_PREFIX = '/api/upload/image/';
/** The path prefix for audio URLs served by the upload API. */ /** The path prefix for audio URLs served by the upload API. */
const AUDIO_PATH_PREFIX = '/api/upload/audio/'; const AUDIO_PATH_PREFIX = '/api/upload/audio/';
const CLEANUP_DELETE_CONCURRENCY = 5;
/** /**
* Extract the R2 object key from a media URL. * Extract the R2 object key from a media URL.
@@ -26,18 +28,17 @@ function mediaUrlToKey(url: string): string | null {
* Delete a list of media files from R2 (best-effort, logs failures). * Delete a list of media files from R2 (best-effort, logs failures).
*/ */
async function deleteMediaFiles(mediaUrls: string[]) { async function deleteMediaFiles(mediaUrls: string[]) {
for (const url of mediaUrls) { const mediaKeys = [...new Set(mediaUrls.map(mediaUrlToKey).filter((key): key is string => Boolean(key)))];
await runWithConcurrency(mediaKeys, CLEANUP_DELETE_CONCURRENCY, async (key) => {
try { try {
const key = mediaUrlToKey(url);
if (key) {
await r2Client.send( await r2Client.send(
new DeleteObjectCommand({ Bucket: R2_BUCKET_NAME, Key: key }) new DeleteObjectCommand({ Bucket: R2_BUCKET_NAME, Key: key })
); );
}
} catch (err) { } catch (err) {
console.error('Failed to delete media from R2:', err); console.error(`Failed to delete media from R2 (key: ${key}):`, err);
}
} }
});
} }
/** /**