mirror of
https://github.com/yusufipk/OpenFrame.git
synced 2026-09-11 17:46:06 +00:00
Improve media cleanup performance with bounded concurrency and aggregated failure reporting
This commit is contained in:
@@ -8,6 +8,9 @@ import { validateShareLinkAccess } from '@/lib/share-links';
|
|||||||
import { getShareSessionFromRequest } from '@/lib/share-session';
|
import { getShareSessionFromRequest } from '@/lib/share-session';
|
||||||
import { apiErrors, successResponse, withCacheControl } from '@/lib/api-response';
|
import { apiErrors, successResponse, withCacheControl } from '@/lib/api-response';
|
||||||
import { getGuestIdentityFromRequest } from '@/lib/guest-identity';
|
import { getGuestIdentityFromRequest } from '@/lib/guest-identity';
|
||||||
|
import { runWithConcurrency } from '@/lib/async-pool';
|
||||||
|
|
||||||
|
const CLEANUP_DELETE_CONCURRENCY = 5;
|
||||||
|
|
||||||
type RouteParams = { params: Promise<{ commentId: string }> };
|
type RouteParams = { params: Promise<{ commentId: string }> };
|
||||||
|
|
||||||
@@ -298,30 +301,31 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) {
|
|||||||
// Clean up media files from R2 (best-effort, don't block on failure)
|
// Clean up media files from R2 (best-effort, don't block on failure)
|
||||||
const AUDIO_PREFIX = '/api/upload/audio/';
|
const AUDIO_PREFIX = '/api/upload/audio/';
|
||||||
const IMAGE_PREFIX = '/api/upload/image/';
|
const IMAGE_PREFIX = '/api/upload/image/';
|
||||||
for (const url of mediaUrls) {
|
const mediaKeys = [...new Set(mediaUrls.map((url) => {
|
||||||
try {
|
|
||||||
// Extract filename using string parsing (safe against ReDoS)
|
// Extract filename using string parsing (safe against ReDoS)
|
||||||
let key: string | null = null;
|
|
||||||
if (url.includes(AUDIO_PREFIX)) {
|
if (url.includes(AUDIO_PREFIX)) {
|
||||||
const filename = url.slice(url.indexOf(AUDIO_PREFIX) + AUDIO_PREFIX.length);
|
const filename = url.slice(url.indexOf(AUDIO_PREFIX) + AUDIO_PREFIX.length);
|
||||||
if (filename) key = `voice/${filename}`;
|
return filename ? `voice/${filename}` : null;
|
||||||
} else if (url.includes(IMAGE_PREFIX)) {
|
|
||||||
const filename = url.slice(url.indexOf(IMAGE_PREFIX) + IMAGE_PREFIX.length);
|
|
||||||
if (filename) key = `images/${filename}`;
|
|
||||||
}
|
}
|
||||||
|
if (url.includes(IMAGE_PREFIX)) {
|
||||||
|
const filename = url.slice(url.indexOf(IMAGE_PREFIX) + IMAGE_PREFIX.length);
|
||||||
|
return filename ? `images/${filename}` : null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}).filter((key): key is string => Boolean(key)))];
|
||||||
|
|
||||||
if (key) {
|
await runWithConcurrency(mediaKeys, CLEANUP_DELETE_CONCURRENCY, async (key) => {
|
||||||
|
try {
|
||||||
await r2Client.send(
|
await r2Client.send(
|
||||||
new DeleteObjectCommand({
|
new DeleteObjectCommand({
|
||||||
Bucket: R2_BUCKET_NAME,
|
Bucket: R2_BUCKET_NAME,
|
||||||
Key: key,
|
Key: key,
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
}
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to delete audio from R2:', err);
|
console.error(`Failed to delete media from R2 (key: ${key}):`, err);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
|
||||||
const response = successResponse({ message: 'Comment deleted' });
|
const response = successResponse({ message: 'Comment deleted' });
|
||||||
return withCacheControl(response, 'private, no-store');
|
return withCacheControl(response, 'private, no-store');
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
/**
|
||||||
|
* Execute async work with a bounded number of in-flight tasks.
|
||||||
|
*/
|
||||||
|
export async function runWithConcurrency<T>(
|
||||||
|
items: T[],
|
||||||
|
limit: number,
|
||||||
|
worker: (item: T) => Promise<void>
|
||||||
|
): Promise<void> {
|
||||||
|
if (items.length === 0) return;
|
||||||
|
|
||||||
|
const concurrency = Math.max(1, Math.floor(limit));
|
||||||
|
let nextIndex = 0;
|
||||||
|
|
||||||
|
async function runWorker(): Promise<void> {
|
||||||
|
while (true) {
|
||||||
|
const currentIndex = nextIndex;
|
||||||
|
nextIndex += 1;
|
||||||
|
|
||||||
|
if (currentIndex >= items.length) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await worker(items[currentIndex]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const workerCount = Math.min(concurrency, items.length);
|
||||||
|
await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
|
||||||
|
}
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { runWithConcurrency } from '@/lib/async-pool';
|
||||||
|
|
||||||
interface BunnyVideoRef {
|
interface BunnyVideoRef {
|
||||||
providerId: string;
|
providerId: string;
|
||||||
videoId: string;
|
videoId: string;
|
||||||
@@ -5,6 +7,7 @@ interface BunnyVideoRef {
|
|||||||
|
|
||||||
const BUNNY_API_BASE = 'https://video.bunnycdn.com';
|
const BUNNY_API_BASE = 'https://video.bunnycdn.com';
|
||||||
const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
|
const BUNNY_VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
|
||||||
|
const BUNNY_DELETE_CONCURRENCY = 5;
|
||||||
|
|
||||||
function getBunnyConfig(): { apiKey: string; libraryId: string } {
|
function getBunnyConfig(): { apiKey: string; libraryId: string } {
|
||||||
const apiKey = process.env.BUNNY_STREAM_API_KEY;
|
const apiKey = process.env.BUNNY_STREAM_API_KEY;
|
||||||
@@ -33,8 +36,9 @@ export async function cleanupBunnyStreamVideos(videoRefs: BunnyVideoRef[]): Prom
|
|||||||
if (bunnyVideoIds.length === 0) return;
|
if (bunnyVideoIds.length === 0) return;
|
||||||
|
|
||||||
const { apiKey, libraryId } = getBunnyConfig();
|
const { apiKey, libraryId } = getBunnyConfig();
|
||||||
|
const failures: Array<{ videoId: string; status: number; bodySnippet: string }> = [];
|
||||||
|
|
||||||
for (const bunnyVideoId of bunnyVideoIds) {
|
await runWithConcurrency(bunnyVideoIds, BUNNY_DELETE_CONCURRENCY, async (bunnyVideoId) => {
|
||||||
const response = await fetch(`${BUNNY_API_BASE}/library/${libraryId}/videos/${encodeURIComponent(bunnyVideoId)}`, {
|
const response = await fetch(`${BUNNY_API_BASE}/library/${libraryId}/videos/${encodeURIComponent(bunnyVideoId)}`, {
|
||||||
method: 'DELETE',
|
method: 'DELETE',
|
||||||
headers: {
|
headers: {
|
||||||
@@ -43,13 +47,23 @@ export async function cleanupBunnyStreamVideos(videoRefs: BunnyVideoRef[]): Prom
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Treat not-found as already deleted.
|
// Treat not-found as already deleted.
|
||||||
if (response.status === 404) continue;
|
if (response.status === 404) return;
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const body = await response.text().catch(() => '');
|
const body = await response.text().catch(() => '');
|
||||||
throw new Error(
|
failures.push({
|
||||||
`Bunny cleanup failed for video ${bunnyVideoId}: ${response.status} ${body.slice(0, 300)}`
|
videoId: bunnyVideoId,
|
||||||
);
|
status: response.status,
|
||||||
|
bodySnippet: body.slice(0, 300),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (failures.length > 0) {
|
||||||
|
const preview = failures
|
||||||
|
.slice(0, 3)
|
||||||
|
.map((failure) => `${failure.videoId} (${failure.status})`)
|
||||||
|
.join(', ');
|
||||||
|
throw new Error(`Bunny cleanup failed for ${failures.length} video(s): ${preview}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-6
@@ -1,11 +1,13 @@
|
|||||||
import { DeleteObjectCommand } from '@aws-sdk/client-s3';
|
import { DeleteObjectCommand } from '@aws-sdk/client-s3';
|
||||||
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
|
import { r2Client, R2_BUCKET_NAME } from '@/lib/r2';
|
||||||
import { db } from '@/lib/db';
|
import { db } from '@/lib/db';
|
||||||
|
import { runWithConcurrency } from '@/lib/async-pool';
|
||||||
|
|
||||||
/** The path prefix for images served by the upload API. */
|
/** The path prefix for images served by the upload API. */
|
||||||
const IMAGE_PATH_PREFIX = '/api/upload/image/';
|
const IMAGE_PATH_PREFIX = '/api/upload/image/';
|
||||||
/** The path prefix for audio URLs served by the upload API. */
|
/** The path prefix for audio URLs served by the upload API. */
|
||||||
const AUDIO_PATH_PREFIX = '/api/upload/audio/';
|
const AUDIO_PATH_PREFIX = '/api/upload/audio/';
|
||||||
|
const CLEANUP_DELETE_CONCURRENCY = 5;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extract the R2 object key from a media URL.
|
* Extract the R2 object key from a media URL.
|
||||||
@@ -26,18 +28,17 @@ function mediaUrlToKey(url: string): string | null {
|
|||||||
* Delete a list of media files from R2 (best-effort, logs failures).
|
* Delete a list of media files from R2 (best-effort, logs failures).
|
||||||
*/
|
*/
|
||||||
async function deleteMediaFiles(mediaUrls: string[]) {
|
async function deleteMediaFiles(mediaUrls: string[]) {
|
||||||
for (const url of mediaUrls) {
|
const mediaKeys = [...new Set(mediaUrls.map(mediaUrlToKey).filter((key): key is string => Boolean(key)))];
|
||||||
|
|
||||||
|
await runWithConcurrency(mediaKeys, CLEANUP_DELETE_CONCURRENCY, async (key) => {
|
||||||
try {
|
try {
|
||||||
const key = mediaUrlToKey(url);
|
|
||||||
if (key) {
|
|
||||||
await r2Client.send(
|
await r2Client.send(
|
||||||
new DeleteObjectCommand({ Bucket: R2_BUCKET_NAME, Key: key })
|
new DeleteObjectCommand({ Bucket: R2_BUCKET_NAME, Key: key })
|
||||||
);
|
);
|
||||||
}
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to delete media from R2:', err);
|
console.error(`Failed to delete media from R2 (key: ${key}):`, err);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user