mirror of
https://github.com/yusufipk/OpenFrame.git
synced 2026-09-11 17:46:06 +00:00
feat: chunked (S3 multipart) uploads for R2/S3 video backend
Self-hosted instances on the R2/S3 backend could only upload a video as a single PUT, which fails behind a Cloudflare proxy/tunnel (100MB request-body cap) and is capped at 5GiB with no resilience. Bunny already avoids this via tus; this brings the R2/S3 path to parity. Files larger than a threshold (default 90MiB) are now split into parts (default 32MiB, min 5MiB) and uploaded directly browser->R2 via presigned UploadPart URLs, then reassembled server-side with CompleteMultipartUpload. Each request stays under the 100MB cap, lifts the size ceiling well past 5GiB, and adds per-chunk retry. Files at/under the threshold keep the existing single-PUT path unchanged. Bunny path is untouched. Thresholds are env-overridable via OPENFRAME_R2_MULTIPART_THRESHOLD_BYTES and OPENFRAME_R2_MULTIPART_PART_SIZE_BYTES. Verified end-to-end against real Cloudflare R2 and a local MinIO behind an nginx 90MB cap (single 141MB PUT 413s on master; 32MB parts pass here). Closes #22
This commit is contained in:
@@ -1,5 +1,13 @@
|
||||
import { captureVideoThumbnail } from '@/lib/client/video-thumbnail';
|
||||
|
||||
export type R2MultipartPart = { partNumber: number; url: string };
|
||||
|
||||
export type R2MultipartInit = {
|
||||
uploadId: string;
|
||||
partSizeBytes: number;
|
||||
parts: R2MultipartPart[];
|
||||
};
|
||||
|
||||
export type R2VideoInitResponse = {
|
||||
presignedPutUrl: string;
|
||||
objectKey: string;
|
||||
@@ -10,8 +18,11 @@ export type R2VideoInitResponse = {
|
||||
thumbnailPresignedPutUrl: string;
|
||||
thumbnailObjectKey: string;
|
||||
thumbnailProxyUrl: string;
|
||||
multipart: R2MultipartInit | null;
|
||||
};
|
||||
|
||||
const PART_RETRY_DELAYS = [0, 2000, 5000, 10000];
|
||||
|
||||
export type R2VideoUploadResult = R2VideoInitResponse & {
|
||||
duration: number | null;
|
||||
thumbnailUrl: string | null;
|
||||
@@ -56,6 +67,128 @@ function uploadBytesWithProgress(
|
||||
});
|
||||
}
|
||||
|
||||
function uploadPartWithProgress(
|
||||
url: string,
|
||||
body: Blob,
|
||||
onPartProgress?: (loadedBytes: number) => void
|
||||
): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('PUT', url);
|
||||
// Intentionally no Content-Type header: it is not part of the presigned
|
||||
// UploadPart signature, and the part body is raw bytes.
|
||||
|
||||
xhr.upload.onprogress = (event) => {
|
||||
if (!onPartProgress || !event.lengthComputable) return;
|
||||
onPartProgress(event.loaded);
|
||||
};
|
||||
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
const etag = xhr.getResponseHeader('ETag');
|
||||
if (!etag) {
|
||||
reject(
|
||||
new Error(
|
||||
'Upload response missing ETag header. Configure bucket CORS to expose the ETag header.'
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
resolve(etag);
|
||||
return;
|
||||
}
|
||||
reject(new Error(`Chunk upload failed with status ${xhr.status}`));
|
||||
};
|
||||
|
||||
xhr.onerror = () => {
|
||||
reject(
|
||||
new Error(
|
||||
'Network error during upload. If you use direct S3/R2 uploads, configure bucket CORS to allow PUT from this site origin.'
|
||||
)
|
||||
);
|
||||
};
|
||||
xhr.onabort = () => reject(new Error('Upload aborted'));
|
||||
|
||||
xhr.send(body);
|
||||
});
|
||||
}
|
||||
|
||||
async function withRetry<T>(fn: () => Promise<T>, delays: number[]): Promise<T> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < delays.length; attempt += 1) {
|
||||
if (attempt > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, delays[attempt]));
|
||||
}
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
throw lastError instanceof Error ? lastError : new Error('Upload failed after retries');
|
||||
}
|
||||
|
||||
async function completeMultipartUpload(
|
||||
projectId: string,
|
||||
objectKey: string,
|
||||
uploadToken: string,
|
||||
parts: Array<{ partNumber: number; etag: string }>
|
||||
): Promise<void> {
|
||||
const res = await fetch(`/api/projects/${projectId}/videos/r2-complete`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ objectKey, uploadToken, parts }),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const payload = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(payload?.error || 'Failed to complete multipart upload');
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadVideoMultipart(
|
||||
projectId: string,
|
||||
file: File,
|
||||
multipart: R2MultipartInit,
|
||||
objectKey: string,
|
||||
uploadToken: string,
|
||||
onProgress?: UploadProgressHandler
|
||||
): Promise<void> {
|
||||
const totalBytes = file.size;
|
||||
const partSize = multipart.partSizeBytes;
|
||||
const loadedPerPart = new Array<number>(multipart.parts.length).fill(0);
|
||||
|
||||
const reportProgress = () => {
|
||||
if (!onProgress) return;
|
||||
const loaded = loadedPerPart.reduce((sum, value) => sum + value, 0);
|
||||
onProgress(Math.min(100, Math.round((loaded / totalBytes) * 100)));
|
||||
};
|
||||
|
||||
const completedParts: Array<{ partNumber: number; etag: string }> = [];
|
||||
|
||||
for (let index = 0; index < multipart.parts.length; index += 1) {
|
||||
const part = multipart.parts[index];
|
||||
const start = (part.partNumber - 1) * partSize;
|
||||
const end = Math.min(start + partSize, totalBytes);
|
||||
const blob = file.slice(start, end);
|
||||
|
||||
const etag = await withRetry(
|
||||
() =>
|
||||
uploadPartWithProgress(part.url, blob, (loadedBytes) => {
|
||||
loadedPerPart[index] = loadedBytes;
|
||||
reportProgress();
|
||||
}),
|
||||
PART_RETRY_DELAYS
|
||||
);
|
||||
|
||||
loadedPerPart[index] = end - start;
|
||||
reportProgress();
|
||||
completedParts.push({ partNumber: part.partNumber, etag });
|
||||
}
|
||||
|
||||
await completeMultipartUpload(projectId, objectKey, uploadToken, completedParts);
|
||||
}
|
||||
|
||||
async function readVideoDuration(file: File): Promise<number | null> {
|
||||
return new Promise((resolve) => {
|
||||
const objectUrl = URL.createObjectURL(file);
|
||||
@@ -151,12 +284,23 @@ export async function uploadVideoToR2(
|
||||
};
|
||||
|
||||
try {
|
||||
await uploadBytesWithProgress(
|
||||
init.presignedPutUrl,
|
||||
file,
|
||||
init.contentType,
|
||||
options?.onProgress
|
||||
);
|
||||
if (init.multipart) {
|
||||
await uploadVideoMultipart(
|
||||
projectId,
|
||||
file,
|
||||
init.multipart,
|
||||
init.objectKey,
|
||||
init.uploadToken,
|
||||
options?.onProgress
|
||||
);
|
||||
} else {
|
||||
await uploadBytesWithProgress(
|
||||
init.presignedPutUrl,
|
||||
file,
|
||||
init.contentType,
|
||||
options?.onProgress
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
await cleanupPendingR2VideoUpload(projectId, cleanupInput);
|
||||
throw error;
|
||||
|
||||
Reference in New Issue
Block a user