feat: chunked (S3 multipart) uploads for R2/S3 video backend

Self-hosted instances on the R2/S3 backend could only upload a video as a
single PUT, which fails behind a Cloudflare proxy/tunnel (100MB request-body
cap) and is capped at 5GiB with no resilience. Bunny already avoids this via
tus; this brings the R2/S3 path to parity.

Files larger than a threshold (default 90MiB) are now split into parts
(default 32MiB, min 5MiB) and uploaded directly browser->R2 via presigned
UploadPart URLs, then reassembled server-side with CompleteMultipartUpload.
Each request stays under the 100MB cap, lifts the size ceiling well past
5GiB, and adds per-chunk retry. Files at/under the threshold keep the
existing single-PUT path unchanged. Bunny path is untouched.

Thresholds are env-overridable via OPENFRAME_R2_MULTIPART_THRESHOLD_BYTES
and OPENFRAME_R2_MULTIPART_PART_SIZE_BYTES.

Verified end-to-end against real Cloudflare R2 and a local MinIO behind an
nginx 90MB cap (single 141MB PUT 413s on master; 32MB parts pass here).

Closes #22
This commit is contained in:
yusufipk
2026-07-10 19:19:07 +07:00
parent 82932c6b22
commit 880d0ac0fa
9 changed files with 513 additions and 12 deletions
+150 -6
View File
@@ -1,5 +1,13 @@
import { captureVideoThumbnail } from '@/lib/client/video-thumbnail';
export type R2MultipartPart = { partNumber: number; url: string };
export type R2MultipartInit = {
uploadId: string;
partSizeBytes: number;
parts: R2MultipartPart[];
};
export type R2VideoInitResponse = {
presignedPutUrl: string;
objectKey: string;
@@ -10,8 +18,11 @@ export type R2VideoInitResponse = {
thumbnailPresignedPutUrl: string;
thumbnailObjectKey: string;
thumbnailProxyUrl: string;
multipart: R2MultipartInit | null;
};
const PART_RETRY_DELAYS = [0, 2000, 5000, 10000];
export type R2VideoUploadResult = R2VideoInitResponse & {
duration: number | null;
thumbnailUrl: string | null;
@@ -56,6 +67,128 @@ function uploadBytesWithProgress(
});
}
function uploadPartWithProgress(
url: string,
body: Blob,
onPartProgress?: (loadedBytes: number) => void
): Promise<string> {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('PUT', url);
// Intentionally no Content-Type header: it is not part of the presigned
// UploadPart signature, and the part body is raw bytes.
xhr.upload.onprogress = (event) => {
if (!onPartProgress || !event.lengthComputable) return;
onPartProgress(event.loaded);
};
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) {
const etag = xhr.getResponseHeader('ETag');
if (!etag) {
reject(
new Error(
'Upload response missing ETag header. Configure bucket CORS to expose the ETag header.'
)
);
return;
}
resolve(etag);
return;
}
reject(new Error(`Chunk upload failed with status ${xhr.status}`));
};
xhr.onerror = () => {
reject(
new Error(
'Network error during upload. If you use direct S3/R2 uploads, configure bucket CORS to allow PUT from this site origin.'
)
);
};
xhr.onabort = () => reject(new Error('Upload aborted'));
xhr.send(body);
});
}
async function withRetry<T>(fn: () => Promise<T>, delays: number[]): Promise<T> {
let lastError: unknown;
for (let attempt = 0; attempt < delays.length; attempt += 1) {
if (attempt > 0) {
await new Promise((resolve) => setTimeout(resolve, delays[attempt]));
}
try {
return await fn();
} catch (error) {
lastError = error;
}
}
throw lastError instanceof Error ? lastError : new Error('Upload failed after retries');
}
async function completeMultipartUpload(
projectId: string,
objectKey: string,
uploadToken: string,
parts: Array<{ partNumber: number; etag: string }>
): Promise<void> {
const res = await fetch(`/api/projects/${projectId}/videos/r2-complete`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ objectKey, uploadToken, parts }),
});
if (!res.ok) {
const payload = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(payload?.error || 'Failed to complete multipart upload');
}
}
async function uploadVideoMultipart(
projectId: string,
file: File,
multipart: R2MultipartInit,
objectKey: string,
uploadToken: string,
onProgress?: UploadProgressHandler
): Promise<void> {
const totalBytes = file.size;
const partSize = multipart.partSizeBytes;
const loadedPerPart = new Array<number>(multipart.parts.length).fill(0);
const reportProgress = () => {
if (!onProgress) return;
const loaded = loadedPerPart.reduce((sum, value) => sum + value, 0);
onProgress(Math.min(100, Math.round((loaded / totalBytes) * 100)));
};
const completedParts: Array<{ partNumber: number; etag: string }> = [];
for (let index = 0; index < multipart.parts.length; index += 1) {
const part = multipart.parts[index];
const start = (part.partNumber - 1) * partSize;
const end = Math.min(start + partSize, totalBytes);
const blob = file.slice(start, end);
const etag = await withRetry(
() =>
uploadPartWithProgress(part.url, blob, (loadedBytes) => {
loadedPerPart[index] = loadedBytes;
reportProgress();
}),
PART_RETRY_DELAYS
);
loadedPerPart[index] = end - start;
reportProgress();
completedParts.push({ partNumber: part.partNumber, etag });
}
await completeMultipartUpload(projectId, objectKey, uploadToken, completedParts);
}
async function readVideoDuration(file: File): Promise<number | null> {
return new Promise((resolve) => {
const objectUrl = URL.createObjectURL(file);
@@ -151,12 +284,23 @@ export async function uploadVideoToR2(
};
try {
await uploadBytesWithProgress(
init.presignedPutUrl,
file,
init.contentType,
options?.onProgress
);
if (init.multipart) {
await uploadVideoMultipart(
projectId,
file,
init.multipart,
init.objectKey,
init.uploadToken,
options?.onProgress
);
} else {
await uploadBytesWithProgress(
init.presignedPutUrl,
file,
init.contentType,
options?.onProgress
);
}
} catch (error) {
await cleanupPendingR2VideoUpload(projectId, cleanupInput);
throw error;