feat: chunked (S3 multipart) uploads for R2/S3 video backend

Self-hosted instances on the R2/S3 backend could only upload a video as a
single PUT, which fails behind a Cloudflare proxy/tunnel (100MB request-body
cap) and is capped at 5GiB with no resilience. Bunny already avoids this via
tus; this brings the R2/S3 path to parity.

Files larger than a threshold (default 90MiB) are now split into parts
(default 32MiB, min 5MiB) and uploaded directly browser->R2 via presigned
UploadPart URLs, then reassembled server-side with CompleteMultipartUpload.
Each request stays under the 100MB cap, lifts the size ceiling well past
5GiB, and adds per-chunk retry. Files at/under the threshold keep the
existing single-PUT path unchanged. Bunny path is untouched.

Thresholds are env-overridable via OPENFRAME_R2_MULTIPART_THRESHOLD_BYTES
and OPENFRAME_R2_MULTIPART_PART_SIZE_BYTES.

Verified end-to-end against real Cloudflare R2 and a local MinIO behind an
nginx 90MB cap (single 141MB PUT 413s on master; 32MB parts pass here).

Closes #22
This commit is contained in:
yusufipk
2026-07-10 19:19:07 +07:00
parent 82932c6b22
commit 880d0ac0fa
9 changed files with 513 additions and 12 deletions
@@ -10,12 +10,20 @@ import {
verifyR2UploadToken,
} from '@/lib/r2-upload-token';
import {
abortMultipartVideoUpload,
createMultipartVideoUpload,
createPresignedImagePutUrl,
createPresignedUploadPartUrl,
createPresignedVideoPutUrl,
deleteR2Object,
deleteVideoObject,
} from '@/lib/r2';
import { getMaxVideoUploadBytes, isS3VideoUploadsEnabled } from '@/lib/feature-flags';
import {
getMaxVideoUploadBytes,
getR2MultipartPartSizeBytes,
getR2MultipartThresholdBytes,
isS3VideoUploadsEnabled,
} from '@/lib/feature-flags';
import {
buildVideoObjectKey,
getVideoExtensionFromMime,
@@ -133,13 +141,52 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
const thumbnailObjectKey = `images/${thumbnailFilename}`;
const thumbnailProxyUrl = `/api/upload/image/${thumbnailFilename}`;
let presignedPutUrl: string;
const useMultipart = sizeBytes > getR2MultipartThresholdBytes();
let presignedPutUrl = '';
let thumbnailPresignedPutUrl: string;
let multipartUploadId: string | null = null;
let multipart: {
uploadId: string;
partSizeBytes: number;
parts: Array<{ partNumber: number; url: string }>;
} | null = null;
try {
[presignedPutUrl, thumbnailPresignedPutUrl] = await Promise.all([
createPresignedVideoPutUrl(objectKey, contentType, sizeBytes),
createPresignedImagePutUrl(thumbnailObjectKey, 'image/jpeg'),
]);
if (useMultipart) {
const partSize = getR2MultipartPartSizeBytes();
const partCount = Number((sizeBytes + partSize - BigInt(1)) / partSize);
multipartUploadId = await createMultipartVideoUpload(objectKey, contentType);
try {
const [parts, thumbnailUrl] = await Promise.all([
Promise.all(
Array.from({ length: partCount }, async (_unused, index) => {
const partNumber = index + 1;
const url = await createPresignedUploadPartUrl(
objectKey,
multipartUploadId as string,
partNumber
);
return { partNumber, url };
})
),
createPresignedImagePutUrl(thumbnailObjectKey, 'image/jpeg'),
]);
multipart = { uploadId: multipartUploadId, partSizeBytes: Number(partSize), parts };
thumbnailPresignedPutUrl = thumbnailUrl;
} catch (error) {
await abortMultipartVideoUpload(objectKey, multipartUploadId).catch(() => undefined);
throw error;
}
} else {
[presignedPutUrl, thumbnailPresignedPutUrl] = await Promise.all([
createPresignedVideoPutUrl(objectKey, contentType, sizeBytes),
createPresignedImagePutUrl(thumbnailObjectKey, 'image/jpeg'),
]);
}
} catch (error) {
await releaseStorageReservation(reserveResult.reservationId, project.workspace.ownerId);
logError('Failed to create presigned video upload URL:', error);
@@ -159,6 +206,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
reservationId: reserveResult.reservationId,
uploadJti,
expiresAt,
multipartUploadId,
});
const uploadToken = createR2UploadToken({
@@ -180,6 +228,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
thumbnailPresignedPutUrl,
thumbnailObjectKey,
thumbnailProxyUrl,
multipart,
});
return withCacheControl(response, 'private, no-store');
@@ -252,6 +301,7 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) {
reservationId: true,
billedUserId: true,
thumbnailObjectKey: true,
multipartUploadId: true,
},
});
if (!uploadSession) {
@@ -278,6 +328,9 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) {
try {
await Promise.all([
uploadSession.multipartUploadId
? abortMultipartVideoUpload(objectKey, uploadSession.multipartUploadId)
: Promise.resolve(),
deleteVideoObject(objectKey),
uploadSession.thumbnailObjectKey.startsWith('images/')
? deleteR2Object(uploadSession.thumbnailObjectKey)