feat: chunked (S3 multipart) uploads for R2/S3 video backend

Self-hosted instances on the R2/S3 backend could only upload a video as a single PUT, which fails behind a Cloudflare proxy/tunnel (100MB request-body cap) and is capped at 5GiB with no resilience. Bunny already avoids this via tus; this brings the R2/S3 path to parity. Files larger than a threshold (default 90MiB) are now split into parts (default 32MiB, min 5MiB) and uploaded directly browser->R2 via presigned UploadPart URLs, then reassembled server-side with CompleteMultipartUpload. Each request stays under the 100MB cap, lifts the size ceiling well past 5GiB, and adds per-chunk retry. Files at/under the threshold keep the existing single-PUT path unchanged. Bunny path is untouched. Thresholds are env-overridable via OPENFRAME_R2_MULTIPART_THRESHOLD_BYTES and OPENFRAME_R2_MULTIPART_PART_SIZE_BYTES. Verified end-to-end against real Cloudflare R2 and a local MinIO behind an nginx 90MB cap (single 141MB PUT 413s on master; 32MB parts pass here). Closes #22
2026-09-11 09:36:08 +00:00 · 2026-07-10 19:19:07 +07:00
parent 82932c6b22
commit 880d0ac0fa
9 changed files with 513 additions and 12 deletions
@@ -0,0 +1,168 @@
+import { NextRequest } from 'next/server';
+import { db } from '@/lib/db';
+import { auth, checkProjectAccess } from '@/lib/auth';
+import { apiErrors, successResponse, withCacheControl } from '@/lib/api-response';
+import { rateLimit } from '@/lib/rate-limit';
+import { parseR2UploadToken, verifyR2UploadToken } from '@/lib/r2-upload-token';
+import { abortMultipartVideoUpload, completeMultipartVideoUpload } from '@/lib/r2';
+import { isS3VideoUploadsEnabled } from '@/lib/feature-flags';
+import { objectKeyToVideoProxyPath } from '@/lib/video-upload-validation';
+import { releaseStorageReservation } from '@/lib/storage-quota';
+import { logError } from '@/lib/logger';
+
+type RouteParams = { params: Promise<{ projectId: string }> };
+
+async function getProjectWithEditAccess(projectId: string, userId: string) {
+  const project = await db.project.findUnique({
+    where: { id: projectId },
+    select: {
+      id: true,
+      ownerId: true,
+      workspaceId: true,
+      visibility: true,
+      workspace: { select: { ownerId: true } },
+    },
+  });
+
+  if (!project) return null;
+
+  const access = await checkProjectAccess(project, userId, { intent: 'manage' });
+  if (!access.canEdit) return null;
+
+  return project;
+}
+
+type IncomingPart = { partNumber: number; etag: string };
+
+function parseParts(raw: unknown): IncomingPart[] | null {
+  if (!Array.isArray(raw) || raw.length === 0 || raw.length > 10000) {
+    return null;
+  }
+
+  const parts: IncomingPart[] = [];
+  const seen = new Set<number>();
+
+  for (const entry of raw) {
+    const partNumber = (entry as { partNumber?: unknown })?.partNumber;
+    const etag = (entry as { etag?: unknown })?.etag;
+
+    if (
+      typeof partNumber !== 'number' ||
+      !Number.isInteger(partNumber) ||
+      partNumber < 1 ||
+      partNumber > 10000 ||
+      seen.has(partNumber)
+    ) {
+      return null;
+    }
+
+    if (typeof etag !== 'string' || etag.trim().length === 0) {
+      return null;
+    }
+
+    seen.add(partNumber);
+    parts.push({ partNumber, etag: etag.trim() });
+  }
+
+  return parts;
+}
+
+// POST /api/projects/[projectId]/videos/r2-complete
+export async function POST(request: NextRequest, { params }: RouteParams) {
+  try {
+    const limited = await rateLimit(request, 'mutate');
+    if (limited) return limited;
+
+    const session = await auth();
+    const { projectId } = await params;
+
+    if (!session?.user?.id) {
+      return apiErrors.unauthorized();
+    }
+
+    if (!isS3VideoUploadsEnabled()) {
+      return apiErrors.badRequest('S3 video uploads are disabled by this host');
+    }
+
+    const project = await getProjectWithEditAccess(projectId, session.user.id);
+    if (!project) {
+      return apiErrors.forbidden('Access denied');
+    }
+
+    const body = await request.json().catch(() => null);
+    const objectKey = typeof body?.objectKey === 'string' ? body.objectKey.trim() : '';
+    const uploadToken = typeof body?.uploadToken === 'string' ? body.uploadToken.trim() : '';
+    const parts = parseParts(body?.parts);
+
+    if (!objectKey || !uploadToken) {
+      return apiErrors.badRequest('objectKey and uploadToken are required');
+    }
+
+    if (!parts) {
+      return apiErrors.badRequest('parts must be a non-empty list of { partNumber, etag }');
+    }
+
+    const tokenPayload = parseR2UploadToken(uploadToken);
+    if (!tokenPayload) {
+      return apiErrors.forbidden('Invalid upload token');
+    }
+
+    const isValidUploadToken = verifyR2UploadToken(uploadToken, {
+      userId: session.user.id,
+      projectId,
+      objectKey,
+      sessionId: tokenPayload.sid,
+      tokenId: tokenPayload.jti,
+    });
+    if (!isValidUploadToken) {
+      return apiErrors.forbidden('Invalid upload token');
+    }
+
+    const uploadSession = await db.videoUploadSession.findFirst({
+      where: {
+        id: tokenPayload.sid,
+        status: 'INITIATED',
+        userId: session.user.id,
+        projectId,
+        objectKey,
+        uploadJti: tokenPayload.jti,
+        expiresAt: { gt: new Date() },
+      },
+      select: {
+        id: true,
+        multipartUploadId: true,
+        reservationId: true,
+        billedUserId: true,
+      },
+    });
+    if (!uploadSession || !uploadSession.multipartUploadId) {
+      return apiErrors.forbidden('Invalid upload token');
+    }
+
+    const proxyUrl = objectKeyToVideoProxyPath(objectKey);
+    if (!proxyUrl) {
+      return apiErrors.badRequest('Invalid object key');
+    }
+
+    try {
+      await completeMultipartVideoUpload(objectKey, uploadSession.multipartUploadId, parts);
+    } catch (error) {
+      logError('Failed to complete R2 multipart upload:', error);
+      await abortMultipartVideoUpload(objectKey, uploadSession.multipartUploadId).catch(
+        () => undefined
+      );
+      await db.videoUploadSession.updateMany({
+        where: { id: uploadSession.id, status: 'INITIATED' },
+        data: { status: 'CANCELLED', consumedAt: new Date() },
+      });
+      await releaseStorageReservation(uploadSession.reservationId, uploadSession.billedUserId);
+      return apiErrors.internalError('Failed to complete multipart upload');
+    }
+
+    const response = successResponse({ objectKey, proxyUrl });
+    return withCacheControl(response, 'private, no-store');
+  } catch (error) {
+    logError('Error completing R2 multipart upload:', error);
+    return apiErrors.internalError('Failed to complete upload');
+  }
+}
@@ -10,12 +10,20 @@ import {
  verifyR2UploadToken,
 } from '@/lib/r2-upload-token';
 import {
+  abortMultipartVideoUpload,
+  createMultipartVideoUpload,
  createPresignedImagePutUrl,
+  createPresignedUploadPartUrl,
  createPresignedVideoPutUrl,
  deleteR2Object,
  deleteVideoObject,
 } from '@/lib/r2';
-import { getMaxVideoUploadBytes, isS3VideoUploadsEnabled } from '@/lib/feature-flags';
+import {
+  getMaxVideoUploadBytes,
+  getR2MultipartPartSizeBytes,
+  getR2MultipartThresholdBytes,
+  isS3VideoUploadsEnabled,
+} from '@/lib/feature-flags';
 import {
  buildVideoObjectKey,
  getVideoExtensionFromMime,
@@ -133,13 +141,52 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
    const thumbnailObjectKey = `images/${thumbnailFilename}`;
    const thumbnailProxyUrl = `/api/upload/image/${thumbnailFilename}`;

-    let presignedPutUrl: string;
+    const useMultipart = sizeBytes > getR2MultipartThresholdBytes();
+
+    let presignedPutUrl = '';
    let thumbnailPresignedPutUrl: string;
+    let multipartUploadId: string | null = null;
+    let multipart: {
+      uploadId: string;
+      partSizeBytes: number;
+      parts: Array<{ partNumber: number; url: string }>;
+    } | null = null;
+
    try {
-      [presignedPutUrl, thumbnailPresignedPutUrl] = await Promise.all([
-        createPresignedVideoPutUrl(objectKey, contentType, sizeBytes),
-        createPresignedImagePutUrl(thumbnailObjectKey, 'image/jpeg'),
-      ]);
+      if (useMultipart) {
+        const partSize = getR2MultipartPartSizeBytes();
+        const partCount = Number((sizeBytes + partSize - BigInt(1)) / partSize);
+
+        multipartUploadId = await createMultipartVideoUpload(objectKey, contentType);
+
+        try {
+          const [parts, thumbnailUrl] = await Promise.all([
+            Promise.all(
+              Array.from({ length: partCount }, async (_unused, index) => {
+                const partNumber = index + 1;
+                const url = await createPresignedUploadPartUrl(
+                  objectKey,
+                  multipartUploadId as string,
+                  partNumber
+                );
+                return { partNumber, url };
+              })
+            ),
+            createPresignedImagePutUrl(thumbnailObjectKey, 'image/jpeg'),
+          ]);
+
+          multipart = { uploadId: multipartUploadId, partSizeBytes: Number(partSize), parts };
+          thumbnailPresignedPutUrl = thumbnailUrl;
+        } catch (error) {
+          await abortMultipartVideoUpload(objectKey, multipartUploadId).catch(() => undefined);
+          throw error;
+        }
+      } else {
+        [presignedPutUrl, thumbnailPresignedPutUrl] = await Promise.all([
+          createPresignedVideoPutUrl(objectKey, contentType, sizeBytes),
+          createPresignedImagePutUrl(thumbnailObjectKey, 'image/jpeg'),
+        ]);
+      }
    } catch (error) {
      await releaseStorageReservation(reserveResult.reservationId, project.workspace.ownerId);
      logError('Failed to create presigned video upload URL:', error);
@@ -159,6 +206,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
      reservationId: reserveResult.reservationId,
      uploadJti,
      expiresAt,
+      multipartUploadId,
    });

    const uploadToken = createR2UploadToken({
@@ -180,6 +228,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
      thumbnailPresignedPutUrl,
      thumbnailObjectKey,
      thumbnailProxyUrl,
+      multipart,
    });

    return withCacheControl(response, 'private, no-store');
@@ -252,6 +301,7 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) {
        reservationId: true,
        billedUserId: true,
        thumbnailObjectKey: true,
+        multipartUploadId: true,
      },
    });
    if (!uploadSession) {
@@ -278,6 +328,9 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) {

    try {
      await Promise.all([
+        uploadSession.multipartUploadId
+          ? abortMultipartVideoUpload(objectKey, uploadSession.multipartUploadId)
+          : Promise.resolve(),
        deleteVideoObject(objectKey),
        uploadSession.thumbnailObjectKey.startsWith('images/')
          ? deleteR2Object(uploadSession.thumbnailObjectKey)