mirror of
https://github.com/yusufipk/OpenFrame.git
synced 2026-09-12 01:46:08 +00:00
feat: chunked (S3 multipart) uploads for R2/S3 video backend
Self-hosted instances on the R2/S3 backend could only upload a video as a single PUT, which fails behind a Cloudflare proxy/tunnel (100MB request-body cap) and is capped at 5GiB with no resilience. Bunny already avoids this via tus; this brings the R2/S3 path to parity. Files larger than a threshold (default 90MiB) are now split into parts (default 32MiB, min 5MiB) and uploaded directly browser->R2 via presigned UploadPart URLs, then reassembled server-side with CompleteMultipartUpload. Each request stays under the 100MB cap, lifts the size ceiling well past 5GiB, and adds per-chunk retry. Files at/under the threshold keep the existing single-PUT path unchanged. Bunny path is untouched. Thresholds are env-overridable via OPENFRAME_R2_MULTIPART_THRESHOLD_BYTES and OPENFRAME_R2_MULTIPART_PART_SIZE_BYTES. Verified end-to-end against real Cloudflare R2 and a local MinIO behind an nginx 90MB cap (single 141MB PUT 413s on master; 32MB parts pass here). Closes #22
This commit is contained in:
@@ -1,5 +1,13 @@
|
||||
import { captureVideoThumbnail } from '@/lib/client/video-thumbnail';
|
||||
|
||||
export type R2MultipartPart = { partNumber: number; url: string };
|
||||
|
||||
export type R2MultipartInit = {
|
||||
uploadId: string;
|
||||
partSizeBytes: number;
|
||||
parts: R2MultipartPart[];
|
||||
};
|
||||
|
||||
export type R2VideoInitResponse = {
|
||||
presignedPutUrl: string;
|
||||
objectKey: string;
|
||||
@@ -10,8 +18,11 @@ export type R2VideoInitResponse = {
|
||||
thumbnailPresignedPutUrl: string;
|
||||
thumbnailObjectKey: string;
|
||||
thumbnailProxyUrl: string;
|
||||
multipart: R2MultipartInit | null;
|
||||
};
|
||||
|
||||
const PART_RETRY_DELAYS = [0, 2000, 5000, 10000];
|
||||
|
||||
export type R2VideoUploadResult = R2VideoInitResponse & {
|
||||
duration: number | null;
|
||||
thumbnailUrl: string | null;
|
||||
@@ -56,6 +67,128 @@ function uploadBytesWithProgress(
|
||||
});
|
||||
}
|
||||
|
||||
function uploadPartWithProgress(
|
||||
url: string,
|
||||
body: Blob,
|
||||
onPartProgress?: (loadedBytes: number) => void
|
||||
): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('PUT', url);
|
||||
// Intentionally no Content-Type header: it is not part of the presigned
|
||||
// UploadPart signature, and the part body is raw bytes.
|
||||
|
||||
xhr.upload.onprogress = (event) => {
|
||||
if (!onPartProgress || !event.lengthComputable) return;
|
||||
onPartProgress(event.loaded);
|
||||
};
|
||||
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
const etag = xhr.getResponseHeader('ETag');
|
||||
if (!etag) {
|
||||
reject(
|
||||
new Error(
|
||||
'Upload response missing ETag header. Configure bucket CORS to expose the ETag header.'
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
resolve(etag);
|
||||
return;
|
||||
}
|
||||
reject(new Error(`Chunk upload failed with status ${xhr.status}`));
|
||||
};
|
||||
|
||||
xhr.onerror = () => {
|
||||
reject(
|
||||
new Error(
|
||||
'Network error during upload. If you use direct S3/R2 uploads, configure bucket CORS to allow PUT from this site origin.'
|
||||
)
|
||||
);
|
||||
};
|
||||
xhr.onabort = () => reject(new Error('Upload aborted'));
|
||||
|
||||
xhr.send(body);
|
||||
});
|
||||
}
|
||||
|
||||
async function withRetry<T>(fn: () => Promise<T>, delays: number[]): Promise<T> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < delays.length; attempt += 1) {
|
||||
if (attempt > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, delays[attempt]));
|
||||
}
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
throw lastError instanceof Error ? lastError : new Error('Upload failed after retries');
|
||||
}
|
||||
|
||||
async function completeMultipartUpload(
|
||||
projectId: string,
|
||||
objectKey: string,
|
||||
uploadToken: string,
|
||||
parts: Array<{ partNumber: number; etag: string }>
|
||||
): Promise<void> {
|
||||
const res = await fetch(`/api/projects/${projectId}/videos/r2-complete`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ objectKey, uploadToken, parts }),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const payload = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(payload?.error || 'Failed to complete multipart upload');
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadVideoMultipart(
|
||||
projectId: string,
|
||||
file: File,
|
||||
multipart: R2MultipartInit,
|
||||
objectKey: string,
|
||||
uploadToken: string,
|
||||
onProgress?: UploadProgressHandler
|
||||
): Promise<void> {
|
||||
const totalBytes = file.size;
|
||||
const partSize = multipart.partSizeBytes;
|
||||
const loadedPerPart = new Array<number>(multipart.parts.length).fill(0);
|
||||
|
||||
const reportProgress = () => {
|
||||
if (!onProgress) return;
|
||||
const loaded = loadedPerPart.reduce((sum, value) => sum + value, 0);
|
||||
onProgress(Math.min(100, Math.round((loaded / totalBytes) * 100)));
|
||||
};
|
||||
|
||||
const completedParts: Array<{ partNumber: number; etag: string }> = [];
|
||||
|
||||
for (let index = 0; index < multipart.parts.length; index += 1) {
|
||||
const part = multipart.parts[index];
|
||||
const start = (part.partNumber - 1) * partSize;
|
||||
const end = Math.min(start + partSize, totalBytes);
|
||||
const blob = file.slice(start, end);
|
||||
|
||||
const etag = await withRetry(
|
||||
() =>
|
||||
uploadPartWithProgress(part.url, blob, (loadedBytes) => {
|
||||
loadedPerPart[index] = loadedBytes;
|
||||
reportProgress();
|
||||
}),
|
||||
PART_RETRY_DELAYS
|
||||
);
|
||||
|
||||
loadedPerPart[index] = end - start;
|
||||
reportProgress();
|
||||
completedParts.push({ partNumber: part.partNumber, etag });
|
||||
}
|
||||
|
||||
await completeMultipartUpload(projectId, objectKey, uploadToken, completedParts);
|
||||
}
|
||||
|
||||
async function readVideoDuration(file: File): Promise<number | null> {
|
||||
return new Promise((resolve) => {
|
||||
const objectUrl = URL.createObjectURL(file);
|
||||
@@ -151,12 +284,23 @@ export async function uploadVideoToR2(
|
||||
};
|
||||
|
||||
try {
|
||||
await uploadBytesWithProgress(
|
||||
init.presignedPutUrl,
|
||||
file,
|
||||
init.contentType,
|
||||
options?.onProgress
|
||||
);
|
||||
if (init.multipart) {
|
||||
await uploadVideoMultipart(
|
||||
projectId,
|
||||
file,
|
||||
init.multipart,
|
||||
init.objectKey,
|
||||
init.uploadToken,
|
||||
options?.onProgress
|
||||
);
|
||||
} else {
|
||||
await uploadBytesWithProgress(
|
||||
init.presignedPutUrl,
|
||||
file,
|
||||
init.contentType,
|
||||
options?.onProgress
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
await cleanupPendingR2VideoUpload(projectId, cleanupInput);
|
||||
throw error;
|
||||
|
||||
@@ -97,3 +97,36 @@ export function getMaxVideoUploadBytes(): bigint {
|
||||
export function isInviteCodeRequired() {
|
||||
return readBooleanEnv('OPENFRAME_REQUIRE_INVITE_CODE', true);
|
||||
}
|
||||
|
||||
function parseBigIntEnv(name: string, defaultValue: bigint, minValue?: bigint): bigint {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) return defaultValue;
|
||||
|
||||
try {
|
||||
const parsed = BigInt(raw);
|
||||
if (parsed <= BigInt(0)) return defaultValue;
|
||||
if (minValue !== undefined && parsed < minValue) return minValue;
|
||||
return parsed;
|
||||
} catch {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
// Files larger than this use S3 multipart upload (chunked) instead of a single PUT.
|
||||
// Default 90 MiB keeps each request under the common 100 MB Cloudflare proxy/tunnel cap.
|
||||
export function getR2MultipartThresholdBytes(): bigint {
|
||||
return parseBigIntEnv(
|
||||
'OPENFRAME_R2_MULTIPART_THRESHOLD_BYTES',
|
||||
BigInt(90) * BigInt(1024) * BigInt(1024)
|
||||
);
|
||||
}
|
||||
|
||||
// Size of each multipart chunk. Clamped to the S3 minimum of 5 MiB for non-final parts.
|
||||
export function getR2MultipartPartSizeBytes(): bigint {
|
||||
const minPartSize = BigInt(5) * BigInt(1024) * BigInt(1024);
|
||||
return parseBigIntEnv(
|
||||
'OPENFRAME_R2_MULTIPART_PART_SIZE_BYTES',
|
||||
BigInt(32) * BigInt(1024) * BigInt(1024),
|
||||
minPartSize
|
||||
);
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ export type CreateR2UploadSessionInput = {
|
||||
reservationId: string | null;
|
||||
uploadJti: string;
|
||||
expiresAt: Date;
|
||||
multipartUploadId?: string | null;
|
||||
};
|
||||
|
||||
export async function createR2UploadSession(input: CreateR2UploadSessionInput) {
|
||||
@@ -26,6 +27,7 @@ export async function createR2UploadSession(input: CreateR2UploadSessionInput) {
|
||||
reservationId: input.reservationId,
|
||||
uploadJti: input.uploadJti,
|
||||
expiresAt: input.expiresAt,
|
||||
multipartUploadId: input.multipartUploadId ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import {
|
||||
AbortMultipartUploadCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
CreateBucketCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
GetBucketCorsCommand,
|
||||
@@ -7,6 +10,7 @@ import {
|
||||
HeadObjectCommand,
|
||||
PutBucketCorsCommand,
|
||||
PutObjectCommand,
|
||||
UploadPartCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
@@ -290,6 +294,94 @@ export async function createPresignedVideoPutUrl(
|
||||
return getSignedUrl(getOrCreateR2PresignClient(), command, { expiresIn: expiresInSeconds });
|
||||
}
|
||||
|
||||
export async function createMultipartVideoUpload(
|
||||
key: string,
|
||||
contentType: string
|
||||
): Promise<string> {
|
||||
if (!key.startsWith(VIDEO_OBJECT_KEY_PREFIX)) {
|
||||
throw new Error('Invalid video object key');
|
||||
}
|
||||
|
||||
const result = await r2Client.send(
|
||||
new CreateMultipartUploadCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
ContentType: contentType,
|
||||
})
|
||||
);
|
||||
|
||||
if (!result.UploadId) {
|
||||
throw new Error('Failed to create multipart upload');
|
||||
}
|
||||
|
||||
return result.UploadId;
|
||||
}
|
||||
|
||||
export async function createPresignedUploadPartUrl(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number,
|
||||
expiresInSeconds = DEFAULT_PRESIGNED_PUT_TTL_SECONDS
|
||||
): Promise<string> {
|
||||
if (!key.startsWith(VIDEO_OBJECT_KEY_PREFIX)) {
|
||||
throw new Error('Invalid video object key');
|
||||
}
|
||||
|
||||
if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > 10000) {
|
||||
throw new Error('Invalid part number');
|
||||
}
|
||||
|
||||
const command = new UploadPartCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: partNumber,
|
||||
});
|
||||
|
||||
return getSignedUrl(getOrCreateR2PresignClient(), command, { expiresIn: expiresInSeconds });
|
||||
}
|
||||
|
||||
export async function completeMultipartVideoUpload(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: Array<{ partNumber: number; etag: string }>
|
||||
): Promise<void> {
|
||||
if (!key.startsWith(VIDEO_OBJECT_KEY_PREFIX)) {
|
||||
throw new Error('Invalid video object key');
|
||||
}
|
||||
|
||||
if (parts.length === 0) {
|
||||
throw new Error('No parts provided for multipart completion');
|
||||
}
|
||||
|
||||
const orderedParts = [...parts]
|
||||
.sort((a, b) => a.partNumber - b.partNumber)
|
||||
.map((part) => ({ PartNumber: part.partNumber, ETag: part.etag }));
|
||||
|
||||
await r2Client.send(
|
||||
new CompleteMultipartUploadCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: { Parts: orderedParts },
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export async function abortMultipartVideoUpload(key: string, uploadId: string): Promise<void> {
|
||||
if (!key.startsWith(VIDEO_OBJECT_KEY_PREFIX)) {
|
||||
throw new Error('Invalid video object key');
|
||||
}
|
||||
|
||||
await r2Client.send(
|
||||
new AbortMultipartUploadCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export async function createPresignedImagePutUrl(
|
||||
key: string,
|
||||
contentType: string,
|
||||
|
||||
Reference in New Issue
Block a user