mirror of
https://github.com/yusufipk/OpenFrame.git
synced 2026-09-11 17:46:06 +00:00
feat: chunked (S3 multipart) uploads for R2/S3 video backend
Self-hosted instances on the R2/S3 backend could only upload a video as a single PUT, which fails behind a Cloudflare proxy/tunnel (100MB request-body cap) and is capped at 5GiB with no resilience. Bunny already avoids this via tus; this brings the R2/S3 path to parity. Files larger than a threshold (default 90MiB) are now split into parts (default 32MiB, min 5MiB) and uploaded directly browser->R2 via presigned UploadPart URLs, then reassembled server-side with CompleteMultipartUpload. Each request stays under the 100MB cap, lifts the size ceiling well past 5GiB, and adds per-chunk retry. Files at/under the threshold keep the existing single-PUT path unchanged. Bunny path is untouched. Thresholds are env-overridable via OPENFRAME_R2_MULTIPART_THRESHOLD_BYTES and OPENFRAME_R2_MULTIPART_PART_SIZE_BYTES. Verified end-to-end against real Cloudflare R2 and a local MinIO behind an nginx 90MB cap (single 141MB PUT 413s on master; 32MB parts pass here). Closes #22
This commit is contained in:
@@ -1,5 +1,8 @@
|
||||
import {
|
||||
AbortMultipartUploadCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
CreateBucketCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
GetBucketCorsCommand,
|
||||
@@ -7,6 +10,7 @@ import {
|
||||
HeadObjectCommand,
|
||||
PutBucketCorsCommand,
|
||||
PutObjectCommand,
|
||||
UploadPartCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
@@ -290,6 +294,94 @@ export async function createPresignedVideoPutUrl(
|
||||
return getSignedUrl(getOrCreateR2PresignClient(), command, { expiresIn: expiresInSeconds });
|
||||
}
|
||||
|
||||
export async function createMultipartVideoUpload(
|
||||
key: string,
|
||||
contentType: string
|
||||
): Promise<string> {
|
||||
if (!key.startsWith(VIDEO_OBJECT_KEY_PREFIX)) {
|
||||
throw new Error('Invalid video object key');
|
||||
}
|
||||
|
||||
const result = await r2Client.send(
|
||||
new CreateMultipartUploadCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
ContentType: contentType,
|
||||
})
|
||||
);
|
||||
|
||||
if (!result.UploadId) {
|
||||
throw new Error('Failed to create multipart upload');
|
||||
}
|
||||
|
||||
return result.UploadId;
|
||||
}
|
||||
|
||||
export async function createPresignedUploadPartUrl(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number,
|
||||
expiresInSeconds = DEFAULT_PRESIGNED_PUT_TTL_SECONDS
|
||||
): Promise<string> {
|
||||
if (!key.startsWith(VIDEO_OBJECT_KEY_PREFIX)) {
|
||||
throw new Error('Invalid video object key');
|
||||
}
|
||||
|
||||
if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > 10000) {
|
||||
throw new Error('Invalid part number');
|
||||
}
|
||||
|
||||
const command = new UploadPartCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: partNumber,
|
||||
});
|
||||
|
||||
return getSignedUrl(getOrCreateR2PresignClient(), command, { expiresIn: expiresInSeconds });
|
||||
}
|
||||
|
||||
export async function completeMultipartVideoUpload(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: Array<{ partNumber: number; etag: string }>
|
||||
): Promise<void> {
|
||||
if (!key.startsWith(VIDEO_OBJECT_KEY_PREFIX)) {
|
||||
throw new Error('Invalid video object key');
|
||||
}
|
||||
|
||||
if (parts.length === 0) {
|
||||
throw new Error('No parts provided for multipart completion');
|
||||
}
|
||||
|
||||
const orderedParts = [...parts]
|
||||
.sort((a, b) => a.partNumber - b.partNumber)
|
||||
.map((part) => ({ PartNumber: part.partNumber, ETag: part.etag }));
|
||||
|
||||
await r2Client.send(
|
||||
new CompleteMultipartUploadCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: { Parts: orderedParts },
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export async function abortMultipartVideoUpload(key: string, uploadId: string): Promise<void> {
|
||||
if (!key.startsWith(VIDEO_OBJECT_KEY_PREFIX)) {
|
||||
throw new Error('Invalid video object key');
|
||||
}
|
||||
|
||||
await r2Client.send(
|
||||
new AbortMultipartUploadCommand({
|
||||
Bucket: R2_BUCKET_NAME,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export async function createPresignedImagePutUrl(
|
||||
key: string,
|
||||
contentType: string,
|
||||
|
||||
Reference in New Issue
Block a user