mirror of
https://github.com/yusufipk/OpenFrame.git
synced 2026-09-11 09:36:08 +00:00
fix(uploads): count a Bunny upload from the moment it is admitted
A Bunny init asked the quota whether it could store zero bytes, which is a question with only one answer. Nothing an upload was about to consume was visible to the next request, so every init inside the same window read the same total and every one of them passed, and an upload that could never fit was only refused after it had been sent. The client now declares the size up front. It is checked against the account's remaining room before Bunny is asked for anything, and held as a reservation the next init has to see. The declaration is a claim rather than proof, so it is signed into the upload token: the same token already binds the video id, which is what makes the reservation safe to release on a caller's say-so, since releasing it costs them the video it belongs to. The declared size is then written onto the version or asset row and the reservation is dropped in the same transaction, because Bunny reports no size at all for a video until it has finished encoding it. On a half hour of footage that is most of an hour during which the upload did not appear on the uploader's own storage page and did not count against the next upload. Per-video accounting now takes the larger of what Bunny reports and what was declared, so the estimate stands in until the real figure arrives and Bunny's wins once it does. Two smaller things came out of the same reading. The asset route's in-transaction fallback compared against the plan limit, so a caller quoting a reservation that no longer existed was measured against 200 GiB even on a trial worth three. And the guest branch reserves without being able to release early, because a guest grant is bound to our video id and the caller's network context rather than to the Bunny video, which would let the reservation be dropped while the upload it stands for carried on.
This commit is contained in:
@@ -5,7 +5,7 @@ import { validateUrl, validateOptionalUrlOrAppPath } from '@/lib/validation';
|
||||
import { rateLimit } from '@/lib/rate-limit';
|
||||
import { notifyProjectOwner } from '@/lib/notifications';
|
||||
import { apiErrors, successResponse, withCacheControl } from '@/lib/api-response';
|
||||
import { verifyBunnyUploadToken } from '@/lib/bunny-upload-token';
|
||||
import { readBunnyUploadGrant } from '@/lib/bunny-upload-token';
|
||||
import { finalizeR2VideoUpload } from '@/lib/r2-video-finalize';
|
||||
import { logError } from '@/lib/logger';
|
||||
|
||||
@@ -65,7 +65,10 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
const video = await db.video.findFirst({
|
||||
where: { id: videoId, projectId },
|
||||
include: {
|
||||
project: true,
|
||||
// The workspace owner comes along because they are the account the
|
||||
// upload is billed to, and the Bunny reservation released below is held
|
||||
// against them rather than against whoever is adding the version.
|
||||
project: { include: { workspace: { select: { ownerId: true } } } },
|
||||
versions: { orderBy: { versionNumber: 'desc' }, take: 1 },
|
||||
},
|
||||
});
|
||||
@@ -135,6 +138,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
const normalizedUploadToken = typeof uploadToken === 'string' ? uploadToken.trim() : '';
|
||||
|
||||
let versionSizeBytes = BigInt(0);
|
||||
let bunnyReservation: string | null = null;
|
||||
let persistedProviderVideoId = normalizedProviderVideoId;
|
||||
let finalizedR2Session: {
|
||||
sessionId: string;
|
||||
@@ -148,14 +152,23 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
return apiErrors.badRequest('Bunny uploads must include providerVideoId and uploadToken');
|
||||
}
|
||||
|
||||
const isValidUploadToken = verifyBunnyUploadToken(normalizedUploadToken, {
|
||||
const grant = readBunnyUploadGrant(normalizedUploadToken, {
|
||||
userId: session.user.id,
|
||||
projectId,
|
||||
videoId: normalizedProviderVideoId,
|
||||
});
|
||||
if (!isValidUploadToken) {
|
||||
if (!grant) {
|
||||
return apiErrors.forbidden('Invalid Bunny upload token');
|
||||
}
|
||||
|
||||
// The size the upload was admitted on, written down here so the account is
|
||||
// charged for it from this moment. Bunny reports nothing at all until it
|
||||
// has finished encoding, which for a half-hour video is the better part of
|
||||
// an hour, and until this row existed those bytes were simply invisible:
|
||||
// the uploader's own storage page read zero and the next upload was
|
||||
// measured against a total that ignored the one before it.
|
||||
versionSizeBytes = grant.declaredSizeBytes ?? BigInt(0);
|
||||
bunnyReservation = grant.reservationId;
|
||||
} else if (normalizedProviderId === 'r2') {
|
||||
const normalizedObjectKey = typeof objectKey === 'string' ? objectKey.trim() : '';
|
||||
if (!normalizedObjectKey || !normalizedUploadToken) {
|
||||
@@ -226,6 +239,14 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
}
|
||||
}
|
||||
|
||||
// Handed over in the same transaction that records the size, so the bytes
|
||||
// are never counted twice and never counted zero times.
|
||||
if (bunnyReservation) {
|
||||
await tx.uploadReservation.deleteMany({
|
||||
where: { id: bunnyReservation, billedUserId: video.project.workspace.ownerId },
|
||||
});
|
||||
}
|
||||
|
||||
return tx.videoVersion.create({
|
||||
data: {
|
||||
versionNumber: nextVersionNumber,
|
||||
|
||||
@@ -5,13 +5,28 @@ import { apiErrors, successResponse, withCacheControl } from '@/lib/api-response
|
||||
import { rateLimit } from '@/lib/rate-limit';
|
||||
import crypto from 'crypto';
|
||||
import { cleanupBunnyStreamVideos } from '@/lib/bunny-stream-cleanup';
|
||||
import { createBunnyUploadToken, verifyBunnyUploadToken } from '@/lib/bunny-upload-token';
|
||||
import { isBunnyUploadsEnabled } from '@/lib/feature-flags';
|
||||
import {
|
||||
createBunnyUploadToken,
|
||||
readBunnyUploadGrant,
|
||||
verifyBunnyUploadToken,
|
||||
} from '@/lib/bunny-upload-token';
|
||||
import { getMaxVideoUploadBytes, isBunnyUploadsEnabled } from '@/lib/feature-flags';
|
||||
import { logError } from '@/lib/logger';
|
||||
import { enforceStorageQuota } from '@/lib/storage-quota';
|
||||
import {
|
||||
enforceStorageQuota,
|
||||
releaseStorageReservation,
|
||||
reserveStorageQuota,
|
||||
} from '@/lib/storage-quota';
|
||||
import { parseDeclaredUploadSize } from '@/lib/upload-size';
|
||||
|
||||
type RouteParams = { params: Promise<{ projectId: string }> };
|
||||
|
||||
// Long enough to outlive a slow upload and Bunny's own reporting delay, matching
|
||||
// what the R2 video path already reserves for. The reservation is what makes
|
||||
// concurrent uploads visible to each other, so it has to stay until the bytes it
|
||||
// stands for are counted, not until the upload finishes.
|
||||
const BUNNY_RESERVATION_TTL_MS = 2 * 60 * 60 * 1000;
|
||||
|
||||
async function getProjectWithEditAccess(projectId: string, userId: string) {
|
||||
const project = await db.project.findUnique({
|
||||
where: { id: projectId },
|
||||
@@ -64,14 +79,37 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
return apiErrors.badRequest('Bunny direct uploads are disabled by this host');
|
||||
}
|
||||
|
||||
const quotaError = await enforceStorageQuota(project.workspace.ownerId, BigInt(0));
|
||||
// The size the client says it is about to upload. It is a claim, not proof,
|
||||
// and the bytes never pass through us to be checked: they go straight to
|
||||
// Bunny, whose own reporting is what eventually settles the account. What
|
||||
// the claim buys is the two things asking for zero bytes could not. An
|
||||
// upload that plainly does not fit is refused before it starts instead of
|
||||
// halfway through, and the reservation written below makes concurrent
|
||||
// uploads visible to each other, where previously every request in the same
|
||||
// two-minute window read the same stale total and every one of them passed.
|
||||
const declaredSize = parseDeclaredUploadSize(body?.sizeBytes, getMaxVideoUploadBytes());
|
||||
if ('error' in declaredSize) {
|
||||
return apiErrors.badRequest(declaredSize.error);
|
||||
}
|
||||
|
||||
const billedUserId = project.workspace.ownerId;
|
||||
const quotaError = await enforceStorageQuota(billedUserId, declaredSize.sizeBytes);
|
||||
if (quotaError) return quotaError;
|
||||
|
||||
const reserveResult = await reserveStorageQuota(
|
||||
billedUserId,
|
||||
declaredSize.sizeBytes,
|
||||
BUNNY_RESERVATION_TTL_MS
|
||||
);
|
||||
if ('error' in reserveResult) return reserveResult.error;
|
||||
const { reservationId } = reserveResult;
|
||||
|
||||
const apiKey = process.env.BUNNY_STREAM_API_KEY;
|
||||
const libraryId =
|
||||
process.env.BUNNY_STREAM_LIBRARY_ID || process.env.NEXT_PUBLIC_BUNNY_STREAM_LIBRARY_ID;
|
||||
|
||||
if (!apiKey || !libraryId) {
|
||||
await releaseStorageReservation(reservationId, billedUserId);
|
||||
return apiErrors.internalError('Bunny Stream is not configured correctly');
|
||||
}
|
||||
|
||||
@@ -87,6 +125,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
});
|
||||
|
||||
if (!bunnyRes.ok) {
|
||||
await releaseStorageReservation(reservationId, billedUserId);
|
||||
logError('Failed to create Bunny Stream video', await bunnyRes.text());
|
||||
return apiErrors.internalError('Failed to initialize video upload with provider');
|
||||
}
|
||||
@@ -94,6 +133,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
const bunnyVideo = await bunnyRes.json();
|
||||
const videoId = bunnyVideo.guid;
|
||||
if (typeof videoId !== 'string' || videoId.length === 0) {
|
||||
await releaseStorageReservation(reservationId, billedUserId);
|
||||
return apiErrors.internalError('Upload provider did not return a valid video identifier');
|
||||
}
|
||||
|
||||
@@ -109,6 +149,8 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
userId: session.user.id,
|
||||
projectId,
|
||||
videoId,
|
||||
reservationId,
|
||||
declaredSizeBytes: declaredSize.sizeBytes,
|
||||
},
|
||||
3600
|
||||
);
|
||||
@@ -164,6 +206,18 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) {
|
||||
return apiErrors.forbidden('Invalid Bunny upload token');
|
||||
}
|
||||
|
||||
// Giving the quota back here rather than waiting for the reservation to
|
||||
// lapse: an abandoned upload that keeps holding gigabytes for two hours is
|
||||
// most of a trial's whole allowance, and the account has nothing to show for
|
||||
// it. Safe to do on a caller's say-so only because the reservation id rides
|
||||
// inside the signed token next to this video id, so releasing it costs the
|
||||
// caller the video it belongs to.
|
||||
await releaseStorageReservation(
|
||||
readBunnyUploadGrant(uploadToken, { userId: session.user.id, projectId, videoId })
|
||||
?.reservationId ?? null,
|
||||
project.workspace.ownerId
|
||||
);
|
||||
|
||||
await cleanupBunnyStreamVideos([{ providerId: 'bunny', videoId }]);
|
||||
|
||||
const response = successResponse({ message: 'Pending upload cleaned up' });
|
||||
|
||||
@@ -5,7 +5,7 @@ import { validateUrl, validateOptionalUrlOrAppPath } from '@/lib/validation';
|
||||
import { rateLimit } from '@/lib/rate-limit';
|
||||
import { notifyProjectOwner } from '@/lib/notifications';
|
||||
import { apiErrors, successResponse, withCacheControl } from '@/lib/api-response';
|
||||
import { verifyBunnyUploadToken } from '@/lib/bunny-upload-token';
|
||||
import { readBunnyUploadGrant } from '@/lib/bunny-upload-token';
|
||||
import { finalizeR2VideoUpload } from '@/lib/r2-video-finalize';
|
||||
import { logError } from '@/lib/logger';
|
||||
import { eventKey, recordEvent } from '@/lib/analytics/record';
|
||||
@@ -77,7 +77,15 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
// Check project access (must be owner, project admin, or workspace admin)
|
||||
const project = await db.project.findUnique({
|
||||
where: { id: projectId },
|
||||
select: { id: true, name: true, ownerId: true, workspaceId: true, visibility: true },
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
ownerId: true,
|
||||
workspaceId: true,
|
||||
visibility: true,
|
||||
// The billed account, which is who the Bunny reservation is held against.
|
||||
workspace: { select: { ownerId: true } },
|
||||
},
|
||||
});
|
||||
|
||||
if (!project) {
|
||||
@@ -135,6 +143,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
const normalizedUploadToken = typeof uploadToken === 'string' ? uploadToken.trim() : '';
|
||||
|
||||
let versionSizeBytes = BigInt(0);
|
||||
let bunnyReservation: string | null = null;
|
||||
let finalizedR2Session: {
|
||||
sessionId: string;
|
||||
reservationId: string | null;
|
||||
@@ -147,14 +156,20 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
return apiErrors.badRequest('Bunny uploads must include videoId and uploadToken');
|
||||
}
|
||||
|
||||
const isValidUploadToken = verifyBunnyUploadToken(normalizedUploadToken, {
|
||||
const grant = readBunnyUploadGrant(normalizedUploadToken, {
|
||||
userId: session.user.id,
|
||||
projectId,
|
||||
videoId: normalizedVideoId,
|
||||
});
|
||||
if (!isValidUploadToken) {
|
||||
if (!grant) {
|
||||
return apiErrors.forbidden('Invalid Bunny upload token');
|
||||
}
|
||||
|
||||
// See the versions route: Bunny reports no size at all until it has
|
||||
// finished encoding, so the size the upload was admitted on is what the
|
||||
// account is charged until a real figure arrives.
|
||||
versionSizeBytes = grant.declaredSizeBytes ?? BigInt(0);
|
||||
bunnyReservation = grant.reservationId;
|
||||
} else if (normalizedProviderId === 'r2') {
|
||||
const normalizedObjectKey = typeof objectKey === 'string' ? objectKey.trim() : '';
|
||||
if (!normalizedObjectKey || !normalizedUploadToken) {
|
||||
@@ -227,6 +242,14 @@ export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
}
|
||||
}
|
||||
|
||||
// Released in the transaction that records the size, so the bytes are never
|
||||
// counted twice and never counted zero times.
|
||||
if (bunnyReservation) {
|
||||
await tx.uploadReservation.deleteMany({
|
||||
where: { id: bunnyReservation, billedUserId: project.workspace.ownerId },
|
||||
});
|
||||
}
|
||||
|
||||
return tx.video.create({
|
||||
data: {
|
||||
title: title.trim(),
|
||||
|
||||
Reference in New Issue
Block a user