mirror of
https://github.com/yusufipk/OpenFrame.git
synced 2026-09-11 09:36:08 +00:00
fix(uploads): count a Bunny upload from the moment it is admitted
A Bunny init asked the quota whether it could store zero bytes, which is a question with only one answer. Nothing an upload was about to consume was visible to the next request, so every init inside the same window read the same total and every one of them passed, and an upload that could never fit was only refused after it had been sent. The client now declares the size up front. It is checked against the account's remaining room before Bunny is asked for anything, and held as a reservation the next init has to see. The declaration is a claim rather than proof, so it is signed into the upload token: the same token already binds the video id, which is what makes the reservation safe to release on a caller's say-so, since releasing it costs them the video it belongs to. The declared size is then written onto the version or asset row and the reservation is dropped in the same transaction, because Bunny reports no size at all for a video until it has finished encoding it. On a half hour of footage that is most of an hour during which the upload did not appear on the uploader's own storage page and did not count against the next upload. Per-video accounting now takes the larger of what Bunny reports and what was declared, so the estimate stands in until the real figure arrives and Bunny's wins once it does. Two smaller things came out of the same reading. The asset route's in-transaction fallback compared against the plan limit, so a caller quoting a reservation that no longer existed was measured against 200 GiB even on a trial worth three. And the guest branch reserves without being able to release early, because a guest grant is bound to our video id and the caller's network context rather than to the Bunny video, which would let the reservation be dropped while the upload it stands for carried on.
This commit is contained in:
+25
-2
@@ -233,6 +233,8 @@ export const getCachedUserBunnyStorage = unstable_cache(
|
||||
where: { providerId: 'bunny' },
|
||||
select: {
|
||||
videoId: true,
|
||||
// What the uploader declared, used as a floor below.
|
||||
sizeBytes: true,
|
||||
video: {
|
||||
select: {
|
||||
project: {
|
||||
@@ -255,10 +257,28 @@ export const getCachedUserBunnyStorage = unstable_cache(
|
||||
select: {
|
||||
providerVideoId: true,
|
||||
billedUserId: true,
|
||||
sizeBytes: true,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
/**
|
||||
* What this video costs us, as the larger of the two numbers we have.
|
||||
*
|
||||
* Bunny reports nothing for a video until it has finished encoding it,
|
||||
* which on a half-hour source is most of an hour, and reading that zero
|
||||
* literally meant an upload was free for as long as it was being
|
||||
* processed: it did not show on the uploader's storage page and it did not
|
||||
* count against the next upload's quota check. The size declared when the
|
||||
* upload was admitted stands in until Bunny has a figure of its own, and
|
||||
* Bunny's wins once it arrives, because the renditions it makes are the
|
||||
* real bill and they are larger than the source.
|
||||
*/
|
||||
const chargeableSize = (reported: number, declared: bigint | null): number => {
|
||||
const declaredBytes = declared === null ? 0 : Number(declared);
|
||||
return reported > declaredBytes ? reported : declaredBytes;
|
||||
};
|
||||
|
||||
const seenVideoIds = new Set<string>();
|
||||
for (const version of bunnyVersions) {
|
||||
const ownerId = version.video.project.workspace.ownerId;
|
||||
@@ -266,7 +286,7 @@ export const getCachedUserBunnyStorage = unstable_cache(
|
||||
if (seenVideoIds.has(dedupeKey)) continue;
|
||||
seenVideoIds.add(dedupeKey);
|
||||
|
||||
const size = bunnyStats.byVideoId[version.videoId] || 0;
|
||||
const size = chargeableSize(bunnyStats.byVideoId[version.videoId] || 0, version.sizeBytes);
|
||||
perUserStorage[ownerId] = (perUserStorage[ownerId] || 0) + size;
|
||||
}
|
||||
|
||||
@@ -277,7 +297,10 @@ export const getCachedUserBunnyStorage = unstable_cache(
|
||||
if (seenVideoIds.has(dedupeKey)) continue;
|
||||
seenVideoIds.add(dedupeKey);
|
||||
|
||||
const size = bunnyStats.byVideoId[asset.providerVideoId] || 0;
|
||||
const size = chargeableSize(
|
||||
bunnyStats.byVideoId[asset.providerVideoId] || 0,
|
||||
asset.sizeBytes
|
||||
);
|
||||
perUserStorage[billedUserId] = (perUserStorage[billedUserId] || 0) + size;
|
||||
}
|
||||
} catch (err) {
|
||||
|
||||
+94
-15
@@ -10,6 +10,27 @@ interface BunnyUploadTokenPayload {
|
||||
vid: string;
|
||||
iat: number;
|
||||
exp: number;
|
||||
/**
|
||||
* The storage reservation this upload holds, when it holds one.
|
||||
*
|
||||
* Carried inside the signature rather than handed to the client as its own
|
||||
* field, because a reservation id the caller can name is a reservation the
|
||||
* caller can drop: it would take two inits and one cancel to release the
|
||||
* quota of an upload that is still running, which is the exact hole the
|
||||
* reservation exists to close. Signed alongside `vid`, releasing it means
|
||||
* presenting the token for that video, which also deletes that video.
|
||||
*/
|
||||
rid?: string;
|
||||
/**
|
||||
* The size the client declared when it asked for this grant, as a decimal
|
||||
* string because JSON has no integer wide enough.
|
||||
*
|
||||
* Signed for the same reason as the reservation: it is written onto the row
|
||||
* the upload creates and counted as storage until Bunny reports a figure of
|
||||
* its own, so a client that could restate it at that point would be declaring
|
||||
* one size to pass the quota check and another to be billed for.
|
||||
*/
|
||||
sz?: string;
|
||||
}
|
||||
|
||||
interface BunnyUploadTokenSubject {
|
||||
@@ -41,12 +62,17 @@ function isValidPayload(value: unknown): value is BunnyUploadTokenPayload {
|
||||
typeof payload.iat === 'number' &&
|
||||
Number.isFinite(payload.iat) &&
|
||||
typeof payload.exp === 'number' &&
|
||||
Number.isFinite(payload.exp)
|
||||
Number.isFinite(payload.exp) &&
|
||||
(payload.rid === undefined || typeof payload.rid === 'string') &&
|
||||
(payload.sz === undefined || typeof payload.sz === 'string')
|
||||
);
|
||||
}
|
||||
|
||||
export function createBunnyUploadToken(
|
||||
subject: BunnyUploadTokenSubject,
|
||||
subject: BunnyUploadTokenSubject & {
|
||||
reservationId?: string | null;
|
||||
declaredSizeBytes?: bigint | null;
|
||||
},
|
||||
ttlSeconds = DEFAULT_TOKEN_TTL_SECONDS
|
||||
): string {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
@@ -57,6 +83,8 @@ export function createBunnyUploadToken(
|
||||
vid: subject.videoId,
|
||||
iat: now,
|
||||
exp: now + ttlSeconds,
|
||||
...(subject.reservationId ? { rid: subject.reservationId } : {}),
|
||||
...(subject.declaredSizeBytes ? { sz: subject.declaredSizeBytes.toString() } : {}),
|
||||
};
|
||||
|
||||
const encodedPayload = Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url');
|
||||
@@ -64,40 +92,91 @@ export function createBunnyUploadToken(
|
||||
return `${encodedPayload}.${signature}`;
|
||||
}
|
||||
|
||||
export function verifyBunnyUploadToken(token: string, subject: BunnyUploadTokenSubject): boolean {
|
||||
/**
|
||||
* The verified payload, or null when the token is not a genuine grant for this
|
||||
* subject. Everything `verifyBunnyUploadToken` promises holds here too; it is
|
||||
* the same check, returning what it read instead of throwing it away.
|
||||
*/
|
||||
function readBunnyUploadToken(
|
||||
token: string,
|
||||
subject: BunnyUploadTokenSubject
|
||||
): BunnyUploadTokenPayload | null {
|
||||
// Resolved before the try. A missing signing secret is a configuration fault, and
|
||||
// swallowing that throw made every upload grant look like a forgery instead.
|
||||
const secret = getBunnyUploadTokenSecret();
|
||||
|
||||
try {
|
||||
const parts = token.split('.');
|
||||
if (parts.length !== 2) return false;
|
||||
if (parts.length !== 2) return null;
|
||||
|
||||
const [encodedPayload, providedSignature] = parts;
|
||||
if (!encodedPayload || !providedSignature) return false;
|
||||
if (!encodedPayload || !providedSignature) return null;
|
||||
|
||||
const expectedSignature = signPayload(encodedPayload, secret);
|
||||
const providedBuffer = Buffer.from(providedSignature, 'utf8');
|
||||
const expectedBuffer = Buffer.from(expectedSignature, 'utf8');
|
||||
|
||||
if (providedBuffer.length !== expectedBuffer.length) return false;
|
||||
if (!crypto.timingSafeEqual(providedBuffer, expectedBuffer)) return false;
|
||||
if (providedBuffer.length !== expectedBuffer.length) return null;
|
||||
if (!crypto.timingSafeEqual(providedBuffer, expectedBuffer)) return null;
|
||||
|
||||
const payloadJson = Buffer.from(encodedPayload, 'base64url').toString('utf8');
|
||||
const payloadUnknown: unknown = JSON.parse(payloadJson);
|
||||
|
||||
if (!isValidPayload(payloadUnknown)) return false;
|
||||
if (!isValidPayload(payloadUnknown)) return null;
|
||||
|
||||
const payload = payloadUnknown;
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
if (payload.exp < now) return false;
|
||||
if (payload.exp < now) return null;
|
||||
|
||||
return (
|
||||
payload.uid === subject.userId &&
|
||||
payload.pid === subject.projectId &&
|
||||
payload.vid === subject.videoId
|
||||
);
|
||||
if (
|
||||
payload.uid !== subject.userId ||
|
||||
payload.pid !== subject.projectId ||
|
||||
payload.vid !== subject.videoId
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return payload;
|
||||
} catch {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function verifyBunnyUploadToken(token: string, subject: BunnyUploadTokenSubject): boolean {
|
||||
return readBunnyUploadToken(token, subject) !== null;
|
||||
}
|
||||
|
||||
export interface BunnyUploadGrant {
|
||||
/** The storage reservation this upload holds, or null if it holds none. */
|
||||
reservationId: string | null;
|
||||
/** What the client said it was uploading, or null on a grant that predates the claim. */
|
||||
declaredSizeBytes: bigint | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* What a genuine grant for this subject carries, or null when the token is not
|
||||
* one.
|
||||
*
|
||||
* Null and empty fields mean the same thing to every caller: there is nothing
|
||||
* here to release and nothing to bill, so a grant issued before either claim
|
||||
* existed keeps working rather than failing an upload in flight.
|
||||
*/
|
||||
export function readBunnyUploadGrant(
|
||||
token: string,
|
||||
subject: BunnyUploadTokenSubject
|
||||
): BunnyUploadGrant | null {
|
||||
const payload = readBunnyUploadToken(token, subject);
|
||||
if (!payload) return null;
|
||||
|
||||
let declaredSizeBytes: bigint | null = null;
|
||||
if (payload.sz) {
|
||||
try {
|
||||
const parsed = BigInt(payload.sz);
|
||||
declaredSizeBytes = parsed > BigInt(0) ? parsed : null;
|
||||
} catch {
|
||||
declaredSizeBytes = null;
|
||||
}
|
||||
}
|
||||
|
||||
return { reservationId: payload.rid ?? null, declaredSizeBytes };
|
||||
}
|
||||
|
||||
@@ -154,7 +154,9 @@ export async function uploadProjectVideo(
|
||||
const initResponse = await fetch(`/api/projects/${projectId}/videos/bunny-init`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ title }),
|
||||
// The server checks this against the quota and holds a reservation for it,
|
||||
// so an upload that cannot fit is turned away before any of it is sent.
|
||||
body: JSON.stringify({ title, sizeBytes: file.size.toString() }),
|
||||
});
|
||||
|
||||
const initPayload = (await initResponse.json().catch(() => null)) as {
|
||||
|
||||
@@ -17,7 +17,7 @@ export const PLAN_STORAGE_LIMIT_BYTES = BigInt(200) * BigInt(1024) * BigInt(1024
|
||||
* directly rather than taking a flag from the caller, so no upload route can
|
||||
* forget to pass it.
|
||||
*/
|
||||
async function getStorageLimitForUser(userId: string): Promise<bigint> {
|
||||
export async function getStorageLimitForUser(userId: string): Promise<bigint> {
|
||||
const user = await db.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { subscriptionStatus: true, stripeCurrentPeriodEnd: true },
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
// The size a client declares before a direct upload starts.
|
||||
//
|
||||
// Shared because the R2 and Bunny paths have to agree on it: both hand the
|
||||
// number to the storage quota before a single byte moves, so a value one of them
|
||||
// would accept and the other would not is a hole in whichever is laxer.
|
||||
|
||||
export type DeclaredUploadSize = { sizeBytes: bigint } | { error: string };
|
||||
|
||||
/**
|
||||
* Reads a declared upload size, refusing anything that is not a whole positive
|
||||
* number of bytes within the host's per-file ceiling.
|
||||
*
|
||||
* The number is the client's word and is treated as such. Overstating it only
|
||||
* spends the caller's own quota, and understating it is caught where the bytes
|
||||
* land: R2 compares the object against the declaration and deletes it on a
|
||||
* mismatch, and Bunny's own storage reporting replaces the estimate once the
|
||||
* upload settles. What is not tolerated is an absent or nonsense value, which is
|
||||
* what asking for zero bytes effectively was.
|
||||
*/
|
||||
export function parseDeclaredUploadSize(raw: unknown, maxBytes: bigint): DeclaredUploadSize {
|
||||
if (typeof raw !== 'number' && typeof raw !== 'string' && typeof raw !== 'bigint') {
|
||||
return { error: 'sizeBytes must be a positive integer' };
|
||||
}
|
||||
|
||||
let sizeBytes: bigint;
|
||||
try {
|
||||
sizeBytes = BigInt(raw);
|
||||
} catch {
|
||||
return { error: 'sizeBytes must be a positive integer' };
|
||||
}
|
||||
|
||||
if (sizeBytes <= BigInt(0)) {
|
||||
return { error: 'sizeBytes must be a positive integer' };
|
||||
}
|
||||
|
||||
if (sizeBytes > maxBytes) {
|
||||
return { error: 'File exceeds the maximum allowed upload size' };
|
||||
}
|
||||
|
||||
return { sizeBytes };
|
||||
}
|
||||
Reference in New Issue
Block a user