Merge pull request #54 from yusufipk/fix/bunny-upload-reservation

fix(uploads): count a Bunny upload from the moment it is admitted

lib/admin-stats.ts

@@ -233,6 +233,8 @@ export const getCachedUserBunnyStorage = unstable_cache(
           where: { providerId: 'bunny' },
           select: {
             videoId: true,
+            // What the uploader declared, used as a floor below.
+            sizeBytes: true,
             video: {
               select: {
                 project: {
@@ -255,10 +257,28 @@ export const getCachedUserBunnyStorage = unstable_cache(
           select: {
             providerVideoId: true,
             billedUserId: true,
+            sizeBytes: true,
           },
         }),
       ]);
 
+      /**
+       * What this video costs us, as the larger of the two numbers we have.
+       *
+       * Bunny reports nothing for a video until it has finished encoding it,
+       * which on a half-hour source is most of an hour, and reading that zero
+       * literally meant an upload was free for as long as it was being
+       * processed: it did not show on the uploader's storage page and it did not
+       * count against the next upload's quota check. The size declared when the
+       * upload was admitted stands in until Bunny has a figure of its own, and
+       * Bunny's wins once it arrives, because the renditions it makes are the
+       * real bill and they are larger than the source.
+       */
+      const chargeableSize = (reported: number, declared: bigint | null): number => {
+        const declaredBytes = declared === null ? 0 : Number(declared);
+        return reported > declaredBytes ? reported : declaredBytes;
+      };
+
       const seenVideoIds = new Set<string>();
       for (const version of bunnyVersions) {
         const ownerId = version.video.project.workspace.ownerId;
@@ -266,7 +286,7 @@ export const getCachedUserBunnyStorage = unstable_cache(
         if (seenVideoIds.has(dedupeKey)) continue;
         seenVideoIds.add(dedupeKey);
 
-        const size = bunnyStats.byVideoId[version.videoId] || 0;
+        const size = chargeableSize(bunnyStats.byVideoId[version.videoId] || 0, version.sizeBytes);
         perUserStorage[ownerId] = (perUserStorage[ownerId] || 0) + size;
       }
 
@@ -277,7 +297,10 @@ export const getCachedUserBunnyStorage = unstable_cache(
         if (seenVideoIds.has(dedupeKey)) continue;
         seenVideoIds.add(dedupeKey);
 
-        const size = bunnyStats.byVideoId[asset.providerVideoId] || 0;
+        const size = chargeableSize(
+          bunnyStats.byVideoId[asset.providerVideoId] || 0,
+          asset.sizeBytes
+        );
         perUserStorage[billedUserId] = (perUserStorage[billedUserId] || 0) + size;
       }
     } catch (err) {

lib/bunny-upload-token.ts

@@ -10,6 +10,27 @@ interface BunnyUploadTokenPayload {
   vid: string;
   iat: number;
   exp: number;
+  /**
+   * The storage reservation this upload holds, when it holds one.
+   *
+   * Carried inside the signature rather than handed to the client as its own
+   * field, because a reservation id the caller can name is a reservation the
+   * caller can drop: it would take two inits and one cancel to release the
+   * quota of an upload that is still running, which is the exact hole the
+   * reservation exists to close. Signed alongside `vid`, releasing it means
+   * presenting the token for that video, which also deletes that video.
+   */
+  rid?: string;
+  /**
+   * The size the client declared when it asked for this grant, as a decimal
+   * string because JSON has no integer wide enough.
+   *
+   * Signed for the same reason as the reservation: it is written onto the row
+   * the upload creates and counted as storage until Bunny reports a figure of
+   * its own, so a client that could restate it at that point would be declaring
+   * one size to pass the quota check and another to be billed for.
+   */
+  sz?: string;
 }
 
 interface BunnyUploadTokenSubject {
@@ -41,12 +62,17 @@ function isValidPayload(value: unknown): value is BunnyUploadTokenPayload {
     typeof payload.iat === 'number' &&
     Number.isFinite(payload.iat) &&
     typeof payload.exp === 'number' &&
-    Number.isFinite(payload.exp)
+    Number.isFinite(payload.exp) &&
+    (payload.rid === undefined || typeof payload.rid === 'string') &&
+    (payload.sz === undefined || typeof payload.sz === 'string')
   );
 }
 
 export function createBunnyUploadToken(
-  subject: BunnyUploadTokenSubject,
+  subject: BunnyUploadTokenSubject & {
+    reservationId?: string | null;
+    declaredSizeBytes?: bigint | null;
+  },
   ttlSeconds = DEFAULT_TOKEN_TTL_SECONDS
 ): string {
   const now = Math.floor(Date.now() / 1000);
@@ -57,6 +83,8 @@ export function createBunnyUploadToken(
     vid: subject.videoId,
     iat: now,
     exp: now + ttlSeconds,
+    ...(subject.reservationId ? { rid: subject.reservationId } : {}),
+    ...(subject.declaredSizeBytes ? { sz: subject.declaredSizeBytes.toString() } : {}),
   };
 
   const encodedPayload = Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url');
@@ -64,40 +92,91 @@ export function createBunnyUploadToken(
   return `${encodedPayload}.${signature}`;
 }
 
-export function verifyBunnyUploadToken(token: string, subject: BunnyUploadTokenSubject): boolean {
+/**
+ * The verified payload, or null when the token is not a genuine grant for this
+ * subject. Everything `verifyBunnyUploadToken` promises holds here too; it is
+ * the same check, returning what it read instead of throwing it away.
+ */
+function readBunnyUploadToken(
+  token: string,
+  subject: BunnyUploadTokenSubject
+): BunnyUploadTokenPayload | null {
   // Resolved before the try. A missing signing secret is a configuration fault, and
   // swallowing that throw made every upload grant look like a forgery instead.
   const secret = getBunnyUploadTokenSecret();
 
   try {
     const parts = token.split('.');
-    if (parts.length !== 2) return false;
+    if (parts.length !== 2) return null;
 
     const [encodedPayload, providedSignature] = parts;
-    if (!encodedPayload || !providedSignature) return false;
+    if (!encodedPayload || !providedSignature) return null;
 
     const expectedSignature = signPayload(encodedPayload, secret);
     const providedBuffer = Buffer.from(providedSignature, 'utf8');
     const expectedBuffer = Buffer.from(expectedSignature, 'utf8');
 
-    if (providedBuffer.length !== expectedBuffer.length) return false;
-    if (!crypto.timingSafeEqual(providedBuffer, expectedBuffer)) return false;
+    if (providedBuffer.length !== expectedBuffer.length) return null;
+    if (!crypto.timingSafeEqual(providedBuffer, expectedBuffer)) return null;
 
     const payloadJson = Buffer.from(encodedPayload, 'base64url').toString('utf8');
     const payloadUnknown: unknown = JSON.parse(payloadJson);
 
-    if (!isValidPayload(payloadUnknown)) return false;
+    if (!isValidPayload(payloadUnknown)) return null;
 
     const payload = payloadUnknown;
     const now = Math.floor(Date.now() / 1000);
-    if (payload.exp < now) return false;
+    if (payload.exp < now) return null;
 
-    return (
-      payload.uid === subject.userId &&
-      payload.pid === subject.projectId &&
-      payload.vid === subject.videoId
-    );
+    if (
+      payload.uid !== subject.userId ||
+      payload.pid !== subject.projectId ||
+      payload.vid !== subject.videoId
+    ) {
+      return null;
+    }
+
+    return payload;
   } catch {
-    return false;
+    return null;
   }
 }
+
+export function verifyBunnyUploadToken(token: string, subject: BunnyUploadTokenSubject): boolean {
+  return readBunnyUploadToken(token, subject) !== null;
+}
+
+export interface BunnyUploadGrant {
+  /** The storage reservation this upload holds, or null if it holds none. */
+  reservationId: string | null;
+  /** What the client said it was uploading, or null on a grant that predates the claim. */
+  declaredSizeBytes: bigint | null;
+}
+
+/**
+ * What a genuine grant for this subject carries, or null when the token is not
+ * one.
+ *
+ * Null and empty fields mean the same thing to every caller: there is nothing
+ * here to release and nothing to bill, so a grant issued before either claim
+ * existed keeps working rather than failing an upload in flight.
+ */
+export function readBunnyUploadGrant(
+  token: string,
+  subject: BunnyUploadTokenSubject
+): BunnyUploadGrant | null {
+  const payload = readBunnyUploadToken(token, subject);
+  if (!payload) return null;
+
+  let declaredSizeBytes: bigint | null = null;
+  if (payload.sz) {
+    try {
+      const parsed = BigInt(payload.sz);
+      declaredSizeBytes = parsed > BigInt(0) ? parsed : null;
+    } catch {
+      declaredSizeBytes = null;
+    }
+  }
+
+  return { reservationId: payload.rid ?? null, declaredSizeBytes };
+}

lib/client/project-video-upload.ts

@@ -154,7 +154,9 @@ export async function uploadProjectVideo(
     const initResponse = await fetch(`/api/projects/${projectId}/videos/bunny-init`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ title }),
+      // The server checks this against the quota and holds a reservation for it,
+      // so an upload that cannot fit is turned away before any of it is sent.
+      body: JSON.stringify({ title, sizeBytes: file.size.toString() }),
     });
 
     const initPayload = (await initResponse.json().catch(() => null)) as {

lib/guest-upload-token.ts

@@ -20,6 +20,25 @@ interface GuestUploadTokenPayload {
   exp: number;
   intent: GuestUploadIntent;
   ctx: string;
+  /**
+   * The provider's own id for the video this grant was issued against.
+   *
+   * `vid` is our video, the one the asset will hang off. That is not enough to
+   * hand a guest the right to release a storage hold: the hold stands for one
+   * particular upload, and a grant that names only our video would let a guest
+   * drop it while the upload it stands for carried on. Binding the provider's id
+   * makes releasing cost the guest the upload itself, which is the same bargain
+   * the signed-in path already makes.
+   */
+  bvid?: string;
+  /** The storage reservation this upload holds, when it holds one. */
+  rid?: string;
+  /**
+   * The size the client declared when it asked for this grant, as a decimal
+   * string because JSON has no integer wide enough. Signed so a guest cannot
+   * declare one size to pass the quota check and another to be billed for.
+   */
+  sz?: string;
 }
 
 interface GuestUploadTokenSubject {
@@ -29,6 +48,20 @@ interface GuestUploadTokenSubject {
   context: string;
 }
 
+interface GuestUploadTokenClaims {
+  /** The provider video id to bind this grant to. */
+  providerVideoId?: string | null;
+  reservationId?: string | null;
+  declaredSizeBytes?: bigint | null;
+}
+
+export interface GuestUploadGrant {
+  /** The storage reservation this upload holds, or null if it holds none. */
+  reservationId: string | null;
+  /** What the client said it was uploading, or null on a grant that predates the claim. */
+  declaredSizeBytes: bigint | null;
+}
+
 const TRUSTED_IP_PATTERN = /^[\da-fA-F.:]+$/;
 
 function getGuestUploadTokenSecret(): string {
@@ -77,7 +110,10 @@ function isValidPayload(value: unknown): value is GuestUploadTokenPayload {
     typeof payload.exp === 'number' &&
     Number.isFinite(payload.exp) &&
     (payload.intent === 'audio' || payload.intent === 'image' || payload.intent === 'bunny') &&
-    typeof payload.ctx === 'string'
+    typeof payload.ctx === 'string' &&
+    (payload.bvid === undefined || typeof payload.bvid === 'string') &&
+    (payload.rid === undefined || typeof payload.rid === 'string') &&
+    (payload.sz === undefined || typeof payload.sz === 'string')
   );
 }
 
@@ -95,7 +131,7 @@ export function deriveGuestUploadContext(
 }
 
 export function createGuestUploadToken(
-  subject: GuestUploadTokenSubject,
+  subject: GuestUploadTokenSubject & GuestUploadTokenClaims,
   ttlSeconds = DEFAULT_GUEST_UPLOAD_TOKEN_TTL_SECONDS
 ): string {
   const now = Math.floor(Date.now() / 1000);
@@ -107,6 +143,9 @@ export function createGuestUploadToken(
     exp: now + ttlSeconds,
     intent: subject.intent,
     ctx: subject.context,
+    ...(subject.providerVideoId ? { bvid: subject.providerVideoId } : {}),
+    ...(subject.reservationId ? { rid: subject.reservationId } : {}),
+    ...(subject.declaredSizeBytes ? { sz: subject.declaredSizeBytes.toString() } : {}),
   };
 
   const encodedPayload = Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url');
@@ -114,39 +153,94 @@ export function createGuestUploadToken(
   return `${encodedPayload}.${signature}`;
 }
 
-export function verifyGuestUploadToken(token: string, subject: GuestUploadTokenSubject): boolean {
+/**
+ * The verified payload, or null when the token is not a genuine grant for this
+ * subject.
+ *
+ * `providerVideoId` is checked only when the grant carries one, so a token
+ * issued before that claim existed keeps working rather than failing an upload
+ * in flight. A grant that does carry one and does not match is a forgery as far
+ * as this is concerned.
+ */
+function readGuestUploadToken(
+  token: string,
+  subject: GuestUploadTokenSubject,
+  providerVideoId?: string | null
+): GuestUploadTokenPayload | null {
   try {
     const parts = token.split('.');
-    if (parts.length !== 2) return false;
+    if (parts.length !== 2) return null;
 
     const [encodedPayload, providedSignature] = parts;
-    if (!encodedPayload || !providedSignature) return false;
+    if (!encodedPayload || !providedSignature) return null;
 
     const expectedSignature = signPayload(encodedPayload);
     const providedBuffer = Buffer.from(providedSignature, 'utf8');
     const expectedBuffer = Buffer.from(expectedSignature, 'utf8');
-    if (providedBuffer.length !== expectedBuffer.length) return false;
-    if (!timingSafeEqual(providedBuffer, expectedBuffer)) return false;
+    if (providedBuffer.length !== expectedBuffer.length) return null;
+    if (!timingSafeEqual(providedBuffer, expectedBuffer)) return null;
 
     const payloadRaw = Buffer.from(encodedPayload, 'base64url').toString('utf8');
     const payloadUnknown: unknown = JSON.parse(payloadRaw);
-    if (!isValidPayload(payloadUnknown)) return false;
+    if (!isValidPayload(payloadUnknown)) return null;
 
     const payload = payloadUnknown;
     const now = Math.floor(Date.now() / 1000);
-    if (payload.exp < now) return false;
+    if (payload.exp < now) return null;
 
-    return (
-      payload.pid === subject.projectId &&
-      payload.vid === subject.videoId &&
-      payload.intent === subject.intent &&
-      payload.ctx === subject.context
-    );
+    if (
+      payload.pid !== subject.projectId ||
+      payload.vid !== subject.videoId ||
+      payload.intent !== subject.intent ||
+      payload.ctx !== subject.context
+    ) {
+      return null;
+    }
+
+    if (payload.bvid !== undefined && payload.bvid !== providerVideoId) return null;
+
+    return payload;
   } catch {
-    return false;
+    return null;
   }
 }
 
+export function verifyGuestUploadToken(
+  token: string,
+  subject: GuestUploadTokenSubject,
+  providerVideoId?: string | null
+): boolean {
+  return readGuestUploadToken(token, subject, providerVideoId) !== null;
+}
+
+/**
+ * What a genuine guest grant for this subject carries, or null when the token is
+ * not one.
+ *
+ * Null and empty fields mean the same thing to every caller: there is nothing
+ * here to release and nothing to bill.
+ */
+export function readGuestUploadGrant(
+  token: string,
+  subject: GuestUploadTokenSubject,
+  providerVideoId?: string | null
+): GuestUploadGrant | null {
+  const payload = readGuestUploadToken(token, subject, providerVideoId);
+  if (!payload) return null;
+
+  let declaredSizeBytes: bigint | null = null;
+  if (payload.sz) {
+    try {
+      const parsed = BigInt(payload.sz);
+      declaredSizeBytes = parsed > BigInt(0) ? parsed : null;
+    } catch {
+      declaredSizeBytes = null;
+    }
+  }
+
+  return { reservationId: payload.rid ?? null, declaredSizeBytes };
+}
+
 export async function enforceGuestUploadQuota(
   request: Request,
   videoId: string,

lib/storage-quota.ts

@@ -17,7 +17,7 @@ export const PLAN_STORAGE_LIMIT_BYTES = BigInt(200) * BigInt(1024) * BigInt(1024
  * directly rather than taking a flag from the caller, so no upload route can
  * forget to pass it.
  */
-async function getStorageLimitForUser(userId: string): Promise<bigint> {
+export async function getStorageLimitForUser(userId: string): Promise<bigint> {
   const user = await db.user.findUnique({
     where: { id: userId },
     select: { subscriptionStatus: true, stripeCurrentPeriodEnd: true },
@@ -29,6 +29,33 @@ async function getStorageLimitForUser(userId: string): Promise<bigint> {
 // TTL for upload reservations: 30 minutes is enough for R2 image/audio uploads
 const RESERVATION_TTL_MS = 30 * 60 * 1000;
 
+/**
+ * What a hold was opened for.
+ *
+ * A reservation is only ever consumed by the flow that opened it, and the
+ * finalize routes match on this as well as on the id. Without it, naming a
+ * reservation would be enough to drop it: the asset route takes a reservation id
+ * from the request body, and every hold an account owns is billed to the same
+ * user, so an image being attached could quietly release a video upload that was
+ * still in flight. The ids are not secret. Two of them are handed to the client
+ * outright, and the rest ride inside signed-but-readable token payloads.
+ */
+export const UPLOAD_RESERVATION_PURPOSES = {
+  /** A comment attachment or standalone image going to R2. */
+  IMAGE: 'IMAGE',
+  /** A voice note going to R2. */
+  AUDIO: 'AUDIO',
+  /** Image and voice attachments weighed together when a comment is posted. */
+  ATTACHMENT: 'ATTACHMENT',
+  /** A presigned direct upload to our own S3-compatible storage. */
+  R2_VIDEO: 'R2_VIDEO',
+  /** A direct upload to Bunny, where the bytes never pass through us. */
+  BUNNY: 'BUNNY',
+} as const;
+
+export type UploadReservationPurpose =
+  (typeof UPLOAD_RESERVATION_PURPOSES)[keyof typeof UPLOAD_RESERVATION_PURPOSES];
+
 // Sentinel error thrown inside a Prisma transaction to signal quota exceeded
 class QuotaExceededError extends Error {}
 
@@ -137,6 +164,7 @@ export async function enforceStorageQuota(
 export async function reserveStorageQuota(
   userId: string,
   incomingSizeBytes: bigint,
+  purpose: UploadReservationPurpose,
   reservationTtlMs: number = RESERVATION_TTL_MS
 ): Promise<{ reservationId: string | null } | { error: NextResponse }> {
   if (!isStripeFeatureEnabled()) {
@@ -199,7 +227,7 @@ export async function reserveStorageQuota(
       }
 
       const reservation = await tx.uploadReservation.create({
-        data: { billedUserId: userId, sizeBytes: incomingSizeBytes, expiresAt },
+        data: { billedUserId: userId, sizeBytes: incomingSizeBytes, expiresAt, purpose },
         select: { id: true },
       });
 
@@ -218,16 +246,22 @@ export async function reserveStorageQuota(
 /**
  * Deletes an upload reservation created by `reserveStorageQuota`.
  * Safe to call with `null` (no-op) for flows where billing is disabled.
+ *
+ * Pass the purpose wherever the caller knows it. A release that names only an id
+ * will delete a hold opened for something else, which is the same hole the
+ * purpose column exists to close.
  */
 export async function releaseStorageReservation(
   reservationId: string | null,
-  billedUserId?: string | null
+  billedUserId?: string | null,
+  purpose?: UploadReservationPurpose
 ): Promise<void> {
   if (!reservationId) return;
   await db.uploadReservation.deleteMany({
     where: {
       id: reservationId,
       ...(billedUserId ? { billedUserId } : {}),
+      ...(purpose ? { purpose } : {}),
     },
   });
 }

lib/upload-size.ts

@@ -0,0 +1,41 @@
+// The size a client declares before a direct upload starts.
+//
+// Shared because the R2 and Bunny paths have to agree on it: both hand the
+// number to the storage quota before a single byte moves, so a value one of them
+// would accept and the other would not is a hole in whichever is laxer.
+
+export type DeclaredUploadSize = { sizeBytes: bigint } | { error: string };
+
+/**
+ * Reads a declared upload size, refusing anything that is not a whole positive
+ * number of bytes within the host's per-file ceiling.
+ *
+ * The number is the client's word and is treated as such. Overstating it only
+ * spends the caller's own quota, and understating it is caught where the bytes
+ * land: R2 compares the object against the declaration and deletes it on a
+ * mismatch, and Bunny's own storage reporting replaces the estimate once the
+ * upload settles. What is not tolerated is an absent or nonsense value, which is
+ * what asking for zero bytes effectively was.
+ */
+export function parseDeclaredUploadSize(raw: unknown, maxBytes: bigint): DeclaredUploadSize {
+  if (typeof raw !== 'number' && typeof raw !== 'string' && typeof raw !== 'bigint') {
+    return { error: 'sizeBytes must be a positive integer' };
+  }
+
+  let sizeBytes: bigint;
+  try {
+    sizeBytes = BigInt(raw);
+  } catch {
+    return { error: 'sizeBytes must be a positive integer' };
+  }
+
+  if (sizeBytes <= BigInt(0)) {
+    return { error: 'sizeBytes must be a positive integer' };
+  }
+
+  if (sizeBytes > maxBytes) {
+    return { error: 'File exceeds the maximum allowed upload size' };
+  }
+
+  return { sizeBytes };
+}