From 7b43932fa66ff3d9940871908e89a7629ec2156a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yusuf=20=C4=B0pek?= Date: Sat, 11 Oct 2025 20:34:25 +0300 Subject: [PATCH] feat: implement image proxy for YouTube CDN images with caching to prevent rate limits --- backend/src/index.ts | 101 ++++++++++++++++++++++++++++++++++ client/app/dashboard/page.tsx | 11 ++-- client/app/overlay/page.tsx | 15 ++--- client/lib/imageProxy.ts | 27 +++++++++ memory-bank/activeContext.md | 5 +- memory-bank/progress.md | 2 + memory-bank/systemPatterns.md | 1 + 7 files changed, 149 insertions(+), 13 deletions(-) create mode 100644 client/lib/imageProxy.ts diff --git a/backend/src/index.ts b/backend/src/index.ts index 11aae49..8b4f7a4 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -3,9 +3,15 @@ import cors from '@fastify/cors'; import EventEmitter from 'eventemitter3'; import type { ChatMessage } from '@shared/chat'; import { bootstrapInnertube, type IngestionContext } from './ingestion/youtubei'; +import crypto from 'crypto'; const MAX_MESSAGES = 500; +// Simple in-memory cache for images +const imageCache = new Map(); +const CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours +const MAX_CACHE_SIZE = 1000; // Maximum number of cached images + export async function startBackend() { const fastify = Fastify({ logger: { @@ -208,6 +214,101 @@ export async function startBackend() { }); }); + // Image proxy endpoint to avoid YouTube CDN rate limits + fastify.get<{ Querystring: { url: string } }>('/proxy/image', async (request, reply) => { + const { url } = request.query; + + if (!url || typeof url !== 'string') { + reply.status(400); + return { error: 'url parameter is required' }; + } + + // Only allow YouTube CDN domains + const allowedDomains = ['yt3.ggpht.com', 'yt4.ggpht.com', 'i.ytimg.com']; + try { + const urlObj = new URL(url); + if (!allowedDomains.includes(urlObj.hostname)) { + reply.status(403); + return { error: 'Only YouTube CDN URLs are allowed' }; + } + } catch (error) { + reply.status(400); + return { error: 'Invalid URL' }; + } + + // Create cache key from URL + const cacheKey = crypto.createHash('md5').update(url).digest('hex'); + + // Check cache + const cached = imageCache.get(cacheKey); + if (cached && (Date.now() - cached.timestamp) < CACHE_TTL) { + reply.header('Content-Type', cached.contentType); + reply.header('Cache-Control', 'public, max-age=86400'); // 24 hours + reply.header('Access-Control-Allow-Origin', '*'); + return reply.send(cached.buffer); + } + + // Fetch from YouTube + try { + const response = await fetch(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Referer': 'https://www.youtube.com/', + } + }); + + if (!response.ok) { + if (response.status === 429) { + console.warn('[Backend] Rate limited by YouTube CDN for:', url); + // Return from cache even if expired, or return error + if (cached) { + reply.header('Content-Type', cached.contentType); + reply.header('Cache-Control', 'public, max-age=86400'); + reply.header('Access-Control-Allow-Origin', '*'); + return reply.send(cached.buffer); + } + } + throw new Error(`Failed to fetch image: ${response.status}`); + } + + const buffer = Buffer.from(await response.arrayBuffer()); + const contentType = response.headers.get('content-type') || 'image/jpeg'; + + // Cache the image + imageCache.set(cacheKey, { + buffer, + contentType, + timestamp: Date.now() + }); + + // Cleanup old cache entries if we exceed max size + if (imageCache.size > MAX_CACHE_SIZE) { + const entries = Array.from(imageCache.entries()); + entries.sort((a, b) => a[1].timestamp - b[1].timestamp); + const toDelete = entries.slice(0, Math.floor(MAX_CACHE_SIZE * 0.2)); // Remove oldest 20% + toDelete.forEach(([key]) => imageCache.delete(key)); + } + + reply.header('Content-Type', contentType); + reply.header('Cache-Control', 'public, max-age=86400'); + reply.header('Access-Control-Allow-Origin', '*'); + return reply.send(buffer); + } catch (error) { + console.error('[Backend] Failed to proxy image:', error); + + // Try to return stale cache if available + if (cached) { + reply.header('Content-Type', cached.contentType); + reply.header('Cache-Control', 'public, max-age=86400'); + reply.header('Access-Control-Allow-Origin', '*'); + return reply.send(cached.buffer); + } + + reply.status(500); + return { error: 'Failed to fetch image' }; + } + }); + const port = Number(process.env.PORT ?? 4100); await fastify.listen({ port, host: '0.0.0.0' }); diff --git a/client/app/dashboard/page.tsx b/client/app/dashboard/page.tsx index 10696ca..5adc510 100644 --- a/client/app/dashboard/page.tsx +++ b/client/app/dashboard/page.tsx @@ -4,6 +4,7 @@ import { useCallback, useEffect, useMemo, useState, useRef } from 'react'; import type { ChatMessage } from '@shared/chat'; import { useTimezone } from '../../lib/TimezoneContext'; import { formatTimestamp } from '../../lib/timezone'; +import { proxyImageUrl } from '../../lib/imageProxy'; // URL regex for detecting links (http/https) const URL_REGEX = /(https?:\/\/[^\s]+)/gi; @@ -607,7 +608,7 @@ function ChatItem({ message, isSelected, onSelect, onLinkClick, isPreviouslySele >
{message.authorPhoto && ( - {message.author} + {message.author} )}
@@ -617,7 +618,7 @@ function ChatItem({ message, isSelected, onSelect, onLinkClick, isPreviouslySele {message.author} {message.badges && message.badges.map((badge, i) => ( badge.imageUrl ? ( - {badge.label} + {badge.label} ) : ( {badge.type === 'moderator' && '🛡️'} @@ -639,7 +640,7 @@ function ChatItem({ message, isSelected, onSelect, onLinkClick, isPreviouslySele

{message.runs.map((r, i) => r.emojiUrl ? ( - {r.emojiAlt + {r.emojiAlt ) : ( ) @@ -670,7 +671,7 @@ function MemberItem({ message, isSelected, onSelect, onLinkClick, isPreviouslySe >

{message.authorPhoto && ( - {message.author} + {message.author} )}
{message.author} @@ -686,7 +687,7 @@ function MemberItem({ message, isSelected, onSelect, onLinkClick, isPreviouslySe

{message.runs.map((r, i) => r.emojiUrl ? ( - {r.emojiAlt + {r.emojiAlt ) : ( ) diff --git a/client/app/overlay/page.tsx b/client/app/overlay/page.tsx index 2fa6e37..fb27a98 100644 --- a/client/app/overlay/page.tsx +++ b/client/app/overlay/page.tsx @@ -4,6 +4,7 @@ import { useEffect, useState, useRef } from 'react'; import type { ChatMessage } from '@shared/chat'; import { useTimezone } from '../../lib/TimezoneContext'; import { formatTimestamp } from '../../lib/timezone'; +import { proxyImageUrl } from '../../lib/imageProxy'; const BACKEND_URL = process.env.NEXT_PUBLIC_BACKEND_URL ?? 'http://localhost:4100'; @@ -74,7 +75,7 @@ export default function OverlayPage() { <>

{message.authorPhoto && ( - {message.author} + {message.author} )} {message.author} - @@ -86,7 +87,7 @@ export default function OverlayPage() {

{message.runs.map((r, i) => r.emojiUrl ? ( - {r.emojiAlt + {r.emojiAlt ) : ( {r.text} ) @@ -100,7 +101,7 @@ export default function OverlayPage() { <>

{message.authorPhoto && ( - {message.author} + {message.author} )}
{message.author} @@ -116,7 +117,7 @@ export default function OverlayPage() {

{message.runs.map((r, i) => r.emojiUrl ? ( - {r.emojiAlt + {r.emojiAlt ) : ( {r.text} ) @@ -130,14 +131,14 @@ export default function OverlayPage() { <>

{message.authorPhoto && ( - {message.author} + {message.author} )}
{message.author} {message.badges && message.badges.map((badge, i) => ( badge.imageUrl ? ( - {badge.label} + {badge.label} ) : ( {badge.type === 'moderator' && '🛡️'} @@ -154,7 +155,7 @@ export default function OverlayPage() {

{message.runs.map((r, i) => r.emojiUrl ? ( - {r.emojiAlt + {r.emojiAlt ) : ( {r.text} ) diff --git a/client/lib/imageProxy.ts b/client/lib/imageProxy.ts new file mode 100644 index 0000000..e1b5c5e --- /dev/null +++ b/client/lib/imageProxy.ts @@ -0,0 +1,27 @@ +const BACKEND_URL = process.env.NEXT_PUBLIC_BACKEND_URL ?? 'http://localhost:4100'; + +/** + * Converts a YouTube CDN image URL to use our backend proxy + * This prevents 429 rate limit errors from YouTube's CDN + */ +export function proxyImageUrl(url: string | undefined): string | undefined { + if (!url) return undefined; + + // Check if it's a YouTube CDN URL + const youtubeCdnDomains = ['yt3.ggpht.com', 'yt4.ggpht.com', 'i.ytimg.com']; + + try { + const urlObj = new URL(url); + if (youtubeCdnDomains.includes(urlObj.hostname)) { + // Proxy through our backend + return `${BACKEND_URL}/proxy/image?url=${encodeURIComponent(url)}`; + } + } catch { + // If URL parsing fails, return as-is + return url; + } + + // Return non-YouTube URLs as-is + return url; +} + diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index faf4fb9..b81c0cf 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -7,6 +7,7 @@ - **Rich message parsing**: Full support for superchats, memberships, badges (moderator, member, verified) - **Timezone-aware timestamps**: All message timestamps display in user's local timezone with proper browser detection - **Visual selection feedback**: Previously selected messages show dimmed state for better user experience +- **Image proxy implemented**: Backend proxies all YouTube CDN images with caching to prevent 429 rate limit errors ## Recent Decisions - Fixed CORS issues by setting headers on raw response object after `reply.hijack()` for SSE endpoint @@ -24,14 +25,16 @@ - **Timezone Support**: Implemented browser timezone detection and proper timestamp formatting across dashboard and overlay using `Intl.DateTimeFormat` with GMT+3 fallback. - **Visual Selection States**: Added three-tier visual feedback system - active (selected), normal, and previously-selected (dimmed) states for better user experience. - **UI Polish**: Fixed pulse animations on initial load, hidden N/A messages, and improved overlay timestamp display. +- **Image Proxy**: Added `/proxy/image` endpoint in backend with in-memory caching (24hr TTL, max 1000 images) to prevent YouTube CDN 429 rate limit errors. All avatars, badges, and emojis now route through proxy with stale-on-error fallback. ## Immediate Next Steps 1. Test with live YouTube stream to verify badge parsing and superchat detection 2. Add search/filter functionality for chat messages 3. Implement error recovery and reconnection logic for stream interruptions 4. Add keyboard shortcuts for quick message selection +5. Consider persistent cache for images (SQLite or file-based) for better reliability ## Open Questions - Whether to add message search/filtering UI controls -- How to handle rate limiting and backoff strategies for long streams - Whether to persist Innertube visitor data between runs +- Should image cache be persistent across restarts? diff --git a/memory-bank/progress.md b/memory-bank/progress.md index d240cc3..1f5bc9c 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -34,6 +34,7 @@ - [ ] Add theme controls and customization options. ## Phase 4 – Reliability & Polish +- [x] Implement image proxy with caching to prevent YouTube CDN 429 errors. - [ ] Expand logging/metrics for long-stream observability. - [ ] Write tests (unit/integration) and contributor documentation. @@ -47,4 +48,5 @@ - **✅ Timestamp Resolution Fixed**: Critical bug resolved - timestamps now display correct current time instead of 1970 epoch - **✅ Timezone Support**: All timestamps display in user's local timezone with browser detection and GMT+3 fallback - **✅ Visual Selection States**: Three-tier feedback system (active/normal/previously-selected) for better UX +- **✅ Image Proxy Implemented**: Backend now proxies all YouTube CDN images (avatars, badges, emojis) with 24hr in-memory cache to prevent 429 rate limit errors - **Next**: Add search/filter, error recovery, and polish UX details diff --git a/memory-bank/systemPatterns.md b/memory-bank/systemPatterns.md index 040add8..50fed8d 100644 --- a/memory-bank/systemPatterns.md +++ b/memory-bank/systemPatterns.md @@ -18,3 +18,4 @@ - **Timezone Context Pattern**: React context provider (`TimezoneContext`) manages browser timezone detection and shares across components for consistent timestamp formatting. - **Selection State Management**: Three-tier visual state system (active/selected, normal, previously-selected) with CSS class composition for clear user feedback. - **Timestamp Resolution**: Backend uses `timestamp_usec` (microseconds) from YouTube data, converts to milliseconds, and frontend formats in user's local timezone. +- **Image Proxy Pattern**: Backend `/proxy/image` endpoint caches YouTube CDN images (avatars, badges, emojis) with MD5-hashed keys, 24hr TTL, and 1000-image LRU eviction. Returns stale cache on 429 errors or network failures. Frontend `proxyImageUrl()` helper transparently rewrites YouTube CDN URLs to use proxy.