mirror of
https://github.com/yusufipk/YTChatHub.git
synced 2026-09-11 10:56:17 +00:00
Rewrite dashboard and overlay, single /events stream, static client served by the backend
Backend: one SSE stream for messages, selection, poll and connection status; reconnect with backoff; no on-disk Innertube session cache; mock only with MOCK_CHAT=1; binds to 127.0.0.1; serves client/out so production runs on one port. Full URLs for links YouTube truncates in chat. Client: components split out, Lucide icons, stream title and status, on-air strip, search, pause, keyboard shortcuts, overlay settings dialog. Overlay themes, position, size, animation and auto-hide via URL, sized for 1080p and scaled with the source. New blueprint theme for the brand background. Removed Tailwind, better-sqlite3, zod, the timezone helpers, Cline memory bank and AGENTS.md. Added ESLint, node:test tests and CI.
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
import type { FastifyInstance, FastifyReply } from 'fastify';
|
||||
import crypto from 'crypto';
|
||||
|
||||
type CachedImage = { buffer: Buffer; contentType: string; timestamp: number };
|
||||
|
||||
const ALLOWED_HOSTS = ['yt3.ggpht.com', 'yt4.ggpht.com', 'i.ytimg.com', 'lh3.googleusercontent.com'];
|
||||
const CACHE_TTL = 1000 * 60 * 60 * 24;
|
||||
const MAX_CACHE_SIZE = 1000;
|
||||
const USER_AGENT =
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36';
|
||||
|
||||
const imageCache = new Map<string, CachedImage>();
|
||||
|
||||
function sendImage(reply: FastifyReply, image: CachedImage) {
|
||||
reply.header('Content-Type', image.contentType);
|
||||
reply.header('Cache-Control', 'public, max-age=86400');
|
||||
reply.header('Access-Control-Allow-Origin', '*');
|
||||
return reply.send(image.buffer);
|
||||
}
|
||||
|
||||
/** GET /proxy/image?url= fetches YouTube CDN images through the backend so the browser does not hit CDN rate limits. */
|
||||
export function registerImageProxy(fastify: FastifyInstance): void {
|
||||
fastify.get<{ Querystring: { url?: string } }>('/proxy/image', async (request, reply) => {
|
||||
const { url } = request.query;
|
||||
if (!url || typeof url !== 'string') {
|
||||
reply.status(400);
|
||||
return { error: 'url parameter is required' };
|
||||
}
|
||||
|
||||
try {
|
||||
if (!ALLOWED_HOSTS.includes(new URL(url).hostname)) {
|
||||
reply.status(403);
|
||||
return { error: 'Only YouTube CDN and Google User Content URLs are allowed' };
|
||||
}
|
||||
} catch {
|
||||
reply.status(400);
|
||||
return { error: 'Invalid URL' };
|
||||
}
|
||||
|
||||
const cacheKey = crypto.createHash('md5').update(url).digest('hex');
|
||||
const cached = imageCache.get(cacheKey);
|
||||
if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
|
||||
return sendImage(reply, cached);
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: { 'User-Agent': USER_AGENT, Referer: 'https://www.youtube.com/' }
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 429) {
|
||||
console.warn('[Backend] Rate limited by YouTube CDN for:', url);
|
||||
if (cached) return sendImage(reply, cached);
|
||||
}
|
||||
throw new Error(`Failed to fetch image: ${response.status}`);
|
||||
}
|
||||
|
||||
const image: CachedImage = {
|
||||
buffer: Buffer.from(await response.arrayBuffer()),
|
||||
contentType: response.headers.get('content-type') || 'image/jpeg',
|
||||
timestamp: Date.now()
|
||||
};
|
||||
imageCache.set(cacheKey, image);
|
||||
|
||||
if (imageCache.size > MAX_CACHE_SIZE) {
|
||||
// Drop the oldest 20% so eviction is not a per-request cost.
|
||||
const entries = Array.from(imageCache.entries()).sort((a, b) => a[1].timestamp - b[1].timestamp);
|
||||
for (const [key] of entries.slice(0, Math.floor(MAX_CACHE_SIZE * 0.2))) imageCache.delete(key);
|
||||
}
|
||||
|
||||
return sendImage(reply, image);
|
||||
} catch (error) {
|
||||
console.error('[Backend] Failed to proxy image:', error);
|
||||
// A stale copy beats a broken avatar.
|
||||
if (cached) return sendImage(reply, cached);
|
||||
reply.status(500);
|
||||
return { error: 'Failed to fetch image' };
|
||||
}
|
||||
});
|
||||
}
|
||||
+222
-404
@@ -1,473 +1,291 @@
|
||||
import Fastify from 'fastify';
|
||||
import cors from '@fastify/cors';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import EventEmitter from 'eventemitter3';
|
||||
import type { ChatMessage, Poll } from '@shared/chat';
|
||||
import { existsSync } from 'fs';
|
||||
import type { OutgoingHttpHeaders, ServerResponse } from 'http';
|
||||
import path from 'path';
|
||||
import type { ChatMessage, ConnectionStatus, Poll, ServerEvent } from '@shared/chat';
|
||||
import { bootstrapInnertube, type IngestionContext } from './ingestion/youtubei';
|
||||
import crypto from 'crypto';
|
||||
import { registerImageProxy } from './imageProxy';
|
||||
import { extractLiveId } from './liveId';
|
||||
|
||||
const MAX_MESSAGES = 500;
|
||||
const MAX_REGULAR_MESSAGES = 200; // Keep fewer regular messages
|
||||
const MAX_REGULAR_MESSAGES = 200;
|
||||
const MAX_SPECIAL_MESSAGES = 500;
|
||||
const RETRY_LIMIT = 30;
|
||||
const RETRY_BASE_MS = 2000;
|
||||
const RETRY_CAP_MS = 30_000;
|
||||
const HEARTBEAT_MS = 15_000;
|
||||
const CORS_ORIGINS = ['http://localhost:3100', 'http://127.0.0.1:3100'];
|
||||
|
||||
// Simple in-memory cache for images
|
||||
const imageCache = new Map<string, { buffer: Buffer; contentType: string; timestamp: number }>();
|
||||
const CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours
|
||||
const MAX_CACHE_SIZE = 1000; // Maximum number of cached images
|
||||
type Bus = EventEmitter<{ event: (event: ServerEvent) => void }>;
|
||||
|
||||
const isSpecial = (m: ChatMessage) => !!(m.superChat || m.membershipGift || m.membershipGiftPurchase);
|
||||
const errorText = (e: unknown) => (e instanceof Error ? e.message : String(e));
|
||||
|
||||
/** Drops the oldest messages of each class past its cap, in place, without allocating. */
|
||||
function trimMessages(store: ChatMessage[]): void {
|
||||
let regular = 0;
|
||||
let special = 0;
|
||||
for (const m of store) isSpecial(m) ? special++ : regular++;
|
||||
let dropRegular = Math.max(0, regular - MAX_REGULAR_MESSAGES);
|
||||
let dropSpecial = Math.max(0, special - MAX_SPECIAL_MESSAGES);
|
||||
if (!dropRegular && !dropSpecial) return;
|
||||
|
||||
let write = 0;
|
||||
for (const m of store) {
|
||||
if (isSpecial(m) ? dropSpecial > 0 : dropRegular > 0) {
|
||||
isSpecial(m) ? dropSpecial-- : dropRegular--;
|
||||
continue;
|
||||
}
|
||||
store[write++] = m;
|
||||
}
|
||||
store.length = write;
|
||||
}
|
||||
|
||||
function writeEvent(res: ServerResponse, event: ServerEvent): void {
|
||||
res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
|
||||
export async function startBackend() {
|
||||
const fastify = Fastify({
|
||||
logger: {
|
||||
level: 'warn', // Only show warnings and errors, not every request
|
||||
}
|
||||
});
|
||||
|
||||
// Register CORS before any routes
|
||||
await fastify.register(cors, {
|
||||
origin: '*',
|
||||
methods: ['GET', 'POST', 'DELETE', 'OPTIONS', 'PUT', 'PATCH'],
|
||||
allowedHeaders: ['Content-Type', 'Authorization'],
|
||||
credentials: false
|
||||
});
|
||||
const fastify = Fastify({ logger: { level: 'warn' } });
|
||||
await fastify.register(cors, { origin: CORS_ORIGINS, methods: ['GET', 'POST', 'DELETE', 'OPTIONS'] });
|
||||
|
||||
const store: ChatMessage[] = [];
|
||||
let currentSelection: ChatMessage | null = null;
|
||||
let currentPoll: Poll | null = null;
|
||||
const overlayEmitter = new EventEmitter<{ update: (message: ChatMessage | null) => void }>();
|
||||
const pollEmitter = new EventEmitter<{ update: (poll: Poll | null) => void }>();
|
||||
|
||||
const rawLiveId = process.env.YOUTUBE_LIVE_ID ?? '';
|
||||
const parsedLiveId = extractLiveId(rawLiveId);
|
||||
const shouldMock = !parsedLiveId;
|
||||
|
||||
const bus: Bus = new EventEmitter();
|
||||
const mockEnabled = process.env.MOCK_CHAT === '1';
|
||||
let selection: ChatMessage | null = null;
|
||||
let poll: Poll | null = null;
|
||||
let status: ConnectionStatus = { state: 'disconnected', liveId: null, title: null, error: null };
|
||||
let ingestion: IngestionContext | null = null;
|
||||
let mockInterval: NodeJS.Timeout | null = null;
|
||||
let retryTimer: NodeJS.Timeout | null = null;
|
||||
let mockTimer: NodeJS.Timeout | null = null;
|
||||
// Bumped whenever the active connection changes so a bootstrap that resolves late is discarded.
|
||||
let generation = 0;
|
||||
|
||||
if (!shouldMock) {
|
||||
try {
|
||||
console.log(`[Backend] Connecting to YouTube Live ID: ${parsedLiveId}`);
|
||||
ingestion = await bootstrapInnertube(parsedLiveId);
|
||||
console.log(`[Backend] ✓ YouTube chat connected successfully`);
|
||||
ingestion.emitter.on('message', (message) => {
|
||||
store.push(message);
|
||||
// Trim regularly to keep regular messages under control
|
||||
// This ensures we don't wait until hitting MAX_MESSAGES
|
||||
trimMessages(store);
|
||||
});
|
||||
ingestion.emitter.on('poll', (poll) => {
|
||||
currentPoll = poll;
|
||||
pollEmitter.emit('update', poll);
|
||||
});
|
||||
ingestion.emitter.on('error', (error) => {
|
||||
console.error('[Backend] Innertube ingestion error:', error);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Backend] Failed to bootstrap Innertube; falling back to mock data:', error);
|
||||
}
|
||||
} else {
|
||||
console.log('[Backend] No YOUTUBE_LIVE_ID found, running in mock mode');
|
||||
const emit = (event: ServerEvent) => bus.emit('event', event);
|
||||
|
||||
function setStatus(patch: Partial<ConnectionStatus>) {
|
||||
status = { ...status, ...patch };
|
||||
emit({ type: 'status', status });
|
||||
}
|
||||
|
||||
if (!ingestion) {
|
||||
mockInterval = seedMockMessages(store, overlayEmitter);
|
||||
function pushMessage(message: ChatMessage) {
|
||||
store.push(message);
|
||||
trimMessages(store);
|
||||
emit({ type: 'message', message });
|
||||
}
|
||||
|
||||
function setSelection(message: ChatMessage | null) {
|
||||
selection = message;
|
||||
emit({ type: 'selection', message });
|
||||
}
|
||||
|
||||
function setPoll(next: Poll | null) {
|
||||
poll = next;
|
||||
emit({ type: 'poll', poll: next });
|
||||
}
|
||||
|
||||
function clearStore() {
|
||||
store.length = 0;
|
||||
emit({ type: 'clear' });
|
||||
if (selection) setSelection(null);
|
||||
if (poll) setPoll(null);
|
||||
}
|
||||
|
||||
function startMock() {
|
||||
if (mockTimer || !mockEnabled) return;
|
||||
console.log('[Backend] MOCK_CHAT=1, generating mock messages');
|
||||
const authors = ['Ada', 'Linus', 'Grace', 'Marge'];
|
||||
let counter = 0;
|
||||
mockTimer = setInterval(() => {
|
||||
pushMessage({
|
||||
id: `mock-${Date.now()}`,
|
||||
author: authors[counter % authors.length],
|
||||
text: `Mock message #${counter}`,
|
||||
publishedAt: new Date().toISOString()
|
||||
});
|
||||
counter += 1;
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
function stopMock() {
|
||||
if (!mockTimer) return;
|
||||
clearInterval(mockTimer);
|
||||
mockTimer = null;
|
||||
}
|
||||
|
||||
// Stops the live chat and any pending retry; the old connection's listeners are dropped.
|
||||
function detach() {
|
||||
generation += 1;
|
||||
if (retryTimer) {
|
||||
clearTimeout(retryTimer);
|
||||
retryTimer = null;
|
||||
}
|
||||
if (ingestion) {
|
||||
ingestion.emitter.removeAllListeners();
|
||||
try {
|
||||
ingestion.liveChat?.stop?.();
|
||||
} catch (error) {
|
||||
console.error('[Backend] Error stopping live chat:', error);
|
||||
}
|
||||
ingestion = null;
|
||||
}
|
||||
}
|
||||
|
||||
function idle(patch: Partial<ConnectionStatus>) {
|
||||
setStatus({ state: 'disconnected', ...patch });
|
||||
startMock();
|
||||
}
|
||||
|
||||
function scheduleRetry(liveId: string, attempt: number, reason: string) {
|
||||
detach();
|
||||
if (attempt > RETRY_LIMIT) {
|
||||
console.error(`[Backend] Giving up on ${liveId} after ${RETRY_LIMIT} attempts: ${reason}`);
|
||||
idle({ error: `Gave up after ${RETRY_LIMIT} reconnect attempts: ${reason}` });
|
||||
return;
|
||||
}
|
||||
const delay = Math.min(RETRY_BASE_MS * 2 ** (attempt - 1), RETRY_CAP_MS);
|
||||
console.warn(`[Backend] Connection lost (${reason}); retry ${attempt}/${RETRY_LIMIT} in ${delay}ms`);
|
||||
setStatus({ state: 'reconnecting', error: reason });
|
||||
retryTimer = setTimeout(() => {
|
||||
retryTimer = null;
|
||||
void connect(liveId, attempt);
|
||||
}, delay);
|
||||
}
|
||||
|
||||
async function connect(liveId: string, attempt = 0): Promise<boolean> {
|
||||
stopMock();
|
||||
const gen = ++generation;
|
||||
if (attempt === 0) setStatus({ state: 'connecting', liveId, title: null, error: null });
|
||||
|
||||
let ctx: IngestionContext;
|
||||
try {
|
||||
ctx = await bootstrapInnertube(liveId);
|
||||
} catch (error) {
|
||||
if (gen !== generation) return false;
|
||||
const reason = errorText(error);
|
||||
console.error(`[Backend] Failed to connect to ${liveId}: ${reason}`);
|
||||
if (attempt === 0) idle({ error: reason });
|
||||
else scheduleRetry(liveId, attempt + 1, reason);
|
||||
return false;
|
||||
}
|
||||
if (gen !== generation) {
|
||||
ctx.liveChat.stop();
|
||||
return false;
|
||||
}
|
||||
|
||||
ingestion = ctx;
|
||||
ctx.emitter.on('message', pushMessage);
|
||||
ctx.emitter.on('poll', setPoll);
|
||||
// youtubei.js retries a failed poll itself, 10 times 2s apart, and emits `end` when it gives up; only then rebuild the session.
|
||||
ctx.emitter.on('end', () => scheduleRetry(liveId, 1, 'live chat ended'));
|
||||
ctx.emitter.on('error', (error) => console.warn(`[Backend] Live chat poll error, library will retry: ${errorText(error)}`));
|
||||
setStatus({ state: 'live', liveId, title: ctx.title, error: null });
|
||||
console.log(`[Backend] Connected to ${liveId}${ctx.title ? ` (${ctx.title})` : ''}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
fastify.get('/health', async () => ({
|
||||
status: 'ok',
|
||||
connection: status,
|
||||
messages: store.length,
|
||||
selection: currentSelection?.id ?? null,
|
||||
mode: ingestion ? 'live' : 'mock',
|
||||
connected: !!ingestion,
|
||||
liveId: ingestion?.videoId ?? null
|
||||
selection: selection?.id ?? null,
|
||||
mock: mockTimer !== null
|
||||
}));
|
||||
|
||||
fastify.post<{ Body: { liveId: string } }>('/chat/connect', async (request, reply) => {
|
||||
const { liveId } = request.body ?? {};
|
||||
fastify.post<{ Body: { liveId?: string } }>('/chat/connect', async (request, reply) => {
|
||||
const liveId = extractLiveId(request.body?.liveId);
|
||||
if (!liveId) {
|
||||
reply.status(400);
|
||||
return { error: 'liveId is required' };
|
||||
}
|
||||
|
||||
const parsedLiveId = extractLiveId(liveId);
|
||||
if (!parsedLiveId) {
|
||||
reply.status(400);
|
||||
return { error: 'Invalid YouTube Live ID or URL' };
|
||||
}
|
||||
|
||||
// Stop mock messages if running
|
||||
if (mockInterval) {
|
||||
clearInterval(mockInterval);
|
||||
mockInterval = null;
|
||||
console.log('[Backend] Stopped mock messages');
|
||||
}
|
||||
|
||||
// Stop existing ingestion if any
|
||||
if (ingestion) {
|
||||
try {
|
||||
ingestion.liveChat?.stop?.();
|
||||
} catch (e) {
|
||||
console.error('[Backend] Error stopping previous connection:', e);
|
||||
}
|
||||
ingestion = null;
|
||||
}
|
||||
|
||||
// Clear messages
|
||||
store.length = 0;
|
||||
|
||||
try {
|
||||
console.log(`[Backend] Connecting to YouTube Live ID: ${parsedLiveId}`);
|
||||
ingestion = await bootstrapInnertube(parsedLiveId);
|
||||
console.log(`[Backend] ✓ YouTube chat connected successfully`);
|
||||
|
||||
ingestion.emitter.on('message', (message) => {
|
||||
store.push(message);
|
||||
// Trim regularly to keep regular messages under control
|
||||
// This ensures we don't wait until hitting MAX_MESSAGES
|
||||
trimMessages(store);
|
||||
});
|
||||
|
||||
ingestion.emitter.on('poll', (poll) => {
|
||||
currentPoll = poll;
|
||||
pollEmitter.emit('update', poll);
|
||||
});
|
||||
|
||||
ingestion.emitter.on('error', (error) => {
|
||||
console.error('[Backend] Innertube ingestion error:', error);
|
||||
});
|
||||
|
||||
return { ok: true, liveId: parsedLiveId };
|
||||
} catch (error) {
|
||||
console.error('[Backend] Failed to connect:', error);
|
||||
detach();
|
||||
clearStore();
|
||||
if (!(await connect(liveId))) {
|
||||
reply.status(500);
|
||||
return { error: 'Failed to connect to YouTube Live chat' };
|
||||
return { error: status.error ?? 'Failed to connect to YouTube Live chat' };
|
||||
}
|
||||
return { ok: true, liveId, title: status.title ?? null };
|
||||
});
|
||||
|
||||
fastify.post('/chat/disconnect', async () => {
|
||||
if (ingestion) {
|
||||
try {
|
||||
ingestion.liveChat?.stop?.();
|
||||
console.log('[Backend] Disconnected from YouTube chat');
|
||||
} catch (e) {
|
||||
console.error('[Backend] Error disconnecting:', e);
|
||||
}
|
||||
ingestion = null;
|
||||
}
|
||||
|
||||
// Start mock messages again
|
||||
if (!mockInterval) {
|
||||
mockInterval = seedMockMessages(store, overlayEmitter);
|
||||
console.log('[Backend] Started mock messages');
|
||||
}
|
||||
|
||||
store.length = 0;
|
||||
currentSelection = null;
|
||||
overlayEmitter.emit('update', null);
|
||||
detach();
|
||||
clearStore();
|
||||
idle({ liveId: null, title: null, error: null });
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
fastify.get('/chat/messages', async () => ({
|
||||
messages: store
|
||||
}));
|
||||
|
||||
fastify.get('/poll/current', async () => ({
|
||||
poll: currentPoll
|
||||
}));
|
||||
fastify.get('/chat/messages', async () => ({ messages: store }));
|
||||
|
||||
fastify.post<{ Body: { id?: string } }>('/overlay/selection', async (request, reply) => {
|
||||
const { id } = request.body ?? {};
|
||||
const id = request.body?.id;
|
||||
if (!id) {
|
||||
reply.status(400);
|
||||
return { error: 'id is required' };
|
||||
}
|
||||
|
||||
const message = store.find((item) => item.id === id);
|
||||
if (!message) {
|
||||
reply.status(404);
|
||||
return { error: 'message not found' };
|
||||
}
|
||||
|
||||
currentSelection = message;
|
||||
overlayEmitter.emit('update', currentSelection);
|
||||
|
||||
setSelection(message);
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
fastify.delete('/overlay/selection', async () => {
|
||||
currentSelection = null;
|
||||
overlayEmitter.emit('update', null);
|
||||
setSelection(null);
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
fastify.get('/poll/stream', async (request, reply) => {
|
||||
fastify.get('/events', (request, reply) => {
|
||||
reply.hijack();
|
||||
|
||||
const res = reply.raw;
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
|
||||
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
|
||||
res.writeHead(200);
|
||||
res.write(': connected\n\n');
|
||||
|
||||
const send = (poll: Poll | null) => {
|
||||
res.write(`event: poll\ndata: ${JSON.stringify({ poll })}\n\n`);
|
||||
};
|
||||
|
||||
const heartbeat = setInterval(() => {
|
||||
res.write('event: heartbeat\ndata: {}\n\n');
|
||||
}, 15000);
|
||||
|
||||
pollEmitter.on('update', send);
|
||||
|
||||
if (currentPoll) {
|
||||
send(currentPoll);
|
||||
}
|
||||
// Headers set by hooks (CORS) are not flushed on a hijacked reply, so copy them.
|
||||
res.writeHead(200, {
|
||||
...(reply.getHeaders() as OutgoingHttpHeaders),
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive'
|
||||
});
|
||||
writeEvent(res, { type: 'init', status, messages: store, selection, poll });
|
||||
|
||||
const forward = (event: ServerEvent) => writeEvent(res, event);
|
||||
bus.on('event', forward);
|
||||
const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
|
||||
request.raw.on('close', () => {
|
||||
clearInterval(heartbeat);
|
||||
pollEmitter.off('update', send);
|
||||
bus.off('event', forward);
|
||||
});
|
||||
});
|
||||
|
||||
fastify.get('/overlay/stream', async (request, reply) => {
|
||||
reply.hijack();
|
||||
registerImageProxy(fastify);
|
||||
|
||||
const res = reply.raw;
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
|
||||
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
|
||||
res.writeHead(200);
|
||||
res.write(': connected\n\n');
|
||||
|
||||
const send = (message: ChatMessage | null) => {
|
||||
res.write(`event: selection\ndata: ${JSON.stringify({ message })}\n\n`);
|
||||
};
|
||||
|
||||
const heartbeat = setInterval(() => {
|
||||
res.write('event: heartbeat\ndata: {}\n\n');
|
||||
}, 15000);
|
||||
|
||||
overlayEmitter.on('update', send);
|
||||
|
||||
if (currentSelection) {
|
||||
send(currentSelection);
|
||||
}
|
||||
|
||||
request.raw.on('close', () => {
|
||||
clearInterval(heartbeat);
|
||||
overlayEmitter.off('update', send);
|
||||
});
|
||||
});
|
||||
|
||||
// Image proxy endpoint to avoid YouTube CDN rate limits
|
||||
fastify.get<{ Querystring: { url: string } }>('/proxy/image', async (request, reply) => {
|
||||
const { url } = request.query;
|
||||
|
||||
if (!url || typeof url !== 'string') {
|
||||
reply.status(400);
|
||||
return { error: 'url parameter is required' };
|
||||
}
|
||||
|
||||
// Only allow YouTube CDN and Google User Content domains
|
||||
const allowedDomains = [
|
||||
'yt3.ggpht.com',
|
||||
'yt4.ggpht.com',
|
||||
'i.ytimg.com',
|
||||
'lh3.googleusercontent.com' // For super stickers
|
||||
];
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
if (!allowedDomains.includes(urlObj.hostname)) {
|
||||
reply.status(403);
|
||||
return { error: 'Only YouTube CDN and Google User Content URLs are allowed' };
|
||||
}
|
||||
} catch (error) {
|
||||
reply.status(400);
|
||||
return { error: 'Invalid URL' };
|
||||
}
|
||||
|
||||
// Create cache key from URL
|
||||
const cacheKey = crypto.createHash('md5').update(url).digest('hex');
|
||||
|
||||
// Check cache
|
||||
const cached = imageCache.get(cacheKey);
|
||||
if (cached && (Date.now() - cached.timestamp) < CACHE_TTL) {
|
||||
reply.header('Content-Type', cached.contentType);
|
||||
reply.header('Cache-Control', 'public, max-age=86400'); // 24 hours
|
||||
reply.header('Access-Control-Allow-Origin', '*');
|
||||
return reply.send(cached.buffer);
|
||||
}
|
||||
|
||||
// Fetch from YouTube
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'Referer': 'https://www.youtube.com/',
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 429) {
|
||||
console.warn('[Backend] Rate limited by YouTube CDN for:', url);
|
||||
// Return from cache even if expired, or return error
|
||||
if (cached) {
|
||||
reply.header('Content-Type', cached.contentType);
|
||||
reply.header('Cache-Control', 'public, max-age=86400');
|
||||
reply.header('Access-Control-Allow-Origin', '*');
|
||||
return reply.send(cached.buffer);
|
||||
}
|
||||
}
|
||||
throw new Error(`Failed to fetch image: ${response.status}`);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const contentType = response.headers.get('content-type') || 'image/jpeg';
|
||||
|
||||
// Cache the image
|
||||
imageCache.set(cacheKey, {
|
||||
buffer,
|
||||
contentType,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
// Cleanup old cache entries if we exceed max size
|
||||
if (imageCache.size > MAX_CACHE_SIZE) {
|
||||
const entries = Array.from(imageCache.entries());
|
||||
entries.sort((a, b) => a[1].timestamp - b[1].timestamp);
|
||||
const toDelete = entries.slice(0, Math.floor(MAX_CACHE_SIZE * 0.2)); // Remove oldest 20%
|
||||
toDelete.forEach(([key]) => imageCache.delete(key));
|
||||
}
|
||||
|
||||
reply.header('Content-Type', contentType);
|
||||
reply.header('Cache-Control', 'public, max-age=86400');
|
||||
reply.header('Access-Control-Allow-Origin', '*');
|
||||
return reply.send(buffer);
|
||||
} catch (error) {
|
||||
console.error('[Backend] Failed to proxy image:', error);
|
||||
|
||||
// Try to return stale cache if available
|
||||
if (cached) {
|
||||
reply.header('Content-Type', cached.contentType);
|
||||
reply.header('Cache-Control', 'public, max-age=86400');
|
||||
reply.header('Access-Control-Allow-Origin', '*');
|
||||
return reply.send(cached.buffer);
|
||||
}
|
||||
|
||||
reply.status(500);
|
||||
return { error: 'Failed to fetch image' };
|
||||
}
|
||||
});
|
||||
// Production serves the exported Next.js client; in dev the client runs on its own port.
|
||||
const clientDir = path.resolve(process.cwd(), 'client/out');
|
||||
if (existsSync(clientDir)) {
|
||||
await fastify.register(fastifyStatic, { root: clientDir, index: ['index.html'], redirect: true });
|
||||
fastify.get('/', (_request, reply) => reply.redirect('/dashboard/'));
|
||||
console.log(`[Backend] Serving client from ${clientDir}`);
|
||||
} else {
|
||||
console.log('[Backend] client/out not found, serving API only');
|
||||
}
|
||||
|
||||
const host = process.env.HOST ?? '127.0.0.1';
|
||||
const port = Number(process.env.PORT ?? 4100);
|
||||
await fastify.listen({ port, host });
|
||||
console.log(`[Backend] Listening on http://${host}:${port}`);
|
||||
|
||||
await fastify.listen({ port, host: '0.0.0.0' });
|
||||
|
||||
console.log(`[Backend] Server listening on http://localhost:${port}`);
|
||||
}
|
||||
|
||||
function extractLiveId(input: string): string {
|
||||
if (!input) return '';
|
||||
|
||||
const trimmed = input.trim();
|
||||
if (/^[a-zA-Z0-9_-]{10,}$/.test(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
try {
|
||||
const url = new URL(trimmed);
|
||||
if (url.searchParams.has('v')) {
|
||||
return url.searchParams.get('v') ?? '';
|
||||
}
|
||||
const pathname = url.pathname.split('/').filter(Boolean).pop();
|
||||
return pathname ?? '';
|
||||
} catch (error) {
|
||||
console.warn('Invalid YOUTUBE_LIVE_ID provided', error);
|
||||
return '';
|
||||
const envLiveId = extractLiveId(process.env.YOUTUBE_LIVE_ID);
|
||||
if (envLiveId) {
|
||||
// Attempt 1 rather than 0 so a failed startup connect goes through the retry schedule instead of giving up.
|
||||
void connect(envLiveId, 1);
|
||||
} else {
|
||||
if (process.env.YOUTUBE_LIVE_ID) console.warn('[Backend] Ignoring invalid YOUTUBE_LIVE_ID');
|
||||
startMock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Intelligently trim messages while preserving superchats and memberships
|
||||
* Regular messages are limited to MAX_REGULAR_MESSAGES
|
||||
* Superchats and memberships are preserved for the entire session
|
||||
*/
|
||||
function trimMessages(store: ChatMessage[]): void {
|
||||
// Count messages by type
|
||||
let regularCount = 0;
|
||||
const specialIndices: number[] = [];
|
||||
|
||||
for (let i = 0; i < store.length; i++) {
|
||||
const message = store[i];
|
||||
const isSpecial = message.superChat || message.membershipGift ||
|
||||
message.membershipGiftPurchase || message.isMember;
|
||||
if (isSpecial) {
|
||||
specialIndices.push(i);
|
||||
} else {
|
||||
regularCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Only trim if we have too many regular messages
|
||||
if (regularCount > MAX_REGULAR_MESSAGES) {
|
||||
const toRemove = regularCount - MAX_REGULAR_MESSAGES;
|
||||
const specialSet = new Set(specialIndices);
|
||||
|
||||
// Remove oldest regular messages (keep special messages)
|
||||
let removed = 0;
|
||||
const newStore: ChatMessage[] = [];
|
||||
|
||||
for (let i = 0; i < store.length; i++) {
|
||||
const isSpecial = specialSet.has(i);
|
||||
|
||||
if (isSpecial) {
|
||||
// Always keep special messages
|
||||
newStore.push(store[i]);
|
||||
} else {
|
||||
// Keep regular messages if we haven't removed enough yet
|
||||
if (removed < toRemove) {
|
||||
removed++;
|
||||
// Skip this message (delete it)
|
||||
} else {
|
||||
newStore.push(store[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Replace store contents
|
||||
store.length = 0;
|
||||
store.push(...newStore);
|
||||
}
|
||||
}
|
||||
|
||||
function seedMockMessages(
|
||||
store: ChatMessage[],
|
||||
overlayEmitter: EventEmitter<{ update: (message: ChatMessage | null) => void }>
|
||||
): NodeJS.Timeout {
|
||||
let counter = 0;
|
||||
const authors = ['Ada', 'Linus', 'Grace', 'Marge'];
|
||||
return setInterval(() => {
|
||||
const message: ChatMessage = {
|
||||
id: `mock-${Date.now()}`,
|
||||
author: authors[counter % authors.length],
|
||||
text: `Mock message #${counter}`,
|
||||
publishedAt: new Date().toISOString()
|
||||
};
|
||||
store.push(message);
|
||||
// Trim regularly to keep regular messages under control
|
||||
trimMessages(store);
|
||||
if (counter % 5 === 0) {
|
||||
overlayEmitter.emit('update', message);
|
||||
}
|
||||
counter += 1;
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
// Only run if this is the main module
|
||||
if (require.main === module) {
|
||||
startBackend().catch((error) => {
|
||||
console.error('Failed to start backend', error);
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolveRunUrl } from './youtubei';
|
||||
|
||||
test('empty input', () => {
|
||||
assert.equal(resolveRunUrl(undefined), undefined);
|
||||
assert.equal(resolveRunUrl(''), undefined);
|
||||
});
|
||||
|
||||
test('plain https URL is returned unchanged', () => {
|
||||
assert.equal(resolveRunUrl('https://example.com/path?a=1&b=2'), 'https://example.com/path?a=1&b=2');
|
||||
assert.equal(resolveRunUrl('http://example.com'), 'http://example.com');
|
||||
});
|
||||
|
||||
test('site-relative path becomes an absolute youtube.com URL', () => {
|
||||
assert.equal(resolveRunUrl('/watch?v=dQw4w9WgXcQ'), 'https://www.youtube.com/watch?v=dQw4w9WgXcQ');
|
||||
assert.equal(resolveRunUrl('/@channel'), 'https://www.youtube.com/@channel');
|
||||
});
|
||||
|
||||
test('redirect wrapper resolves to the encoded q parameter', () => {
|
||||
const wrapped =
|
||||
'https://www.youtube.com/redirect?event=live_chat&redir_token=TOKEN&q=https%3A%2F%2Fexample.com%2Fp%3Fa%3D1%26b%3D2&html_redirect=1';
|
||||
assert.equal(resolveRunUrl(wrapped), 'https://example.com/p?a=1&b=2');
|
||||
assert.equal(resolveRunUrl('/redirect?q=https%3A%2F%2Fexample.com'), 'https://example.com');
|
||||
assert.equal(resolveRunUrl('https://www.youtube.com/redirect?event=x'), 'https://www.youtube.com/redirect?event=x');
|
||||
});
|
||||
|
||||
test('unparseable input', () => {
|
||||
assert.equal(resolveRunUrl('not a url'), undefined);
|
||||
});
|
||||
+164
-274
@@ -1,75 +1,121 @@
|
||||
import type { ChatMessage, Badge, SuperChatInfo, Poll } from '@shared/chat';
|
||||
import type { ChatMessage, Badge, MessageRun, SuperChatInfo, Poll } from '@shared/chat';
|
||||
import EventEmitter from 'eventemitter3';
|
||||
import Innertube, { UniversalCache } from 'youtubei.js';
|
||||
import Innertube from 'youtubei.js';
|
||||
|
||||
export type IngestionContext = {
|
||||
client: Innertube;
|
||||
liveChat: any;
|
||||
videoId: string;
|
||||
title: string | null;
|
||||
emitter: ChatEventEmitter;
|
||||
};
|
||||
|
||||
export type ContinuationState = {
|
||||
messages: ChatMessage[];
|
||||
nextToken: string | null;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
export type ChatEventEmitter = EventEmitter<{
|
||||
message: (message: ChatMessage) => void;
|
||||
poll: (poll: Poll | null) => void;
|
||||
error: (error: unknown) => void;
|
||||
end: () => void;
|
||||
}>;
|
||||
|
||||
const defaultTimeout = 1500;
|
||||
const SEEN_IDS_CAP = 5000;
|
||||
|
||||
// Track message IDs to ensure uniqueness
|
||||
// YouTube reuses message ids in busy chats; a per-id counter keeps ours unique.
|
||||
const seenIds = new Map<string, number>();
|
||||
|
||||
function generateUniqueId(baseId: string): string {
|
||||
if (seenIds.size > SEEN_IDS_CAP) seenIds.clear();
|
||||
const count = seenIds.get(baseId) ?? 0;
|
||||
seenIds.set(baseId, count + 1);
|
||||
|
||||
// If this is the first occurrence of this ID, return it as-is
|
||||
if (count === 0) {
|
||||
return baseId;
|
||||
}
|
||||
|
||||
// Otherwise, append a counter suffix to ensure uniqueness
|
||||
return `${baseId}#${count}`;
|
||||
return count === 0 ? baseId : `${baseId}#${count}`;
|
||||
}
|
||||
|
||||
function resolveMessageRuns(item: any) {
|
||||
const runs: { text?: string; emojiUrl?: string; emojiAlt?: string }[] = [];
|
||||
/**
|
||||
* Turns the raw navigation URL of a text run into the absolute destination.
|
||||
* YouTube wraps external links in /redirect?q=<url> and uses site-relative paths for its own.
|
||||
*/
|
||||
export function resolveRunUrl(raw: string | undefined): string | undefined {
|
||||
if (!raw) return undefined;
|
||||
const absolute = raw.startsWith('/') ? `https://www.youtube.com${raw}` : raw;
|
||||
try {
|
||||
const url = new URL(absolute);
|
||||
if (url.hostname.endsWith('youtube.com') && url.pathname === '/redirect') {
|
||||
return url.searchParams.get('q') ?? absolute;
|
||||
}
|
||||
return absolute;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Reads the run endpoint youtubei.js exposes on TextRun. Verified against the library types, not yet against a live chat carrying a long link.
|
||||
function runUrl(run: any): string | undefined {
|
||||
return resolveRunUrl(
|
||||
run?.endpoint?.payload?.url ??
|
||||
run?.endpoint?.metadata?.url ??
|
||||
run?.navigationEndpoint?.urlEndpoint?.url ??
|
||||
run?.navigationEndpoint?.commandMetadata?.webCommandMetadata?.url
|
||||
);
|
||||
}
|
||||
|
||||
// The visible text of a link run may be shortened; the endpoint keeps the full URL.
|
||||
function isTruncated(text: string, url: string): boolean {
|
||||
return text.endsWith('...') || text.endsWith('…') || text.length < url.length;
|
||||
}
|
||||
|
||||
function resolveMessageRuns(item: any): MessageRun[] {
|
||||
const runs: MessageRun[] = [];
|
||||
const rawRuns = item?.message?.runs;
|
||||
if (Array.isArray(rawRuns)) {
|
||||
for (const run of rawRuns) {
|
||||
if (run?.emoji) {
|
||||
const url = run.emoji?.image?.[0]?.url;
|
||||
runs.push({
|
||||
emojiUrl: url,
|
||||
emojiAlt: run.emoji?.shortcuts?.[0] || run.emoji?.emoji_id || ''
|
||||
});
|
||||
} else if (run?.text) {
|
||||
runs.push({ text: String(run.text) });
|
||||
}
|
||||
if (!Array.isArray(rawRuns)) return runs;
|
||||
for (const run of rawRuns) {
|
||||
if (run?.emoji) {
|
||||
runs.push({
|
||||
emojiUrl: run.emoji?.image?.[0]?.url,
|
||||
emojiAlt: run.emoji?.shortcuts?.[0] || run.emoji?.emoji_id || ''
|
||||
});
|
||||
} else if (run?.text) {
|
||||
const url = runUrl(run);
|
||||
runs.push(url ? { text: String(run.text), url } : { text: String(run.text) });
|
||||
}
|
||||
}
|
||||
return runs;
|
||||
}
|
||||
|
||||
function resolveMessageText(item: any): string {
|
||||
const message = item?.message;
|
||||
if (!message) return '';
|
||||
if (typeof message === 'string') return message;
|
||||
|
||||
if (Array.isArray(message.runs)) {
|
||||
return message.runs
|
||||
.map((run: any) => {
|
||||
if (run?.emoji) {
|
||||
return (run.emoji.is_custom && run.emoji.shortcuts?.[0]) || run.emoji.emoji_id || run.text || '';
|
||||
}
|
||||
const text = String(run?.text ?? '');
|
||||
const url = runUrl(run);
|
||||
return url && isTruncated(text, url) ? url : text;
|
||||
})
|
||||
.join('');
|
||||
}
|
||||
|
||||
return typeof message.toString === 'function' ? message.toString() : '';
|
||||
}
|
||||
|
||||
export async function bootstrapInnertube(videoId: string): Promise<IngestionContext> {
|
||||
if (!videoId) {
|
||||
throw new Error('YOUTUBE_LIVE_ID is required to bootstrap Innertube');
|
||||
}
|
||||
seenIds.clear();
|
||||
|
||||
console.log('[Ingestion] Creating Innertube client...');
|
||||
const client = await Innertube.create({
|
||||
cache: new UniversalCache(false)
|
||||
});
|
||||
// No cache: even UniversalCache(false) writes session data to os.tmpdir(), and a session captured during a
|
||||
// network hiccup came back poisoned on every reconnect (YouTube answered 400 "unusual traffic" until it was deleted).
|
||||
// The player is skipped because chat never needs stream formats.
|
||||
const client = await Innertube.create({ retrieve_player: false });
|
||||
|
||||
console.log('[Ingestion] Fetching video info...');
|
||||
const info = await client.getInfo(videoId);
|
||||
const title = info.basic_info?.title ?? null;
|
||||
|
||||
console.log('[Ingestion] Getting live chat...');
|
||||
const liveChat = info.getLiveChat();
|
||||
@@ -81,7 +127,6 @@ export async function bootstrapInnertube(videoId: string): Promise<IngestionCont
|
||||
const emitter: ChatEventEmitter = new EventEmitter();
|
||||
|
||||
liveChat.on('chat-update', (action: any) => {
|
||||
// Handle poll updates (just detect active/closed state)
|
||||
if (action?.type === 'UpdateLiveChatPollAction') {
|
||||
const pollId = action?.poll_to_update?.live_chat_poll_id;
|
||||
if (pollId) {
|
||||
@@ -90,7 +135,6 @@ export async function bootstrapInnertube(videoId: string): Promise<IngestionCont
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle poll closing
|
||||
if (action?.type === 'CloseLiveChatActionPanelAction' || action?.type === 'RemoveBannerForLiveChatCommand') {
|
||||
emitter.emit('poll', null);
|
||||
return;
|
||||
@@ -104,85 +148,19 @@ export async function bootstrapInnertube(videoId: string): Promise<IngestionCont
|
||||
|
||||
liveChat.on('error', (err: unknown) => {
|
||||
const msg = (err as any)?.message || String(err);
|
||||
// Ignore known non-fatal parser drift issues that YouTube.js auto-generates
|
||||
if (msg && (
|
||||
msg.includes('LiveChatReportModerationStateCommand not found') ||
|
||||
msg.includes('CloseLiveChatActionPanelAction not found')
|
||||
)) {
|
||||
return; // ignore noisy parser drift that YouTube.js JITs around
|
||||
}
|
||||
// Unknown renderer types are parser drift that youtubei.js generates stubs for; not a connection problem.
|
||||
if (/\w+(Command|Action) not found/.test(msg)) return;
|
||||
console.error('[Ingestion] Live chat error:', err);
|
||||
emitter.emit('error', err);
|
||||
});
|
||||
|
||||
liveChat.on('end', () => emitter.emit('end'));
|
||||
|
||||
console.log('[Ingestion] Starting live chat listener...');
|
||||
liveChat.start();
|
||||
console.log('[Ingestion] Live chat listener started');
|
||||
|
||||
return {
|
||||
client,
|
||||
liveChat,
|
||||
videoId,
|
||||
emitter
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchChatBatch(
|
||||
ctx: IngestionContext,
|
||||
options?: { windowMs?: number }
|
||||
): Promise<ContinuationState> {
|
||||
const windowMs = options?.windowMs ?? defaultTimeout;
|
||||
const collected: ChatMessage[] = [];
|
||||
|
||||
const listener = (message: ChatMessage) => {
|
||||
collected.push(message);
|
||||
};
|
||||
|
||||
ctx.emitter.on('message', listener);
|
||||
|
||||
await delay(windowMs);
|
||||
|
||||
ctx.emitter.off('message', listener);
|
||||
|
||||
const timeoutMs = ctx.liveChat?.continuation?.timeout_ms;
|
||||
const validTimeout = typeof timeoutMs === 'number' && !isNaN(timeoutMs) && timeoutMs > 0
|
||||
? timeoutMs
|
||||
: defaultTimeout;
|
||||
|
||||
return {
|
||||
messages: collected,
|
||||
nextToken: ctx.liveChat?.continuation?.token ?? null,
|
||||
timeoutMs: validTimeout
|
||||
};
|
||||
}
|
||||
|
||||
function resolveMessageText(item: any): string {
|
||||
if (!item?.message) return '';
|
||||
|
||||
if (typeof item.message === 'string') {
|
||||
return item.message;
|
||||
}
|
||||
|
||||
if (typeof item.message?.toString === 'function') {
|
||||
return item.message.toString();
|
||||
}
|
||||
|
||||
if (Array.isArray(item.message?.runs)) {
|
||||
return item.message.runs.map((run: any) => {
|
||||
// Handle emoji objects - for custom emojis, use the shortcut text
|
||||
if (run.emoji) {
|
||||
// For custom emojis, use the first shortcut (e.g., ":_heçkır:")
|
||||
if (run.emoji.is_custom && run.emoji.shortcuts?.[0]) {
|
||||
return run.emoji.shortcuts[0];
|
||||
}
|
||||
// For standard emojis, use emoji_id (e.g., "❤")
|
||||
return run.emoji.emoji_id || run.text || '';
|
||||
}
|
||||
return run.text ?? '';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
return '';
|
||||
return { client, liveChat, videoId, title, emitter };
|
||||
}
|
||||
|
||||
function resolveTimestamp(timestamp: number | string | undefined): string {
|
||||
@@ -193,68 +171,32 @@ function resolveTimestamp(timestamp: number | string | undefined): string {
|
||||
const numeric = typeof timestamp === 'string' ? Number(timestamp) : timestamp;
|
||||
|
||||
if (Number.isFinite(numeric)) {
|
||||
// YouTube timestamps are in microseconds (16 digits) or milliseconds (13 digits)
|
||||
// If it's microseconds (>= 1e15), divide by 1000 to get milliseconds
|
||||
// If it's milliseconds (>= 1e12), use as is
|
||||
let millis: number;
|
||||
if (numeric >= 1e15) {
|
||||
// Microseconds - convert to milliseconds
|
||||
millis = numeric / 1000;
|
||||
} else if (numeric >= 1e12) {
|
||||
// Already milliseconds
|
||||
millis = numeric;
|
||||
} else {
|
||||
// Fallback for smaller numbers
|
||||
millis = numeric;
|
||||
}
|
||||
|
||||
// YouTube sends microseconds (16 digits) or milliseconds (13 digits).
|
||||
const millis = numeric >= 1e15 ? numeric / 1000 : numeric;
|
||||
return new Date(millis).toISOString();
|
||||
}
|
||||
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function extractBadges(item: any): Badge[] {
|
||||
function extractBadges(list: any): Badge[] {
|
||||
const badges: Badge[] = [];
|
||||
if (!Array.isArray(list)) return badges;
|
||||
|
||||
if (!item.author?.badges) return badges;
|
||||
|
||||
for (const badge of item.author.badges) {
|
||||
const label = badge.tooltip ?? badge.label ?? '';
|
||||
for (const badge of list) {
|
||||
const label: string = badge.tooltip ?? badge.label ?? '';
|
||||
const iconType: string = badge.icon_type ?? '';
|
||||
const imageUrl = badge.custom_thumbnail?.[0]?.url;
|
||||
const lower = label.toLowerCase();
|
||||
|
||||
if (label.toLowerCase().includes('moderator')) {
|
||||
if (iconType === 'MODERATOR' || lower.includes('moderator')) {
|
||||
badges.push({ type: 'moderator', label, imageUrl });
|
||||
} else if (label.toLowerCase().includes('member')) {
|
||||
} else if (lower.includes('member')) {
|
||||
badges.push({ type: 'member', label, imageUrl });
|
||||
} else if (label.toLowerCase().includes('verified')) {
|
||||
badges.push({ type: 'verified', label, imageUrl });
|
||||
} else if (label) {
|
||||
badges.push({ type: 'custom', label, imageUrl });
|
||||
}
|
||||
}
|
||||
|
||||
return badges;
|
||||
}
|
||||
|
||||
function extractBadgesFromHeader(header: any): Badge[] {
|
||||
const badges: Badge[] = [];
|
||||
|
||||
if (!header?.author_badges) return badges;
|
||||
|
||||
for (const badge of header.author_badges) {
|
||||
const label = badge.tooltip ?? '';
|
||||
const iconType = badge.icon_type ?? '';
|
||||
const imageUrl = badge.custom_thumbnail?.[0]?.url;
|
||||
|
||||
if (label.toLowerCase().includes('moderator') || iconType === 'MODERATOR') {
|
||||
badges.push({ type: 'moderator', label, imageUrl });
|
||||
} else if (label.toLowerCase().includes('member')) {
|
||||
badges.push({ type: 'member', label, imageUrl });
|
||||
} else if (label.toLowerCase().includes('verified') || iconType === 'VERIFIED') {
|
||||
} else if (iconType === 'VERIFIED' || lower.includes('verified')) {
|
||||
badges.push({ type: 'verified', label, imageUrl });
|
||||
} else if (iconType === 'OWNER') {
|
||||
badges.push({ type: 'custom', label: label || 'Owner', imageUrl });
|
||||
badges.push({ type: 'owner', label: label || 'Owner', imageUrl });
|
||||
} else if (label) {
|
||||
badges.push({ type: 'custom', label, imageUrl });
|
||||
}
|
||||
@@ -264,13 +206,11 @@ function extractBadgesFromHeader(header: any): Badge[] {
|
||||
}
|
||||
|
||||
function extractLeaderboardRank(item: any): number | undefined {
|
||||
// Check for before_content_buttons array (where leaderboard badge appears)
|
||||
if (!Array.isArray(item.before_content_buttons)) return undefined;
|
||||
|
||||
for (const button of item.before_content_buttons) {
|
||||
// Look for CROWN icon (leaderboard indicator)
|
||||
// The CROWN button carries the rank as "#3".
|
||||
if (button.icon_name === 'CROWN' && button.title) {
|
||||
// Title is like "#3", "#1", etc.
|
||||
const match = String(button.title).match(/#(\d+)/);
|
||||
if (match && match[1]) {
|
||||
return parseInt(match[1], 10);
|
||||
@@ -294,8 +234,8 @@ function extractSuperChatInfo(item: any): SuperChatInfo | undefined {
|
||||
|
||||
let amountText = '';
|
||||
const candidates = [
|
||||
item.purchase_amount, // Correct property name from youtubei.js docs
|
||||
item.purchase_amount_text, // Fallback
|
||||
item.purchase_amount,
|
||||
item.purchase_amount_text,
|
||||
item.purchaseAmountText,
|
||||
item.header?.purchase_amount,
|
||||
item.header?.purchase_amount_text,
|
||||
@@ -325,25 +265,21 @@ function extractSuperChatInfo(item: any): SuperChatInfo | undefined {
|
||||
let currency = '';
|
||||
|
||||
if (amountText) {
|
||||
// Regex to capture currency symbol/code and amount
|
||||
// Handles: $5.00, €5,00, 5,00 €, 5.00 USD, TRY5.00, TRY 55, etc.
|
||||
// Handles $5.00, 5,00 EUR, TRY 55 and similar: symbol or code on either side of the number.
|
||||
const match = amountText.match(/([\$\€\£\¥\₹\₺]|[A-Z]{2,3})?\s*([\d,\.]+)\s*([\$\€\£\¥\₹\₺]|[A-Z]{2,3})?/);
|
||||
if (match) {
|
||||
// Prefer currency symbol/code before the number, fallback to after
|
||||
currency = match[1] || match[3] || '';
|
||||
amount = match[2];
|
||||
} else {
|
||||
// Fallback: use the whole text if no pattern matches
|
||||
amount = amountText;
|
||||
}
|
||||
}
|
||||
|
||||
// If we still don't have an amount, use a default
|
||||
if (!amount) {
|
||||
amount = 'Super Chat';
|
||||
}
|
||||
|
||||
let rawColor = item.body_background_color ?? item.bodyBackgroundColor ?? item.headerBackgroundColor;
|
||||
const rawColor = item.body_background_color ?? item.bodyBackgroundColor ?? item.headerBackgroundColor;
|
||||
let color = '#1e3a8a';
|
||||
if (rawColor != null) {
|
||||
if (typeof rawColor === 'number') {
|
||||
@@ -355,16 +291,13 @@ function extractSuperChatInfo(item: any): SuperChatInfo | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
// Extract super sticker image URL if present
|
||||
let stickerUrl: string | undefined;
|
||||
let stickerAlt: string | undefined;
|
||||
|
||||
if (Array.isArray(item.sticker) && item.sticker.length > 0) {
|
||||
// Prefer larger image (first in array is usually largest)
|
||||
const stickerThumb = item.sticker[0];
|
||||
if (stickerThumb?.url) {
|
||||
// URLs from YouTube might be protocol-relative (//domain.com)
|
||||
// Convert to absolute HTTPS URL
|
||||
// Sticker URLs may be protocol-relative.
|
||||
let url = String(stickerThumb.url);
|
||||
if (url.startsWith('//')) {
|
||||
url = 'https:' + url;
|
||||
@@ -374,7 +307,6 @@ function extractSuperChatInfo(item: any): SuperChatInfo | undefined {
|
||||
stickerUrl = url;
|
||||
}
|
||||
|
||||
// Extract accessibility label for alt text
|
||||
if (item.sticker_accessibility_label) {
|
||||
stickerAlt = String(item.sticker_accessibility_label);
|
||||
}
|
||||
@@ -397,9 +329,9 @@ function normalizeAction(action: any): ChatMessage | null {
|
||||
const item = action.item;
|
||||
if (!item) return null;
|
||||
|
||||
// Normalize various live chat events
|
||||
const itemType = String(item.type || '').trim();
|
||||
const messageText = resolveMessageText(item).toLowerCase();
|
||||
const resolvedText = resolveMessageText(item);
|
||||
const messageText = resolvedText.toLowerCase();
|
||||
|
||||
const isText = itemType === 'LiveChatTextMessage';
|
||||
const isPaid = !!extractSuperChatInfo(item);
|
||||
@@ -407,118 +339,76 @@ function normalizeAction(action: any): ChatMessage | null {
|
||||
const isGiftPurchase = itemType === 'LiveChatSponsorshipsGiftPurchaseAnnouncement';
|
||||
const isGiftReceived = itemType === 'LiveChatSponsorshipsGiftRedemptionAnnouncement';
|
||||
|
||||
// Check if the message text indicates it's a gift recipient message
|
||||
const isGiftRecipientMessage =
|
||||
messageText.includes('received a gift membership') ||
|
||||
messageText.includes('received a membership gift') ||
|
||||
messageText.includes('received a gift') ||
|
||||
/received\s+a\s+.*membership.*by/i.test(messageText);
|
||||
|
||||
// Ignore gift received messages - we only care about the purchaser
|
||||
// Only the purchaser of a gift is shown, not each recipient.
|
||||
if (isGiftReceived || isGiftRecipientMessage) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (isText || isPaid || isMembership || isGiftPurchase) {
|
||||
// For gift purchases, author info is in header
|
||||
const authorSource = isGiftPurchase ? item.header : item;
|
||||
const badges = isGiftPurchase ? extractBadgesFromHeader(item.header) : extractBadges(item);
|
||||
const isModerator = badges.some(b => b.type === 'moderator');
|
||||
const isMember = badges.some(b => b.type === 'member');
|
||||
const isVerified = badges.some(b => b.type === 'verified');
|
||||
|
||||
// Extract membership level for new members, upgrades, and milestones
|
||||
let membershipLevel: string | undefined;
|
||||
if (isMembership) {
|
||||
const subtext: string = item.header_subtext?.text || '';
|
||||
const primaryText: string = item.header_primary_text?.text || '';
|
||||
|
||||
// Try to capture level name from common phrases
|
||||
// e.g., "Welcome to User!" => "User"
|
||||
// e.g., "Upgraded membership to Superuser!" => "Superuser"
|
||||
let levelFromSub: string | undefined;
|
||||
const welcomeMatch = subtext.match(/Welcome to\s+(.+?)!/i);
|
||||
const upgradeMatch = subtext.match(/Upgraded membership to\s+(.+?)!/i);
|
||||
if (welcomeMatch?.[1]) {
|
||||
levelFromSub = welcomeMatch[1].trim();
|
||||
} else if (upgradeMatch?.[1]) {
|
||||
levelFromSub = upgradeMatch[1].trim();
|
||||
}
|
||||
|
||||
membershipLevel = levelFromSub || primaryText || 'New member';
|
||||
} else if (isGiftPurchase) {
|
||||
// For gift purchases, we don't need membership level
|
||||
membershipLevel = undefined;
|
||||
} else if (isPaid) {
|
||||
membershipLevel = undefined;
|
||||
}
|
||||
|
||||
// Extract gift count for gift purchases
|
||||
let giftCount: number | undefined;
|
||||
if (isGiftPurchase) {
|
||||
const primaryText = item.header?.primary_text?.text || '';
|
||||
// Extract number from "Sent 1 Yusuf İpek gift memberships"
|
||||
const countMatch = primaryText.match(/sent\s+(\d+)\s+/i);
|
||||
if (countMatch) {
|
||||
giftCount = parseInt(countMatch[1], 10);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract channel ID - for gifts it's in author_external_channel_id
|
||||
const authorChannelId = isGiftPurchase
|
||||
? item.author_external_channel_id
|
||||
: item.author?.id;
|
||||
|
||||
// Extract author name and photo
|
||||
const authorName = isGiftPurchase
|
||||
? (item.header?.author_name?.text || 'Unknown')
|
||||
: String(item.author?.name ?? 'Unknown');
|
||||
|
||||
const authorPhoto = isGiftPurchase
|
||||
? item.header?.author_photo?.[0]?.url
|
||||
: item.author?.thumbnails?.[0]?.url;
|
||||
|
||||
// Build text with fallback: if no user message, show header subtext/primary for membership events
|
||||
const resolvedText = resolveMessageText(item);
|
||||
const membershipFallbackText = isMembership
|
||||
? (item.header_subtext?.text || item.header_primary_text?.text || '')
|
||||
: '';
|
||||
|
||||
// Use timestamp_usec (microseconds) as it's more accurate than timestamp (milliseconds)
|
||||
const timestampToUse = item.timestamp_usec ?? item.timestamp;
|
||||
|
||||
// Extract leaderboard rank if present
|
||||
const leaderboardRank = extractLeaderboardRank(item);
|
||||
|
||||
// Generate a unique ID - YouTube message IDs might not be unique in high-frequency chats
|
||||
// so we track seen IDs and append a counter if needed
|
||||
const baseId = String(item.id ?? item.timestamp_usec ?? Date.now());
|
||||
const uniqueId = generateUniqueId(baseId);
|
||||
|
||||
return {
|
||||
id: uniqueId,
|
||||
author: authorName,
|
||||
authorPhoto,
|
||||
authorChannelId: authorChannelId ? String(authorChannelId) : undefined,
|
||||
text: resolvedText || membershipFallbackText,
|
||||
runs: (() => { const r = resolveMessageRuns(item); return r.length ? r : undefined; })(),
|
||||
publishedAt: resolveTimestamp(timestampToUse),
|
||||
badges: badges.length > 0 ? badges : undefined,
|
||||
isModerator,
|
||||
isMember,
|
||||
isVerified,
|
||||
superChat: extractSuperChatInfo(item),
|
||||
membershipGift: isMembership,
|
||||
membershipGiftPurchase: isGiftPurchase,
|
||||
membershipLevel,
|
||||
giftCount,
|
||||
leaderboardRank
|
||||
};
|
||||
if (!(isText || isPaid || isMembership || isGiftPurchase)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
// Gift purchase announcements carry author data in the header.
|
||||
const badges = extractBadges(isGiftPurchase ? item.header?.author_badges : item.author?.badges);
|
||||
const isModerator = badges.some(b => b.type === 'moderator');
|
||||
const isMember = badges.some(b => b.type === 'member');
|
||||
const isVerified = badges.some(b => b.type === 'verified');
|
||||
|
||||
function delay(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
let membershipLevel: string | undefined;
|
||||
if (isMembership) {
|
||||
const subtext: string = item.header_subtext?.text || '';
|
||||
const primaryText: string = item.header_primary_text?.text || '';
|
||||
// "Welcome to Level!" or "Upgraded membership to Level!"
|
||||
const levelMatch = subtext.match(/(?:Welcome to|Upgraded membership to)\s+(.+?)!/i);
|
||||
membershipLevel = levelMatch?.[1]?.trim() || primaryText || 'New member';
|
||||
}
|
||||
|
||||
let giftCount: number | undefined;
|
||||
if (isGiftPurchase) {
|
||||
// "Sent 5 Channel gift memberships"
|
||||
const countMatch = String(item.header?.primary_text?.text || '').match(/sent\s+(\d+)\s+/i);
|
||||
if (countMatch) {
|
||||
giftCount = parseInt(countMatch[1], 10);
|
||||
}
|
||||
}
|
||||
|
||||
const authorChannelId = isGiftPurchase ? item.author_external_channel_id : item.author?.id;
|
||||
const authorName = isGiftPurchase
|
||||
? (item.header?.author_name?.text || 'Unknown')
|
||||
: String(item.author?.name ?? 'Unknown');
|
||||
const authorPhoto = isGiftPurchase
|
||||
? item.header?.author_photo?.[0]?.url
|
||||
: item.author?.thumbnails?.[0]?.url;
|
||||
|
||||
const membershipFallbackText = isMembership
|
||||
? (item.header_subtext?.text || item.header_primary_text?.text || '')
|
||||
: '';
|
||||
|
||||
const runs = resolveMessageRuns(item);
|
||||
|
||||
return {
|
||||
id: generateUniqueId(String(item.id ?? item.timestamp_usec ?? Date.now())),
|
||||
author: authorName,
|
||||
authorPhoto,
|
||||
authorChannelId: authorChannelId ? String(authorChannelId) : undefined,
|
||||
text: resolvedText || membershipFallbackText,
|
||||
runs: runs.length ? runs : undefined,
|
||||
publishedAt: resolveTimestamp(item.timestamp_usec ?? item.timestamp),
|
||||
badges: badges.length > 0 ? badges : undefined,
|
||||
isModerator,
|
||||
isMember,
|
||||
isVerified,
|
||||
superChat: extractSuperChatInfo(item),
|
||||
membershipGift: isMembership,
|
||||
membershipGiftPurchase: isGiftPurchase,
|
||||
membershipLevel,
|
||||
giftCount,
|
||||
leaderboardRank: extractLeaderboardRank(item)
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { extractLiveId } from './liveId';
|
||||
|
||||
const ID = 'dQw4w9WgXcQ';
|
||||
|
||||
test('bare id', () => {
|
||||
assert.equal(extractLiveId(ID), ID);
|
||||
assert.equal(extractLiveId(` ${ID}\n`), ID);
|
||||
});
|
||||
|
||||
test('watch URLs', () => {
|
||||
assert.equal(extractLiveId(`https://www.youtube.com/watch?v=${ID}`), ID);
|
||||
assert.equal(extractLiveId(`https://www.youtube.com/watch?v=${ID}&t=42s&feature=share`), ID);
|
||||
assert.equal(extractLiveId(`https://m.youtube.com/watch?feature=share&v=${ID}`), ID);
|
||||
assert.equal(extractLiveId(`youtube.com/watch?v=${ID}`), ID);
|
||||
});
|
||||
|
||||
test('short and live URLs', () => {
|
||||
assert.equal(extractLiveId(`https://youtu.be/${ID}`), ID);
|
||||
assert.equal(extractLiveId(`https://youtu.be/${ID}?si=abc123`), ID);
|
||||
assert.equal(extractLiveId(`https://www.youtube.com/live/${ID}`), ID);
|
||||
assert.equal(extractLiveId(`https://www.youtube.com/live/${ID}?feature=share`), ID);
|
||||
assert.equal(extractLiveId(`https://www.youtube.com/embed/${ID}`), ID);
|
||||
});
|
||||
|
||||
test('junk', () => {
|
||||
assert.equal(extractLiveId(undefined), '');
|
||||
assert.equal(extractLiveId(''), '');
|
||||
assert.equal(extractLiveId('hello world'), '');
|
||||
assert.equal(extractLiveId('dQw4w9WgXc'), '');
|
||||
assert.equal(extractLiveId('https://www.youtube.com/'), '');
|
||||
assert.equal(extractLiveId('https://www.youtube.com/watch?v=short'), '');
|
||||
assert.equal(extractLiveId(`https://example.com/${ID}`), '');
|
||||
assert.equal(extractLiveId('http://[bad'), '');
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
const VIDEO_ID = /^[A-Za-z0-9_-]{11}$/;
|
||||
const YOUTUBE_HOST = /(^|\.)(youtube\.com|youtube-nocookie\.com|youtu\.be)$/;
|
||||
|
||||
/**
|
||||
* Extracts the 11-character video id from a bare id or any YouTube URL
|
||||
* (watch?v=, youtu.be/, /live/, /embed/, with or without scheme). Returns '' for anything else.
|
||||
*/
|
||||
export function extractLiveId(input: string | undefined): string {
|
||||
const trimmed = (input ?? '').trim();
|
||||
if (!trimmed) return '';
|
||||
if (VIDEO_ID.test(trimmed)) return trimmed;
|
||||
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(/^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
if (!YOUTUBE_HOST.test(url.hostname)) return '';
|
||||
|
||||
const candidate = url.searchParams.get('v') ?? url.pathname.split('/').filter(Boolean).pop() ?? '';
|
||||
return VIDEO_ID.test(candidate) ? candidate : '';
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"declaration": true,
|
||||
"emitDeclarationOnly": false
|
||||
}
|
||||
"declaration": true
|
||||
},
|
||||
"exclude": ["src/**/*.test.ts"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user