refactor: migrate from pnpm workspaces to single-package structure with shared types

This commit is contained in:
Yusuf İpek
2025-10-05 04:42:47 +03:00
parent ddda9fb9dd
commit 99e1762068
32 changed files with 945 additions and 183 deletions
-10
View File
@@ -1,10 +0,0 @@
:root {
color-scheme: dark light;
}
body {
margin: 0;
font-family: system-ui, sans-serif;
background: #0f0f0f;
color: #f5f5f5;
}
-8
View File
@@ -1,8 +0,0 @@
export default function HomePage() {
return (
<main>
<h1>youtube-client</h1>
<p>Operator dashboard under construction.</p>
</main>
);
}
-3
View File
@@ -1,3 +0,0 @@
/// <reference types="next" />
/// <reference types="next/types/global" />
/// <reference types="next/image-types/global" />
-12
View File
@@ -1,12 +0,0 @@
{
"name": "client",
"version": "0.1.0",
"private": true,
"description": "Operator dashboard and overlay UI.",
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint"
}
}
-16
View File
@@ -1,16 +0,0 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"jsx": "preserve",
"types": ["next", "next/types/global", "next/image-types/global"],
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"noEmit": true
},
"include": [
"next-env.d.ts",
"**/*.ts",
"**/*.tsx"
],
"exclude": ["node_modules", "dist"]
}
+187
View File
@@ -0,0 +1,187 @@
import Fastify from 'fastify';
import cors from '@fastify/cors';
import EventEmitter from 'eventemitter3';
import type { ChatMessage } from '@shared/chat';
import { bootstrapInnertube, type IngestionContext } from './ingestion/youtubei';
const MAX_MESSAGES = 500;
export async function startBackend() {
const fastify = Fastify({
logger: true
});
await fastify.register(cors, { origin: true });
const store: ChatMessage[] = [];
let currentSelection: ChatMessage | null = null;
const overlayEmitter = new EventEmitter<{ update: (message: ChatMessage | null) => void }>();
const rawLiveId = process.env.YOUTUBE_LIVE_ID ?? '';
const parsedLiveId = extractLiveId(rawLiveId);
const shouldMock = !parsedLiveId;
let ingestion: IngestionContext | null = null;
if (!shouldMock) {
try {
console.log(`[Backend] Connecting to YouTube Live ID: ${parsedLiveId}`);
ingestion = await bootstrapInnertube(parsedLiveId);
console.log(`[Backend] ✓ YouTube chat connected successfully`);
ingestion.emitter.on('message', (message) => {
console.log(`[Chat] ${message.author}: ${message.text}`);
store.push(message);
if (store.length > MAX_MESSAGES) {
store.splice(0, store.length - MAX_MESSAGES);
}
});
ingestion.emitter.on('error', (error) => {
console.error('[Backend] Innertube ingestion error:', error);
});
} catch (error) {
console.error('[Backend] Failed to bootstrap Innertube; falling back to mock data:', error);
}
} else {
console.log('[Backend] No YOUTUBE_LIVE_ID found, running in mock mode');
}
if (!ingestion) {
seedMockMessages(store, overlayEmitter);
}
fastify.get('/health', async () => ({
status: 'ok',
messages: store.length,
selection: currentSelection?.id ?? null,
mode: ingestion ? 'live' : 'mock'
}));
fastify.get('/chat/messages', async () => ({
messages: store
}));
fastify.post<{ Body: { id?: string } }>('/overlay/selection', async (request, reply) => {
const { id } = request.body ?? {};
if (!id) {
reply.status(400);
return { error: 'id is required' };
}
const message = store.find((item) => item.id === id);
if (!message) {
reply.status(404);
return { error: 'message not found' };
}
currentSelection = message;
overlayEmitter.emit('update', currentSelection);
return { ok: true };
});
fastify.delete('/overlay/selection', async () => {
currentSelection = null;
overlayEmitter.emit('update', null);
return { ok: true };
});
fastify.options('/overlay/stream', async (request, reply) => {
reply.headers({
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type'
});
reply.status(204).send();
});
fastify.get('/overlay/stream', async (request, reply) => {
reply.hijack();
const res = reply.raw;
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
res.writeHead(200);
res.write(': connected\n\n');
const send = (message: ChatMessage | null) => {
res.write(`event: selection\ndata: ${JSON.stringify({ message })}\n\n`);
};
const heartbeat = setInterval(() => {
res.write('event: heartbeat\ndata: {}\n\n');
}, 15000);
overlayEmitter.on('update', send);
if (currentSelection) {
send(currentSelection);
}
request.raw.on('close', () => {
clearInterval(heartbeat);
overlayEmitter.off('update', send);
});
});
const port = Number(process.env.PORT ?? 4100);
await fastify.listen({ port, host: '0.0.0.0' });
console.log(`[Backend] Server listening on http://localhost:${port}`);
}
function extractLiveId(input: string): string {
if (!input) return '';
const trimmed = input.trim();
if (/^[a-zA-Z0-9_-]{10,}$/.test(trimmed)) {
return trimmed;
}
try {
const url = new URL(trimmed);
if (url.searchParams.has('v')) {
return url.searchParams.get('v') ?? '';
}
const pathname = url.pathname.split('/').filter(Boolean).pop();
return pathname ?? '';
} catch (error) {
console.warn('Invalid YOUTUBE_LIVE_ID provided', error);
return '';
}
}
function seedMockMessages(
store: ChatMessage[],
overlayEmitter: EventEmitter<{ update: (message: ChatMessage | null) => void }>
) {
let counter = 0;
const authors = ['Ada', 'Linus', 'Grace', 'Marge'];
setInterval(() => {
const message: ChatMessage = {
id: `mock-${Date.now()}`,
author: authors[counter % authors.length],
text: `Mock message #${counter}`,
publishedAt: new Date().toISOString()
};
store.push(message);
if (store.length > MAX_MESSAGES) {
store.splice(0, store.length - MAX_MESSAGES);
}
if (counter % 5 === 0) {
overlayEmitter.emit('update', message);
}
counter += 1;
}, 2000);
}
if (import.meta.url === `file://${process.argv[1]}`) {
startBackend().catch((error) => {
console.error('Failed to start backend', error);
process.exit(1);
});
}
+153
View File
@@ -0,0 +1,153 @@
import type { ChatMessage } from '@shared/chat';
import EventEmitter from 'eventemitter3';
import Innertube, { UniversalCache } from 'youtubei.js';
export type IngestionContext = {
client: Innertube;
liveChat: any;
videoId: string;
emitter: ChatEventEmitter;
};
export type ContinuationState = {
messages: ChatMessage[];
nextToken: string | null;
timeoutMs: number;
};
export type ChatEventEmitter = EventEmitter<{
message: (message: ChatMessage) => void;
error: (error: unknown) => void;
}>;
const defaultTimeout = 1500;
export async function bootstrapInnertube(videoId: string): Promise<IngestionContext> {
if (!videoId) {
throw new Error('YOUTUBE_LIVE_ID is required to bootstrap Innertube');
}
console.log('[Ingestion] Creating Innertube client...');
const client = await Innertube.create({
cache: new UniversalCache(false)
});
console.log('[Ingestion] Fetching video info...');
const info = await client.getInfo(videoId);
console.log('[Ingestion] Getting live chat...');
const liveChat = info.getLiveChat();
if (!liveChat) {
throw new Error('This video does not have an active live chat');
}
const emitter: ChatEventEmitter = new EventEmitter();
liveChat.on('chat-update', (action: any) => {
const normalized = normalizeAction(action);
if (normalized) {
emitter.emit('message', normalized);
}
});
liveChat.on('error', (err: unknown) => {
console.error('[Ingestion] Live chat error:', err);
emitter.emit('error', err);
});
console.log('[Ingestion] Starting live chat listener...');
liveChat.start();
console.log('[Ingestion] Live chat listener started');
return {
client,
liveChat,
videoId,
emitter
};
}
export async function fetchChatBatch(
ctx: IngestionContext,
options?: { windowMs?: number }
): Promise<ContinuationState> {
const windowMs = options?.windowMs ?? defaultTimeout;
const collected: ChatMessage[] = [];
const listener = (message: ChatMessage) => {
collected.push(message);
};
ctx.emitter.on('message', listener);
await delay(windowMs);
ctx.emitter.off('message', listener);
return {
messages: collected,
nextToken: ctx.liveChat?.continuation?.token ?? null,
timeoutMs: ctx.liveChat?.continuation?.timeout_ms ?? defaultTimeout
};
}
function resolveMessageText(item: any): string {
if (!item?.message) return '';
if (typeof item.message === 'string') {
return item.message;
}
if (typeof item.message?.toString === 'function') {
return item.message.toString();
}
if (Array.isArray(item.message?.runs)) {
return item.message.runs.map((run: any) => run.text ?? '').join('');
}
return '';
}
function resolveTimestamp(timestamp: number | string | undefined): string {
if (!timestamp) {
return new Date().toISOString();
}
const numeric = typeof timestamp === 'string' ? Number(timestamp) : timestamp;
if (Number.isFinite(numeric)) {
const millis = numeric > 1e12 ? numeric / 1000 : numeric;
return new Date(millis).toISOString();
}
return new Date().toISOString();
}
function normalizeAction(action: any): ChatMessage | null {
if (!action || action.type !== 'AddChatItemAction') {
return null;
}
const item = action.item;
if (!item) return null;
if (
item.type === 'LiveChatTextMessage' ||
item.type === 'LiveChatPaidMessage' ||
item.type === 'LiveChatMembershipItem'
) {
return {
id: String(item.id ?? item.timestamp_usec ?? Date.now()),
author: String(item.author?.name ?? 'Unknown'),
text: resolveMessageText(item),
publishedAt: resolveTimestamp(item.timestamp ?? item.timestamp_usec)
};
}
return null;
}
function delay(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
+12
View File
@@ -0,0 +1,12 @@
{
"extends": "../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ES2022",
"noEmit": false,
"allowSyntheticDefaultImports": true
},
"include": ["src"]
}
+164
View File
@@ -0,0 +1,164 @@
'use client';
import { useCallback, useEffect, useMemo, useState } from 'react';
import type { ChatMessage } from '@shared/chat';
const BACKEND_URL = process.env.NEXT_PUBLIC_BACKEND_URL ?? 'http://localhost:4100';
const POLL_INTERVAL = 2500;
export default function DashboardPage() {
const { messages, refresh, error: pollError } = useChatMessages();
const { selection, status: overlayStatus } = useOverlaySelection();
const handleSelect = useCallback(
async (message: ChatMessage) => {
try {
await fetch(`${BACKEND_URL}/overlay/selection`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ id: message.id })
});
} catch (error) {
console.error('Failed to select message', error);
}
},
[]
);
const handleClear = useCallback(async () => {
try {
await fetch(`${BACKEND_URL}/overlay/selection`, { method: 'DELETE' });
} catch (error) {
console.error('Failed to clear selection', error);
}
}, []);
useEffect(() => {
const timer = setInterval(() => {
refresh();
}, POLL_INTERVAL);
return () => clearInterval(timer);
}, [refresh]);
const statusHint = useMemo(() => {
if (pollError) return 'Backend unreachable';
if (overlayStatus === 'connecting') return 'Connecting to overlay…';
if (overlayStatus === 'error') return 'Overlay stream disconnected';
return 'Live';
}, [pollError, overlayStatus]);
return (
<main className="dashboard">
<header className="dashboard__header">
<div>
<h1>Operator Dashboard</h1>
<p className="muted">Click a message to push it to the OBS overlay stream.</p>
</div>
<span className={`status status--${overlayStatus}`}>{statusHint}</span>
</header>
<section className="dashboard__content">
<article className="panel">
<header className="panel__title">Live Chat</header>
<div className="chatList">
{messages.map((message) => (
<button
key={message.id}
className={
selection?.id === message.id ? 'chatItem chatItem--active' : 'chatItem'
}
onClick={() => handleSelect(message)}
>
<span className="chatItem__author">{message.author}</span>
<span className="chatItem__text">{message.text}</span>
<time>{new Date(message.publishedAt).toLocaleTimeString()}</time>
</button>
))}
{messages.length === 0 && <p className="muted">Waiting for chat messages</p>}
</div>
</article>
<article className="panel overlayPreview">
<header className="panel__title">Overlay Preview</header>
{selection ? (
<div className="overlayPreview__card">
<span className="overlayPreview__author">{selection.author}</span>
<p>{selection.text}</p>
<time>{new Date(selection.publishedAt).toLocaleTimeString()}</time>
<button className="secondary" onClick={handleClear}>
Clear selection
</button>
</div>
) : (
<div className="overlayPreview__empty">
<p>No message selected yet.</p>
<button className="secondary" onClick={handleClear}>
Reset
</button>
</div>
)}
</article>
</section>
</main>
);
}
function useChatMessages() {
const [messages, setMessages] = useState<ChatMessage[]>([]);
const [error, setError] = useState<Error | null>(null);
const refresh = useCallback(async () => {
try {
const response = await fetch(`${BACKEND_URL}/chat/messages`);
if (!response.ok) throw new Error(`Request failed: ${response.status}`);
const data = await response.json();
setMessages(Array.isArray(data.messages) ? data.messages : []);
setError(null);
} catch (err) {
setError(err as Error);
}
}, []);
useEffect(() => {
refresh();
}, [refresh]);
return { messages, refresh, error };
}
type OverlayStatus = 'connecting' | 'live' | 'error';
type SelectionPayload = {
message: ChatMessage | null;
};
function useOverlaySelection() {
const [selection, setSelection] = useState<ChatMessage | null>(null);
const [status, setStatus] = useState<OverlayStatus>('connecting');
useEffect(() => {
const source = new EventSource(`${BACKEND_URL}/overlay/stream`);
const onSelection = (event: MessageEvent) => {
try {
const payload: SelectionPayload = JSON.parse(event.data);
setSelection(payload.message);
setStatus('live');
} catch (error) {
console.error('Failed to parse selection payload', error);
}
};
source.addEventListener('selection', onSelection as EventListener);
source.addEventListener('heartbeat', () => setStatus('live'));
source.onerror = () => setStatus('error');
return () => {
source.removeEventListener('selection', onSelection as EventListener);
source.close();
};
}, []);
return { selection, status };
}
+238
View File
@@ -0,0 +1,238 @@
:root {
color-scheme: dark;
}
body {
margin: 0;
font-family: 'Inter', system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
background: #050608;
color: #f4f4f5;
min-height: 100vh;
}
a {
color: inherit;
text-decoration: none;
}
main {
min-height: 100vh;
}
.landing {
display: grid;
place-items: center;
padding: 4rem 1.5rem;
}
.panel {
background: rgba(21, 23, 28, 0.85);
border: 1px solid rgba(255, 255, 255, 0.08);
border-radius: 16px;
padding: 2.5rem;
max-width: 720px;
width: min(100%, 720px);
box-shadow: 0 12px 40px rgba(0, 0, 0, 0.35);
}
.panel__title {
font-size: 1.1rem;
font-weight: 600;
margin-bottom: 1rem;
}
.primary,
.secondary,
.chatItem {
font: inherit;
border: none;
cursor: pointer;
}
.primary {
display: inline-block;
margin-top: 1.5rem;
padding: 0.85rem 1.6rem;
border-radius: 999px;
background: linear-gradient(135deg, #ff3b30, #ff9500);
color: #fff;
font-weight: 600;
transition: transform 120ms ease, box-shadow 120ms ease;
}
.primary:hover {
transform: translateY(-2px);
box-shadow: 0 8px 24px rgba(255, 99, 71, 0.35);
}
.secondary {
margin-top: 1rem;
padding: 0.6rem 1.2rem;
border-radius: 999px;
background: rgba(255, 255, 255, 0.08);
color: #f4f4f5;
}
.muted {
color: rgba(244, 244, 245, 0.6);
margin-top: 0.6rem;
}
.dashboard {
display: flex;
flex-direction: column;
gap: 1.5rem;
padding: 2rem clamp(1rem, 5vw, 3rem);
}
.dashboard__header {
display: flex;
justify-content: space-between;
align-items: center;
gap: 1rem;
}
.dashboard__content {
display: grid;
grid-template-columns: minmax(0, 2fr) minmax(0, 1fr);
gap: 1.5rem;
}
@media (max-width: 960px) {
.dashboard__content {
grid-template-columns: 1fr;
}
}
.chatList {
display: flex;
flex-direction: column;
gap: 0.75rem;
max-height: 70vh;
overflow-y: auto;
}
.chatItem {
display: grid;
grid-template-columns: auto 1fr auto;
gap: 0.75rem;
align-items: baseline;
padding: 0.85rem 1rem;
border-radius: 12px;
background: rgba(255, 255, 255, 0.04);
text-align: left;
transition: background 120ms ease, transform 120ms ease;
}
.chatItem:hover {
background: rgba(255, 255, 255, 0.1);
transform: translateY(-2px);
}
.chatItem--active {
outline: 2px solid rgba(255, 149, 0, 0.65);
background: rgba(255, 149, 0, 0.12);
}
.chatItem__author {
font-weight: 600;
color: #f97316;
}
.chatItem__text {
opacity: 0.9;
}
.chatItem time {
font-size: 0.8rem;
opacity: 0.6;
}
.overlayPreview {
display: flex;
flex-direction: column;
justify-content: space-between;
}
.overlayPreview__card {
display: flex;
flex-direction: column;
gap: 0.75rem;
background: rgba(255, 255, 255, 0.05);
padding: 1.5rem;
border-radius: 14px;
min-height: 240px;
}
.overlayPreview__author {
font-weight: 600;
color: #7dd3fc;
}
.overlayPreview__empty {
min-height: 240px;
display: grid;
place-items: center;
background: rgba(255, 255, 255, 0.03);
border-radius: 14px;
gap: 1rem;
}
.overlay {
min-height: 100vh;
display: grid;
place-items: center;
background: rgba(0, 0, 0, 0);
}
.overlay__card {
padding: 1.5rem 2rem;
border-radius: 18px;
background: rgba(15, 23, 42, 0.85);
color: #f8fafc;
max-width: 960px;
width: min(90vw, 960px);
box-shadow: 0 16px 60px rgba(15, 23, 42, 0.35);
}
.overlay__author {
display: block;
font-size: 1.1rem;
font-weight: 700;
margin-bottom: 0.75rem;
color: #38bdf8;
}
.overlay__text {
font-size: clamp(1.4rem, 2.5vw, 2rem);
line-height: 1.4;
}
.overlay__placeholder {
padding: 1rem 1.4rem;
border-radius: 12px;
background: rgba(255, 255, 255, 0.08);
color: rgba(255, 255, 255, 0.75);
}
.status {
padding: 0.4rem 0.8rem;
border-radius: 999px;
font-size: 0.85rem;
font-weight: 600;
}
.status--connecting {
background: rgba(125, 211, 252, 0.2);
color: #7dd3fc;
}
.status--live {
background: rgba(74, 222, 128, 0.2);
color: #4ade80;
}
.status--error {
background: rgba(248, 113, 113, 0.2);
color: #f87171;
}
+52
View File
@@ -0,0 +1,52 @@
'use client';
import { useEffect, useState } from 'react';
import type { ChatMessage } from '@shared/chat';
const BACKEND_URL = process.env.NEXT_PUBLIC_BACKEND_URL ?? 'http://localhost:4100';
type SelectionPayload = {
message: ChatMessage | null;
};
export default function OverlayPage() {
const [message, setMessage] = useState<ChatMessage | null>(null);
const [connected, setConnected] = useState(false);
useEffect(() => {
const source = new EventSource(`${BACKEND_URL}/overlay/stream`);
const onSelection = (event: MessageEvent) => {
try {
const payload: SelectionPayload = JSON.parse(event.data);
setMessage(payload.message);
setConnected(true);
} catch (error) {
console.error('overlay: failed to parse payload', error);
}
};
source.addEventListener('selection', onSelection as EventListener);
source.addEventListener('heartbeat', () => setConnected(true));
source.onerror = () => setConnected(false);
return () => {
source.removeEventListener('selection', onSelection as EventListener);
source.close();
};
}, []);
return (
<main className="overlay">
{message ? (
<div className="overlay__card">
<span className="overlay__author">{message.author}</span>
<p className="overlay__text">{message.text}</p>
</div>
) : (
<div className="overlay__placeholder">
<span>{connected ? 'Awaiting selection…' : 'Reconnecting…'}</span>
</div>
)}
</main>
);
}
+18
View File
@@ -0,0 +1,18 @@
import Link from 'next/link';
export default function HomePage() {
return (
<main className="landing">
<section className="panel">
<h1>YouTube Chat Client</h1>
<p>
Launch the dashboard to monitor live chat and control the overlay that feeds OBS.
</p>
<Link className="primary" href="/dashboard">
Open Dashboard
</Link>
<p className="muted">Overlay preview lives at /overlay for the OBS browser source.</p>
</section>
</main>
);
}
+6
View File
@@ -0,0 +1,6 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
/// <reference path="./.next/types/routes.d.ts" />
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
+34
View File
@@ -0,0 +1,34 @@
{
"extends": "../tsconfig.base.json",
"compilerOptions": {
"jsx": "preserve",
"types": [
"next",
"next/types/global",
"next/image-types/global"
],
"lib": [
"dom",
"dom.iterable",
"esnext"
],
"allowJs": true,
"noEmit": true,
"incremental": true,
"plugins": [
{
"name": "next"
}
]
},
"include": [
"**/*.ts",
"**/*.tsx",
"next-env.d.ts",
".next/types/**/*.ts"
],
"exclude": [
"node_modules",
"dist"
]
}
+10 -10
View File
@@ -1,19 +1,19 @@
# Active Context
## Current Focus
- Maintain Memory Bank documentation and scaffold the monorepo structure for the YouTube Live chat client.
- Define onboarding flow for Innertube-based chat ingestion and configuration.
- Simplified project layout: single pnpm package with `client/`, `backend/`, and `shared/` folders; no workspaces.
- Backend and UI skeletons run via `pnpm dev`, ready for real stream integration and UX polish.
## Recent Decisions
- Switch from official YouTube Data API to Innertube (`youtubei.js`) ingestion to avoid quota issues.
- Use Next.js for operator UI and OBS overlay, with a separate backend worker for polling and realtime events.
- Prefer Server-Sent Events for one-way overlay updates; keep WebSocket option in mind for future enhancements.
- Removed pnpm workspaces to reduce setup friction; all dependencies now live in the root `package.json`.
- Adopted `tsx` for running the backend in dev, so we avoid ESM loader quirks from `ts-node`.
- Maintained Innertube (`youtubei.js`) ingestion with mock fallback to keep development unblocked without credentials.
## Immediate Next Steps
1. Initialize `pnpm` workspace with `apps/client`, `packages/backend`, and `packages/shared` directories. (Scaffolded.)
2. Configure baseline project files: `package.json`, `pnpm-workspace.yaml`, TS configs, linting setup. (Scaffolded.)
3. Stub backend poller using `youtubei.js` to verify dev scripts once dependencies are installed.
1. Verify `pnpm install` + `pnpm dev` on a clean machine, ensuring backend and client start smoothly.
2. Harden backend ingestion (error handling, reconnection/backoff) now that the runtime setup is stable.
3. Flesh out operator dashboard UX (filters/search, live status indicators) and document configuration in README/onboarding notes.
## Open Questions
- How to persist or refresh Innertube context data (visitor data, API key) between sessions for reliability.
- Whether to include optional SQLite persistence from the outset or add once basic flow is working.
- Whether to persist Innertube visitor data between runs to reduce boot time and API churn.
- When to introduce optional persistence (SQLite) given `better-sqlite3` is now a direct runtime dependency.
+11 -11
View File
@@ -2,23 +2,23 @@
## Phase 0 Foundations
- [x] Create Memory Bank documentation.
- [x] Scaffold pnpm workspace structure.
- [ ] Commit baseline configs and ensure dev scripts run.
- [x] Simplify project layout (single package, shared types via alias).
- [ ] Document setup instructions for the new command set.
## Phase 1 Core Infrastructure
- [ ] Implement Innertube client bootstrap (retrieve context, manage continuation tokens).
- [ ] Build backend poller with message normalization and rate/error handling.
- [ ] Expose REST+SSE endpoints for chat and overlay delivery.
- [x] Implement Innertube client bootstrap (retrieve context, manage continuation tokens).
- [ ] Build backend poller with full normalization, error/backoff handling, and persistence hooks.
- [x] Expose REST+SSE endpoints for chat and overlay delivery.
## Phase 2 Operator Dashboard
- [ ] Implement chat feed UI with filters/search and live updates.
- [ ] Provide message selection controls and status indicators.
- [ ] Implement chat feed UI with filters/search and live status.
- [x] Provide message selection controls and overlay preview basics.
- [ ] Handle error states (rate limits, disconnects) gracefully in UI.
## Phase 3 OBS Overlay Experience
- [ ] Create minimal overlay page that consumes SSE stream.
- [ ] Style overlay for readability and ensure quick updates in OBS browser source.
- [ ] Add local preview within dashboard for operator verification.
- [x] Create minimal overlay page that consumes SSE stream.
- [ ] Style overlay for production readability and ensure OBS compatibility testing.
- [ ] Add local preview enhancements (animations, theme controls).
## Phase 4 Reliability & Polish
- [ ] Add optional persistence (SQLite) and crash recovery.
@@ -26,4 +26,4 @@
- [ ] Write tests (unit/integration) and contributor documentation.
## Current Status
- Memory Bank established; repository scaffolding in place; backend ingestion stubs pending.
- Single-package setup in place; backend/frontend run via unified scripts. Awaiting validation on clean install, ingestion hardening, and richer dashboard UX.
+12 -12
View File
@@ -1,18 +1,18 @@
# System Patterns
## Architecture Overview
- **Monorepo Layout:** `pnpm` workspaces with `apps/client` (Next.js dashboard + overlay), `packages/backend` (Node ingestion + realtime gateway), `packages/shared` (types, schemas).
- **Project Layout:** Single pnpm package with three top-level folders: `client/` (Next.js dashboard + overlay), `backend/` (Node ingestion + realtime gateway), and `shared/` (typescript definitions shared between both).
- **Data Flow:**
1. Backend worker polls YouTube Live chat through the Innertube (youtubei) API, maintaining continuation tokens.
2. Messages stored in-memory (and optionally SQLite) and emitted over an internal event bus.
3. Client dashboard fetches chat via HTTP (React Query) and pushes selection back via REST.
4. Overlay page listens to Server-Sent Events stream for the currently highlighted message.
- **Realtime Delivery:** SSE chosen for one-directional updates to OBS browser source; can swap to WebSocket if bidirectional control is required later.
- **Configuration:** Environment variables drive stream IDs and optional auth tokens; local secrets persisted in `.env.local` or config files.
1. Backend worker polls YouTube Live chat through the Innertube (`youtubei.js`) API, maintaining continuation tokens.
2. Messages are stored in-memory (optionally persisted later) and emitted over an internal event bus.
3. Client dashboard fetches chat data via REST and pushes selection updates via REST.
4. Overlay page consumes a Server-Sent Events stream to stay in sync with the selected message.
- **Realtime Delivery:** SSE for one-directional updates to OBS browser source; leave room to switch to WebSockets if we need bidirectional control later.
- **Configuration:** `.env.local` (or process env) supplies `YOUTUBE_LIVE_ID` and optional Innertube overrides; backend picks mock mode automatically when unset.
## Key Patterns & Practices
- Abstract ingestion behind an interface so alternate providers (official API, headless browser) can be swapped in quickly.
- Cache Innertube visitor data and API keys locally to reduce startup latency and handle rotations gracefully.
- Use Zod schemas in shared package to validate external responses and internal payloads.
- Centralized error reporting/logging with structured logs for monitoring during streams.
- Graceful degradation: exponential backoff on fetch failures, last-known overlay message cached to disk to survive restarts.
- Abstract ingestion behind a module (`backend/src/ingestion/youtubei.ts`) so alternate providers (official API, headless browser) can be swapped in quickly.
- Cache Innertube visitor data and API keys locally when we extend functionality, keeping startup fast and resilient to key rotations.
- Use shared TypeScript definitions via the `@shared` path alias to maintain type safety across backend and client.
- Centralized logging in the backend with structured payloads for easier debugging during long streams.
- Graceful degradation: backoff strategies for fetch failures and mock-data fallback keep the UI usable even without credentials.
+15 -10
View File
@@ -1,17 +1,22 @@
# Tech Context
## Primary Stack
- **Frontend:** Next.js 14 (App Router) + React 18 + TypeScript, styled with Tailwind CSS and optional shadcn/ui components.
- **Backend Worker:** Node.js (Fastify) with `youtubei.js` for Innertube chat ingestion, `better-sqlite3` for persistence, EventEmitter for internal pub/sub.
- **Realtime:** Server-Sent Events for overlay updates; potential future WebSocket support via `ws`.
- **Tooling:** `pnpm` for workspace management, ESLint + Prettier, Zod for schema validation, Vitest/Playwright for testing (to be introduced later).
- **Frontend:** Next.js 15 (App Router) + React 19 + TypeScript, styled with handcrafted CSS for now (Tailwind/shadcn still optional future add-ons).
- **Backend Worker:** Node.js service run with `tsx`, using `youtubei.js` for Innertube chat ingestion, `fastify` + `@fastify/cors` for REST/SSE transport, and `eventemitter3` for internal pub/sub.
- **Shared Types:** Simple TypeScript module in `shared/chat.ts`, imported via the `@shared/*` path alias defined in `tsconfig.base.json`.
- **Realtime:** Server-Sent Events for overlay updates; WebSockets remain a future enhancement option.
## Tooling & Commands
- Single root `package.json`; `pnpm install` manages all deps.
- `pnpm dev` runs backend (`tsx backend/src/index.ts`) and client (`next dev client`) concurrently via `concurrently`.
- `pnpm build` compiles the backend with `tsc` and builds the Next app; `pnpm start:backend` / `pnpm start:client` serve production bundles separately.
## Environment & Dependencies
- No official API quota required; ingestion relies on Innertube visitor tokens produced at runtime.
- Required env values: `YOUTUBE_LIVE_ID` (or stream URL), optional overrides for Innertube API key/context if we need to pin versions.
- Local `.env.local` file manages configuration; sample `.env.example` committed for contributors.
- Requires `YOUTUBE_LIVE_ID` (or full URL) to enable real chat ingestion; omitted value triggers mock mode.
- Optional overrides for Innertube API keys/version can be supplied through env vars when needed.
- `better-sqlite3` is included for future persistence work but not yet wired in.
## Constraints & Considerations
- Innertube endpoints change occasionally; design ingestion to update keys dynamically and fall back to alternate strategies if responses shift.
- Application expected to run on Windows/macOS/Linux desktops used for streaming; keep dependencies cross-platform and avoid native build steps when possible.
- No external database by default; design backend to operate fully in-process with optional local persistence.
- Innertube endpoints may break; keep ingestion module adaptable and plan for a headless-browser fallback.
- Project expected to run on the streamers machine; dependencies must remain cross-platform and avoid heavyweight native builds where possible.
- No separate package boundaries anymore, so TypeScript path aliases and import hygiene are important to prevent tangled relative paths.
+27 -4
View File
@@ -6,9 +6,32 @@
"license": "MIT",
"packageManager": "[email protected]",
"scripts": {
"dev": "pnpm -r dev",
"build": "pnpm -r build",
"lint": "pnpm -r lint",
"start": "pnpm -r start"
"dev": "concurrently \"pnpm dev:backend\" \"pnpm dev:client\"",
"dev:backend": "tsx --env-file=.env backend/src/index.ts",
"dev:client": "next dev client",
"build": "pnpm build:backend && pnpm build:client",
"build:backend": "tsc -p backend/tsconfig.build.json",
"build:client": "next build client",
"start:backend": "node backend/dist/index.js",
"start:client": "next start client",
"lint": "next lint client"
},
"dependencies": {
"@fastify/cors": "^11.1.0",
"better-sqlite3": "^12.4.1",
"eventemitter3": "^5.0.1",
"fastify": "^5.6.1",
"next": "^15.5.4",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"youtubei.js": "^15.1.1",
"zod": "^4.1.11"
},
"devDependencies": {
"@types/node": "^20.16.5",
"@types/react": "19.2.0",
"concurrently": "^8.2.2",
"tsx": "^4.19.1",
"typescript": "^5.9.3"
}
}
-14
View File
@@ -1,14 +0,0 @@
{
"name": "backend",
"version": "0.1.0",
"private": true,
"description": "YouTube chat poller and realtime bridge.",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"dev": "ts-node src/index.ts",
"build": "tsc -p tsconfig.build.json",
"start": "node dist/index.js",
"lint": "eslint 'src/**/*.{ts,tsx}'"
}
}
-12
View File
@@ -1,12 +0,0 @@
import { bootstrapInnertube, fetchChatBatch } from './ingestion/youtubei';
export async function startBackend() {
console.log('Starting backend worker (youtubei ingestion pending).');
const state = await bootstrapInnertube();
console.log('Initial continuation state', state);
await fetchChatBatch(state);
}
if (require.main === module) {
void startBackend();
}
@@ -1,18 +0,0 @@
export type ContinuationState = {
token: string | null;
apiKey?: string;
};
export async function bootstrapInnertube(): Promise<ContinuationState> {
// TODO: fetch initial visitor data and live chat continuation token via youtubei.js
return { token: null };
}
export async function fetchChatBatch(state: ContinuationState) {
// TODO: use youtubei.js to fetch messages with the provided continuation token
// return both normalized messages and the next continuation token
return {
messages: [],
next: state.token
};
}
-10
View File
@@ -1,10 +0,0 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"module": "CommonJS",
"target": "ES2021",
"noEmit": false
},
"include": ["src"]
}
-14
View File
@@ -1,14 +0,0 @@
{
"name": "shared",
"version": "0.1.0",
"private": true,
"description": "Shared types and schemas.",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"dev": "tsc --watch -p tsconfig.json",
"build": "tsc -p tsconfig.json",
"start": "node dist/index.js",
"lint": "eslint 'src/**/*.{ts,tsx}'"
}
}
-12
View File
@@ -1,12 +0,0 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"module": "ESNext",
"target": "ES2021",
"declaration": true,
"declarationMap": true,
"noEmit": false
},
"include": ["src"]
}
-3
View File
@@ -1,3 +0,0 @@
packages:
- 'apps/*'
- 'packages/*'
+4 -1
View File
@@ -12,6 +12,9 @@
"isolatedModules": true,
"noEmit": true,
"types": ["node"],
"baseUrl": "."
"baseUrl": ".",
"paths": {
"@shared/*": ["shared/*"]
}
}
}
+2 -3
View File
@@ -1,8 +1,7 @@
{
"files": [],
"references": [
{ "path": "apps/client" },
{ "path": "packages/backend" },
{ "path": "packages/shared" }
{ "path": "client" },
{ "path": "backend" }
]
}