From 311b5f392b88e1446a24ce32668338b0c4619d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yusuf=20=C4=B0pek?= Date: Sat, 25 Oct 2025 21:59:55 +0300 Subject: [PATCH] feat(youtubei): ensure unique message IDs in high-frequency chats Add ID deduplication mechanism to handle cases where YouTube message IDs might not be unique in high-frequency chat scenarios. The system now tracks seen IDs and appends a counter suffix when duplicates are encountered, preventing message ID collisions. --- backend/src/ingestion/youtubei.ts | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/backend/src/ingestion/youtubei.ts b/backend/src/ingestion/youtubei.ts index 8d94930..156bbf1 100644 --- a/backend/src/ingestion/youtubei.ts +++ b/backend/src/ingestion/youtubei.ts @@ -23,6 +23,22 @@ export type ChatEventEmitter = EventEmitter<{ const defaultTimeout = 1500; +// Track message IDs to ensure uniqueness +const seenIds = new Map(); + +function generateUniqueId(baseId: string): string { + const count = seenIds.get(baseId) ?? 0; + seenIds.set(baseId, count + 1); + + // If this is the first occurrence of this ID, return it as-is + if (count === 0) { + return baseId; + } + + // Otherwise, append a counter suffix to ensure uniqueness + return `${baseId}#${count}`; +} + function resolveMessageRuns(item: any) { const runs: { text?: string; emojiUrl?: string; emojiAlt?: string }[] = []; const rawRuns = item?.message?.runs; @@ -473,9 +489,14 @@ function normalizeAction(action: any): ChatMessage | null { // Extract leaderboard rank if present const leaderboardRank = extractLeaderboardRank(item); - + + // Generate a unique ID - YouTube message IDs might not be unique in high-frequency chats + // so we track seen IDs and append a counter if needed + const baseId = String(item.id ?? item.timestamp_usec ?? Date.now()); + const uniqueId = generateUniqueId(baseId); + return { - id: String(item.id ?? item.timestamp_usec ?? Date.now()), + id: uniqueId, author: authorName, authorPhoto, authorChannelId: authorChannelId ? String(authorChannelId) : undefined,