From 2b93adcba519aada0ed4f2ff6eb8236f27f794de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yusuf=20=C4=B0pek?= Date: Mon, 10 Nov 2025 20:06:23 +0300 Subject: [PATCH] feat: add Fedora package sync API endpoint - Implemented API route with progress tracking for fetching and storing Fedora packages - Created FedoraPackageFetcher service to scrape packages.fedoraproject.org with rate limiting - Added batch processing and duplicate detection for efficient database storage --- src/app/api/sync-fedora/route.ts | 86 +++++++++++++ src/services/fedoraPackageFetcher.ts | 177 +++++++++++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 src/app/api/sync-fedora/route.ts create mode 100644 src/services/fedoraPackageFetcher.ts diff --git a/src/app/api/sync-fedora/route.ts b/src/app/api/sync-fedora/route.ts new file mode 100644 index 0000000..ec9bacf --- /dev/null +++ b/src/app/api/sync-fedora/route.ts @@ -0,0 +1,86 @@ +import { NextResponse } from 'next/server' +import { FedoraPackageFetcher } from '@/services/fedoraPackageFetcher' + +export const dynamic = 'force-dynamic' +export const maxDuration = 300 // 5 minutes timeout + +let syncInProgress = false +let syncStatus = { + status: 'idle' as 'idle' | 'fetching' | 'storing' | 'complete' | 'error', + fetchProgress: 0, + fetchTotal: 0, + storeProgress: 0, + storeTotal: 0, + currentPackage: '', + error: null as string | null +} + +export async function GET() { + return NextResponse.json(syncStatus) +} + +export async function POST() { + if (syncInProgress) { + return NextResponse.json( + { error: 'Sync already in progress' }, + { status: 409 } + ) + } + + syncInProgress = true + syncStatus = { + status: 'fetching', + fetchProgress: 0, + fetchTotal: 0, + storeProgress: 0, + storeTotal: 0, + currentPackage: '', + error: null + } + + // Start sync in background + ;(async () => { + try { + const fetcher = new FedoraPackageFetcher() + + // Fetch packages + console.log('🔄 Starting Fedora package fetch...') + const packages = await fetcher.fetchAllPackages( + (current, total, packageName) => { + syncStatus.fetchProgress = current + syncStatus.fetchTotal = total + syncStatus.currentPackage = packageName + } + ) + + console.log(`✅ Fetched ${packages.length} Fedora packages`) + + // Store packages + syncStatus.status = 'storing' + syncStatus.storeTotal = packages.length + + await fetcher.storePackages( + packages, + (current, total) => { + syncStatus.storeProgress = current + syncStatus.storeTotal = total + } + ) + + syncStatus.status = 'complete' + console.log('✅ Fedora sync completed successfully') + + } catch (error) { + console.error('❌ Fedora sync error:', error) + syncStatus.status = 'error' + syncStatus.error = error instanceof Error ? error.message : 'Unknown error' + } finally { + syncInProgress = false + } + })() + + return NextResponse.json({ + message: 'Fedora package sync started', + status: syncStatus + }) +} diff --git a/src/services/fedoraPackageFetcher.ts b/src/services/fedoraPackageFetcher.ts new file mode 100644 index 0000000..8eb76f5 --- /dev/null +++ b/src/services/fedoraPackageFetcher.ts @@ -0,0 +1,177 @@ +interface FedoraPackage { + name: string + version: string + description: string + repository: string +} + +export class FedoraPackageFetcher { + private baseUrl = 'https://packages.fedoraproject.org' + + /** + * Fetch all Fedora packages with progress callback + */ + async fetchAllPackages( + onProgress?: (current: number, total: number, packageName: string) => void + ): Promise { + console.log('🎩 Starting Fedora package fetch...') + + const allPackages: FedoraPackage[] = [] + + try { + // Step 1: Get all index prefixes + console.log('📄 Fetching index page...') + const indexResponse = await fetch(`${this.baseUrl}/index-static.html`) + if (!indexResponse.ok) { + throw new Error(`HTTP error! status: ${indexResponse.status}`) + } + + const indexHtml = await indexResponse.text() + const prefixes = this.extractPrefixes(indexHtml) + + console.log(`📦 Found ${prefixes.length} index prefixes`) + + // Step 2: Fetch packages from each prefix + let processedPrefixes = 0 + for (const prefix of prefixes) { + try { + const prefixUrl = `${this.baseUrl}/index/${prefix}.html` + console.log(`📄 Fetching prefix ${prefix}... (${processedPrefixes + 1}/${prefixes.length})`) + + const response = await fetch(prefixUrl) + if (!response.ok) { + console.error(`❌ Failed to fetch prefix ${prefix}: ${response.status}`) + continue + } + + const html = await response.text() + const packages = this.extractPackagesFromPrefix(html, prefix) + + allPackages.push(...packages) + processedPrefixes++ + + if (onProgress && packages.length > 0) { + onProgress(allPackages.length, allPackages.length, packages[0].name) + } + + // Rate limiting: wait 100ms between requests + await new Promise(resolve => setTimeout(resolve, 100)) + + } catch (error) { + console.error(`❌ Error fetching prefix ${prefix}:`, error) + } + } + + console.log(`✅ Fetch completed! Total packages: ${allPackages.length}`) + return allPackages + + } catch (error) { + console.error('❌ Error fetching Fedora packages:', error) + throw error + } + } + + /** + * Extract prefix links from index page + */ + private extractPrefixes(html: string): string[] { + const prefixes: string[] = [] + + // Match: 0a + const prefixRegex = //gi + + let match + while ((match = prefixRegex.exec(html)) !== null) { + prefixes.push(match[1]) + } + + return prefixes + } + + /** + * Extract package names from a prefix page + */ + private extractPackagesFromPrefix(html: string, prefix: string): FedoraPackage[] { + const packages: FedoraPackage[] = [] + + // Match:
  • ffmpeg-free
    • + const packageRegex = /
  • ([^<]+)<\/a><\/li>/gi + + let match + while ((match = packageRegex.exec(html)) !== null) { + const name = match[1].trim() + + if (name) { + packages.push({ + name, + version: 'latest', // We'll get version from API later if needed + description: `Fedora package: ${name}`, + repository: 'official' + }) + } + } + + return packages + } + + /** + * Store packages in database + */ + async storePackages( + packages: FedoraPackage[], + onProgress?: (current: number, total: number) => void + ): Promise { + const { query } = await import('@/lib/database/config') + + console.log(`💾 Storing ${packages.length} Fedora packages in database...`) + + const batchSize = 100 + let stored = 0 + let updated = 0 + let skipped = 0 + + for (let i = 0; i < packages.length; i += batchSize) { + const batch = packages.slice(i, i + batchSize) + + for (const pkg of batch) { + try { + // Check if package already exists + const existingResult = await query( + 'SELECT id, version FROM packages WHERE name = $1 AND platform_id = $2', + [pkg.name, 'fedora'] + ) + + if (existingResult.rows.length === 0) { + // Insert new package + await query( + `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) + VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, + [pkg.name, pkg.description, pkg.version, 'fedora', 'cli', pkg.repository, 0, true] + ) + stored++ + } else if (existingResult.rows[0].version !== pkg.version) { + // Update if version changed + await query( + `UPDATE packages + SET version = $1, description = $2, repository = $3, updated_at = NOW() + WHERE id = $4`, + [pkg.version, pkg.description, pkg.repository, existingResult.rows[0].id] + ) + updated++ + } else { + skipped++ + } + + if (onProgress && (stored + updated + skipped) % 100 === 0) { + onProgress(stored + updated + skipped, packages.length) + } + + } catch (error) { + console.error(`Error storing package ${pkg.name}:`, error) + } + } + } + + console.log(`✅ Stored ${stored} new packages, updated ${updated}, skipped ${skipped}`) + } +}