feat: add Fedora package sync API endpoint

- Implemented API route with progress tracking for fetching and storing Fedora packages
- Created FedoraPackageFetcher service to scrape packages.fedoraproject.org with rate limiting
- Added batch processing and duplicate detection for efficient database storage
This commit is contained in:
Yusuf İpek
2025-11-10 20:06:23 +03:00
parent fdd5d9ef63
commit 2b93adcba5
2 changed files with 263 additions and 0 deletions
+86
View File
@@ -0,0 +1,86 @@
import { NextResponse } from 'next/server'
import { FedoraPackageFetcher } from '@/services/fedoraPackageFetcher'
export const dynamic = 'force-dynamic'
export const maxDuration = 300 // 5 minutes timeout
let syncInProgress = false
let syncStatus = {
status: 'idle' as 'idle' | 'fetching' | 'storing' | 'complete' | 'error',
fetchProgress: 0,
fetchTotal: 0,
storeProgress: 0,
storeTotal: 0,
currentPackage: '',
error: null as string | null
}
export async function GET() {
return NextResponse.json(syncStatus)
}
export async function POST() {
if (syncInProgress) {
return NextResponse.json(
{ error: 'Sync already in progress' },
{ status: 409 }
)
}
syncInProgress = true
syncStatus = {
status: 'fetching',
fetchProgress: 0,
fetchTotal: 0,
storeProgress: 0,
storeTotal: 0,
currentPackage: '',
error: null
}
// Start sync in background
;(async () => {
try {
const fetcher = new FedoraPackageFetcher()
// Fetch packages
console.log('🔄 Starting Fedora package fetch...')
const packages = await fetcher.fetchAllPackages(
(current, total, packageName) => {
syncStatus.fetchProgress = current
syncStatus.fetchTotal = total
syncStatus.currentPackage = packageName
}
)
console.log(`✅ Fetched ${packages.length} Fedora packages`)
// Store packages
syncStatus.status = 'storing'
syncStatus.storeTotal = packages.length
await fetcher.storePackages(
packages,
(current, total) => {
syncStatus.storeProgress = current
syncStatus.storeTotal = total
}
)
syncStatus.status = 'complete'
console.log('✅ Fedora sync completed successfully')
} catch (error) {
console.error('❌ Fedora sync error:', error)
syncStatus.status = 'error'
syncStatus.error = error instanceof Error ? error.message : 'Unknown error'
} finally {
syncInProgress = false
}
})()
return NextResponse.json({
message: 'Fedora package sync started',
status: syncStatus
})
}