feat: add multiple package sync sources and API integration

- Added undici dependency and implemented three package fetcher variants (simple text parsing, v2 with better error handling, and official repository sync) - Integrated real API calls in PlatformSelector and PackageBrowser components with fallback to mock data - Extended sync API route to support multiple data sources (debian-simple, ubuntu-simple, debian-official-v2, etc.) with platform initialization
2026-09-11 18:46:07 +00:00 · 2025-11-10 19:12:46 +03:00
parent 596bba0832
commit 08f35aedbc
25 changed files with 2152 additions and 102 deletions
@@ -0,0 +1,74 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { PackageService } from '@/services/packageService'
+
+export async function GET(request: NextRequest) {
+  try {
+    const { searchParams } = new URL(request.url)
+    const platform_id = searchParams.get('platform_id') || 'debian'
+    
+    console.log('🐛 Debug: Fetching packages for platform:', platform_id)
+    
+    // Test 1: Simple query without filters
+    const result1 = await PackageService.getMany({
+      platform_id,
+      limit: 10
+    })
+    
+    console.log('🐛 Debug: Simple query result:', {
+      total: result1.total,
+      packageCount: result1.packages.length,
+      firstPackage: result1.packages[0]?.name || 'None'
+    })
+    
+    // Test 2: Query with search
+    const result2 = await PackageService.getMany({
+      platform_id,
+      search: 'firefox',
+      limit: 10
+    })
+    
+    console.log('🐛 Debug: Search query result:', {
+      total: result2.total,
+      packageCount: result2.packages.length,
+      firstPackage: result2.packages[0]?.name || 'None'
+    })
+    
+    // Test 3: Raw SQL query
+    const rawQuery = `
+      SELECT COUNT(*) as total, 
+             MIN(p.name) as first_package
+      FROM packages p 
+      WHERE p.platform_id = $1 AND p.is_active = true
+    `
+    
+    const rawResult = await query(rawQuery, [platform_id])
+    console.log('🐛 Debug: Raw SQL result:', rawResult.rows[0])
+    
+    return NextResponse.json({
+      simpleQuery: {
+        total: result1.total,
+        packages: result1.packages.slice(0, 3).map(p => ({
+          id: p.id,
+          name: p.name,
+          version: p.version
+        }))
+      },
+      searchQuery: {
+        total: result2.total,
+        packages: result2.packages.slice(0, 3).map(p => ({
+          id: p.id,
+          name: p.name,
+          version: p.version
+        }))
+      },
+      rawSql: rawResult.rows[0]
+    })
+    
+  } catch (error) {
+    console.error('🐛 Debug: Error:', error)
+    return NextResponse.json({
+      error: error instanceof Error ? error.message : 'Unknown error',
+      stack: error instanceof Error ? error.stack : undefined
+    }, { status: 500 })
+  }
+}
@@ -0,0 +1,18 @@
+import { NextResponse } from 'next/server'
+import { PlatformInitializer } from '@/services/platformInitializer'
+
+export async function POST() {
+  try {
+    await PlatformInitializer.initializePlatforms()
+    return NextResponse.json({ 
+      success: true, 
+      message: 'Platforms initialized successfully' 
+    })
+  } catch (error) {
+    console.error('Error initializing platforms:', error)
+    return NextResponse.json(
+      { error: 'Failed to initialize platforms' },
+      { status: 500 }
+    )
+  }
+}
@@ -0,0 +1,93 @@
+import { NextRequest } from 'next/server'
+import { SimplePackageFetcher } from '@/services/simplePackageFetcher'
+import { PlatformInitializer } from '@/services/platformInitializer'
+import { setSyncInProgress, setSyncProgress } from '@/app/api/sync-status/route'
+
+export async function GET(request: NextRequest) {
+  const encoder = new TextEncoder()
+  let cancelled = false
+  
+  // Set sync as in progress
+  setSyncInProgress(true)
+  
+  // Create a readable stream for SSE
+  const stream = new ReadableStream({
+    async start(controller) {
+      try {
+        // Helper function to send progress updates
+        const sendProgress = (message: string, progress?: number, total?: number) => {
+          if (cancelled) return
+          
+          const data = JSON.stringify({ 
+            message, 
+            progress, 
+            total,
+            timestamp: new Date().toISOString()
+          })
+          controller.enqueue(encoder.encode(`data: ${data}\n\n`))
+          setSyncProgress(message, progress, total)
+        }
+
+        sendProgress('🔄 Starting package synchronization...')
+
+        // Initialize platforms
+        sendProgress('📋 Initializing platforms...')
+        await PlatformInitializer.initializePlatforms()
+        sendProgress('✅ Platforms initialized')
+
+        // Fetch Debian packages
+        sendProgress('🐧 Fetching Debian packages...')
+        await SimplePackageFetcher.fetchDebianPackagesWithProgress(sendProgress)
+        
+        if (cancelled) {
+          sendProgress('❌ Sync cancelled during Debian processing')
+          controller.close()
+          return
+        }
+        
+        sendProgress('✅ Debian packages completed')
+
+        // Fetch Ubuntu packages
+        sendProgress('🐧 Fetching Ubuntu packages...')
+        await SimplePackageFetcher.fetchUbuntuPackagesWithProgress(sendProgress)
+        
+        if (cancelled) {
+          sendProgress('❌ Sync cancelled during Ubuntu processing')
+          controller.close()
+          return
+        }
+        
+        sendProgress('✅ Ubuntu packages completed')
+
+        // Final success message
+        sendProgress('🎉 All packages synchronized successfully!')
+
+        // Close the stream
+        controller.close()
+        setSyncInProgress(false)
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error'
+        const errorData = JSON.stringify({ 
+          error: errorMessage,
+          timestamp: new Date().toISOString()
+        })
+        controller.enqueue(encoder.encode(`data: ${errorData}\n\n`))
+        controller.close()
+        setSyncInProgress(false)
+      }
+    },
+    
+    cancel() {
+      cancelled = true
+      setSyncInProgress(false)
+    }
+  })
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive',
+    },
+  })
+}
@@ -0,0 +1,40 @@
+import { NextRequest, NextResponse } from 'next/server'
+
+// Global sync state (in production, use Redis or database)
+let syncInProgress = false
+let syncProgress = { message: '', progress: 0, total: 100 }
+
+export async function GET() {
+  return NextResponse.json({
+    inProgress: syncInProgress,
+    progress: syncProgress
+  })
+}
+
+export async function POST(request: NextRequest) {
+  const body = await request.json()
+  const { action } = body
+
+  if (action === 'cancel') {
+    syncInProgress = false
+    syncProgress = { message: 'Sync cancelled', progress: 0, total: 100 }
+    return NextResponse.json({ message: 'Sync cancelled successfully' })
+  }
+
+  if (action === 'start') {
+    syncInProgress = true
+    syncProgress = { message: 'Starting sync...', progress: 0, total: 100 }
+    return NextResponse.json({ message: 'Sync started' })
+  }
+
+  return NextResponse.json({ error: 'Invalid action' }, { status: 400 })
+}
+
+// Export functions for other endpoints to use
+export function setSyncProgress(message: string, progress?: number, total?: number) {
+  syncProgress = { message, progress: progress || 0, total: total || 100 }
+}
+
+export function setSyncInProgress(inProgress: boolean) {
+  syncInProgress = inProgress
+}
@@ -1,20 +1,90 @@
 import { NextRequest, NextResponse } from 'next/server'
 import { MetadataFetcher } from '@/services/metadataFetcher'
+import { DebianPackageFetcher } from '@/services/debianPackageFetcher'
+import { PackageFetcherV2 } from '@/services/packageFetcherV2'
+import { SimplePackageFetcher } from '@/services/simplePackageFetcher'
+import { PlatformInitializer } from '@/services/platformInitializer'

 export async function POST(request: NextRequest) {
  try {
    const body = await request.json()
-    const { platform_id, all_platforms } = body
+    const { platform_id, all_platforms, source } = body

-    if (all_platforms) {
-      // Sync all platforms
+    // Initialize platforms first
+    await PlatformInitializer.initializePlatforms()
+
+    if (source === 'debian-simple') {
+      // Sync Debian packages (simple text parsing, no gzip)
+      await SimplePackageFetcher.fetchDebianPackages()
+      return NextResponse.json({ 
+        message: 'Debian packages synced (simple text parsing)',
+        timestamp: new Date().toISOString()
+      })
+    } else if (source === 'ubuntu-simple') {
+      // Sync Ubuntu packages (simple text parsing, no gzip)
+      await SimplePackageFetcher.fetchUbuntuPackages()
+      return NextResponse.json({ 
+        message: 'Ubuntu packages synced (simple text parsing)',
+        timestamp: new Date().toISOString()
+      })
+    } else if (source === 'all-simple') {
+      // Sync all Debian-based packages (simple text parsing)
+      await SimplePackageFetcher.syncAll()
+      return NextResponse.json({ 
+        message: 'All Debian-based packages synced (simple text parsing)',
+        timestamp: new Date().toISOString()
+      })
+    } else if (source === 'debian-official-v2') {
+      // Sync Debian packages from official repository (v2 with better error handling)
+      await PackageFetcherV2.fetchDebianPackages()
+      return NextResponse.json({ 
+        message: 'Debian packages synced from official repository (v2)',
+        timestamp: new Date().toISOString()
+      })
+    } else if (source === 'ubuntu-official-v2') {
+      // Sync Ubuntu packages from official repository (v2 with better error handling)
+      await PackageFetcherV2.fetchUbuntuPackages()
+      return NextResponse.json({ 
+        message: 'Ubuntu packages synced from official repository (v2)',
+        timestamp: new Date().toISOString()
+      })
+    } else if (source === 'all-official-v2') {
+      // Sync all Debian-based packages from official repositories (v2)
+      await PackageFetcherV2.syncAll()
+      return NextResponse.json({ 
+        message: 'All Debian-based packages synced from official repositories (v2)',
+        timestamp: new Date().toISOString()
+      })
+    } else if (source === 'debian-official') {
+      // Sync Debian packages from official repository
+      await DebianPackageFetcher.fetchDebianPackages()
+      return NextResponse.json({ 
+        message: 'Debian packages synced from official repository',
+        timestamp: new Date().toISOString()
+      })
+    } else if (source === 'ubuntu-official') {
+      // Sync Ubuntu packages from official repository
+      await DebianPackageFetcher.fetchUbuntuPackages()
+      return NextResponse.json({ 
+        message: 'Ubuntu packages synced from official repository',
+        timestamp: new Date().toISOString()
+      })
+    } else if (source === 'all-official') {
+      // Sync all Debian-based packages from official repositories
+      await DebianPackageFetcher.syncAll()
+      return NextResponse.json({ 
+        message: 'All Debian-based packages synced from official repositories',
+        timestamp: new Date().toISOString()
+      })
+    } else if (all_platforms) {
+      // Sync all platforms (legacy method)
      await MetadataFetcher.syncAllPlatforms()
      return NextResponse.json({ 
        message: 'All platforms synced successfully',
        timestamp: new Date().toISOString()
      })
    } else if (platform_id) {
-      // Sync specific platform
+      // Sync specific platform (legacy method)
      await MetadataFetcher.syncPlatform(platform_id)
      return NextResponse.json({ 
        message: `Platform ${platform_id} synced successfully`,
@@ -22,7 +92,7 @@ export async function POST(request: NextRequest) {
      })
    } else {
      return NextResponse.json(
-        { error: 'Either platform_id or all_platforms must be specified' },
+        { error: 'Either platform_id, all_platforms, or source must be specified' },
        { status: 400 }
      )
    }
@@ -0,0 +1,46 @@
+import { NextResponse } from 'next/server'
+import { query } from '@/lib/database/config'
+
+export async function GET() {
+  try {
+    // Test 1: Database connection
+    const dbTest = await query('SELECT NOW() as current_time', [])
+    
+    // Test 2: Packages table count
+    const packageCount = await query('SELECT COUNT(*) as count FROM packages', [])
+    
+    // Test 3: Debian packages count
+    const debianCount = await query('SELECT COUNT(*) as count FROM packages WHERE platform_id = $1', ['debian'])
+    
+    // Test 4: Active packages count
+    const activeCount = await query('SELECT COUNT(*) as count FROM packages WHERE is_active = true', [])
+    
+    // Test 5: Sample packages
+    const samplePackages = await query(`
+      SELECT id, name, version, platform_id 
+      FROM packages 
+      WHERE platform_id = $1 AND is_active = true 
+      LIMIT 5
+    `, ['debian'])
+    
+    return NextResponse.json({
+      database: {
+        connected: true,
+        currentTime: dbTest.rows[0].current_time
+      },
+      packages: {
+        total: packageCount.rows[0].count,
+        debian: debianCount.rows[0].count,
+        active: activeCount.rows[0].count,
+        sample: samplePackages.rows
+      }
+    })
+    
+  } catch (error) {
+    console.error('Database test error:', error)
+    return NextResponse.json({
+      error: error instanceof Error ? error.message : 'Unknown error',
+      database: { connected: false }
+    }, { status: 500 })
+  }
+}
@@ -0,0 +1,84 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { PackageFetcherV2 } from '@/services/packageFetcherV2'
+import { PlatformInitializer } from '@/services/platformInitializer'
+
+export async function GET() {
+  try {
+    console.log('🧪 Starting test sync...')
+    
+    // Test 1: Initialize platforms
+    await PlatformInitializer.initializePlatforms()
+    console.log('✅ Platforms initialized')
+    
+    // Test 2: Try to fetch a small sample
+    console.log('🔄 Testing Debian package fetch...')
+    
+    // Just test the fetch and parse, don't store in DB
+    const response = await fetch('https://packages.debian.org/stable/allpackages?format=txt.gz', {
+      headers: {
+        'Accept-Encoding': 'gzip, deflate',
+        'User-Agent': 'RepoHub-Package-Fetcher/1.0'
+      }
+    })
+    
+    if (!response.ok) {
+      throw new Error(`Failed to fetch: ${response.statusText}`)
+    }
+    
+    console.log('✅ Response received, size:', response.headers.get('content-length'))
+    
+    const buffer = await response.arrayBuffer()
+    console.log('✅ Buffer received, size:', buffer.byteLength)
+    
+    // Check magic bytes
+    const bytes = new Uint8Array(buffer.slice(0, 2))
+    console.log('🔍 Magic bytes:', bytes[0].toString(16), bytes[1].toString(16))
+    
+    // Try decompression
+    const { gunzip } = await import('zlib')
+    const decompressed = await new Promise<Buffer>((resolve, reject) => {
+      gunzip(new Uint8Array(buffer), (err, result) => {
+        if (err) {
+          console.error('❌ Gunzip error:', err)
+          reject(err)
+        } else {
+          console.log('✅ Gunzip successful, size:', result.length)
+          resolve(result)
+        }
+      })
+    })
+    
+    const text = decompressed.toString('utf-8')
+    console.log('✅ Text decoded, length:', text.length)
+    
+    // Parse first 10 lines
+    const lines = text.split('\n').slice(0, 20)
+    console.log('📋 First 20 lines:')
+    lines.forEach((line, i) => console.log(`${i + 1}: ${line}`))
+    
+    // Test parsing
+    const samplePackages = PackageFetcherV2['parseDebianPackageList'](text)
+    console.log('📊 Sample packages parsed:', samplePackages.length)
+    
+    if (samplePackages.length > 0) {
+      console.log('📦 First package:', samplePackages[0])
+    }
+    
+    return NextResponse.json({
+      success: true,
+      message: 'Test completed successfully',
+      bufferSize: buffer.byteLength,
+      textSize: text.length,
+      samplePackages: samplePackages.length,
+      firstPackage: samplePackages[0] || null
+    })
+    
+  } catch (error) {
+    console.error('❌ Test failed:', error)
+    return NextResponse.json({
+      success: false,
+      error: error instanceof Error ? error.message : 'Unknown error',
+      stack: error instanceof Error ? error.stack : undefined
+    }, { status: 500 })
+  }
+}