RepoHub/src/services/debianPackageFetcher.ts

// Using Node.js built-in fetch instead of undici for better compatibility
// import { fetch } from 'undici'
import { PackageService } from './packageService'
import { CreatePackageInput } from '@/models/Package'

interface DebianPackageInfo {
  name: string
  version: string
  description?: string
  section?: string
  architecture?: string[]
}

export class DebianPackageFetcher {
  private static readonly DEBIAN_PACKAGES_URL = 'https://packages.debian.org/stable/allpackages?format=txt.gz'
  private static readonly UBUNTU_PACKAGES_URL = 'https://packages.ubuntu.com/noble/allpackages?format=txt.gz'

  // Fetch and parse Debian packages
  static async fetchDebianPackages(): Promise<void> {
    console.log('🔄 Fetching Debian packages from official repository...')
    
    try {
      const response = await fetch(this.DEBIAN_PACKAGES_URL)
      
      if (!response.ok) {
        throw new Error(`Failed to fetch Debian packages: ${response.statusText}`)
      }

      // Get the compressed data
      const buffer = await response.arrayBuffer()
      const compressed = new Uint8Array(buffer)
      
      // Decompress if input is gzipped (magic 0x1f 0x8b); otherwise treat as plain text
      let text: string
      if (compressed.length >= 2 && compressed[0] === 0x1f && compressed[1] === 0x8b) {
        const { gunzip } = await import('zlib')
        const decompressed = await new Promise<Buffer>((resolve, reject) => {
          gunzip(compressed, (err, result) => {
            if (err) {
              console.error('Gunzip error:', err)
              reject(err)
            } else {
              resolve(result)
            }
          })
        })
        text = decompressed.toString('utf-8')
      } else {
        text = Buffer.from(compressed).toString('utf-8')
      }
      const packages = this.parseDebianPackageList(text)
      
      console.log(`📊 Parsed ${packages.length} Debian packages`)
      
      // Store in database
      await this.storePackages('debian', packages)
      
      console.log('✅ Debian packages fetched and stored successfully')
    } catch (error) {
      console.error('❌ Error fetching Debian packages:', error)
      throw error
    }
  }

  // Fetch and parse Ubuntu packages
  static async fetchUbuntuPackages(): Promise<void> {
    console.log('🔄 Fetching Ubuntu packages from official repository...')
    
    try {
      const response = await fetch(this.UBUNTU_PACKAGES_URL)
      
      if (!response.ok) {
        throw new Error(`Failed to fetch Ubuntu packages: ${response.statusText}`)
      }

      // Get the compressed data
      const buffer = await response.arrayBuffer()
      const compressed = new Uint8Array(buffer)
      
      // Decompress if input is gzipped (magic 0x1f 0x8b); otherwise treat as plain text
      let text: string
      if (compressed.length >= 2 && compressed[0] === 0x1f && compressed[1] === 0x8b) {
        const { gunzip } = await import('zlib')
        const decompressed = await new Promise<Buffer>((resolve, reject) => {
          gunzip(compressed, (err, result) => {
            if (err) {
              console.error('Gunzip error:', err)
              reject(err)
            } else {
              resolve(result)
            }
          })
        })
        text = decompressed.toString('utf-8')
      } else {
        text = Buffer.from(compressed).toString('utf-8')
      }
      const packages = this.parseUbuntuPackageList(text)
      
      console.log(`📊 Parsed ${packages.length} Ubuntu packages`)
      
      // Store in database
      await this.storePackages('ubuntu', packages)
      
      console.log('✅ Ubuntu packages fetched and stored successfully')
    } catch (error) {
      console.error('❌ Error fetching Ubuntu packages:', error)
      throw error
    }
  }

  // Parse Debian package list format
  private static parseDebianPackageList(text: string): CreatePackageInput[] {
    const packages: CreatePackageInput[] = []
    const lines = text.split('\n')
    
    // Skip header lines until we find package entries
    let packageStartIndex = 0
    for (let i = 0; i < lines.length; i++) {
      if (lines[i].match(/^[a-z0-9][a-z0-9+.-]*\s+\([^)]+\)/)) {
        packageStartIndex = i
        break
      }
    }

    for (let i = packageStartIndex; i < lines.length; i++) {
      const line = lines[i].trim()
      if (!line) continue

      // Parse package line format: "package-name (version) description"
      const match = line.match(/^([a-z0-9][a-z0-9+.-]*)\s+\(([^)]+)\)\s*(.+)?$/)
      
      if (match) {
        const [, name, version, description] = match
        
        packages.push({
          id: `debian-${name}`,
          name: name,
          description: description?.trim(),
          version: version.trim(),
          platform_id: 'debian',
          type: this.determinePackageType(name, description),
          repository: 'official',
          popularity_score: Math.floor(Math.random() * 100) // Placeholder - would be calculated from download stats
        })
      }
    }

    return packages
  }

  // Parse Ubuntu package list format (with repository info)
  private static parseUbuntuPackageList(text: string): CreatePackageInput[] {
    const packages: CreatePackageInput[] = []
    const lines = text.split('\n')
    
    // Skip header lines until we find package entries
    let packageStartIndex = 0
    for (let i = 0; i < lines.length; i++) {
      if (lines[i].match(/^[a-z0-9][a-z0-9+.-]*\s+\([^)]+\)\s+\[.*\]/)) {
        packageStartIndex = i
        break
      }
    }

    for (let i = packageStartIndex; i < lines.length; i++) {
      const line = lines[i].trim()
      if (!line) continue

      // Parse package line format: "package-name (version) [repository] description"
      const match = line.match(/^([a-z0-9][a-z0-9+.-]*)\s+\(([^)]+)\)\s+\[([^\]]+)\]\s*(.+)?$/)
      
      if (match) {
        const [, name, version, repository, description] = match
        
        packages.push({
          id: `ubuntu-${name}`,
          name: name,
          description: description?.trim(),
          version: version.trim(),
          platform_id: 'ubuntu',
          type: this.determinePackageType(name, description),
          repository: this.mapUbuntuRepository(repository),
          popularity_score: Math.floor(Math.random() * 100) // Placeholder - would be calculated from download stats
        })
      }
    }

    return packages
  }

  // Map Ubuntu repository to our enum
  private static mapUbuntuRepository(repo: string): 'official' | 'third-party' {
    switch (repo.toLowerCase()) {
      case 'main':
      case 'restricted':
      case 'universe':
      case 'multiverse':
        return 'official'
      default:
        return 'third-party'
    }
  }

  // Determine if package is GUI or CLI based on name and description
  private static determinePackageType(name: string, description?: string): 'gui' | 'cli' {
    const guiKeywords = [
      'gui', 'gtk', 'qt', 'x11', 'desktop', 'window', 'display',
      'graphical', 'visual', 'image', 'video', 'audio', 'media',
      'browser', 'editor', 'viewer', 'player', 'manager',
      'game', 'games', 'steam', 'wine'
    ]
    
    const cliKeywords = [
      'cli', 'command', 'terminal', 'console', 'shell', 'bash',
      'tool', 'utility', 'daemon', 'service', 'server', 'client',
      'lib', 'dev', 'debug', 'build', 'compile'
    ]
    
    const searchText = `${name} ${description || ''}`.toLowerCase()
    
    for (const keyword of guiKeywords) {
      if (searchText.includes(keyword)) return 'gui'
    }
    
    for (const keyword of cliKeywords) {
      if (searchText.includes(keyword)) return 'cli'
    }
    
    // Default to CLI for system packages
    return 'cli'
  }

  // Store packages in database
  private static async storePackages(platformId: string, packages: CreatePackageInput[]): Promise<void> {
    console.log(`💾 Storing ${packages.length} packages for platform ${platformId}...`)
    
    let added = 0
    let updated = 0
    let batchSize = 100
    let processed = 0

    // Process in batches to avoid overwhelming the database
    for (let i = 0; i < packages.length; i += batchSize) {
      const batch = packages.slice(i, i + batchSize)
      
      for (const pkg of batch) {
        try {
          // Check if package exists
          const existing = await PackageService.getById(pkg.id)
          
          if (existing) {
            // Update existing package
            await PackageService.update(pkg.id, {
              description: pkg.description,
              version: pkg.version,
              type: pkg.type,
              repository: pkg.repository,
              popularity_score: pkg.popularity_score
            })
            updated++
          } else {
            // Create new package
            await PackageService.create(pkg)
            added++
          }
        } catch (error) {
          console.error(`Error storing package ${pkg.id}:`, error)
        }
        
        processed++
        
        // Progress indicator
        if (processed % 100 === 0) {
          console.log(`📈 Processed ${processed}/${packages.length} packages...`)
        }
      }
    }
    
    console.log(`📈 Final: Added: ${added}, Updated: ${updated}`)
  }

  // Sync both Debian and Ubuntu packages
  static async syncAll(): Promise<void> {
    console.log('🔄 Starting sync of all Debian-based packages...')
    
    try {
      await this.fetchDebianPackages()
      await this.fetchUbuntuPackages()
      
      console.log('✅ All Debian-based packages synced successfully')
    } catch (error) {
      console.error('❌ Error during sync:', error)
      throw error
    }
  }
}