mirror of
https://github.com/yusufipk/RepoHub.git
synced 2026-09-11 10:36:07 +00:00
feat: add multiple package sync sources and API integration
- Added undici dependency and implemented three package fetcher variants (simple text parsing, v2 with better error handling, and official repository sync) - Integrated real API calls in PlatformSelector and PackageBrowser components with fallback to mock data - Extended sync API route to support multiple data sources (debian-simple, ubuntu-simple, debian-official-v2, etc.) with platform initialization
This commit is contained in:
Yusuf İpek
@@ -0,0 +1,295 @@
|
||||
// Using Node.js built-in fetch instead of undici for better compatibility
|
||||
// import { fetch } from 'undici'
|
||||
import { PackageService } from './packageService'
|
||||
import { CreatePackageInput } from '@/models/Package'
|
||||
|
||||
interface DebianPackageInfo {
|
||||
name: string
|
||||
version: string
|
||||
description?: string
|
||||
section?: string
|
||||
architecture?: string[]
|
||||
}
|
||||
|
||||
export class DebianPackageFetcher {
|
||||
private static readonly DEBIAN_PACKAGES_URL = 'https://packages.debian.org/stable/allpackages?format=txt.gz'
|
||||
private static readonly UBUNTU_PACKAGES_URL = 'https://packages.ubuntu.com/noble/allpackages?format=txt.gz'
|
||||
|
||||
// Fetch and parse Debian packages
|
||||
static async fetchDebianPackages(): Promise<void> {
|
||||
console.log('🔄 Fetching Debian packages from official repository...')
|
||||
|
||||
try {
|
||||
const response = await fetch(this.DEBIAN_PACKAGES_URL)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Debian packages: ${response.statusText}`)
|
||||
}
|
||||
|
||||
// Get the compressed data
|
||||
const buffer = await response.arrayBuffer()
|
||||
const compressed = new Uint8Array(buffer)
|
||||
|
||||
// Decompress using Node.js zlib with proper error handling
|
||||
const { gunzip } = await import('zlib')
|
||||
const decompressed = await new Promise<Buffer>((resolve, reject) => {
|
||||
gunzip(compressed, {
|
||||
finishFlush: 1, // Z_SYNC_FLUSH
|
||||
windowBits: 15 + 16 // Enable gzip header decoding
|
||||
}, (err, result) => {
|
||||
if (err) {
|
||||
console.error('Gunzip error:', err)
|
||||
reject(err)
|
||||
} else {
|
||||
resolve(result)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
const text = decompressed.toString('utf-8')
|
||||
const packages = this.parseDebianPackageList(text)
|
||||
|
||||
console.log(`📊 Parsed ${packages.length} Debian packages`)
|
||||
|
||||
// Store in database
|
||||
await this.storePackages('debian', packages)
|
||||
|
||||
console.log('✅ Debian packages fetched and stored successfully')
|
||||
} catch (error) {
|
||||
console.error('❌ Error fetching Debian packages:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch and parse Ubuntu packages
|
||||
static async fetchUbuntuPackages(): Promise<void> {
|
||||
console.log('🔄 Fetching Ubuntu packages from official repository...')
|
||||
|
||||
try {
|
||||
const response = await fetch(this.UBUNTU_PACKAGES_URL)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Ubuntu packages: ${response.statusText}`)
|
||||
}
|
||||
|
||||
// Get the compressed data
|
||||
const buffer = await response.arrayBuffer()
|
||||
const compressed = new Uint8Array(buffer)
|
||||
|
||||
// Decompress using Node.js zlib with proper error handling
|
||||
const { gunzip } = await import('zlib')
|
||||
const decompressed = await new Promise<Buffer>((resolve, reject) => {
|
||||
gunzip(compressed, {
|
||||
finishFlush: 1, // Z_SYNC_FLUSH
|
||||
windowBits: 15 + 16 // Enable gzip header decoding
|
||||
}, (err, result) => {
|
||||
if (err) {
|
||||
console.error('Gunzip error:', err)
|
||||
reject(err)
|
||||
} else {
|
||||
resolve(result)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
const text = decompressed.toString('utf-8')
|
||||
const packages = this.parseUbuntuPackageList(text)
|
||||
|
||||
console.log(`📊 Parsed ${packages.length} Ubuntu packages`)
|
||||
|
||||
// Store in database
|
||||
await this.storePackages('ubuntu', packages)
|
||||
|
||||
console.log('✅ Ubuntu packages fetched and stored successfully')
|
||||
} catch (error) {
|
||||
console.error('❌ Error fetching Ubuntu packages:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Parse Debian package list format
|
||||
private static parseDebianPackageList(text: string): CreatePackageInput[] {
|
||||
const packages: CreatePackageInput[] = []
|
||||
const lines = text.split('\n')
|
||||
|
||||
// Skip header lines until we find package entries
|
||||
let packageStartIndex = 0
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].match(/^[a-z0-9][a-z0-9+.-]*\s+\([^)]+\)/)) {
|
||||
packageStartIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = packageStartIndex; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue
|
||||
|
||||
// Parse package line format: "package-name (version) description"
|
||||
const match = line.match(/^([a-z0-9][a-z0-9+.-]*)\s+\(([^)]+)\)\s*(.+)?$/)
|
||||
|
||||
if (match) {
|
||||
const [, name, version, description] = match
|
||||
|
||||
packages.push({
|
||||
id: `debian-${name}`,
|
||||
name: name,
|
||||
description: description?.trim(),
|
||||
version: version.trim(),
|
||||
platform_id: 'debian',
|
||||
type: this.determinePackageType(name, description),
|
||||
repository: 'official',
|
||||
popularity_score: Math.floor(Math.random() * 100) // Placeholder - would be calculated from download stats
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return packages
|
||||
}
|
||||
|
||||
// Parse Ubuntu package list format (with repository info)
|
||||
private static parseUbuntuPackageList(text: string): CreatePackageInput[] {
|
||||
const packages: CreatePackageInput[] = []
|
||||
const lines = text.split('\n')
|
||||
|
||||
// Skip header lines until we find package entries
|
||||
let packageStartIndex = 0
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].match(/^[a-z0-9][a-z0-9+.-]*\s+\([^)]+\)\s+\[.*\]/)) {
|
||||
packageStartIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = packageStartIndex; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue
|
||||
|
||||
// Parse package line format: "package-name (version) [repository] description"
|
||||
const match = line.match(/^([a-z0-9][a-z0-9+.-]*)\s+\(([^)]+)\)\s+\[([^\]]+)\]\s*(.+)?$/)
|
||||
|
||||
if (match) {
|
||||
const [, name, version, repository, description] = match
|
||||
|
||||
packages.push({
|
||||
id: `ubuntu-${name}`,
|
||||
name: name,
|
||||
description: description?.trim(),
|
||||
version: version.trim(),
|
||||
platform_id: 'ubuntu',
|
||||
type: this.determinePackageType(name, description),
|
||||
repository: this.mapUbuntuRepository(repository),
|
||||
popularity_score: Math.floor(Math.random() * 100) // Placeholder - would be calculated from download stats
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return packages
|
||||
}
|
||||
|
||||
// Map Ubuntu repository to our enum
|
||||
private static mapUbuntuRepository(repo: string): 'official' | 'third-party' {
|
||||
switch (repo.toLowerCase()) {
|
||||
case 'main':
|
||||
case 'restricted':
|
||||
case 'universe':
|
||||
case 'multiverse':
|
||||
return 'official'
|
||||
default:
|
||||
return 'third-party'
|
||||
}
|
||||
}
|
||||
|
||||
// Determine if package is GUI or CLI based on name and description
|
||||
private static determinePackageType(name: string, description?: string): 'gui' | 'cli' {
|
||||
const guiKeywords = [
|
||||
'gui', 'gtk', 'qt', 'x11', 'desktop', 'window', 'display',
|
||||
'graphical', 'visual', 'image', 'video', 'audio', 'media',
|
||||
'browser', 'editor', 'viewer', 'player', 'manager',
|
||||
'game', 'games', 'steam', 'wine'
|
||||
]
|
||||
|
||||
const cliKeywords = [
|
||||
'cli', 'command', 'terminal', 'console', 'shell', 'bash',
|
||||
'tool', 'utility', 'daemon', 'service', 'server', 'client',
|
||||
'lib', 'dev', 'debug', 'build', 'compile'
|
||||
]
|
||||
|
||||
const searchText = `${name} ${description || ''}`.toLowerCase()
|
||||
|
||||
for (const keyword of guiKeywords) {
|
||||
if (searchText.includes(keyword)) return 'gui'
|
||||
}
|
||||
|
||||
for (const keyword of cliKeywords) {
|
||||
if (searchText.includes(keyword)) return 'cli'
|
||||
}
|
||||
|
||||
// Default to CLI for system packages
|
||||
return 'cli'
|
||||
}
|
||||
|
||||
// Store packages in database
|
||||
private static async storePackages(platformId: string, packages: CreatePackageInput[]): Promise<void> {
|
||||
console.log(`💾 Storing ${packages.length} packages for platform ${platformId}...`)
|
||||
|
||||
let added = 0
|
||||
let updated = 0
|
||||
let batchSize = 100
|
||||
let processed = 0
|
||||
|
||||
// Process in batches to avoid overwhelming the database
|
||||
for (let i = 0; i < packages.length; i += batchSize) {
|
||||
const batch = packages.slice(i, i + batchSize)
|
||||
|
||||
for (const pkg of batch) {
|
||||
try {
|
||||
// Check if package exists
|
||||
const existing = await PackageService.getById(pkg.id)
|
||||
|
||||
if (existing) {
|
||||
// Update existing package
|
||||
await PackageService.update(pkg.id, {
|
||||
description: pkg.description,
|
||||
version: pkg.version,
|
||||
type: pkg.type,
|
||||
repository: pkg.repository,
|
||||
popularity_score: pkg.popularity_score
|
||||
})
|
||||
updated++
|
||||
} else {
|
||||
// Create new package
|
||||
await PackageService.create(pkg)
|
||||
added++
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error storing package ${pkg.id}:`, error)
|
||||
}
|
||||
|
||||
processed++
|
||||
|
||||
// Progress indicator
|
||||
if (processed % 100 === 0) {
|
||||
console.log(`📈 Processed ${processed}/${packages.length} packages...`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`📈 Final: Added: ${added}, Updated: ${updated}`)
|
||||
}
|
||||
|
||||
// Sync both Debian and Ubuntu packages
|
||||
static async syncAll(): Promise<void> {
|
||||
console.log('🔄 Starting sync of all Debian-based packages...')
|
||||
|
||||
try {
|
||||
await this.fetchDebianPackages()
|
||||
await this.fetchUbuntuPackages()
|
||||
|
||||
console.log('✅ All Debian-based packages synced successfully')
|
||||
} catch (error) {
|
||||
console.error('❌ Error during sync:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
import { PackageService } from './packageService'
|
||||
import { CreatePackageInput } from '@/models/Package'
|
||||
|
||||
export class PackageFetcherV2 {
|
||||
private static readonly DEBIAN_PACKAGES_URL = 'https://packages.debian.org/stable/allpackages?format=txt.gz'
|
||||
private static readonly UBUNTU_PACKAGES_URL = 'https://packages.ubuntu.com/noble/allpackages?format=txt.gz'
|
||||
|
||||
// Fetch and parse Debian packages with better error handling
|
||||
static async fetchDebianPackages(): Promise<void> {
|
||||
console.log('🔄 Fetching Debian packages from official repository...')
|
||||
|
||||
try {
|
||||
// Use Node.js built-in fetch with proper headers
|
||||
const response = await fetch(this.DEBIAN_PACKAGES_URL, {
|
||||
headers: {
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'User-Agent': 'RepoHub-Package-Fetcher/1.0'
|
||||
}
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Debian packages: ${response.statusText}`)
|
||||
}
|
||||
|
||||
// Get the response as buffer
|
||||
const buffer = await response.arrayBuffer()
|
||||
|
||||
// Try to decompress with multiple methods
|
||||
let text: string
|
||||
|
||||
try {
|
||||
// Method 1: Try direct text (in case it's not compressed)
|
||||
text = new TextDecoder().decode(buffer)
|
||||
|
||||
// Check if it looks like gzip (starts with gzip magic bytes)
|
||||
const bytes = new Uint8Array(buffer.slice(0, 2))
|
||||
if (bytes[0] === 0x1f && bytes[1] === 0x8b) {
|
||||
throw new Error('Data is compressed, need to decompress')
|
||||
}
|
||||
} catch {
|
||||
// Method 2: Try zlib decompression
|
||||
try {
|
||||
const { gunzip } = await import('zlib')
|
||||
const decompressed = await new Promise<Buffer>((resolve, reject) => {
|
||||
gunzip(new Uint8Array(buffer), (err, result) => {
|
||||
if (err) reject(err)
|
||||
else resolve(result)
|
||||
})
|
||||
})
|
||||
text = decompressed.toString('utf-8')
|
||||
} catch (gzipError) {
|
||||
// Method 3: Try streaming decompression
|
||||
try {
|
||||
const { createGunzip } = await import('zlib')
|
||||
const { Readable } = await import('stream')
|
||||
const { pipeline } = await import('stream/promises')
|
||||
|
||||
const readable = Readable.from([new Uint8Array(buffer)])
|
||||
const gunzip = createGunzip()
|
||||
const chunks: Buffer[] = []
|
||||
|
||||
gunzip.on('data', (chunk) => chunks.push(chunk))
|
||||
|
||||
await pipeline(readable, gunzip)
|
||||
text = Buffer.concat(chunks).toString('utf-8')
|
||||
} catch (streamError) {
|
||||
throw new Error(`All decompression methods failed. Gzip error: ${gzipError}, Stream error: ${streamError}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const packages = this.parseDebianPackageList(text)
|
||||
console.log(`📊 Parsed ${packages.length} Debian packages`)
|
||||
|
||||
// Store in database
|
||||
await this.storePackages('debian', packages)
|
||||
|
||||
console.log('✅ Debian packages fetched and stored successfully')
|
||||
} catch (error) {
|
||||
console.error('❌ Error fetching Debian packages:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch and parse Ubuntu packages with better error handling
|
||||
static async fetchUbuntuPackages(): Promise<void> {
|
||||
console.log('🔄 Fetching Ubuntu packages from official repository...')
|
||||
|
||||
try {
|
||||
// Use Node.js built-in fetch with proper headers
|
||||
const response = await fetch(this.UBUNTU_PACKAGES_URL, {
|
||||
headers: {
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'User-Agent': 'RepoHub-Package-Fetcher/1.0'
|
||||
}
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Ubuntu packages: ${response.statusText}`)
|
||||
}
|
||||
|
||||
// Get the response as buffer
|
||||
const buffer = await response.arrayBuffer()
|
||||
|
||||
// Try to decompress with multiple methods
|
||||
let text: string
|
||||
|
||||
try {
|
||||
// Method 1: Try direct text (in case it's not compressed)
|
||||
text = new TextDecoder().decode(buffer)
|
||||
|
||||
// Check if it looks like gzip (starts with gzip magic bytes)
|
||||
const bytes = new Uint8Array(buffer.slice(0, 2))
|
||||
if (bytes[0] === 0x1f && bytes[1] === 0x8b) {
|
||||
throw new Error('Data is compressed, need to decompress')
|
||||
}
|
||||
} catch {
|
||||
// Method 2: Try zlib decompression
|
||||
try {
|
||||
const { gunzip } = await import('zlib')
|
||||
const decompressed = await new Promise<Buffer>((resolve, reject) => {
|
||||
gunzip(new Uint8Array(buffer), (err, result) => {
|
||||
if (err) reject(err)
|
||||
else resolve(result)
|
||||
})
|
||||
})
|
||||
text = decompressed.toString('utf-8')
|
||||
} catch (gzipError) {
|
||||
// Method 3: Try streaming decompression
|
||||
try {
|
||||
const { createGunzip } = await import('zlib')
|
||||
const { Readable } = await import('stream')
|
||||
const { pipeline } = await import('stream/promises')
|
||||
|
||||
const readable = Readable.from([new Uint8Array(buffer)])
|
||||
const gunzip = createGunzip()
|
||||
const chunks: Buffer[] = []
|
||||
|
||||
gunzip.on('data', (chunk) => chunks.push(chunk))
|
||||
|
||||
await pipeline(readable, gunzip)
|
||||
text = Buffer.concat(chunks).toString('utf-8')
|
||||
} catch (streamError) {
|
||||
throw new Error(`All decompression methods failed. Gzip error: ${gzipError}, Stream error: ${streamError}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const packages = this.parseUbuntuPackageList(text)
|
||||
console.log(`📊 Parsed ${packages.length} Ubuntu packages`)
|
||||
|
||||
// Store in database
|
||||
await this.storePackages('ubuntu', packages)
|
||||
|
||||
console.log('✅ Ubuntu packages fetched and stored successfully')
|
||||
} catch (error) {
|
||||
console.error('❌ Error fetching Ubuntu packages:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Parse Debian package list format
|
||||
private static parseDebianPackageList(text: string): CreatePackageInput[] {
|
||||
const packages: CreatePackageInput[] = []
|
||||
const lines = text.split('\n')
|
||||
|
||||
// Skip header lines until we find package entries
|
||||
let packageStartIndex = 0
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].match(/^[a-z0-9][a-z0-9+.-]*\s+\([^)]+\)/)) {
|
||||
packageStartIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = packageStartIndex; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue
|
||||
|
||||
// Parse package line format: "package-name (version) description"
|
||||
const match = line.match(/^([a-z0-9][a-z0-9+.-]*)\s+\(([^)]+)\)\s*(.+)?$/)
|
||||
|
||||
if (match) {
|
||||
const [, name, version, description] = match
|
||||
|
||||
packages.push({
|
||||
id: `debian-${name}`,
|
||||
name: name,
|
||||
description: description?.trim(),
|
||||
version: version.trim(),
|
||||
platform_id: 'debian',
|
||||
type: this.determinePackageType(name, description),
|
||||
repository: 'official',
|
||||
popularity_score: Math.floor(Math.random() * 100)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return packages
|
||||
}
|
||||
|
||||
// Parse Ubuntu package list format (with repository info)
|
||||
private static parseUbuntuPackageList(text: string): CreatePackageInput[] {
|
||||
const packages: CreatePackageInput[] = []
|
||||
const lines = text.split('\n')
|
||||
|
||||
// Skip header lines until we find package entries
|
||||
let packageStartIndex = 0
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].match(/^[a-z0-9][a-z0-9+.-]*\s+\([^)]+\)\s+\[.*\]/)) {
|
||||
packageStartIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = packageStartIndex; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue
|
||||
|
||||
// Parse package line format: "package-name (version) [repository] description"
|
||||
const match = line.match(/^([a-z0-9][a-z0-9+.-]*)\s+\(([^)]+)\)\s+\[([^\]]+)\]\s*(.+)?$/)
|
||||
|
||||
if (match) {
|
||||
const [, name, version, repository, description] = match
|
||||
|
||||
packages.push({
|
||||
id: `ubuntu-${name}`,
|
||||
name: name,
|
||||
description: description?.trim(),
|
||||
version: version.trim(),
|
||||
platform_id: 'ubuntu',
|
||||
type: this.determinePackageType(name, description),
|
||||
repository: this.mapUbuntuRepository(repository),
|
||||
popularity_score: Math.floor(Math.random() * 100)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return packages
|
||||
}
|
||||
|
||||
// Map Ubuntu repository to our enum
|
||||
private static mapUbuntuRepository(repo: string): 'official' | 'third-party' {
|
||||
switch (repo.toLowerCase()) {
|
||||
case 'main':
|
||||
case 'restricted':
|
||||
case 'universe':
|
||||
case 'multiverse':
|
||||
return 'official'
|
||||
default:
|
||||
return 'third-party'
|
||||
}
|
||||
}
|
||||
|
||||
// Determine if package is GUI or CLI based on name and description
|
||||
private static determinePackageType(name: string, description?: string): 'gui' | 'cli' {
|
||||
const guiKeywords = [
|
||||
'gui', 'gtk', 'qt', 'x11', 'desktop', 'window', 'display',
|
||||
'graphical', 'visual', 'image', 'video', 'audio', 'media',
|
||||
'browser', 'editor', 'viewer', 'player', 'manager',
|
||||
'game', 'games', 'steam', 'wine'
|
||||
]
|
||||
|
||||
const cliKeywords = [
|
||||
'cli', 'command', 'terminal', 'console', 'shell', 'bash',
|
||||
'tool', 'utility', 'daemon', 'service', 'server', 'client',
|
||||
'lib', 'dev', 'debug', 'build', 'compile'
|
||||
]
|
||||
|
||||
const searchText = `${name} ${description || ''}`.toLowerCase()
|
||||
|
||||
for (const keyword of guiKeywords) {
|
||||
if (searchText.includes(keyword)) return 'gui'
|
||||
}
|
||||
|
||||
for (const keyword of cliKeywords) {
|
||||
if (searchText.includes(keyword)) return 'cli'
|
||||
}
|
||||
|
||||
return 'cli'
|
||||
}
|
||||
|
||||
// Store packages in database
|
||||
private static async storePackages(platformId: string, packages: CreatePackageInput[]): Promise<void> {
|
||||
console.log(`💾 Storing ${packages.length} packages for platform ${platformId}...`)
|
||||
|
||||
let added = 0
|
||||
let updated = 0
|
||||
let batchSize = 100
|
||||
let processed = 0
|
||||
|
||||
for (let i = 0; i < packages.length; i += batchSize) {
|
||||
const batch = packages.slice(i, i + batchSize)
|
||||
|
||||
for (const pkg of batch) {
|
||||
try {
|
||||
const existing = await PackageService.getById(pkg.id)
|
||||
|
||||
if (existing) {
|
||||
await PackageService.update(pkg.id, {
|
||||
description: pkg.description,
|
||||
version: pkg.version,
|
||||
type: pkg.type,
|
||||
repository: pkg.repository,
|
||||
popularity_score: pkg.popularity_score
|
||||
})
|
||||
updated++
|
||||
} else {
|
||||
await PackageService.create(pkg)
|
||||
added++
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error storing package ${pkg.id}:`, error)
|
||||
}
|
||||
|
||||
processed++
|
||||
|
||||
if (processed % 100 === 0) {
|
||||
console.log(`📈 Processed ${processed}/${packages.length} packages...`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`📈 Final: Added: ${added}, Updated: ${updated}`)
|
||||
}
|
||||
|
||||
// Sync both Debian and Ubuntu packages
|
||||
static async syncAll(): Promise<void> {
|
||||
console.log('🔄 Starting sync of all Debian-based packages...')
|
||||
|
||||
try {
|
||||
await this.fetchDebianPackages()
|
||||
await this.fetchUbuntuPackages()
|
||||
|
||||
console.log('✅ All Debian-based packages synced successfully')
|
||||
} catch (error) {
|
||||
console.error('❌ Error during sync:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,10 +43,10 @@ export class PackageService {
|
||||
|
||||
if (search) {
|
||||
whereConditions.push(`(
|
||||
to_tsvector('english', p.name) @@ to_tsquery('english', $${paramIndex}) OR
|
||||
to_tsvector('english', p.description) @@ to_tsquery('english', $${paramIndex})
|
||||
p.name ILIKE $${paramIndex} OR
|
||||
p.description ILIKE $${paramIndex}
|
||||
)`)
|
||||
values.push(search.split(' ').join(' & '))
|
||||
values.push(`%${search}%`)
|
||||
paramIndex++
|
||||
}
|
||||
|
||||
@@ -69,27 +69,26 @@ export class PackageService {
|
||||
const countResult = await query(countQuery, values)
|
||||
const total = parseInt(countResult.rows[0].total)
|
||||
|
||||
// Get packages with joins
|
||||
// Get packages with joins (simplified)
|
||||
const packagesQuery = `
|
||||
SELECT
|
||||
p.*,
|
||||
p.id,
|
||||
p.name,
|
||||
p.description,
|
||||
p.version,
|
||||
p.platform_id,
|
||||
p.type,
|
||||
p.repository,
|
||||
p.popularity_score,
|
||||
p.is_active,
|
||||
p.created_at,
|
||||
p.updated_at,
|
||||
pl.name as platform_name,
|
||||
pl.package_manager as platform_package_manager,
|
||||
pl.icon as platform_icon,
|
||||
c.name as category_name,
|
||||
l.name as license_name,
|
||||
l.url as license_url,
|
||||
COALESCE(
|
||||
ARRAY_AGG(DISTINCT pt.tag) FILTER (WHERE pt.tag IS NOT NULL),
|
||||
ARRAY[]::VARCHAR[]
|
||||
) as tags
|
||||
pl.icon as platform_icon
|
||||
FROM packages p
|
||||
LEFT JOIN platforms pl ON p.platform_id = pl.id
|
||||
LEFT JOIN categories c ON p.category_id = c.id
|
||||
LEFT JOIN licenses l ON p.license_id = l.id
|
||||
LEFT JOIN package_tags pt ON p.id = pt.package_id
|
||||
${whereClause}
|
||||
GROUP BY p.id, pl.name, pl.package_manager, pl.icon, c.name, l.name, l.url
|
||||
${orderClause}
|
||||
LIMIT $${paramIndex++} OFFSET $${paramIndex++}
|
||||
`
|
||||
@@ -101,18 +100,12 @@ export class PackageService {
|
||||
const packages = packagesResult.rows.map((row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
description: row.description || 'No description available',
|
||||
version: row.version,
|
||||
platform_id: row.platform_id,
|
||||
category_id: row.category_id,
|
||||
license_id: row.license_id,
|
||||
type: row.type,
|
||||
repository: row.repository,
|
||||
homepage_url: row.homepage_url,
|
||||
download_url: row.download_url,
|
||||
last_updated: row.last_updated,
|
||||
downloads_count: row.downloads_count,
|
||||
popularity_score: row.popularity_score,
|
||||
type: row.type || 'cli',
|
||||
repository: row.repository || 'official',
|
||||
popularity_score: row.popularity_score || 0,
|
||||
is_active: row.is_active,
|
||||
created_at: row.created_at,
|
||||
updated_at: row.updated_at,
|
||||
@@ -122,16 +115,7 @@ export class PackageService {
|
||||
package_manager: row.platform_package_manager,
|
||||
icon: row.platform_icon
|
||||
} : undefined,
|
||||
category: row.category_name ? {
|
||||
id: row.category_id,
|
||||
name: row.category_name
|
||||
} : undefined,
|
||||
license: row.license_name ? {
|
||||
id: row.license_id,
|
||||
name: row.license_name,
|
||||
url: row.license_url
|
||||
} : undefined,
|
||||
tags: row.tags || []
|
||||
tags: [] // Empty tags for now
|
||||
}))
|
||||
|
||||
return { packages, total }
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { PlatformService } from './platformService'
|
||||
|
||||
export class PlatformInitializer {
|
||||
// Initialize all platforms in database
|
||||
static async initializePlatforms(): Promise<void> {
|
||||
console.log('🔄 Initializing platforms in database...')
|
||||
|
||||
const platforms = [
|
||||
{
|
||||
id: 'debian',
|
||||
name: 'Debian',
|
||||
package_manager: 'apt',
|
||||
icon: '🐧'
|
||||
},
|
||||
{
|
||||
id: 'ubuntu',
|
||||
name: 'Ubuntu',
|
||||
package_manager: 'apt',
|
||||
icon: '🐧'
|
||||
},
|
||||
{
|
||||
id: 'fedora',
|
||||
name: 'Fedora',
|
||||
package_manager: 'dnf',
|
||||
icon: '🎩'
|
||||
},
|
||||
{
|
||||
id: 'arch',
|
||||
name: 'Arch Linux',
|
||||
package_manager: 'pacman',
|
||||
icon: '🏛️'
|
||||
},
|
||||
{
|
||||
id: 'windows',
|
||||
name: 'Windows',
|
||||
package_manager: 'winget',
|
||||
icon: '🪟'
|
||||
},
|
||||
{
|
||||
id: 'macos',
|
||||
name: 'macOS',
|
||||
package_manager: 'homebrew',
|
||||
icon: '🍎'
|
||||
}
|
||||
]
|
||||
|
||||
for (const platform of platforms) {
|
||||
try {
|
||||
const existing = await PlatformService.getById(platform.id)
|
||||
|
||||
if (!existing) {
|
||||
await PlatformService.create(platform)
|
||||
console.log(`✅ Created platform: ${platform.name}`)
|
||||
} else {
|
||||
// Update existing platform to ensure correct name
|
||||
await PlatformService.update(platform.id, platform)
|
||||
console.log(`🔄 Updated platform: ${platform.name}`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`❌ Error initializing platform ${platform.id}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('✅ Platform initialization completed')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
import { PackageService } from './packageService'
|
||||
import { CreatePackageInput } from '@/models/Package'
|
||||
|
||||
export class SimplePackageFetcher {
|
||||
private static readonly DEBIAN_PACKAGES_URL = 'https://packages.debian.org/stable/allpackages?format=txt.gz'
|
||||
private static readonly UBUNTU_PACKAGES_URL = 'https://packages.ubuntu.com/noble/allpackages?format=txt.gz'
|
||||
|
||||
// Fetch and parse Debian packages (simple text parsing)
|
||||
static async fetchDebianPackages(): Promise<void> {
|
||||
await this.fetchDebianPackagesWithProgress(() => {})
|
||||
}
|
||||
|
||||
// Fetch with progress callback
|
||||
static async fetchDebianPackagesWithProgress(
|
||||
onProgress: (message: string, progress?: number, total?: number) => void
|
||||
): Promise<void> {
|
||||
onProgress('🔄 Fetching Debian packages from official repository...')
|
||||
|
||||
try {
|
||||
onProgress('📡 Downloading package list...')
|
||||
const response = await fetch(this.DEBIAN_PACKAGES_URL, {
|
||||
headers: {
|
||||
'User-Agent': 'RepoHub-Package-Fetcher/1.0'
|
||||
}
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Debian packages: ${response.statusText}`)
|
||||
}
|
||||
|
||||
// Get as text directly (not compressed!)
|
||||
const text = await response.text()
|
||||
onProgress(`📊 Downloaded ${Math.round(text.length / 1024 / 1024)}MB of package data`)
|
||||
|
||||
const packages = this.parseDebianPackageListWithProgress(text, onProgress)
|
||||
onProgress(`📊 Parsed ${packages.length} Debian packages`)
|
||||
|
||||
// Store in database
|
||||
await this.storePackagesWithProgress('debian', packages, onProgress)
|
||||
|
||||
onProgress('✅ Debian packages completed')
|
||||
} catch (error) {
|
||||
onProgress(`❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch and parse Ubuntu packages (simple text parsing)
|
||||
static async fetchUbuntuPackages(): Promise<void> {
|
||||
await this.fetchUbuntuPackagesWithProgress(() => {})
|
||||
}
|
||||
|
||||
// Fetch with progress callback
|
||||
static async fetchUbuntuPackagesWithProgress(
|
||||
onProgress: (message: string, progress?: number, total?: number) => void
|
||||
): Promise<void> {
|
||||
onProgress('🔄 Fetching Ubuntu packages from official repository...')
|
||||
|
||||
try {
|
||||
onProgress('📡 Downloading package list...')
|
||||
const response = await fetch(this.UBUNTU_PACKAGES_URL, {
|
||||
headers: {
|
||||
'User-Agent': 'RepoHub-Package-Fetcher/1.0'
|
||||
}
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Ubuntu packages: ${response.statusText}`)
|
||||
}
|
||||
|
||||
// Get as text directly (not compressed!)
|
||||
const text = await response.text()
|
||||
onProgress(`📊 Downloaded ${Math.round(text.length / 1024 / 1024)}MB of package data`)
|
||||
|
||||
const packages = this.parseUbuntuPackageListWithProgress(text, onProgress)
|
||||
onProgress(`📊 Parsed ${packages.length} Ubuntu packages`)
|
||||
|
||||
// Store in database
|
||||
await this.storePackagesWithProgress('ubuntu', packages, onProgress)
|
||||
|
||||
onProgress('✅ Ubuntu packages completed')
|
||||
} catch (error) {
|
||||
onProgress(`❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Parse Debian package list format
|
||||
private static parseDebianPackageList(text: string): CreatePackageInput[] {
|
||||
return this.parseDebianPackageListWithProgress(text, () => {})
|
||||
}
|
||||
|
||||
// Parse with progress callback
|
||||
private static parseDebianPackageListWithProgress(
|
||||
text: string,
|
||||
onProgress: (message: string, progress?: number, total?: number) => void
|
||||
): CreatePackageInput[] {
|
||||
const packages: CreatePackageInput[] = []
|
||||
const lines = text.split('\n')
|
||||
|
||||
console.log('📋 Analyzing package format...')
|
||||
|
||||
// Find where actual packages start
|
||||
let packageStartIndex = 0
|
||||
let sampleLines = []
|
||||
|
||||
for (let i = 0; i < Math.min(50, lines.length); i++) {
|
||||
const line = lines[i].trim()
|
||||
if (line && !line.startsWith('All') && !line.startsWith('Generated') && !line.startsWith('Copyright')) {
|
||||
sampleLines.push(line)
|
||||
if (packageStartIndex === 0 && line.match(/^[a-z0-9]/)) {
|
||||
packageStartIndex = i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('📋 Sample lines:', sampleLines.slice(0, 5))
|
||||
console.log('📋 Package start index:', packageStartIndex)
|
||||
|
||||
let processedCount = 0
|
||||
const totalLines = lines.length - packageStartIndex
|
||||
|
||||
for (let i = packageStartIndex; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue
|
||||
|
||||
// Skip header lines
|
||||
if (line.startsWith('All') || line.startsWith('Generated') || line.startsWith('Copyright')) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Try multiple regex patterns
|
||||
let match = null
|
||||
|
||||
// Pattern 1: "package-name (version) description"
|
||||
match = line.match(/^([a-z0-9][a-z0-9+.-]*)\s+\(([^)]+)\)\s*(.+)?$/)
|
||||
|
||||
if (match) {
|
||||
const [, name, version, description] = match
|
||||
|
||||
packages.push({
|
||||
id: `debian-${name}`,
|
||||
name: name,
|
||||
description: description?.trim(),
|
||||
version: version.trim(),
|
||||
platform_id: 'debian',
|
||||
type: this.determinePackageType(name, description),
|
||||
repository: 'official',
|
||||
popularity_score: Math.floor(Math.random() * 100)
|
||||
})
|
||||
|
||||
processedCount++
|
||||
|
||||
// Send progress every 1000 packages
|
||||
if (processedCount % 1000 === 0) {
|
||||
const progress = Math.round((processedCount / totalLines) * 100)
|
||||
onProgress(`📦 Parsed ${processedCount} packages...`, progress, 100)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
onProgress(`📦 Parsed ${processedCount} total packages`, 100, 100)
|
||||
return packages
|
||||
}
|
||||
|
||||
// Parse Ubuntu package list format
|
||||
private static parseUbuntuPackageList(text: string): CreatePackageInput[] {
|
||||
return this.parseUbuntuPackageListWithProgress(text, () => {})
|
||||
}
|
||||
|
||||
// Parse with progress callback
|
||||
private static parseUbuntuPackageListWithProgress(
|
||||
text: string,
|
||||
onProgress: (message: string, progress?: number, total?: number) => void
|
||||
): CreatePackageInput[] {
|
||||
const packages: CreatePackageInput[] = []
|
||||
const lines = text.split('\n')
|
||||
|
||||
onProgress('📋 Analyzing Ubuntu package format...')
|
||||
|
||||
// Find where actual packages start
|
||||
let packageStartIndex = 0
|
||||
|
||||
for (let i = 0; i < Math.min(50, lines.length); i++) {
|
||||
const line = lines[i].trim()
|
||||
if (line && !line.startsWith('All') && !line.startsWith('Generated') && !line.startsWith('Copyright')) {
|
||||
if (packageStartIndex === 0 && line.match(/^[a-z0-9]/)) {
|
||||
packageStartIndex = i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let processedCount = 0
|
||||
const totalLines = lines.length - packageStartIndex
|
||||
|
||||
for (let i = packageStartIndex; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue
|
||||
|
||||
// Skip header lines
|
||||
if (line.startsWith('All') || line.startsWith('Generated') || line.startsWith('Copyright')) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Try Ubuntu pattern: "package-name (version) [repository] description"
|
||||
let match = line.match(/^([a-z0-9][a-z0-9+.-]*)\s+\(([^)]+)\)\s+\[([^\]]+)\]\s*(.+)?$/)
|
||||
|
||||
if (match) {
|
||||
const [, name, version, repository, description] = match
|
||||
|
||||
packages.push({
|
||||
id: `ubuntu-${name}`,
|
||||
name: name,
|
||||
description: description?.trim(),
|
||||
version: version.trim(),
|
||||
platform_id: 'ubuntu',
|
||||
type: this.determinePackageType(name, description),
|
||||
repository: this.mapUbuntuRepository(repository),
|
||||
popularity_score: Math.floor(Math.random() * 100)
|
||||
})
|
||||
|
||||
processedCount++
|
||||
|
||||
// Send progress every 1000 packages
|
||||
if (processedCount % 1000 === 0) {
|
||||
const progress = Math.round((processedCount / totalLines) * 100)
|
||||
onProgress(`📦 Parsed ${processedCount} packages...`, progress, 100)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
onProgress(`📦 Parsed ${processedCount} total packages`, 100, 100)
|
||||
return packages
|
||||
}
|
||||
|
||||
// Map Ubuntu repository to our enum
|
||||
private static mapUbuntuRepository(repo: string): 'official' | 'third-party' {
|
||||
switch (repo.toLowerCase()) {
|
||||
case 'main':
|
||||
case 'restricted':
|
||||
case 'universe':
|
||||
case 'multiverse':
|
||||
return 'official'
|
||||
default:
|
||||
return 'third-party'
|
||||
}
|
||||
}
|
||||
|
||||
// Determine if package is GUI or CLI based on name and description
|
||||
private static determinePackageType(name: string, description?: string): 'gui' | 'cli' {
|
||||
const guiKeywords = [
|
||||
'gui', 'gtk', 'qt', 'x11', 'desktop', 'window', 'display',
|
||||
'graphical', 'visual', 'image', 'video', 'audio', 'media',
|
||||
'browser', 'editor', 'viewer', 'player', 'manager',
|
||||
'game', 'games', 'steam', 'wine'
|
||||
]
|
||||
|
||||
const cliKeywords = [
|
||||
'cli', 'command', 'terminal', 'console', 'shell', 'bash',
|
||||
'tool', 'utility', 'daemon', 'service', 'server', 'client',
|
||||
'lib', 'dev', 'debug', 'build', 'compile'
|
||||
]
|
||||
|
||||
const searchText = `${name} ${description || ''}`.toLowerCase()
|
||||
|
||||
for (const keyword of guiKeywords) {
|
||||
if (searchText.includes(keyword)) return 'gui'
|
||||
}
|
||||
|
||||
for (const keyword of cliKeywords) {
|
||||
if (searchText.includes(keyword)) return 'cli'
|
||||
}
|
||||
|
||||
return 'cli'
|
||||
}
|
||||
|
||||
// Store packages in database
|
||||
private static async storePackages(platformId: string, packages: CreatePackageInput[]): Promise<void> {
|
||||
await this.storePackagesWithProgress(platformId, packages, () => {})
|
||||
}
|
||||
|
||||
// Store with progress callback
|
||||
private static async storePackagesWithProgress(
|
||||
platformId: string,
|
||||
packages: CreatePackageInput[],
|
||||
onProgress: (message: string, progress?: number, total?: number) => void
|
||||
): Promise<void> {
|
||||
onProgress(`💾 Storing ${packages.length} packages for platform ${platformId}...`)
|
||||
|
||||
let added = 0
|
||||
let skipped = 0
|
||||
let processed = 0
|
||||
const batchSize = 50 // Smaller batches for better performance
|
||||
|
||||
// Process in batches
|
||||
for (let i = 0; i < packages.length; i += batchSize) {
|
||||
const batch = packages.slice(i, i + batchSize)
|
||||
|
||||
for (const pkg of batch) {
|
||||
try {
|
||||
const existing = await PackageService.getById(pkg.id)
|
||||
|
||||
if (existing) {
|
||||
// Skip if version is the same (no need to update)
|
||||
if (existing.version === pkg.version) {
|
||||
skipped++
|
||||
processed++
|
||||
continue
|
||||
}
|
||||
|
||||
// Only update if version changed
|
||||
await PackageService.update(pkg.id, {
|
||||
description: pkg.description,
|
||||
version: pkg.version,
|
||||
type: pkg.type,
|
||||
repository: pkg.repository,
|
||||
popularity_score: pkg.popularity_score
|
||||
})
|
||||
} else {
|
||||
// Create new package
|
||||
await PackageService.create(pkg)
|
||||
added++
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error storing package ${pkg.id}:`, error)
|
||||
}
|
||||
|
||||
processed++
|
||||
}
|
||||
|
||||
// Send progress every batch
|
||||
if (processed % 100 === 0) {
|
||||
const progress = Math.round((processed / packages.length) * 100)
|
||||
onProgress(`💾 Processed ${processed} packages... (${added} new, ${skipped} skipped)`, progress, 100)
|
||||
}
|
||||
|
||||
// Small delay to prevent overwhelming the database
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
}
|
||||
|
||||
onProgress(`💾 Completed: ${added} new, ${skipped} skipped packages`, 100, 100)
|
||||
}
|
||||
|
||||
// Sync both Debian and Ubuntu packages
|
||||
static async syncAll(): Promise<void> {
|
||||
console.log('🔄 Starting sync of all Debian-based packages...')
|
||||
|
||||
try {
|
||||
await this.fetchDebianPackages()
|
||||
await this.fetchUbuntuPackages()
|
||||
|
||||
console.log('✅ All Debian-based packages synced successfully')
|
||||
} catch (error) {
|
||||
console.error('❌ Error during sync:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user