mirror of
https://github.com/yusufipk/RepoHub.git
synced 2026-09-11 18:46:07 +00:00
feat: add architecture sync script and cheerio dependency
- Added sync:arch npm script to synchronize architecture documentation - Installed cheerio for HTML parsing capabilities - Alphabetized package.json dependencies for better maintainability
This commit is contained in:
Yusuf İpek
@@ -0,0 +1,36 @@
|
||||
import { NextResponse } from 'next/server'
|
||||
import { query } from '@/lib/database/config'
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
// Check if arch platform exists
|
||||
const platformResult = await query(
|
||||
'SELECT * FROM platforms WHERE id = $1',
|
||||
['arch']
|
||||
)
|
||||
|
||||
// Count arch packages
|
||||
const countResult = await query(
|
||||
'SELECT COUNT(*) FROM packages WHERE platform_id = $1',
|
||||
['arch']
|
||||
)
|
||||
|
||||
// Get sample packages
|
||||
const sampleResult = await query(
|
||||
'SELECT name, version, description FROM packages WHERE platform_id = $1 LIMIT 5',
|
||||
['arch']
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
platform: platformResult.rows[0] || null,
|
||||
totalPackages: parseInt(countResult.rows[0].count),
|
||||
samplePackages: sampleResult.rows
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Debug error:', error)
|
||||
return NextResponse.json(
|
||||
{ error: error instanceof Error ? error.message : 'Unknown error' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { NextResponse } from 'next/server'
|
||||
import { ArchPackageFetcher } from '@/services/archPackageFetcher'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const maxDuration = 300 // 5 minutes timeout
|
||||
|
||||
let syncInProgress = false
|
||||
let syncStatus = {
|
||||
status: 'idle' as 'idle' | 'fetching' | 'storing' | 'complete' | 'error',
|
||||
fetchProgress: 0,
|
||||
fetchTotal: 0,
|
||||
storeProgress: 0,
|
||||
storeTotal: 0,
|
||||
currentPackage: '',
|
||||
error: null as string | null
|
||||
}
|
||||
|
||||
export async function GET() {
|
||||
return NextResponse.json(syncStatus)
|
||||
}
|
||||
|
||||
export async function POST() {
|
||||
if (syncInProgress) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Sync already in progress' },
|
||||
{ status: 409 }
|
||||
)
|
||||
}
|
||||
|
||||
syncInProgress = true
|
||||
syncStatus = {
|
||||
status: 'fetching',
|
||||
fetchProgress: 0,
|
||||
fetchTotal: 0,
|
||||
storeProgress: 0,
|
||||
storeTotal: 0,
|
||||
currentPackage: '',
|
||||
error: null
|
||||
}
|
||||
|
||||
// Start sync in background
|
||||
;(async () => {
|
||||
try {
|
||||
const fetcher = new ArchPackageFetcher()
|
||||
|
||||
// Fetch packages
|
||||
console.log('🔄 Starting Arch Linux package fetch...')
|
||||
const packages = await fetcher.fetchAllPackages(
|
||||
(current, total, packageName) => {
|
||||
syncStatus.fetchProgress = current
|
||||
syncStatus.fetchTotal = total
|
||||
syncStatus.currentPackage = packageName
|
||||
}
|
||||
)
|
||||
|
||||
console.log(`✅ Fetched ${packages.length} Arch packages`)
|
||||
|
||||
// Store packages
|
||||
syncStatus.status = 'storing'
|
||||
syncStatus.storeTotal = packages.length
|
||||
|
||||
await fetcher.storePackages(
|
||||
packages,
|
||||
(current, total) => {
|
||||
syncStatus.storeProgress = current
|
||||
syncStatus.storeTotal = total
|
||||
}
|
||||
)
|
||||
|
||||
syncStatus.status = 'complete'
|
||||
console.log('✅ Arch Linux sync completed successfully')
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Arch sync error:', error)
|
||||
syncStatus.status = 'error'
|
||||
syncStatus.error = error instanceof Error ? error.message : 'Unknown error'
|
||||
} finally {
|
||||
syncInProgress = false
|
||||
}
|
||||
})()
|
||||
|
||||
return NextResponse.json({
|
||||
message: 'Arch Linux package sync started',
|
||||
status: syncStatus
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
interface ArchPackage {
|
||||
name: string
|
||||
version: string
|
||||
description: string
|
||||
architecture: string
|
||||
repository: string
|
||||
lastUpdated: string
|
||||
}
|
||||
|
||||
// Simple HTML parser without external dependencies
|
||||
function parseHTML(html: string) {
|
||||
return {
|
||||
querySelector: (selector: string) => {
|
||||
const regex = new RegExp(`<${selector}[^>]*>([\\s\\S]*?)<\\/${selector}>`, 'gi')
|
||||
const match = regex.exec(html)
|
||||
return match ? { textContent: match[1].replace(/<[^>]*>/g, '').trim() } : null
|
||||
},
|
||||
querySelectorAll: (selector: string) => {
|
||||
const results: any[] = []
|
||||
const regex = new RegExp(`<${selector}[^>]*>([\\s\\S]*?)<\\/${selector}>`, 'gi')
|
||||
let match
|
||||
while ((match = regex.exec(html)) !== null) {
|
||||
results.push({ innerHTML: match[1], textContent: match[1].replace(/<[^>]*>/g, '').trim() })
|
||||
}
|
||||
return results
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class ArchPackageFetcher {
|
||||
private baseUrl = 'https://archlinux.org/packages'
|
||||
|
||||
/**
|
||||
* Fetch all Arch Linux packages with progress callback
|
||||
*/
|
||||
async fetchAllPackages(
|
||||
onProgress?: (current: number, total: number, packageName: string) => void
|
||||
): Promise<ArchPackage[]> {
|
||||
console.log('🔍 Starting Arch Linux package fetch...')
|
||||
|
||||
const allPackages: ArchPackage[] = []
|
||||
let currentPage = 1
|
||||
let totalPages = 1
|
||||
|
||||
try {
|
||||
// Fetch first page to get total pages
|
||||
const firstPageUrl = `${this.baseUrl}/?page=1`
|
||||
console.log(`📄 Fetching first page: ${firstPageUrl}`)
|
||||
|
||||
const firstPageResponse = await fetch(firstPageUrl)
|
||||
if (!firstPageResponse.ok) {
|
||||
throw new Error(`HTTP error! status: ${firstPageResponse.status}`)
|
||||
}
|
||||
|
||||
const firstPageHtml = await firstPageResponse.text()
|
||||
|
||||
// Parse total packages and pages
|
||||
const statsMatch = firstPageHtml.match(/(\d+)\s+matching packages[\s\S]*?Page \d+ of (\d+)/)
|
||||
const totalPackages = statsMatch ? parseInt(statsMatch[1]) : 0
|
||||
totalPages = statsMatch ? parseInt(statsMatch[2]) : 1
|
||||
|
||||
console.log(`📦 Total packages: ${totalPackages}`)
|
||||
console.log(`📄 Total pages: ${totalPages}`)
|
||||
|
||||
// Parse first page packages
|
||||
const firstPagePackages = this.parsePackagesFromHtml(firstPageHtml)
|
||||
allPackages.push(...firstPagePackages)
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(allPackages.length, totalPackages, firstPagePackages[0]?.name || 'N/A')
|
||||
}
|
||||
|
||||
// Fetch remaining pages
|
||||
for (currentPage = 2; currentPage <= totalPages; currentPage++) {
|
||||
const pageUrl = `${this.baseUrl}/?page=${currentPage}`
|
||||
console.log(`📄 Fetching page ${currentPage}/${totalPages}...`)
|
||||
|
||||
try {
|
||||
const response = await fetch(pageUrl)
|
||||
if (!response.ok) {
|
||||
console.error(`❌ Failed to fetch page ${currentPage}: ${response.status}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const html = await response.text()
|
||||
const packages = this.parsePackagesFromHtml(html)
|
||||
|
||||
allPackages.push(...packages)
|
||||
|
||||
if (onProgress && packages.length > 0) {
|
||||
onProgress(allPackages.length, totalPackages, packages[0].name)
|
||||
}
|
||||
|
||||
// Rate limiting: wait 100ms between requests
|
||||
await new Promise(resolve => setTimeout(resolve, 100))
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error fetching page ${currentPage}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Fetch completed! Total packages: ${allPackages.length}`)
|
||||
return allPackages
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error fetching Arch packages:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse packages from HTML using regex
|
||||
*/
|
||||
private parsePackagesFromHtml(html: string): ArchPackage[] {
|
||||
const packages: ArchPackage[] = []
|
||||
|
||||
// Match all table rows
|
||||
const rowRegex = /<tr>\s*<td>([^<]+)<\/td>\s*<td>([^<]+)<\/td>\s*<td><a[^>]*>([^<]+)<\/a><\/td>\s*<td>([^<]+)<\/td>\s*<td[^>]*>([^<]*)<\/td>\s*<td>([^<]*)<\/td>/gi
|
||||
|
||||
let match
|
||||
while ((match = rowRegex.exec(html)) !== null) {
|
||||
try {
|
||||
const architecture = match[1].trim()
|
||||
const repository = match[2].trim()
|
||||
const name = match[3].trim()
|
||||
const version = match[4].trim()
|
||||
const description = match[5].trim()
|
||||
const lastUpdated = match[6].trim()
|
||||
|
||||
if (name && version) {
|
||||
packages.push({
|
||||
name,
|
||||
version,
|
||||
description: description || 'No description available',
|
||||
architecture,
|
||||
repository: repository.toLowerCase(),
|
||||
lastUpdated
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error parsing package row:', error)
|
||||
}
|
||||
}
|
||||
|
||||
return packages
|
||||
}
|
||||
|
||||
/**
|
||||
* Store packages in database
|
||||
*/
|
||||
async storePackages(
|
||||
packages: ArchPackage[],
|
||||
onProgress?: (current: number, total: number) => void
|
||||
): Promise<void> {
|
||||
const { query } = await import('@/lib/database/config')
|
||||
|
||||
console.log(`💾 Storing ${packages.length} Arch packages in database...`)
|
||||
|
||||
const batchSize = 100
|
||||
let stored = 0
|
||||
let updated = 0
|
||||
let skipped = 0
|
||||
|
||||
for (let i = 0; i < packages.length; i += batchSize) {
|
||||
const batch = packages.slice(i, i + batchSize)
|
||||
|
||||
for (const pkg of batch) {
|
||||
try {
|
||||
// Check if package already exists
|
||||
const existingResult = await query(
|
||||
'SELECT id, version FROM packages WHERE name = $1 AND platform_id = $2',
|
||||
[pkg.name, 'arch']
|
||||
)
|
||||
|
||||
const repository = (pkg.repository === 'core' || pkg.repository === 'extra')
|
||||
? 'official'
|
||||
: 'third-party'
|
||||
|
||||
if (existingResult.rows.length === 0) {
|
||||
// Insert new package (id will be auto-generated by SERIAL)
|
||||
await query(
|
||||
`INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active)
|
||||
VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`,
|
||||
[pkg.name, pkg.description, pkg.version, 'arch', 'cli', repository, 0, true]
|
||||
)
|
||||
stored++
|
||||
} else if (existingResult.rows[0].version !== pkg.version) {
|
||||
// Update if version changed
|
||||
await query(
|
||||
`UPDATE packages
|
||||
SET version = $1, description = $2, repository = $3, updated_at = NOW()
|
||||
WHERE id = $4`,
|
||||
[pkg.version, pkg.description, repository, existingResult.rows[0].id]
|
||||
)
|
||||
updated++
|
||||
} else {
|
||||
skipped++
|
||||
}
|
||||
|
||||
if (onProgress && (stored + updated + skipped) % 100 === 0) {
|
||||
onProgress(stored + updated + skipped, packages.length)
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Error storing package ${pkg.name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Stored ${stored} new packages, updated ${updated}, skipped ${skipped}`)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user