From 560e78cb9616cecdd66aa84581aa1c3384b4c426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yusuf=20=C4=B0pek?= Date: Wed, 12 Nov 2025 20:43:50 +0300 Subject: [PATCH] feat: add AUR repository support for Arch Linux packages - Extended database schema and models to include 'aur' as a valid repository type alongside 'official' and 'third-party' - Added repository filter UI in package browser for Arch platform to distinguish between official and AUR packages - Enhanced init-db script with migration system to track and apply schema changes automatically --- docs/SYNC_SECURITY.md | 211 -------------------- scripts/init-db.js | 78 +++++++- scripts/migrate-add-aur-to-repository.sql | 5 + src/app/api/packages/route.ts | 2 +- src/app/api/sync-aur/route.ts | 52 +++++ src/components/PackageBrowserV2.tsx | 32 +++- src/lib/database/schema.sql | 2 +- src/models/Package.ts | 8 +- src/services/archPackageFetcher.ts | 4 +- src/services/aurPackageFetcher.ts | 223 ++++++++++++++++++++++ src/types/index.ts | 4 +- 11 files changed, 395 insertions(+), 226 deletions(-) delete mode 100644 docs/SYNC_SECURITY.md create mode 100644 scripts/migrate-add-aur-to-repository.sql create mode 100644 src/app/api/sync-aur/route.ts create mode 100644 src/services/aurPackageFetcher.ts diff --git a/docs/SYNC_SECURITY.md b/docs/SYNC_SECURITY.md deleted file mode 100644 index 58e4d4d..0000000 --- a/docs/SYNC_SECURITY.md +++ /dev/null @@ -1,211 +0,0 @@ -# RepoHub Sync Security Configuration - -Bu doküman, RepoHub sync işlemlerinin güvenliği ve otomatikleştirilmesi için yapılandırmayı açıklar. - -## Güvenlik Yapılandırması - -### Environment Değişkenleri - -`.env.local` dosyasına aşağı değişkenleri ekleyin: - -```bash -# Sync Configuration -# Enable sync operations only from the server itself (set to 'true' on production server) -SYNC_SERVER_ONLY=true -# Automatic sync frequency in days (set to 0 to disable automatic sync) -AUTO_SYNC_DAYS=1 -# Secret key to authorize sync operations from server -SYNC_SECRET_KEY=your_very_secure_secret_key_here -``` - -### Değişkenlerin Açıklaması - -- **SYNC_SERVER_ONLY**: `true` olarak ayarlandığında, sync işlemleri sadece sunucudan yapılabilir -- **AUTO_SYNC_DAYS**: Otomatik sync sıklığı (gün olarak). `0` = otomatik sync kapalı -- **SYNC_SECRET_KEY**: Sunucu sync işlemleri için gerekli gizli anahtar - -## Sunucu Sync İşlemleri - -### 1. Manuel Sync (Sunucudan) - -Sunucuda sync işlemleri yapmak için: - -```bash -# Tüm platformları sync et -./scripts/server-sync.sh auto-sync - -# Sadece Windows paketlerini sync et -./scripts/server-sync.sh sync-winget - -# Sync durumunu kontrol et -./scripts/server-sync.sh status -``` - -### 2. Cron Job ile Otomatik Sync - -Otomatik sync kurulumu için: - -```bash -# Günlük otomatik sync kur -sudo ./scripts/cron-setup.sh install - -# Haftalık otomatik sync kur (her 7 günde bir) -sudo AUTO_SYNC_DAYS=7 ./scripts/cron-setup.sh install - -# Cron job durumunu kontrol et -sudo ./scripts/cron-setup.sh status - -# Cron job'ı kaldır -sudo ./scripts/cron-setup.sh remove -``` - -## API Güvenliği - -### Sync Endpoint'leri - -Tüm sync endpoint'leri artık güvenlik kontrolü yapar: - -- `/api/sync` - Genel sync endpoint'i -- `/api/sync-winget` - Windows paket sync'i -- `/api/sync-homebrew` - macOS paket sync'i -- `/api/sync-fedora` - Fedora paket sync'i -- `/api/sync-arch` - Arch paket sync'i -- `/api/auto-sync` - Otomatik sync - -### Güvenlik Kontrolü - -**Server-only modda (`SYNC_SERVER_ONLY=true`):** - -1. **Secret Key Kontrolü**: Request header'ında `x-sync-secret` olmalı -2. **IP Kontrolü**: İstek localhost'tan gelmeli veya doğru secret key içermeli - -**Normal modda (`SYNC_SERVER_ONLY=false`):** - -- Herkes sync işlemi yapabilir (geliştirme için) - -### Örnek API Kullanımı - -```bash -# Server-only modda sync yapmak -curl -X POST \ - -H "Content-Type: application/json" \ - -H "x-sync-secret: your_secret_key" \ - -d '{}' \ - http://localhost:3002/api/sync-winget - -# Sync durumunu kontrol et -curl -H "x-sync-secret: your_secret_key" \ - http://localhost:3002/api/sync -``` - -## Güvenlik İpuçları - -### 1. Secret Key Güvenliği - -- Güçlü ve rastgele bir secret key kullanın -- Secret key'i `.env.local` dosyasında saklayın, asla kod içine koymayın -- Secret key'i düzenli olarak değiştirin - -### 2. Sunucu Güvenliği - -- Sync script'lerini sadece sunucuda çalıştırın -- Cron job'ları root kullanıcısı olarak ayarlayın -- Log dosyalarını düzenli olarak kontrol edin - -### 3. Ağ Güvenliği - -- Sync endpoint'lerini firewall ile koruyun -- Sadece localhost'tan erişime izin verin -- SSL/TLS kullanın (production'da) - -## Monitoring ve Logging - -### Log Dosyaları - -- **Sync Log**: `/var/log/repohub-sync.log` -- **Cron Log**: `/var/log/cron.log` (sistem bağımlı) - -### Log İzleme - -```bash -# Son 20 sync log satırını göster -tail -20 /var/log/repohub-sync.log - -# Real-time log izleme -tail -f /var/log/repohub-sync.log - -# Cron job loglarını kontrol et -sudo tail -20 /var/log/cron.log -``` - -## Troubleshooting - -### Yaygın Sorunlar - -1. **403 Forbidden Error** - - `SYNC_SECRET_KEY` doğru ayarlanmamış - - Header'da `x-sync-secret` eksik - -2. **Cron Job Çalışmıyor** - - Script executable değil - - Environment değişkenleri eksik - - Log dosyası izinleri yanlış - -3. **Sync Başarısız** - - API URL yanlış - - Ağ bağlantısı sorunu - - Disk alanı yetersiz - -### Debug Komutları - -```bash -# Sync script test et -./scripts/server-sync.sh test - -# Cron job durumunu kontrol et -sudo ./scripts/cron-setup.sh status - -# Environment değişkenlerini kontrol et -env | grep SYNC -``` - -## Production Dağıtımı - -### Adım 1: Environment Ayarı - -```bash -# .env.local dosyasını production'a kopyala -cp .env.local.example .env.local -# .env.local dosyasını production değerleriyle düzenle -``` - -### Adım 2: Secret Key Oluştur - -```bash -# Güçlü secret key oluştur -openssl rand -hex 32 -# Bu değeri .env.local dosyasına ekle -``` - -### Adım 3: Cron Job Kur - -```bash -# Production cron job kur -sudo AUTO_SYNC_DAYS=1 ./scripts/cron-setup.sh install -``` - -### Adım 4: Test Et - -```bash -# Sync işlemini test et -./scripts/server-sync.sh test - -# Otomatik sync'i test et -curl -X POST \ - -H "Content-Type: application/json" \ - -H "x-sync-secret: your_secret_key" \ - -d '{}' \ - http://localhost:3002/api/auto-sync -``` - -Bu yapılandırma ile sync işlemleriniz güvenli ve otomatik hale gelecektir. diff --git a/scripts/init-db.js b/scripts/init-db.js index d897401..c3d37ae 100644 --- a/scripts/init-db.js +++ b/scripts/init-db.js @@ -36,7 +36,7 @@ async function main() { await admin.end().catch(() => {}) } - // Connect to the target DB and apply schema + // Connect to the target DB and apply schema, then run migrations const db = new Pool({ host: DB_HOST, port: DB_PORT, @@ -59,10 +59,82 @@ async function main() { } else { console.error('❌ Schema apply error:', err) process.exitCode = 1 + // continue to attempt migrations anyway } - } finally { - await db.end().catch(() => {}) + } + + // Ensure schema_migrations table exists + try { + await db.query(` + CREATE TABLE IF NOT EXISTS schema_migrations ( + id SERIAL PRIMARY KEY, + filename TEXT UNIQUE NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + `) + } catch (err) { + console.error('❌ Failed to ensure schema_migrations table:', err) + process.exitCode = 1 } + + // Discover migration files + try { + const migrationsDir = path.join(__dirname, 'migrations') + const migrationFiles = [] + + if (fs.existsSync(migrationsDir)) { + for (const f of fs.readdirSync(migrationsDir)) { + if (f.endsWith('.sql')) { + migrationFiles.push({ filename: f, fullpath: path.join(migrationsDir, f) }) + } + } + } + + // Back-compat: also pick up legacy root-level migrations named migrate-*.sql + for (const f of fs.readdirSync(__dirname)) { + if (f.startsWith('migrate-') && f.endsWith('.sql')) { + migrationFiles.push({ filename: f, fullpath: path.join(__dirname, f) }) + } + } + + // Sort by filename for deterministic order (e.g., 001_*, 002_* ...) + migrationFiles.sort((a, b) => a.filename.localeCompare(b.filename)) + + if (migrationFiles.length) { + console.log(`🚀 Running migrations (${migrationFiles.length} found)...`) + } else { + console.log('ℹ️ No migrations found') + } + + for (const m of migrationFiles) { + try { + const { rows } = await db.query('SELECT 1 FROM schema_migrations WHERE filename = $1', [m.filename]) + if (rows.length) { + console.log(`↪️ Skipping already applied migration: ${m.filename}`) + continue + } + + const sql = fs.readFileSync(m.fullpath, 'utf8') + console.log(`➡️ Applying migration: ${m.filename}`) + await db.query('BEGIN') + await db.query(sql) + await db.query('INSERT INTO schema_migrations (filename) VALUES ($1)', [m.filename]) + await db.query('COMMIT') + console.log(`✅ Migration applied: ${m.filename}`) + } catch (err) { + console.error(`❌ Migration failed: ${m.filename}`, err) + try { await db.query('ROLLBACK') } catch {} + process.exitCode = 1 + break + } + } + } catch (err) { + console.error('❌ Failed during migration discovery/execution:', err) + process.exitCode = 1 + } + + // Done + await db.end().catch(() => {}) } main().catch((e) => { diff --git a/scripts/migrate-add-aur-to-repository.sql b/scripts/migrate-add-aur-to-repository.sql new file mode 100644 index 0000000..1f5e05e --- /dev/null +++ b/scripts/migrate-add-aur-to-repository.sql @@ -0,0 +1,5 @@ +-- Add 'aur' to repository check constraint +ALTER TABLE packages DROP CONSTRAINT IF EXISTS packages_repository_check; +ALTER TABLE packages + ADD CONSTRAINT packages_repository_check + CHECK (repository IN ('official','third-party','aur')); diff --git a/src/app/api/packages/route.ts b/src/app/api/packages/route.ts index 7126928..786ce33 100644 --- a/src/app/api/packages/route.ts +++ b/src/app/api/packages/route.ts @@ -11,7 +11,7 @@ export async function GET(request: NextRequest) { category_id: searchParams.get('category_id') ? parseInt(searchParams.get('category_id')!) : undefined, type: searchParams.get('type') as 'gui' | 'cli' | undefined, - repository: searchParams.get('repository') as 'official' | 'third-party' | undefined, + repository: searchParams.get('repository') as 'official' | 'third-party' | 'aur' | undefined, search: searchParams.get('search') || undefined, limit: searchParams.get('limit') ? parseInt(searchParams.get('limit')!) : undefined, diff --git a/src/app/api/sync-aur/route.ts b/src/app/api/sync-aur/route.ts new file mode 100644 index 0000000..5a5e136 --- /dev/null +++ b/src/app/api/sync-aur/route.ts @@ -0,0 +1,52 @@ +import { NextRequest, NextResponse } from 'next/server' +import { SyncAuth } from '@/lib/sync/auth' +import { AurPackageFetcher } from '@/services/aurPackageFetcher' + +export const dynamic = 'force-dynamic' +export const maxDuration = 300 + +let syncInProgress = false +let syncStatus: { + status: 'idle' | 'running' | 'complete' | 'error' + error: string | null +} = { status: 'idle', error: null } + +export async function GET() { + return NextResponse.json(syncStatus) +} + +export async function POST(request: NextRequest) { + const authResult = await SyncAuth.isSyncAllowed(request) + if (!authResult.allowed) { + return NextResponse.json( + { error: 'Sync operation not allowed', reason: authResult.reason }, + { status: 403 } + ) + } + + if (syncInProgress) { + return NextResponse.json( + { error: 'Sync already in progress' }, + { status: 409 } + ) + } + + syncInProgress = true + syncStatus = { status: 'running', error: null } + + ;(async () => { + try { + const fetcher = new AurPackageFetcher() + const pkgs = await fetcher.fetchAllPackages() + await fetcher.storePackages(pkgs) + syncStatus.status = 'complete' + } catch (err: any) { + syncStatus.status = 'error' + syncStatus.error = err?.message || 'Unknown error' + } finally { + syncInProgress = false + } + })() + + return NextResponse.json({ message: 'AUR sync started', status: syncStatus }) +} diff --git a/src/components/PackageBrowserV2.tsx b/src/components/PackageBrowserV2.tsx index 8937dc7..e2da6a3 100644 --- a/src/components/PackageBrowserV2.tsx +++ b/src/components/PackageBrowserV2.tsx @@ -31,10 +31,12 @@ export function PackageBrowserV2({ const [totalCount, setTotalCount] = useState(0) const [searchQuery, setSearchQuery] = useState('') const [typeFilter, setTypeFilter] = useState('') + const [repositoryFilter, setRepositoryFilter] = useState('') const isDebianUbuntu = useMemo(() => { const id = selectedPlatform?.id || '' return id === 'debian' || id === 'ubuntu' }, [selectedPlatform]) + const isArch = useMemo(() => (selectedPlatform?.id || '') === 'arch', [selectedPlatform]) const [scrollPosition, setScrollPosition] = useState(0) const searchInputRef = useRef(null) @@ -66,6 +68,9 @@ export function PackageBrowserV2({ if (isDebianUbuntu && typeFilter && typeFilter !== 'all') { params.type = typeFilter as 'gui' | 'cli' } + if (isArch && repositoryFilter && repositoryFilter !== 'all') { + params.repository = repositoryFilter as 'official' | 'aur' + } console.log('🔍 Frontend: API params:', params) const result = await apiClient.getPackages(params) @@ -86,7 +91,7 @@ export function PackageBrowserV2({ } loadPackages() - }, [selectedPlatform, typeFilter, isDebianUbuntu]) + }, [selectedPlatform, typeFilter, repositoryFilter, isDebianUbuntu, isArch]) // Debounced search to prevent focus loss useEffect(() => { @@ -111,6 +116,9 @@ export function PackageBrowserV2({ if (isDebianUbuntu && typeFilter && typeFilter !== 'all') { params.type = typeFilter as 'gui' | 'cli' } + if (isArch && repositoryFilter && repositoryFilter !== 'all') { + params.repository = repositoryFilter as 'official' | 'aur' + } console.log('🔍 Frontend: Debounced API params:', params) const result = await apiClient.getPackages(params) @@ -130,7 +138,7 @@ export function PackageBrowserV2({ }, 300) // 300ms debounce return () => clearTimeout(timeoutId) - }, [searchQuery, isDebianUbuntu, typeFilter, selectedPlatform]) + }, [searchQuery, isDebianUbuntu, typeFilter, isArch, repositoryFilter, selectedPlatform]) // Load more packages const loadMore = async () => { @@ -151,6 +159,9 @@ export function PackageBrowserV2({ if (isDebianUbuntu && typeFilter && typeFilter !== 'all') { params.type = typeFilter as 'gui' | 'cli' } + if (isArch && repositoryFilter && repositoryFilter !== 'all') { + params.repository = repositoryFilter as 'official' | 'aur' + } const result = await apiClient.getPackages(params) setPackages(prev => [...prev, ...result.packages]) @@ -257,6 +268,18 @@ export function PackageBrowserV2({ )} + {isArch && ( + + )} {/* Package List */} @@ -311,6 +334,11 @@ export function PackageBrowserV2({ {pkg.type.toUpperCase()} )} + {pkg.repository === 'aur' && ( + + AUR + + )} {pkg.repository === 'third-party' && ( third-party diff --git a/src/lib/database/schema.sql b/src/lib/database/schema.sql index 2609ad0..af4d00b 100644 --- a/src/lib/database/schema.sql +++ b/src/lib/database/schema.sql @@ -37,7 +37,7 @@ CREATE TABLE packages ( category_id INTEGER REFERENCES categories(id), license_id INTEGER REFERENCES licenses(id), type VARCHAR(10) CHECK (type IN ('gui', 'cli')), - repository VARCHAR(20) CHECK (repository IN ('official', 'third-party')), + repository VARCHAR(20) CHECK (repository IN ('official', 'third-party', 'aur')), homepage_url VARCHAR(255), download_url VARCHAR(255), last_updated TIMESTAMP WITH TIME ZONE, diff --git a/src/models/Package.ts b/src/models/Package.ts index 91da092..97da78f 100644 --- a/src/models/Package.ts +++ b/src/models/Package.ts @@ -7,7 +7,7 @@ export interface Package { category_id?: number license_id?: number type?: 'gui' | 'cli' - repository?: 'official' | 'third-party' + repository?: 'official' | 'third-party' | 'aur' homepage_url?: string download_url?: string last_updated?: Date @@ -53,7 +53,7 @@ export interface CreatePackageInput { category_id?: number license_id?: number type?: 'gui' | 'cli' - repository?: 'official' | 'third-party' + repository?: 'official' | 'third-party' | 'aur' homepage_url?: string download_url?: string popularity_score?: number @@ -66,7 +66,7 @@ export interface UpdatePackageInput { category_id?: number license_id?: number type?: 'gui' | 'cli' - repository?: 'official' | 'third-party' + repository?: 'official' | 'third-party' | 'aur' homepage_url?: string download_url?: string popularity_score?: number @@ -77,7 +77,7 @@ export interface PackageFilter { platform_id?: string category_id?: number type?: 'gui' | 'cli' - repository?: 'official' | 'third-party' + repository?: 'official' | 'third-party' | 'aur' search?: string limit?: number offset?: number diff --git a/src/services/archPackageFetcher.ts b/src/services/archPackageFetcher.ts index 9cf8b0b..50659b8 100644 --- a/src/services/archPackageFetcher.ts +++ b/src/services/archPackageFetcher.ts @@ -177,7 +177,7 @@ export class ArchPackageFetcher { await query( `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, - [pkg.name, pkg.description, pkg.version, 'arch', null, null, 0, true] + [pkg.name, pkg.description, pkg.version, 'arch', null, 'official', 0, true] ) stored++ } else if (existingResult.rows[0].version !== pkg.version) { @@ -186,7 +186,7 @@ export class ArchPackageFetcher { `UPDATE packages SET version = $1, description = $2, repository = $3, updated_at = NOW() WHERE id = $4`, - [pkg.version, pkg.description, null, existingResult.rows[0].id] + [pkg.version, pkg.description, 'official', existingResult.rows[0].id] ) updated++ } else { diff --git a/src/services/aurPackageFetcher.ts b/src/services/aurPackageFetcher.ts new file mode 100644 index 0000000..133030e --- /dev/null +++ b/src/services/aurPackageFetcher.ts @@ -0,0 +1,223 @@ +interface AurPackage { + name: string + version: string + description: string +} + +type AurRpcResult = { + ID: number + Name: string + Version: string + Description: string +} + +type AurRpcResponse = { + version: number + type: string + resultcount: number + results: AurRpcResult[] +} + +export class AurPackageFetcher { + private rpcBase = 'https://aur.archlinux.org/rpc/?v=5&type=search&by=name&arg=' + private htmlBase = 'https://aur.archlinux.org/packages' + + private async sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)) + } + + private async fetchWithUA(url: string) { + return fetch(url, { + headers: { + 'user-agent': 'RepoHubBot/0.1 (+https://repohub.local) NodeFetch', + 'accept': 'text/html,application/json;q=0.9,*/*;q=0.8', + 'accept-language': 'en-US,en;q=0.9' + } + }) + } + + async fetchAllPackages(onProgress?: (current: number, total: number, sampleName: string) => void): Promise { + console.log('🅰️ Starting AUR package fetch via RPC...') + + const queries = 'abcdefghijklmnopqrstuvwxyz0123456789'.split('') + const seen = new Set() + const out: AurPackage[] = [] + let totalApprox = 0 + + for (let i = 0; i < queries.length; i++) { + const q = queries[i] + const url = `${this.rpcBase}${encodeURIComponent(q)}` + try { + const res = await this.fetchWithUA(url) + if (!res.ok) { + console.warn(`AUR RPC failed for '${q}': ${res.status}`) + continue + } + const data = (await res.json()) as AurRpcResponse + totalApprox += data.resultcount || 0 + for (const r of data.results || []) { + if (!seen.has(r.Name)) { + seen.add(r.Name) + out.push({ + name: r.Name, + version: r.Version || 'latest', + description: r.Description || `AUR package: ${r.Name}` + }) + } + } + if (onProgress) onProgress(out.length, totalApprox || out.length, out[out.length - 1]?.name || q) + // polite delay + await this.sleep(120) + } catch (e) { + console.error(`AUR RPC error for '${q}':`, e) + } + } + + // If RPC yielded nothing (or environment blocks RPC), fallback to HTML scraping + if (out.length === 0) { + console.log('ℹ️ AUR RPC returned 0 results. Falling back to HTML scraping...') + const htmlResults = await this.fetchAllPackagesHtml(onProgress) + console.log(`✅ AUR HTML fetch completed! Rows: ${htmlResults.length}`) + return htmlResults + } + + console.log(`✅ AUR RPC fetch completed! Unique packages: ${out.length}`) + return out + } + + private async fetchAllPackagesHtml(onProgress?: (current: number, total: number, sampleName: string) => void): Promise { + const results: AurPackage[] = [] + let offset = 0 + const perPage = 50 + let total = 0 + + // Try first page to detect total + const firstUrl = `${this.htmlBase}?O=${offset}&SeB=nd&SB=p` + console.log(`📄 AUR HTML: fetching first page ${firstUrl}`) + const firstRes = await this.fetchWithUA(firstUrl) + if (!firstRes.ok) throw new Error(`AUR HTML HTTP ${firstRes.status}`) + const firstHtml = await firstRes.text() + total = this.extractTotal(firstHtml) || 0 + const firstRows = this.parseAurRows(firstHtml) + results.push(...firstRows) + if (onProgress && firstRows.length) onProgress(results.length, total || results.length, results[results.length - 1].name) + + // Iterate subsequent pages until no rows + while (true) { + offset += perPage + const url = `${this.htmlBase}?O=${offset}&SeB=nd&SB=p` + console.log(`📄 AUR HTML: fetching offset ${offset}`) + const res = await this.fetchWithUA(url) + if (!res.ok) break + const html = await res.text() + const rows = this.parseAurRows(html) + console.log(`🧩 AUR HTML: parsed ${rows.length} rows at offset ${offset}`) + if (rows.length === 0) break + results.push(...rows) + if (onProgress) onProgress(results.length, total || results.length, rows[rows.length - 1].name) + await this.sleep(100) + } + + return results + } + + private extractTotal(html: string): number | null { + // Matches: "101725 packages found.\n Page 1 of 2035." + const m = html.match(/([0-9][0-9,.]*)\s+packages\s+found\./i) + if (m) { + const n = parseInt(m[1].replace(/[,\.]/g, '')) + return Number.isFinite(n) ? n : null + } + return null + } + + private parseAurRows(html: string): AurPackage[] { + const out: AurPackage[] = [] + const rowBlockRegex = /]*>([\s\S]*?)<\/tr>/gi + let rowMatch: RegExpExecArray | null + while ((rowMatch = rowBlockRegex.exec(html)) !== null) { + const row = rowMatch[1] + const nameMatch = row.match(/]*>\s*([^<]+)\s*<\/a>/i) + if (!nameMatch) continue + const name = nameMatch[1].trim() + // find version as the first after the anchor + const afterAnchorIndex = row.indexOf(nameMatch[0]) + nameMatch[0].length + const afterAnchor = row.slice(afterAnchorIndex) + const versionMatch = afterAnchor.match(/]*>\s*([^<]+)\s*<\/td>/i) + const version = (versionMatch?.[1] || '').trim() + const descMatch = row.match(/]*class="[^"]*wrap[^"]*"[^>]*>\s*([\s\S]*?)\s*<\/td>/i) + let description = (descMatch?.[1] || '').replace(/<[^>]*>/g, '').trim() + if (!description && name) description = `AUR package: ${name}` + if (name) { + out.push({ name, version: version || 'latest', description }) + } + } + return out + } + + async storePackages(packages: AurPackage[], onProgress?: (current: number, total: number) => void): Promise { + const { query } = await import('@/lib/database/config') + console.log(`💾 Storing ${packages.length} AUR packages in database...`) + + const batchSize = 100 + let stored = 0 + let updated = 0 + let skipped = 0 + + // Preflight: ensure DB constraint allows 'aur' + try { + const check = await query( + "SELECT pg_get_constraintdef(oid) AS def FROM pg_constraint WHERE conname = 'packages_repository_check'" + ) + const def = check.rows?.[0]?.def as string | undefined + if (!def || !def.toLowerCase().includes("'aur'")) { + throw new Error( + "Database constraint 'packages_repository_check' does not include 'aur'. Please run scripts/migrate-add-aur-to-repository.sql before syncing AUR." + ) + } + } catch (prefErr) { + console.error('[AUR] Preflight constraint check failed:', prefErr) + throw prefErr + } + + for (let i = 0; i < packages.length; i += batchSize) { + const batch = packages.slice(i, i + batchSize) + for (const pkg of batch) { + try { + const existing = await query( + 'SELECT id, version FROM packages WHERE name = $1 AND platform_id = $2', + [pkg.name, 'arch'] + ) + if (existing.rows.length === 0) { + await query( + `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) + VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, + [pkg.name, pkg.description, pkg.version, 'arch', null, 'aur', 0, true] + ) + stored++ + } else if (existing.rows[0].version !== pkg.version) { + await query( + `UPDATE packages SET version = $1, description = $2, repository = $3, updated_at = NOW() WHERE id = $4`, + [pkg.version, pkg.description, 'aur', existing.rows[0].id] + ) + updated++ + } else { + skipped++ + } + if (onProgress && (stored + updated + skipped) % 100 === 0) { + onProgress(stored + updated + skipped, packages.length) + } + } catch (e: any) { + if (e?.code === '23514') { + console.error('[AUR] Constraint violation detected. Abort to avoid wasting time.') + throw new Error( + "Insert failed due to 'packages_repository_check'. Run scripts/migrate-add-aur-to-repository.sql to allow 'aur'." + ) + } + console.error(`Error storing AUR package ${pkg.name}:`, e) + } + } + } + console.log(`✅ AUR stored: ${stored} new, ${updated} updated, ${skipped} skipped`) + } +} diff --git a/src/types/index.ts b/src/types/index.ts index dd81990..717867e 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -16,7 +16,7 @@ export interface Package { type: 'gui' | 'cli' platform?: string | Platform platform_id?: string - repository: 'official' | 'third-party' + repository: 'official' | 'third-party' | 'aur' download_url?: string lastUpdated?: string downloads?: number @@ -29,7 +29,7 @@ export interface FilterOptions { platform_id?: string category_id?: number type?: 'gui' | 'cli' - repository?: 'official' | 'third-party' + repository?: 'official' | 'third-party' | 'aur' search?: string limit?: number offset?: number