feat: add AUR repository support for Arch Linux packages

- Extended database schema and models to include 'aur' as a valid repository type alongside 'official' and 'third-party' - Added repository filter UI in package browser for Arch platform to distinguish between official and AUR packages - Enhanced init-db script with migration system to track and apply schema changes automatically
2026-09-11 10:36:07 +00:00 · 2025-11-12 20:43:50 +03:00
parent cce1098282
commit 560e78cb96
11 changed files with 395 additions and 226 deletions
@@ -11,7 +11,7 @@ export async function GET(request: NextRequest) {
      category_id: searchParams.get('category_id') ? 
        parseInt(searchParams.get('category_id')!) : undefined,
      type: searchParams.get('type') as 'gui' | 'cli' | undefined,
-      repository: searchParams.get('repository') as 'official' | 'third-party' | undefined,
+      repository: searchParams.get('repository') as 'official' | 'third-party' | 'aur' | undefined,
      search: searchParams.get('search') || undefined,
      limit: searchParams.get('limit') ? 
        parseInt(searchParams.get('limit')!) : undefined,
@@ -0,0 +1,52 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { SyncAuth } from '@/lib/sync/auth'
+import { AurPackageFetcher } from '@/services/aurPackageFetcher'
+
+export const dynamic = 'force-dynamic'
+export const maxDuration = 300
+
+let syncInProgress = false
+let syncStatus: {
+  status: 'idle' | 'running' | 'complete' | 'error'
+  error: string | null
+} = { status: 'idle', error: null }
+
+export async function GET() {
+  return NextResponse.json(syncStatus)
+}
+
+export async function POST(request: NextRequest) {
+  const authResult = await SyncAuth.isSyncAllowed(request)
+  if (!authResult.allowed) {
+    return NextResponse.json(
+      { error: 'Sync operation not allowed', reason: authResult.reason },
+      { status: 403 }
+    )
+  }
+
+  if (syncInProgress) {
+    return NextResponse.json(
+      { error: 'Sync already in progress' },
+      { status: 409 }
+    )
+  }
+
+  syncInProgress = true
+  syncStatus = { status: 'running', error: null }
+
+  ;(async () => {
+    try {
+      const fetcher = new AurPackageFetcher()
+      const pkgs = await fetcher.fetchAllPackages()
+      await fetcher.storePackages(pkgs)
+      syncStatus.status = 'complete'
+    } catch (err: any) {
+      syncStatus.status = 'error'
+      syncStatus.error = err?.message || 'Unknown error'
+    } finally {
+      syncInProgress = false
+    }
+  })()
+
+  return NextResponse.json({ message: 'AUR sync started', status: syncStatus })
+}
@@ -31,10 +31,12 @@ export function PackageBrowserV2({
  const [totalCount, setTotalCount] = useState(0)
  const [searchQuery, setSearchQuery] = useState('')
  const [typeFilter, setTypeFilter] = useState<string>('')
+  const [repositoryFilter, setRepositoryFilter] = useState<string>('')
  const isDebianUbuntu = useMemo(() => {
    const id = selectedPlatform?.id || ''
    return id === 'debian' || id === 'ubuntu'
  }, [selectedPlatform])
+  const isArch = useMemo(() => (selectedPlatform?.id || '') === 'arch', [selectedPlatform])
  const [scrollPosition, setScrollPosition] = useState(0)
  const searchInputRef = useRef<HTMLInputElement>(null)

@@ -66,6 +68,9 @@ export function PackageBrowserV2({
        if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
          params.type = typeFilter as 'gui' | 'cli'
        }
+        if (isArch && repositoryFilter && repositoryFilter !== 'all') {
+          params.repository = repositoryFilter as 'official' | 'aur'
+        }
        
        console.log('🔍 Frontend: API params:', params)
        const result = await apiClient.getPackages(params)
@@ -86,7 +91,7 @@ export function PackageBrowserV2({
    }

    loadPackages()
-  }, [selectedPlatform, typeFilter, isDebianUbuntu])
+  }, [selectedPlatform, typeFilter, repositoryFilter, isDebianUbuntu, isArch])

  // Debounced search to prevent focus loss
  useEffect(() => {
@@ -111,6 +116,9 @@ export function PackageBrowserV2({
          if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
            params.type = typeFilter as 'gui' | 'cli'
          }
+          if (isArch && repositoryFilter && repositoryFilter !== 'all') {
+            params.repository = repositoryFilter as 'official' | 'aur'
+          }
          
          console.log('🔍 Frontend: Debounced API params:', params)
          const result = await apiClient.getPackages(params)
@@ -130,7 +138,7 @@ export function PackageBrowserV2({
    }, 300) // 300ms debounce

    return () => clearTimeout(timeoutId)
-  }, [searchQuery, isDebianUbuntu, typeFilter, selectedPlatform])
+  }, [searchQuery, isDebianUbuntu, typeFilter, isArch, repositoryFilter, selectedPlatform])

  // Load more packages
  const loadMore = async () => {
@@ -151,6 +159,9 @@ export function PackageBrowserV2({
      if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
        params.type = typeFilter as 'gui' | 'cli'
      }
+      if (isArch && repositoryFilter && repositoryFilter !== 'all') {
+        params.repository = repositoryFilter as 'official' | 'aur'
+      }
      
      const result = await apiClient.getPackages(params)
      setPackages(prev => [...prev, ...result.packages])
@@ -257,6 +268,18 @@ export function PackageBrowserV2({
              </SelectContent>
            </Select>
          )}
+          {isArch && (
+            <Select value={repositoryFilter || "all"} onValueChange={(value) => setRepositoryFilter(value === "all" ? "" : value)}>
+              <SelectTrigger className="w-full sm:w-40">
+                <SelectValue placeholder="Repository" />
+              </SelectTrigger>
+              <SelectContent>
+                <SelectItem value="all">All</SelectItem>
+                <SelectItem value="official">Official</SelectItem>
+                <SelectItem value="aur">AUR</SelectItem>
+              </SelectContent>
+            </Select>
+          )}
          </div>

          {/* Package List */}
@@ -311,6 +334,11 @@ export function PackageBrowserV2({
                            {pkg.type.toUpperCase()}
                          </span>
                        )}
+                        {pkg.repository === 'aur' && (
+                          <span className="text-xs text-muted-foreground">
+                            AUR
+                          </span>
+                        )}
                        {pkg.repository === 'third-party' && (
                          <span className="text-xs text-muted-foreground">
                            third-party
@@ -37,7 +37,7 @@ CREATE TABLE packages (
    category_id INTEGER REFERENCES categories(id),
    license_id INTEGER REFERENCES licenses(id),
    type VARCHAR(10) CHECK (type IN ('gui', 'cli')),
-    repository VARCHAR(20) CHECK (repository IN ('official', 'third-party')),
+    repository VARCHAR(20) CHECK (repository IN ('official', 'third-party', 'aur')),
    homepage_url VARCHAR(255),
    download_url VARCHAR(255),
    last_updated TIMESTAMP WITH TIME ZONE,
@@ -7,7 +7,7 @@ export interface Package {
  category_id?: number
  license_id?: number
  type?: 'gui' | 'cli'
-  repository?: 'official' | 'third-party'
+  repository?: 'official' | 'third-party' | 'aur'
  homepage_url?: string
  download_url?: string
  last_updated?: Date
@@ -53,7 +53,7 @@ export interface CreatePackageInput {
  category_id?: number
  license_id?: number
  type?: 'gui' | 'cli'
-  repository?: 'official' | 'third-party'
+  repository?: 'official' | 'third-party' | 'aur'
  homepage_url?: string
  download_url?: string
  popularity_score?: number
@@ -66,7 +66,7 @@ export interface UpdatePackageInput {
  category_id?: number
  license_id?: number
  type?: 'gui' | 'cli'
-  repository?: 'official' | 'third-party'
+  repository?: 'official' | 'third-party' | 'aur'
  homepage_url?: string
  download_url?: string
  popularity_score?: number
@@ -77,7 +77,7 @@ export interface PackageFilter {
  platform_id?: string
  category_id?: number
  type?: 'gui' | 'cli'
-  repository?: 'official' | 'third-party'
+  repository?: 'official' | 'third-party' | 'aur'
  search?: string
  limit?: number
  offset?: number
@@ -177,7 +177,7 @@ export class ArchPackageFetcher {
            await query(
              `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active)
               VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`,
-              [pkg.name, pkg.description, pkg.version, 'arch', null, null, 0, true]
+              [pkg.name, pkg.description, pkg.version, 'arch', null, 'official', 0, true]
            )
            stored++
          } else if (existingResult.rows[0].version !== pkg.version) {
@@ -186,7 +186,7 @@ export class ArchPackageFetcher {
              `UPDATE packages 
               SET version = $1, description = $2, repository = $3, updated_at = NOW()
               WHERE id = $4`,
-              [pkg.version, pkg.description, null, existingResult.rows[0].id]
+              [pkg.version, pkg.description, 'official', existingResult.rows[0].id]
            )
            updated++
          } else {
@@ -0,0 +1,223 @@
+interface AurPackage {
+  name: string
+  version: string
+  description: string
+}
+
+type AurRpcResult = {
+  ID: number
+  Name: string
+  Version: string
+  Description: string
+}
+
+type AurRpcResponse = {
+  version: number
+  type: string
+  resultcount: number
+  results: AurRpcResult[]
+}
+
+export class AurPackageFetcher {
+  private rpcBase = 'https://aur.archlinux.org/rpc/?v=5&type=search&by=name&arg='
+  private htmlBase = 'https://aur.archlinux.org/packages'
+
+  private async sleep(ms: number) {
+    return new Promise((r) => setTimeout(r, ms))
+  }
+
+  private async fetchWithUA(url: string) {
+    return fetch(url, {
+      headers: {
+        'user-agent': 'RepoHubBot/0.1 (+https://repohub.local) NodeFetch',
+        'accept': 'text/html,application/json;q=0.9,*/*;q=0.8',
+        'accept-language': 'en-US,en;q=0.9'
+      }
+    })
+  }
+
+  async fetchAllPackages(onProgress?: (current: number, total: number, sampleName: string) => void): Promise<AurPackage[]> {
+    console.log('🅰️  Starting AUR package fetch via RPC...')
+
+    const queries = 'abcdefghijklmnopqrstuvwxyz0123456789'.split('')
+    const seen = new Set<string>()
+    const out: AurPackage[] = []
+    let totalApprox = 0
+
+    for (let i = 0; i < queries.length; i++) {
+      const q = queries[i]
+      const url = `${this.rpcBase}${encodeURIComponent(q)}`
+      try {
+        const res = await this.fetchWithUA(url)
+        if (!res.ok) {
+          console.warn(`AUR RPC failed for '${q}': ${res.status}`)
+          continue
+        }
+        const data = (await res.json()) as AurRpcResponse
+        totalApprox += data.resultcount || 0
+        for (const r of data.results || []) {
+          if (!seen.has(r.Name)) {
+            seen.add(r.Name)
+            out.push({
+              name: r.Name,
+              version: r.Version || 'latest',
+              description: r.Description || `AUR package: ${r.Name}`
+            })
+          }
+        }
+        if (onProgress) onProgress(out.length, totalApprox || out.length, out[out.length - 1]?.name || q)
+        // polite delay
+        await this.sleep(120)
+      } catch (e) {
+        console.error(`AUR RPC error for '${q}':`, e)
+      }
+    }
+
+    // If RPC yielded nothing (or environment blocks RPC), fallback to HTML scraping
+    if (out.length === 0) {
+      console.log('ℹ️ AUR RPC returned 0 results. Falling back to HTML scraping...')
+      const htmlResults = await this.fetchAllPackagesHtml(onProgress)
+      console.log(`✅ AUR HTML fetch completed! Rows: ${htmlResults.length}`)
+      return htmlResults
+    }
+
+    console.log(`✅ AUR RPC fetch completed! Unique packages: ${out.length}`)
+    return out
+  }
+
+  private async fetchAllPackagesHtml(onProgress?: (current: number, total: number, sampleName: string) => void): Promise<AurPackage[]> {
+    const results: AurPackage[] = []
+    let offset = 0
+    const perPage = 50
+    let total = 0
+
+    // Try first page to detect total
+    const firstUrl = `${this.htmlBase}?O=${offset}&SeB=nd&SB=p`
+    console.log(`📄 AUR HTML: fetching first page ${firstUrl}`)
+    const firstRes = await this.fetchWithUA(firstUrl)
+    if (!firstRes.ok) throw new Error(`AUR HTML HTTP ${firstRes.status}`)
+    const firstHtml = await firstRes.text()
+    total = this.extractTotal(firstHtml) || 0
+    const firstRows = this.parseAurRows(firstHtml)
+    results.push(...firstRows)
+    if (onProgress && firstRows.length) onProgress(results.length, total || results.length, results[results.length - 1].name)
+
+    // Iterate subsequent pages until no rows
+    while (true) {
+      offset += perPage
+      const url = `${this.htmlBase}?O=${offset}&SeB=nd&SB=p`
+      console.log(`📄 AUR HTML: fetching offset ${offset}`)
+      const res = await this.fetchWithUA(url)
+      if (!res.ok) break
+      const html = await res.text()
+      const rows = this.parseAurRows(html)
+      console.log(`🧩 AUR HTML: parsed ${rows.length} rows at offset ${offset}`)
+      if (rows.length === 0) break
+      results.push(...rows)
+      if (onProgress) onProgress(results.length, total || results.length, rows[rows.length - 1].name)
+      await this.sleep(100)
+    }
+
+    return results
+  }
+
+  private extractTotal(html: string): number | null {
+    // Matches: "101725 packages found.\n  Page 1 of 2035."
+    const m = html.match(/([0-9][0-9,.]*)\s+packages\s+found\./i)
+    if (m) {
+      const n = parseInt(m[1].replace(/[,\.]/g, ''))
+      return Number.isFinite(n) ? n : null
+    }
+    return null
+  }
+
+  private parseAurRows(html: string): AurPackage[] {
+    const out: AurPackage[] = []
+    const rowBlockRegex = /<tr[^>]*>([\s\S]*?)<\/tr>/gi
+    let rowMatch: RegExpExecArray | null
+    while ((rowMatch = rowBlockRegex.exec(html)) !== null) {
+      const row = rowMatch[1]
+      const nameMatch = row.match(/<a\s+href="\/packages\/[^"]+"[^>]*>\s*([^<]+)\s*<\/a>/i)
+      if (!nameMatch) continue
+      const name = nameMatch[1].trim()
+      // find version as the first <td> after the anchor
+      const afterAnchorIndex = row.indexOf(nameMatch[0]) + nameMatch[0].length
+      const afterAnchor = row.slice(afterAnchorIndex)
+      const versionMatch = afterAnchor.match(/<td[^>]*>\s*([^<]+)\s*<\/td>/i)
+      const version = (versionMatch?.[1] || '').trim()
+      const descMatch = row.match(/<td[^>]*class="[^"]*wrap[^"]*"[^>]*>\s*([\s\S]*?)\s*<\/td>/i)
+      let description = (descMatch?.[1] || '').replace(/<[^>]*>/g, '').trim()
+      if (!description && name) description = `AUR package: ${name}`
+      if (name) {
+        out.push({ name, version: version || 'latest', description })
+      }
+    }
+    return out
+  }
+
+  async storePackages(packages: AurPackage[], onProgress?: (current: number, total: number) => void): Promise<void> {
+    const { query } = await import('@/lib/database/config')
+    console.log(`💾 Storing ${packages.length} AUR packages in database...`)
+
+    const batchSize = 100
+    let stored = 0
+    let updated = 0
+    let skipped = 0
+
+    // Preflight: ensure DB constraint allows 'aur'
+    try {
+      const check = await query(
+        "SELECT pg_get_constraintdef(oid) AS def FROM pg_constraint WHERE conname = 'packages_repository_check'"
+      )
+      const def = check.rows?.[0]?.def as string | undefined
+      if (!def || !def.toLowerCase().includes("'aur'")) {
+        throw new Error(
+          "Database constraint 'packages_repository_check' does not include 'aur'. Please run scripts/migrate-add-aur-to-repository.sql before syncing AUR."
+        )
+      }
+    } catch (prefErr) {
+      console.error('[AUR] Preflight constraint check failed:', prefErr)
+      throw prefErr
+    }
+
+    for (let i = 0; i < packages.length; i += batchSize) {
+      const batch = packages.slice(i, i + batchSize)
+      for (const pkg of batch) {
+        try {
+          const existing = await query(
+            'SELECT id, version FROM packages WHERE name = $1 AND platform_id = $2',
+            [pkg.name, 'arch']
+          )
+          if (existing.rows.length === 0) {
+            await query(
+              `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active)
+               VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`,
+              [pkg.name, pkg.description, pkg.version, 'arch', null, 'aur', 0, true]
+            )
+            stored++
+          } else if (existing.rows[0].version !== pkg.version) {
+            await query(
+              `UPDATE packages SET version = $1, description = $2, repository = $3, updated_at = NOW() WHERE id = $4`,
+              [pkg.version, pkg.description, 'aur', existing.rows[0].id]
+            )
+            updated++
+          } else {
+            skipped++
+          }
+          if (onProgress && (stored + updated + skipped) % 100 === 0) {
+            onProgress(stored + updated + skipped, packages.length)
+          }
+        } catch (e: any) {
+          if (e?.code === '23514') {
+            console.error('[AUR] Constraint violation detected. Abort to avoid wasting time.')
+            throw new Error(
+              "Insert failed due to 'packages_repository_check'. Run scripts/migrate-add-aur-to-repository.sql to allow 'aur'."
+            )
+          }
+          console.error(`Error storing AUR package ${pkg.name}:`, e)
+        }
+      }
+    }
+    console.log(`✅ AUR stored: ${stored} new, ${updated} updated, ${skipped} skipped`)
+  }
+}
@@ -16,7 +16,7 @@ export interface Package {
  type: 'gui' | 'cli'
  platform?: string | Platform
  platform_id?: string
-  repository: 'official' | 'third-party'
+  repository: 'official' | 'third-party' | 'aur'
  download_url?: string
  lastUpdated?: string
  downloads?: number
@@ -29,7 +29,7 @@ export interface FilterOptions {
  platform_id?: string
  category_id?: number
  type?: 'gui' | 'cli'
-  repository?: 'official' | 'third-party'
+  repository?: 'official' | 'third-party' | 'aur'
  search?: string
  limit?: number
  offset?: number