diff --git a/package.json b/package.json index 8aeb170..69614d2 100644 --- a/package.json +++ b/package.json @@ -9,38 +9,40 @@ "lint": "next lint", "type-check": "tsc --noEmit", "test:db": "node scripts/test-db.js", - "init:backend": "chmod +x scripts/init-backend.sh && ./scripts/init-backend.sh" + "init:backend": "chmod +x scripts/init-backend.sh && ./scripts/init-backend.sh", + "sync:arch": "npx tsx scripts/sync-arch.ts" }, "dependencies": { - "next": "14.0.4", - "react": "^18", - "react-dom": "^18", - "lucide-react": "^0.294.0", - "@radix-ui/react-select": "^2.0.0", "@radix-ui/react-checkbox": "^1.0.4", "@radix-ui/react-dialog": "^1.0.5", - "@radix-ui/react-tabs": "^1.0.4", "@radix-ui/react-label": "^2.0.2", + "@radix-ui/react-select": "^2.0.0", "@radix-ui/react-slot": "^1.0.2", + "@radix-ui/react-tabs": "^1.0.4", + "cheerio": "^1.1.2", "class-variance-authority": "^0.7.0", "clsx": "^2.0.0", + "lucide-react": "^0.294.0", + "next": "14.0.4", + "pg": "^8.11.3", + "react": "^18", + "react-dom": "^18", + "react-query": "^3.39.3", "tailwind-merge": "^2.1.0", "tailwindcss-animate": "^1.0.7", - "react-query": "^3.39.3", - "zustand": "^4.4.7", - "pg": "^8.11.3", - "undici": "^6.6.2" + "undici": "^6.6.2", + "zustand": "^4.4.7" }, "devDependencies": { - "typescript": "^5", "@types/node": "^20", + "@types/pg": "^8.10.9", "@types/react": "^18", "@types/react-dom": "^18", - "@types/pg": "^8.10.9", "autoprefixer": "^10.0.1", + "eslint": "^8", + "eslint-config-next": "14.0.4", "postcss": "^8", "tailwindcss": "^3.3.0", - "eslint": "^8", - "eslint-config-next": "14.0.4" + "typescript": "^5" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 41dd768..ffb83ed 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: '@radix-ui/react-tabs': specifier: ^1.0.4 version: 1.1.13(@types/react-dom@18.3.7(@types/react@18.3.26))(@types/react@18.3.26)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + cheerio: + specifier: ^1.1.2 + version: 1.1.2 class-variance-authority: specifier: ^0.7.0 version: 0.7.1 @@ -899,6 +902,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -948,6 +954,13 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + + cheerio@1.1.2: + resolution: {integrity: sha512-IkxPpb5rS/d1IiLbHMgfPuS0FgiWTtFIm/Nj+2woXDLTZ7fOT2eqzgYbdMlLweqlHbsZjxEChoVK+7iph7jyQg==} + engines: {node: '>=20.18.1'} + chokidar@3.6.0: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} @@ -980,6 +993,13 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + css-select@5.2.2: + resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==} + + css-what@6.2.2: + resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==} + engines: {node: '>= 6'} + cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -1055,6 +1075,19 @@ packages: resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} engines: {node: '>=6.0.0'} + dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + + domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + + domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + + domutils@3.2.2: + resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==} + dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -1071,6 +1104,17 @@ packages: emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + encoding-sniffer@0.2.1: + resolution: {integrity: sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==} + + entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + + entities@6.0.1: + resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==} + engines: {node: '>=0.12'} + es-abstract@1.24.0: resolution: {integrity: sha512-WSzPgsdLtTcQwm4CROfS5ju2Wa1QQcVeT37jFjYzdFz1r9ahadC8B8/a4qxJxM+09F18iumCdRmlr96ZYkQvEg==} engines: {node: '>= 0.4'} @@ -1387,6 +1431,13 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + htmlparser2@10.0.0: + resolution: {integrity: sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==} + + iconv-lite@0.6.3: + resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} + engines: {node: '>=0.10.0'} + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -1693,6 +1744,9 @@ packages: resolution: {integrity: sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==} engines: {node: '>=0.10.0'} + nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -1758,6 +1812,15 @@ packages: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} + parse5-htmlparser2-tree-adapter@7.1.0: + resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==} + + parse5-parser-stream@7.1.2: + resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==} + + parse5@7.3.0: + resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -2031,6 +2094,9 @@ packages: resolution: {integrity: sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==} engines: {node: '>= 0.4'} + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + scheduler@0.23.2: resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==} @@ -2265,6 +2331,10 @@ packages: resolution: {integrity: sha512-hU/10obOIu62MGYjdskASR3CUAiYaFTtC9Pa6vHyf//mAipSvSQg6od2CnJswq7fvzNS3zJhxoRkgNVaHurWKw==} engines: {node: '>=18.17'} + undici@7.16.0: + resolution: {integrity: sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==} + engines: {node: '>=20.18.1'} + unload@2.2.0: resolution: {integrity: sha512-B60uB5TNBLtN6/LsgAf3udH9saB5p7gqJwcFfbOEZ8BcBHnGwCf6G/TGiEqkRAxX7zAFIUtzdrXQSdL3Q/wqNA==} @@ -2312,6 +2382,14 @@ packages: resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} engines: {node: '>=10.13.0'} + whatwg-encoding@3.1.1: + resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} + engines: {node: '>=18'} + + whatwg-mimetype@4.0.0: + resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} + engines: {node: '>=18'} + which-boxed-primitive@1.1.1: resolution: {integrity: sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==} engines: {node: '>= 0.4'} @@ -3125,6 +3203,8 @@ snapshots: binary-extensions@2.3.0: {} + boolbase@1.0.0: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -3189,6 +3269,29 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + cheerio-select@2.1.0: + dependencies: + boolbase: 1.0.0 + css-select: 5.2.2 + css-what: 6.2.2 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + + cheerio@1.1.2: + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.2.2 + encoding-sniffer: 0.2.1 + htmlparser2: 10.0.0 + parse5: 7.3.0 + parse5-htmlparser2-tree-adapter: 7.1.0 + parse5-parser-stream: 7.1.2 + undici: 7.16.0 + whatwg-mimetype: 4.0.0 + chokidar@3.6.0: dependencies: anymatch: 3.1.3 @@ -3225,6 +3328,16 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + css-select@5.2.2: + dependencies: + boolbase: 1.0.0 + css-what: 6.2.2 + domhandler: 5.0.3 + domutils: 3.2.2 + nth-check: 2.1.1 + + css-what@6.2.2: {} + cssesc@3.0.0: {} csstype@3.1.3: {} @@ -3291,6 +3404,24 @@ snapshots: dependencies: esutils: 2.0.3 + dom-serializer@2.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + + domelementtype@2.3.0: {} + + domhandler@5.0.3: + dependencies: + domelementtype: 2.3.0 + + domutils@3.2.2: + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dunder-proto@1.0.1: dependencies: call-bind-apply-helpers: 1.0.2 @@ -3305,6 +3436,15 @@ snapshots: emoji-regex@9.2.2: {} + encoding-sniffer@0.2.1: + dependencies: + iconv-lite: 0.6.3 + whatwg-encoding: 3.1.1 + + entities@4.5.0: {} + + entities@6.0.1: {} + es-abstract@1.24.0: dependencies: array-buffer-byte-length: 1.0.2 @@ -3792,6 +3932,17 @@ snapshots: dependencies: function-bind: 1.1.2 + htmlparser2@10.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + entities: 6.0.1 + + iconv-lite@0.6.3: + dependencies: + safer-buffer: 2.1.2 + ignore@5.3.2: {} import-fresh@3.3.1: @@ -4097,6 +4248,10 @@ snapshots: normalize-range@0.1.2: {} + nth-check@2.1.1: + dependencies: + boolbase: 1.0.0 + object-assign@4.1.1: {} object-hash@3.0.0: {} @@ -4176,6 +4331,19 @@ snapshots: dependencies: callsites: 3.1.0 + parse5-htmlparser2-tree-adapter@7.1.0: + dependencies: + domhandler: 5.0.3 + parse5: 7.3.0 + + parse5-parser-stream@7.1.2: + dependencies: + parse5: 7.3.0 + + parse5@7.3.0: + dependencies: + entities: 6.0.1 + path-exists@4.0.0: {} path-is-absolute@1.0.1: {} @@ -4426,6 +4594,8 @@ snapshots: es-errors: 1.3.0 is-regex: 1.2.1 + safer-buffer@2.1.2: {} + scheduler@0.23.2: dependencies: loose-envify: 1.4.0 @@ -4722,6 +4892,8 @@ snapshots: undici@6.22.0: {} + undici@7.16.0: {} + unload@2.2.0: dependencies: '@babel/runtime': 7.28.4 @@ -4787,6 +4959,12 @@ snapshots: glob-to-regexp: 0.4.1 graceful-fs: 4.2.11 + whatwg-encoding@3.1.1: + dependencies: + iconv-lite: 0.6.3 + + whatwg-mimetype@4.0.0: {} + which-boxed-primitive@1.1.1: dependencies: is-bigint: 1.1.0 diff --git a/src/app/api/debug-arch/route.ts b/src/app/api/debug-arch/route.ts new file mode 100644 index 0000000..a4a1692 --- /dev/null +++ b/src/app/api/debug-arch/route.ts @@ -0,0 +1,36 @@ +import { NextResponse } from 'next/server' +import { query } from '@/lib/database/config' + +export async function GET() { + try { + // Check if arch platform exists + const platformResult = await query( + 'SELECT * FROM platforms WHERE id = $1', + ['arch'] + ) + + // Count arch packages + const countResult = await query( + 'SELECT COUNT(*) FROM packages WHERE platform_id = $1', + ['arch'] + ) + + // Get sample packages + const sampleResult = await query( + 'SELECT name, version, description FROM packages WHERE platform_id = $1 LIMIT 5', + ['arch'] + ) + + return NextResponse.json({ + platform: platformResult.rows[0] || null, + totalPackages: parseInt(countResult.rows[0].count), + samplePackages: sampleResult.rows + }) + } catch (error) { + console.error('Debug error:', error) + return NextResponse.json( + { error: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ) + } +} diff --git a/src/app/api/sync-arch/route.ts b/src/app/api/sync-arch/route.ts new file mode 100644 index 0000000..292afc3 --- /dev/null +++ b/src/app/api/sync-arch/route.ts @@ -0,0 +1,86 @@ +import { NextResponse } from 'next/server' +import { ArchPackageFetcher } from '@/services/archPackageFetcher' + +export const dynamic = 'force-dynamic' +export const maxDuration = 300 // 5 minutes timeout + +let syncInProgress = false +let syncStatus = { + status: 'idle' as 'idle' | 'fetching' | 'storing' | 'complete' | 'error', + fetchProgress: 0, + fetchTotal: 0, + storeProgress: 0, + storeTotal: 0, + currentPackage: '', + error: null as string | null +} + +export async function GET() { + return NextResponse.json(syncStatus) +} + +export async function POST() { + if (syncInProgress) { + return NextResponse.json( + { error: 'Sync already in progress' }, + { status: 409 } + ) + } + + syncInProgress = true + syncStatus = { + status: 'fetching', + fetchProgress: 0, + fetchTotal: 0, + storeProgress: 0, + storeTotal: 0, + currentPackage: '', + error: null + } + + // Start sync in background + ;(async () => { + try { + const fetcher = new ArchPackageFetcher() + + // Fetch packages + console.log('🔄 Starting Arch Linux package fetch...') + const packages = await fetcher.fetchAllPackages( + (current, total, packageName) => { + syncStatus.fetchProgress = current + syncStatus.fetchTotal = total + syncStatus.currentPackage = packageName + } + ) + + console.log(`✅ Fetched ${packages.length} Arch packages`) + + // Store packages + syncStatus.status = 'storing' + syncStatus.storeTotal = packages.length + + await fetcher.storePackages( + packages, + (current, total) => { + syncStatus.storeProgress = current + syncStatus.storeTotal = total + } + ) + + syncStatus.status = 'complete' + console.log('✅ Arch Linux sync completed successfully') + + } catch (error) { + console.error('❌ Arch sync error:', error) + syncStatus.status = 'error' + syncStatus.error = error instanceof Error ? error.message : 'Unknown error' + } finally { + syncInProgress = false + } + })() + + return NextResponse.json({ + message: 'Arch Linux package sync started', + status: syncStatus + }) +} diff --git a/src/services/archPackageFetcher.ts b/src/services/archPackageFetcher.ts new file mode 100644 index 0000000..1110731 --- /dev/null +++ b/src/services/archPackageFetcher.ts @@ -0,0 +1,212 @@ +interface ArchPackage { + name: string + version: string + description: string + architecture: string + repository: string + lastUpdated: string +} + +// Simple HTML parser without external dependencies +function parseHTML(html: string) { + return { + querySelector: (selector: string) => { + const regex = new RegExp(`<${selector}[^>]*>([\\s\\S]*?)<\\/${selector}>`, 'gi') + const match = regex.exec(html) + return match ? { textContent: match[1].replace(/<[^>]*>/g, '').trim() } : null + }, + querySelectorAll: (selector: string) => { + const results: any[] = [] + const regex = new RegExp(`<${selector}[^>]*>([\\s\\S]*?)<\\/${selector}>`, 'gi') + let match + while ((match = regex.exec(html)) !== null) { + results.push({ innerHTML: match[1], textContent: match[1].replace(/<[^>]*>/g, '').trim() }) + } + return results + } + } +} + +export class ArchPackageFetcher { + private baseUrl = 'https://archlinux.org/packages' + + /** + * Fetch all Arch Linux packages with progress callback + */ + async fetchAllPackages( + onProgress?: (current: number, total: number, packageName: string) => void + ): Promise { + console.log('🔍 Starting Arch Linux package fetch...') + + const allPackages: ArchPackage[] = [] + let currentPage = 1 + let totalPages = 1 + + try { + // Fetch first page to get total pages + const firstPageUrl = `${this.baseUrl}/?page=1` + console.log(`📄 Fetching first page: ${firstPageUrl}`) + + const firstPageResponse = await fetch(firstPageUrl) + if (!firstPageResponse.ok) { + throw new Error(`HTTP error! status: ${firstPageResponse.status}`) + } + + const firstPageHtml = await firstPageResponse.text() + + // Parse total packages and pages + const statsMatch = firstPageHtml.match(/(\d+)\s+matching packages[\s\S]*?Page \d+ of (\d+)/) + const totalPackages = statsMatch ? parseInt(statsMatch[1]) : 0 + totalPages = statsMatch ? parseInt(statsMatch[2]) : 1 + + console.log(`📦 Total packages: ${totalPackages}`) + console.log(`📄 Total pages: ${totalPages}`) + + // Parse first page packages + const firstPagePackages = this.parsePackagesFromHtml(firstPageHtml) + allPackages.push(...firstPagePackages) + + if (onProgress) { + onProgress(allPackages.length, totalPackages, firstPagePackages[0]?.name || 'N/A') + } + + // Fetch remaining pages + for (currentPage = 2; currentPage <= totalPages; currentPage++) { + const pageUrl = `${this.baseUrl}/?page=${currentPage}` + console.log(`📄 Fetching page ${currentPage}/${totalPages}...`) + + try { + const response = await fetch(pageUrl) + if (!response.ok) { + console.error(`❌ Failed to fetch page ${currentPage}: ${response.status}`) + continue + } + + const html = await response.text() + const packages = this.parsePackagesFromHtml(html) + + allPackages.push(...packages) + + if (onProgress && packages.length > 0) { + onProgress(allPackages.length, totalPackages, packages[0].name) + } + + // Rate limiting: wait 100ms between requests + await new Promise(resolve => setTimeout(resolve, 100)) + + } catch (error) { + console.error(`❌ Error fetching page ${currentPage}:`, error) + } + } + + console.log(`✅ Fetch completed! Total packages: ${allPackages.length}`) + return allPackages + + } catch (error) { + console.error('❌ Error fetching Arch packages:', error) + throw error + } + } + + /** + * Parse packages from HTML using regex + */ + private parsePackagesFromHtml(html: string): ArchPackage[] { + const packages: ArchPackage[] = [] + + // Match all table rows + const rowRegex = /\s*([^<]+)<\/td>\s*([^<]+)<\/td>\s*]*>([^<]+)<\/a><\/td>\s*([^<]+)<\/td>\s*]*>([^<]*)<\/td>\s*([^<]*)<\/td>/gi + + let match + while ((match = rowRegex.exec(html)) !== null) { + try { + const architecture = match[1].trim() + const repository = match[2].trim() + const name = match[3].trim() + const version = match[4].trim() + const description = match[5].trim() + const lastUpdated = match[6].trim() + + if (name && version) { + packages.push({ + name, + version, + description: description || 'No description available', + architecture, + repository: repository.toLowerCase(), + lastUpdated + }) + } + } catch (error) { + console.error('Error parsing package row:', error) + } + } + + return packages + } + + /** + * Store packages in database + */ + async storePackages( + packages: ArchPackage[], + onProgress?: (current: number, total: number) => void + ): Promise { + const { query } = await import('@/lib/database/config') + + console.log(`💾 Storing ${packages.length} Arch packages in database...`) + + const batchSize = 100 + let stored = 0 + let updated = 0 + let skipped = 0 + + for (let i = 0; i < packages.length; i += batchSize) { + const batch = packages.slice(i, i + batchSize) + + for (const pkg of batch) { + try { + // Check if package already exists + const existingResult = await query( + 'SELECT id, version FROM packages WHERE name = $1 AND platform_id = $2', + [pkg.name, 'arch'] + ) + + const repository = (pkg.repository === 'core' || pkg.repository === 'extra') + ? 'official' + : 'third-party' + + if (existingResult.rows.length === 0) { + // Insert new package (id will be auto-generated by SERIAL) + await query( + `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) + VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, + [pkg.name, pkg.description, pkg.version, 'arch', 'cli', repository, 0, true] + ) + stored++ + } else if (existingResult.rows[0].version !== pkg.version) { + // Update if version changed + await query( + `UPDATE packages + SET version = $1, description = $2, repository = $3, updated_at = NOW() + WHERE id = $4`, + [pkg.version, pkg.description, repository, existingResult.rows[0].id] + ) + updated++ + } else { + skipped++ + } + + if (onProgress && (stored + updated + skipped) % 100 === 0) { + onProgress(stored + updated + skipped, packages.length) + } + + } catch (error) { + console.error(`Error storing package ${pkg.name}:`, error) + } + } + } + + console.log(`✅ Stored ${stored} new packages, updated ${updated}, skipped ${skipped}`) + } +}