mirror of
https://github.com/yusufipk/RepoHub.git
synced 2026-09-11 10:36:07 +00:00
feat: Implement request cancellation for package fetching using AbortController in PackageBrowserV2 and update API client to support it.
This commit is contained in:
@@ -17,11 +17,11 @@ interface PackageBrowserProps {
|
|||||||
onFiltersChange: (filters: FilterOptions) => void
|
onFiltersChange: (filters: FilterOptions) => void
|
||||||
}
|
}
|
||||||
|
|
||||||
export function PackageBrowserV2({
|
export function PackageBrowserV2({
|
||||||
selectedPlatform,
|
selectedPlatform,
|
||||||
selectedPackages,
|
selectedPackages,
|
||||||
onPackageToggle,
|
onPackageToggle,
|
||||||
onFiltersChange
|
onFiltersChange
|
||||||
}: PackageBrowserProps) {
|
}: PackageBrowserProps) {
|
||||||
const { t } = useLocale()
|
const { t } = useLocale()
|
||||||
const [packages, setPackages] = useState<Package[]>([])
|
const [packages, setPackages] = useState<Package[]>([])
|
||||||
@@ -42,6 +42,8 @@ export function PackageBrowserV2({
|
|||||||
|
|
||||||
// Load packages when platform changes
|
// Load packages when platform changes
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
const controller = new AbortController()
|
||||||
|
|
||||||
const loadPackages = async () => {
|
const loadPackages = async () => {
|
||||||
if (!selectedPlatform) {
|
if (!selectedPlatform) {
|
||||||
setPackages([])
|
setPackages([])
|
||||||
@@ -52,7 +54,7 @@ export function PackageBrowserV2({
|
|||||||
setLoading(true)
|
setLoading(true)
|
||||||
setPackages([])
|
setPackages([])
|
||||||
setHasMore(true)
|
setHasMore(true)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
console.log('🔍 Frontend: Fetching initial packages for', selectedPlatform.id)
|
console.log('🔍 Frontend: Fetching initial packages for', selectedPlatform.id)
|
||||||
const params: any = {
|
const params: any = {
|
||||||
@@ -60,20 +62,20 @@ export function PackageBrowserV2({
|
|||||||
limit: 50,
|
limit: 50,
|
||||||
offset: 0
|
offset: 0
|
||||||
}
|
}
|
||||||
|
|
||||||
if (searchQuery && searchQuery.trim()) {
|
if (searchQuery && searchQuery.trim()) {
|
||||||
params.search = searchQuery.trim()
|
params.search = searchQuery.trim()
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
|
if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
|
||||||
params.type = typeFilter as 'gui' | 'cli'
|
params.type = typeFilter as 'gui' | 'cli'
|
||||||
}
|
}
|
||||||
if (isArch && repositoryFilter && repositoryFilter !== 'all') {
|
if (isArch && repositoryFilter && repositoryFilter !== 'all') {
|
||||||
params.repository = repositoryFilter as 'official' | 'aur'
|
params.repository = repositoryFilter as 'official' | 'aur'
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('🔍 Frontend: API params:', params)
|
console.log('🔍 Frontend: API params:', params)
|
||||||
const result = await apiClient.getPackages(params)
|
const result = await apiClient.getPackages(params, controller.signal)
|
||||||
console.log('📦 Frontend: Received initial packages:', {
|
console.log('📦 Frontend: Received initial packages:', {
|
||||||
total: result.total,
|
total: result.total,
|
||||||
packageCount: result.packages.length,
|
packageCount: result.packages.length,
|
||||||
@@ -83,47 +85,57 @@ export function PackageBrowserV2({
|
|||||||
setTotalCount(result.total)
|
setTotalCount(result.total)
|
||||||
setHasMore(result.packages.length < result.total)
|
setHasMore(result.packages.length < result.total)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (error instanceof Error && error.name === 'AbortError') {
|
||||||
|
console.log('Request aborted')
|
||||||
|
return
|
||||||
|
}
|
||||||
console.error('Failed to load packages:', error)
|
console.error('Failed to load packages:', error)
|
||||||
setPackages([])
|
setPackages([])
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false)
|
if (!controller.signal.aborted) {
|
||||||
|
setLoading(false)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
loadPackages()
|
loadPackages()
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
controller.abort()
|
||||||
|
}
|
||||||
}, [selectedPlatform, typeFilter, repositoryFilter, isDebianUbuntu, isArch])
|
}, [selectedPlatform, typeFilter, repositoryFilter, isDebianUbuntu, isArch])
|
||||||
|
|
||||||
// Debounced search to prevent focus loss
|
// Debounced search to prevent focus loss
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!selectedPlatform) return
|
if (!selectedPlatform) return
|
||||||
|
|
||||||
const timeoutId = setTimeout(() => {
|
const timeoutId = setTimeout(() => {
|
||||||
const loadPackages = async () => {
|
const loadPackages = async () => {
|
||||||
// DON'T set loading to true - it causes re-render and focus loss
|
// DON'T set loading to true - it causes re-render and focus loss
|
||||||
// setLoading(true)
|
// setLoading(true)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const params: any = {
|
const params: any = {
|
||||||
platform_id: selectedPlatform.id,
|
platform_id: selectedPlatform.id,
|
||||||
limit: 50,
|
limit: 50,
|
||||||
offset: 0
|
offset: 0
|
||||||
}
|
}
|
||||||
|
|
||||||
if (searchQuery && searchQuery.trim()) {
|
if (searchQuery && searchQuery.trim()) {
|
||||||
params.search = searchQuery.trim()
|
params.search = searchQuery.trim()
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
|
if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
|
||||||
params.type = typeFilter as 'gui' | 'cli'
|
params.type = typeFilter as 'gui' | 'cli'
|
||||||
}
|
}
|
||||||
if (isArch && repositoryFilter && repositoryFilter !== 'all') {
|
if (isArch && repositoryFilter && repositoryFilter !== 'all') {
|
||||||
params.repository = repositoryFilter as 'official' | 'aur'
|
params.repository = repositoryFilter as 'official' | 'aur'
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('🔍 Frontend: Debounced API params:', params)
|
console.log('🔍 Frontend: Debounced API params:', params)
|
||||||
const result = await apiClient.getPackages(params)
|
const result = await apiClient.getPackages(params)
|
||||||
console.log('📦 Frontend: Debounced result:', result.packages.length)
|
console.log('📦 Frontend: Debounced result:', result.packages.length)
|
||||||
|
|
||||||
// Update packages without triggering loading state
|
// Update packages without triggering loading state
|
||||||
setPackages(result.packages)
|
setPackages(result.packages)
|
||||||
setTotalCount(result.total)
|
setTotalCount(result.total)
|
||||||
@@ -143,7 +155,7 @@ export function PackageBrowserV2({
|
|||||||
// Load more packages
|
// Load more packages
|
||||||
const loadMore = async () => {
|
const loadMore = async () => {
|
||||||
if (!selectedPlatform || loadingMore || !hasMore) return
|
if (!selectedPlatform || loadingMore || !hasMore) return
|
||||||
|
|
||||||
setLoadingMore(true)
|
setLoadingMore(true)
|
||||||
try {
|
try {
|
||||||
const params: any = {
|
const params: any = {
|
||||||
@@ -151,18 +163,18 @@ export function PackageBrowserV2({
|
|||||||
limit: 50,
|
limit: 50,
|
||||||
offset: packages.length
|
offset: packages.length
|
||||||
}
|
}
|
||||||
|
|
||||||
if (searchQuery && searchQuery.trim()) {
|
if (searchQuery && searchQuery.trim()) {
|
||||||
params.search = searchQuery.trim()
|
params.search = searchQuery.trim()
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
|
if (isDebianUbuntu && typeFilter && typeFilter !== 'all') {
|
||||||
params.type = typeFilter as 'gui' | 'cli'
|
params.type = typeFilter as 'gui' | 'cli'
|
||||||
}
|
}
|
||||||
if (isArch && repositoryFilter && repositoryFilter !== 'all') {
|
if (isArch && repositoryFilter && repositoryFilter !== 'all') {
|
||||||
params.repository = repositoryFilter as 'official' | 'aur'
|
params.repository = repositoryFilter as 'official' | 'aur'
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await apiClient.getPackages(params)
|
const result = await apiClient.getPackages(params)
|
||||||
setPackages(prev => [...prev, ...result.packages])
|
setPackages(prev => [...prev, ...result.packages])
|
||||||
setHasMore(packages.length + result.packages.length < result.total)
|
setHasMore(packages.length + result.packages.length < result.total)
|
||||||
@@ -256,30 +268,30 @@ export function PackageBrowserV2({
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Type Filter (Debian/Ubuntu only) */}
|
{/* Type Filter (Debian/Ubuntu only) */}
|
||||||
{isDebianUbuntu && (
|
{isDebianUbuntu && (
|
||||||
<Select value={typeFilter || "all"} onValueChange={(value) => setTypeFilter(value === "all" ? "" : value)}>
|
<Select value={typeFilter || "all"} onValueChange={(value) => setTypeFilter(value === "all" ? "" : value)}>
|
||||||
<SelectTrigger className="w-full sm:w-40">
|
<SelectTrigger className="w-full sm:w-40">
|
||||||
<SelectValue placeholder="Type" />
|
<SelectValue placeholder="Type" />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
<SelectContent>
|
<SelectContent>
|
||||||
<SelectItem value="all">All Types</SelectItem>
|
<SelectItem value="all">All Types</SelectItem>
|
||||||
<SelectItem value="gui">GUI</SelectItem>
|
<SelectItem value="gui">GUI</SelectItem>
|
||||||
<SelectItem value="cli">CLI</SelectItem>
|
<SelectItem value="cli">CLI</SelectItem>
|
||||||
</SelectContent>
|
</SelectContent>
|
||||||
</Select>
|
</Select>
|
||||||
)}
|
)}
|
||||||
{isArch && (
|
{isArch && (
|
||||||
<Select value={repositoryFilter || "all"} onValueChange={(value) => setRepositoryFilter(value === "all" ? "" : value)}>
|
<Select value={repositoryFilter || "all"} onValueChange={(value) => setRepositoryFilter(value === "all" ? "" : value)}>
|
||||||
<SelectTrigger className="w-full sm:w-40">
|
<SelectTrigger className="w-full sm:w-40">
|
||||||
<SelectValue placeholder="Repository" />
|
<SelectValue placeholder="Repository" />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
<SelectContent>
|
<SelectContent>
|
||||||
<SelectItem value="all">All</SelectItem>
|
<SelectItem value="all">All</SelectItem>
|
||||||
<SelectItem value="official">Official</SelectItem>
|
<SelectItem value="official">Official</SelectItem>
|
||||||
<SelectItem value="aur">AUR</SelectItem>
|
<SelectItem value="aur">AUR</SelectItem>
|
||||||
</SelectContent>
|
</SelectContent>
|
||||||
</Select>
|
</Select>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Package List */}
|
{/* Package List */}
|
||||||
@@ -303,11 +315,10 @@ export function PackageBrowserV2({
|
|||||||
packages.map((pkg) => (
|
packages.map((pkg) => (
|
||||||
<div
|
<div
|
||||||
key={pkg.id}
|
key={pkg.id}
|
||||||
className={`flex items-start space-x-3 p-3 rounded-lg border transition-colors cursor-pointer ${
|
className={`flex items-start space-x-3 p-3 rounded-lg border transition-colors cursor-pointer ${isPackageSelected(pkg)
|
||||||
isPackageSelected(pkg)
|
? 'border-primary bg-primary/5'
|
||||||
? 'border-primary bg-primary/5'
|
|
||||||
: 'border-border hover:bg-secondary/50'
|
: 'border-border hover:bg-secondary/50'
|
||||||
}`}
|
}`}
|
||||||
onClick={() => onPackageToggle(pkg)}
|
onClick={() => onPackageToggle(pkg)}
|
||||||
>
|
>
|
||||||
<Checkbox
|
<Checkbox
|
||||||
@@ -350,7 +361,7 @@ export function PackageBrowserV2({
|
|||||||
</div>
|
</div>
|
||||||
))
|
))
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Load More Button */}
|
{/* Load More Button */}
|
||||||
{hasMore && (
|
{hasMore && (
|
||||||
<div className="text-center py-4">
|
<div className="text-center py-4">
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import {
|
|||||||
|
|
||||||
const API_BASE_URL =
|
const API_BASE_URL =
|
||||||
process.env.NEXT_PUBLIC_API_URL &&
|
process.env.NEXT_PUBLIC_API_URL &&
|
||||||
process.env.NEXT_PUBLIC_API_URL.trim() !== ""
|
process.env.NEXT_PUBLIC_API_URL.trim() !== ""
|
||||||
? process.env.NEXT_PUBLIC_API_URL.replace(/\/$/, "")
|
? process.env.NEXT_PUBLIC_API_URL.replace(/\/$/, "")
|
||||||
: "/api";
|
: "/api";
|
||||||
|
|
||||||
@@ -47,7 +47,8 @@ class ApiClient {
|
|||||||
|
|
||||||
// Package operations
|
// Package operations
|
||||||
async getPackages(
|
async getPackages(
|
||||||
filters: FilterOptions = {}
|
filters: FilterOptions = {},
|
||||||
|
signal?: AbortSignal
|
||||||
): Promise<{ packages: Package[]; total: number }> {
|
): Promise<{ packages: Package[]; total: number }> {
|
||||||
const params = new URLSearchParams();
|
const params = new URLSearchParams();
|
||||||
|
|
||||||
@@ -64,7 +65,8 @@ class ApiClient {
|
|||||||
|
|
||||||
const query = params.toString() ? `?${params.toString()}` : "";
|
const query = params.toString() ? `?${params.toString()}` : "";
|
||||||
return this.request<{ packages: Package[]; total: number }>(
|
return this.request<{ packages: Package[]; total: number }>(
|
||||||
`/packages${query}`
|
`/packages${query}`,
|
||||||
|
{ signal }
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,29 +7,11 @@ interface ArchPackage {
|
|||||||
lastUpdated: string
|
lastUpdated: string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Simple HTML parser without external dependencies
|
|
||||||
function parseHTML(html: string) {
|
|
||||||
return {
|
|
||||||
querySelector: (selector: string) => {
|
|
||||||
const regex = new RegExp(`<${selector}[^>]*>([\\s\\S]*?)<\\/${selector}>`, 'gi')
|
|
||||||
const match = regex.exec(html)
|
|
||||||
return match ? { textContent: match[1].replace(/<[^>]*>/g, '').trim() } : null
|
|
||||||
},
|
|
||||||
querySelectorAll: (selector: string) => {
|
|
||||||
const results: any[] = []
|
|
||||||
const regex = new RegExp(`<${selector}[^>]*>([\\s\\S]*?)<\\/${selector}>`, 'gi')
|
|
||||||
let match
|
|
||||||
while ((match = regex.exec(html)) !== null) {
|
|
||||||
results.push({ innerHTML: match[1], textContent: match[1].replace(/<[^>]*>/g, '').trim() })
|
|
||||||
}
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ArchPackageFetcher {
|
export class ArchPackageFetcher {
|
||||||
private baseUrl = 'https://archlinux.org/packages'
|
private baseUrl = 'https://archlinux.org/packages'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch all Arch Linux packages with progress callback
|
* Fetch all Arch Linux packages with progress callback
|
||||||
*/
|
*/
|
||||||
@@ -37,86 +19,86 @@ export class ArchPackageFetcher {
|
|||||||
onProgress?: (current: number, total: number, packageName: string) => void
|
onProgress?: (current: number, total: number, packageName: string) => void
|
||||||
): Promise<ArchPackage[]> {
|
): Promise<ArchPackage[]> {
|
||||||
console.log('🔍 Starting Arch Linux package fetch...')
|
console.log('🔍 Starting Arch Linux package fetch...')
|
||||||
|
|
||||||
const allPackages: ArchPackage[] = []
|
const allPackages: ArchPackage[] = []
|
||||||
let currentPage = 1
|
let currentPage = 1
|
||||||
let totalPages = 1
|
let totalPages = 1
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Fetch first page to get total pages
|
// Fetch first page to get total pages
|
||||||
const firstPageUrl = `${this.baseUrl}/?page=1`
|
const firstPageUrl = `${this.baseUrl}/?page=1`
|
||||||
console.log(`📄 Fetching first page: ${firstPageUrl}`)
|
console.log(`📄 Fetching first page: ${firstPageUrl}`)
|
||||||
|
|
||||||
const firstPageResponse = await fetch(firstPageUrl)
|
const firstPageResponse = await fetch(firstPageUrl)
|
||||||
if (!firstPageResponse.ok) {
|
if (!firstPageResponse.ok) {
|
||||||
throw new Error(`HTTP error! status: ${firstPageResponse.status}`)
|
throw new Error(`HTTP error! status: ${firstPageResponse.status}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
const firstPageHtml = await firstPageResponse.text()
|
const firstPageHtml = await firstPageResponse.text()
|
||||||
|
|
||||||
// Parse total packages and pages
|
// Parse total packages and pages
|
||||||
const statsMatch = firstPageHtml.match(/(\d+)\s+matching packages[\s\S]*?Page \d+ of (\d+)/)
|
const statsMatch = firstPageHtml.match(/(\d+)\s+matching packages[\s\S]*?Page \d+ of (\d+)/)
|
||||||
const totalPackages = statsMatch ? parseInt(statsMatch[1]) : 0
|
const totalPackages = statsMatch ? parseInt(statsMatch[1]) : 0
|
||||||
totalPages = statsMatch ? parseInt(statsMatch[2]) : 1
|
totalPages = statsMatch ? parseInt(statsMatch[2]) : 1
|
||||||
|
|
||||||
console.log(`📦 Total packages: ${totalPackages}`)
|
console.log(`📦 Total packages: ${totalPackages}`)
|
||||||
console.log(`📄 Total pages: ${totalPages}`)
|
console.log(`📄 Total pages: ${totalPages}`)
|
||||||
|
|
||||||
// Parse first page packages
|
// Parse first page packages
|
||||||
const firstPagePackages = this.parsePackagesFromHtml(firstPageHtml)
|
const firstPagePackages = this.parsePackagesFromHtml(firstPageHtml)
|
||||||
allPackages.push(...firstPagePackages)
|
allPackages.push(...firstPagePackages)
|
||||||
|
|
||||||
if (onProgress) {
|
if (onProgress) {
|
||||||
onProgress(allPackages.length, totalPackages, firstPagePackages[0]?.name || 'N/A')
|
onProgress(allPackages.length, totalPackages, firstPagePackages[0]?.name || 'N/A')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch remaining pages
|
// Fetch remaining pages
|
||||||
for (currentPage = 2; currentPage <= totalPages; currentPage++) {
|
for (currentPage = 2; currentPage <= totalPages; currentPage++) {
|
||||||
const pageUrl = `${this.baseUrl}/?page=${currentPage}`
|
const pageUrl = `${this.baseUrl}/?page=${currentPage}`
|
||||||
console.log(`📄 Fetching page ${currentPage}/${totalPages}...`)
|
console.log(`📄 Fetching page ${currentPage}/${totalPages}...`)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch(pageUrl)
|
const response = await fetch(pageUrl)
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
console.error(`❌ Failed to fetch page ${currentPage}: ${response.status}`)
|
console.error(`❌ Failed to fetch page ${currentPage}: ${response.status}`)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
const html = await response.text()
|
const html = await response.text()
|
||||||
const packages = this.parsePackagesFromHtml(html)
|
const packages = this.parsePackagesFromHtml(html)
|
||||||
|
|
||||||
allPackages.push(...packages)
|
allPackages.push(...packages)
|
||||||
|
|
||||||
if (onProgress && packages.length > 0) {
|
if (onProgress && packages.length > 0) {
|
||||||
onProgress(allPackages.length, totalPackages, packages[0].name)
|
onProgress(allPackages.length, totalPackages, packages[0].name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rate limiting: wait 100ms between requests
|
// Rate limiting: wait 100ms between requests
|
||||||
await new Promise(resolve => setTimeout(resolve, 100))
|
await new Promise(resolve => setTimeout(resolve, 100))
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`❌ Error fetching page ${currentPage}:`, error)
|
console.error(`❌ Error fetching page ${currentPage}:`, error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`✅ Fetch completed! Total packages: ${allPackages.length}`)
|
console.log(`✅ Fetch completed! Total packages: ${allPackages.length}`)
|
||||||
return allPackages
|
return allPackages
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('❌ Error fetching Arch packages:', error)
|
console.error('❌ Error fetching Arch packages:', error)
|
||||||
throw error
|
throw error
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse packages from HTML using regex
|
* Parse packages from HTML using regex
|
||||||
*/
|
*/
|
||||||
private parsePackagesFromHtml(html: string): ArchPackage[] {
|
private parsePackagesFromHtml(html: string): ArchPackage[] {
|
||||||
const packages: ArchPackage[] = []
|
const packages: ArchPackage[] = []
|
||||||
|
|
||||||
// Match all table rows
|
// Match all table rows
|
||||||
const rowRegex = /<tr>\s*<td>([^<]+)<\/td>\s*<td>([^<]+)<\/td>\s*<td><a[^>]*>([^<]+)<\/a><\/td>\s*<td>([^<]+)<\/td>\s*<td[^>]*>([^<]*)<\/td>\s*<td>([^<]*)<\/td>/gi
|
const rowRegex = /<tr>\s*<td>([^<]+)<\/td>\s*<td>([^<]+)<\/td>\s*<td><a[^>]*>([^<]+)<\/a><\/td>\s*<td>([^<]+)<\/td>\s*<td[^>]*>([^<]*)<\/td>\s*<td>([^<]*)<\/td>/gi
|
||||||
|
|
||||||
let match
|
let match
|
||||||
while ((match = rowRegex.exec(html)) !== null) {
|
while ((match = rowRegex.exec(html)) !== null) {
|
||||||
try {
|
try {
|
||||||
@@ -126,7 +108,7 @@ export class ArchPackageFetcher {
|
|||||||
const version = match[4].trim()
|
const version = match[4].trim()
|
||||||
const description = match[5].trim()
|
const description = match[5].trim()
|
||||||
const lastUpdated = match[6].trim()
|
const lastUpdated = match[6].trim()
|
||||||
|
|
||||||
if (name && version) {
|
if (name && version) {
|
||||||
packages.push({
|
packages.push({
|
||||||
name,
|
name,
|
||||||
@@ -141,10 +123,10 @@ export class ArchPackageFetcher {
|
|||||||
console.error('Error parsing package row:', error)
|
console.error('Error parsing package row:', error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return packages
|
return packages
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Store packages in database
|
* Store packages in database
|
||||||
*/
|
*/
|
||||||
@@ -153,17 +135,17 @@ export class ArchPackageFetcher {
|
|||||||
onProgress?: (current: number, total: number) => void
|
onProgress?: (current: number, total: number) => void
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const { query } = await import('@/lib/database/config')
|
const { query } = await import('@/lib/database/config')
|
||||||
|
|
||||||
console.log(`💾 Storing ${packages.length} Arch packages in database...`)
|
console.log(`💾 Storing ${packages.length} Arch packages in database...`)
|
||||||
|
|
||||||
const batchSize = 100
|
const batchSize = 100
|
||||||
let stored = 0
|
let stored = 0
|
||||||
let updated = 0
|
let updated = 0
|
||||||
let skipped = 0
|
let skipped = 0
|
||||||
|
|
||||||
for (let i = 0; i < packages.length; i += batchSize) {
|
for (let i = 0; i < packages.length; i += batchSize) {
|
||||||
const batch = packages.slice(i, i + batchSize)
|
const batch = packages.slice(i, i + batchSize)
|
||||||
|
|
||||||
for (const pkg of batch) {
|
for (const pkg of batch) {
|
||||||
try {
|
try {
|
||||||
// Check if package already exists
|
// Check if package already exists
|
||||||
@@ -171,7 +153,7 @@ export class ArchPackageFetcher {
|
|||||||
'SELECT id, version FROM packages WHERE name = $1 AND platform_id = $2',
|
'SELECT id, version FROM packages WHERE name = $1 AND platform_id = $2',
|
||||||
[pkg.name, 'arch']
|
[pkg.name, 'arch']
|
||||||
)
|
)
|
||||||
|
|
||||||
if (existingResult.rows.length === 0) {
|
if (existingResult.rows.length === 0) {
|
||||||
// Insert new package (id will be auto-generated by SERIAL)
|
// Insert new package (id will be auto-generated by SERIAL)
|
||||||
await query(
|
await query(
|
||||||
@@ -197,17 +179,17 @@ export class ArchPackageFetcher {
|
|||||||
)
|
)
|
||||||
skipped++
|
skipped++
|
||||||
}
|
}
|
||||||
|
|
||||||
if (onProgress && (stored + updated + skipped) % 100 === 0) {
|
if (onProgress && (stored + updated + skipped) % 100 === 0) {
|
||||||
onProgress(stored + updated + skipped, packages.length)
|
onProgress(stored + updated + skipped, packages.length)
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error storing package ${pkg.name}:`, error)
|
console.error(`Error storing package ${pkg.name}:`, error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`✅ Stored ${stored} new packages, updated ${updated}, skipped ${skipped}`)
|
console.log(`✅ Stored ${stored} new packages, updated ${updated}, skipped ${skipped}`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user