diff --git a/.env.example b/.env.example index ff9eaac..19c2513 100644 --- a/.env.example +++ b/.env.example @@ -24,3 +24,8 @@ SYNC_SERVER_ONLY=false AUTO_SYNC_DAYS=1 # Secret key to authorize sync operations from server SYNC_SECRET_KEY=your_sync_secret_key_here + +# Prune Configuration +# Number of days to wait before pruning packages +PRUNE_GRACE_DAYS=7 +PRUNE_HARD_DELETE_DAYS=1 \ No newline at end of file diff --git a/scripts/migrations/002_add_last_seen_to_packages.sql b/scripts/migrations/002_add_last_seen_to_packages.sql new file mode 100644 index 0000000..3a9696e --- /dev/null +++ b/scripts/migrations/002_add_last_seen_to_packages.sql @@ -0,0 +1,10 @@ +-- Add last_seen_at to track soft-deletion by sync scope +ALTER TABLE packages + ADD COLUMN IF NOT EXISTS last_seen_at TIMESTAMPTZ; + +-- Backfill existing rows so they are not immediately purged until their next scoped sync +UPDATE packages SET last_seen_at = NOW() WHERE last_seen_at IS NULL; + +-- Helpful index for prune queries +CREATE INDEX IF NOT EXISTS idx_packages_seen_scope + ON packages (platform_id, repository, is_active, last_seen_at); diff --git a/src/app/api/sync-arch/route.ts b/src/app/api/sync-arch/route.ts index ae4065d..094919e 100644 --- a/src/app/api/sync-arch/route.ts +++ b/src/app/api/sync-arch/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { ArchPackageFetcher } from '@/services/archPackageFetcher' +import { query } from '@/lib/database/config' import { SyncAuth } from '@/lib/sync/auth' export const dynamic = 'force-dynamic' @@ -51,6 +52,7 @@ export async function POST(request: NextRequest) { // Start sync in background ;(async () => { try { + const runStartedAt = new Date() const fetcher = new ArchPackageFetcher() // Fetch packages @@ -76,6 +78,31 @@ export async function POST(request: NextRequest) { syncStatus.storeTotal = total } ) + // Prune packages not seen in this run (Arch official scope) + const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10) + const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000) + await query( + `UPDATE packages + SET is_active = false + WHERE platform_id = $1 + AND repository = $2 + AND is_active = true + AND (last_seen_at IS NULL OR last_seen_at < $3)`, + ['arch', 'official', cutoff] + ) + // Optional hard delete of long-inactive packages + const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10) + if (hardDays > 0) { + const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000) + await query( + `DELETE FROM packages + WHERE platform_id = $1 + AND repository = $2 + AND is_active = false + AND last_seen_at IS NOT NULL AND last_seen_at < $3`, + ['arch', 'official', deleteCutoff] + ) + } syncStatus.status = 'complete' console.log('✅ Arch Linux sync completed successfully') diff --git a/src/app/api/sync-aur/route.ts b/src/app/api/sync-aur/route.ts index 5a5e136..5f9fc57 100644 --- a/src/app/api/sync-aur/route.ts +++ b/src/app/api/sync-aur/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { SyncAuth } from '@/lib/sync/auth' +import { query } from '@/lib/database/config' import { AurPackageFetcher } from '@/services/aurPackageFetcher' export const dynamic = 'force-dynamic' @@ -36,9 +37,35 @@ export async function POST(request: NextRequest) { ;(async () => { try { + const runStartedAt = new Date() const fetcher = new AurPackageFetcher() const pkgs = await fetcher.fetchAllPackages() await fetcher.storePackages(pkgs) + // Prune not-seen AUR packages for this run + const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10) + const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000) + await query( + `UPDATE packages + SET is_active = false + WHERE platform_id = $1 + AND repository = $2 + AND is_active = true + AND (last_seen_at IS NULL OR last_seen_at < $3)`, + ['arch', 'aur', cutoff] + ) + // Optional hard delete of long-inactive AUR packages + const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10) + if (hardDays > 0) { + const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000) + await query( + `DELETE FROM packages + WHERE platform_id = $1 + AND repository = $2 + AND is_active = false + AND last_seen_at IS NOT NULL AND last_seen_at < $3`, + ['arch', 'aur', deleteCutoff] + ) + } syncStatus.status = 'complete' } catch (err: any) { syncStatus.status = 'error' diff --git a/src/app/api/sync-debian/route.ts b/src/app/api/sync-debian/route.ts index 19857bd..c8151bc 100644 --- a/src/app/api/sync-debian/route.ts +++ b/src/app/api/sync-debian/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { DebianPackageFetcher } from '@/services/debianPackageFetcher' +import { query } from '@/lib/database/config' import { SyncAuth } from '@/lib/sync/auth' export const dynamic = 'force-dynamic' @@ -36,7 +37,30 @@ export async function POST(request: NextRequest) { ;(async () => { try { + const runStartedAt = new Date() await DebianPackageFetcher.fetchDebianPackages() + // Prune not-seen Debian packages (platform debian) + const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10) + const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000) + await query( + `UPDATE packages + SET is_active = false + WHERE platform_id = $1 + AND is_active = true + AND (last_seen_at IS NULL OR last_seen_at < $2)`, + ['debian', cutoff] + ) + const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10) + if (hardDays > 0) { + const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000) + await query( + `DELETE FROM packages + WHERE platform_id = $1 + AND is_active = false + AND last_seen_at IS NOT NULL AND last_seen_at < $2`, + ['debian', deleteCutoff] + ) + } syncStatus.status = 'complete' } catch (error) { syncStatus.status = 'error' diff --git a/src/app/api/sync-fedora/route.ts b/src/app/api/sync-fedora/route.ts index c18d918..db46c0c 100644 --- a/src/app/api/sync-fedora/route.ts +++ b/src/app/api/sync-fedora/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { FedoraPackageFetcher } from '@/services/fedoraPackageFetcher' +import { query } from '@/lib/database/config' import { SyncAuth } from '@/lib/sync/auth' export const dynamic = 'force-dynamic' @@ -51,6 +52,7 @@ export async function POST(request: NextRequest) { // Start sync in background ;(async () => { try { + const runStartedAt = new Date() const fetcher = new FedoraPackageFetcher() // Fetch packages @@ -76,6 +78,30 @@ export async function POST(request: NextRequest) { syncStatus.storeTotal = total } ) + // Prune packages not seen in this run (Fedora; repository is NULL) + const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10) + const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000) + await query( + `UPDATE packages + SET is_active = false + WHERE platform_id = $1 + AND repository IS NULL + AND is_active = true + AND (last_seen_at IS NULL OR last_seen_at < $2)`, + ['fedora', cutoff] + ) + const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10) + if (hardDays > 0) { + const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000) + await query( + `DELETE FROM packages + WHERE platform_id = $1 + AND repository IS NULL + AND is_active = false + AND last_seen_at IS NOT NULL AND last_seen_at < $2`, + ['fedora', deleteCutoff] + ) + } syncStatus.status = 'complete' console.log('✅ Fedora sync completed successfully') diff --git a/src/app/api/sync-homebrew/route.ts b/src/app/api/sync-homebrew/route.ts index c808af6..0398304 100644 --- a/src/app/api/sync-homebrew/route.ts +++ b/src/app/api/sync-homebrew/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { HomebrewPackageFetcher } from '@/services/homebrewPackageFetcher' +import { query } from '@/lib/database/config' import { SyncAuth } from '@/lib/sync/auth' export const dynamic = 'force-dynamic' @@ -51,6 +52,7 @@ export async function POST(request: NextRequest) { // Start sync in background ;(async () => { try { + const runStartedAt = new Date() const fetcher = new HomebrewPackageFetcher() // Fetch packages @@ -76,6 +78,30 @@ export async function POST(request: NextRequest) { syncStatus.storeTotal = total } ) + // Prune not-seen Homebrew packages (platform macos, repository IS NULL) + const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10) + const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000) + await query( + `UPDATE packages + SET is_active = false + WHERE platform_id = $1 + AND repository IS NULL + AND is_active = true + AND (last_seen_at IS NULL OR last_seen_at < $2)`, + ['macos', cutoff] + ) + const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10) + if (hardDays > 0) { + const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000) + await query( + `DELETE FROM packages + WHERE platform_id = $1 + AND repository IS NULL + AND is_active = false + AND last_seen_at IS NOT NULL AND last_seen_at < $2`, + ['macos', deleteCutoff] + ) + } syncStatus.status = 'complete' console.log('✅ Homebrew sync completed successfully') diff --git a/src/app/api/sync-ubuntu/route.ts b/src/app/api/sync-ubuntu/route.ts index e95b1c9..61634ee 100644 --- a/src/app/api/sync-ubuntu/route.ts +++ b/src/app/api/sync-ubuntu/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { DebianPackageFetcher } from '@/services/debianPackageFetcher' +import { query } from '@/lib/database/config' import { SyncAuth } from '@/lib/sync/auth' export const dynamic = 'force-dynamic' @@ -36,7 +37,30 @@ export async function POST(request: NextRequest) { ;(async () => { try { + const runStartedAt = new Date() await DebianPackageFetcher.fetchUbuntuPackages() + // Prune not-seen Ubuntu packages (platform ubuntu) + const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10) + const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000) + await query( + `UPDATE packages + SET is_active = false + WHERE platform_id = $1 + AND is_active = true + AND (last_seen_at IS NULL OR last_seen_at < $2)`, + ['ubuntu', cutoff] + ) + const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10) + if (hardDays > 0) { + const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000) + await query( + `DELETE FROM packages + WHERE platform_id = $1 + AND is_active = false + AND last_seen_at IS NOT NULL AND last_seen_at < $2`, + ['ubuntu', deleteCutoff] + ) + } syncStatus.status = 'complete' } catch (error) { syncStatus.status = 'error' diff --git a/src/app/api/sync-winget/route.ts b/src/app/api/sync-winget/route.ts index 6bbf6db..e998181 100644 --- a/src/app/api/sync-winget/route.ts +++ b/src/app/api/sync-winget/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { WingetPackageFetcher } from '@/services/wingetPackageFetcher' +import { query } from '@/lib/database/config' import { SyncAuth } from '@/lib/sync/auth' export const dynamic = 'force-dynamic' @@ -51,6 +52,7 @@ export async function POST(request: NextRequest) { // Start sync in background ;(async () => { try { + const runStartedAt = new Date() const fetcher = new WingetPackageFetcher() // Fetch packages @@ -76,6 +78,30 @@ export async function POST(request: NextRequest) { syncStatus.storeTotal = total } ) + // Prune not-seen Winget packages (platform windows, repository IS NULL) + const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10) + const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000) + await query( + `UPDATE packages + SET is_active = false + WHERE platform_id = $1 + AND repository IS NULL + AND is_active = true + AND (last_seen_at IS NULL OR last_seen_at < $2)`, + ['windows', cutoff] + ) + const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10) + if (hardDays > 0) { + const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000) + await query( + `DELETE FROM packages + WHERE platform_id = $1 + AND repository IS NULL + AND is_active = false + AND last_seen_at IS NOT NULL AND last_seen_at < $2`, + ['windows', deleteCutoff] + ) + } syncStatus.status = 'complete' console.log('✅ Winget sync completed successfully') diff --git a/src/services/archPackageFetcher.ts b/src/services/archPackageFetcher.ts index 50659b8..ee0028e 100644 --- a/src/services/archPackageFetcher.ts +++ b/src/services/archPackageFetcher.ts @@ -175,8 +175,8 @@ export class ArchPackageFetcher { if (existingResult.rows.length === 0) { // Insert new package (id will be auto-generated by SERIAL) await query( - `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) - VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, + `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active, last_seen_at) + VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, NOW())`, [pkg.name, pkg.description, pkg.version, 'arch', null, 'official', 0, true] ) stored++ @@ -184,12 +184,17 @@ export class ArchPackageFetcher { // Update if version changed await query( `UPDATE packages - SET version = $1, description = $2, repository = $3, updated_at = NOW() + SET version = $1, description = $2, repository = $3, last_seen_at = NOW(), is_active = true, updated_at = NOW() WHERE id = $4`, [pkg.version, pkg.description, 'official', existingResult.rows[0].id] ) updated++ } else { + // Still mark as seen to avoid pruning + await query( + `UPDATE packages SET last_seen_at = NOW(), is_active = true WHERE id = $1`, + [existingResult.rows[0].id] + ) skipped++ } diff --git a/src/services/aurPackageFetcher.ts b/src/services/aurPackageFetcher.ts index 133030e..ed6a12c 100644 --- a/src/services/aurPackageFetcher.ts +++ b/src/services/aurPackageFetcher.ts @@ -190,18 +190,23 @@ export class AurPackageFetcher { ) if (existing.rows.length === 0) { await query( - `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) - VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, + `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active, last_seen_at) + VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, NOW())`, [pkg.name, pkg.description, pkg.version, 'arch', null, 'aur', 0, true] ) stored++ } else if (existing.rows[0].version !== pkg.version) { await query( - `UPDATE packages SET version = $1, description = $2, repository = $3, updated_at = NOW() WHERE id = $4`, + `UPDATE packages SET version = $1, description = $2, repository = $3, last_seen_at = NOW(), is_active = true, updated_at = NOW() WHERE id = $4`, [pkg.version, pkg.description, 'aur', existing.rows[0].id] ) updated++ } else { + // Still mark as seen to avoid pruning + await query( + `UPDATE packages SET last_seen_at = NOW(), is_active = true WHERE id = $1`, + [existing.rows[0].id] + ) skipped++ } if (onProgress && (stored + updated + skipped) % 100 === 0) { diff --git a/src/services/fedoraPackageFetcher.ts b/src/services/fedoraPackageFetcher.ts index 6ed5231..1c31946 100644 --- a/src/services/fedoraPackageFetcher.ts +++ b/src/services/fedoraPackageFetcher.ts @@ -144,8 +144,8 @@ export class FedoraPackageFetcher { if (existingResult.rows.length === 0) { // Insert new package await query( - `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) - VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, + `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active, last_seen_at) + VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, NOW())`, [pkg.name, pkg.description, pkg.version, 'fedora', null, null, 0, true] ) stored++ @@ -153,12 +153,17 @@ export class FedoraPackageFetcher { // Update if version changed await query( `UPDATE packages - SET version = $1, description = $2, repository = $3, updated_at = NOW() + SET version = $1, description = $2, repository = $3, last_seen_at = NOW(), is_active = true, updated_at = NOW() WHERE id = $4`, [pkg.version, pkg.description, null, existingResult.rows[0].id] ) updated++ } else { + // Still mark as seen to avoid pruning + await query( + `UPDATE packages SET last_seen_at = NOW(), is_active = true WHERE id = $1`, + [existingResult.rows[0].id] + ) skipped++ } diff --git a/src/services/homebrewPackageFetcher.ts b/src/services/homebrewPackageFetcher.ts index 158c762..ddf30a5 100644 --- a/src/services/homebrewPackageFetcher.ts +++ b/src/services/homebrewPackageFetcher.ts @@ -143,8 +143,8 @@ export class HomebrewPackageFetcher { if (existingResult.rows.length === 0) { // Insert new package await query( - `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) - VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, + `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active, last_seen_at) + VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, NOW())`, [pkg.name, pkg.description, pkg.version, 'macos', null, null, 0, true] ) stored++ @@ -152,12 +152,17 @@ export class HomebrewPackageFetcher { // Update if version changed await query( `UPDATE packages - SET version = $1, description = $2, updated_at = NOW() + SET version = $1, description = $2, last_seen_at = NOW(), is_active = true, updated_at = NOW() WHERE id = $3`, [pkg.version, pkg.description, existingResult.rows[0].id] ) updated++ } else { + // Still mark as seen to avoid pruning + await query( + `UPDATE packages SET last_seen_at = NOW(), is_active = true WHERE id = $1`, + [existingResult.rows[0].id] + ) skipped++ } diff --git a/src/services/packageService.ts b/src/services/packageService.ts index 840ed11..87dfced 100644 --- a/src/services/packageService.ts +++ b/src/services/packageService.ts @@ -210,8 +210,8 @@ export class PackageService { `INSERT INTO packages ( id, name, description, version, platform_id, category_id, license_id, type, repository, homepage_url, download_url, - popularity_score - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + popularity_score, last_seen_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NOW()) RETURNING *`, [id, name, description, version, platform_id, category_id, license_id, type, repository, homepage_url, download_url, @@ -238,6 +238,11 @@ export class PackageService { } } + // Always mark as seen and active on update + fields.push('last_seen_at = NOW()') + fields.push('is_active = true') + fields.push('updated_at = NOW()') + if (fields.length === 0) { return this.getById(id) } diff --git a/src/services/wingetPackageFetcher.ts b/src/services/wingetPackageFetcher.ts index 222d89f..018436b 100644 --- a/src/services/wingetPackageFetcher.ts +++ b/src/services/wingetPackageFetcher.ts @@ -230,8 +230,8 @@ export class WingetPackageFetcher { if (existingResult.rows.length === 0) { // Insert new package await query( - `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active) - VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8)`, + `INSERT INTO packages (id, name, description, version, platform_id, type, repository, popularity_score, is_active, last_seen_at) + VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, NOW())`, [pkg.name, pkg.description, pkg.version, 'windows', null, null, 0, true] ) stored++ @@ -239,12 +239,17 @@ export class WingetPackageFetcher { // Update if version changed await query( `UPDATE packages - SET version = $1, description = $2, updated_at = NOW() + SET version = $1, description = $2, last_seen_at = NOW(), is_active = true, updated_at = NOW() WHERE id = $3`, [pkg.version, pkg.description, existingResult.rows[0].id] ) updated++ } else { + // Still mark as seen to avoid pruning + await query( + `UPDATE packages SET last_seen_at = NOW(), is_active = true WHERE id = $1`, + [existingResult.rows[0].id] + ) skipped++ }