yusufipek/RepoHub
1
0
Fork 0
mirror of https://github.com/yusufipk/RepoHub.git synced 2026-09-11 18:46:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add package pruning system with configurable grace periods

Browse Source 
- Implemented automatic pruning of stale packages across all platform sync endpoints (Arch, AUR, Debian, Ubuntu, Fedora, Homebrew, Winget)
- Added `last_seen_at` tracking to mark packages as seen during sync runs and identify inactive packages
- Introduced `PRUNE_GRACE_DAYS` and `PRUNE_HARD_DELETE_DAYS` environment variables for configurable soft-delete and hard-delete thresholds
This commit is contained in:
Yusuf İpek
2025-11-13 01:54:23 +03:00
parent 560e78cb96
commit ea792a8a7f
15 changed files with 242 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/api/sync-arch/route.ts
+27
View File
@@ -1,5 +1,6 @@
import { NextRequest, NextResponse } from 'next/server'
import { ArchPackageFetcher } from '@/services/archPackageFetcher'
import { query } from '@/lib/database/config'
import { SyncAuth } from '@/lib/sync/auth'
export const dynamic = 'force-dynamic'
@@ -51,6 +52,7 @@ export async function POST(request: NextRequest) {
// Start sync in background
;(async () => {
try {
const runStartedAt = new Date()
const fetcher = new ArchPackageFetcher()
// Fetch packages
@@ -76,6 +78,31 @@ export async function POST(request: NextRequest) {
syncStatus.storeTotal = total
}
)
// Prune packages not seen in this run (Arch official scope)
const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10)
const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000)
await query(
`UPDATE packages
SET is_active = false
WHERE platform_id = $1
AND repository = $2
AND is_active = true
AND (last_seen_at IS NULL OR last_seen_at < $3)`,
['arch', 'official', cutoff]
)
// Optional hard delete of long-inactive packages
const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10)
if (hardDays > 0) {
const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000)
await query(
`DELETE FROM packages
WHERE platform_id = $1
AND repository = $2
AND is_active = false
AND last_seen_at IS NOT NULL AND last_seen_at < $3`,
['arch', 'official', deleteCutoff]
)
}
syncStatus.status = 'complete'
console.log('✅ Arch Linux sync completed successfully')
src/app/api/sync-aur/route.ts
+27
View File
@@ -1,5 +1,6 @@
import { NextRequest, NextResponse } from 'next/server'
import { SyncAuth } from '@/lib/sync/auth'
import { query } from '@/lib/database/config'
import { AurPackageFetcher } from '@/services/aurPackageFetcher'
export const dynamic = 'force-dynamic'
@@ -36,9 +37,35 @@ export async function POST(request: NextRequest) {
;(async () => {
try {
const runStartedAt = new Date()
const fetcher = new AurPackageFetcher()
const pkgs = await fetcher.fetchAllPackages()
await fetcher.storePackages(pkgs)
// Prune not-seen AUR packages for this run
const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10)
const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000)
await query(
`UPDATE packages
SET is_active = false
WHERE platform_id = $1
AND repository = $2
AND is_active = true
AND (last_seen_at IS NULL OR last_seen_at < $3)`,
['arch', 'aur', cutoff]
)
// Optional hard delete of long-inactive AUR packages
const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10)
if (hardDays > 0) {
const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000)
await query(
`DELETE FROM packages
WHERE platform_id = $1
AND repository = $2
AND is_active = false
AND last_seen_at IS NOT NULL AND last_seen_at < $3`,
['arch', 'aur', deleteCutoff]
)
}
syncStatus.status = 'complete'
} catch (err: any) {
syncStatus.status = 'error'
src/app/api/sync-debian/route.ts
+24
View File
@@ -1,5 +1,6 @@
import { NextRequest, NextResponse } from 'next/server'
import { DebianPackageFetcher } from '@/services/debianPackageFetcher'
import { query } from '@/lib/database/config'
import { SyncAuth } from '@/lib/sync/auth'
export const dynamic = 'force-dynamic'
@@ -36,7 +37,30 @@ export async function POST(request: NextRequest) {
;(async () => {
try {
const runStartedAt = new Date()
await DebianPackageFetcher.fetchDebianPackages()
// Prune not-seen Debian packages (platform debian)
const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10)
const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000)
await query(
`UPDATE packages
SET is_active = false
WHERE platform_id = $1
AND is_active = true
AND (last_seen_at IS NULL OR last_seen_at < $2)`,
['debian', cutoff]
)
const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10)
if (hardDays > 0) {
const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000)
await query(
`DELETE FROM packages
WHERE platform_id = $1
AND is_active = false
AND last_seen_at IS NOT NULL AND last_seen_at < $2`,
['debian', deleteCutoff]
)
}
syncStatus.status = 'complete'
} catch (error) {
syncStatus.status = 'error'
src/app/api/sync-fedora/route.ts
+26
View File
@@ -1,5 +1,6 @@
import { NextRequest, NextResponse } from 'next/server'
import { FedoraPackageFetcher } from '@/services/fedoraPackageFetcher'
import { query } from '@/lib/database/config'
import { SyncAuth } from '@/lib/sync/auth'
export const dynamic = 'force-dynamic'
@@ -51,6 +52,7 @@ export async function POST(request: NextRequest) {
// Start sync in background
;(async () => {
try {
const runStartedAt = new Date()
const fetcher = new FedoraPackageFetcher()
// Fetch packages
@@ -76,6 +78,30 @@ export async function POST(request: NextRequest) {
syncStatus.storeTotal = total
}
)
// Prune packages not seen in this run (Fedora; repository is NULL)
const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10)
const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000)
await query(
`UPDATE packages
SET is_active = false
WHERE platform_id = $1
AND repository IS NULL
AND is_active = true
AND (last_seen_at IS NULL OR last_seen_at < $2)`,
['fedora', cutoff]
)
const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10)
if (hardDays > 0) {
const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000)
await query(
`DELETE FROM packages
WHERE platform_id = $1
AND repository IS NULL
AND is_active = false
AND last_seen_at IS NOT NULL AND last_seen_at < $2`,
['fedora', deleteCutoff]
)
}
syncStatus.status = 'complete'
console.log('✅ Fedora sync completed successfully')
src/app/api/sync-homebrew/route.ts
+26
View File
@@ -1,5 +1,6 @@
import { NextRequest, NextResponse } from 'next/server'
import { HomebrewPackageFetcher } from '@/services/homebrewPackageFetcher'
import { query } from '@/lib/database/config'
import { SyncAuth } from '@/lib/sync/auth'
export const dynamic = 'force-dynamic'
@@ -51,6 +52,7 @@ export async function POST(request: NextRequest) {
// Start sync in background
;(async () => {
try {
const runStartedAt = new Date()
const fetcher = new HomebrewPackageFetcher()
// Fetch packages
@@ -76,6 +78,30 @@ export async function POST(request: NextRequest) {
syncStatus.storeTotal = total
}
)
// Prune not-seen Homebrew packages (platform macos, repository IS NULL)
const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10)
const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000)
await query(
`UPDATE packages
SET is_active = false
WHERE platform_id = $1
AND repository IS NULL
AND is_active = true
AND (last_seen_at IS NULL OR last_seen_at < $2)`,
['macos', cutoff]
)
const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10)
if (hardDays > 0) {
const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000)
await query(
`DELETE FROM packages
WHERE platform_id = $1
AND repository IS NULL
AND is_active = false
AND last_seen_at IS NOT NULL AND last_seen_at < $2`,
['macos', deleteCutoff]
)
}
syncStatus.status = 'complete'
console.log('✅ Homebrew sync completed successfully')
src/app/api/sync-ubuntu/route.ts
+24
View File
@@ -1,5 +1,6 @@
import { NextRequest, NextResponse } from 'next/server'
import { DebianPackageFetcher } from '@/services/debianPackageFetcher'
import { query } from '@/lib/database/config'
import { SyncAuth } from '@/lib/sync/auth'
export const dynamic = 'force-dynamic'
@@ -36,7 +37,30 @@ export async function POST(request: NextRequest) {
;(async () => {
try {
const runStartedAt = new Date()
await DebianPackageFetcher.fetchUbuntuPackages()
// Prune not-seen Ubuntu packages (platform ubuntu)
const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10)
const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000)
await query(
`UPDATE packages
SET is_active = false
WHERE platform_id = $1
AND is_active = true
AND (last_seen_at IS NULL OR last_seen_at < $2)`,
['ubuntu', cutoff]
)
const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10)
if (hardDays > 0) {
const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000)
await query(
`DELETE FROM packages
WHERE platform_id = $1
AND is_active = false
AND last_seen_at IS NOT NULL AND last_seen_at < $2`,
['ubuntu', deleteCutoff]
)
}
syncStatus.status = 'complete'
} catch (error) {
syncStatus.status = 'error'
src/app/api/sync-winget/route.ts
+26
View File
@@ -1,5 +1,6 @@
import { NextRequest, NextResponse } from 'next/server'
import { WingetPackageFetcher } from '@/services/wingetPackageFetcher'
import { query } from '@/lib/database/config'
import { SyncAuth } from '@/lib/sync/auth'
export const dynamic = 'force-dynamic'
@@ -51,6 +52,7 @@ export async function POST(request: NextRequest) {
// Start sync in background
;(async () => {
try {
const runStartedAt = new Date()
const fetcher = new WingetPackageFetcher()
// Fetch packages
@@ -76,6 +78,30 @@ export async function POST(request: NextRequest) {
syncStatus.storeTotal = total
}
)
// Prune not-seen Winget packages (platform windows, repository IS NULL)
const graceDays = parseInt(process.env.PRUNE_GRACE_DAYS || '0', 10)
const cutoff = new Date(runStartedAt.getTime() - (graceDays > 0 ? graceDays : 0) * 24 * 60 * 60 * 1000)
await query(
`UPDATE packages
SET is_active = false
WHERE platform_id = $1
AND repository IS NULL
AND is_active = true
AND (last_seen_at IS NULL OR last_seen_at < $2)`,
['windows', cutoff]
)
const hardDays = parseInt(process.env.PRUNE_HARD_DELETE_DAYS || '0', 10)
if (hardDays > 0) {
const deleteCutoff = new Date(runStartedAt.getTime() - hardDays * 24 * 60 * 60 * 1000)
await query(
`DELETE FROM packages
WHERE platform_id = $1
AND repository IS NULL
AND is_active = false
AND last_seen_at IS NOT NULL AND last_seen_at < $2`,
['windows', deleteCutoff]
)
}
syncStatus.status = 'complete'
console.log('✅ Winget sync completed successfully')