SPB Git

Repository Public

index.ts 5.2 KB 91 lines typescript

1import "../../scripts/_env";2import { readdir, rm, stat } from "node:fs/promises";3import path from "node:path";4import { sql } from "../lib/db";5import { env } from "../lib/env";6import { claimLyrics, transcribeTrack } from "../server/lyrics";7import { buildRendition, processUpload, pruneOrphans, reanalyseTrack, uploadTmpPath } from "../server/ingest";89/**10 * Background worker (one process): import queue, compatibility renditions, re-analysis, housekeeping.11 * The queue is the `uploads` table itself (FOR UPDATE SKIP LOCKED), so nothing is lost on a restart.12 */13let stopping = false;14const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));1516async function claimUpload(): Promise<string | null> {17  const [u] = await sql<{ id: string }[]>`18    UPDATE uploads SET status = 'validating', attempts = attempts + 1, updated_at = now()19    WHERE id = (SELECT id FROM uploads WHERE status = 'received' ORDER BY created_at, filename FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id`;20  return u?.id ?? null;21}22async function claimRendition(): Promise<string | null> {23  const [t] = await sql<{ id: string }[]>`24    UPDATE tracks SET compat_state = 'working' WHERE id = (SELECT id FROM tracks WHERE compat_state = 'queued' ORDER BY updated_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id`;25  return t?.id ?? null;26}27async function claimAnalysis(): Promise<string | null> {28  const [t] = await sql<{ id: string }[]>`29    UPDATE tracks SET analysis_requested = false WHERE id = (SELECT id FROM tracks WHERE analysis_requested ORDER BY updated_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id`;30  return t?.id ?? null;31}3233/** Work interrupted by a crash or restart goes back to the queue (three attempts, then a visible error). */34async function recoverStale() {35  await sql`UPDATE uploads SET status = 'received', updated_at = now() WHERE status IN ('validating', 'extracting') AND updated_at < now() - interval '15 minutes' AND attempts < 3`;36  await sql`UPDATE uploads SET status = 'error', error_code = 'processing_interrupted', updated_at = now() WHERE status IN ('validating', 'extracting') AND updated_at < now() - interval '15 minutes'`;37  await sql`UPDATE tracks SET compat_state = 'queued' WHERE compat_state = 'working' AND updated_at < now() - interval '45 minutes'`;38  await sql`UPDATE lyrics SET state = 'queued' WHERE state = 'working' AND updated_at < now() - interval '45 minutes'`;39}4041async function housekeeping() {42  const days = env.uploadRetentionDays;43  // Abandoned uploads and old import rows (retention policy), then their scratch files.44  await sql`UPDATE uploads SET status = 'cancelled', updated_at = now() WHERE status = 'uploading' AND updated_at < now() - interval '2 days'`;45  await sql`DELETE FROM uploads WHERE status IN ('ready', 'cancelled') AND updated_at < now() - ${days} * interval '1 day'`;46  await sql`DELETE FROM uploads WHERE status IN ('error', 'duplicate') AND updated_at < now() - ${days} * interval '1 day'`;47  await sql`DELETE FROM webauthn_challenges WHERE expires_at < now()`;48  await sql`DELETE FROM rate_limits WHERE window_start < now() - interval '1 day'`;49  await sql`DELETE FROM setup_tokens WHERE expires_at < now() - interval '30 days'`;50  await sql`DELETE FROM sessions WHERE expires_at < now() - interval '30 days' OR revoked_at < now() - interval '30 days'`;51  const live = new Set((await sql<{ id: string }[]>`SELECT id FROM uploads WHERE status NOT IN ('ready', 'cancelled')`).map((r) => path.basename(uploadTmpPath(r.id))));52  for (const dir of ["uploads", "work"]) {53    const d = path.join(env.tmpDir, dir);54    for (const f of await readdir(d).catch(() => [] as string[])) {55      if (dir === "uploads" && live.has(f)) continue;56      const s = await stat(path.join(d, f)).catch(() => null);57      if (s && Date.now() - s.mtimeMs > 6 * 3600_000) await rm(path.join(d, f), { force: true });58    }59  }60  await pruneOrphans();61}6263async function main() {64  console.log(`[worker] started (storage ${env.storageDriver}, renditions ${env.renditions}, lyrics ${env.lyrics})`);65  let lastBeat = 0, lastHouse = 0;66  while (!stopping) {67    try {68      const now = Date.now();69      if (now - lastBeat > 20_000) { await sql`INSERT INTO worker_heartbeat (name) VALUES ('worker') ON CONFLICT (name) DO UPDATE SET beat_at = now()`; await recoverStale(); lastBeat = now; }70      if (now - lastHouse > 3600_000) { await housekeeping(); lastHouse = now; }71      const up = await claimUpload();72      if (up) { await processUpload(up); continue; }73      const an = await claimAnalysis();74      if (an) { await reanalyseTrack(an).catch((e) => console.error("[worker] analysis failed", an, e instanceof Error ? e.message : e)); continue; }75      const r = await claimRendition();76      if (r) { await buildRendition(r); continue; }77      const ly = await claimLyrics();78      if (ly) { await transcribeTrack(ly); continue; }79      await sleep(1500);80    } catch (e) {81      console.error("[worker] loop error", e instanceof Error ? `${e.name}: ${e.message}` : String(e));82      await sleep(5000);83    }84  }85  await sql.end({ timeout: 5 });86  console.log("[worker] stopped");87}8889for (const sig of ["SIGINT", "SIGTERM"] as const) process.on(sig, () => { stopping = true; });90main().catch((e) => { console.error(e); process.exit(1); });91
Files main
Menu

Theme