import type { ConnectorHealth } from '@rareindex/shared'; import type { HealthContext } from './types.js'; /** * Derive a ConnectorHealth snapshot (§105) from recent runs. Anomaly detectors that need data * distributions (price drift, duplicate explosion) run in the normalizer and are passed in as * run anomalies. */ export function healthFromRuns(ctx: HealthContext, schemaVersion: string): ConnectorHealth { const now = Date.now(); const runs24h = ctx.recentRuns.filter((r) => now - r.startedAt.getTime() <= 24 * 3600_000); const attempted = runs24h.reduce((a, r) => a + r.pagesAttempted, 0); const success = runs24h.reduce((a, r) => a + r.pagesSuccess, 0); const rawRecords = runs24h.reduce((a, r) => a + r.recordsRaw, 0); const dupes = runs24h.reduce((a, r) => a + r.recordsDuplicate, 0); const fc = sumEngine(runs24h, 'firecrawl'); const sf = sumEngine(runs24h, 'scrapfly'); const lastOk = [...ctx.recentRuns].reverse().find((r) => r.status === 'success' || r.status === 'partial'); const lastErr = [...ctx.recentRuns].reverse().find((r) => r.error)?.error ?? null; const anomalies = [...new Set(runs24h.flatMap((r) => r.anomalies))]; const successRate = attempted > 0 ? success / attempted : null; const failing = ctx.recentRuns.slice(-3).length >= 3 && ctx.recentRuns.slice(-3).every((r) => r.status === 'failed'); let status: ConnectorHealth['status'] = 'unknown'; if (ctx.recentRuns.length === 0) status = 'unknown'; else if (failing) status = 'failing'; else if (successRate !== null && successRate < 0.5) status = 'failing'; else if ((successRate !== null && successRate < 0.85) || anomalies.length > 0) status = 'degraded'; else status = 'healthy'; return { connector: ctx.meta.id, status, success_rate_24h: successRate === null ? null : round3(successRate), pages_attempted: attempted, pages_success: success, firecrawl_success_rate: fc.attempts ? round3(fc.success / fc.attempts) : null, scrapfly_fallback_rate: attempted ? round3(sf.attempts / attempted) : null, parse_failure_rate: rawRecords ? round3(runs24h.reduce((a, r) => a + (r.anomalies.filter((x) => x.startsWith('parse_failure')).length), 0) / rawRecords) : null, last_success: lastOk ? lastOk.startedAt.toISOString() : null, last_error: lastErr, schema_version: schemaVersion, records_24h: rawRecords, duplicates_24h: dupes, anomalies, }; } function sumEngine(runs: HealthContext['recentRuns'], engine: string) { return runs.reduce( (acc, r) => { const s = r.engineStats[engine]; if (s) { acc.attempts += s.attempts; acc.success += s.success; } return acc; }, { attempts: 0, success: 0 }, ); } function round3(x: number): number { return Math.round(x * 1000) / 1000; } /** * Detect abnormal price distributions between a baseline and a new batch (§105): median shift * beyond a factor or a collapse of dispersion suggests a parser/currency problem. */ export function priceDistributionAnomaly(baseline: number[], batch: number[], factor = 3): string | null { if (baseline.length < 20 || batch.length < 10) return null; const med = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]!; }; const b = med(baseline); const n = med(batch); if (b > 0 && (n / b > factor || b / n > factor)) return `price_distribution_shift: median ${b.toFixed(2)} → ${n.toFixed(2)}`; return null; } /** Duplicate explosion: share of duplicate raw records in a run is far above baseline. */ export function duplicateExplosion(records: number, duplicates: number, threshold = 0.9): string | null { if (records < 50) return null; const share = duplicates / records; return share >= threshold ? `duplicate_explosion: ${(share * 100).toFixed(0)}% duplicates` : null; }