1import type { QualityField } from './types.js';23/** Extraction quality weights (§199). */4export const QUALITY_WEIGHTS: Record<QualityField, number> = {5 title: 0.2,6 price: 0.2,7 status: 0.15,8 date: 0.1,9 images: 0.1,10 description: 0.05,11 identifiers: 0.1,12 category: 0.05,13 currency: 0.05,14};1516export const MIN_QUALITY = 0.6;1718function present(v: unknown): boolean {19 if (v === null || v === undefined) return false;20 if (typeof v === 'string') return v.trim().length > 0;21 if (Array.isArray(v)) return v.length > 0;22 if (typeof v === 'number') return Number.isFinite(v);23 if (v instanceof Date) return !Number.isNaN(v.getTime());24 if (typeof v === 'object') return Object.keys(v as object).length > 0;25 return Boolean(v);26}2728/**29 * Score 0–1 = weighted share of expected fields that were found. When `expect` is omitted all30 * weighted fields count; otherwise weights are renormalised over the expected subset so that a31 * catalog page (no price/status) can still reach 1.0.32 */33export function qualityScore(found: Partial<Record<QualityField, unknown>> | null | undefined, expect?: QualityField[]): number {34 if (!found) return 0;35 const fields = (expect && expect.length ? expect : (Object.keys(QUALITY_WEIGHTS) as QualityField[]));36 let total = 0;37 let got = 0;38 for (const f of fields) {39 const w = QUALITY_WEIGHTS[f] ?? 0;40 total += w;41 if (present(found[f])) got += w;42 }43 return total > 0 ? Math.round((got / total) * 1000) / 1000 : 0;44}4546/** Heuristic score for a document when the connector provides no parser: did we get real content? */47export function documentQuality(doc: { html?: string | null; markdown?: string | null; json?: unknown; httpStatus?: number | null }): number {48 if (doc.httpStatus && (doc.httpStatus >= 400 || doc.httpStatus === 0)) return 0;49 if (doc.json && present(doc.json)) return 0.9;50 const text = doc.markdown ?? doc.html ?? '';51 if (!text) return 0;52 const len = text.length;53 const blocked = /access denied|captcha|verify you are human|attention required|cloudflare|just a moment|request blocked|unusual traffic|enable javascript and cookies/i.test(text.slice(0, 4000));54 if (blocked && len < 20_000) return 0.1;55 if (len < 500) return 0.2;56 if (len < 2_000) return 0.5;57 return 0.8;58}59