spb/rareindex

1import type { QualityField } from './types.js';23/** Extraction quality weights (§199). */4export const QUALITY_WEIGHTS: Record<QualityField, number> = {5  title: 0.2,6  price: 0.2,7  status: 0.15,8  date: 0.1,9  images: 0.1,10  description: 0.05,11  identifiers: 0.1,12  category: 0.05,13  currency: 0.05,14};1516export const MIN_QUALITY = 0.6;1718function present(v: unknown): boolean {19  if (v === null || v === undefined) return false;20  if (typeof v === 'string') return v.trim().length > 0;21  if (Array.isArray(v)) return v.length > 0;22  if (typeof v === 'number') return Number.isFinite(v);23  if (v instanceof Date) return !Number.isNaN(v.getTime());24  if (typeof v === 'object') return Object.keys(v as object).length > 0;25  return Boolean(v);26}2728/**29 * Score 0–1 = weighted share of expected fields that were found. When `expect` is omitted all30 * weighted fields count; otherwise weights are renormalised over the expected subset so that a31 * catalog page (no price/status) can still reach 1.0.32 */33export function qualityScore(found: Partial<Record<QualityField, unknown>> | null | undefined, expect?: QualityField[]): number {34  if (!found) return 0;35  const fields = (expect && expect.length ? expect : (Object.keys(QUALITY_WEIGHTS) as QualityField[]));36  let total = 0;37  let got = 0;38  for (const f of fields) {39    const w = QUALITY_WEIGHTS[f] ?? 0;40    total += w;41    if (present(found[f])) got += w;42  }43  return total > 0 ? Math.round((got / total) * 1000) / 1000 : 0;44}4546const CHALLENGE_RE = /access denied|captcha|verify you are human|attention required|just a moment\.\.\.|request blocked|unusual traffic|enable javascript and cookies to continue|<title>\s*(?:blocked|forbidden|challenge|security check)|_cf_chl_opt|cf-challenge|px-captcha|distil_r_captcha|akamai.*?bot manager|are you a robot|pardon our interruption/i;4748/**49 * Detect anti-bot challenge / block pages (Cloudflare, Akamai, PerimeterX, DataDome…). Counted in50 * health as "challenge events" (SPEC §12). Short pages with challenge markers are challenges; long51 * pages that merely mention "cloudflare" in a footer are not.52 */53export function isChallengePage(doc: { html?: string | null; markdown?: string | null; json?: unknown; httpStatus?: number | null }): boolean {54  if (doc.json && present(doc.json)) return false;55  const text = doc.markdown ?? doc.html ?? '';56  if (!text) return doc.httpStatus === 403 || doc.httpStatus === 429 || doc.httpStatus === 503;57  const head = text.slice(0, 6000);58  const hit = CHALLENGE_RE.test(head);59  if (!hit) return false;60  return text.length < 25_000 || doc.httpStatus === 403 || doc.httpStatus === 429 || doc.httpStatus === 503;61}6263/** Heuristic score for a document when the connector provides no parser: did we get real content? */64export function documentQuality(doc: { html?: string | null; markdown?: string | null; json?: unknown; httpStatus?: number | null; buffer?: Uint8Array | null }): number {65  if (doc.httpStatus && (doc.httpStatus >= 400 || doc.httpStatus === 0)) return 0;66  if (doc.json && present(doc.json)) return 0.9;67  if (doc.buffer && doc.buffer.byteLength > 0) return 0.9;68  const text = doc.markdown ?? doc.html ?? '';69  if (!text) return 0;70  const len = text.length;71  const blocked = isChallengePage(doc);72  if (blocked && len < 20_000) return 0.1;73  if (len < 500) return 0.2;74  if (len < 2_000) return 0.5;75  return 0.8;76}77