TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { canonicalizeText, sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { NormalizeError, type WebSensorConnector } from "./types";45/**6 * PDF connector — official documents published as PDF (central-bank statements, regulatory notices,7 * tariff schedules, price lists, terms). Text is extracted with pdf.js (via `unpdf`, no native8 * dependency), then canonicalized like any text page so diffs and heuristics are shared.9 *10 * Config: { maxPages?: 40, keepLayout?: false }11 * A PDF that yields < 40 chars of text (scanned image) gets extraction confidence 0.2 — the pipeline12 * then relies on the raw hash only (changed / unchanged), never on text diffs.13 */14export class PdfConnector implements WebSensorConnector {15 mode = "text" as const;16 metadata(): ConnectorMetadata {17 return { key: "pdf", name: "PDF document", sensorTypes: ["FILE", "PDF_INDEX"], description: "Text extraction (pdf.js) + canonical text diff for official PDF documents", version: "1.0.0" };18 }19 async fetch(endpoint: SensorEndpoint): Promise<Observation> {20 const cfg = endpoint.config as { headers?: Record<string, string>; timeoutMs?: number };21 return httpFetchWithRetry(endpoint.id, endpoint.url, { etag: endpoint.etag, lastModified: endpoint.lastModified, accept: "application/pdf, */*;q=0.5", headers: cfg.headers, timeoutMs: cfg.timeoutMs ?? 60_000, maxBytes: 40 * 1024 * 1024 });22 }23 async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {24 if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");25 if (obs.body.subarray(0, 5).toString("latin1") !== "%PDF-") {26 if (/^\s*<(!doctype|html)/i.test(obs.body.subarray(0, 300).toString("utf8"))) throw new NormalizeError("html_not_pdf", "Endpoint returned HTML instead of a PDF");27 throw new NormalizeError("bad_pdf", "Body does not start with %PDF-");28 }29 const cfg = endpoint.config as { maxPages?: number };30 const { text, pages, meta } = await extractPdfText(obs.body, cfg.maxPages ?? 40);31 const t = canonicalizeText(text);32 const thin = t.text.length < 40;33 return {34 mode: "text",35 text: t.text,36 title: meta.title ?? endpoint.name,37 rawHash: sha256(obs.body),38 canonicalHash: t.canonicalHash,39 semanticHash: t.semanticHash,40 publishedAt: meta.modified ?? meta.created ?? null,41 extractionConfidence: thin ? 0.2 : 0.85,42 extra: { pages, chars: t.text.length, thin, producer: meta.producer, author: meta.author },43 };44 }45}4647export async function extractPdfText(buf: Buffer, maxPages: number): Promise<{ text: string; pages: number; meta: { title?: string; author?: string; producer?: string; created?: Date | null; modified?: Date | null } }> {48 const { getDocumentProxy, extractText, getMeta } = await import("unpdf");49 const data = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);50 const pdf = await getDocumentProxy(data);51 const total = pdf.numPages;52 const parts: string[] = [];53 const limit = Math.min(total, maxPages);54 if (limit === total) {55 const r = await extractText(pdf, { mergePages: false });56 parts.push(...(r.text as string[]));57 } else {58 for (let p = 1; p <= limit; p++) {59 const page = await pdf.getPage(p);60 const content = await page.getTextContent();61 parts.push((content.items as { str?: string; hasEOL?: boolean }[]).map((it) => (it.str ?? "") + (it.hasEOL ? "\n" : "")).join(" "));62 }63 }64 let meta: { title?: string; author?: string; producer?: string; created?: Date | null; modified?: Date | null } = {};65 try {66 const m = await getMeta(pdf);67 const info = (m.info ?? {}) as Record<string, unknown>;68 meta = { title: str(info.Title), author: str(info.Author), producer: str(info.Producer), created: pdfDate(info.CreationDate), modified: pdfDate(info.ModDate) };69 } catch {70 /* metadata optional */71 }72 const text = parts73 .map((p) => p.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n"))74 .join("\n\n")75 .trim();76 return { text, pages: total, meta };77}7879function str(v: unknown): string | undefined {80 return typeof v === "string" && v.trim() ? v.trim() : undefined;81}8283/** PDF date `D:YYYYMMDDHHmmSS+HH'mm'` → Date */84function pdfDate(v: unknown): Date | null {85 if (typeof v !== "string") return null;86 const m = v.match(/D:(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?([Z+-])?(\d{2})?'?(\d{2})?/);87 if (!m) return null;88 const [, y, mo = "01", d = "01", h = "00", mi = "00", s = "00", tz, tzh = "00", tzm = "00"] = m;89 const iso = `${y}-${mo}-${d}T${h}:${mi}:${s}${tz === "Z" || !tz ? "Z" : `${tz}${tzh}:${tzm}`}`;90 const dt = new Date(iso);91 return Number.isNaN(dt.getTime()) ? null : dt;92}93