TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { canonicalizeHtml, canonicalizeText, canonicalJson, sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { NormalizeError, type WebSensorConnector } from "./types";45/**6 * Generic HTTP connector: HTML pages (canonical text), plain text and JSON documents.7 * Config: { keepChrome?: boolean, selector?: string, accept?: string, headers?: {} }8 */9export class HttpConnector implements WebSensorConnector {10 metadata(): ConnectorMetadata {11 return { key: "http", name: "Generic HTTP", sensorTypes: ["HTML", "JSON", "XML", "HTTP_HEADERS", "FILE"], description: "Conditional GET + canonical extraction for HTML/JSON/text", version: "1.0.0" };12 }1314 async fetch(endpoint: SensorEndpoint): Promise<Observation> {15 const cfg = endpoint.config as { accept?: string; headers?: Record<string, string>; method?: "GET" | "HEAD"; timeoutMs?: number };16 return httpFetchWithRetry(endpoint.id, endpoint.url, {17 method: cfg.method ?? "GET",18 etag: endpoint.etag,19 lastModified: endpoint.lastModified,20 accept: cfg.accept ?? (endpoint.type === "JSON" ? "application/json, */*;q=0.5" : undefined),21 headers: cfg.headers,22 timeoutMs: cfg.timeoutMs,23 });24 }2526 async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {27 if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");28 const ct = (obs.meta.contentType ?? "").toLowerCase();29 const text = obs.body.toString("utf8");30 const cfg = endpoint.config as { keepChrome?: boolean; jsonPath?: string; ignoreKeys?: string[] };3132 if (endpoint.type === "HTTP_HEADERS" || obs.meta.method === "HEAD") {33 const h = { ...obs.meta.headers };34 delete h.date;35 delete h.age;36 delete h["cf-ray"];37 const j = canonicalJson(h);38 return { mode: "json", json: h, rawHash: sha256(j), canonicalHash: sha256(j), semanticHash: simhash(j), extractionConfidence: 1 };39 }4041 if (endpoint.type === "JSON" || ct.includes("json") || /^\s*[[{]/.test(text.slice(0, 50))) {42 let parsed: unknown;43 try {44 parsed = JSON.parse(text);45 } catch {46 if (endpoint.type === "JSON") throw new NormalizeError("bad_json", "Response is not valid JSON");47 parsed = undefined;48 }49 if (parsed !== undefined) {50 let node: unknown = parsed;51 if (cfg.jsonPath) node = getPath(node, cfg.jsonPath);52 if (cfg.ignoreKeys?.length) node = stripKeys(node, new Set(cfg.ignoreKeys));53 const j = canonicalJson(node);54 return { mode: "json", json: node as object | null, rawHash: sha256(text), canonicalHash: sha256(j), semanticHash: simhash(j), extractionConfidence: 1 };55 }56 }5758 if (ct.includes("html") || /<\s*(!doctype|html|body|div|p)\b/i.test(text.slice(0, 4000))) {59 const c = canonicalizeHtml(text, obs.meta.finalUrl, { keepChrome: cfg.keepChrome });60 if (c.text.length < 40 && c.headings.length === 0) {61 // Probably a JS shell, a challenge page, or an interstitial: low extraction confidence.62 return { mode: "text", text: c.text, title: c.title, headings: c.headings, links: c.links, rawHash: c.rawHash, canonicalHash: c.canonicalHash, semanticHash: c.semanticHash, extractionConfidence: 0.3, publishedAt: parseMetaDate(c.meta), extra: { structure: c.structure, meta: c.meta, thin: true } };63 }64 return { mode: "text", text: c.text, title: c.title, headings: c.headings, links: c.links, rawHash: c.rawHash, canonicalHash: c.canonicalHash, semanticHash: c.semanticHash, extractionConfidence: 0.75, publishedAt: parseMetaDate(c.meta), extra: { structure: c.structure, meta: c.meta } };65 }6667 const t = canonicalizeText(text);68 return { mode: "text", text: t.text, rawHash: t.rawHash, canonicalHash: t.canonicalHash, semanticHash: t.semanticHash, extractionConfidence: 0.9 };69 }70}7172function parseMetaDate(meta: Record<string, string>): Date | null {73 const s = meta["article:modified_time"] ?? meta["article:published_time"] ?? meta["last-modified"];74 if (!s) return null;75 const d = new Date(s);76 return Number.isNaN(d.getTime()) ? null : d;77}7879export function getPath(obj: unknown, path: string): unknown {80 let cur = obj;81 for (const part of path.split(".").filter(Boolean)) {82 if (cur === null || cur === undefined) return undefined;83 const m = part.match(/^(\w+)?\[(\d+)\]$/);84 if (m) {85 if (m[1]) cur = (cur as Record<string, unknown>)[m[1]];86 cur = Array.isArray(cur) ? cur[Number(m[2])] : undefined;87 } else cur = (cur as Record<string, unknown>)[part];88 }89 return cur;90}9192function stripKeys(v: unknown, keys: Set<string>): unknown {93 if (Array.isArray(v)) return v.map((x) => stripKeys(x, keys));94 if (v && typeof v === "object") {95 const out: Record<string, unknown> = {};96 for (const [k, val] of Object.entries(v as Record<string, unknown>)) if (!keys.has(k)) out[k] = stripKeys(val, keys);97 return out;98 }99 return v;100}101