spb/market-atlas

1import { XMLParser } from "fast-xml-parser";2import { sha256 } from "./fingerprint.js";34const xmlParser = new XMLParser({5  ignoreAttributes: false,6  attributeNamePrefix: "@_",7  textNodeName: "#text",8  parseTagValue: false,9  trimValues: true,10  cdataPropName: "#cdata",11  removeNSPrefix: false,12});1314export function parseXml<T = Record<string, unknown>>(text: string): T {15  return xmlParser.parse(text) as T;16}1718export interface FeedItem {19  id: string | null;20  title: string;21  link: string | null;22  published: string | null;23  updated: string | null;24  summary: string | null;25  categories: Array<{ term: string; label?: string; scheme?: string }>;26  /** Any namespaced or custom element (e.g. ndaq:HaltDate) kept verbatim. */27  extra: Record<string, unknown>;28}2930const asArray = <T>(v: T | T[] | undefined | null): T[] => (v == null ? [] : Array.isArray(v) ? v : [v]);31const text = (v: unknown): string | null => {32  if (v == null) return null;33  if (typeof v === "string") return v;34  if (typeof v === "number") return String(v);35  if (typeof v === "object") {36    const o = v as Record<string, unknown>;37    if (typeof o["#cdata"] === "string") return o["#cdata"] as string;38    if (typeof o["#text"] === "string") return o["#text"] as string;39    if (typeof o["@_href"] === "string") return o["@_href"] as string;40  }41  return null;42};4344/** Parse RSS 2.0 or Atom into a common item list. */45export function parseFeed(xml: string): { title: string | null; items: FeedItem[] } {46  const doc = parseXml<Record<string, any>>(xml);47  if (doc.feed) {48    const feed = doc.feed;49    const items = asArray(feed.entry).map((e: Record<string, unknown>) => {50      const links = asArray(e.link as any);51      const alt = links.find((l: any) => l?.["@_rel"] === "alternate") ?? links[0];52      return {53        id: text(e.id),54        title: text(e.title) ?? "",55        link: alt ? text(alt) : null,56        published: text(e.published),57        updated: text(e.updated),58        summary: text(e.summary) ?? text(e.content),59        categories: asArray(e.category as any).map((c: any) => ({ term: String(c?.["@_term"] ?? ""), label: c?.["@_label"], scheme: c?.["@_scheme"] })),60        extra: e as Record<string, unknown>,61      } satisfies FeedItem;62    });63    return { title: text(feed.title), items };64  }65  const channel = doc.rss?.channel ?? doc.channel;66  if (channel) {67    const items = asArray(channel.item).map((e: Record<string, unknown>) => ({68      id: text(e.guid) ?? text(e.link),69      title: text(e.title) ?? "",70      link: text(e.link),71      published: text(e.pubDate),72      updated: null,73      summary: text(e.description),74      categories: asArray(e.category as any).map((c: any) => ({ term: text(c) ?? "" })),75      extra: e as Record<string, unknown>,76    }));77    return { title: text(channel.title), items };78  }79  return { title: null, items: [] };80}8182/** Delimited text (pipe/comma/tab) → array of records keyed by header. Handles simple quoted fields. */83export function parseDelimited(textInput: string, delimiter = ","): Record<string, string>[] {84  const lines = textInput.replace(/\r\n?/g, "\n").split("\n").filter((l) => l.length > 0);85  if (!lines.length) return [];86  const split = (line: string): string[] => {87    const out: string[] = [];88    let cur = "";89    let q = false;90    for (let i = 0; i < line.length; i++) {91      const ch = line[i]!;92      if (ch === '"') {93        if (q && line[i + 1] === '"') {94          cur += '"';95          i++;96        } else q = !q;97      } else if (ch === delimiter && !q) {98        out.push(cur);99        cur = "";100      } else cur += ch;101    }102    out.push(cur);103    return out;104  };105  const header = split(lines[0]!).map((h) => h.trim());106  const rows: Record<string, string>[] = [];107  for (const line of lines.slice(1)) {108    const cells = split(line);109    if (cells.length < 2) continue;110    const rec: Record<string, string> = {};111    header.forEach((h, i) => (rec[h] = (cells[i] ?? "").trim()));112    rows.push(rec);113  }114  return rows;115}116117/** Locale-tolerant number parser: "1,234.5", "1 234,5", "(12.3)", "12.3%", "N/A" → number|null. */118export function parseNumber(v: unknown): number | null {119  if (v == null) return null;120  if (typeof v === "number") return Number.isFinite(v) ? v : null;121  if (typeof v !== "string") return null;122  let s = v.trim();123  if (!s || /^(n\/?a|null|undefined|-|—|–)$/i.test(s)) return null;124  let neg = false;125  if (/^\(.*\)$/.test(s)) {126    neg = true;127    s = s.slice(1, -1);128  }129  s = s.replace(/[%$€£¥\s]/g, "").replace(/[+]/g, "");130  if (/^-?\d{1,3}(\.\d{3})+(,\d+)?$/.test(s)) s = s.replace(/\./g, "").replace(",", ".");131  else if (/^-?\d+,\d{1,2}$/.test(s) || /^-?\d+,\d{4,}$/.test(s)) s = s.replace(",", "."); // "12,5" decimal comma; "1,000" stays thousands132  else s = s.replace(/,/g, "");133  const n = Number(s);134  if (!Number.isFinite(n)) return null;135  return neg ? -n : n;136}137138export function stripHtml(html: string): string {139  return html140    .replace(/<script[\s\S]*?<\/script>/gi, " ")141    .replace(/<style[\s\S]*?<\/style>/gi, " ")142    .replace(/<[^>]+>/g, " ")143    .replace(/&nbsp;/g, " ")144    .replace(/&amp;/g, "&")145    .replace(/&lt;/g, "<")146    .replace(/&gt;/g, ">")147    .replace(/&#39;|&apos;/g, "'")148    .replace(/&quot;/g, '"')149    .replace(/\s+/g, " ")150    .trim();151}152153/** Extract simple HTML tables as arrays of cell texts (no nested tables). */154export function extractTables(html: string): string[][][] {155  const tables: string[][][] = [];156  const tableRe = /<table[\s\S]*?<\/table>/gi;157  for (const t of html.match(tableRe) ?? []) {158    const rows: string[][] = [];159    for (const tr of t.match(/<tr[\s\S]*?<\/tr>/gi) ?? []) {160      const cells = (tr.match(/<t[hd][^>]*>[\s\S]*?<\/t[hd]>/gi) ?? []).map((c) => stripHtml(c));161      if (cells.length) rows.push(cells);162    }163    if (rows.length) tables.push(rows);164  }165  return tables;166}167168/**169 * Change-detection fingerprints for HTML documents: whole normalized text + per-section hashes170 * (split on headings) so that a page can be compared section by section.171 */172export function htmlFingerprints(html: string): { document: string; sections: Record<string, string> } {173  const normalized = stripHtml(html).toLowerCase();174  const sections: Record<string, string> = {};175  const parts = html.split(/<h[1-4][^>]*>/i);176  parts.forEach((p, i) => {177    const title = stripHtml(p.split(/<\/h[1-4]>/i)[0] ?? "").slice(0, 60) || `section-${i}`;178    sections[title] = sha256(stripHtml(p).toLowerCase()).slice(0, 16);179  });180  return { document: sha256(normalized).slice(0, 24), sections };181}182