spb/market-atlas

1import type { HttpClient } from "@market-atlas/connector-sdk";2import { redactUrl, schemaFingerprint } from "@market-atlas/connector-sdk";34/**5 * Discovery prototype (network-level, no browser yet): fetches a public page unauthenticated,6 * inventories the data endpoints it references (JSON/XHR URLs, WebSocket URLs, SSE, embedded7 * JSON state such as __NEXT_DATA__), and scores which ones look like market-data feeds.8 * Output is a *candidate* report for the rights-review workflow — never an activated connector.9 */10export interface DiscoveryReport {11  url: string;12  status: number;13  content_type: string | null;14  bytes: number;15  websocket_urls: string[];16  sse_urls: string[];17  json_endpoints: Array<{ url: string; score: number; hints: string[] }>;18  embedded_state: Array<{ kind: string; fingerprint: string; price_like_fields: string[]; symbol_like_fields: string[]; sample_keys: string[] }>;19  instrument_hints: string[];20  notes: string[];21}2223const PRICE_KEYS = /^(last|lastprice|last_price|price|lp|px|close|c|regularmarketprice|current_price|bid|ask|mid|value|rate)$/i;24const SYMBOL_KEYS = /^(symbol|sym|ticker|s|code|instrument|isin|pair|product_id|instid)$/i;25const TS_KEYS = /^(ts|t|time|timestamp|datetime|eventtime|updated|last_trade_time)$/i;2627export async function runDiscovery(url: string, http: HttpClient): Promise<DiscoveryReport> {28  const res = await http.getText(url, { timeoutMs: 20_000, conditional: false, headers: { accept: "text/html,application/json;q=0.9,*/*;q=0.8" } });29  const html = res.text;30  const notes: string[] = [];31  const wsUrls = uniq([...html.matchAll(/wss?:\/\/[^\s"'<>\\)]+/g)].map((m) => redactUrl(m[0])));32  const sse = uniq([...html.matchAll(/EventSource\((["'`])([^"'`]+)\1/g)].map((m) => m[2]!));33  const urls = uniq([...html.matchAll(/https?:\/\/[^\s"'<>\\)]+/g)].map((m) => m[0]))34    .filter((u) => /api|quote|price|market|ticker|chart|data|feed|graphql|json|v\d\//i.test(u) && !/\.(png|jpg|jpeg|gif|svg|css|woff2?|ico)(\?|$)/i.test(u))35    .map((u) => redactUrl(u));36  const jsonEndpoints = urls37    .map((u) => {38      const hints: string[] = [];39      let score = 0;40      if (/quote|price|ticker|last/i.test(u)) (score += 40), hints.push("quote/price keyword");41      if (/graphql/i.test(u)) (score += 20), hints.push("GraphQL");42      if (/\.json(\?|$)|format=json/i.test(u)) (score += 15), hints.push("JSON");43      if (/api\.|\/api\//i.test(u)) (score += 15), hints.push("API path");44      if (/chart|history|candles|bars/i.test(u)) (score += 10), hints.push("historical/chart");45      if (/symbol=|ticker=|sym=/i.test(u)) (score += 20), hints.push("symbol parameter");46      return { url: u, score, hints };47    })48    .filter((e) => e.score > 0)49    .sort((a, b) => b.score - a.score)50    .slice(0, 40);5152  const embedded: DiscoveryReport["embedded_state"] = [];53  const stateBlocks: Array<[string, RegExp]> = [54    ["__NEXT_DATA__", /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i],55    ["__NUXT__", /window\.__NUXT__\s*=\s*([\s\S]*?);\s*<\/script>/i],56    ["__INITIAL_STATE__", /window\.__INITIAL_STATE__\s*=\s*([\s\S]*?);\s*<\/script>/i],57    ["__PRELOADED_STATE__", /window\.__PRELOADED_STATE__\s*=\s*([\s\S]*?);\s*<\/script>/i],58    ["ld+json", /<script type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/i],59  ];60  for (const [kind, re] of stateBlocks) {61    const m = html.match(re);62    if (!m?.[1]) continue;63    try {64      const obj = JSON.parse(m[1]);65      const keys = new Set<string>();66      const priceLike = new Set<string>();67      const symbolLike = new Set<string>();68      walk(obj, "", 0, (path, key, val) => {69        keys.add(key);70        if (PRICE_KEYS.test(key) && (typeof val === "number" || (typeof val === "string" && /^-?\d+(\.\d+)?$/.test(val)))) priceLike.add(path);71        if (SYMBOL_KEYS.test(key) && typeof val === "string") symbolLike.add(path);72      });73      embedded.push({ kind, fingerprint: schemaFingerprint(obj), price_like_fields: [...priceLike].slice(0, 20), symbol_like_fields: [...symbolLike].slice(0, 20), sample_keys: [...keys].slice(0, 60) });74    } catch {75      notes.push(`${kind} block present but not parseable as JSON`);76    }77  }78  if (res.headers["content-type"]?.includes("application/json")) {79    try {80      const obj = JSON.parse(html);81      const priceLike = new Set<string>();82      const symbolLike = new Set<string>();83      const tsLike = new Set<string>();84      walk(obj, "", 0, (path, key, val) => {85        if (PRICE_KEYS.test(key) && typeof val !== "object") priceLike.add(path);86        if (SYMBOL_KEYS.test(key) && typeof val === "string") symbolLike.add(path);87        if (TS_KEYS.test(key)) tsLike.add(path);88      });89      embedded.push({ kind: "response-json", fingerprint: schemaFingerprint(obj), price_like_fields: [...priceLike], symbol_like_fields: [...symbolLike], sample_keys: [...tsLike].map((t) => `ts:${t}`) });90      notes.push("URL returns JSON directly — candidate XHR/FETCH connector");91    } catch {92      /* ignore */93    }94  }95  const instrumentHints = uniq([...html.matchAll(/\b(?:NASDAQ|NYSE|TSX|LSE|XETRA|BINANCE|COINBASE):([A-Z.\-]{1,12})\b/g)].map((m) => m[0])).slice(0, 20);96  if (wsUrls.length) notes.push("WebSocket endpoints referenced — verify subscription protocol and terms before use");97  if (!wsUrls.length && !jsonEndpoints.length && !embedded.length) notes.push("No data endpoints found statically; the page may load data via bundled JavaScript (browser-based discovery required)");98  notes.push("Rights status must be classified manually (RIGHTS_REVIEW) before any connector reaches staging.");99  return {100    url: redactUrl(url),101    status: res.status,102    content_type: res.headers["content-type"] ?? null,103    bytes: html.length,104    websocket_urls: wsUrls,105    sse_urls: sse,106    json_endpoints: jsonEndpoints,107    embedded_state: embedded,108    instrument_hints: instrumentHints,109    notes,110  };111}112113function walk(v: unknown, path: string, depth: number, fn: (path: string, key: string, val: unknown) => void) {114  if (depth > 8 || v == null) return;115  if (Array.isArray(v)) {116    for (const item of v.slice(0, 5)) walk(item, `${path}[]`, depth + 1, fn);117    return;118  }119  if (typeof v === "object") {120    for (const [k, val] of Object.entries(v as Record<string, unknown>)) {121      const p = path ? `${path}.${k}` : k;122      fn(p, k, val);123      walk(val, p, depth + 1, fn);124    }125  }126}127128const uniq = <T>(xs: T[]) => [...new Set(xs)];129