spb/market-atlas

1import { describe, expect, it } from "vitest";2import { HttpClient, RateLimiter, TokenBucket, assertPublicUrl, extractTables, htmlFingerprints, isPrivateAddress, parseDelimited, parseFeed, parseNumber, redactObject, redactUrl, schemaFingerprint } from "./index.js";3import { fakeFetch } from "./testing.js";45describe("rate limiter", () => {6  it("token bucket paces requests", async () => {7    const b = new TokenBucket(50, 2);8    const t0 = Date.now();9    for (let i = 0; i < 6; i++) await b.acquire();10    expect(Date.now() - t0).toBeGreaterThanOrEqual(60); // 4 extra tokens at 50/s ≈ 80 ms11  });12  it("limiter keys by host", () => {13    const l = new RateLimiter({ ratePerSec: 1 });14    l.configure("a.example", 10);15    expect(l.bucket("A.EXAMPLE").ratePerSec).toBe(10);16    expect(l.bucket("b.example").ratePerSec).toBe(1);17  });18});1920describe("http client", () => {21  it("uses ETag for conditional requests and returns cached body on 304", async () => {22    let calls = 0;23    const fetchImpl = (async (_url: string | URL | Request, init?: RequestInit) => {24      calls++;25      const h = (init?.headers ?? {}) as Record<string, string>;26      if (h["if-none-match"] === '"v1"') return new Response(null, { status: 304 });27      return new Response('{"a":1}', { status: 200, headers: { etag: '"v1"' } });28    }) as typeof fetch;29    const c = new HttpClient({ userAgent: "t", limiter: new RateLimiter({ ratePerSec: 1000 }), fetchImpl });30    const r1 = await c.getJson<{ a: number }>("https://x.example/data");31    const r2 = await c.getJson<{ a: number }>("https://x.example/data");32    expect(r1.data.a).toBe(1);33    expect(r2.response.notModified).toBe(true);34    expect(r2.data.a).toBe(1);35    expect(calls).toBe(2);36  });37  it("retries 5xx then throws HttpError with redacted URL", async () => {38    const c = new HttpClient({ userAgent: "t", limiter: new RateLimiter({ ratePerSec: 1000 }), fetchImpl: fakeFetch({ boom: { status: 503, body: "x" } }) });39    await expect(c.getText("https://x.example/boom?api_key=SECRET", { retries: 1 })).rejects.toThrow(/HTTP 503.*api_key=\[redacted\]/);40  });41});4243describe("ssrf guard", () => {44  it("blocks private and internal targets", async () => {45    await expect(assertPublicUrl("http://127.0.0.1/")).rejects.toThrow();46    await expect(assertPublicUrl("http://10.0.0.5/x")).rejects.toThrow();47    await expect(assertPublicUrl("http://localhost:8080/")).rejects.toThrow();48    await expect(assertPublicUrl("ftp://example.com/")).rejects.toThrow();49    await expect(assertPublicUrl("http://169.254.169.254/latest/meta-data")).rejects.toThrow();50    await expect(assertPublicUrl("https://m3u96a.maclustr.io/")).rejects.toThrow();51    expect(isPrivateAddress("192.168.2.1")).toBe(true);52    expect(isPrivateAddress("8.8.8.8")).toBe(false);53    expect(isPrivateAddress("fd00::1")).toBe(true);54  });55});5657describe("parsers", () => {58  it("schema fingerprint ignores values but not shape", () => {59    const a = schemaFingerprint({ sym: "AAPL", lp: "1.0", v: 1, ts: 2 });60    const b = schemaFingerprint({ sym: "MSFT", lp: "2.0", v: 9, ts: 3 });61    const c = schemaFingerprint({ ticker: "AAPL", lastPrice: 1, eventTime: 2 });62    expect(a).toBe(b);63    expect(a).not.toBe(c);64  });65  it("parses numbers in several locales", () => {66    expect(parseNumber("1,234.5")).toBe(1234.5);67    expect(parseNumber("1.234,5")).toBe(1234.5);68    expect(parseNumber("(12.3)")).toBe(-12.3);69    expect(parseNumber("12.3%")).toBe(12.3);70    expect(parseNumber("N/A")).toBeNull();71    expect(parseNumber("$1,000")).toBe(1000);72  });73  it("parses pipe-delimited and quoted csv", () => {74    const rows = parseDelimited('a|b\n1|"x|y"\n2|z', "|");75    expect(rows).toEqual([{ a: "1", b: "x|y" }, { a: "2", b: "z" }]);76  });77  it("parses RSS and Atom into items", () => {78    const rss = parseFeed('<rss><channel><title>T</title><item><title>A</title><link>http://a</link><guid>g1</guid><pubDate>Fri, 11 Sep 2026 04:00:00 GMT</pubDate></item></channel></rss>');79    expect(rss.items[0]).toMatchObject({ title: "A", link: "http://a", id: "g1" });80    const atom = parseFeed('<feed xmlns="http://www.w3.org/2005/Atom"><title>F</title><entry><title>E</title><id>i1</id><link rel="alternate" href="http://e"/><updated>2026-09-11T00:00:00Z</updated><category term="8-K" label="form type"/></entry></feed>');81    expect(atom.items[0]).toMatchObject({ title: "E", link: "http://e", id: "i1" });82    expect(atom.items[0]!.categories[0]!.term).toBe("8-K");83  });84  it("extracts tables and fingerprints html sections", () => {85    const html = "<h1>Title</h1><table><tr><th>a</th><th>b</th></tr><tr><td>1</td><td>2</td></tr></table><h2>Other</h2><p>x</p>";86    expect(extractTables(html)).toEqual([[["a", "b"], ["1", "2"]]]);87    const fp1 = htmlFingerprints(html);88    const fp2 = htmlFingerprints(html.replace("<p>x</p>", "<p>y</p>"));89    expect(fp1.document).not.toBe(fp2.document);90    expect(fp1.sections["Title"]).toBe(fp2.sections["Title"]);91    expect(fp1.sections["Other"]).not.toBe(fp2.sections["Other"]);92  });93  it("redacts secrets", () => {94    expect(redactUrl("https://x/y?token=abc&z=1")).toBe("https://x/y?token=[redacted]&z=1");95    expect(redactObject({ headers: { Authorization: "Bearer xyz", accept: "*/*" }, cookie: "a=b", price: 1 })).toEqual({ headers: { Authorization: "[redacted]", accept: "*/*" }, cookie: "[redacted]", price: 1 });96  });97});98