spb/market-atlas

1import { describe, expect, it } from "vitest";2import { dirname, join } from "node:path";3import { fileURLToPath } from "node:url";4import { loadFixture, makeTestContext, normalizeFixture, fakeFetch } from "@market-atlas/connector-sdk";5import { ConnectorMetadataSchema } from "@market-atlas/market-model";6import { CONNECTORS, SOURCES } from "./index.js";7import { coinbaseWs } from "./coinbase-ws/index.js";8import { krakenWs } from "./kraken-ws/index.js";9import { binanceWs } from "./binance-ws/index.js";10import { okxWs } from "./okx-ws/index.js";11import { cboeDelayedQuotes } from "./cboe-delayed-quotes/index.js";12import { ecbFrankfurter } from "./ecb-frankfurter/index.js";13import { bankOfCanada } from "./bank-of-canada/index.js";14import { usTreasuryYieldCurve } from "./us-treasury-yield-curve/index.js";15import { secEdgarFilings } from "./sec-edgar-filings/index.js";16import { secCompanyTickers, titleCase } from "./sec-company-tickers/index.js";17import { nasdaqSymbolDirectory, cleanName } from "./nasdaq-symbol-directory/index.js";18import { nasdaqTradeHalts } from "./nasdaq-trade-halts/index.js";19import { nasdaqMarketCalendar, parseLongDate } from "./nasdaq-market-calendar/index.js";20import { hfmarketdata } from "./hfmarketdata/index.js";2122const here = dirname(fileURLToPath(import.meta.url));23const fx = (dir: string, name: string) => loadFixture(join(here, dir, "fixtures"), name);2425describe("registry", () => {26  it("every connector has valid metadata, a known source, docs and fixtures", () => {27    const ids = new Set<string>();28    for (const c of CONNECTORS) {29      expect(ConnectorMetadataSchema.safeParse(c.metadata).success).toBe(true);30      expect(ids.has(c.metadata.id)).toBe(false);31      ids.add(c.metadata.id);32      expect(SOURCES.some((s) => s.id === c.metadata.sourceId)).toBe(true);33      expect(c.metadata.rightsStatus).not.toBe("UNKNOWN");34      expect(c.fixturesDir).toBe("fixtures");35    }36  });37  it("covers the V1 source-type diversity goal", () => {38    const types = new Set(CONNECTORS.map((c) => c.metadata.sourceType));39    for (const t of ["WEBSOCKET", "XHR", "OFFICIAL_API", "HTML", "RSS", "BULK_FILE", "XML"]) expect(types.has(t as never)).toBe(true);40  });41});4243describe("crypto WebSocket connectors", () => {44  it("coinbase ticker → observations with exchange timestamp", () => {45    const b = normalizeFixture(coinbaseWs, "ticker", fx("coinbase-ws", "ticker.json"));46    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;47    expect(last.value).toBe(77279.97);48    expect(last.symbol).toBe("BTC-USD");49    expect(last.instrumentHint?.base).toBe("BTC");50    expect(last.sourceTimestamp).toBe(Date.parse("2026-09-12T07:54:03.482921Z"));51    expect(last.timestampTrust).toBe("EXCHANGE");52    expect(b.observations.map((o) => o.field)).toContain("BID");53    expect(b.observations.map((o) => o.field)).toContain("VOLUME");54  });55  it("coinbase malformed payload yields no observations and does not throw", () => {56    const b = normalizeFixture(coinbaseWs, "ticker", fx("coinbase-ws", "malformed.json"));57    expect(b.observations).toHaveLength(0);58  });59  it("kraken snapshot → connector-timed observations", () => {60    const b = normalizeFixture(krakenWs, "ticker", fx("kraken-ws", "ticker.json"));61    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;62    expect(last.value).toBe(77281.3);63    expect(last.sourceTimestamp).toBeNull();64    expect(last.timestampTrust).toBe("CONNECTOR");65    expect(b.observations.find((o) => o.field === "VWAP")?.value).toBe(77900.1);66  });67  it("binance miniTicker → USDT instrument", () => {68    const b = normalizeFixture(binanceWs, "miniTicker", fx("binance-ws", "miniTicker.json"));69    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;70    expect(last.value).toBeCloseTo(77298.87);71    expect(last.instrumentHint?.quote).toBe("USDT");72    expect(last.sourceTimestamp).toBe(1789199657016);73  });74  it("okx tickers → observations", () => {75    const b = normalizeFixture(okxWs, "tickers", fx("okx-ws", "tickers.json"));76    expect(b.observations.find((o) => o.field === "LAST_PRICE")?.value).toBe(77303.9);77    expect(b.observations.find((o) => o.field === "ASK")?.value).toBe(77303.9);78    expect(b.observations[0]?.sourceTimestamp).toBe(1789199660123);79  });80  it("unexpected schema (wrong kind) is ignored", () => {81    expect(normalizeFixture(okxWs, "trades", { arg: { channel: "trades" }, data: [{ px: "1" }] }).observations).toHaveLength(0);82  });83  it("streaming connectors open exactly one socket and subscribe on open", async () => {84    for (const def of [coinbaseWs, krakenWs, binanceWs, okxWs]) {85      const ctx = makeTestContext(def);86      await def.start!(ctx);87      expect(ctx.sockets).toHaveLength(1);88      ctx.sockets[0]!.close();89    }90  });91});9293describe("cboe-delayed-quotes", () => {94  it("equity quote → DELAYED observations with Eastern timestamps converted to UTC", () => {95    const b = normalizeFixture(cboeDelayedQuotes, "quote", fx("cboe-delayed-quotes", "quote.json"));96    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;97    expect(last.value).toBe(332.58);98    expect(last.realtimeStatus).toBe("DELAYED");99    expect(last.rightsStatus).toBe("DELAYED");100    // 2026-09-11T15:59:59 America/New_York (EDT, UTC-4) → 19:59:59Z101    expect(new Date(last.sourceTimestamp!).toISOString()).toBe("2026-09-11T19:59:59.000Z");102    expect(b.observations.find((o) => o.field === "PREVIOUS_CLOSE")?.value).toBe(332.27);103    expect(b.observations.find((o) => o.field === "VOLUME")?.value).toBe(50716865);104    expect(b.observations.find((o) => o.field === "IMPLIED_VOLATILITY")?.value).toBe(23.849);105  });106  it("index quote keeps the _SPX source symbol and INDEX hint", () => {107    const b = normalizeFixture(cboeDelayedQuotes, "quote", fx("cboe-delayed-quotes", "index.json"));108    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;109    expect(last.symbol).toBe("_SPX");110    expect(last.instrumentHint?.assetClass).toBe("INDEX");111    expect(b.observations.find((o) => o.field === "VOLUME")?.value).toBe(0);112  });113  it("never-traded symbol (zeros) yields no price observations", () => {114    const b = normalizeFixture(cboeDelayedQuotes, "quote", fx("cboe-delayed-quotes", "closed_market.json"));115    expect(b.observations.filter((o) => o.field === "LAST_PRICE" || o.field === "BID")).toHaveLength(0);116  });117  it("poll tolerates a failing symbol", async () => {118    const ctx = makeTestContext(cboeDelayedQuotes, {119      symbols: ["AAPL", "BROKEN"],120      fetchImpl: fakeFetch({ "/AAPL.json": JSON.stringify(fx("cboe-delayed-quotes", "quote.json")), "/BROKEN.json": { status: 500, body: "boom" } }),121    });122    const raws = await cboeDelayedQuotes.poll!(ctx);123    expect(raws).toHaveLength(1);124    expect(ctx.errors).toHaveLength(1);125  });126});127128describe("official sources", () => {129  it("frankfurter USD base → conventional pairs, latest + previous close, EUR pairs skipped", () => {130    const b = normalizeFixture(ecbFrankfurter, "timeseries", fx("ecb-frankfurter", "timeseries.json"));131    const usdcad = b.observations.filter((o) => o.symbol === "USDCAD");132    expect(usdcad.find((o) => o.field === "LAST_PRICE")?.value).toBe(1.3858);133    expect(usdcad.find((o) => o.field === "PREVIOUS_CLOSE")?.value).toBe(1.3871);134    expect(b.observations.some((o) => o.symbol === "EURUSD")).toBe(false); // EUR pairs come from the EUR-base call135    const gbpusd = b.observations.find((o) => o.symbol === "GBPUSD" && o.field === "LAST_PRICE")!;136    expect(gbpusd.value).toBeCloseTo(1 / 0.7403, 6);137    expect(gbpusd.realtimeStatus).toBe("END_OF_DAY");138    expect(gbpusd.rightsStatus).toBe("OFFICIAL_OPEN_DATA");139  });140  it("bank of canada fx + rates", () => {141    const fxb = normalizeFixture(bankOfCanada, "fx", fx("bank-of-canada", "fx.json"));142    expect(fxb.observations.find((o) => o.symbol === "USDCAD" && o.field === "LAST_PRICE")?.value).toBe(1.3855);143    expect(fxb.observations.find((o) => o.symbol === "EURCAD" && o.field === "PREVIOUS_CLOSE")?.value).toBe(1.606);144    const rates = normalizeFixture(bankOfCanada, "rates", fx("bank-of-canada", "rates.json"));145    expect(rates.observations.find((o) => o.symbol === "CA_POLICY_RATE" && o.field === "RATE")?.value).toBe(2.25);146    expect(rates.observations.find((o) => o.symbol === "CA_10Y" && o.field === "YIELD")?.value).toBe(3.24);147    expect(rates.observations.find((o) => o.symbol === "CA_10Y" && o.field === "PREVIOUS_CLOSE")?.value).toBe(3.21);148  });149  it("treasury XML → 13 tenors with yields and previous close", () => {150    const b = normalizeFixture(usTreasuryYieldCurve, "month", fx("us-treasury-yield-curve", "month.xml"));151    const us10 = b.observations.filter((o) => o.symbol === "US10Y");152    expect(us10.find((o) => o.field === "YIELD")?.value).toBe(4.02);153    expect(us10.find((o) => o.field === "PREVIOUS_CLOSE")?.value).toBe(4.05);154    expect(us10.find((o) => o.field === "YIELD")?.instrumentHint?.assetClass).toBe("TREASURY");155    expect(b.observations.some((o) => o.symbol === "US4M")).toBe(false); // absent tenor is skipped156  });157});158159describe("regulatory / directory / halts", () => {160  it("edgar atom → filings; Form 4 reporting is not material, 8-K is", () => {161    const b = normalizeFixture(secEdgarFilings, "atom", fx("sec-edgar-filings", "atom.xml"));162    expect(b.filings).toHaveLength(2);163    const f4 = b.filings!.find((f) => f.formType === "4")!;164    expect(f4.cik).toBe("1734770");165    expect(f4.metadata?.material).toBe(false);166    const k8 = b.filings!.find((f) => f.formType === "8-K")!;167    expect(k8.companyName).toBe("Apple Inc.");168    expect(k8.cik).toBe("320193");169    expect(k8.metadata?.material).toBe(true);170    expect(k8.url).toContain("0000320193-26-000099");171  });172  it("sec directory → listed created, OTC enrich-only", () => {173    const b = normalizeFixture(secCompanyTickers, "directory", fx("sec-company-tickers", "directory.json"));174    const brk = b.instruments!.find((i) => i.symbol === "BRK.B")!;175    expect(brk.hint.exchangeId).toBe("xnys");176    expect(brk.hint.cik).toBe("1067983");177    expect(brk.aliases).toContain("BRK-B");178    expect(b.instruments!.find((i) => i.symbol === "SOTC")?.createIfMissing).toBe(false);179    expect(titleCase("MICROSOFT CORP")).toBe("Microsoft Corp");180    expect(titleCase("Apple Inc.")).toBe("Apple Inc.");181  });182  it("nasdaq directories → instruments with venues, ETF flags and no test issues", () => {183    const n = normalizeFixture(nasdaqSymbolDirectory, "nasdaqlisted", fx("nasdaq-symbol-directory", "nasdaqlisted.txt"));184    expect(n.instruments!.map((i) => i.symbol)).toEqual(["AAAP", "AACG", "AAPL", "QQQ"]);185    expect(n.instruments!.find((i) => i.symbol === "AAPL")?.hint.name).toBe("Apple Inc.");186    expect(n.instruments!.find((i) => i.symbol === "QQQ")?.hint.assetClass).toBe("ETF");187    expect(n.instruments!.find((i) => i.symbol === "AACG")?.hint.securityType).toBe("ADR");188    const o = normalizeFixture(nasdaqSymbolDirectory, "otherlisted", fx("nasdaq-symbol-directory", "otherlisted.txt"));189    const brk = o.instruments!.find((i) => i.symbol === "BRK.B")!;190    expect(brk.hint.exchangeId).toBe("xnys");191    expect(brk.aliases).toEqual(expect.arrayContaining(["BRK-B", "BRK B"]));192    expect(o.instruments!.find((i) => i.symbol === "SPY")?.hint.exchangeId).toBe("arcx");193    expect(o.instruments!.find((i) => i.symbol === "BAC-PB")?.hint.securityType).toBe("PREFERRED");194    expect(cleanName("Alcoa Corporation Common Stock ")).toBe("Alcoa Corporation");195  });196  it("halts rss → halt + resume events with Eastern → UTC conversion", () => {197    const b = normalizeFixture(nasdaqTradeHalts, "rss", fx("nasdaq-trade-halts", "rss.xml"));198    const halts = b.events!.filter((e) => e.type === "TRADING_HALT");199    expect(halts).toHaveLength(2);200    const hubc = halts.find((e) => e.symbols?.[0] === "HUBC")!;201    expect(new Date(hubc.timestamp).toISOString()).toBe("2026-09-11T23:50:00.000Z");202    expect(hubc.data?.reason).toBe("News pending");203    const resume = b.events!.find((e) => e.type === "TRADING_RESUME")!;204    expect(resume.symbols).toEqual(["ABCD"]);205    expect(new Date(resume.timestamp).toISOString()).toBe("2026-09-11T14:22:31.000Z");206    expect(b.events!.find((e) => e.symbols?.[0] === "ABCD" && e.type === "TRADING_HALT")?.severity).toBe("NOTICE");207  });208  it("calendar html → holidays for every US venue, early close parsed, change event only when changed", () => {209    const html = fx("nasdaq-market-calendar", "page.html") as string;210    const first = nasdaqMarketCalendar.normalize({ connectorId: "nasdaq-market-calendar", sourceId: "nasdaq-trader", kind: "page", payload: html, receivedAt: Date.now(), meta: { changed: true, first: true } });211    expect(first.events).toHaveLength(0);212    expect(first.holidays!.filter((h) => h.exchangeId === "xnys")).toHaveLength(7);213    const early = first.holidays!.find((h) => h.date === "2026-11-27" && h.exchangeId === "xnas")!;214    expect(early.kind).toBe("EARLY_CLOSE");215    expect(early.closeTime).toBe("13:00");216    expect(first.holidays!.find((h) => h.date === "2026-01-19")?.name).toBe("Martin Luther King, Jr. Day");217    const changed = nasdaqMarketCalendar.normalize({ connectorId: "nasdaq-market-calendar", sourceId: "nasdaq-trader", kind: "page", payload: html, receivedAt: Date.now(), meta: { changed: true, first: false, document_hash: "abc", changed_sections: ["x"] } });218    expect(changed.events).toHaveLength(1);219    expect(changed.events![0]!.type).toBe("DOCUMENT_CHANGED");220    expect(parseLongDate("December 25, 2026")).toBe("2026-12-25");221  });222});223224describe("hfmarketdata-daily", () => {225  it("daily bars → 1d bars + END_OF_DAY reference observations", () => {226    const b = hfmarketdata.normalize({ connectorId: "hfmarketdata-daily", sourceId: "hfmarketdata", kind: "bars", payload: fx("hfmarketdata", "bars.json"), receivedAt: Date.now(), meta: { symbol: "AAPL" } });227    expect(b.bars).toHaveLength(3);228    expect(b.bars![0]!.ts).toBe(Date.parse("2026-09-01T00:00:00Z"));229    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;230    expect(last.value).toBe(328.21);231    expect(last.realtimeStatus).toBe("END_OF_DAY");232    expect(b.observations.find((o) => o.field === "PREVIOUS_CLOSE")?.value).toBe(324.96);233    // 2026-09-03 16:00 America/New_York → 20:00Z234    expect(new Date(last.sourceTimestamp!).toISOString()).toBe("2026-09-03T20:00:00.000Z");235  });236  it("continuous futures → COMMODITY instrument with contract metadata", () => {237    const b = hfmarketdata.normalize({ connectorId: "hfmarketdata-daily", sourceId: "hfmarketdata", kind: "continuous", payload: fx("hfmarketdata", "continuous.json"), receivedAt: Date.now(), meta: { symbol: "CL=F" } });238    expect(b.bars).toHaveLength(2);239    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;240    expect(last.value).toBe(91.01);241    expect(last.instrumentHint?.assetClass).toBe("COMMODITY");242    expect(last.meta?.contract).toBe("CLV26");243  });244  it("unknown symbol meta yields nothing", () => {245    expect(hfmarketdata.normalize({ connectorId: "hfmarketdata-daily", sourceId: "hfmarketdata", kind: "bars", payload: { data: [] }, receivedAt: 0, meta: { symbol: "NOPE" } }).observations).toHaveLength(0);246  });247});248249describe("source mesh v2 — venues, live FX and proxies", async () => {250  const { bitstampWs } = await import("./bitstamp-ws/index.js");251  const { geminiWs } = await import("./gemini-ws/index.js");252  const { bitfinexWs } = await import("./bitfinex-ws/index.js");253  const { bybitWs } = await import("./bybit-ws/index.js");254  const { gateWs } = await import("./gate-ws/index.js");255  const { cryptocomWs } = await import("./cryptocom-ws/index.js");256  const { kucoinWs } = await import("./kucoin-ws/index.js");257  const { nasdaqQuoteApi, parseNasdaqTime } = await import("./nasdaq-quote-api/index.js");258  const { planPair } = await import("./_shared/pairs.js");259260  it("plans venue pairs: crypto, real fiat markets, stablecoin proxies (with inversion)", () => {261    expect(planPair("BTC", "USD", "x").hint.assetClass).toBe("CRYPTO");262    const fx = planPair("EUR", "USD", "kraken");263    expect(fx.hint.assetClass).toBe("FOREX");264    expect(fx.type).toBe("TRADE");265    const inv = planPair("USDT", "EUR", "okx");266    expect(inv.type).toBe("STABLECOIN_PROXY");267    expect(inv.hint.base).toBe("EUR");268    expect(inv.invert).toBe(true);269    const brl = planPair("USDT", "BRL", "binance");270    expect(brl.hint.base).toBe("USD");271    expect(brl.hint.quote).toBe("BRL");272    expect(brl.invert).toBe(false);273  });274  it("kraken fiat ticker → FOREX TRADE observation on the conventional pair", () => {275    const b = normalizeFixture(krakenWs, "ticker", { channel: "ticker", type: "update", data: [{ symbol: "EUR/USD", bid: 1.1724, ask: 1.1726, last: 1.1725, volume: 120000.5, high: 1.175, low: 1.17 }] });276    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;277    expect(last.instrumentHint?.assetClass).toBe("FOREX");278    expect(last.observationType).toBe("TRADE");279    expect(last.value).toBe(1.1725);280  });281  it("okx USDT-EUR ticker → inverted EURUSD stablecoin proxy", () => {282    const b = normalizeFixture(okxWs, "tickers", { arg: { channel: "tickers", instId: "USDT-EUR" }, data: [{ instId: "USDT-EUR", last: "0.8530", bidPx: "0.8529", askPx: "0.8531", ts: "1789199660123" }] });283    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;284    expect(last.instrumentHint?.base).toBe("EUR");285    expect(last.observationType).toBe("STABLECOIN_PROXY");286    expect(last.value).toBeCloseTo(1 / 0.853, 6);287    const bid = b.observations.find((o) => o.field === "BID")!;288    expect(bid.value).toBeCloseTo(1 / 0.8531, 6); // venue ask becomes our bid289    expect(b.observations.some((o) => o.field === "VOLUME")).toBe(false);290  });291  it("bitstamp trade + book (EUR/USD fiat market)", () => {292    const t = normalizeFixture(bitstampWs, "trade", fx("bitstamp-ws", "trade.json"));293    expect(t.observations[0]).toMatchObject({ field: "LAST_PRICE", value: 77203.59, observationType: "TRADE" });294    expect(t.observations[0]!.sourceTimestamp).toBe(1789273894413);295    const bk = normalizeFixture(bitstampWs, "book", fx("bitstamp-ws", "book.json"));296    expect(bk.observations.find((o) => o.field === "BID")?.value).toBe(1.17252);297    expect(bk.observations[0]!.instrumentHint?.assetClass).toBe("FOREX");298  });299  it("gemini, bitfinex, bybit, gate, crypto.com, kucoin tickers", () => {300    expect(normalizeFixture(geminiWs, "trade", fx("gemini-ws", "trade.json")).observations[0]).toMatchObject({ field: "LAST_PRICE", value: 77212.78, sourceTimestamp: 1789274085234 });301    const bf = normalizeFixture(bitfinexWs, "ticker", fx("bitfinex-ws", "ticker.json"));302    expect(bf.observations.find((o) => o.field === "LAST_PRICE")?.value).toBe(77118);303    expect(bf.observations.find((o) => o.field === "BID")?.value).toBe(77123);304    expect(normalizeFixture(bybitWs, "ticker", fx("bybit-ws", "ticker.json")).observations.find((o) => o.field === "LAST_PRICE")?.value).toBe(77229.4);305    expect(normalizeFixture(gateWs, "ticker", fx("gate-ws", "ticker.json")).observations.find((o) => o.field === "ASK")?.value).toBe(77227.8);306    expect(normalizeFixture(cryptocomWs, "ticker", fx("cryptocom-ws", "ticker.json")).observations.find((o) => o.field === "LAST_PRICE")?.value).toBe(77219.2);307    const ku = normalizeFixture(kucoinWs, "ticker", fx("kucoin-ws", "ticker.json"));308    expect(ku.observations.find((o) => o.field === "LAST_PRICE")?.value).toBe(77225.1);309    expect(ku.observations[0]!.sourceTimestamp).toBe(1789274087087);310  });311  it("nasdaq.com quote is a restricted validator with parsed Eastern timestamps", () => {312    const b = normalizeFixture(nasdaqQuoteApi, "info", fx("nasdaq-quote-api", "info.json"));313    const last = b.observations.find((o) => o.field === "LAST_PRICE")!;314    expect(last.value).toBe(332.27);315    expect(last.rightsStatus).toBe("PUBLIC_RESTRICTED_REDISTRIBUTION");316    expect(new Date(last.sourceTimestamp!).toISOString()).toBe("2026-09-10T20:00:00.000Z");317    expect(b.observations.find((o) => o.field === "VOLUME")?.value).toBe(50716997);318    expect(parseNasdaqTime("Sep 12, 2026 9:45 AM ET")).toBe(Date.parse("2026-09-12T13:45:00Z"));319    expect(parseNasdaqTime("garbage")).toBeNull();320  });321  it("every streaming connector still opens exactly one socket", async () => {322    for (const def of [bitstampWs, geminiWs, bitfinexWs, bybitWs, gateWs, cryptocomWs]) {323      const ctx = makeTestContext(def);324      await def.start!(ctx);325      expect(ctx.sockets).toHaveLength(1);326      ctx.sockets[0]!.close();327    }328  });329});330