spb/market-atlas

1import { defineConnector, raw, type NormalizedBatch, type ProposedBar, type ProposedInstrument } from "@market-atlas/connector-sdk";2import type { InstrumentHint, NormalizedObservation, RawObservation } from "@market-atlas/market-model";3import { zonedTimeToUtc } from "@market-atlas/market-model";4import { US_EQUITIES, US_ETFS, US_INDICES, equityHint, indexHint } from "../_shared/us-universe.js";5import { fxHint } from "../ecb-frankfurter/index.js";6import { cryptoHint } from "../_shared/crypto.js";78const BASE = "https://www.hfmarketdata.io/v1";9const ET = "America/New_York";1011interface Target {12  path: string; // request path after /v113  symbol: string; // Market Atlas source symbol14  hint: InstrumentHint;15  kind: "bars" | "continuous";16  sessionCloseLocal: string; // local close time of the daily bar17  timezone: string;18}1920const FUTURES: Array<[string, string, string]> = [21  ["GC", "Gold futures (continuous)", "xcme"],22  ["SI", "Silver futures (continuous)", "xcme"],23  ["CL", "WTI crude oil futures (continuous)", "xcme"],24  ["NG", "Henry Hub natural gas futures (continuous)", "xcme"],25  ["HG", "Copper futures (continuous)", "xcme"],26  ["ZC", "Corn futures (continuous)", "xcme"],27  ["ZW", "Wheat futures (continuous)", "xcme"],28  ["ZS", "Soybean futures (continuous)", "xcme"],29  ["ES", "E-mini S&P 500 futures (continuous)", "xcme"],30  ["NQ", "E-mini Nasdaq-100 futures (continuous)", "xcme"],31  ["ZN", "10-year T-note futures (continuous)", "xcme"],32  ["6E", "Euro FX futures (continuous)", "xcme"],33  ["BTC", "Bitcoin futures (continuous)", "xcme"],34];35const FX_PAIRS = ["EURUSD", "USDJPY", "GBPUSD", "USDCHF", "USDCAD", "AUDUSD", "NZDUSD", "EURGBP", "EURJPY", "USDMXN", "USDCNH"];36const CRYPTO = ["BTC", "ETH", "SOL", "XRP", "ADA", "DOGE", "LTC", "LINK", "AVAX", "DOT"];3738export const commodityHint = (root: string, name: string, exchangeId: string): InstrumentHint => ({39  assetClass: ["ES", "NQ", "ZN", "6E", "BTC"].includes(root) ? "FUTURE" : "COMMODITY",40  name,41  exchangeId,42  mic: "XCME",43  currency: "USD",44  country: "US",45  securityType: "CONTINUOUS_FUTURE",46  metadata: { featured: ["GC", "CL", "SI", "NG", "ES"].includes(root), roll: "volume", adjust: "none" },47});4849const TARGETS: Target[] = [50  ...US_EQUITIES.map(([s, n, v]) => ({ path: `/bars/stock/${encodeURIComponent(s)}?timeframe=1day&limit=400&order=desc`, symbol: s, hint: equityHint(s, n, v, "EQUITY"), kind: "bars" as const, sessionCloseLocal: "16:00", timezone: ET })),51  ...US_ETFS.map(([s, n, v]) => ({ path: `/bars/etf/${s}?timeframe=1day&limit=400&order=desc`, symbol: s, hint: equityHint(s, n, v, "ETF"), kind: "bars" as const, sessionCloseLocal: "16:00", timezone: ET })),52  ...US_INDICES.filter((i) => i.hfmd).map((i) => ({ path: `/bars/index/${i.hfmd}?timeframe=1day&limit=400&order=desc`, symbol: i.cboe, hint: indexHint(i), kind: "bars" as const, sessionCloseLocal: "16:00", timezone: ET })),53  ...FX_PAIRS.map((p) => ({ path: `/bars/fx/${p}?timeframe=1day&limit=400&order=desc`, symbol: p, hint: fxHint(p.slice(0, 3), p.slice(3)), kind: "bars" as const, sessionCloseLocal: "17:00", timezone: ET })),54  ...CRYPTO.map((c) => ({ path: `/bars/crypto/${c}?timeframe=1day&limit=400&order=desc`, symbol: `${c}-USD`, hint: cryptoHint(c, "USD", "coinbase"), kind: "bars" as const, sessionCloseLocal: "23:59:59", timezone: "UTC" })),55  ...FUTURES.map(([root, name, ex]) => ({ path: `/futures/${root}/continuous?timeframe=1day&limit=1000&roll=volume&adjust=none`, symbol: `${root}=F`, hint: commodityHint(root, name, ex), kind: "continuous" as const, sessionCloseLocal: "17:00", timezone: "America/Chicago" })),56];5758const seeds: ProposedInstrument[] = FUTURES.map(([root, name, ex]) => ({ symbol: `${root}=F`, hint: commodityHint(root, name, ex), aliases: [root, `${root}1!`] }));5960/**61 * HF Market Data (Market Atlas' sister platform, own FirstRate-derived lake): daily OHLCV history for62 * the curated universe + CME continuous futures (gold, oil, gas, grains, index/rates futures).63 * Provides chart history (bars, producer = this connector) and END_OF_DAY reference values.64 */65export const hfmarketdata = defineConnector({66  metadata: {67    id: "hfmarketdata-daily",68    name: "HF Market Data — daily bars & continuous futures",69    version: "1.0.0",70    sourceId: "hfmarketdata",71    organization: "spboucher.ai",72    sourceType: "OFFICIAL_API",73    jurisdiction: "CA",74    rightsStatus: "LICENSED",75    realtimeStatus: "END_OF_DAY",76    expectedLatencyMs: null,77    supportsStreaming: false,78    supportsHistorical: true,79    assetClasses: ["EQUITY", "ETF", "INDEX", "FOREX", "CRYPTO", "COMMODITY", "FUTURE"],80    exchanges: ["xcme"],81    homepage: "https://www.hfmarketdata.io",82    description: "Daily OHLCV bars (up to 400 sessions) for the curated US universe, major FX pairs, crypto and CME continuous futures from the HF Market Data lake (updated weekly from FirstRate). Backfills chart history and provides end-of-day reference closes.",83    rightsNotes: "Licensed sister platform (same operator). Redistribution of daily bars permitted under the HF Market Data data licence.",84    termsUrl: "https://www.hfmarketdata.io/data-license",85    sourceFamily: "firstrate",86    enabled: true,87  },88  seeds,89  defaultSymbols: TARGETS.map((t) => t.symbol),90  rateLimits: { "www.hfmarketdata.io": 1.5 },91  schedule: { intervalMs: 6 * 60 * 60_000 },92  async poll(ctx) {93    const key = ctx.secret("HFMD_API_KEY");94    const headers: Record<string, string> = key ? { authorization: `Bearer ${key}` } : {};95    const out: RawObservation[] = [];96    // ~120 requests every 6 h through the 1.5 req/s bucket (≈ 80 s per poll) — far below quotas.97    // Continuous futures return the oldest rows first → ask from ~400 sessions back.98    const from = new Date(ctx.now() - 560 * 86_400_000).toISOString().slice(0, 10);99    for (const t of TARGETS) {100      try {101        const url = t.kind === "continuous" ? `${BASE}${t.path}&from=${from}` : `${BASE}${t.path}`;102        const { data, response } = await ctx.http.getJson(url, { headers, timeoutMs: 45_000 });103        if (response.notModified) continue;104        out.push(raw("hfmarketdata-daily", "hfmarketdata", t.kind, data, { symbol: t.symbol }));105      } catch (err) {106        ctx.reportError(err, { symbol: t.symbol });107      }108    }109    return out;110  },111  normalize(r): NormalizedBatch {112    const symbol = r.meta?.symbol as string | undefined;113    const target = TARGETS.find((t) => t.symbol === symbol);114    if (!target) return { observations: [] };115    const p = r.payload as { data?: Array<Record<string, unknown>> };116    if (!Array.isArray(p.data)) return { observations: [] };117    const rows = p.data118      .map((d) => ({ date: String(d.datetime ?? "").slice(0, 10), open: num(d.open), high: num(d.high), low: num(d.low), close: num(d.close), volume: num(d.volume), contract: typeof d.symbol === "string" ? d.symbol : null }))119      .filter((d) => /^\d{4}-\d{2}-\d{2}$/.test(d.date) && d.open != null && d.high != null && d.low != null && d.close != null)120      .sort((a, b) => a.date.localeCompare(b.date));121    const bars: ProposedBar[] = rows.map((d) => ({ symbol: target.symbol, instrumentHint: target.hint, resolution: "1d", ts: Date.parse(`${d.date}T00:00:00Z`), open: d.open!, high: d.high!, low: d.low!, close: d.close!, volume: d.volume }));122    const last = rows[rows.length - 1];123    const prev = rows[rows.length - 2];124    const observations: NormalizedObservation[] = [];125    if (last) {126      const ts = zonedTimeToUtc(`${last.date}T${target.sessionCloseLocal.length === 5 ? `${target.sessionCloseLocal}:00` : target.sessionCloseLocal}`, target.timezone);127      const base = { symbol: target.symbol, instrumentHint: target.hint, currency: target.hint.currency ?? "USD", observationType: "EOD_CLOSE" as const, sourceTimestamp: ts, timestampTrust: "SOURCE" as const, rightsStatus: "LICENSED" as const, realtimeStatus: "END_OF_DAY" as const, meta: { session: last.date, contract: last.contract } };128      observations.push({ ...base, field: "LAST_PRICE", value: last.close! }, { ...base, field: "CLOSE", value: last.close! }, { ...base, field: "OPEN", value: last.open! }, { ...base, field: "HIGH", value: last.high! }, { ...base, field: "LOW", value: last.low! });129      if (last.volume != null) observations.push({ ...base, field: "VOLUME", value: last.volume });130      if (prev) observations.push({ ...base, field: "PREVIOUS_CLOSE", value: prev.close!, meta: { session: prev.date } });131    }132    return { observations, bars, stats: { bars: bars.length } };133  },134  async healthCheck(ctx) {135    const { data } = await ctx.http.getJson<{ datasets?: unknown }>(`${BASE}/status`);136    return { ok: !!data?.datasets };137  },138  fixturesDir: "fixtures",139});140141const num = (v: unknown): number | null => {142  const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;143  return Number.isFinite(n) ? n : null;144};145