spb/market-atlas

1import { defineConnector, parseDelimited, raw, type NormalizedBatch, type ProposedInstrument } from "@market-atlas/connector-sdk";23const NASDAQ_LISTED = "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt";4const OTHER_LISTED = "https://www.nasdaqtrader.com/dynamic/SymDir/otherlisted.txt";5/** otherlisted.txt "Exchange" column codes. */6const OTHER_EXCHANGES: Record<string, { exchangeId: string; mic: string }> = {7  N: { exchangeId: "xnys", mic: "XNYS" },8  A: { exchangeId: "xase", mic: "XASE" },9  P: { exchangeId: "arcx", mic: "ARCX" },10  Z: { exchangeId: "bats", mic: "BATS" },11  V: { exchangeId: "iexg", mic: "IEXG" },12};1314/**15 * Nasdaq Trader symbol directories (nasdaqlisted.txt + otherlisted.txt): the complete list of16 * US-listed securities with ETF flag and listing venue. Populates the instrument master daily.17 */18export const nasdaqSymbolDirectory = defineConnector({19  metadata: {20    id: "nasdaq-symbol-directory",21    name: "Nasdaq Trader — US symbol directory",22    version: "1.0.0",23    sourceId: "nasdaq-trader",24    organization: "Nasdaq, Inc.",25    sourceType: "BULK_FILE",26    jurisdiction: "US",27    rightsStatus: "PUBLIC_ATTRIBUTED",28    realtimeStatus: "END_OF_DAY",29    expectedLatencyMs: null,30    supportsStreaming: false,31    supportsHistorical: false,32    assetClasses: ["EQUITY", "ETF"],33    exchanges: ["xnas", "xnys", "xase", "arcx", "bats"],34    homepage: "https://www.nasdaqtrader.com/trader.aspx?id=symboldirdefs",35    description: "Daily pipe-delimited directories of every Nasdaq-listed and other US exchange-listed security (~11,000 rows): symbol, security name, listing venue, ETF flag, test-issue flag. Seeds the US instrument master.",36    rightsNotes: "Public reference files published by Nasdaq Trader; displayed with attribution.",37    termsUrl: "https://www.nasdaqtrader.com/Trader.aspx?id=Terms",38    sourceFamily: "nasdaq",39    enabled: true,40  },41  rateLimits: { "www.nasdaqtrader.com": 1 },42  schedule: { intervalMs: 24 * 60 * 60_000 },43  async poll(ctx) {44    const out = [];45    for (const [kind, url] of [["nasdaqlisted", NASDAQ_LISTED], ["otherlisted", OTHER_LISTED]] as const) {46      const res = await ctx.http.getText(url, { timeoutMs: 60_000 });47      if (res.notModified) continue;48      out.push(raw("nasdaq-symbol-directory", "nasdaq-trader", kind, res.text));49    }50    return out;51  },52  normalize(r): NormalizedBatch {53    if (typeof r.payload !== "string") return { observations: [] };54    const rows = parseDelimited(r.payload, "|").filter((row) => !String(Object.values(row)[0] ?? "").startsWith("File Creation Time"));55    const instruments: ProposedInstrument[] = [];56    if (r.kind === "nasdaqlisted") {57      for (const row of rows) {58        if (row["Test Issue"] === "Y" || !row.Symbol) continue;59        const etf = row.ETF === "Y";60        instruments.push(proposal(row.Symbol!, row["Security Name"] ?? row.Symbol!, "xnas", "XNAS", etf, { market_category: row["Market Category"], financial_status: row["Financial Status"] }));61      }62    } else if (r.kind === "otherlisted") {63      for (const row of rows) {64        if (row["Test Issue"] === "Y" || !row["ACT Symbol"]) continue;65        const ex = OTHER_EXCHANGES[row.Exchange ?? ""];66        if (!ex) continue;67        const etf = row.ETF === "Y";68        instruments.push(proposal(row["ACT Symbol"]!, row["Security Name"] ?? row["ACT Symbol"]!, ex.exchangeId, ex.mic, etf, { cqs_symbol: row["CQS Symbol"], nasdaq_symbol: row["NASDAQ Symbol"] }, [row["CQS Symbol"], row["NASDAQ Symbol"]]));69      }70    }71    return { observations: [], instruments, stats: { rows: rows.length } };72  },73  fixturesDir: "fixtures",74});7576function proposal(symbol: string, rawName: string, exchangeId: string, mic: string, etf: boolean, metadata: Record<string, unknown>, extraAliases: Array<string | undefined> = []): ProposedInstrument {77  const name = cleanName(rawName);78  const sym = symbol.trim();79  const aliases = [...new Set([sym.replace(".", "-"), sym.replace(".", "/"), ...extraAliases.filter((a): a is string => !!a && a !== sym)])].filter((a) => a !== sym);80  const securityType = classify(rawName);81  return {82    symbol: sym,83    aliases,84    hint: { assetClass: etf ? "ETF" : securityType === "COMMON_STOCK" || securityType === "ADR" ? "EQUITY" : "OTHER", name, exchangeId, mic, currency: "USD", country: "US", securityType: etf ? "ETF" : securityType, companyName: !etf && securityType === "COMMON_STOCK" ? name : null, metadata },85  };86}8788export function cleanName(n: string): string {89  return n90    .trim()91    .replace(/\s+-\s+(Common Stock|Class [A-Z] Common Stock|Ordinary Shares|Common Shares|American Depositary Shares.*|Class [A-Z] Ordinary Shares?.*|Units?.*|Warrants?.*|Depositary Shares.*)$/i, "")92    .replace(/\s+Common Stock$/i, "")93    .replace(/\s{2,}/g, " ")94    .trim();95}9697export function classify(n: string): string {98  const s = n.toLowerCase();99  if (/warrant/.test(s)) return "WARRANT";100  if (/\bunits?\b/.test(s)) return "UNIT";101  if (/\bright(s)?\b/.test(s)) return "RIGHT";102  if (/american depositary|\badr\b|\bads\b/.test(s)) return "ADR";103  if (/preferred|preference|depositary share/.test(s)) return "PREFERRED";104  if (/\bnotes?\b|debenture|bond/.test(s)) return "DEBT";105  return "COMMON_STOCK";106}107