spb/market-atlas

1import { defineConnector, parseFeed, raw, stripHtml, type NormalizedBatch } from "@market-atlas/connector-sdk";2import { parseTimestamp } from "@market-atlas/market-model";34const FEED = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcurrent&type=&company=&dateb=&owner=include&start=0&count=100&output=atom";5/** Forms that produce a public FILING_PUBLISHED event; the rest is stored but not surfaced as an event. */6export const MATERIAL_FORMS = new Set(["8-K", "8-K/A", "10-K", "10-K/A", "10-Q", "10-Q/A", "6-K", "20-F", "40-F", "S-1", "S-1/A", "F-1", "424B4", "SC 13D", "SC 13D/A", "SC 13G", "SC 13G/A", "DEF 14A", "DEFM14A", "425", "SC TO-T", "S-4", "NT 10-K", "NT 10-Q"]);78/**9 * SEC EDGAR "latest filings" Atom feed (public, no key; SEC asks for an identifying User-Agent10 * and ≤ 10 req/s — Market Atlas polls once every 2 minutes). Each entry → one Filing + a11 * FILING_PUBLISHED event for material forms.12 */13export const secEdgarFilings = defineConnector({14  metadata: {15    id: "sec-edgar-filings",16    name: "SEC EDGAR — latest filings feed",17    version: "1.0.0",18    sourceId: "sec-edgar",19    organization: "U.S. Securities and Exchange Commission",20    sourceType: "RSS",21    jurisdiction: "US",22    rightsStatus: "OFFICIAL_OPEN_DATA",23    realtimeStatus: "REALTIME",24    expectedLatencyMs: 120_000,25    supportsStreaming: false,26    supportsHistorical: false,27    assetClasses: ["EQUITY"],28    exchanges: [],29    homepage: "https://www.sec.gov/cgi-bin/browse-edgar?action=getcurrent",30    description: "Real-time list of documents filed on EDGAR (all forms). Each filing is stored with accession number, form type, filer name and CIK, and linked to instruments through the SEC company/ticker directory. Material forms (8-K, 10-K/Q, 6-K, S-1, 13D/G…) become FILING_PUBLISHED events.",31    rightsNotes: "U.S. government data (public domain). SEC fair-access policy: identifying User-Agent, max 10 requests/second.",32    termsUrl: "https://www.sec.gov/os/accessing-edgar-data",33    sourceFamily: "sec",34    enabled: true,35  },36  rateLimits: { "www.sec.gov": 2 },37  schedule: { intervalMs: 2 * 60_000 },38  async poll(ctx) {39    const res = await ctx.http.getText(FEED, { timeoutMs: 30_000, headers: { accept: "application/atom+xml, application/xml" } });40    if (res.notModified) return [];41    return [raw("sec-edgar-filings", "sec-edgar", "atom", res.text, { status: res.status })];42  },43  normalize(r): NormalizedBatch {44    if (r.kind !== "atom" || typeof r.payload !== "string") return { observations: [] };45    const feed = parseFeed(r.payload);46    const filings: NonNullable<NormalizedBatch["filings"]> = [];47    for (const item of feed.items) {48      // Title: "4 - Ault & Company, Inc. (0001734770) (Reporting)"49      const m = item.title.match(/^(.+?)\s+-\s+(.+?)\s+\((\d{10})\)\s*(?:\((\w+)\))?$/);50      const formType = (item.categories.find((c) => c.label === "form type")?.term ?? m?.[1] ?? "").trim();51      const companyName = (m?.[2] ?? item.title).trim();52      const cik = m?.[3] ? m[3].replace(/^0+/, "") : null;53      const role = m?.[4] ?? null;54      const accn = item.id?.match(/accession-number=([\d-]+)/)?.[1] ?? item.link?.match(/(\d{10}-\d{2}-\d{6})/)?.[1] ?? null;55      if (!accn || !formType) continue;56      const summary = item.summary ? stripHtml(item.summary) : "";57      const filedDate = summary.match(/Filed:\s*(\d{4}-\d{2}-\d{2})/)?.[1] ?? null;58      const filedAt = parseTimestamp(item.updated ?? item.published) ?? (filedDate ? Date.parse(filedDate) : null);59      if (!filedAt) continue;60      filings.push({61        id: `${accn}:${cik ?? "x"}`,62        cik,63        companyName,64        formType,65        filedAt,66        url: item.link ?? `https://www.sec.gov/Archives/edgar/data/${cik ?? ""}/${accn.replace(/-/g, "")}/${accn}-index.htm`,67        metadata: { accession_number: accn, role, filed_date: filedDate, size: summary.match(/Size:\s*([\d.]+\s*\w+)/)?.[1] ?? null, material: MATERIAL_FORMS.has(formType) && role !== "Reporting" },68      });69    }70    return { observations: [], filings, stats: { entries: feed.items.length } };71  },72  fixturesDir: "fixtures",73});74