TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { NormalizeError, type WebSensorConnector } from "./types";45/**6 * SEC EDGAR connector — company submissions API (`https://data.sec.gov/submissions/CIK##########.json`).7 * The API is columnar (`filings.recent.form[]`, `filings.recent.filingDate[]`…), which the generic8 * `jsonlist` connector cannot read; this connector rebuilds one item per filing keyed by accession9 * number, so a new 8-K / 10-Q / S-1 / 13D becomes a `filing` list event with a direct document link.10 *11 * Rules of the road (sec.gov/developer): a descriptive User-Agent with contact e-mail is mandatory,12 * ≤ 10 requests/second overall. The engine's per-host concurrency (2) plus tier B/C intervals keep us13 * far below that.14 *15 * Config: { forms?: ["8-K", "10-K", "10-Q", "S-1", "6-K", "SC 13D", "DEF 14A"], excludeForms?: ["4", "144", "3", "5"], maxItems?: 60 }16 * Default: all forms except insider forms 3/4/5 and 144 notices (noise).17 */18const DEFAULT_EXCLUDE = new Set(["3", "4", "5", "3/A", "4/A", "5/A", "144", "144/A"]);19const UA = process.env.WS_EDGAR_USER_AGENT ?? "WebSensor (contact@websensor.io)";2021const FORM_LABEL: Record<string, string> = {22 "8-K": "Current report",23 "10-K": "Annual report",24 "10-Q": "Quarterly report",25 "20-F": "Annual report (foreign issuer)",26 "40-F": "Annual report (Canadian issuer)",27 "6-K": "Report of foreign issuer",28 "S-1": "Registration statement (IPO)",29 "S-3": "Shelf registration",30 "S-4": "Merger registration",31 "S-8": "Employee plan registration",32 "424B4": "Prospectus (final)",33 "424B5": "Prospectus supplement",34 "DEF 14A": "Proxy statement",35 "DEFA14A": "Additional proxy materials",36 "SC 13D": "Beneficial ownership >5% (active)",37 "SC 13G": "Beneficial ownership >5% (passive)",38 "13F-HR": "Institutional holdings",39 "SD": "Specialized disclosure",40 "11-K": "Employee plan annual report",41 "NT 10-K": "Late filing notice (10-K)",42 "NT 10-Q": "Late filing notice (10-Q)",43 "25-NSE": "Delisting notice",44 "8-K/A": "Current report (amended)",45 "10-K/A": "Annual report (amended)",46 "10-Q/A": "Quarterly report (amended)",47};4849/** 8-K item numbers → human labels (Regulation S-K). */50const ITEM_LABEL: Record<string, string> = {51 "1.01": "Entry into a material agreement",52 "1.02": "Termination of a material agreement",53 "1.03": "Bankruptcy or receivership",54 "1.05": "Material cybersecurity incident",55 "2.01": "Completion of acquisition or disposition",56 "2.02": "Results of operations and financial condition",57 "2.03": "Creation of a direct financial obligation",58 "2.04": "Triggering events (acceleration of obligation)",59 "2.05": "Costs associated with exit or disposal activities",60 "2.06": "Material impairments",61 "3.01": "Delisting notice or failure to satisfy listing rule",62 "3.02": "Unregistered sales of equity securities",63 "3.03": "Material modification to rights of security holders",64 "4.01": "Changes in registrant's certifying accountant",65 "4.02": "Non-reliance on previously issued financial statements (restatement)",66 "5.01": "Changes in control of registrant",67 "5.02": "Departure/appointment of directors or officers; compensation",68 "5.03": "Amendments to articles/bylaws; change in fiscal year",69 "5.07": "Submission of matters to a vote of security holders",70 "7.01": "Regulation FD disclosure",71 "8.01": "Other events",72 "9.01": "Financial statements and exhibits",73};7475export class EdgarConnector implements WebSensorConnector {76 mode = "list" as const;77 metadata(): ConnectorMetadata {78 return { key: "edgar", name: "SEC EDGAR filings", sensorTypes: ["REST_API", "JSON"], description: "Company submissions (data.sec.gov) → filings list keyed by accession number, 8-K items decoded", version: "1.0.0" };79 }80 async fetch(endpoint: SensorEndpoint): Promise<Observation> {81 const obs = await httpFetchWithRetry(endpoint.id, endpoint.url, { etag: endpoint.etag, lastModified: endpoint.lastModified, accept: "application/json", headers: { "user-agent": UA, host: "data.sec.gov" }, userAgent: UA, timeoutMs: 40_000, maxBytes: 30 * 1024 * 1024 });82 return obs;83 }84 async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {85 if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");86 const text = obs.body.toString("utf8");87 let j: Record<string, unknown>;88 try {89 j = JSON.parse(text) as Record<string, unknown>;90 } catch {91 throw new NormalizeError("bad_json", "EDGAR response is not JSON");92 }93 const recent = ((j.filings as Record<string, unknown> | undefined)?.recent ?? {}) as Record<string, unknown[]>;94 const forms = (recent.form ?? []) as string[];95 if (!Array.isArray(forms)) throw new NormalizeError("bad_shape", "filings.recent.form missing");96 const cfg = endpoint.config as { forms?: string[]; excludeForms?: string[]; maxItems?: number };97 const include = cfg.forms ? new Set(cfg.forms.map((f) => f.toUpperCase())) : null;98 const exclude = new Set((cfg.excludeForms ?? [...DEFAULT_EXCLUDE]).map((f) => f.toUpperCase()));99 const cik = String(j.cik ?? "").padStart(10, "0");100 const cikNum = String(Number(cik));101 const company = String(j.name ?? "");102 const tickers = (j.tickers ?? []) as string[];103 const col = (k: string, i: number): string => String((recent[k] as unknown[] | undefined)?.[i] ?? "");104 const items: { key: string; [k: string]: unknown }[] = [];105 for (let i = 0; i < forms.length && items.length < (cfg.maxItems ?? 60); i++) {106 const form = forms[i]!.toUpperCase();107 if (exclude.has(form)) continue;108 if (include && !include.has(form)) continue;109 const acc = col("accessionNumber", i);110 const accPath = acc.replace(/-/g, "");111 const doc = col("primaryDocument", i);112 const itemsRaw = col("items", i);113 const itemLabels = itemsRaw114 .split(",")115 .map((s) => s.trim())116 .filter(Boolean)117 .map((n) => `${n} ${ITEM_LABEL[n] ?? ""}`.trim());118 const filingDate = col("filingDate", i);119 const accepted = col("acceptanceDateTime", i);120 const desc = col("primaryDocDescription", i);121 const label = FORM_LABEL[form] ?? form;122 const title = `${company}: ${form} — ${label}${itemLabels.length ? ` (${itemLabels.map((l) => l.split(" ")[0]).join(", ")})` : ""}`;123 const summary = [desc && desc !== form ? desc : "", itemLabels.length ? `Items: ${itemLabels.join("; ")}` : "", col("reportDate", i) ? `Period: ${col("reportDate", i)}` : ""].filter(Boolean).join(". ");124 items.push({125 key: acc,126 form,127 formLabel: label,128 items: itemsRaw,129 title,130 summary: summary.slice(0, 800),131 url: doc ? `https://www.sec.gov/Archives/edgar/data/${cikNum}/${accPath}/${doc}` : `https://www.sec.gov/Archives/edgar/data/${cikNum}/${accPath}/`,132 indexUrl: `https://www.sec.gov/Archives/edgar/data/${cikNum}/${accPath}/${acc}-index.htm`,133 filingDate,134 publishedAt: accepted && !Number.isNaN(new Date(accepted).getTime()) ? new Date(accepted).toISOString() : filingDate ? new Date(filingDate + "T12:00:00Z").toISOString() : null,135 size: Number(col("size", i)) || 0,136 xbrl: col("isXBRL", i) === "1",137 });138 }139 const canonical = items.map((i) => `${i.key}\t${String(i.form)}\t${String(i.items)}`).join("\n");140 const newest = items.map((i) => i.publishedAt as string | null).filter((x): x is string => Boolean(x)).sort().at(-1);141 return {142 mode: "list",143 items,144 compareFields: ["form", "items"],145 title: `${company} (${tickers.join(", ") || "CIK " + cikNum}) — EDGAR filings`,146 rawHash: sha256(text),147 canonicalHash: sha256(canonical),148 semanticHash: simhash(items.map((i) => String(i.title)).join("\n")),149 publishedAt: newest ? new Date(newest) : null,150 extractionConfidence: 1,151 extra: { cik: cikNum, company, tickers, exchanges: j.exchanges, sic: j.sic, sicDescription: j.sicDescription, fiscalYearEnd: j.fiscalYearEnd, stateOfIncorporation: j.stateOfIncorporation, totalRecent: forms.length, filings: items.length },152 };153 }154}155