spb/datacenterindex

1import type { ConnectorConfig, ConnectorContext, DiscoveredUrl, ExtractedRecord, Parser, RawDocument } from "@dci/connectors";2import { GenericConnector } from "@dci/connectors";3import { cleanText } from "@dci/core";4import { EXTRACTOR_VERSION } from "./extract-project.js";5import { detectOperators } from "./operators-lexicon.js";6import { US_STATES } from "./locations-lexicon.js";78/**9 * SEC EDGAR full-text search (efts.sec.gov/LATEST/search-index) — public JSON API, no key. The API ignores10 * `dateRange=30d`, so the connector computes `dateRange=custom&startdt=…&enddt=…` at discovery time11 * (`implementation: news_edgar_fts`, params.queries / params.lookbackDays / params.forms). One `news_event`12 * per filing (exhibit preferred), no fetch of the filing itself — links point to the SEC archive.13 * SEC fair-access policy: identify the crawler with a contact address (DEFAULT_UA does), ≤ 10 req/s.14 */15interface EdgarHit {16  _id: string; // "0001079973-26-000696:ex99x2.htm"17  _source: { ciks?: string[]; display_names?: string[]; file_date?: string; form?: string; root_forms?: string[]; file_type?: string; file_description?: string; adsh?: string; items?: string[]; biz_locations?: string[]; biz_states?: string[]; period_ending?: string; sics?: string[] };18}19interface EdgarResponse { hits?: { total?: { value?: number }; hits?: EdgarHit[] }; query?: unknown }2021const iso = (d: Date): string => d.toISOString().slice(0, 10);2223export class EdgarFtsConnector extends GenericConnector {24  async discover(_ctx: ConnectorContext): Promise<DiscoveredUrl[]> {25    const p = (this.cfg.params ?? {}) as { queries?: unknown; forms?: unknown; lookbackDays?: unknown };26    const queries = Array.isArray(p.queries) && p.queries.length ? p.queries.map(String) : ['"data center"'];27    const forms = typeof p.forms === "string" && p.forms ? p.forms : "8-K";28    const lookback = Number.isFinite(Number(p.lookbackDays)) && Number(p.lookbackDays) > 0 ? Number(p.lookbackDays) : 30;29    const end = new Date();30    const start = new Date(end.getTime() - lookback * 86_400_000);31    return queries.map((q, i) => ({32      url: `https://efts.sec.gov/LATEST/search-index?q=${encodeURIComponent(q)}&forms=${encodeURIComponent(forms)}&dateRange=custom&startdt=${iso(start)}&enddt=${iso(end)}`,33      group: "filings",34      pageType: "financial_disclosure",35      priority: 90 - i,36      discoveredFrom: null,37      meta: { query: q, forms, lookbackDays: lookback },38    }));39  }40}4142export function edgarFilingUrl(hit: EdgarHit): { url: string; adsh: string; file: string } | null {43  const [adshRaw, file] = hit._id.split(":");44  const adsh = hit._source.adsh ?? adshRaw;45  const cik = hit._source.ciks?.[0];46  if (!adsh || !cik) return null;47  const folder = adsh.replace(/-/g, "");48  const url = file ? `https://www.sec.gov/Archives/edgar/data/${Number(cik)}/${folder}/${file}` : `https://www.sec.gov/Archives/edgar/data/${Number(cik)}/${folder}/`;49  return { url, adsh, file: file ?? "" };50}5152function companyName(display: string | undefined): string | null {53  if (!display) return null;54  return cleanText(display.replace(/\s*\([A-Z0-9.]{1,10}\)\s*\(CIK \d+\)\s*$/, "").replace(/\s*\(CIK \d+\)\s*$/, ""));55}5657/** Rank exhibits above the cover 8-K so one filing yields the press-release exhibit when it exists. */58function rank(hit: EdgarHit): number {59  const ft = (hit._source.file_type ?? "").toUpperCase();60  if (/^EX-99/.test(ft)) return 3;61  if (/^EX-/.test(ft)) return 2;62  return 1;63}6465export const edgarFtsParser: Parser = {66  name: "news_edgar_fts_v1",67  version: EXTRACTOR_VERSION,68  parse(doc: RawDocument, ctx: ConnectorContext): ExtractedRecord[] {69    let res: EdgarResponse;70    try { res = JSON.parse(doc.text) as EdgarResponse; } catch { ctx.log("warn", `news_edgar_fts_v1: response is not JSON (${doc.finalUrl})`); return []; }71    const hits = res.hits?.hits ?? [];72    const query = typeof doc.meta?.query === "string" ? doc.meta.query : new URL(doc.finalUrl).searchParams.get("q") ?? "";73    const byFiling = new Map<string, EdgarHit>();74    for (const h of hits) {75      const adsh = h._source.adsh ?? h._id.split(":")[0]!;76      const prev = byFiling.get(adsh);77      if (!prev || rank(h) > rank(prev)) byFiling.set(adsh, h);78    }79    const out: ExtractedRecord[] = [];80    for (const h of byFiling.values()) {81      const link = edgarFilingUrl(h);82      if (!link) continue;83      const s = h._source;84      const company = companyName(s.display_names?.[0]);85      if (!company) continue;86      const form = s.root_forms?.[0] ?? s.form ?? "8-K";87      const fileType = s.file_type && s.file_type.toUpperCase() !== form.toUpperCase() ? s.file_type : null;88      const title = `${company}: Form ${form}${fileType ? ` (${fileType}${s.file_description && !/^EXHIBIT/i.test(s.file_description) ? ` — ${s.file_description}` : ""})` : ""}`;89      const loc = s.biz_locations?.[0] ?? null;90      const state = s.biz_states?.[0] ?? null;91      const summary = cleanText(`${form} filed ${s.file_date ?? "(date unknown)"} by ${(s.display_names ?? [company]).map((d) => companyName(d)).filter(Boolean).join(", ")}${s.items?.length ? `; items ${s.items.join(", ")}` : ""}${loc ? `; registrant located in ${loc}` : ""}. Matched EDGAR full-text search ${query}.`);92      const operators = detectOperators(company).map((o) => o.name);93      const isUs = state ? Object.prototype.hasOwnProperty.call(US_STATES, state.toUpperCase()) : false;94      const city = loc ? cleanText(loc.split(",")[0]) : null;95      out.push({96        kind: "news_event",97        key: `${ctx.connectorId}:news:${link.adsh}`,98        url: link.url,99        pageType: "financial_disclosure",100        certainty: 0.7,101        data: {102          title, url: link.url, publishedAt: s.file_date ?? null, summary, pageType: "financial_disclosure", eventType: "news",103          mw: [], status: null, money: null, text: null,104          operators, countries: isUs ? ["US"] : [], cities: city && isUs ? [city] : [],105          filing: { adsh: link.adsh, form, fileType: s.file_type ?? null, items: s.items ?? [], cik: s.ciks?.[0] ?? null, periodEnding: s.period_ending ?? null, query },106          relevance: "strong", extractor: EXTRACTOR_VERSION,107        },108        methods: { title: "edgar:display_names", publishedAt: "edgar:file_date", summary: "edgar:metadata", operators: "lexicon:operator" },109      });110    }111    ctx.log("info", `news_edgar_fts_v1: ${hits.length} hits → ${out.length} filings (${query})`);112    return out;113  },114};115116export function edgarImplementation(cfg: ConnectorConfig): EdgarFtsConnector { return new EdgarFtsConnector(cfg); }117