spb/datacenterindex
Public
HTML 53.9%
TypeScript 44.5%
JavaScript 0.6%
SQL 0.5%
1/**2 * Golden tests: archived production bodies (apps/worker/fixtures/<connectorId>/<name>.{html,json,xml,pdf}) run through3 * extract → normalize → validate with the real YAML config and the real registered parsers — no network.4 * Expectations live next to each body in `<name>.expected.json` (schema: docs/CONNECTORS.md § Fixtures). Adding a5 * fixture = adding two files; nothing here needs editing.6 *7 * `_todo` blocks inside an expected entity (or the announcement) hold the CORRECT value the parser does not produce8 * yet: they run as `it.fails`, so fixing the parser flips them red until the field is moved out of `_todo`.9 */10import { existsSync, readdirSync, readFileSync } from "node:fs";11import { dirname, join, resolve } from "node:path";12import { fileURLToPath } from "node:url";13import { describe, expect, it } from "vitest";14import type { NormalizedEntity } from "@dci/core";15import { GenericConnector, THIRD_PARTY_SUMMARY_MAX, createTestContext, getImplementation, parseConnectorConfig, validateEntities, type Connector, type ExtractedRecord, type RawDocument } from "@dci/connectors";16import { register as registerOperators1 } from "./operators1/index.js";17import { register as registerOperators2 } from "./operators2/index.js";18import { register as registerCloud } from "./cloud/index.js";19import { register as registerDatasets } from "./datasets/index.js";20import { register as registerNews } from "./news/index.js";21import { articleContent } from "./news/article-parser.js";22import { extractAnnouncement, type Announcement } from "./news/extract-project.js";2324const HERE = dirname(fileURLToPath(import.meta.url));25const FIXTURES_DIR = resolve(HERE, "../../fixtures");26const CONFIG_DIR = resolve(HERE, "../../../../config/connectors");27const BODY_EXT = [".html", ".htm", ".json", ".xml", ".pdf", ".txt", ".md"];2829type Todo = Record<string, unknown> & { _reason?: string };30interface ExpectedEntity { key: string; _todo?: Todo; [field: string]: unknown }31interface ExpectedAnnouncement { _todo?: Todo; [field: string]: unknown }32interface Expected {33 url: string; documentId: string; fetchedAt: string; contentType: string; note?: string;34 /** asserted count per entityType (absent key = not asserted) */35 counts?: Partial<Record<NormalizedEntity["entityType"], number>>;36 facilities?: ExpectedEntity[]; projects?: ExpectedEntity[]; newsEvents?: ExpectedEntity[]; cloudRegions?: ExpectedEntity[]; operators?: ExpectedEntity[];37 /** kind = news publisher: summaries ≤ THIRD_PARTY_SUMMARY_MAX and no article text in the records */38 thirdParty?: boolean;39 /** `extractAnnouncement(title, text)` facts (news / government fixtures) */40 announcement?: ExpectedAnnouncement;41}42interface Fixture { connectorId: string; name: string; bodyPath: string; expected: Expected }4344let registered = false;45function registerParsers(): void {46 if (registered) return;47 registered = true;48 for (const reg of [registerOperators1, registerOperators2, registerCloud, registerDatasets, registerNews]) {49 try { reg(); } catch (e) { if (!/already registered/i.test((e as Error).message)) throw e; }50 }51}5253function listFixtures(): Fixture[] {54 if (!existsSync(FIXTURES_DIR)) return [];55 const out: Fixture[] = [];56 for (const dir of readdirSync(FIXTURES_DIR, { withFileTypes: true })) {57 if (!dir.isDirectory()) continue;58 const files = readdirSync(join(FIXTURES_DIR, dir.name));59 for (const f of files.filter((x) => x.endsWith(".expected.json")).sort()) {60 const name = f.slice(0, -".expected.json".length);61 const body = files.find((x) => x !== f && x.startsWith(`${name}.`) && BODY_EXT.some((ext) => x.endsWith(ext)));62 if (!body) throw new Error(`fixture ${dir.name}/${name}: expected.json without a body file (${BODY_EXT.join("|")})`);63 out.push({ connectorId: dir.name, name, bodyPath: join(FIXTURES_DIR, dir.name, body), expected: JSON.parse(readFileSync(join(FIXTURES_DIR, dir.name, f), "utf8")) as Expected });64 }65 }66 return out;67}6869function loadConnector(connectorId: string): Connector {70 const file = join(CONFIG_DIR, `${connectorId}.yaml`);71 if (!existsSync(file)) throw new Error(`no config/connectors/${connectorId}.yaml for fixture dir ${connectorId}`);72 const cfg = parseConnectorConfig(readFileSync(file, "utf8"), file);73 const impl = cfg.implementation ? getImplementation(cfg.implementation) : undefined;74 if (cfg.implementation && !impl) throw new Error(`implementation "${cfg.implementation}" not registered`);75 return impl ? impl(cfg) : new GenericConnector(cfg);76}7778function rawDocument(fx: Fixture): RawDocument {79 const body = readFileSync(fx.bodyPath);80 const { url, contentType, fetchedAt } = fx.expected;81 return { url, finalUrl: url, fetchedAt, status: 200, contentType, body, text: /pdf/i.test(contentType) ? "" : body.toString("utf8"), headers: {}, etag: null, lastModified: null, notModified: false, fetcher: "cache", level: 1, durationMs: 0, credits: 0 };82}8384/** "mentions.countriesIso2" → value; missing path → undefined */85function get(obj: unknown, path: string): unknown {86 return path.split(".").reduce<unknown>((o, k) => (o != null && typeof o === "object" ? (o as Record<string, unknown>)[k] : undefined), obj);87}8889/** `null` in expected.json means "absent or null" (normalized entities use null for unknown fields). */90function normalize(v: unknown): unknown { return v === undefined ? null : v; }9192interface Run { records: ExtractedRecord[]; entities: NormalizedEntity[]; errors: Array<{ key: string; field: string | undefined; message: string }>; warnings: number; announcement: Announcement | null; published: string | null }93async function runFixture(fx: Fixture): Promise<Run> {94 registerParsers();95 const connector = loadConnector(fx.connectorId);96 const cfg = (connector as GenericConnector).cfg ?? parseConnectorConfig(readFileSync(join(CONFIG_DIR, `${fx.connectorId}.yaml`), "utf8"));97 const ctx = createTestContext(cfg);98 ctx.log = () => {}; // keep vitest output clean; the connector never fetches during extract/normalize99 const doc = rawDocument(fx);100 const records = await connector.extract(ctx, doc);101 const entities = await connector.normalize(ctx, records);102 const report = validateEntities(entities);103 const errors = report.issues.filter((i) => i.level === "error").map(({ key, field, message }) => ({ key, field, message }));104 let announcement: Announcement | null = null, published: string | null = null;105 if (fx.expected.announcement) {106 const content = await articleContent(rawDocument(fx));107 published = content.published;108 announcement = extractAnnouncement(content.title, content.text, { publishedAt: content.published });109 }110 return { records, entities, errors, warnings: report.issues.length - errors.length, announcement, published };111}112113/** Flattened view of an Announcement for expected.json (`announcement.<key>`). */114function announcementFacts(a: Announcement, published: string | null): Record<string, unknown> {115 return {116 title: a.title, published, class: a.classification.class, mayCreateProject: a.classification.mayCreateProject, physical: a.classification.physical,117 headlineMw: a.headlineMw, mwAll: a.mwAll, status: a.status, operator: a.operator?.name ?? null, operators: a.operators.map((o) => o.name),118 country: a.location?.country ?? null, city: a.location?.city ?? null, region: a.location?.region ?? null,119 money: a.money, investmentUsd: a.investmentUsd, acreage: a.acreage, phaseCount: a.phaseCount, expectedOpening: a.expectedOpening, projectName: a.projectName, explicitName: a.explicitName,120 relevance: a.relevance, leadRelevance: a.leadRelevance, titleSignal: a.titleSignal, nonProjectTitle: a.nonProjectTitle, pageType: a.pageType, eventType: a.eventType ?? null,121 };122}123124const ENTITY_LISTS: Array<[keyof Expected, NormalizedEntity["entityType"]]> = [["facilities", "facility"], ["projects", "project"], ["newsEvents", "news_event"], ["cloudRegions", "cloud_region"], ["operators", "operator"]];125126const fixtures = listFixtures();127128describe("connector fixtures (golden)", () => {129 it("every fixture directory has a connector config and complete metadata", () => {130 expect(fixtures.length).toBeGreaterThan(0);131 for (const fx of fixtures) {132 expect(existsSync(join(CONFIG_DIR, `${fx.connectorId}.yaml`)), `${fx.connectorId}: config/connectors/${fx.connectorId}.yaml`).toBe(true);133 for (const k of ["url", "documentId", "fetchedAt", "contentType"] as const) expect(typeof fx.expected[k], `${fx.connectorId}/${fx.name}: ${k}`).toBe("string");134 expect(fx.expected.url, `${fx.connectorId}/${fx.name}: url`).toMatch(/^https?:\/\//);135 expect(fx.expected.documentId, `${fx.connectorId}/${fx.name}: documentId`).toMatch(/^doc_[0-9a-f]{16}$/);136 expect(Number.isFinite(Date.parse(fx.expected.fetchedAt)), `${fx.connectorId}/${fx.name}: fetchedAt`).toBe(true);137 }138 });139140 for (const fx of fixtures) {141 const label = `${fx.connectorId}/${fx.name}`;142 const exp = fx.expected;143 let cached: Promise<Run> | null = null;144 const run = () => (cached ??= runFixture(fx));145146 describe(label, () => {147 it("extracts, normalizes and validates as expected.json says", async () => {148 const r = await run();149 expect(r.errors, `validation errors for ${label}`).toEqual([]);150 for (const [type, n] of Object.entries(exp.counts ?? {})) expect(r.entities.filter((e) => e.entityType === type).length, `${type} count`).toBe(n);151 for (const [listKey, entityType] of ENTITY_LISTS) {152 for (const want of (exp[listKey] as ExpectedEntity[] | undefined) ?? []) {153 const got = r.entities.find((e) => e.entityType === entityType && e.key === want.key);154 expect(got, `${entityType} ${want.key} present (have: ${r.entities.filter((e) => e.entityType === entityType).map((e) => e.key).join(", ")})`).toBeDefined();155 for (const [field, value] of Object.entries(want)) {156 if (field === "key" || field === "_todo") continue;157 expect(normalize(get(got, field)), `${want.key}.${field}`).toEqual(normalize(value));158 }159 }160 }161 if (exp.thirdParty) {162 for (const e of r.entities) if (e.entityType === "news_event") expect((e.summary ?? "").length, `${e.key} summary ≤ ${THIRD_PARTY_SUMMARY_MAX}`).toBeLessThanOrEqual(THIRD_PARTY_SUMMARY_MAX);163 for (const rec of r.records) if (rec.kind === "news_event") expect(rec.data.text ?? null, `${rec.key} must not store third-party article text`).toBeNull();164 }165 if (exp.announcement) {166 expect(r.announcement).not.toBeNull();167 const facts = announcementFacts(r.announcement!, r.published);168 for (const [field, value] of Object.entries(exp.announcement)) {169 if (field === "_todo") continue;170 expect(field in facts, `announcement.${field} is not a known fact (${Object.keys(facts).join(", ")})`).toBe(true);171 expect(normalize(facts[field]), `announcement.${field}`).toEqual(normalize(value));172 }173 }174 });175176 // Known parser gaps: the CORRECT value is asserted and the test is expected to fail until the parser is fixed.177 for (const [listKey, entityType] of ENTITY_LISTS) {178 for (const want of (exp[listKey] as ExpectedEntity[] | undefined) ?? []) {179 for (const [field, value] of Object.entries(want._todo ?? {})) {180 if (field === "_reason") continue;181 it.fails(`known gap — ${want.key}.${field} should be ${JSON.stringify(value)} (${want._todo?._reason ?? "no reason given"})`, async () => {182 const r = await run();183 const got = r.entities.find((e) => e.entityType === entityType && e.key === want.key);184 expect(normalize(get(got, field))).toEqual(normalize(value));185 });186 }187 }188 }189 for (const [field, value] of Object.entries(exp.announcement?._todo ?? {})) {190 if (field === "_reason") continue;191 it.fails(`known gap — announcement.${field} should be ${JSON.stringify(value)} (${exp.announcement?._todo?._reason ?? "no reason given"})`, async () => {192 const r = await run();193 expect(normalize(announcementFacts(r.announcement!, r.published)[field])).toEqual(normalize(value));194 });195 }196 });197 }198});199