spb/datacenterindex
Public
HTML 53.9%
TypeScript 44.5%
JavaScript 0.6%
SQL 0.5%
1/**2 * Ingest smoke test against the local databases (Postgres + ClickHouse). SYNTHETIC DATA ONLY — every record is3 * prefixed "Smoke Test", uses connector id "smoke-test" and is deleted at the end.4 *5 * set -a; source .env; set +a; node node_modules/tsx/dist/cli.mjs scripts/ingest-smoke.ts6 *7 * Run 1: 6 entities (3 facilities — one is a duplicate that must auto-merge —, 1 project, 1 news event, 1 cloud region).8 * Run 2: same entities with one changed MW → 0 created, exactly 1 event (capacity_changed).9 * Then computeRankings / computeDailyMetrics / refreshStats, cleanup, recompute so the database is left clean.10 */11import { closeDb, getDb, sql } from "@dci/db";12import { ensureClickHouse, getClickHouse } from "@dci/db/clickhouse";13import type { NormalizedCloudRegion, NormalizedEntity, NormalizedFacility, NormalizedNewsEvent, NormalizedProject, Provenance } from "@dci/core";14import { ingestEntities, type IngestRun } from "../apps/worker/src/ingest/index.js";15import { computeRankings, refreshStats } from "../apps/worker/src/rankings.js";16import { computeDailyMetrics } from "../apps/worker/src/metrics.js";17import { seedReferenceData } from "../apps/worker/src/seeds/index.js";1819const CONNECTOR = "smoke-test";20const SOURCE = "src_smoke-test";21const run: IngestRun = { connectorId: CONNECTOR, sourceId: SOURCE, runId: `run_smoke_${Date.now().toString(36)}`, sourceKind: "operator", sourcePriority: 2, dryRun: false };22const doc = { documentId: "doc_smoke_test", url: "https://smoke-test.invalid/facilities", pageType: "facility_index", fetchedAt: new Date().toISOString() };2324function prov(url: string, extra: Partial<Provenance> = {}): Provenance {25 const now = new Date().toISOString();26 return { sourceId: SOURCE, connectorId: CONNECTOR, url, firstObserved: now, lastObserved: now, retrievedAt: now, confidence: "high", method: "smoke-test", extractorVersion: "smoke", ...extra };27}2829function entities(mwA: number): NormalizedEntity[] {30 const facA: NormalizedFacility = {31 entityType: "facility",32 key: "smoke-test:fac:1",33 name: "Smoke Test DC12",34 operatorName: "Smoke Test Operator",35 address: "1 Smoke Test Court, Ashburn, VA 20147",36 city: "Ashburn",37 regionName: "Virginia",38 countryIso2: "US",39 geo: { lat: 39.0401, lng: -77.4899, precision: "exact", source: "smoke-test" },40 status: "operational",41 facilityType: "colocation",42 itCapacityMw: mwA,43 openedOn: "2016-05",44 website: "https://smoke-test.invalid/dc12",45 carriers: ["Smoke Test Carrier Networks"],46 ixps: ["SMOKE-IX"],47 externalIds: { smoke_test: "fac-1" },48 provenance: prov("https://smoke-test.invalid/dc12"),49 };50 // duplicate of A from a "second page": must auto-merge (same operator, code DC12, 30 m apart)51 const facB: NormalizedFacility = {52 entityType: "facility",53 key: "smoke-test:fac:2",54 name: "Smoke Test DC12 Data Center",55 operatorName: "Smoke Test Operator",56 city: "Ashburn",57 countryIso2: "US",58 geo: { lat: 39.0403, lng: -77.4901, precision: "exact", source: "smoke-test" },59 status: "operational",60 totalPowerMw: 60,61 provenance: prov("https://smoke-test.invalid/dc12-alt"),62 };63 const facC: NormalizedFacility = {64 entityType: "facility",65 key: "smoke-test:fac:3",66 name: "Smoke Test Montréal Campus",67 operatorName: "Smoke Test Operator",68 city: "Montréal",69 regionName: "Quebec",70 countryIso2: "CA",71 geo: { lat: 45.5017, lng: -73.5673, precision: "city", source: "smoke-test" },72 status: "under_construction",73 facilityType: "hyperscale",74 plannedPowerMw: 120,75 isAi: true,76 facts: [{ field: "plannedPowerMw", value: 120, provenance: prov("https://smoke-test.invalid/montreal", { isEstimate: true, method: "smoke-estimate" }) }],77 provenance: prov("https://smoke-test.invalid/montreal"),78 };79 const project: NormalizedProject = {80 entityType: "project",81 key: "smoke-test:prj:1",82 name: "Smoke Test Hyperscale Campus Phase 2",83 operatorName: "Smoke Test Operator",84 facilityKey: "smoke-test:fac:3",85 city: "Montréal",86 countryIso2: "CA",87 status: "announced",88 announcedOn: "2026-08",89 expectedOpening: "2028-Q2",90 plannedMw: 200,91 investmentUsd: 1.2e9,92 sourceUrl: "https://smoke-test.invalid/news/1",93 timeline: [{ date: "2026-08-15", type: "project_announced", description: "Phase 2 announced (smoke test)", url: "https://smoke-test.invalid/news/1" }],94 provenance: prov("https://smoke-test.invalid/projects/phase-2"),95 };96 const news: NormalizedNewsEvent = {97 entityType: "news_event",98 key: "smoke-test:news:1",99 title: "Smoke Test Operator announces 200 MW campus in Montréal, Canada",100 url: "https://smoke-test.invalid/news/1",101 publishedAt: "2026-08-15T12:00:00Z",102 summary: "Synthetic news item used by the ingest smoke test.",103 pageType: "press_release",104 eventType: "project_announced",105 mentions: { operators: ["Smoke Test Operator"], countriesIso2: ["CA"], mw: [200] },106 provenance: prov("https://smoke-test.invalid/news/1"),107 };108 const region: NormalizedCloudRegion = {109 entityType: "cloud_region",110 key: "smoke-test:region:smoke-east-1",111 providerName: "Smoke Test Cloud",112 code: "smoke-east-1",113 name: "Smoke East (Ashburn)",114 city: "Ashburn",115 countryIso2: "US",116 geo: { lat: 39.04, lng: -77.49, precision: "city", source: "smoke-test" },117 availabilityZones: 3,118 launchedOn: "2020",119 status: "operational",120 provenance: prov("https://smoke-test.invalid/regions"),121 };122 return [facA, facB, facC, project, news, region];123}124125function assert(cond: unknown, msg: string): void {126 if (!cond) throw new Error(`ASSERTION FAILED: ${msg}`);127}128129async function count(q: ReturnType<typeof sql>): Promise<number> {130 const r = await getDb().execute(q);131 return Number(r[0]?.n ?? 0);132}133134async function cleanup(): Promise<void> {135 const db = getDb();136 const keyed = async (type: string) => (await db.execute(sql`select entity_id from entity_keys where connector_id = ${CONNECTOR} and entity_type = ${type}`)).map((r) => String(r.entity_id));137 const facIds = await keyed("facility");138 const prjIds = await keyed("project");139 const crIds = await keyed("cloud_region");140 const ixIds = await keyed("ixp");141 await db.execute(sql`delete from events where source_id = ${SOURCE}`);142 await db.execute(sql`delete from provenance where connector_id = ${CONNECTOR}`);143 await db.execute(sql`delete from news_items where connector_id = ${CONNECTOR}`);144 await db.execute(sql`delete from entity_matches where connector_id = ${CONNECTOR}`);145 await db.execute(sql`delete from project_timeline where source_id = ${SOURCE}`);146 if (prjIds.length) await db.execute(sql`delete from projects where id in ${prjIds}`);147 if (crIds.length) await db.execute(sql`delete from cloud_regions where id in ${crIds}`);148 if (facIds.length) await db.execute(sql`delete from facilities where id in ${facIds}`); // aliases/tenants/ixps cascade149 await db.execute(sql`delete from ixps where name = 'SMOKE-IX' ${ixIds.length ? sql`or id in ${ixIds}` : sql``}`);150 await db.execute(sql`delete from entity_keys where connector_id = ${CONNECTOR}`);151 await db.execute(sql`delete from facility_tenants where operator_id in (select id from operators where name like 'Smoke Test%')`);152 await db.execute(sql`delete from operators where name like 'Smoke Test%'`);153 await db.execute(sql`delete from sources where id = ${SOURCE}`);154 try {155 await getClickHouse().command({ query: `ALTER TABLE observations DELETE WHERE connector_id = 'smoke-test'` });156 } catch {157 /* ClickHouse optional */158 }159}160161async function main() {162 const db = getDb();163 try {164 await ensureClickHouse();165 } catch (e) {166 console.warn(`clickhouse unavailable: ${(e as Error).message}`);167 }168 const seeded = await seedReferenceData(db);169 console.log(`seeds: ${seeded.countries} countries, ${seeded.metros} metros`);170 await cleanup(); // in case a previous run aborted171172 try {173 // ---- run 1174 const s1 = await ingestEntities(run, entities(36), doc);175 console.log("run 1", JSON.stringify({ ...s1, changes: s1.changes.length, refs: s1.refs.length }));176 assert(s1.rejected === 0, `run 1 rejected=${s1.rejected}`);177 const facCount = await count(sql`select count(*)::int as n from facilities f where f.id in (select entity_id from entity_keys where connector_id = ${CONNECTOR} and entity_type = 'facility') and f.merged_into is null`);178 assert(facCount === 2, `expected 2 facilities after merge, got ${facCount}`);179 const merged = await count(sql`select count(*)::int as n from entity_matches where connector_id = ${CONNECTOR} and status = 'auto_merged'`);180 assert(merged === 1, `expected 1 auto_merged match, got ${merged}`);181 const keys = await db.execute(sql`select key, entity_id from entity_keys where connector_id = ${CONNECTOR} and key in ('smoke-test:fac:1','smoke-test:fac:2')`);182 assert(keys.length === 2 && keys[0]!.entity_id === keys[1]!.entity_id, "fac:1 and fac:2 must point to the same facility");183 const facA = (await db.execute(sql`select f.* from facilities f join entity_keys k on k.entity_id = f.id where k.key = 'smoke-test:fac:1'`))[0]!;184 assert(Number(facA.it_capacity_mw) === 36 && Number(facA.total_power_mw) === 60, "merged facility keeps IT 36 MW and total 60 MW");185 assert(facA.metro_id != null, "Ashburn facility assigned to a metro");186 assert(Number(facA.carriers_count) === 1 && Number(facA.ixp_count) === 1, `tenants/ixps counted (${facA.carriers_count}/${facA.ixp_count})`);187 assert(Number(facA.completeness) >= 70, `completeness ${facA.completeness} ≥ 70`);188 const facC = (await db.execute(sql`select f.* from facilities f join entity_keys k on k.entity_id = f.id where k.key = 'smoke-test:fac:3'`))[0]!;189 assert(facC.mw_is_estimate === true, "estimate-only MW flagged mw_is_estimate");190 assert(facC.country_iso2 === "CA" && facC.metro_id != null, "Montréal facility → CA + metro");191 const discovered = await count(sql`select count(*)::int as n from events where source_id = ${SOURCE} and event_type = 'facility_discovered'`);192 assert(discovered === 2, `expected 2 facility_discovered events, got ${discovered}`);193 const news = await count(sql`select count(*)::int as n from news_items where connector_id = ${CONNECTOR} and 'CA' = any(country_iso2s) and mw = 200`);194 assert(news === 1, "news item linked to CA with 200 MW");195 const newsEvt = await count(sql`select count(*)::int as n from events where source_id = ${SOURCE} and entity_type = 'news_event'`);196 assert(newsEvt === 1, "news item with eventType produced one event");197 const prj = (await db.execute(sql`select p.*, (select count(*) from project_timeline t where t.project_id = p.id) as tl from projects p join entity_keys k on k.entity_id = p.id where k.key = 'smoke-test:prj:1'`))[0]!;198 assert(prj.facility_id === facC.id && Number(prj.tl) === 1, "project linked to facility with 1 timeline row");199 const provRows = await count(sql`select count(*)::int as n from provenance where connector_id = ${CONNECTOR} and is_current`);200 assert(provRows >= 25, `provenance rows ${provRows} ≥ 25`);201202 // ---- run 2 (idempotent except one MW change)203 const s2 = await ingestEntities(run, entities(48), doc);204 console.log("run 2", JSON.stringify({ ...s2, changes: s2.changes, refs: s2.refs.length }));205 assert(s2.created === 0, `run 2 created=${s2.created} (expected 0)`);206 assert(s2.rejected === 0, `run 2 rejected=${s2.rejected}`);207 assert(s2.events === 1, `run 2 events=${s2.events} (expected exactly 1 capacity_changed)`);208 assert(s2.changes.length === 1 && s2.changes[0]!.field === "itCapacityMw" && s2.changes[0]!.newValue === 48, "the only change is itCapacityMw → 48");209 const cap = (await db.execute(sql`select significance, title from events where source_id = ${SOURCE} and event_type = 'capacity_changed' and new_value = '48'::jsonb`))[0]!;210 assert(Number(cap.significance) === 80, `+33 % MW change → significance 80 (got ${cap.significance})`);211 const facCount2 = await count(sql`select count(*)::int as n from facilities f where f.id in (select entity_id from entity_keys where connector_id = ${CONNECTOR} and entity_type = 'facility') and f.merged_into is null`);212 assert(facCount2 === 2, "still 2 facilities");213 const provRows2 = await count(sql`select count(*)::int as n from provenance where connector_id = ${CONNECTOR} and is_current`);214 assert(provRows2 === provRows, `provenance row count unchanged on re-ingest (${provRows} → ${provRows2})`);215216 // ---- run 3: pure repeat → nothing at all217 const s3 = await ingestEntities(run, entities(48), doc);218 assert(s3.created === 0 && s3.events === 0 && s3.changes.length === 0, `run 3 must be a no-op (created=${s3.created} events=${s3.events})`);219220 // ---- dry run must not persist221 const before = await count(sql`select count(*)::int as n from facilities where name like 'Smoke Test%'`);222 await ingestEntities({ ...run, dryRun: true }, [{ ...(entities(48)[0] as NormalizedFacility), key: "smoke-test:fac:dry", name: "Smoke Test Dry Run", geo: { lat: 47.6, lng: -122.3, precision: "exact", source: "smoke-test" }, externalIds: {} }], doc);223 const after = await count(sql`select count(*)::int as n from facilities where name like 'Smoke Test%'`);224 assert(before === after, "dry run persisted nothing");225226 // ---- rankings / metrics / stats227 const r = await computeRankings(db);228 console.log("rankings", r);229 assert(r.rankings >= 20 && r.rows > 0, "rankings computed");230 const top = (await db.execute(sql`select rows from rankings where key = 'countries_facilities' and is_current`))[0]!;231 assert(Array.isArray(top.rows) && (top.rows as Array<{ id: string }>).some((x) => x.id === "US"), "US appears in countries_facilities");232 const m = await computeDailyMetrics(db);233 console.log("metrics", m);234 assert(m.rows > 0, "daily metrics written");235 const st = await refreshStats(db);236 console.log("stats", st);237 const opStats = (await db.execute(sql`select stats from operators where name = 'Smoke Test Operator'`))[0]!;238 assert(Number((opStats.stats as Record<string, unknown>).facilityCount) === 2, "operator stats facilityCount = 2");239 console.log("\nSMOKE TEST PASSED");240 } finally {241 await cleanup();242 // leave aggregates consistent with the clean database243 await computeRankings(db);244 await computeDailyMetrics(db);245 await refreshStats(db);246 const left = await count(sql`select (select count(*) from facilities where name like 'Smoke Test%') + (select count(*) from operators where name like 'Smoke Test%') + (select count(*) from events where source_id = ${SOURCE}) + (select count(*) from provenance where connector_id = ${CONNECTOR}) as n`);247 console.log(`cleanup: ${left} smoke rows left`);248 await closeDb();249 }250}251252main().catch((e) => {253 console.error(e);254 process.exit(1);255});256