spb/datacenterindex

1/**2 * Project-pipeline precision audit. Every project is re-judged from the headline / summary of the article it came3 * from, with the same gates the extractors now apply:4 *5 *   veto:title          people / awards / partnerships / financing / market research / policy / opinion headline6 *   existing-facility   opening, "we are online", delivered — the article is about an operating facility7 *   fallback:no-signal  created by the GenericConnector fallback (key …:project) without a MW figure AND a pipeline8 *                       status in the headline (the new fallback gate)9 *   news-kind:fallback  fallback project from a third-party publisher (the fallback no longer creates those)10 *   keep                passes every gate11 *12 *   set -a; source .env; set +a13 *   node node_modules/tsx/dist/cli.mjs scripts/quality-projects.ts               # report14 *   node node_modules/tsx/dist/cli.mjs scripts/quality-projects.ts --apply       # delete flagged projects (keys, provenance, timeline, events)15 *   node node_modules/tsx/dist/cli.mjs scripts/quality-projects.ts --sample 60   # print a random sample for hand classification16 */17import { closeDb, getDb, sql } from "@dci/db";18import { fallbackProjectGate } from "@dci/connectors";19import { isNonProjectTitle, OPENING_TITLE_RE, STATUS_RULES } from "../apps/worker/src/connectors/news/extract-project.js";2021const args = process.argv.slice(2);22const APPLY = args.includes("--apply");23const SAMPLE = args.includes("--sample") ? Number(args[args.indexOf("--sample") + 1] ?? 60) : 0;24const FUTURE_OPEN_RE = /\b(to|will|set to|plans? to|due to|expected to|slated to) (open|be online|go live)\b|\b(open|opening|online|live) (in|by) (Q[1-4]|20\d\d|early|late|mid)\b/i;2526interface P { id: string; key: string | null; connector: string | null; kind: string | null; name: string; status: string; mw: number | null; title: string | null; summary: string | null }2728function verdict(p: P): string {29  const title = p.title ?? p.name;30  if (isNonProjectTitle(title)) return "veto:title";31  if (OPENING_TITLE_RE.test(title) && !FUTURE_OPEN_RE.test(title) && !STATUS_RULES.slice(0, 3).some(([re]) => re.test(title))) return "existing-facility";32  if (p.key?.endsWith(":project")) {33    if (p.kind === "news") return "news-kind:fallback";34    if (!fallbackProjectGate(title, p.summary)) return "fallback:no-signal";35  }36  return "keep";37}3839async function main(): Promise<void> {40  const db = getDb();41  const rows = await db.execute(sql`42    select p.id, p.name, p.status, p.planned_mw, k.key, k.connector_id, s.kind, n.title, n.summary43    from projects p44    left join entity_keys k on k.entity_id = p.id and k.entity_type = 'project'45    left join sources s on s.connector_id = k.connector_id46    left join news_items n on n.url = p.source_url47    order by p.created_at`);48  const projects: P[] = rows.map((r) => ({ id: String(r.id), key: r.key == null ? null : String(r.key), connector: r.connector_id == null ? null : String(r.connector_id), kind: r.kind == null ? null : String(r.kind), name: String(r.name), status: String(r.status), mw: r.planned_mw == null ? null : Number(r.planned_mw), title: r.title == null ? null : String(r.title), summary: r.summary == null ? null : String(r.summary) }));4950  if (SAMPLE) {51    const shuffled = [...projects].sort(() => Math.random() - 0.5).slice(0, SAMPLE);52    for (const p of shuffled) console.log(`${p.id} | ${p.connector} | ${p.status} | ${p.mw ?? "-"} MW | ${p.name} | ${p.title ?? "-"} | ${(p.summary ?? "").slice(0, 140)}`);53    return;54  }5556  const counts = new Map<string, number>();57  const flagged: Array<P & { verdict: string }> = [];58  for (const p of projects) {59    const v = verdict(p);60    counts.set(v, (counts.get(v) ?? 0) + 1);61    if (v !== "keep") flagged.push({ ...p, verdict: v });62  }63  console.log(`projects: ${projects.length}`);64  for (const [v, n] of [...counts].sort((a, b) => b[1] - a[1])) console.log(`  ${v.padEnd(20)} ${n}`);65  console.log("\nflagged:");66  for (const p of flagged) console.log(`  ${p.verdict.padEnd(20)} ${p.connector ?? "-"} | ${p.status} | ${p.mw ?? "-"} MW | ${p.name.slice(0, 70)} | ${(p.title ?? "").slice(0, 90)}`);67  const kept = projects.length - flagged.length;68  console.log(`\nprecision proxy: kept ${kept}/${projects.length} (${((kept / Math.max(1, projects.length)) * 100).toFixed(0)} % of stored projects pass the headline gates)`);6970  if (!APPLY) { console.log("(dry run — pass --apply to delete the flagged projects)"); return; }71  let deleted = 0;72  for (const p of flagged) {73    await db.transaction(async (tx) => {74      await tx.execute(sql`update events set project_id = null where project_id = ${p.id}`);75      await tx.execute(sql`delete from events where entity_type = 'project' and entity_id = ${p.id}`);76      await tx.execute(sql`update news_items set project_id = null where project_id = ${p.id}`);77      await tx.execute(sql`delete from provenance where entity_type = 'project' and entity_id = ${p.id}`);78      await tx.execute(sql`delete from entity_keys where entity_type = 'project' and entity_id = ${p.id}`);79      await tx.execute(sql`delete from projects where id = ${p.id}`); // project_timeline cascades80    });81    deleted++;82  }83  console.log(`deleted ${deleted} projects`);84}8586main().catch((e) => { console.error(e); process.exitCode = 1; }).finally(() => closeDb());87