Classifier: site nouns separated from development verbs (status fallback needs both); quality-apply --hide-unconfirmed after reprocess
2 changed files +29 −2
modified
packages/core/src/claims.ts
+4 −2
@@ -336,7 +336,9 @@ const CLASS_RULES: Array<[ProjectClass, RegExp]> = [ | ||
| 336 | 336 | ]; |
| 337 | 337 | |
| 338 | 338 | /** Verbs / nouns that describe physical development (build, construct, break ground, expand, permit, approve, file, acquire land, energize…). */ |
| 339 | −export const DEVELOPMENT_VERB_RE = /\b(build|builds|building|built|construct\w*|develop\w*|break\w* ground|broke ground|groundbreaking|expan\w+|open|opens|opening|opened|permit\w*|approv\w+|file[sd]?|filing|rezon\w+|acquire[sd]? (?:\d[\d,.]*[ -]?(?:acre|hectare)s?|land|a site|the site|a parcel)|land (?:purchase|acquisition)|energi[sz]\w+|deliver\w*|commission\w*|top\w* out|takes? shape|rising|going up|nears? completion|commits? (?:at least |up to |over |more than )?[€$£]\d|invest\w* (?:at least |up to |over |more than )?[€$£]\d|(?:ai |digital |cloud |compute |sovereign )infrastructure|phase|campus(?:es)?|mega-?campus(?:es)?|data ?cent(?:er|re)s?|data hall|facilit(?:y|ies)|site plan|planning application|zoning|\d[\d.,]*\s?(?:mw|gw)\s+(?:[\w€$£,]+\s){0,3}(?:campus|project|data ?cent(?:er|re)|facility|site|hub|megacampus))\b/i; | |
| 339 | +export const DEVELOPMENT_VERB_RE = /\b(build|builds|building|built|construct\w*|develop\w*|break\w* ground|broke ground|groundbreaking|expan\w+|open|opens|opening|opened|permit\w*|approv\w+|file[sd]?|filing|rezon\w+|acquire[sd]? (?:\d[\d,.]*[ -]?(?:acre|hectare)s?|land|a site|the site|a parcel)|land (?:purchase|acquisition)|energi[sz]\w+|deliver\w*|commission\w*|top\w* out|takes? shape|rising|going up|nears? completion|commits? (?:at least |up to |over |more than )?[€$£]\d|invest\w* (?:at least |up to |over |more than )?[€$£]\d|unveil\w* (?:plans?|a |an |its |new |\d|[€$£])|announc\w* (?:plans? (?:for|to)|a new|new|its (?:first|second|next)|\d+[ -]?(?:mw|gw))|plans? (?:for|to build|to develop|a new|new|\d+[ -]?(?:mw|gw))|propos\w* (?:a |an |new |\d)|planned|proposed|site plan|planning application|zoning|\d[\d.,]*\s?(?:mw|gw)\s+(?:[\w€$£,]+\s){0,3}(?:campus|project|data ?cent(?:er|re)|facility|site|hub|megacampus))\b/i; | |
| 340 | +/** Site nouns — a headline must name a site AND carry a development verb before a status word can make it a project. */ | |
| 341 | +export const SITE_NOUN_RE = /\b(data ?cent(?:er|re)s?|campus(?:es)?|mega-?campus(?:es)?|facility|facilities|ai factor(?:y|ies)|hub|site|data hall|(?:ai |digital |cloud |compute |sovereign )infrastructure)\b/i; | |
| 340 | 342 | |
| 341 | 343 | export interface ProjectClassInput { |
| 342 | 344 | title: string | null | undefined; |
@@ -398,7 +400,7 @@ export function classifyProjectEvent(i: ProjectClassInput): ProjectClassificatio | ||
| 398 | 400 | // status-derived fallbacks need the headline itself to talk about building / a site ("Southern's 17 GW pipeline puts AI power |
| 399 | 401 | // demand into utility math" has an "announced" status word somewhere in the body but is an analysis piece) |
| 400 | 402 | const titleVerb = DEVELOPMENT_VERB_RE.test(title); |
| 401 | − const titleSite = /\b(data ?cent(?:er|re)s?|campus(?:es)?|facility|facilities|ai factory|hub|site)\b/i.test(title); | |
| 403 | + const titleSite = SITE_NOUN_RE.test(title); | |
| 402 | 404 | if (cls === "UNKNOWN" && i.status && /^(announced|proposed|rumored)$/.test(i.status) && titleVerb && titleSite) { cls = "NEW_BUILD"; reasons.push("status:announced→NEW_BUILD"); } |
| 403 | 405 | if (cls === "UNKNOWN" && i.status === "under_construction" && titleSite) { cls = "CONSTRUCTION_START"; reasons.push("status:construction"); } |
| 404 | 406 | if (cls === "UNKNOWN" && (i.status === "permitting" || i.status === "approved") && titleSite) { cls = "PERMIT"; reasons.push(`status:${i.status}`); } |
modified
scripts/quality-apply.ts
+25 −0
@@ -9,6 +9,7 @@ | ||
| 9 | 9 | * node node_modules/tsx/dist/cli.mjs scripts/quality-apply.ts --null-unbacked-mw # after `dci reprocess … --stale` |
| 10 | 10 | * node node_modules/tsx/dist/cli.mjs scripts/quality-apply.ts --fix-slugs # non-Latin operator slugs ("item") |
| 11 | 11 | * node node_modules/tsx/dist/cli.mjs scripts/quality-apply.ts --link-campuses # OSM buildings inside a campus footprint |
| 12 | + * node node_modules/tsx/dist/cli.mjs scripts/quality-apply.ts --hide-unconfirmed # after reprocess: source re-extracted, no project record / no claim → hide | |
| 12 | 13 | * node node_modules/tsx/dist/cli.mjs scripts/quality-apply.ts --all |
| 13 | 14 | */ |
| 14 | 15 | import { closeDb, getDb, sql } from "@dci/db"; |
@@ -65,6 +66,28 @@ async function nullUnbackedMw(): Promise<void> { | ||
| 65 | 66 | console.log(`\nunbacked figures: ${mw.length} planned_mw · ${inv.length} investment_usd ${has("--null-unbacked-mw") ? "nulled" : "would be nulled"}`); |
| 66 | 67 | } |
| 67 | 68 | |
| 69 | +/** | |
| 70 | + * After a full reprocess, a live project whose source document was re-extracted with the current parser but produced | |
| 71 | + * no claim at all (not even a status claim) is one the new pipeline no longer recognises as a project — the article | |
| 72 | + * was vetoed by class or evidence. Hide it (kept for audit; the news item and its event remain). | |
| 73 | + */ | |
| 74 | +async function hideUnconfirmed(): Promise<void> { | |
| 75 | + const db = getDb(); | |
| 76 | + const rows = await db.execute<{ id: string; slug: string; name: string; planned_mw: number | null; url: string; ev: string | null; cur: string | null }>(sql` | |
| 77 | + select p.id, p.slug, p.name, p.planned_mw, p.source_url as url, d.extractor_version as ev, c.parser_version as cur | |
| 78 | + from projects p | |
| 79 | + join documents d on d.canonical_url = p.source_url or d.url = p.source_url | |
| 80 | + join connectors c on c.id = d.connector_id | |
| 81 | + where p.merged_into is null and not p.hidden and p.project_class is null | |
| 82 | + and d.extractor_version = c.parser_version and d.extract_ok | |
| 83 | + and not exists (select 1 from claims cl where cl.subject_type = 'project' and cl.subject_id = p.id) | |
| 84 | + and not exists (select 1 from entity_keys k where k.entity_type = 'project' and k.entity_id = p.id and k.created_at > now() - interval '1 day') | |
| 85 | + and p.updated_at < d.updated_at`); | |
| 86 | + for (const r of rows) console.log(` hide ${r.slug.padEnd(70)} ${String(r.planned_mw ?? "").padStart(7)} MW re-extraction produced no project record «${r.name.slice(0, 70)}»`); | |
| 87 | + if (has("--hide-unconfirmed")) for (const r of rows) await db.transaction((tx) => hideProject(tx, r.id, "re-extraction with the current parser produced no project record (class veto / evidence threshold)", "quality-apply")); | |
| 88 | + console.log(`\nunconfirmed projects: ${rows.length} ${has("--hide-unconfirmed") ? "hidden" : "would be hidden"}`); | |
| 89 | +} | |
| 90 | + | |
| 68 | 91 | async function fixSlugs(): Promise<void> { |
| 69 | 92 | const db = getDb(); |
| 70 | 93 | const rows = await db.execute<{ id: string; name: string; slug: string; website: string | null; aliases: string[] }>(sql`select id, name, slug, website, aliases from operators where slug ~ '^(item|op|operator)(-\\d+)?$' or slug = ''`); |
@@ -113,6 +136,8 @@ async function main(): Promise<void> { | ||
| 113 | 136 | await hideFalsePositives(); |
| 114 | 137 | console.log("\n── unbacked figures"); |
| 115 | 138 | await nullUnbackedMw(); |
| 139 | + console.log("\n── unconfirmed after re-extraction"); | |
| 140 | + await hideUnconfirmed(); | |
| 116 | 141 | console.log("\n── operator slugs"); |
| 117 | 142 | await fixSlugs(); |
| 118 | 143 | console.log("\n── campus containment"); |
| 119 | 144 | |