SPB Git forge
38commits 1branches 0releases
338.7 MBsize
maindefault branch
1 h agolast push
HTML 53.9% TypeScript 44.5% JavaScript 0.6% SQL 0.5%

Classifier: verb boundary fix, more development/financing/lease cues, trust the lead for primary-source newsrooms; quality-apply passes source kind

Simon-Pierre Boucher committed 20 days ago (Sep 12, 2026) parent 0efbc40

6 changed files +16 −13

modified apps/worker/src/connectors/news/article-parser.ts +2 −1
@@ -179,7 +179,8 @@ export const newsArticleParser: Parser = {
179 179 let content: ArticleContent;
180 180 try { content = await articleContent(doc); } catch (e) { ctx.log("warn", `news_article_v1: cannot read ${doc.finalUrl}: ${(e as Error).message}`); return []; }
181 181 if (!content.text || content.text.length < 80) return [];
182 − const a = extractAnnouncement(content.title, content.text, { publishedAt: content.published });
182 + const primary = params._sourceKind != null && ["operator", "cloud_provider", "government", "utility"].includes(String(params._sourceKind));
183 + const a = extractAnnouncement(content.title, content.text, { publishedAt: content.published, trustLead: primary || p.keepText });
183 184 if (a.relevance === "none") { ctx.log("debug", `news_article_v1: not about data centers, skipped ${doc.finalUrl}`); return []; }
184 185 // Third-party publishers (keepText false): an article only weakly about data centers, or whose data-center vocabulary
185 186 // sits deep in the body, is not indexed at all. The sentinel keeps the GenericConnector fallback from archiving it.
modified apps/worker/src/connectors/news/extract-project.ts +2 −2
@@ -407,7 +407,7 @@ export function classifyAnnouncement(title: string | null, text: string): { page
407 407 }
408 408
409 409 // ─── main ──────────────────────────────────────────────────────────────────────────────────────────
410 −export function extractAnnouncement(rawTitle: string | null, rawText: string, opts: { publishedAt?: string | null; now?: Date } = {}): Announcement {
410 +export function extractAnnouncement(rawTitle: string | null, rawText: string, opts: { publishedAt?: string | null; now?: Date; /** primary source (operator / government / utility newsroom): the lead may supply class + verb */ trustLead?: boolean } = {}): Announcement {
411 411 // "118-acre", "192-MW", "1.2-GW" → the core parsers expect a space (or nothing) between figure and unit
412 412 const unhyphen = (s: string) => s.replace(/(\d)[-‑–](?=(?:acres?|hectares?|ha|MW|GW|kW|megawatts?|gigawatts?)\b)/gi, "$1 ");
413 413 const title = unhyphen(cleanTitle(rawTitle));
@@ -491,7 +491,7 @@ export function extractAnnouncement(rawTitle: string | null, rawText: string, op
491 491 if (nonProject && !["acquisition", "closure", "cloud_region", "financial_disclosure"].includes(pageType)) { pageType = "press_release"; eventType = "news"; methods.pageType = `${methods.pageType}+veto:title`; }
492 492
493 493 // classification BEFORE anything becomes a project: people / money / deal / market headlines never do
494 − const classification = classifyProjectEvent({ title, lead: head, hasExplicitName: !!explicitName, hasOperator: !!operator, hasLocation: !!(location?.city || location?.region || location?.country), status: st.status });
494 + const classification = classifyProjectEvent({ title, lead: head, hasExplicitName: !!explicitName, hasOperator: !!operator, hasLocation: !!(location?.city || location?.region || location?.country), status: st.status, trustLead: !!opts.trustLead });
495 495 // evidence + scope + semantics of the headline figures (the sentence decides; no sentence → the claim is not site-scoped)
496 496 const campusHint: ClaimScope = /\b(campus|park|complex|hub)\b/i.test(`${explicitName ?? ""} ${title}`) ? "campus" : "facility";
497 497 const mwEvidence = headlineMw != null ? findEvidence(`${title}. ${body}`, headlineMw, "mw") : null;
modified deploy/bin/post-upgrade.sh +1 −1
@@ -15,7 +15,7 @@ has() { local s; for s in "${steps[@]}"; do [[ $s == "$1" ]] && return 0; done;
15 15
16 16 # `cli` service runs the worker image; scripts/ is not baked into the image → bind-mount it
17 17 CLI="$(compose_prefix data) run --rm -T"
18 −CLI_SCRIPTS="$(compose_prefix data) run --rm -T -v $REMOTE_DIR/scripts:/app/scripts cli node /app/node_modules/tsx/dist/cli.mjs /app/scripts"
18 +CLI_SCRIPTS="$(compose_prefix data) run --rm -T -v $REMOTE_DIR/scripts:/app/scripts -v $REMOTE_DIR/packages/core/src:/app/packages/core/src cli node /app/node_modules/tsx/dist/cli.mjs /app/scripts"
19 19 dci() { remote data "$CLI cli cli $*"; }
20 20 script() { remote data "$CLI_SCRIPTS/$*"; }
21 21 psqlq() { remote data "docker exec -i dci-postgres-1 psql -U dci -d dci -Atc \"$1\""; }
modified packages/connectors/src/generic.ts +1 −1
@@ -59,7 +59,7 @@ export class GenericConnector implements Connector {
59 59 for (const [name, ex] of extractors) {
60 60 if (ex.parser) {
61 61 const parser = getParser(ex.parser);
62 − const recs = await parser.parse(doc, ctx, ex.params);
62 + const recs = await parser.parse(doc, ctx, { ...(ex.params ?? {}), _sourceKind: this.sourceKind });
63 63 for (const r of recs) records.push({ ...r, pageType: r.pageType ?? pageType });
64 64 continue;
65 65 }
modified packages/core/src/claims.ts +6 −4
@@ -323,9 +323,9 @@ const CLASS_RULES: Array<[ProjectClass, RegExp]> = [
323 323 ["POWER_AGREEMENT", /\b(ppa\b|power purchase agreement|(?:virtual|corporate|long-term|\d+-year) (?:power|energy) (?:purchase )?agreement|(?:energy|power|electricity) (?:supply|offtake) (?:agreement|deal|contract)|offtake|(?:signs?|signed|inks?|inked|secures?|secured) (?:a )?(?:\d+[ -]?(?:mw|gw) )?(?:of )?(?:solar|wind|nuclear|geothermal|hydro|gas|renewable|clean|carbon-free) (?:power|energy|capacity|supply)|(?:solar|wind|nuclear|geothermal|hydro|gas|battery|fuel cell|smr|reactor)s? (?:to power|will power|powering|deal|plant to supply|farm to supply)|behind-the-meter|on-site (?:generation|power plant|gas plant|solar)|microgrid|small modular reactor)\b/i],
324 324 ["SUSTAINABILITY", /\b(sustainability (?:report|goals?|targets?|commitment|strategy)|net[- ]zero|carbon[- ]neutral|carbon[- ]free|esg\b|renewable (?:energy )?(?:certificates?|credits?|target|goal|commitment)|100% renewable|water (?:positive|stewardship|usage|conservation)|emissions? (?:reduction|target)|leed (?:gold|platinum|certif)|green (?:building|certification)|energy efficiency (?:program|initiative)|climate (?:pledge|commitment)|biodiversity|tree planting|heat (?:reuse|recovery) (?:program|scheme|initiative))\b/i],
325 325 ["GRID_CONNECTION", /\b(grid connection|grid-?connected|interconnection (?:agreement|request|capacity|queue|study)|connection agreement|(?:new |expanded )?substation|transmission (?:line|upgrade|project|capacity)|energi[sz]ation|energi[sz]ed|large[- ]load (?:tariff|request|agreement|customer)|load (?:request|study|letter)|(?:secured|reserved|allocated|approved) (?:\d+[ -]?(?:mw|gw) of )?(?:grid |utility )?(?:power|capacity) (?:from|with) (?:the utility|[A-Z][\w&]+ (?:energy|power|electric|utility)))\b/i],
326 − ["FINANCING", /\b(financ(?:es|ing|ed)|refinanc\w+|(?:debt|notes?|bond|equity|securit\w+) (?:issuance|offering|placement|raise)|raises? (?:\$|€|£|us\$)|raised (?:\$|€|£|us\$)|series [a-e]\b|funding (?:round|arranged|secured)|loan|credit facility|green bond|bonds?\b|notes? offering|debt (?:facility|package|raise)|equity (?:raise|investment|stake)|ipo\b|capital raise|construction (?:loan|financing)|securitization|abs\b|term loan|(?:secures?|secured|closes?|closed) (?:\$|€|£|us\$)[\d.,]+ ?(?:m|bn|b|million|billion)? (?:in )?(?:financing|loan|facility|debt|funding))\b/i],
326 + ["FINANCING", /\b(financ(?:es|ing|ed)|refinanc\w+|backs? [A-Z][\w'’.-]+|guarantees?|(?:debt|notes?|bond|equity|securit\w+) (?:issuance|offering|placement|raise)|raises? (?:\$|€|£|us\$)|raised (?:\$|€|£|us\$)|series [a-e]\b|funding (?:round|arranged|secured)|loan|credit facility|green bond|bonds?\b|notes? offering|debt (?:facility|package|raise)|equity (?:raise|investment|stake)|ipo\b|capital raise|construction (?:loan|financing)|securitization|abs\b|term loan|(?:secures?|secured|closes?|closed) (?:\$|€|£|us\$)[\d.,]+ ?(?:m|bn|b|million|billion)? (?:in )?(?:financing|loan|facility|debt|funding))\b/i],
327 327 ["ACQUISITION", /\b(acqui(?:res?|red|sition) (?:of )?(?:[A-Z][\w&'.-]+ )+(?:group|holdings|inc|ltd|llc|limited|corp|sa|ag|plc|pte|data ?cent(?:er|re)s?|portfolio|platform|company|business)|(?:acquires?|acquired|acquiring|buys?|bought|buying|purchases?|purchased|purchasing|takes? over|took over) (?:a |an |the |its |two |three |four |five |six |\d+ )?(?:\d+[ -]?(?:mw|gw) )?(?:(?:hyperscale|colocation|carrier-neutral|edge|operational|existing) )?(?:data ?cent(?:er|re)s?|facility|facilities|portfolio|platform|operator|business|company|stake|campus from|site from)|acquisition of|takeover|buyout|merger|merges? with|to be acquired|stake in|sells?|sold|divest\w+|sale of)\b/i],
328 − ["CUSTOMER_AGREEMENT", /\b((?:signs?|signed|secures?|secured|inks?|inked|lands?|landed|wins?|won|announces?) (?:a |an |its |the )?(?:multi-year |long-term |major |anchor |hyperscale |\d+[ -]?(?:mw|gw) )?(?:colocation |hosting |capacity |wholesale |pre-?)?(?:lease|leases|tenant|customer|contract|agreement|deal) (?:with|for|from)|(?:lease|leases|leased) (?:\d+[ -]?(?:mw|gw)|[\d,]+ (?:sq|square))|(?:selects?|selected|chooses?|chose|picks?|picked|taps?|tapped) [A-Z][\w&'.-]+ (?:for|as|to)|(?:hosting|colocation|lease|leasing|services?|master|supply|framework|capacity) agreements?|pre-?leas\w+|fully leased|anchor tenant|moves? into|deploys? (?:in|at|with)|expands? (?:its )?(?:footprint|presence) (?:in|at|with) [A-Z][\w&'.-]+(?:'s)? (?:data ?cent(?:er|re)|facility|campus))\b/i],
328 + ["CUSTOMER_AGREEMENT", /\b([$€£][\d.,]+ ?(?:bn|b|m|million|billion)? (?:\w+ )?leases?|\bleases?\b(?! (?:land|a site|the site))|(?:signs?|signed|secures?|secured|inks?|inked|lands?|landed|wins?|won|announces?) (?:a |an |its |the )?(?:multi-year |long-term |major |anchor |hyperscale |\d+[ -]?(?:mw|gw) )?(?:colocation |hosting |capacity |wholesale |pre-?)?(?:lease|leases|tenant|customer|contract|agreement|deal) (?:with|for|from)|(?:lease|leases|leased) (?:\d+[ -]?(?:mw|gw)|[\d,]+ (?:sq|square))|(?:selects?|selected|chooses?|chose|picks?|picked|taps?|tapped) [A-Z][\w&'.-]+ (?:for|as|to)|(?:hosting|colocation|lease|leasing|services?|master|supply|framework|capacity) agreements?|pre-?leas\w+|fully leased|anchor tenant|moves? into|deploys? (?:in|at|with)|expands? (?:its )?(?:footprint|presence) (?:in|at|with) [A-Z][\w&'.-]+(?:'s)? (?:data ?cent(?:er|re)|facility|campus))\b/i],
329 329 ["PARTNERSHIP", /\b(partnership|partners? with|joint venture|\bjv\b|join(?:s|ed)? forces|team(?:s|ed)? up|collaborat(?:es?|ion|ing)|alliance|memorandum of understanding|\bmou\b|strategic (?:agreement|relationship|cooperation)|framework agreement)\b/i],
330 330 ["PRODUCT_NEWS", /\b(launches? (?:a )?(?:new )?(?:service|platform|product|portal|program|programme|offering|solution|marketplace|tool|api|feature)|introduces? (?:a )?(?:new )?(?:service|platform|product|offering|solution)|now available|available (?:now|in|from)|unveils? (?:a )?(?:new )?(?:service|platform|product|offering|solution|brand|logo|website)|rebrand|new (?:brand|logo|website|identity)|integrat(?:es|ion) with|certified by|(?:earns?|achieves?|receives?) (?:[A-Z]\w+ )?(?:certification|rating|accreditation|iso \d))\b/i],
331 331 ["LAND_ACQUISITION", /\b((?:acquires?|acquired|buys?|bought|purchases?|purchased|secures?|secured|closes? on|closed on|options?) (?:a |an |the |its |additional |another )?(?:\d[\d,.]*[ -]?(?:acre|hectare|ha)s?|land|site|parcel|plot|property|farmland|lot)|land (?:purchase|acquisition|deal|bank)|(?:acre|hectare) (?:site|parcel|plot|property) (?:acquired|purchased|bought)|site (?:acquisition|purchase)|zoned land|land for (?:a |its |the )?(?:new )?(?:data ?cent(?:er|re)|campus))\b/i],
@@ -336,7 +336,7 @@ const CLASS_RULES: Array<[ProjectClass, RegExp]> = [
336 336 ];
337 337
338 338 /** Verbs / nouns that describe physical development (build, construct, break ground, expand, permit, approve, file, acquire land, energize…). */
339 −export const DEVELOPMENT_VERB_RE = /\b(build|builds|building|built|construct\w*|develop\w*|break\w* ground|broke ground|groundbreaking|expan\w+|open|opens|opening|opened|permit\w*|approv\w+|file[sd]?|filing|rezon\w+|acquire[sd]? (?:\d[\d,.]*[ -]?(?:acre|hectare)s?|land|a site|the site|a parcel)|land (?:purchase|acquisition)|energi[sz]\w+|deliver\w*|commission\w*|top\w* out|takes? shape|rising|going up|nears? completion|commits? (?:at least |up to |over |more than )?[€$£]\d|invest\w* (?:at least |up to |over |more than )?[€$£]\d|unveil\w* (?:plans?|a |an |its |new |\d|[€$£])|announc\w* (?:plans? (?:for|to)|a new|new|its (?:first|second|next)|\d+[ -]?(?:mw|gw))|plans? (?:for|to build|to develop|a new|new|\d+[ -]?(?:mw|gw))|propos\w* (?:a |an |new |\d)|planned|proposed|site plan|planning application|zoning|\d[\d.,]*\s?(?:mw|gw)\s+(?:[\w€$£,]+\s){0,3}(?:campus|project|data ?cent(?:er|re)|facility|site|hub|megacampus))\b/i;
339 +export const DEVELOPMENT_VERB_RE = /\b(build|builds|building|built|build-?out|construct\w*|develop\w*|break\w* ground|broke ground|groundbreaking|expan\w+|open|opens|opening|opened|permit\w*|approv\w+|file[sd]?|filing|rezon\w+|acquire[sd]? (?:\d[\d,.]*[ -]?(?:acre|hectare)s?|land|a site|the site|a parcel)|land (?:purchase|acquisition)|energi[sz]\w+|deliver\w*|commission\w*|top\w* out|takes? shape|rising|going up|nears? completion|advances?|scales? up|commits? (?:at least |up to |over |more than )?[€$£]\d|invest\w* (?:at least |up to |over |more than )?[€$£]\d|(?:triples?|doubles?|increases?) (?:its |their )?investment|investment in|unveil\w* (?:plans?|a |an |its |new |\d|[€$£])|announc\w* (?:plans? (?:for|to)|a new|new|its (?:first|second|next)|\d+[ -]?(?:mw|gw))|announcement of|plans?\b|planned|propos\w* (?:a |an |new |\d)|proposed|(?:a |two |three |four |second |third |its first |first |new |newest |another )(?:new )?(?:hyperscale |ai |ai-ready |\d+[ -]?(?:mw|gw) )?(?:campus(?:es)?|data ?cent(?:er|re)s?|facilit(?:y|ies)|data halls?)|site plan|planning application|zoning|\d[\d.,]*\s?(?:mw|gw)\s+(?:[\w€$£,]+\s){0,3}(?:campus|project|data ?cent(?:er|re)|facility|site|hub|megacampus))(?![A-Za-z])/i;
340 340 /** Site nouns — a headline must name a site AND carry a development verb before a status word can make it a project. */
341 341 export const SITE_NOUN_RE = /\b(data ?cent(?:er|re)s?|campus(?:es)?|mega-?campus(?:es)?|facility|facilities|ai factor(?:y|ies)|hub|site|data hall|(?:ai |digital |cloud |compute |sovereign )infrastructure)\b/i;
342 342
@@ -352,6 +352,8 @@ export interface ProjectClassInput {
352 352 hasLocation?: boolean;
353 353 /** the title-derived status (pipeline / operational / null) */
354 354 status?: string | null;
355 + /** the source is the operator / a government / a utility itself: the lead may supply the class and the verb ("Hello, Tulsa!") */
356 + trustLead?: boolean;
355 357 }
356 358
357 359 export interface ProjectClassification {
@@ -407,7 +409,7 @@ export function classifyProjectEvent(i: ProjectClassInput): ProjectClassificatio
407 409 if (cls === "UNKNOWN" && i.status === "expansion" && titleSite) { cls = "EXPANSION"; reasons.push("status:expansion"); }
408 410
409 411 const classFromTitle = reasons.some((r) => r.startsWith("title:"));
410 − const verb = titleVerb || (classFromTitle && !!lead && DEVELOPMENT_VERB_RE.test(lead));
412 + const verb = titleVerb || ((classFromTitle || !!i.trustLead) && !!lead && DEVELOPMENT_VERB_RE.test(lead));
411 413 const name = !!i.hasExplicitName;
412 414 const operator = !!i.hasOperator;
413 415 const location = !!i.hasLocation;
modified scripts/quality-apply.ts +4 −4
@@ -23,15 +23,15 @@ const has = (f: string) => ALL || args.includes(f);
23 23
24 24 async function hideFalsePositives(): Promise<void> {
25 25 const db = getDb();
26 − const rows = await db.execute<{ id: string; name: string; slug: string; planned_mw: number | null; project_class: string | null; status: string; title: string | null; summary: string | null; operator_id: string | null; city: string | null; country_iso2: string | null }>(sql`
27 − select p.id, p.name, p.slug, p.planned_mw, p.project_class, p.status, n.title, n.summary, p.operator_id, p.city, p.country_iso2
28 − from projects p left join news_items n on n.url = p.source_url
26 + const rows = await db.execute<{ id: string; name: string; slug: string; planned_mw: number | null; project_class: string | null; status: string; title: string | null; source_kind: string | null; summary: string | null; operator_id: string | null; city: string | null; country_iso2: string | null }>(sql`
27 + select p.id, p.name, p.slug, p.planned_mw, p.project_class, p.status, n.title, n.summary, p.operator_id, p.city, p.country_iso2, c.kind as source_kind
28 + from projects p left join news_items n on n.url = p.source_url left join connectors c on c.id = n.connector_id
29 29 where p.merged_into is null and not p.hidden`);
30 30 let hidden = 0, kept = 0;
31 31 const reasons: Record<string, number> = {};
32 32 for (const r of rows) {
33 33 const title = r.title ?? r.name;
34 − const cls = classifyProjectEvent({ title, lead: r.summary ?? "", hasOperator: !!r.operator_id, hasLocation: !!(r.city || r.country_iso2), hasExplicitName: !/ \| |—| - /.test(r.name) && r.name.length < 90, status: r.status });
34 + const cls = classifyProjectEvent({ title, lead: r.summary ?? "", hasOperator: !!r.operator_id, hasLocation: !!(r.city || r.country_iso2), hasExplicitName: !/ \| |—| - /.test(r.name) && r.name.length < 90, status: r.status, trustLead: ["operator", "cloud_provider", "government", "utility"].includes(r.source_kind ?? "") });
35 35 let reason: string | null = null;
36 36 if (!PHYSICAL_CLASSES.has(cls.class)) reason = `class:${cls.class}`;
37 37 else if (isNonProjectTitle(title)) reason = "veto:title";
38 38