spb/datacenterindex

1import type { ConnectorContext, ExtractedRecord, Parser, RawDocument } from "@dci/connectors";2import { cleanText } from "@dci/core";3import { articleContent, buildRecords, indexSentinel, shouldSkipDocument, type ArticleContent } from "./article-parser.js";4import { EXTRACTOR_VERSION, extractAnnouncement, parseDateLoose, type Announcement } from "./extract-project.js";5import { detectOperators } from "./operators-lexicon.js";67/**8 * `news_planning_pdf_v1` — public planning documents (staff reports, decision notices, rezoning applications,9 * committee agendas) as PDF or HTML. Public records: the text is kept by default (`keepText: true`).10 * Emits a `news_event` (planning_filed / planning_approved) and a `project` with status permitting / approved.11 * A refused application yields the event only (a refusal is not a cancellation by the developer).12 */13export type PlanningDecision = "approved" | "refused" | "pending";1415const DECISION_LINE_RE = /\b(?:decision|outcome|resolution|recommendation|status)\s*[:\-–]\s*(approved?|approve with conditions|granted?|permitted|refused?|rejected|denied|declined|withdrawn|pending|under consideration|awaiting decision)\b/i;16const APPROVED_RE = /\b(approved|approval granted|permission (?:is |be |has been |was )?granted|grant(?:ed|s)? (?:planning |conditional |outline )?(?:permission|approval|consent)|resolved to approve|resolves to approve|recommend(?:ed|ation)?(?: that the application be| to)? approv(?:e|al|ed)|approve(?:d)? with conditions|conditional approval|be approved|voted to approve|unanimously approved|planning permission granted|consent granted)\b/i;17const REFUSED_RE = /\b(refused|refusal|rejected|denied|declined|recommend(?:ed|ation)?(?: that the application be| to)? refus(?:e|al|ed)|not approved|dismissed|permission (?:is |be |has been |was )?refused)\b/i;18const APPLICANT_RE = /\b(?:applicant(?:'s)? name|name of applicant|applicant|on behalf of|proposed by|developer|submitted by|agent for|owner\/applicant|applicant\/owner|property owner|petitioner)\s*[:\-–]?\s*([A-Z][A-Za-z0-9&,.'’ -]{2,80}?)(?=\s{2,}|\n|\s(?:Site|Address|Location|Proposal|Description|Agent|Ward|Parish|Case|Reference|Ref|Application|Date|Tel|Email)\b|[;|]|\.\s|$)/m;19const ADDRESS_RE = /\b(\d{1,5}[A-Za-z]?\s+(?:[A-Z][A-Za-z0-9'.-]+\s){1,4}(?:Road|Rd|Street|St|Avenue|Ave|Drive|Dr|Lane|Ln|Way|Boulevard|Blvd|Parkway|Pkwy|Highway|Hwy|Court|Ct|Place|Pl|Trail|Trl|Circle|Cir|Loop|Route|Rte|Pike|Turnpike)\b\.?)/;20const REF_RE = /\b(?:application|planning|case|file|reference|ref\.?|permit|rezoning|petition)\s*(?:no\.?|number|#|ref\.?|:)?\s*[:#]?\s*((?:[A-Z]{1,5}[-/ ])?\d{1,4}[-/]\d{2,6}(?:[-/][A-Z0-9]{1,8})?)\b/i;21const DOC_DATE_RE = /\b(?:date(?: of (?:decision|issue|report|meeting|validation|receipt))?|decision date|meeting date|issued|dated|hearing date)\s*[:\-–]?\s*((?:\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\.?,?\s+20\d\d)|(?:(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\.?\s+\d{1,2},?\s+20\d\d)|20\d\d-\d{2}-\d{2}|\d{1,2}\/\d{1,2}\/20\d\d)/i;2223export function detectDecision(text: string): { decision: PlanningDecision; method: string } {24  const head = text.slice(0, 20_000);25  const line = head.match(DECISION_LINE_RE);26  if (line) {27    const v = line[1]!.toLowerCase();28    if (/refus|reject|deni|declin|withdrawn/.test(v)) return { decision: "refused", method: "regex:decision_line" };29    if (/approv|grant|permitted/.test(v)) return { decision: "approved", method: "regex:decision_line" };30    return { decision: "pending", method: "regex:decision_line" };31  }32  const refused = (head.match(new RegExp(REFUSED_RE.source, "gi")) ?? []).length;33  const approved = (head.match(new RegExp(APPROVED_RE.source, "gi")) ?? []).length;34  if (refused && refused >= approved) return { decision: "refused", method: "regex:decision_keywords" };35  if (approved) return { decision: "approved", method: "regex:decision_keywords" };36  return { decision: "pending", method: "regex:decision_default" };37}3839export function detectApplicant(text: string): { name: string | null; method: string } {40  const ops = detectOperators(text.slice(0, 20_000));41  const m = text.slice(0, 20_000).match(APPLICANT_RE);42  if (m) {43    const raw = cleanText(m[1]!.replace(/[,.;:\s]+$/, ""));44    if (raw && raw.length >= 3 && !/^(the|a|an|n\/a|none|same|as above)$/i.test(raw)) {45      // prefer the canonical lexicon name when the applicant line names a known operator46      const inLine = detectOperators(raw);47      return { name: inLine[0]?.name ?? raw.slice(0, 120), method: inLine.length ? "regex:applicant+lexicon" : "regex:applicant" };48    }49  }50  return { name: ops[0]?.name ?? null, method: ops.length ? "lexicon:operator" : "none" };51}5253export const planningPdfParser: Parser = {54  name: "news_planning_pdf_v1",55  version: EXTRACTOR_VERSION,56  pageTypes: ["planning_document"],57  async parse(doc: RawDocument, ctx: ConnectorContext, params: Record<string, unknown> = {}): Promise<ExtractedRecord[]> {58    if (shouldSkipDocument(doc)) return [indexSentinel(ctx.connectorId, doc.finalUrl)];59    let content: ArticleContent;60    try { content = await articleContent(doc); } catch (e) { ctx.log("warn", `news_planning_pdf_v1: cannot read ${doc.finalUrl}: ${(e as Error).message}`); return []; }61    if (!content.text || content.text.length < 120) return [];62    const text = content.text;63    const a = extractAnnouncement(content.title, text, { publishedAt: content.published });64    if (a.relevance !== "strong") { ctx.log("debug", `news_planning_pdf_v1: not a data center document ${doc.finalUrl}`); return []; }6566    const decision = detectDecision(text);67    const applicant = detectApplicant(text);68    const address = text.slice(0, 20_000).match(ADDRESS_RE)?.[1] ?? null;69    const reference = text.slice(0, 20_000).match(REF_RE)?.[1] ?? null;70    if (!content.published) {71      const dm = text.slice(0, 6000).match(DOC_DATE_RE);72      if (dm) { const d = parseDateLoose(dm[1]!.replace(/(\d{1,2})\/(\d{1,2})\/(20\d\d)/, (_, mo: string, da: string, y: string) => `${y}-${mo.padStart(2, "0")}-${da.padStart(2, "0")}`)); if (d) content = { ...content, published: d, publishedMethod: "text:document_date" }; }73    }7475    const status = decision.decision === "approved" ? "approved" : decision.decision === "pending" ? "permitting" : null;76    const eventType = decision.decision === "approved" ? "planning_approved" : "planning_filed";77    const operatorName = applicant.name ?? a.operator?.name ?? null;78    const adjusted: Announcement = {79      ...a,80      pageType: "planning_document",81      eventType,82      status: status ?? "cancelled", // refused → no project (qualifiesAsProject rejects "cancelled"); the event keeps the decision83      operator: a.operator ?? (operatorName ? { name: operatorName, kind: "other", index: 0, matched: operatorName } : null),84      methods: { ...a.methods, status: decision.method, operatorName: applicant.method !== "none" ? applicant.method : a.methods.operatorName ?? "none", pageType: "parser:planning" },85    };86    if (adjusted.operator && !adjusted.explicitName) {87      const city = adjusted.location?.city;88      adjusted.projectName = city ? `${adjusted.operator.name} ${city} data center (planning)` : `${adjusted.operator.name} data center planning application`;89      adjusted.methods.name = "generated:planning";90    }91    // A planning document describes exactly one application: a project is warranted as soon as the decision status is92    // known and the document gives a size (MW or acreage) or names the applicant.93    const forceProject = status != null && (adjusted.headlineMw != null || adjusted.acreage != null || operatorName != null);94    return buildRecords({95      connectorId: ctx.connectorId, url: doc.finalUrl, a: adjusted, content,96      params: { keepText: params.keepText !== false, minProjectMw: 0.5, minInvestmentUsd: 1, minAcres: 1, lenient: true },97      pageTypeOverride: "planning_document",98      extraEvent: { decision: decision.decision, applicant: operatorName, address, reference },99      extraProject: address ? { address } : {},100      certaintyEvent: 0.75,101      forceProject,102    });103  },104};105