spb/datacenterindex

1import { load, type CheerioAPI } from "cheerio";2import type { GeoPrecision, NormalizedCloudRegion, NormalizedEntity, PageType } from "@dci/core";3import { cleanText, countryFromText, parsePartialDate, validLatLng, urlFingerprint } from "@dci/core";4import { GenericConnector, type ConnectorConfig, type ConnectorContext, type DiscoveredUrl, type ExtractedRecord, type RawDocument } from "@dci/connectors";5import { lookupCity, countryFromStateName, stateDisplayName, normKey, type CityEntry } from "./city-coords.js";67/** A cheerio selection (avoids depending on domhandler types directly). */8type Sel = ReturnType<CheerioAPI>;910/** Text of a cheerio node, whitespace-collapsed, BOM/zero-width characters removed. */11export function txt(el: Sel): string {12  return cleanText(el.text().replace(/[\uFEFF\u200B\u00A0]/g, " ")) ?? "";13}1415/** Rows of an HTML table as arrays of cell texts (th + td). */16export function tableRows($: CheerioAPI, table: Sel): string[][] {17  return table18    .find("tr")19    .toArray()20    .map((tr) => $(tr).find("th,td").toArray().map((c) => txt($(c))));21}2223export function loadHtml(doc: RawDocument): CheerioAPI { return load(doc.text); }2425/** "Canada (Central)", "Mexico (Central)": a compass word, not a place. */26export function isDirectional(label: string): boolean {27  return /^(central|north|south|east|west|north ?east|north ?west|south ?east|south ?west|northern|southern|eastern|western|\d)$/i.test(label.trim());28}2930/** "US East (N. Virginia)" → "N. Virginia"; "Singapore" → null */31export function parenthetical(name: string): string | null {32  const m = name.match(/\(([^()]+)\)/);33  return m ? cleanText(m[1]) : null;34}3536export interface Place {37  city: string | null;38  regionName: string | null;39  countryIso2: string | null;40  lat: number | null;41  lng: number | null;42  geoPrecision: GeoPrecision | null;43}4445/**46 * Resolve a provider-published place label ("St. Ghislain", "N. Virginia", "Virginia", "Ireland") into47 * city / region / country / curated coordinates. Never geocodes: unknown labels yield a country at best.48 */49export function resolvePlace(label: string | null | undefined, isoHint?: string | null, regionHint?: string | null): Place {50  const empty: Place = { city: null, regionName: null, countryIso2: isoHint ?? null, lat: null, lng: null, geoPrecision: null };51  if (!label || isDirectional(label)) return empty;52  let hit: CityEntry | null = lookupCity(label, isoHint ?? undefined) ?? (isoHint ? null : lookupCity(label));53  // Same town name in another state ("Lebanon", "Columbus", "Douglas County"): refuse the table entry when the54  // provider's own state/province disagrees — better no coordinates than the wrong town.55  if (hit && regionHint && hit.region && normKey(hit.region) !== normKey(regionHint)) hit = null;56  if (hit && (!isoHint || hit.iso === isoHint)) {57    return { city: hit.city, regionName: hit.region ?? null, countryIso2: hit.iso, lat: hit.lat, lng: hit.lng, geoPrecision: hit.precision ?? "city" };58  }59  // A US state / Canadian province name → region only, no coordinates (we do not place markers on state centroids)60  const stateIso = countryFromStateName(label);61  if (stateIso) return { city: null, regionName: stateDisplayName(label), countryIso2: stateIso, lat: null, lng: null, geoPrecision: null };62  // A country name → country only63  const iso = countryFromText(label);64  if (iso) return { city: null, regionName: null, countryIso2: iso, lat: null, lng: null, geoPrecision: null };65  return { ...empty, city: cleanText(label) };66}6768/** Build a cloud_region ExtractedRecord with the field names CloudConnector.normalize expects. */69export function regionRecord(opts: {70  connectorId: string;71  providerName: string;72  code: string;73  name: string;74  place?: Partial<Place> | null;75  availabilityZones?: number | null;76  launchedOn?: string | null;77  status?: NormalizedCloudRegion["status"];78  isSovereign?: boolean;79  externalIds?: Record<string, string | number>;80  url: string;81  methods?: Record<string, string>;82  certainty?: number;83  keyPrefix?: string;84  /** where the coordinates came from when not the curated city table (e.g. "provider-marker:azure") */85  geoSource?: string;86}): ExtractedRecord {87  const p = opts.place ?? {};88  return {89    kind: "cloud_region",90    key: `${opts.keyPrefix ?? opts.connectorId}:${opts.code.toLowerCase()}`,91    url: opts.url,92    pageType: "cloud_region",93    certainty: opts.certainty ?? 0.95,94    methods: opts.methods ?? { code: "html:table", name: "html:table" },95    data: {96      providerName: opts.providerName,97      code: opts.code,98      name: opts.name,99      city: p.city ?? null,100      regionName: p.regionName ?? null,101      countryIso2: p.countryIso2 ?? null,102      lat: p.lat ?? null,103      lng: p.lng ?? null,104      geoPrecision: p.geoPrecision ?? null,105      availabilityZones: opts.availabilityZones ?? null,106      launchedOn: opts.launchedOn ?? null,107      status: opts.status ?? "operational",108      isSovereign: opts.isSovereign ?? false,109      externalIds: opts.externalIds ?? {},110      geoSource: opts.geoSource ?? null,111    },112  };113}114115const REGION_STATUSES: ReadonlySet<string> = new Set(["announced", "operational", "restricted", "retired"]);116117/**118 * Cloud-provider connector: the generic pipeline plus a `cloud_region` normalizer that keeps119 * externalIds, the full status vocabulary (announced | operational | restricted | retired), the120 * curated geo precision and the provider's own source URL. Facilities / news / projects go through121 * the generic normalizer unchanged.122 */123export class CloudConnector extends GenericConnector {124  override async normalize(ctx: ConnectorContext, records: ExtractedRecord[]): Promise<NormalizedEntity[]> {125    const regions = records.filter((r) => r.kind === "cloud_region");126    const others = records.filter((r) => r.kind !== "cloud_region");127    const out: NormalizedEntity[] = others.length ? await super.normalize(ctx, others) : [];128    const d = this.cfg.defaults ?? {};129    for (const r of regions) {130      const x = r.data;131      const str = (k: string): string | null => (x[k] == null ? null : cleanText(String(x[k])));132      const num = (k: string): number | null => { const v = x[k]; if (v == null || v === "") return null; const n = typeof v === "number" ? v : Number(String(v).replace(/[^\d.-]/g, "")); return Number.isFinite(n) ? n : null; };133      const code = str("code"), name = str("name");134      if (!code || !name) continue;135      const lat = num("lat"), lng = num("lng");136      const precision = (str("geoPrecision") as GeoPrecision | null) ?? "city";137      const statusRaw = str("status");138      const status = (statusRaw && REGION_STATUSES.has(statusRaw) ? statusRaw : "operational") as NormalizedCloudRegion["status"];139      const prov = ctx.provenance(r.url, { method: r.methods ? Object.values(r.methods).slice(0, 3).join(",") : undefined, extractorVersion: this.parserVersion, note: precision === "city" || precision === null ? "coordinates are city-level markers from a curated city table, not facility locations" : `coordinates are ${precision}-level markers, not facility locations` });140      const region: NormalizedCloudRegion = {141        entityType: "cloud_region",142        key: r.key,143        providerName: str("providerName") ?? d.providerName ?? this.sourceName,144        code,145        name,146        city: str("city"),147        regionName: str("regionName"),148        countryIso2: str("countryIso2") ?? d.countryIso2 ?? null,149        geo: validLatLng(lat, lng) ? { lat: lat!, lng: lng!, precision, source: str("geoSource") ?? `curated-city-table:${this.id}` } : null,150        availabilityZones: num("availabilityZones"),151        launchedOn: str("launchedOn") ? parsePartialDate(str("launchedOn")) : null,152        status,153        isSovereign: x.isSovereign === true,154        sourceUrl: str("sourceUrl") ?? r.url,155        externalIds: (x.externalIds as Record<string, string | number> | undefined) ?? {},156        provenance: prov,157      };158      out.push(region);159    }160    return out;161  }162}163164/**165 * Newsroom connector for hyperscaler feeds: items are filtered on their RSS title/summary166 * (params.include / params.exclude regexes) BEFORE fetching, so a 100-item "What's New" feed only costs167 * the handful of infrastructure-related fetches. When the generic extractor yields nothing for a168 * matched item, a news_event is still produced from the feed metadata (title, summary, date).169 */170export class CloudNewsroomConnector extends CloudConnector {171  private readonly include: RegExp | null;172  private readonly exclude: RegExp | null;173  constructor(cfg: ConnectorConfig) {174    super(cfg);175    const p = cfg.params ?? {};176    this.include = typeof p.include === "string" ? new RegExp(p.include, "i") : null;177    this.exclude = typeof p.exclude === "string" ? new RegExp(p.exclude, "i") : null;178  }179180  /** Feed items are judged on title + summary; URLs found without feed metadata (sitemaps) pass through to the generic relevance gate. */181  private matches(u: DiscoveredUrl): boolean {182    if (!u.meta?.title) return true;183    const hay = `${String(u.meta.title)} ${String(u.meta.summary ?? "").replace(/<[^>]+>/g, " ")}`;184    if (this.exclude && this.exclude.test(hay)) return false;185    return this.include ? this.include.test(hay) : true;186  }187188  override async discover(ctx: ConnectorContext): Promise<DiscoveredUrl[]> {189    const all = await super.discover(ctx);190    const kept = all.filter((u) => u.group !== "newsroom" || this.matches(u)).map((u) => (u.meta?.title ? { ...u, meta: { ...(u.meta ?? {}), feedMatched: true } } : u));191    ctx.log("info", `newsroom filter: ${kept.length}/${all.length} items match`);192    return kept;193  }194195  override async extract(ctx: ConnectorContext, doc: RawDocument): Promise<ExtractedRecord[]> {196    const recs = await super.extract(ctx, doc);197    if (recs.length || doc.error || doc.status >= 400 || !doc.meta?.feedMatched) return recs;198    const title = cleanText(String(doc.meta?.title ?? "")) ?? cleanText(String(doc.meta?.pageTitle ?? "")) ?? doc.finalUrl;199    const summary = cleanText(String(doc.meta?.summary ?? "").replace(/<[^>]+>/g, " "));200    const published = doc.meta?.published ? parsePartialDate(String(doc.meta.published)) ?? isoDay(String(doc.meta.published)) : null;201    const pageType: PageType = (doc.meta?.pageType as PageType | undefined) && doc.meta?.pageType !== "unknown" ? (doc.meta!.pageType as PageType) : "press_release";202    return [{ kind: "news_event", key: `${this.id}:news:${urlFingerprint(doc.finalUrl)}`, url: doc.finalUrl, pageType, certainty: 0.5, methods: { title: "rss:title", summary: "rss:description", publishedAt: "rss:pubDate" }, data: { title, url: doc.finalUrl, publishedAt: published, summary, pageType, eventType: "news" } }];203  }204}205206function isoDay(s: string): string | null {207  const d = new Date(s);208  return Number.isNaN(d.getTime()) ? null : d.toISOString().slice(0, 10);209}210