spb/datacenterindex

1import type { Parser, ExtractedRecord, RawDocument } from "@dci/connectors";2import { countryFromText, cleanText, slugify } from "@dci/core";3import { loadHtml, tableRows, resolvePlace, regionRecord, type Place } from "./shared.js";4import { countryFromStateName, stateDisplayName, lookupCity } from "./city-coords.js";56/**7 * Google Cloud regions — https://cloud.google.com/compute/docs/regions-zones (static docs table):8 *   Zones | Location | Machine types | CPUs | … ; zone "europe-west1-b", location "St. Ghislain, Belgium, Europe".9 * Regions are derived by grouping zones on their prefix; the zone count is the availability-zone count.10 * (The marketing page /about/locations is a JS shell fed by data.json which carries no geography → not used.)11 */12const CONTINENTS = new Set(["europe", "apac", "north america", "south america", "middle east", "africa", "asia"]);1314/** "Montréal, Québec, North America" → { city, region, iso } */15function gcpLocation(loc: string): { city: string; region: string | null; iso: string | null } {16  const parts = loc.split(",").map((s) => cleanText(s) ?? "").filter((s) => s && !CONTINENTS.has(s.toLowerCase()));17  const city = parts[0] ?? loc;18  let region: string | null = null;19  let iso: string | null = null;20  for (const p of parts.slice(1)) {21    if (countryFromStateName(p)) { region = stateDisplayName(p); iso = iso ?? countryFromStateName(p); continue; }22    if (/^england|scotland|wales$/i.test(p)) { region = p; iso = "GB"; continue; }23    iso = iso ?? countryFromText(p);24    if (!countryFromText(p)) region = region ?? p;25  }26  if (!iso) iso = lookupCity(city)?.iso ?? countryFromText(city);27  return { city, region, iso };28}2930export const gcpRegionsParser: Parser = {31  name: "cloud_gcp_regions_v1",32  version: "1.0.0",33  pageTypes: ["cloud_region"],34  parse(doc: RawDocument, ctx): ExtractedRecord[] {35    const $ = loadHtml(doc);36    const regions = new Map<string, { zones: Set<string>; location: string }>();37    for (const row of tableRows($, $("table"))) {38      const zone = row[0]?.trim();39      const location = row[1];40      if (!zone || !location || !/^[a-z]+-[a-z]+\d+-[a-z]$/.test(zone)) continue;41      const code = zone.replace(/-[a-z]$/, "");42      const r = regions.get(code) ?? { zones: new Set<string>(), location };43      r.zones.add(zone);44      regions.set(code, r);45    }46    const out: ExtractedRecord[] = [];47    for (const [code, r] of regions) {48      const loc = gcpLocation(r.location);49      let place: Place = resolvePlace(loc.city, loc.iso, loc.region);50      if (!place.countryIso2) place.countryIso2 = loc.iso;51      if (!place.city) place.city = loc.city;52      if (!place.regionName && loc.region) place.regionName = loc.region;53      out.push(54        regionRecord({55          connectorId: ctx.connectorId,56          keyPrefix: "gcp:region",57          providerName: "Google Cloud",58          code,59          name: `${loc.city} (${code})`,60          place,61          availabilityZones: r.zones.size,62          status: "operational",63          externalIds: { gcp_region: code, gcp_zones: [...r.zones].sort().join(","), published_location: r.location },64          url: doc.finalUrl,65          methods: { code: "html:table.Zones(prefix)", availabilityZones: "count:zones", city: "html:table.Location+curated-city-table" },66        }),67      );68    }69    ctx.log("info", `gcp: ${out.length} regions from ${[...regions.values()].reduce((n, r) => n + r.zones.size, 0)} zones`);70    return out;71  },72};7374/**75 * Google's own data center sites — https://www.google.com/about/datacenters/locations/76 * The list is embedded server-side in <div id="data-centers-data"> as a comma-joined list of JSON objects:77 *   { location: "Council Bluffs, Iowa", latitude, longitude (often null), inDevelopment, continent, detailPage }78 * → facilities operated by Google (hyperscale). `inDevelopment: true` is Google's own label; we map it to79 *   "announced" (pipeline) because the page does not say whether construction has started.80 */81interface GoogleSite { location?: string; latitude?: number | null; longitude?: number | null; inDevelopment?: boolean; continent?: string; detailPage?: string }8283export const googleDatacentersParser: Parser = {84  name: "cloud_google_datacenters_v1",85  version: "1.0.0",86  parse(doc: RawDocument, ctx): ExtractedRecord[] {87    const $ = loadHtml(doc);88    const raw = $("#data-centers-data").text().trim();89    if (!raw) { ctx.log("warn", "google datacenters: #data-centers-data not found"); return []; }90    let sites: GoogleSite[] = [];91    try { sites = JSON.parse(raw.startsWith("[") ? raw : `[${raw}]`) as GoogleSite[]; } catch (e) { ctx.log("warn", `google datacenters: bad JSON ${(e as Error).message}`); return []; }92    const out: ExtractedRecord[] = [];93    for (const s of sites) {94      if (!s.location) continue;95      const [cityPart, ...rest] = s.location.split(",").map((x) => x.trim());96      const tail = rest.join(", ");97      const isoFromTail = countryFromStateName(tail) ?? countryFromText(tail) ?? null;98      let place: Place = resolvePlace(cityPart!, isoFromTail, countryFromStateName(tail) ? tail : null);99      if (!place.city) place.city = cityPart!;100      if (!place.countryIso2) place.countryIso2 = isoFromTail ?? lookupCity(cityPart!)?.iso ?? countryFromText(s.location);101      if (!place.regionName && tail && countryFromStateName(tail)) place.regionName = stateDisplayName(tail);102      // Google's own coordinates (when present) are used only when the curated table has none; they are site-area markers → "approximate"103      if (place.lat == null && typeof s.latitude === "number" && typeof s.longitude === "number") place = { ...place, lat: s.latitude, lng: s.longitude, geoPrecision: "approximate" };104      const detail = s.detailPage ? new URL(s.detailPage.replace(/^\.\/(\.\.\/)*/, "/about/datacenters/"), "https://www.google.com").toString() : null;105      const slug = slugify(s.location);106      out.push({107        kind: "facility",108        key: `google:dc:${slug}`,109        url: doc.finalUrl,110        pageType: "facility_index",111        certainty: 0.9,112        methods: { name: "embedded-json:location", status: "embedded-json:inDevelopment", lat: place.geoPrecision === "approximate" ? "embedded-json:latitude" : "curated-city-table" },113        data: {114          name: `Google ${s.location} data center`,115          operatorName: "Google",116          city: place.city,117          regionName: place.regionName,118          countryIso2: place.countryIso2,119          lat: place.lat,120          lng: place.lng,121          geoPrecision: place.geoPrecision ?? "city",122          status: s.inDevelopment ? "announced" : "operational",123          facilityType: "hyperscale",124          isHyperscale: true,125          website: detail ?? doc.finalUrl,126          description: s.inDevelopment ? `Listed by Google as a data center location "in development" (${s.continent ?? ""}).` : `Listed by Google as an operating data center location (${s.continent ?? ""}).`,127          externalIds: { google_location: s.location, ...(detail ? { google_detail_page: detail } : {}) },128        },129      });130    }131    ctx.log("info", `google datacenters: ${out.length} sites (${out.filter((r) => r.data.status === "announced").length} in development)`);132    return out;133  },134};135