import type { Parser, ExtractedRecord, RawDocument } from "@dci/connectors"; import { countryFromText, cleanText, slugify } from "@dci/core"; import { loadHtml, tableRows, resolvePlace, regionRecord, type Place } from "./shared.js"; import { countryFromStateName, stateDisplayName, lookupCity } from "./city-coords.js"; /** * Google Cloud regions — https://cloud.google.com/compute/docs/regions-zones (static docs table): * Zones | Location | Machine types | CPUs | … ; zone "europe-west1-b", location "St. Ghislain, Belgium, Europe". * Regions are derived by grouping zones on their prefix; the zone count is the availability-zone count. * (The marketing page /about/locations is a JS shell fed by data.json which carries no geography → not used.) */ const CONTINENTS = new Set(["europe", "apac", "north america", "south america", "middle east", "africa", "asia"]); /** "Montréal, Québec, North America" → { city, region, iso } */ function gcpLocation(loc: string): { city: string; region: string | null; iso: string | null } { const parts = loc.split(",").map((s) => cleanText(s) ?? "").filter((s) => s && !CONTINENTS.has(s.toLowerCase())); const city = parts[0] ?? loc; let region: string | null = null; let iso: string | null = null; for (const p of parts.slice(1)) { if (countryFromStateName(p)) { region = stateDisplayName(p); iso = iso ?? countryFromStateName(p); continue; } if (/^england|scotland|wales$/i.test(p)) { region = p; iso = "GB"; continue; } iso = iso ?? countryFromText(p); if (!countryFromText(p)) region = region ?? p; } if (!iso) iso = lookupCity(city)?.iso ?? countryFromText(city); return { city, region, iso }; } export const gcpRegionsParser: Parser = { name: "cloud_gcp_regions_v1", version: "1.0.0", pageTypes: ["cloud_region"], parse(doc: RawDocument, ctx): ExtractedRecord[] { const $ = loadHtml(doc); const regions = new Map; location: string }>(); for (const row of tableRows($, $("table"))) { const zone = row[0]?.trim(); const location = row[1]; if (!zone || !location || !/^[a-z]+-[a-z]+\d+-[a-z]$/.test(zone)) continue; const code = zone.replace(/-[a-z]$/, ""); const r = regions.get(code) ?? { zones: new Set(), location }; r.zones.add(zone); regions.set(code, r); } const out: ExtractedRecord[] = []; for (const [code, r] of regions) { const loc = gcpLocation(r.location); let place: Place = resolvePlace(loc.city, loc.iso, loc.region); if (!place.countryIso2) place.countryIso2 = loc.iso; if (!place.city) place.city = loc.city; if (!place.regionName && loc.region) place.regionName = loc.region; out.push( regionRecord({ connectorId: ctx.connectorId, keyPrefix: "gcp:region", providerName: "Google Cloud", code, name: `${loc.city} (${code})`, place, availabilityZones: r.zones.size, status: "operational", externalIds: { gcp_region: code, gcp_zones: [...r.zones].sort().join(","), published_location: r.location }, url: doc.finalUrl, methods: { code: "html:table.Zones(prefix)", availabilityZones: "count:zones", city: "html:table.Location+curated-city-table" }, }), ); } ctx.log("info", `gcp: ${out.length} regions from ${[...regions.values()].reduce((n, r) => n + r.zones.size, 0)} zones`); return out; }, }; /** * Google's own data center sites — https://www.google.com/about/datacenters/locations/ * The list is embedded server-side in
as a comma-joined list of JSON objects: * { location: "Council Bluffs, Iowa", latitude, longitude (often null), inDevelopment, continent, detailPage } * → facilities operated by Google (hyperscale). `inDevelopment: true` is Google's own label; we map it to * "announced" (pipeline) because the page does not say whether construction has started. */ interface GoogleSite { location?: string; latitude?: number | null; longitude?: number | null; inDevelopment?: boolean; continent?: string; detailPage?: string } export const googleDatacentersParser: Parser = { name: "cloud_google_datacenters_v1", version: "1.0.0", parse(doc: RawDocument, ctx): ExtractedRecord[] { const $ = loadHtml(doc); const raw = $("#data-centers-data").text().trim(); if (!raw) { ctx.log("warn", "google datacenters: #data-centers-data not found"); return []; } let sites: GoogleSite[] = []; try { sites = JSON.parse(raw.startsWith("[") ? raw : `[${raw}]`) as GoogleSite[]; } catch (e) { ctx.log("warn", `google datacenters: bad JSON ${(e as Error).message}`); return []; } const out: ExtractedRecord[] = []; for (const s of sites) { if (!s.location) continue; const [cityPart, ...rest] = s.location.split(",").map((x) => x.trim()); const tail = rest.join(", "); const isoFromTail = countryFromStateName(tail) ?? countryFromText(tail) ?? null; let place: Place = resolvePlace(cityPart!, isoFromTail, countryFromStateName(tail) ? tail : null); if (!place.city) place.city = cityPart!; if (!place.countryIso2) place.countryIso2 = isoFromTail ?? lookupCity(cityPart!)?.iso ?? countryFromText(s.location); if (!place.regionName && tail && countryFromStateName(tail)) place.regionName = stateDisplayName(tail); // Google's own coordinates (when present) are used only when the curated table has none; they are site-area markers → "approximate" if (place.lat == null && typeof s.latitude === "number" && typeof s.longitude === "number") place = { ...place, lat: s.latitude, lng: s.longitude, geoPrecision: "approximate" }; const detail = s.detailPage ? new URL(s.detailPage.replace(/^\.\/(\.\.\/)*/, "/about/datacenters/"), "https://www.google.com").toString() : null; const slug = slugify(s.location); out.push({ kind: "facility", key: `google:dc:${slug}`, url: doc.finalUrl, pageType: "facility_index", certainty: 0.9, methods: { name: "embedded-json:location", status: "embedded-json:inDevelopment", lat: place.geoPrecision === "approximate" ? "embedded-json:latitude" : "curated-city-table" }, data: { name: `Google ${s.location} data center`, operatorName: "Google", city: place.city, regionName: place.regionName, countryIso2: place.countryIso2, lat: place.lat, lng: place.lng, geoPrecision: place.geoPrecision ?? "city", status: s.inDevelopment ? "announced" : "operational", facilityType: "hyperscale", isHyperscale: true, website: detail ?? doc.finalUrl, description: s.inDevelopment ? `Listed by Google as a data center location "in development" (${s.continent ?? ""}).` : `Listed by Google as an operating data center location (${s.continent ?? ""}).`, externalIds: { google_location: s.location, ...(detail ? { google_detail_page: detail } : {}) }, }, }); } ctx.log("info", `google datacenters: ${out.length} sites (${out.filter((r) => r.data.status === "announced").length} in development)`); return out; }, };