spb/datacenterindex

1import type { Parser, ExtractedRecord, RawDocument, ConnectorContext } from "@dci/connectors";2import { countryFromText } from "@dci/core";3import { loadHtml, tableRows, parenthetical, resolvePlace, regionRecord, type Place } from "./shared.js";4import { countryFromStateName, stateDisplayName } from "./city-coords.js";56/**7 * AWS — static documentation pages (no JS):8 *   Regions:     https://docs.aws.amazon.com/global-infrastructure/latest/regions/aws-regions.html9 *                table Code | Name | AZs | Geography | Opt-in status10 *   AZs:         https://docs.aws.amazon.com/global-infrastructure/latest/regions/aws-availability-zones.html11 *                table AZ ID | Region | Geography ("Virginia, United States of America") → state per region12 *   Local Zones: https://docs.aws.amazon.com/local-zones/latest/ug/available-local-zones.html13 *                table Group Long Name | Zone Name | Zone ID | Network Border Group | Parent Region | Parent Zone ID | Geography14 */15const AZ_PAGE = "https://docs.aws.amazon.com/global-infrastructure/latest/regions/aws-availability-zones.html";1617/** "Virginia, United States of America" → { state: "Virginia", iso: "US" } ; "Japan" → { iso: "JP" } */18function geography(g: string): { state: string | null; iso: string | null } {19  const parts = g.split(",").map((s) => s.trim()).filter(Boolean);20  if (parts.length >= 2) {21    const state = parts[0]!;22    const iso = countryFromText(parts.slice(1).join(" ")) ?? countryFromStateName(state);23    return { state: countryFromStateName(state) ? stateDisplayName(state) : state, iso };24  }25  return { state: null, iso: countryFromText(g) };26}2728/** Per-region state/province names from the Availability Zones docs page (best effort; cached in state). */29async function regionStates(ctx: ConnectorContext): Promise<Map<string, string>> {30  const cached = await ctx.getState<Record<string, string>>("aws:regionStates");31  if (cached) return new Map(Object.entries(cached));32  const map = new Map<string, string>();33  const doc = await ctx.fetch(AZ_PAGE, { group: "regions" });34  if (doc.error || doc.status !== 200) { ctx.log("warn", `aws AZ page unavailable (${doc.error?.code ?? doc.status}); regionName left null`); return map; }35  const $ = loadHtml(doc);36  for (const row of tableRows($, $("table"))) {37    if (row.length < 3 || row[0] === "AZ ID") continue;38    const region = row[1]!.replace(/[†*]/g, "").trim();39    const { state } = geography(row[2]!);40    if (state && !map.has(region)) map.set(region, state);41  }42  await ctx.setState("aws:regionStates", Object.fromEntries(map));43  return map;44}4546export const awsDocsRegionsParser: Parser = {47  name: "cloud_aws_docs_regions_v1",48  version: "1.0.0",49  pageTypes: ["cloud_region"],50  async parse(doc: RawDocument, ctx): Promise<ExtractedRecord[]> {51    const $ = loadHtml(doc);52    const states = await regionStates(ctx);53    const out: ExtractedRecord[] = [];54    for (const row of tableRows($, $("table"))) {55      if (row.length < 4) continue;56      const code = row[0]!.trim();57      if (!/^[a-z]{2}-[a-z]+-\d$/.test(code)) continue;58      const name = row[1]!;59      const azs = Number.parseInt(row[2]!.replace(/[^\d]/g, ""), 10);60      const geo = geography(row[3]!);61      const optIn = row[4] ?? null;62      const label = parenthetical(name);63      const state = states.get(code) ?? geo.state;64      // Place: the parenthetical when it is a city/metro (checked against the docs' own state); otherwise country / state only65      let place: Place = resolvePlace(label, geo.iso, state);66      if (!place.city && !place.regionName && geo.iso) place = { ...place, countryIso2: geo.iso };67      if (!place.regionName && state) place.regionName = state;68      if (!place.countryIso2) place.countryIso2 = geo.iso;69      const externalIds: Record<string, string | number> = { aws_region: code, kind: "region" };70      if (optIn) externalIds.opt_in = optIn;71      if (label) externalIds.published_location = label;72      out.push(73        regionRecord({74          connectorId: ctx.connectorId,75          keyPrefix: "aws:region",76          providerName: "AWS",77          code,78          name,79          place,80          availabilityZones: Number.isFinite(azs) && azs > 0 ? azs : null,81          status: "operational",82          externalIds,83          url: doc.finalUrl,84          methods: { code: "html:table.Code", name: "html:table.Name", availabilityZones: "html:table.AZs", countryIso2: "html:table.Geography", city: "curated-city-table" },85        }),86      );87    }88    ctx.log("info", `aws: ${out.length} regions`);89    return out;90  },91};9293export const awsLocalZonesParser: Parser = {94  name: "cloud_aws_local_zones_v1",95  version: "1.0.0",96  pageTypes: ["cloud_region"],97  parse(doc: RawDocument, ctx): ExtractedRecord[] {98    const $ = loadHtml(doc);99    const groups = new Map<string, { name: string; parent: string; geography: string; zones: string[]; zoneIds: string[] }>();100    for (const row of tableRows($, $("table"))) {101      if (row.length < 7 || row[0] === "Local Zone Group Long Name") continue;102      const [longName, zoneName, zoneId, group, parent, , geo] = row as [string, string, string, string, string, string, string];103      if (!/^[a-z]{2}-[a-z]+-\d-[a-z]{3}-\d$/.test(group)) continue;104      const g = groups.get(group) ?? { name: longName.replace(/\*+$/, "").trim(), parent, geography: geo, zones: [], zoneIds: [] };105      g.zones.push(zoneName); g.zoneIds.push(zoneId);106      groups.set(group, g);107    }108    const out: ExtractedRecord[] = [];109    for (const [group, g] of groups) {110      const geo = geography(g.geography);111      const label = parenthetical(g.name);112      // Local Zones are named after large metros; no state hint on purpose: AWS files "New York City" under New Jersey.113      let place: Place = resolvePlace(label, geo.iso);114      if (!place.countryIso2) place.countryIso2 = geo.iso;115      if (geo.state) place.regionName = geo.state;116      out.push(117        regionRecord({118          connectorId: ctx.connectorId,119          keyPrefix: "aws:localzone",120          providerName: "AWS",121          code: group,122          name: `${g.name} (Local Zone)`,123          place,124          availabilityZones: g.zones.length,125          status: "operational",126          externalIds: { kind: "local_zone", aws_parent_region: g.parent, aws_zone_names: g.zones.join(","), aws_zone_ids: g.zoneIds.join(","), ...(label ? { published_location: label } : {}) },127          url: doc.finalUrl,128          methods: { code: "html:table.NetworkBorderGroup", name: "html:table.GroupLongName", availabilityZones: "count:zones", countryIso2: "html:table.Geography", city: "curated-city-table" },129          certainty: 0.9,130        }),131      );132    }133    ctx.log("info", `aws: ${out.length} local zone groups`);134    return out;135  },136};137