spb/countryatlas

1#!/usr/bin/env python32"""Build registry/countries.yaml from the World Bank country list + mledoze/countries.34World Bank  : region, income group, capital, lat/long, WB aggregates (excluded here, listed in groups.yaml).5mledoze     : ISO numeric, official name, subregion, currency, area, UN membership, flag, borders, languages, demonym.67Hand overrides survive regeneration: edit registry/countries.overrides.yaml (keyed by iso3).8Usage: python3 scripts/build_country_registry.py  [--offline]  (offline = reuse cached /tmp files)9"""10from __future__ import annotations1112import json13import re14import sys15import unicodedata16import urllib.request17from pathlib import Path1819import yaml2021ROOT = Path(__file__).resolve().parents[1]22OUT = ROOT / "registry" / "countries.yaml"23OVERRIDES = ROOT / "registry" / "countries.overrides.yaml"24CACHE = Path("/tmp/countryatlas-registry")25CACHE.mkdir(parents=True, exist_ok=True)2627WB_URL = "https://api.worldbank.org/v2/country?format=json&per_page=400"28MLEDOZE_URL = "https://raw.githubusercontent.com/mledoze/countries/master/countries.json"2930CONTINENT_BY_REGION = {31    "Africa": "Africa",32    "Americas": "Americas",33    "Asia": "Asia",34    "Europe": "Europe",35    "Oceania": "Oceania",36    "Antarctic": "Antarctica",37}3839# Short display names preferred over the WB/mledoze defaults40SHORT_NAME_OVERRIDES = {41    "USA": "United States",42    "GBR": "United Kingdom",43    "RUS": "Russia",44    "KOR": "South Korea",45    "PRK": "North Korea",46    "IRN": "Iran",47    "SYR": "Syria",48    "VEN": "Venezuela",49    "BOL": "Bolivia",50    "TZA": "Tanzania",51    "LAO": "Laos",52    "VNM": "Vietnam",53    "EGY": "Egypt",54    "YEM": "Yemen",55    "GMB": "Gambia",56    "BHS": "Bahamas",57    "COD": "DR Congo",58    "COG": "Republic of the Congo",59    "CIV": "Côte d'Ivoire",60    "CZE": "Czechia",61    "SVK": "Slovakia",62    "MKD": "North Macedonia",63    "MDA": "Moldova",64    "KGZ": "Kyrgyzstan",65    "BRN": "Brunei",66    "FSM": "Micronesia",67    "STP": "São Tomé and Príncipe",68    "TUR": "Türkiye",69    "HKG": "Hong Kong",70    "MAC": "Macao",71    "PSE": "Palestine",72    "TWN": "Taiwan",73    "CPV": "Cabo Verde",74    "SWZ": "Eswatini",75    "TLS": "Timor-Leste",76    "VIR": "U.S. Virgin Islands",77    "VGB": "British Virgin Islands",78    "SXM": "Sint Maarten",79    "MAF": "Saint Martin",80    "CUW": "Curaçao",81    "XKX": "Kosovo",82}838485def fetch(url: str, name: str, offline: bool) -> bytes:86    p = CACHE / name87    if offline and p.exists():88        return p.read_bytes()89    req = urllib.request.Request(url, headers={"User-Agent": "CountryAtlas registry builder (contact@countryatlas.co)"})90    with urllib.request.urlopen(req, timeout=60) as r:91        data = r.read()92    p.write_bytes(data)93    return data949596def slugify(name: str) -> str:97    s = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode()98    s = s.lower().replace("&", "and").replace("'", "")99    s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")100    return s101102103def main() -> None:104    offline = "--offline" in sys.argv105    wb = json.loads(fetch(WB_URL, "wb-countries.json", offline))[1]106    ml = json.loads(fetch(MLEDOZE_URL, "mledoze.json", offline))107    ml_by3 = {c["cca3"]: c for c in ml}108    overrides = yaml.safe_load(OVERRIDES.read_text()) if OVERRIDES.exists() else {}109    overrides = overrides or {}110111    countries: list[dict] = []112    seen = set()113    for c in wb:114        if c["region"]["id"] == "NA" or not c["region"]["id"]:115            continue  # aggregate116        iso3 = c["id"]117        m = ml_by3.get(iso3)118        if iso3 == "XKX":  # Kosovo: mledoze uses UNK119            m = ml_by3.get("UNK")120        if iso3 == "CHI":  # Channel Islands (WB pseudo-country) — keep as territory using Jersey/Guernsey info121            m = None122        name = SHORT_NAME_OVERRIDES.get(iso3) or (m["name"]["common"] if m else c["name"])123        cur_code, cur_name = None, None124        if m and m.get("currencies"):125            cur_code = sorted(m["currencies"].keys())[0]126            cur_name = m["currencies"][cur_code].get("name")127        region_ml = m["region"] if m else None128        entry = {129            "id": iso3,130            "iso2": c["iso2Code"],131            "iso3": iso3,132            "iso_numeric": (m or {}).get("ccn3") or None,133            "slug": slugify(name),134            "short_name": name,135            "official_name": (m["name"]["official"] if m else c["name"]),136            "capital": (m["capital"][0] if m and m.get("capital") else c.get("capitalCity") or None),137            "continent": CONTINENT_BY_REGION.get(region_ml or "", None),138            "region_wb": c["region"]["id"],139            "region_wb_name": c["region"]["value"].strip(),140            "subregion": (m or {}).get("subregion") or None,141            "income_group": c["incomeLevel"]["id"] if c["incomeLevel"]["id"] not in ("", "INX") else None,142            "income_group_name": c["incomeLevel"]["value"] if c["incomeLevel"]["id"] not in ("", "INX") else None,143            "currency_code": cur_code,144            "currency_name": cur_name,145            "area_km2": (m or {}).get("area"),146            "latitude": float(c["latitude"]) if c.get("latitude") else ((m or {}).get("latlng") or [None, None])[0],147            "longitude": float(c["longitude"]) if c.get("longitude") else ((m or {}).get("latlng") or [None, None])[1],148            "flag_emoji": (m or {}).get("flag") or "".join(chr(0x1F1E6 + ord(ch) - 65) for ch in c["iso2Code"] if ch.isalpha()),149            "un_member": bool((m or {}).get("unMember", False)),150            "independent": bool((m or {}).get("independent", False)),151            "landlocked": bool((m or {}).get("landlocked", False)),152            "borders": (m or {}).get("borders") or [],153            "languages": sorted(((m or {}).get("languages") or {}).values()),154            "demonym": (((m or {}).get("demonyms") or {}).get("eng") or {}).get("m") or None,155            "status": "country" if (m and m.get("independent")) else "territory",156            "kind": "country",157        }158        # WB lists some territories with income levels; status from independence159        entry.update(overrides.get(iso3, {}))160        if entry["slug"] in seen:161            raise SystemExit(f"duplicate slug {entry['slug']} ({iso3})")162        seen.add(entry["slug"])163        countries.append(entry)164165    # Taiwan is not in the WB list — add from mledoze so the registry is complete (data will be sparse).166    if "TWN" not in {c["id"] for c in countries} and "TWN" in ml_by3:167        m = ml_by3["TWN"]168        countries.append({169            "id": "TWN", "iso2": "TW", "iso3": "TWN", "iso_numeric": m.get("ccn3"), "slug": "taiwan",170            "short_name": "Taiwan", "official_name": m["name"]["official"], "capital": m["capital"][0],171            "continent": "Asia", "region_wb": "EAS", "region_wb_name": "East Asia & Pacific", "subregion": m.get("subregion"),172            "income_group": "HIC", "income_group_name": "High income", "currency_code": "TWD", "currency_name": "New Taiwan dollar",173            "area_km2": m.get("area"), "latitude": m["latlng"][0], "longitude": m["latlng"][1], "flag_emoji": m.get("flag"),174            "un_member": False, "independent": False, "landlocked": False, "borders": [], "languages": sorted(m["languages"].values()),175            "demonym": "Taiwanese", "status": "territory", "kind": "country",176            **overrides.get("TWN", {}),177        })178179    countries.sort(key=lambda x: x["short_name"])180    header = (181        "# Canonical CountryAtlas country registry — GENERATED by scripts/build_country_registry.py\n"182        "# Sources: World Bank country API (region, income, capital, coordinates) + mledoze/countries (ISO numeric, official name,\n"183        "# subregion, currency, area, UN membership, flag, borders, languages). Edit registry/countries.overrides.yaml, not this file.\n"184    )185    OUT.write_text(header + yaml.safe_dump({"countries": countries}, allow_unicode=True, sort_keys=False, width=120))186    n_country = sum(1 for c in countries if c["status"] == "country")187    print(f"wrote {OUT} — {len(countries)} entries ({n_country} independent countries, {len(countries)-n_country} territories)")188189190if __name__ == "__main__":191    main()192