spb/datacenterindex

1/**2 * World Bank Indicators API (https://api.worldbank.org/v2) → NormalizedCountryStat per ISO-2 country.3 * License: CC BY 4.0 (World Bank open data terms). Indicators:4 *   SP.POP.TOTL        population, total5 *   NY.GDP.MKTP.CD     GDP (current US$)6 *   EG.ELC.RNEW.ZS     renewable electricity output (% of total electricity output)   → renewableShare (percent, 0–100)7 *   EG.USE.ELEC.KH.PC  electric power consumption (kWh per capita)                     → electricityTwh = kWh/capita × population(same year) / 1e9 (derived, flagged estimate)8 *9 * One composed document: the country list (to drop aggregates) + the four indicator series (mrnev=1 = most10 * recent non-empty value) + a population time series so the derived electricity figure uses the population of11 * the same year as the per-capita value. All calls go through ctx.fetch, memoized per run.12 */13import type { ConnectorContext, DiscoveredUrl, ExtractedRecord, RawDocument } from "@dci/connectors";14import type { NormalizedCountryStat, NormalizedEntity, Provenance } from "@dci/core";15import { round } from "@dci/core";16import { DatasetConnector, fetchJsonOnce, iso2, num, str } from "./shared.js";1718interface WbCountry { id: string; iso2Code: string; name: string; region: { id: string; value: string }; incomeLevel?: { id: string; value: string } }19interface WbRow { indicator: { id: string; value: string }; country: { id: string; value: string }; countryiso3code: string; date: string; value: number | null }20type WbResponse<T> = [{ page: number; pages: number; per_page: number | string; total: number; lastupdated?: string }, T[] | null];2122const INDICATORS = { population: "SP.POP.TOTL", gdpUsd: "NY.GDP.MKTP.CD", renewableShare: "EG.ELC.RNEW.ZS", kwhPerCapita: "EG.USE.ELEC.KH.PC" } as const;2324export class WorldBankConnector extends DatasetConnector {25  private api(): string { return String(this.param("apiBase", "https://api.worldbank.org/v2")).replace(/\/+$/, ""); }26  private countriesUrl(): string { return `${this.api()}/country?format=json&per_page=400`; }27  private indicatorUrl(code: string): string { return `${this.api()}/country/all/indicator/${code}?format=json&per_page=20000&mrnev=1`; }28  private populationSeriesUrl(): string {29    const years = Math.max(3, this.param<number>("populationYears", 15));30    const to = new Date().getUTCFullYear();31    return `${this.api()}/country/all/indicator/${INDICATORS.population}?format=json&per_page=20000&date=${to - years}:${to}`;32  }33  private async json<T>(ctx: ConnectorContext, url: string): Promise<{ doc: RawDocument; rows: T[] | null; meta: WbResponse<T>[0] | null }> {34    const { doc, json } = await fetchJsonOnce<WbResponse<T>>(ctx, url, { pauseMs: this.param<number>("minDelayMs", 1500), timeoutMs: 90_000 });35    if (!Array.isArray(json) || json.length < 2 || !Array.isArray(json[1])) {36      if (json && !Array.isArray(json[1])) ctx.log("warn", `World Bank ${url}: no data (${JSON.stringify(json[0]).slice(0, 200)})`);37      return { doc, rows: null, meta: Array.isArray(json) ? json[0] : null };38    }39    if (json[0].pages > 1) ctx.log("warn", `World Bank ${url}: ${json[0].pages} pages, only the first was read — raise per_page`);40    return { doc, rows: json[1], meta: json[0] };41  }4243  async discover(_ctx: ConnectorContext): Promise<DiscoveredUrl[]> {44    return [{ url: this.countriesUrl(), group: "dataset", pageType: "dataset", priority: 100, meta: { object: "country-stats" } }];45  }4647  async fetch(ctx: ConnectorContext, u: DiscoveredUrl): Promise<RawDocument> {48    const { doc } = await this.json<WbCountry>(ctx, u.url);49    doc.group = u.group; doc.pageType = "dataset"; doc.meta = { ...(doc.meta ?? {}), ...(u.meta ?? {}) };50    if (doc.error || doc.status >= 400) return doc;51    for (const code of Object.values(INDICATORS)) await this.json(ctx, this.indicatorUrl(code));52    await this.json(ctx, this.populationSeriesUrl());53    return doc;54  }5556  async extract(ctx: ConnectorContext, doc: RawDocument): Promise<ExtractedRecord[]> {57    if (doc.error || doc.notModified || doc.status >= 400 || !doc.body.length) return [];58    let countries: WbCountry[];59    try { const j = JSON.parse(doc.text) as WbResponse<WbCountry>; countries = Array.isArray(j[1]) ? j[1] : []; } catch { ctx.log("warn", "World Bank country list: invalid JSON"); return []; }60    const real = new Map<string, WbCountry>();61    for (const c of countries) { const code = iso2(c.iso2Code); if (!code || c.region?.value === "Aggregates" || c.region?.id === "NA") continue; real.set(code, c); }62    ctx.log("info", `World Bank: ${countries.length} entries, ${real.size} countries/economies after dropping aggregates`);63    const series: Record<string, { rows: WbRow[]; url: string; updated: string | null }> = {};64    for (const [field, code] of Object.entries(INDICATORS)) {65      const url = this.indicatorUrl(code);66      const { rows, meta } = await this.json<WbRow>(ctx, url);67      if (!rows) { ctx.log("error", `World Bank ${code} unavailable — aborting document (no partial country stats)`); return []; }68      series[field] = { rows, url, updated: meta?.lastupdated ?? null };69    }70    const popSeriesUrl = this.populationSeriesUrl();71    const popSeries = await this.json<WbRow>(ctx, popSeriesUrl);72    const popByCountryYear = new Map<string, number>();73    for (const r of popSeries.rows ?? []) { const c = iso2(r.country?.id); if (c && r.value != null) popByCountryYear.set(`${c}:${r.date}`, r.value); }74    if (!popSeries.rows) ctx.log("warn", "World Bank population series unavailable — electricityTwh will not be derived");7576    const byCountry = new Map<string, Record<string, unknown>>();77    for (const [field, s] of Object.entries(series)) {78      for (const r of s.rows) {79        const c = iso2(r.country?.id);80        if (!c || !real.has(c) || r.value == null) continue;81        const d = byCountry.get(c) ?? byCountry.set(c, { iso2: c, name: real.get(c)!.name, iso3: real.get(c)!.id }).get(c)!;82        d[field] = r.value; d[`${field}Year`] = num(r.date); d[`${field}Url`] = s.url; d[`${field}Updated`] = s.updated;83        if (field === "kwhPerCapita") d.populationSameYear = popByCountryYear.get(`${c}:${r.date}`) ?? null;84      }85    }86    return [...byCountry.values()].map((d) => ({ kind: "country" as const, key: String(d.iso2), data: d, methods: { _all: "dataset-field:worldbank" }, certainty: 1, url: doc.finalUrl, pageType: "dataset" as const }));87  }8889  async normalize(ctx: ConnectorContext, records: ExtractedRecord[]): Promise<NormalizedEntity[]> {90    const out: NormalizedEntity[] = [];91    for (const r of records) {92      const d = r.data as Record<string, unknown>;93      const code = iso2(d.iso2); if (!code) continue;94      const prov = (url: string, method: string, extra: Partial<Provenance> = {}): Provenance => ctx.provenance(url, { method, extractorVersion: this.parserVersion, ...extra });95      const population = num(d.population), gdpUsd = num(d.gdpUsd), renewableShare = num(d.renewableShare), kwhPc = num(d.kwhPerCapita), popSameYear = num(d.populationSameYear);96      const electricityTwh = kwhPc != null && popSameYear != null ? round((kwhPc * popSameYear) / 1e9, 3) : null;97      if (population == null && gdpUsd == null && renewableShare == null && electricityTwh == null) continue;98      const years = [d.populationYear, d.gdpUsdYear, d.renewableShareYear, d.kwhPerCapitaYear].map((y) => num(y)).filter((y): y is number => y != null);99      // NormalizedCountryStat has no per-field `facts` channel: the per-indicator years and the derivation of100      // electricityTwh are recorded in the provenance note (entity-level provenance URL = country list document).101      const note = [102        `${str(d.name) ?? code} (${str(d.iso3) ?? "?"})`,103        population != null ? `population ${INDICATORS.population} year=${num(d.populationYear) ?? "?"}` : null,104        gdpUsd != null ? `gdpUsd ${INDICATORS.gdpUsd} year=${num(d.gdpUsdYear) ?? "?"}` : null,105        renewableShare != null ? `renewableShare ${INDICATORS.renewableShare} (% of electricity output) year=${num(d.renewableShareYear) ?? "?"}` : null,106        electricityTwh != null ? `electricityTwh DERIVED (estimate) = ${INDICATORS.kwhPerCapita} × ${INDICATORS.population} same year / 1e9, year=${num(d.kwhPerCapitaYear) ?? "?"}` : null,107      ].filter(Boolean).join("; ");108      const ent: NormalizedCountryStat = {109        entityType: "country", key: code, iso2: code,110        population, gdpUsd, electricityTwh, renewableShare,111        year: years.length ? Math.max(...years) : null,112        // isEstimate stays unset at entity level (population/GDP are measured); the derived field is flagged in method + note113        provenance: prov(r.url, electricityTwh != null ? "dataset-field:worldbank+derived-estimate:electricityTwh" : "dataset-field:worldbank", { note }),114      };115      out.push(ent);116    }117    ctx.log("info", `World Bank: ${out.length} country stats`);118    return out;119  }120}121