spb/datacenterindex

1/**2 * Wikidata connector — SPARQL (https://query.wikidata.org/sparql), items that are instances of3 * "data center" (Q671224) or one of its subclasses (P279*). Data is CC0.4 *5 * Note: the Q-id sometimes quoted for data center, Q1063421, is a French general; Q671224 is the class.6 * discover() runs one COUNT query and emits one page URL per `params.pageSize` items (LIMIT/OFFSET on an7 * ordered sub-select). Each page is one GET with the query in the URL; the result is grouped one row per8 * item (SAMPLE/GROUP_CONCAT) so multi-valued properties never duplicate a facility.9 */10import type { ConnectorContext, DiscoveredUrl, ExtractedRecord, RawDocument } from "@dci/connectors";11import type { Fact, FacilityStatus, FacilityType, NormalizedEntity, NormalizedFacility, Provenance } from "@dci/core";12import { inferFacilityType, validLatLng } from "@dci/core";13import { DatasetConnector, httpUrl, iso2, num, str, uniqNames } from "./shared.js";1415interface SparqlBinding { type: string; value: string; "xml:lang"?: string; datatype?: string }16interface SparqlResult { head: { vars: string[] }; results: { bindings: Array<Record<string, SparqlBinding>> } }1718const ACCEPT = "application/sparql-results+json";1920export class WikidataConnector extends DatasetConnector {21  private endpoint(): string { return String(this.param("endpoint", "https://query.wikidata.org/sparql")); }22  private classQid(): string { return String(this.param("classQid", "Q671224")); }23  private languages(): string { return String(this.param("labelLanguages", "en,fr,de,es,nl,it,pt,ja,zh,mul")); }24  private sparqlUrl(query: string): string { return `${this.endpoint()}?query=${encodeURIComponent(query.replace(/\s+/g, " ").trim())}`; }2526  private countQuery(): string { return `SELECT (COUNT(DISTINCT ?item) AS ?n) WHERE { ?item wdt:P31/wdt:P279* wd:${this.classQid()} . }`; }27  private pageQuery(limit: number, offset: number): string {28    return `SELECT ?item ?itemLabel ?itemDescription29      (SAMPLE(?coord) AS ?coord) (SAMPLE(?inception) AS ?inception) (SAMPLE(?incPrec) AS ?incPrec) (SAMPLE(?dissolved) AS ?dissolved)30      (GROUP_CONCAT(DISTINCT ?operatorLabel; separator="|") AS ?operators) (GROUP_CONCAT(DISTINCT ?ownerLabel; separator="|") AS ?owners)31      (GROUP_CONCAT(DISTINCT ?iso2; separator="|") AS ?iso2s) (SAMPLE(?website) AS ?website)32      (GROUP_CONCAT(DISTINCT ?locatedLabel; separator="|") AS ?located) (GROUP_CONCAT(DISTINCT ?classLabel; separator="|") AS ?classes)33      (GROUP_CONCAT(DISTINCT ?altLabel; separator="|") AS ?altLabels) (SAMPLE(?enwiki) AS ?enwiki)34    WHERE {35      { SELECT DISTINCT ?item WHERE { ?item wdt:P31/wdt:P279* wd:${this.classQid()} . } ORDER BY ?item LIMIT ${limit} OFFSET ${offset} }36      OPTIONAL { ?item wdt:P625 ?coord . }37      OPTIONAL { ?item p:P571/psv:P571 ?incNode . ?incNode wikibase:timeValue ?inception ; wikibase:timePrecision ?incPrec . }38      OPTIONAL { ?item wdt:P576 ?dissolved . }39      OPTIONAL { ?item wdt:P137 ?operator . }40      OPTIONAL { ?item wdt:P127 ?owner . }41      OPTIONAL { ?item wdt:P17 ?country . ?country wdt:P297 ?iso2 . }42      OPTIONAL { ?item wdt:P856 ?website . }43      OPTIONAL { ?item wdt:P131 ?located . }44      OPTIONAL { ?item wdt:P31 ?class . }45      OPTIONAL { ?item skos:altLabel ?altLabel . FILTER(LANG(?altLabel) = "en") }46      OPTIONAL { ?enwiki schema:about ?item ; schema:isPartOf <https://en.wikipedia.org/> . }47      SERVICE wikibase:label { bd:serviceParam wikibase:language "${this.languages()}". ?item rdfs:label ?itemLabel . ?item schema:description ?itemDescription . ?operator rdfs:label ?operatorLabel . ?owner rdfs:label ?ownerLabel . ?located rdfs:label ?locatedLabel . ?class rdfs:label ?classLabel . }48    } GROUP BY ?item ?itemLabel ?itemDescription`;49  }5051  async discover(ctx: ConnectorContext): Promise<DiscoveredUrl[]> {52    const pageSize = Math.max(100, Math.min(5000, this.param<number>("pageSize", 2000)));53    const countDoc = await ctx.fetch(this.sparqlUrl(this.countQuery()), { group: "dataset", accept: ACCEPT, timeoutMs: 90_000 });54    let total: number | null = null;55    if (!countDoc.error && countDoc.status === 200) {56      try { total = num((JSON.parse(countDoc.text) as SparqlResult).results.bindings[0]?.n?.value); } catch { total = null; }57    }58    if (total == null) { ctx.log("warn", `Wikidata COUNT query failed (${countDoc.error?.code ?? countDoc.status}); falling back to a single page`); total = pageSize; }59    const pages = Math.max(1, Math.ceil(total / pageSize));60    ctx.log("info", `Wikidata: ${total} items in class tree of ${this.classQid()} → ${pages} page(s) of ${pageSize}`);61    return Array.from({ length: pages }, (_, i) => ({ url: this.sparqlUrl(this.pageQuery(pageSize, i * pageSize)), group: "dataset", pageType: "dataset" as const, priority: 100 - i, meta: { page: i, pageSize, total } }));62  }6364  async fetch(ctx: ConnectorContext, u: DiscoveredUrl): Promise<RawDocument> {65    const doc = await ctx.fetch(u.url, { group: u.group, accept: ACCEPT, timeoutMs: 90_000 });66    doc.group = u.group; doc.pageType = "dataset"; doc.meta = { ...(doc.meta ?? {}), ...(u.meta ?? {}) };67    if (doc.status === 429) ctx.log("warn", `Wikidata SPARQL rate limited (retry-after ${doc.headers["retry-after"] ?? "?"}s)`);68    return doc;69  }7071  async extract(ctx: ConnectorContext, doc: RawDocument): Promise<ExtractedRecord[]> {72    if (doc.error || doc.notModified || doc.status >= 400 || !doc.body.length) return [];73    let res: SparqlResult;74    try { res = JSON.parse(doc.text) as SparqlResult; } catch { ctx.log("warn", `Wikidata page ${String(doc.meta?.page)}: invalid JSON`); return []; }75    const out: ExtractedRecord[] = [];76    let unlabeled = 0;77    for (const b of res.results?.bindings ?? []) {78      const qid = b.item?.value.split("/").pop();79      if (!qid || !/^Q\d+$/.test(qid)) continue;80      const label = str(b.itemLabel?.value);81      if (!label || label === qid) { unlabeled++; continue; } // no label in any configured language → not a usable name82      const data: Record<string, unknown> = { qid };83      for (const [k, v] of Object.entries(b)) data[k] = v.value;84      out.push({ kind: "facility", key: `wikidata:${qid}`, data, methods: { _all: "sparql", geo: "wikidata:P625" }, certainty: 1, url: `https://www.wikidata.org/wiki/${qid}`, pageType: "dataset" });85    }86    if (unlabeled) ctx.log("info", `Wikidata page ${String(doc.meta?.page)}: ${unlabeled} item(s) without a label skipped`);87    return out;88  }8990  async normalize(ctx: ConnectorContext, records: ExtractedRecord[]): Promise<NormalizedEntity[]> {91    const out: NormalizedEntity[] = [];92    const nowYear = new Date().getUTCFullYear();93    for (const r of records) {94      const d = r.data as Record<string, string | undefined>;95      const name = str(d.itemLabel); if (!name) continue;96      const prov = (method: string, extra: Partial<Provenance> = {}): Provenance => ctx.provenance(r.url, { method, extractorVersion: this.parserVersion, ...extra });97      const split = (s: string | undefined) => (s ?? "").split("|").map((x) => x.trim()).filter(Boolean);98      const coord = d.coord?.match(/Point\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i);99      const lng = coord ? Number(coord[1]) : null, lat = coord ? Number(coord[2]) : null;100      const isoList = split(d.iso2s).map((x) => iso2(x)).filter((x): x is string => Boolean(x));101      const operators = split(d.operators), owners = split(d.owners), located = split(d.located), classes = split(d.classes);102      let openedOn = partialFromWikidata(d.inception, num(d.incPrec));103      // P571 on a building/organisation item often predates the data center use (e.g. 1930s telephone exchanges) → not a DC opening date104      if (openedOn && Number(openedOn.slice(0, 4)) < 1950) { ctx.log("debug", `${d.qid}: ignoring inception ${openedOn} (< 1950)`); openedOn = null; }105      const dissolved = partialFromWikidata(d.dissolved, null);106      let status: FacilityStatus | null = null;107      if (dissolved) status = "closed";108      else if (openedOn && Number(openedOn.slice(0, 4)) > nowYear) status = "announced";109      const classText = classes.join(" ");110      const facilityType: FacilityType | null = /high performance computing|supercomputing/i.test(classText) ? "hpc" : /\bAI data center/i.test(classText) ? "ai" : /colocation/i.test(classText) ? "colocation" : inferFacilityType(`${name} ${d.itemDescription ?? ""} ${classText}`);111      const facts: Fact[] = [];112      if (openedOn) facts.push({ field: "openedOn", value: openedOn, provenance: prov("wikidata:P571") });113      if (lat != null && lng != null) facts.push({ field: "geo", value: { lat, lng }, provenance: prov("wikidata:P625") });114      const externalIds: Record<string, string | number> = { wikidata: d.qid! };115      if (d.enwiki) externalIds.wikipedia_en = d.enwiki;116      const ent: NormalizedFacility = {117        entityType: "facility", key: r.key, name,118        aliases: uniqNames(split(d.altLabels), [name]),119        operatorName: operators[0] ?? null, ownerName: owners[0] ?? null,120        city: located[0] ?? null, // P131 = located in the administrative territorial entity (municipality most of the time, sometimes a county/district)121        countryIso2: isoList.length === 1 ? isoList[0]! : null,122        geo: validLatLng(lat, lng) ? { lat: lat!, lng: lng!, precision: "street", source: "dataset:wikidata" } : null,123        status, facilityType, openedOn, website: httpUrl(d.website),124        externalIds, description: str(d.itemDescription), facts, provenance: prov("sparql:wikidata_dc_v1"),125      };126      // multiple operators / countries: the first operator is kept as operatorName, the country is left null when ambiguous; the full lists are kept in the description127      if (operators.length > 1 || isoList.length > 1) ent.description = [ent.description, operators.length > 1 ? `Operators (Wikidata P137): ${operators.join(", ")}` : null, isoList.length > 1 ? `Countries (Wikidata P17): ${isoList.join(", ")}` : null].filter(Boolean).join(" — ") || null;128      out.push(ent);129    }130    return out;131  }132}133134/** Wikidata time value + precision (9 = year, 10 = month, 11 = day; anything coarser → year only) → partial date string. */135function partialFromWikidata(value: string | undefined, precision: number | null): string | null {136  if (!value) return null;137  const m = value.match(/^([+-]?\d{4,})-(\d{2})-(\d{2})/);138  if (!m) return null;139  const year = Number(m[1]);140  if (!Number.isFinite(year) || year < 1000 || year > 2200) return null;141  const y = String(year).padStart(4, "0");142  if (precision != null && precision >= 11) return `${y}-${m[2]}-${m[3]}`;143  if (precision != null && precision === 10) return `${y}-${m[2]}`;144  return y; // unknown precision (simple value) or year precision: never claim a day Wikidata did not assert145}146