spb/company-atlas
Public
1#!/usr/bin/env python2"""Polite Wikidata harvester for the Company Atlas seed registry (docs/SEEDS.md).34Stages (each resumable; every SPARQL result is cached under data/seed/wikidata/sparql/<sha256>.json):5 candidates company-class × sitelink-band queries + per-country and per-industry boost queries → data/seed/candidates.json6 select normalise websites, drop generic hosts / duplicate domains, diversify (US ≤ 40 %, others ≤ 12 %, country minimums)7 → data/seed/selected.json8 details batched detail queries (labels, legal names, industries, HQ, coordinates, tickers, LEI, CIK, parent, logo, employees)9 → data/seed/details.json10 assemble importance + tiers + industry mapping + parent/domain conflicts → registry/companies/wikidata-<region>.ndjson + README.md11 all the four stages in sequence (default)1213Politeness: User-Agent CompanyAtlasBot/0.1, one query at a time, ≥ 2 s between queries, 60 s server timeout, retries with backoff.14"""15from __future__ import annotations1617import argparse18import csv19import hashlib20import json21import logging22import math23import re24import sys25import time26from collections import Counter, defaultdict27from datetime import UTC, datetime28from pathlib import Path29from typing import Any30from urllib.parse import quote, urlparse3132import httpx3334ROOT = Path(__file__).resolve().parents[1]35sys.path.insert(0, str(ROOT / "src"))3637from companyatlas.registry.industries import load_industries, map_industry, top_level_of, top_level_slugs38from companyatlas.urls import registrable_domain3940log = logging.getLogger("seed_wikidata")4142SPARQL_URL = "https://query.wikidata.org/sparql"43USER_AGENT = "CompanyAtlasBot/0.1 (contact@spboucher.ai)"44MIN_INTERVAL_S = 2.045TIMEOUT_S = 60.046MAX_TRIES = 647DATA_DIR = ROOT / "data" / "seed"48CACHE_DIR = DATA_DIR / "wikidata" / "sparql"49REGISTRY_DIR = ROOT / "registry"50OUT_DIR = REGISTRY_DIR / "companies"51COUNTRIES_CSV = REGISTRY_DIR / "countries.csv"5253# Wikidata classes queried with wdt:P31 (no P279* — the subclass tree of "business" is too broad to page). Banded classes are large.54CLASSES: dict[str, str] = {55 "Q4830453": "business", "Q6881511": "enterprise", "Q891723": "public company", "Q783794": "company", "Q1589009": "privately held company",56 "Q167037": "corporation", "Q18388277": "technology company", "Q1058914": "software company", "Q22687": "bank", "Q46970": "airline",57 "Q786820": "automobile manufacturer", "Q6500733": "pharmaceutical company", "Q210167": "video game developer",58 "Q1137109": "video game publisher", "Q1762059": "film production company", "Q18127": "record label", "Q2085381": "publisher",59 "Q613142": "law firm", "Q740752": "transport company", "Q2401749": "telecommunications company", "Q131734": "brewery",60 "Q507619": "retail chain", "Q936518": "aerospace manufacturer", "Q206361": "conglomerate", "Q249556": "railway company",61 "Q2005696": "real estate company", "Q708676": "shipyard", "Q1668024": "internet service", "Q1320047": "book publisher",62 "Q19967801": "online service",63}64BANDED_CLASSES = {"Q4830453", "Q6881511", "Q891723", "Q46970"}65# Country boosts run only when the class stage yielded fewer than BOOST_MARGIN × minimum candidates for that country.66BOOST_MARGIN = 2.067SITELINK_BANDS: list[tuple[int, int | None]] = [(80, None), (50, 80), (35, 50), (25, 35), (18, 25), (12, 18), (8, 12), (5, 8)]68MIN_SITELINKS = 569BOOST_MIN_SITELINKS = 270BOOST_LIMIT = 250071INDUSTRY_BOOST_LIMIT = 6007273# Diversification (spec: "seed diversified companies across US, Canada, Europe, UK, Japan, South Korea, India, Australia, LatAm, ME, Africa, SEA")74US_CAP_SHARE = 0.4075OTHER_CAP_SHARE = 0.1276UNKNOWN_COUNTRY_SHARE = 0.0377COUNTRY_MINIMUMS: dict[str, int] = {78 **dict.fromkeys(["CA", "GB", "DE", "FR", "JP", "KR", "IN", "AU"], 150),79 **dict.fromkeys(["BR", "MX", "AE", "SA", "ZA", "NG", "SG", "ID", "NL", "SE", "CH", "ES", "IT", "CN", "TW", "HK"], 60),80}81INDUSTRY_MINIMUM = 6082INDUSTRY_RESERVE = 150 # extra candidates per top-level industry fetched in the detail stage to satisfy INDUSTRY_MINIMUM83# English labels of Wikidata items commonly used as P452 (industry) values, per top-level industry, for the industry-boost queries.84INDUSTRY_BOOST_LABELS: dict[str, list[str]] = {85 "technology": ["information technology", "electronics industry", "consumer electronics", "computer hardware", "electronics"],86 "financial-services": ["financial services", "insurance", "banking", "asset management", "investment banking", "payment system"],87 "real-estate": ["real estate", "real estate development", "property management", "real estate industry"],88 "construction": ["construction", "construction industry", "civil engineering", "building construction", "engineering"],89 "retail": ["retail", "retailing", "supermarket", "department store", "wholesale", "grocery store"],90 "consumer-goods": ["consumer goods", "cosmetics industry", "toy industry", "furniture industry", "fast-moving consumer goods",91 "household goods", "luxury goods", "personal care", "home appliance"],92 "energy": ["energy industry", "energy", "electric power industry", "nuclear power", "oil industry", "petroleum industry",93 "renewable energy", "solar energy", "wind power"],94 "utilities": ["public utility", "electric utility", "water industry", "waste management", "electricity generation",95 "electric power distribution", "water supply"],96 "mining": ["mining", "mining industry", "steel industry", "metallurgy", "gold mining", "steelmaking", "metal industry"],97 "chemicals": ["chemical industry", "chemicals", "petrochemical industry", "specialty chemicals", "plastics industry", "fertilizer"],98 "materials": ["glass industry", "paper industry", "packaging industry", "pulp and paper industry", "forestry", "building material",99 "cement industry", "wood industry"],100 "manufacturing": ["manufacturing", "mechanical engineering", "industrial machinery", "machine industry", "electrical engineering",101 "machine tool", "industrial engineering", "heavy industry", "shipbuilding"],102 "automotive": ["automotive industry", "automobile", "automotive", "motor vehicle", "auto parts"],103 "aerospace-defense": ["aerospace industry", "arms industry", "defense industry", "aerospace", "aviation industry", "space industry",104 "defence industry"],105 "transportation": ["transport", "rail transport", "public transport", "logistics", "transportation", "freight transport", "shipping",106 "maritime transport", "railway"],107 "telecommunications": ["telecommunications industry", "telecommunications", "telecommunication", "mobile telephony",108 "internet service provider"],109 "media": ["mass media", "publishing", "advertising", "broadcasting", "entertainment industry", "film industry", "video game industry",110 "music industry", "media industry", "newspaper"],111 "healthcare": ["health care industry", "health care", "medical technology", "hospital", "pharmaceutical industry", "biotechnology",112 "medical device", "healthcare industry", "medical equipment"],113 "hospitality": ["hospitality industry", "hotel industry", "hotel", "restaurant", "hospitality", "catering", "restaurant chain"],114 "travel": ["tourism", "travel agency", "travel industry", "travel", "tourism industry", "online travel agency", "tour operator"],115 "education": ["education", "educational technology", "higher education", "e-learning", "education industry", "educational services"],116 "professional-services": ["professional services", "consulting", "management consulting", "accounting", "outsourcing", "staffing",117 "business services", "legal services", "information technology consulting", "human resources"],118 "agriculture": ["agriculture", "agribusiness", "agricultural industry", "forestry", "fishing industry", "food industry", "farming",119 "aquaculture", "agricultural machinery"],120}121122# Hosts that are never a company's own website (social profiles, blogs, stores, code forges, encyclopaedias, site builders …).123GENERIC_DOMAINS = {124 "facebook.com", "fb.com", "linkedin.com", "twitter.com", "x.com", "instagram.com", "youtube.com", "youtu.be", "wikipedia.org",125 "wikimedia.org", "wikidata.org", "blogspot.com", "blogspot.co.uk", "wordpress.com", "tumblr.com", "medium.com", "github.com", "gitlab.com",126 "sourceforge.net", "archive.org", "tiktok.com", "vk.com", "weibo.com", "t.me", "telegram.me", "bit.ly", "wix.com", "wixsite.com",127 "weebly.com", "squarespace.com", "webnode.com", "jimdo.com", "jimdosite.com", "godaddysites.com", "carrd.co", "notion.site", "substack.com",128 "patreon.com", "itch.io", "steampowered.com", "bandcamp.com", "soundcloud.com", "imdb.com", "myspace.com", "flickr.com", "pinterest.com",129 "twitch.tv", "discord.gg", "discord.com", "bilibili.com", "tistory.com", "ameblo.jp", "fc2.com", "livejournal.com", "geocities.com",130 "angelfire.com", "tripod.com", "netlify.app", "vercel.app", "herokuapp.com", "github.io", "pages.dev", "web.app", "firebaseapp.com",131 "glitch.me", "strikingly.com", "yolasite.com", "webs.com", "linktr.ee", "about.me", "crunchbase.com", "bloomberg.com", "sec.gov",132 "yelp.com", "tripadvisor.com", "foursquare.com", "goo.gl", "ow.ly", "tinyurl.com", "wa.me", "whatsapp.com", "line.me", "kakao.com",133 "spotify.com", "deezer.com", "vimeo.com", "dailymotion.com", "behance.net", "dribbble.com", "etsy.com", "ebay.com", "aliexpress.com",134 "taobao.com", "tmall.com", "rakuten.co.jp", "shopee.com", "mercadolibre.com", "google.co.uk", "googleusercontent.com", "gstatic.com",135 "webflow.io", "mystrikingly.com", "site123.me", "simplesite.com", "ucoz.ru", "narod.ru", "hatenablog.com", "note.com", "wixstatic.com",136 "shopify.com", "myshopify.com", "bigcartel.com", "storenvy.com", "yahoo.co.jp", "yahoo.com", "aol.com", "cargo.site", "format.com",137 "blogger.com", "mixi.jp", "naver.me", "cafe24.com", "modoo.at", "over-blog.com", "canalblog.com", "skyrock.com", "free.fr", "orange.fr",138 "wanadoo.fr", "pagesperso-orange.fr", "t-online.de", "web.de", "gmx.de", "chello.at", "bplaced.net", "beepworld.de", "npage.de",139 "altervista.org", "xoom.it", "libero.it", "interfree.it", "terra.com.br", "uol.com.br", "ig.com.br", "sapo.pt", "webcindario.com",140 "iespana.es", "galeon.com", "hpage.com", "wordpress.org", "js.org", "readthedocs.io", "gitbook.io", "gumroad.com", "ko-fi.com",141 "onlyfans.com", "reddit.com", "quora.com", "scribd.com", "issuu.com", "slideshare.net", "docs.google.com", "drive.google.com",142 "sites.google.com", "play.google.com", "apps.apple.com", "itunes.apple.com", "amazon.com", "amazon.co.uk", "amazon.de", "amazon.co.jp",143 "amazon.fr", "amazon.ca", "amazon.in", "amazon.com.br", "amzn.to", "microsoft.com", "apple.com", "google.com", "naver.com", "daum.net",144 "qq.com", "163.com", "sina.com.cn", "baidu.com", "sohu.com", "douyin.com", "kuaishou.com", "zhihu.com", "xiaohongshu.com",145}146# Registrable domains that are themselves seed companies: only the bare/www host counts as that company's site (not sub-brands/store pages).147PLATFORM_ROOTS = {"google.com": "www.google.com", "apple.com": "www.apple.com", "amazon.com": "www.amazon.com", "microsoft.com": "www.microsoft.com",148 "naver.com": "www.naver.com", "yahoo.com": "www.yahoo.com", "qq.com": "www.qq.com", "baidu.com": "www.baidu.com",149 "163.com": "www.163.com", "sohu.com": "www.sohu.com", "sina.com.cn": "www.sina.com.cn", "daum.net": "www.daum.net",150 "kakao.com": "www.kakao.com", "shopify.com": "www.shopify.com", "spotify.com": "www.spotify.com", "reddit.com": "www.reddit.com",151 "ebay.com": "www.ebay.com", "etsy.com": "www.etsy.com", "yelp.com": "www.yelp.com", "tripadvisor.com": "www.tripadvisor.com",152 "linkedin.com": "www.linkedin.com", "facebook.com": "www.facebook.com", "instagram.com": "www.instagram.com",153 "youtube.com": "www.youtube.com", "twitter.com": "twitter.com", "x.com": "x.com", "tiktok.com": "www.tiktok.com",154 "github.com": "github.com", "gitlab.com": "gitlab.com", "medium.com": "medium.com", "substack.com": "substack.com",155 "patreon.com": "www.patreon.com", "twitch.tv": "www.twitch.tv", "pinterest.com": "www.pinterest.com", "vimeo.com": "vimeo.com",156 "soundcloud.com": "soundcloud.com", "bandcamp.com": "bandcamp.com", "imdb.com": "www.imdb.com", "crunchbase.com": "www.crunchbase.com",157 "bloomberg.com": "www.bloomberg.com", "wix.com": "www.wix.com", "squarespace.com": "www.squarespace.com", "weebly.com": "www.weebly.com",158 "godaddy.com": "www.godaddy.com", "wordpress.com": "wordpress.com", "tumblr.com": "www.tumblr.com", "flickr.com": "www.flickr.com",159 "quora.com": "www.quora.com", "scribd.com": "www.scribd.com", "issuu.com": "issuu.com", "discord.com": "discord.com",160 "telegram.org": "telegram.org", "whatsapp.com": "www.whatsapp.com", "line.me": "line.me", "bilibili.com": "www.bilibili.com",161 "weibo.com": "weibo.com", "vk.com": "vk.com", "zhihu.com": "www.zhihu.com", "aliexpress.com": "www.aliexpress.com",162 "taobao.com": "www.taobao.com", "tmall.com": "www.tmall.com", "rakuten.co.jp": "www.rakuten.co.jp", "shopee.com": "shopee.com",163 "mercadolibre.com": "www.mercadolibre.com", "archive.org": "archive.org", "sourceforge.net": "sourceforge.net", "itch.io": "itch.io",164 "steampowered.com": "store.steampowered.com", "deezer.com": "www.deezer.com", "dailymotion.com": "www.dailymotion.com",165 "behance.net": "www.behance.net", "dribbble.com": "dribbble.com", "notion.so": "www.notion.so", "gumroad.com": "gumroad.com",166 "onlyfans.com": "onlyfans.com", "yahoo.co.jp": "www.yahoo.co.jp", "aol.com": "www.aol.com", "free.fr": "www.free.fr",167 "orange.fr": "www.orange.fr", "t-online.de": "www.t-online.de", "web.de": "web.de", "gmx.de": "www.gmx.de", "uol.com.br": "www.uol.com.br",168 "terra.com.br": "www.terra.com.br", "sapo.pt": "www.sapo.pt", "libero.it": "www.libero.it", "douyin.com": "www.douyin.com",169 "kuaishou.com": "www.kuaishou.com", "xiaohongshu.com": "www.xiaohongshu.com", "note.com": "note.com", "cafe24.com": "www.cafe24.com",170 "hatenablog.com": "hatenablog.com", "mixi.jp": "mixi.jp", "myspace.com": "myspace.com", "livejournal.com": "www.livejournal.com",171 "webflow.com": "webflow.com", "carrd.co": "carrd.co", "linktr.ee": "linktr.ee", "about.me": "about.me", "netlify.com": "www.netlify.com",172 "vercel.com": "vercel.com", "heroku.com": "www.heroku.com", "glitch.com": "glitch.com", "readthedocs.org": "readthedocs.org",173 "gitbook.com": "www.gitbook.com", "ko-fi.com": "ko-fi.com", "foursquare.com": "foursquare.com", "slideshare.net": "www.slideshare.net"}174175176# ------------------------------------------------------------------------------------------------------------ SPARQL client177class Sparql:178 def __init__(self, *, refresh: bool = False) -> None:179 self.client = httpx.Client(headers={"User-Agent": USER_AGENT, "Accept": "application/sparql-results+json"}, timeout=TIMEOUT_S + 15,180 follow_redirects=True)181 self.last_call = 0.0182 self.refresh = refresh183 self.queries = 0184 self.cached = 0185 CACHE_DIR.mkdir(parents=True, exist_ok=True)186187 def query(self, sparql: str, *, label: str = "") -> list[dict[str, str]]:188 key = hashlib.sha256(sparql.encode("utf-8")).hexdigest()189 path = CACHE_DIR / f"{key}.json"190 if path.exists() and not self.refresh:191 self.cached += 1192 return json.loads(path.read_text(encoding="utf-8"))["bindings"]193 delay = 5.0194 last_err = ""195 gateway_timeouts = 0196 for attempt in range(1, MAX_TRIES + 1):197 wait = MIN_INTERVAL_S - (time.monotonic() - self.last_call)198 if wait > 0:199 time.sleep(wait)200 t0 = time.monotonic()201 try:202 r = self.client.get(SPARQL_URL, params={"query": sparql, "format": "json"})203 self.last_call = time.monotonic()204 self.queries += 1205 if r.status_code == 200:206 try:207 # strict=False: a handful of Wikidata literals contain raw control characters.208 payload = json.loads(r.text, strict=False)209 except json.JSONDecodeError:210 # The endpoint streams results and, on a server-side timeout, appends a Java stack trace to a *200* body211 # (which the gateway may even cache). Treat a truncated body as a timeout so callers can split or skip.212 if "TimeoutException" in r.text or "SPARQL-QUERY" in r.text:213 raise TimeoutError(f"server timeout (truncated body) {label}") from None214 last_err = "truncated/invalid JSON body"215 continue216 rows = [{k: v["value"] for k, v in b.items()} for b in payload["results"]["bindings"]]217 path.write_text(json.dumps({"label": label, "fetched_at": datetime.now(UTC).isoformat(), "query": sparql, "bindings": rows},218 ensure_ascii=False), encoding="utf-8")219 log.info("sparql ok %s rows=%d %.1fs", label, len(rows), self.last_call - t0)220 return rows221 last_err = f"HTTP {r.status_code}: {r.text[:160]!r}"222 if r.status_code == 504:223 # Gateway timeout = the query ran past the 60 s server limit. One retry (load varies), then let the caller split it.224 gateway_timeouts += 1225 if gateway_timeouts >= 2:226 raise TimeoutError(f"gateway timeout {label}")227 if r.status_code == 429:228 retry_after = r.headers.get("Retry-After")229 delay = max(delay, float(retry_after)) if retry_after and retry_after.isdigit() else max(delay, 30.0)230 if r.status_code == 400:231 raise RuntimeError(f"bad query {label}: {r.text[:500]}")232 if r.status_code == 500 and "TimeoutException" in r.text:233 raise TimeoutError(f"server timeout {label}")234 except (httpx.TimeoutException, httpx.TransportError) as e:235 self.last_call = time.monotonic()236 last_err = f"{type(e).__name__}: {e}"237 log.warning("sparql retry %d/%d %s (%s) sleeping %.0fs", attempt, MAX_TRIES, label, last_err, delay)238 time.sleep(delay)239 delay = min(delay * 2, 120.0)240 raise RuntimeError(f"sparql failed {label}: {last_err}")241242243def values_clause(var: str, qids: list[str]) -> str:244 return f"VALUES ?{var} {{ {' '.join('wd:' + q for q in qids)} }}"245246247def class_values() -> str:248 return values_clause("cls", list(CLASSES))249250251# Truthy website (best rank, deprecated excluded). Dissolution is fetched as an OPTIONAL and filtered client-side: `FILTER NOT EXISTS`252# sub-selects over tens of thousands of bindings are what pushed the big class scans past the 60 s server limit.253WEBSITE_BLOCK = """254 ?item wdt:P856 ?web .255 OPTIONAL { ?item wdt:P576 ?dissolved }256 OPTIONAL { ?item wdt:P17 ?c . ?c wdt:P297 ?iso }257"""258259260def q_class_band(cls: str, lo: int, hi: int | None) -> str:261 band = f"?sl >= {lo}" + (f" && ?sl < {hi}" if hi else "")262 return f"""SELECT ?item ?sl ?web ?iso WHERE {{263 ?item wdt:P31 wd:{cls} ; wikibase:sitelinks ?sl .264 FILTER({band})265 {WEBSITE_BLOCK}266}}"""267268269def q_country_qids(isos: list[str]) -> str:270 vals = " ".join(json.dumps(x) for x in isos)271 return f"""SELECT ?iso ?country WHERE {{ VALUES ?iso {{ {vals} }} ?country wdt:P297 ?iso . FILTER NOT EXISTS {{ ?country wdt:P576 [] }} }}"""272273274def q_country_boost(iso: str, country_qid: str) -> str:275 """Country-first scan (P17 → website → class) with the optimizer pinned. Cheap for countries with few items in Wikidata; the276 caller only issues it for countries still short of their minimum after the class stage and tolerates a timeout."""277 return f"""SELECT ?item ?sl ?web ?iso WHERE {{278 hint:Query hint:optimizer "None" .279 ?item wdt:P17 wd:{country_qid} .280 ?item wdt:P856 ?web .281 ?item wdt:P31 ?cls .282 {class_values()}283 ?item wikibase:sitelinks ?sl . FILTER(?sl >= {BOOST_MIN_SITELINKS})284 OPTIONAL {{ ?item wdt:P576 ?dissolved }}285 BIND("{iso}" AS ?iso)286}}"""287288289def q_industry_boost(labels: list[str]) -> str:290 vals = " ".join(json.dumps(x) + "@en" for x in labels)291 return f"""SELECT ?item ?sl ?web ?iso ?indLabel WHERE {{292 hint:Query hint:optimizer "None" .293 VALUES ?indLabel {{ {vals} }}294 ?ind rdfs:label ?indLabel .295 ?item wdt:P452 ?ind .296 ?item wikibase:sitelinks ?sl .297 FILTER(?sl >= {BOOST_MIN_SITELINKS})298 ?item wdt:P31 ?cls .299 {class_values()}300 {WEBSITE_BLOCK}301}} ORDER BY DESC(?sl) LIMIT {INDUSTRY_BOOST_LIMIT}"""302303304def q_labels(qids: list[str]) -> str:305 return f"""SELECT ?item ?itemLabel ?itemDescription ?itemAltLabel WHERE {{306 {values_clause("item", qids)}307 SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en". }}308}}"""309310311# Plain (non-aggregated) detail queries, reduced client-side: a GROUP BY with a dozen SAMPLE() aggregates over nested OPTIONALs makes312# Blazegraph throw StackOverflowError.313def q_misc(qids: list[str]) -> str:314 return f"""SELECT ?item ?inception ?coord ?lei ?cik ?logo WHERE {{315 {values_clause("item", qids)}316 OPTIONAL {{ ?item wdt:P571 ?inception }}317 OPTIONAL {{ ?item wdt:P625 ?coord }}318 OPTIONAL {{ ?item wdt:P1278 ?lei }}319 OPTIONAL {{ ?item wdt:P5531 ?cik }}320 OPTIONAL {{ ?item wdt:P154 ?logo }}321}}"""322323324def q_hq_parent(qids: list[str]) -> str:325 return f"""SELECT ?item ?hq ?hqEnd ?hqLabel ?hqcoord ?hqRegionLabel ?hqiso ?parent ?parentEnd ?parentLabel WHERE {{326 {values_clause("item", qids)}327 OPTIONAL {{328 ?item p:P159 ?hqs . ?hqs ps:P159 ?hq .329 OPTIONAL {{ ?hqs pq:P582 ?hqEnd }}330 OPTIONAL {{ ?hq rdfs:label ?hqLabel FILTER(LANG(?hqLabel) = "en") }}331 OPTIONAL {{ ?hq wdt:P625 ?hqcoord }}332 OPTIONAL {{ ?hq wdt:P131 ?hqRegion . ?hqRegion rdfs:label ?hqRegionLabel FILTER(LANG(?hqRegionLabel) = "en") }}333 OPTIONAL {{ ?hq wdt:P17 ?hqc . ?hqc wdt:P297 ?hqiso }}334 }}335 OPTIONAL {{336 ?item p:P749 ?ps . ?ps ps:P749 ?parent .337 OPTIONAL {{ ?ps pq:P582 ?parentEnd }}338 OPTIONAL {{ ?parent rdfs:label ?parentLabel FILTER(LANG(?parentLabel) = "en") }}339 }}340}}"""341342343def q_names_industries(qids: list[str]) -> str:344 return f"""SELECT ?item ?legal ?indLabel WHERE {{345 {values_clause("item", qids)}346 OPTIONAL {{ ?item wdt:P1448 ?legal }}347 OPTIONAL {{ ?item wdt:P452 ?ind . ?ind rdfs:label ?indLabel FILTER(LANG(?indLabel) = "en") }}348}}"""349350351def q_tickers_employees(qids: list[str]) -> str:352 return f"""SELECT ?item ?ticker ?exchangeLabel ?ticker2 ?employees ?empDate WHERE {{353 {values_clause("item", qids)}354 OPTIONAL {{355 ?item p:P414 ?exs . ?exs ps:P414 ?exchange . FILTER NOT EXISTS {{ ?exs pq:P582 [] }}356 OPTIONAL {{ ?exs pq:P249 ?ticker }}357 OPTIONAL {{ ?exchange rdfs:label ?exchangeLabel FILTER(LANG(?exchangeLabel) = "en") }}358 }}359 OPTIONAL {{ ?item wdt:P249 ?ticker2 }}360 OPTIONAL {{ ?item p:P1128 ?es . ?es ps:P1128 ?employees . OPTIONAL {{ ?es pq:P585 ?empDate }} }}361}}"""362363364# ------------------------------------------------------------------------------------------------------------ helpers365def qid_of(uri: str) -> str:366 return uri.rsplit("/", 1)[-1]367368369def load_countries() -> dict[str, dict[str, str]]:370 with COUNTRIES_CSV.open(encoding="utf-8") as f:371 return {r["code"]: r for r in csv.DictReader(f)}372373374def normalise_website(url: str) -> tuple[str, str] | None:375 """→ (website `https://host`, registrable domain) or None when unusable/generic."""376 url = (url or "").strip()377 if not url:378 return None379 if "://" not in url:380 url = "https://" + url381 try:382 p = urlparse(url)383 except ValueError:384 return None385 host = (p.hostname or "").lower().strip().rstrip(".")386 if not host or "." not in host or re.fullmatch(r"[\d.]+", host) or ":" in host:387 return None388 if p.scheme not in ("http", "https"):389 return None390 try:391 host.encode("idna")392 except UnicodeError:393 return None394 dom = registrable_domain(host)395 if not dom or "." not in dom:396 return None397 path = (p.path or "/").rstrip("/")398 if dom in PLATFORM_ROOTS:399 canonical_host = PLATFORM_ROOTS[dom]400 if host not in (canonical_host, dom, "www." + dom) or path:401 return None402 return f"https://{canonical_host}", dom403 if dom in GENERIC_DOMAINS or host in GENERIC_DOMAINS:404 return None405 if host.endswith((".blogspot.com", ".wordpress.com", ".github.io", ".wixsite.com", ".weebly.com", ".tumblr.com")):406 return None407 return f"https://{host}", dom408409410def parse_point(value: str | None) -> tuple[float, float] | None:411 if not value or not value.startswith("Point("):412 return None413 try:414 lon, lat = value[6:-1].split()415 return round(float(lat), 5), round(float(lon), 5)416 except ValueError:417 return None418419420def parse_year(value: str | None) -> int | None:421 if not value:422 return None423 m = re.match(r"^(-?\d{1,4})-", value)424 if not m:425 return None426 y = int(m.group(1))427 return y if 1000 <= y <= datetime.now(UTC).year else None428429430def parse_int(value: str | None) -> int | None:431 try:432 return int(float(value)) if value not in (None, "") else None433 except ValueError:434 return None435436437def commons_url(filename: str | None) -> str | None:438 if not filename:439 return None440 name = filename.rsplit("/", 1)[-1].replace(" ", "_")441 return f"https://commons.wikimedia.org/wiki/Special:FilePath/{quote(name)}"442443444def dump_json(path: Path, data: Any) -> None:445 path.parent.mkdir(parents=True, exist_ok=True)446 path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")447448449def load_json(path: Path) -> Any:450 return json.loads(path.read_text(encoding="utf-8"))451452453# ------------------------------------------------------------------------------------------------------------ stage: candidates454def stage_candidates(sp: Sparql, *, refresh: bool = False) -> dict[str, dict[str, Any]]:455 out_path = DATA_DIR / "candidates.json"456 cands: dict[str, dict[str, Any]] = {}457 dissolved: set[str] = set()458459 def add(row: dict[str, str], *, cls: str | None = None, boost_country: str | None = None, boost_industry: str | None = None) -> None:460 if row.get("dissolved"):461 dissolved.add(qid_of(row["item"]))462 return463 qid = qid_of(row["item"])464 c = cands.setdefault(qid, {"sitelinks": 0, "websites": [], "isos": [], "classes": [], "boost_countries": [], "boost_industries": []})465 c["sitelinks"] = max(c["sitelinks"], parse_int(row.get("sl")) or 0)466 if row.get("web") and row["web"] not in c["websites"]:467 c["websites"].append(row["web"])468 iso = (row.get("iso") or "").upper()469 if iso and iso not in c["isos"]:470 c["isos"].append(iso)471 if cls and cls not in c["classes"]:472 c["classes"].append(cls)473 if boost_country and boost_country not in c["boost_countries"]:474 c["boost_countries"].append(boost_country)475 if boost_industry and boost_industry not in c["boost_industries"]:476 c["boost_industries"].append(boost_industry)477478 def run_band(cls: str, lo: int, hi: int | None) -> None:479 label = f"class {cls} {CLASSES[cls]} sl[{lo},{hi or '∞'})"480 try:481 rows = sp.query(q_class_band(cls, lo, hi), label=label)482 except TimeoutError:483 if hi is None:484 hi = 400485 if hi - lo <= 1:486 raise487 mid = (lo + hi) // 2488 log.warning("splitting band %s → [%d,%d) [%d,%d)", label, lo, mid, mid, hi)489 run_band(cls, lo, mid)490 run_band(cls, mid, hi)491 return492 for r in rows:493 add(r, cls=cls)494495 for cls in CLASSES:496 if cls in BANDED_CLASSES:497 for lo, hi in SITELINK_BANDS:498 run_band(cls, lo, hi)499 else:500 run_band(cls, MIN_SITELINKS, None)501 log.info("candidates so far: %d", len(cands))502 iso_to_qid = {r["iso"]: qid_of(r["country"]) for r in sp.query(q_country_qids(list(COUNTRY_MINIMUMS)), label="country qids")}503 per_country: Counter[str] = Counter(c["isos"][0] for c in cands.values() if c["isos"])504 for iso, minimum in COUNTRY_MINIMUMS.items():505 if per_country[iso] >= minimum * BOOST_MARGIN or iso not in iso_to_qid:506 log.info("country %s: %d candidates (min %d) — no boost", iso, per_country[iso], minimum)507 continue508 try:509 rows = sp.query(q_country_boost(iso, iso_to_qid[iso]), label=f"country boost {iso}")510 except (RuntimeError, TimeoutError) as e:511 log.warning("country boost %s failed (%s) — known gap, see README", iso, e)512 continue513 for r in rows:514 add(r, boost_country=iso)515 for slug, labels in INDUSTRY_BOOST_LABELS.items():516 for r in sp.query(q_industry_boost(labels), label=f"industry boost {slug}"):517 add(r, boost_industry=slug)518 for qid in dissolved:519 cands.pop(qid, None)520 dump_json(out_path, cands)521 log.info("candidates: %d (%d dissolved dropped) → %s", len(cands), len(dissolved), out_path)522 return cands523524525# ------------------------------------------------------------------------------------------------------------ stage: select526def prefilter(cands: dict[str, dict[str, Any]], countries: dict[str, dict[str, str]]) -> tuple[list[dict[str, Any]], dict[str, int]]:527 """Normalise websites, pick one country, drop generic hosts and duplicate registrable domains (highest sitelinks wins)."""528 stats: Counter[str] = Counter()529 rows: list[dict[str, Any]] = []530 for qid, c in cands.items():531 picked = None532 for w in c["websites"]:533 picked = normalise_website(w)534 if picked:535 break536 if not picked:537 stats["dropped_generic_or_invalid_website"] += 1538 continue539 website, dom = picked540 isos = [i for i in c["isos"] if i in countries]541 country = isos[0] if isos else None542 rows.append({"wikidata_id": qid, "sitelinks": c["sitelinks"], "website": website, "canonical_domain": dom, "country": country,543 "country_candidates": isos, "classes": c["classes"], "boost_industries": c["boost_industries"]})544 rows.sort(key=lambda r: (-r["sitelinks"], int(r["wikidata_id"][1:])))545 by_domain: dict[str, dict[str, Any]] = {}546 for r in rows:547 keeper = by_domain.get(r["canonical_domain"])548 if keeper is None:549 by_domain[r["canonical_domain"]] = r550 else:551 keeper.setdefault("domain_conflicts", []).append(r["wikidata_id"])552 stats["dropped_duplicate_domain"] += 1553 kept = list(by_domain.values())554 stats["kept"] = len(kept)555 return kept, dict(stats)556557558def select_diversified(rows: list[dict[str, Any]], *, target: int, countries: dict[str, dict[str, str]]) -> list[dict[str, Any]]:559 """Sitelinks-ordered selection with country caps and minimum coverage. Rows must be sorted by sitelinks desc."""560 caps: dict[str | None, int] = defaultdict(lambda: int(target * OTHER_CAP_SHARE))561 caps["US"] = int(target * US_CAP_SHARE)562 caps[None] = int(target * UNKNOWN_COUNTRY_SHARE)563 chosen: dict[str, dict[str, Any]] = {}564 per_country: Counter[str | None] = Counter()565566 def take(r: dict[str, Any]) -> None:567 chosen[r["wikidata_id"]] = r568 per_country[r["country"]] += 1569570 by_country: dict[str | None, list[dict[str, Any]]] = defaultdict(list)571 for r in rows:572 by_country[r["country"]].append(r)573 for iso, minimum in COUNTRY_MINIMUMS.items():574 for r in by_country.get(iso, [])[:minimum]:575 take(r)576 for r in rows:577 if len(chosen) >= target:578 break579 if r["wikidata_id"] in chosen or r["sitelinks"] < MIN_SITELINKS:580 continue581 if per_country[r["country"]] >= caps[r["country"]]:582 continue583 take(r)584 return list(chosen.values())585586587def stage_select(*, target: int) -> list[dict[str, Any]]:588 countries = load_countries()589 cands = load_json(DATA_DIR / "candidates.json")590 kept, stats = prefilter(cands, countries)591 log.info("prefilter: %s", stats)592 selected = select_diversified(kept, target=target, countries=countries)593 # Pool for the industry guarantee: best boost candidates per industry, fetched in the details stage too (≤ INDUSTRY_RESERVE each).594 chosen_ids = {r["wikidata_id"] for r in selected}595 reserve: list[dict[str, Any]] = []596 per_ind: Counter[str] = Counter()597 for r in kept:598 if r["wikidata_id"] in chosen_ids:599 continue600 for slug in r["boost_industries"]:601 if per_ind[slug] < INDUSTRY_RESERVE:602 per_ind[slug] += 1603 reserve.append(r)604 break605 dump_json(DATA_DIR / "selected.json", {"selected": selected, "reserve": reserve, "stats": stats, "target": target})606 log.info("selected %d (+%d reserve for industry minimums) → %s", len(selected), len(reserve), DATA_DIR / "selected.json")607 return selected608609610# ------------------------------------------------------------------------------------------------------------ stage: details611def stage_details(sp: Sparql, *, batch: int = 100) -> dict[str, dict[str, Any]]:612 sel = load_json(DATA_DIR / "selected.json")613 qids = [r["wikidata_id"] for r in sel["selected"]] + [r["wikidata_id"] for r in sel["reserve"]]614 out_path = DATA_DIR / "details.json"615 details: dict[str, dict[str, Any]] = load_json(out_path) if out_path.exists() else {}616 todo = [q for q in qids if q not in details]617 log.info("details: %d to fetch (%d cached)", len(todo), len(qids) - len(todo))618 for i in range(0, len(todo), batch):619 chunk = todo[i:i + batch]620 tag = f"details {i + len(chunk)}/{len(todo)}"621 d: dict[str, dict[str, Any]] = {q: {"legal_names": [], "industry_labels": [], "tickers": [], "employees_obs": []} for q in chunk}622 for r in sp.query(q_labels(chunk), label=f"{tag} labels"):623 q = qid_of(r["item"])624 d[q]["label"] = r.get("itemLabel")625 d[q]["description"] = r.get("itemDescription")626 d[q]["alt_labels"] = [a.strip() for a in (r.get("itemAltLabel") or "").split(",") if a.strip()]627 for r in sp.query(q_misc(chunk), label=f"{tag} misc"):628 m = d[qid_of(r["item"])]629 for key in ("inception", "coord", "lei", "cik", "logo"):630 if r.get(key) and (not m.get(key) or (key == "inception" and r[key] < m[key])):631 m[key] = r[key]632 grouped: dict[str, list[dict[str, str]]] = defaultdict(list)633 for r in sp.query(q_hq_parent(chunk), label=f"{tag} hq/parent"):634 grouped[qid_of(r["item"])].append(r)635 for q, rs in grouped.items():636 hqs = [r for r in rs if r.get("hq")]637 current = [r for r in hqs if not r.get("hqEnd")] or hqs638 if current:639 r = current[0]640 d[q].update({"hq": qid_of(r["hq"]), "hq_label": r.get("hqLabel"), "hq_coord": r.get("hqcoord"),641 "hq_region": r.get("hqRegionLabel"), "hq_iso": r.get("hqiso")})642 parents = [r for r in rs if r.get("parent") and not r.get("parentEnd")]643 if parents:644 d[q].update({"parent": qid_of(parents[0]["parent"]), "parent_label": parents[0].get("parentLabel")})645 for r in sp.query(q_names_industries(chunk), label=f"{tag} names/industries"):646 q = qid_of(r["item"])647 if r.get("legal") and r["legal"] not in d[q]["legal_names"]:648 d[q]["legal_names"].append(r["legal"])649 if r.get("indLabel") and r["indLabel"] not in d[q]["industry_labels"]:650 d[q]["industry_labels"].append(r["indLabel"])651 for r in sp.query(q_tickers_employees(chunk), label=f"{tag} tickers/employees"):652 q = qid_of(r["item"])653 t = r.get("ticker") or r.get("ticker2")654 if t:655 pair = [t, r.get("exchangeLabel")]656 if pair not in d[q]["tickers"]:657 d[q]["tickers"].append(pair)658 if r.get("employees"):659 obs = [r["employees"], r.get("empDate")]660 if obs not in d[q]["employees_obs"]:661 d[q]["employees_obs"].append(obs)662 details.update(d)663 dump_json(out_path, details)664 log.info("details: %d entries → %s", len(details), out_path)665 return details666667668# ------------------------------------------------------------------------------------------------------------ stage: assemble669def importance_score(sitelinks: int, employees: int | None, public: bool, ticker: str | None) -> float:670 s_sl = min(1.0, math.log1p(max(sitelinks, 0)) / math.log1p(300))671 s_emp = min(1.0, math.log10((employees or 0) + 1) / 6) if employees else 0.0672 score = 0.6 * s_sl + 0.2 * s_emp + 0.1 * (1 if public else 0) + 0.1 * (1 if ticker else 0)673 return round(min(1.0, max(0.02, score)), 4)674675676def assign_tiers(rows: list[dict[str, Any]]) -> None:677 rows.sort(key=lambda r: (-r["importance"], -r["sitelinks"], r["wikidata_id"]))678 for i, r in enumerate(rows):679 r["tier"] = 1 if i < 150 else 2 if i < 950 else 3 if i < 3450 else 4680681682def build_row(sel: dict[str, Any], d: dict[str, Any], harvested_at: str) -> dict[str, Any]:683 label = d.get("label") or ""684 if not label or re.fullmatch(r"Q\d+", label):685 label = (d.get("legal_names") or [sel["canonical_domain"]])[0]686 legal = next((x for x in d.get("legal_names", []) if re.search(r"[A-Za-z]", x)), None) or (d.get("legal_names") or [None])[0]687 aliases = [a for a in d.get("alt_labels", []) if a and a != label and len(a) <= 80][:8]688 emp = None689 obs = d.get("employees_obs") or []690 if obs:691 obs = sorted(obs, key=lambda o: (o[1] or ""), reverse=True)692 emp = parse_int(obs[0][0])693 tickers = d.get("tickers") or []694 ticker, exchange = (tickers[0][0], tickers[0][1]) if tickers else (None, None)695 public = "Q891723" in sel.get("classes", []) or bool(ticker) or bool(exchange)696 coord = parse_point(d.get("coord")) or parse_point(d.get("hq_coord"))697 country = sel["country"]698 if d.get("hq_iso") and d["hq_iso"] in (sel.get("country_candidates") or []):699 country = d["hq_iso"]700 industry_labels = d.get("industry_labels") or []701 slugs = map_industry(industry_labels)702 if not slugs and sel.get("boost_industries"):703 slugs = [sel["boost_industries"][0]]704 if not slugs:705 slugs = map_industry([label, d.get("description") or ""] + [CLASSES.get(c, "") for c in sel.get("classes", [])], limit=2)706 return {707 "wikidata_id": sel["wikidata_id"], "display_name": label.strip(), "legal_name": legal, "aliases": aliases, "website": sel["website"],708 "canonical_domain": sel["canonical_domain"], "country": country, "hq_city": d.get("hq_label"), "hq_region": d.get("hq_region"),709 "lat": coord[0] if coord else None, "lon": coord[1] if coord else None, "industries": slugs, "industry_labels": industry_labels[:8],710 "founded_year": parse_year(d.get("inception")), "employees": emp, "public_company": public, "ticker": ticker, "exchange": exchange,711 "lei": d.get("lei"), "sec_cik": str(d["cik"]).lstrip("0") or None if d.get("cik") else None,712 "parent": {"wikidata_id": d["parent"], "name": d.get("parent_label")} if d.get("parent") else None,713 "logo_url": commons_url(d.get("logo")), "description": (d.get("description") or None), "sitelinks": sel["sitelinks"],714 "importance": importance_score(sel["sitelinks"], emp, public, ticker), "tier": 4, "source": "wikidata", "harvested_at": harvested_at,715 "domain_conflicts": sel.get("domain_conflicts") or [],716 }717718719def stage_assemble() -> list[dict[str, Any]]:720 countries = load_countries()721 sel = load_json(DATA_DIR / "selected.json")722 details = load_json(DATA_DIR / "details.json")723 harvested_at = datetime.now(UTC).replace(microsecond=0).isoformat()724 selected = [r for r in sel["selected"] if r["wikidata_id"] in details]725 reserve = [r for r in sel["reserve"] if r["wikidata_id"] in details]726 rows = [build_row(s, details[s["wikidata_id"]], harvested_at) for s in selected]727 # Industry minimums: top up from the reserve where a top-level industry is below the floor.728 counts: Counter[str] = Counter()729 for r in rows:730 for top in {top_level_of(s) for s in r["industries"]}:731 counts[top] += 1732 chosen = {r["wikidata_id"] for r in rows}733 reserve_rows = [build_row(s, details[s["wikidata_id"]], harvested_at) for s in reserve]734 reserve_rows.sort(key=lambda r: -r["sitelinks"])735 for top in top_level_slugs():736 for rr in reserve_rows:737 if counts[top] >= INDUSTRY_MINIMUM:738 break739 if rr["wikidata_id"] in chosen or top not in {top_level_of(s) for s in rr["industries"]}:740 continue741 rows.append(rr)742 chosen.add(rr["wikidata_id"])743 for t in {top_level_of(s) for s in rr["industries"]}:744 counts[t] += 1745 # Parent/child sharing a domain: parent wins (regardless of sitelinks); duplicate domains (should not happen after prefilter) drop the lower.746 by_domain: dict[str, dict[str, Any]] = {}747 for r in sorted(rows, key=lambda r: -r["sitelinks"]):748 by_domain.setdefault(r["canonical_domain"], r)749 ids = {r["wikidata_id"] for r in by_domain.values()}750 dropped: list[dict[str, Any]] = []751 for r in list(by_domain.values()):752 p = r.get("parent")753 if p and p["wikidata_id"] in ids and p["wikidata_id"] != r["wikidata_id"]:754 parent_row = next((x for x in by_domain.values() if x["wikidata_id"] == p["wikidata_id"]), None)755 if parent_row and parent_row["canonical_domain"] == r["canonical_domain"]:756 dropped.append({"wikidata_id": r["wikidata_id"], "reason": "shares_parent_domain", "parent": p["wikidata_id"]})757 for r in rows:758 if r["wikidata_id"] not in ids:759 dropped.append({"wikidata_id": r["wikidata_id"], "reason": "duplicate_domain", "domain": r["canonical_domain"]})760 drop_ids = {x["wikidata_id"] for x in dropped}761 final = [r for r in by_domain.values() if r["wikidata_id"] not in drop_ids]762 for r in final:763 r["notes"] = [{"related_domain_conflict": q} for q in r.pop("domain_conflicts", [])] or []764 if not r["notes"]:765 r.pop("notes")766 assign_tiers(final)767 # Write per region.768 OUT_DIR.mkdir(parents=True, exist_ok=True)769 for old in OUT_DIR.glob("wikidata-*.ndjson"):770 old.unlink()771 by_region: dict[str, list[dict[str, Any]]] = defaultdict(list)772 for r in final:773 region = (countries.get(r["country"] or "", {}).get("region") or "other").lower().replace(" ", "-")774 by_region[region].append(r)775 for region, items in sorted(by_region.items()):776 items.sort(key=lambda r: (r["tier"], -r["importance"], r["wikidata_id"]))777 with (OUT_DIR / f"wikidata-{region}.ndjson").open("w", encoding="utf-8") as f:778 for r in items:779 f.write(json.dumps(r, ensure_ascii=False) + "\n")780 dump_json(DATA_DIR / "dropped.json", dropped)781 write_readme(final, countries)782 log.info("assembled %d companies into %d region files (%d dropped) → %s", len(final), len(by_region), len(dropped), OUT_DIR)783 return final784785786def write_readme(rows: list[dict[str, Any]], countries: dict[str, dict[str, str]]) -> None:787 names = {i.slug: i.name for i in load_industries()}788 by_country = Counter(r["country"] or "??" for r in rows)789 by_tier = Counter(r["tier"] for r in rows)790 by_top: Counter[str] = Counter()791 by_ind: Counter[str] = Counter()792 for r in rows:793 for s in r["industries"]:794 by_ind[s] += 1795 for t in {top_level_of(s) for s in r["industries"]}:796 by_top[t] += 1797 no_ind = sum(1 for r in rows if not r["industries"])798 public = sum(1 for r in rows if r["public_company"])799 region = Counter((countries.get(r["country"] or "", {}).get("region") or "other") for r in rows)800 generated = rows[0]["harvested_at"] if rows else ""801 intro = (f"Generated by `scripts/seed_wikidata.py` on {generated}. **{len(rows)} companies**, {public} public, {no_ind} without an "802 f"industry mapping, {len(by_country)} countries.")803 files_note = ("Files: one NDJSON per UN region (`wikidata-<region>.ndjson`), one JSON object per line — see `docs/SEEDS.md` for the "804 "schema, the diversification rules and how to add companies.")805 lines = [806 "# Seed registry — Wikidata harvest", "", intro, "", files_note, "",807 "## Tiers", "", "| Tier | Companies |", "|---|---|",808 *[f"| {t} ({ {1: 'global', 2: 'major', 3: 'notable', 4: 'long tail'}[t] }) | {by_tier[t]} |" for t in sorted(by_tier)], "",809 "## Regions", "", "| Region | Companies |", "|---|---|", *[f"| {k} | {v} |" for k, v in region.most_common()], "",810 "## Countries", "", "| Code | Country | Companies | Share |", "|---|---|---|---|",811 *[f"| {c} | {countries.get(c, {}).get('name', 'unknown')} | {n} | {100 * n / len(rows):.1f} % |" for c, n in by_country.most_common()], "",812 "## Top-level industries (a company counts once per top-level sector)", "", "| Industry | Companies |", "|---|---|",813 *[f"| {names.get(k, k)} | {v} |" for k, v in by_top.most_common()], "",814 "## All industries", "", "| Slug | Companies |", "|---|---|", *[f"| {k} | {v} |" for k, v in by_ind.most_common()], "",815 ]816 (OUT_DIR / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8")817818819# ------------------------------------------------------------------------------------------------------------ main820def main() -> int:821 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)822 ap.add_argument("stage", nargs="?", default="all", choices=["candidates", "select", "details", "assemble", "all"])823 ap.add_argument("--target", type=int, default=8000, help="companies to select before detail fetching (default 8000)")824 ap.add_argument("--batch", type=int, default=100, help="QIDs per detail query")825 ap.add_argument("--refresh", action="store_true", help="ignore the SPARQL cache")826 ap.add_argument("-v", "--verbose", action="store_true")827 args = ap.parse_args()828 logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(asctime)s %(levelname)s %(message)s")829 logging.getLogger("httpx").setLevel(logging.WARNING)830 DATA_DIR.mkdir(parents=True, exist_ok=True)831 sp = Sparql(refresh=args.refresh)832 t0 = time.monotonic()833 if args.stage in ("candidates", "all"):834 stage_candidates(sp)835 if args.stage in ("select", "all"):836 stage_select(target=args.target)837 if args.stage in ("details", "all"):838 stage_details(sp, batch=args.batch)839 if args.stage in ("assemble", "all"):840 stage_assemble()841 log.info("done in %.0fs (%d live queries, %d cached)", time.monotonic() - t0, sp.queries, sp.cached)842 return 0843844845if __name__ == "__main__":846 sys.exit(main())847