spb/company-atlas
Public
Python 66.3%
TypeScript 22.7%
JavaScript 8.6%
HTML 1.4%
CSS 0.7%
1#!/usr/bin/env python2"""Polite Wikidata harvester for the Company Atlas seed registry (docs/SEEDS.md).34Stages (each resumable; every SPARQL result is cached under data/seed/wikidata/sparql/<sha256>.json):5 candidates company-class × sitelink-band queries (sitelinks ≥ 3), every listed company (P414 per exchange / P249), companies with6 ≥ 500 employees (P1128) or a revenue (P2139) per class, per-country and per-industry boost queries → data/seed/candidates.json7 dissolved P576 / P582 check over every candidate → data/seed/dissolved.json (excluded by `select`)8 select normalise websites, drop generic hosts / duplicate domains, diversify (listed companies first, US ≤ 35 %, others ≤ 12 %,9 country minimums) → data/seed/selected.json10 details batched detail queries (labels, legal names, industries, HQ, coordinates, tickers, LEI, CIK, parent, logo, employees)11 → data/seed/details.json12 assemble importance + tiers + industry mapping + parent/domain conflicts → registry/companies/wikidata-<region>.ndjson + README.md13 all the four stages in sequence (default)1415Politeness: User-Agent CompanyAtlasBot/0.1, one query at a time, ≥ 2 s between queries, 60 s server timeout, retries with backoff.16"""17from __future__ import annotations1819import argparse20import csv21import hashlib22import json23import logging24import math25import re26import sys27import time28from collections import Counter, defaultdict29from datetime import UTC, datetime30from pathlib import Path31from typing import Any32from urllib.parse import quote, unquote, urlparse3334import httpx3536ROOT = Path(__file__).resolve().parents[1]37sys.path.insert(0, str(ROOT / "src"))3839from companyatlas.registry.industries import load_industries, map_industry, top_level_of, top_level_slugs40from companyatlas.urls import registrable_domain4142log = logging.getLogger("seed_wikidata")4344SPARQL_URL = "https://query.wikidata.org/sparql"45USER_AGENT = "CompanyAtlasBot/0.1 (contact@spboucher.ai)"46MIN_INTERVAL_S = 2.047TIMEOUT_S = 60.048MAX_TRIES = 649DATA_DIR = ROOT / "data" / "seed"50CACHE_DIR = DATA_DIR / "wikidata" / "sparql"51REGISTRY_DIR = ROOT / "registry"52OUT_DIR = REGISTRY_DIR / "companies"53COUNTRIES_CSV = REGISTRY_DIR / "countries.csv"5455# Wikidata classes queried with wdt:P31 (no P279* — the subclass tree of "business" is too broad to page). Banded classes are large.56# Labels are the English Wikidata labels (checked 2026-09-13: the 2026-09-12 list had six wrong labels, and three QIDs that are not57# company classes at all — Q708676 "charitable organization", Q1668024 "service on Internet", Q19967801 "online service" — were dropped).58CLASSES: dict[str, str] = {59 "Q4830453": "business", "Q6881511": "enterprise", "Q891723": "public company", "Q783794": "company", "Q1589009": "privately held company",60 "Q167037": "corporation", "Q18388277": "technology company", "Q1058914": "software company", "Q22687": "bank", "Q46970": "airline",61 "Q786820": "automobile manufacturer", "Q6500733": "printing company", "Q19644607": "pharmaceutical company", "Q210167": "video game developer",62 "Q1137109": "video game publisher", "Q1762059": "film production company", "Q18127": "record label", "Q2085381": "publishing house",63 "Q613142": "law firm", "Q740752": "transport company", "Q2401749": "telecommunications company", "Q131734": "brewery",64 "Q507619": "retail chain", "Q936518": "aerospace manufacturer", "Q206361": "concern", "Q778575": "conglomerate", "Q249556": "railway company",65 "Q2005696": "commercial vehicle manufacturer", "Q190928": "shipyard", "Q1320047": "book publisher",66 # Added 2026-09-13 for the 30 k harvest: the classes most used by listed / large companies outside the generic ones above.67 "Q134161": "joint-stock company", "Q1480166": "kabushiki gaisha", "Q60997538": "kōkai gaisha", "Q219577": "holding company",68 "Q650241": "financial institution", "Q730038": "credit institution", "Q2143354": "insurance company", "Q697852": "real estate investment trust",69 "Q658255": "subsidiary company", "Q270791": "state-owned enterprise", "Q161726": "multinational corporation", "Q18534542": "restaurant chain",70 "Q1631129": "hotel chain", "Q64027599": "gas station chain",71}72BANDED_CLASSES = {"Q4830453", "Q6881511", "Q891723", "Q46970"}73# Country boosts run only when the class stage yielded fewer than BOOST_MARGIN × minimum candidates for that country.74BOOST_MARGIN = 2.075SITELINK_BANDS: list[tuple[int, int | None]] = [(80, None), (50, 80), (35, 50), (25, 35), (18, 25), (12, 18), (8, 12), (5, 8), (3, 5)]76MIN_SITELINKS = 5 # single-query classes were harvested at ≥ 5 first (2026-09-12); the (3, 5) band is added for every class77LOW_BAND: tuple[int, int] = (3, 5)78SELECT_MIN_SITELINKS = 3 # class-band candidates need ≥ 3 sitelinks to be selected …79FALLBACK_MIN_SITELINKS = 2 # … unless the target is still short, then ≥ 2 (industry / country boosts reach down to 2)80EMPLOYEES_MIN = 500 # P1128 band: companies with at least this many employees, any sitelink count81BOOST_MIN_SITELINKS = 282BOOST_LIMIT = 250083INDUSTRY_BOOST_LIMIT = 60084EXCHANGE_GROUP_MAX = 1500 # listed-company queries group small exchanges until ≈ this many P414 statements85SPLIT_DEPTH_MAX = 2 # a query that still times out is split by the last (then the last two) digits of the QID86CLASS_BATCH = 300 # QIDs per P31 class query in the details stage8788# Non-company exclusion (assemble, 2026-09-13): an item carrying one of these P31 classes is dropped unless it has a current stock-exchange89# listing or a ticker (listed sports clubs such as Manchester United or Juventus stay). Companies = business / enterprise / company /90# public company / corporation / state-owned enterprise / cooperative / bank / insurer / airline / manufacturer / retailer … (CLASSES).91NON_COMPANY_CLASSES: dict[str, str] = {92 # museums, libraries, archives93 "Q33506": "museum", "Q207694": "art museum", "Q2087181": "historic house museum", "Q17431399": "national museum", "Q7075": "library",94 "Q28564": "public library", "Q26271642": "library network", "Q166118": "archives",95 # education96 "Q3918": "university", "Q3914": "school", "Q9826": "high school", "Q159334": "secondary school", "Q189004": "college",97 "Q2385804": "educational institution", "Q875538": "public university", "Q902104": "private university", "Q1336920": "community college",98 "Q269770": "boarding school", "Q2418495": "independent school", "Q615150": "land-grant university", "Q62078547": "public research university",99 "Q23002039": "public educational institution of the United States", "Q23002054": "private not-for-profit educational institution",100 "Q38723": "higher education institution", "Q1371037": "institute of technology", "Q3354859": "collegiate university",101 # government102 "Q327333": "government agency", "Q192350": "ministry", "Q732717": "law enforcement agency", "Q2659904": "government organization",103 "Q7188": "government", "Q4287745": "medical organization",104 # nonprofit sector105 "Q79913": "non-governmental organization", "Q708676": "charitable organization", "Q157031": "foundation", "Q163740": "nonprofit organization",106 "Q18325436": "501(c)(3) organization", "Q48204": "voluntary association", "Q15911314": "association", "Q829080": "professional association",107 "Q955824": "learned society", "Q155271": "think tank", "Q431603": "advocacy group", "Q1666019": "pressure group",108 "Q5774403": "historical society", "Q1021488": "community foundation", "Q1785733": "environmental organization",109 "Q1899015": "conservation organization", "Q336473": "aid agency", "Q4438121": "sports organization", "Q1530022": "religious organization",110 "Q94670589": "Christian organization", "Q7278": "political party", "Q178790": "labor union", "Q16917": "hospital",111 # sports clubs and teams (kept only when listed)112 "Q476028": "association football club", "Q847017": "sports club", "Q12973014": "sports team", "Q13393265": "basketball team",113 "Q14752149": "amateur football club", "Q18558301": "college sports team",114 # channels115 "Q17558136": "YouTube channel",116}117# Description rule: matches this (and none of the company words below) with no ticker, exchange, employee count or revenue → dropped.118NON_COMPANY_DESC_RE = re.compile(r"\b(museum|university|school|ministry|agency|charity|foundation|association|club|church|channel)\b", re.IGNORECASE)119COMPANY_DESC_RE = re.compile(120 r"\b(compan(?:y|ies)|corporation|manufacturer|enterprise|firm|conglomerate|retailer|bank|insurer|insurance|developer|publisher|publishing|producer|"121 r"operator|provider|chain|holding|startup|studio|label|brand|business|airline|carrier|maker|supplier|distributor|wholesaler|contractor|"122 r"consultancy|consulting|(?:advertising|travel|news|talent|marketing|staffing|recruitment|employment|real[- ]estate|estate|creative|design|"123 r"digital|literary|model(?:ing|ling)?|public relations|PR|media|shipping|rating|photo|press|ad|insurance|shipping)\s+agenc(?:y|ies))\b", re.IGNORECASE)124125# Diversification (spec: "seed diversified companies across US, Canada, Europe, UK, Japan, South Korea, India, Australia, LatAm, ME, Africa, SEA")126US_CAP_SHARE = 0.35127OTHER_CAP_SHARE = 0.12128UNKNOWN_COUNTRY_SHARE = 0.03129# Tiers by quantile of importance: top 1 % → tier 1 (≈ 300 at 30 k), next 6.67 % → tier 2 (≈ 2,000), next 26.67 % → tier 3 (≈ 8,000), rest 4.130TIER_SHARES: tuple[float, float, float] = (0.01, 0.0667, 0.2667)131COUNTRY_MINIMUMS: dict[str, int] = {132 **dict.fromkeys(["CA", "GB", "DE", "FR", "JP", "KR", "IN", "AU"], 150),133 **dict.fromkeys(["BR", "MX", "AE", "SA", "ZA", "NG", "SG", "ID", "NL", "SE", "CH", "ES", "IT", "CN", "TW", "HK"], 60),134}135# Share-of-target caps for narrow classes that Wikidata over-represents among high-sitelink items (airlines, record labels, publishers…).136CLASS_CAPS: dict[str, float] = {"Q46970": 0.05, "Q18127": 0.03, "Q210167": 0.04, "Q1137109": 0.02, "Q2085381": 0.03, "Q1320047": 0.02,137 "Q1762059": 0.03, "Q190928": 0.02, "Q131734": 0.02, "Q249556": 0.03}138INDUSTRY_MINIMUM = 60139INDUSTRY_RESERVE = 150 # extra candidates per top-level industry fetched in the detail stage to satisfy INDUSTRY_MINIMUM140# English labels of Wikidata items commonly used as P452 (industry) values, per top-level industry, for the industry-boost queries.141INDUSTRY_BOOST_LABELS: dict[str, list[str]] = {142 "technology": ["information technology", "electronics industry", "consumer electronics", "computer hardware", "electronics"],143 "financial-services": ["financial services", "insurance", "banking", "asset management", "investment banking", "payment system"],144 "real-estate": ["real estate", "real estate development", "property management", "real estate industry"],145 "construction": ["construction", "construction industry", "civil engineering", "building construction", "engineering"],146 "retail": ["retail", "retailing", "supermarket", "department store", "wholesale", "grocery store"],147 "consumer-goods": ["consumer goods", "cosmetics industry", "toy industry", "furniture industry", "fast-moving consumer goods",148 "household goods", "luxury goods", "personal care", "home appliance"],149 "energy": ["energy industry", "energy", "electric power industry", "nuclear power", "oil industry", "petroleum industry",150 "renewable energy", "solar energy", "wind power"],151 "utilities": ["public utility", "electric utility", "water industry", "waste management", "electricity generation",152 "electric power distribution", "water supply"],153 "mining": ["mining", "mining industry", "steel industry", "metallurgy", "gold mining", "steelmaking", "metal industry"],154 "chemicals": ["chemical industry", "chemicals", "petrochemical industry", "specialty chemicals", "plastics industry", "fertilizer"],155 "materials": ["glass industry", "paper industry", "packaging industry", "pulp and paper industry", "forestry", "building material",156 "cement industry", "wood industry"],157 "manufacturing": ["manufacturing", "mechanical engineering", "industrial machinery", "machine industry", "electrical engineering",158 "machine tool", "industrial engineering", "heavy industry", "shipbuilding"],159 "automotive": ["automotive industry", "automobile", "automotive", "motor vehicle", "auto parts"],160 "aerospace-defense": ["aerospace industry", "arms industry", "defense industry", "aerospace", "aviation industry", "space industry",161 "defence industry"],162 "transportation": ["transport", "rail transport", "public transport", "logistics", "transportation", "freight transport", "shipping",163 "maritime transport", "railway"],164 "telecommunications": ["telecommunications industry", "telecommunications", "telecommunication", "mobile telephony",165 "internet service provider"],166 "media": ["mass media", "publishing", "advertising", "broadcasting", "entertainment industry", "film industry", "video game industry",167 "music industry", "media industry", "newspaper"],168 "healthcare": ["health care industry", "health care", "medical technology", "hospital", "pharmaceutical industry", "biotechnology",169 "medical device", "healthcare industry", "medical equipment"],170 "hospitality": ["hospitality industry", "hotel industry", "hotel", "restaurant", "hospitality", "catering", "restaurant chain"],171 "travel": ["tourism", "travel agency", "travel industry", "travel", "tourism industry", "online travel agency", "tour operator"],172 "education": ["education", "educational technology", "higher education", "e-learning", "education industry", "educational services"],173 "professional-services": ["professional services", "consulting", "management consulting", "accounting", "outsourcing", "staffing",174 "business services", "legal services", "information technology consulting", "human resources"],175 "agriculture": ["agriculture", "agribusiness", "agricultural industry", "forestry", "fishing industry", "food industry", "farming",176 "aquaculture", "agricultural machinery"],177}178179# Hosts that are never a company's own website (social profiles, blogs, stores, code forges, encyclopaedias, site builders …).180GENERIC_DOMAINS = {181 "facebook.com", "fb.com", "linkedin.com", "twitter.com", "x.com", "instagram.com", "youtube.com", "youtu.be", "wikipedia.org",182 "wikimedia.org", "wikidata.org", "blogspot.com", "blogspot.co.uk", "wordpress.com", "tumblr.com", "medium.com", "github.com", "gitlab.com",183 "sourceforge.net", "archive.org", "tiktok.com", "vk.com", "weibo.com", "t.me", "telegram.me", "bit.ly", "wix.com", "wixsite.com",184 "weebly.com", "squarespace.com", "webnode.com", "jimdo.com", "jimdosite.com", "godaddysites.com", "carrd.co", "notion.site", "substack.com",185 "patreon.com", "itch.io", "steampowered.com", "bandcamp.com", "soundcloud.com", "imdb.com", "myspace.com", "flickr.com", "pinterest.com",186 "twitch.tv", "discord.gg", "discord.com", "bilibili.com", "tistory.com", "ameblo.jp", "fc2.com", "livejournal.com", "geocities.com",187 "angelfire.com", "tripod.com", "netlify.app", "vercel.app", "herokuapp.com", "github.io", "pages.dev", "web.app", "firebaseapp.com",188 "glitch.me", "strikingly.com", "yolasite.com", "webs.com", "linktr.ee", "about.me", "crunchbase.com", "bloomberg.com", "sec.gov",189 "yelp.com", "tripadvisor.com", "foursquare.com", "goo.gl", "ow.ly", "tinyurl.com", "wa.me", "whatsapp.com", "line.me", "kakao.com",190 "spotify.com", "deezer.com", "vimeo.com", "dailymotion.com", "behance.net", "dribbble.com", "etsy.com", "ebay.com", "aliexpress.com",191 "taobao.com", "tmall.com", "rakuten.co.jp", "shopee.com", "mercadolibre.com", "google.co.uk", "googleusercontent.com", "gstatic.com",192 "webflow.io", "mystrikingly.com", "site123.me", "simplesite.com", "ucoz.ru", "narod.ru", "hatenablog.com", "note.com", "wixstatic.com",193 "shopify.com", "myshopify.com", "bigcartel.com", "storenvy.com", "yahoo.co.jp", "yahoo.com", "aol.com", "cargo.site", "format.com",194 "blogger.com", "mixi.jp", "naver.me", "cafe24.com", "modoo.at", "over-blog.com", "canalblog.com", "skyrock.com", "free.fr", "orange.fr",195 "wanadoo.fr", "pagesperso-orange.fr", "t-online.de", "web.de", "gmx.de", "chello.at", "bplaced.net", "beepworld.de", "npage.de",196 "altervista.org", "xoom.it", "libero.it", "interfree.it", "terra.com.br", "uol.com.br", "ig.com.br", "sapo.pt", "webcindario.com",197 "iespana.es", "galeon.com", "hpage.com", "wordpress.org", "js.org", "readthedocs.io", "gitbook.io", "gumroad.com", "ko-fi.com",198 "onlyfans.com", "reddit.com", "quora.com", "scribd.com", "issuu.com", "slideshare.net", "docs.google.com", "drive.google.com",199 "sites.google.com", "play.google.com", "apps.apple.com", "itunes.apple.com", "amazon.com", "amazon.co.uk", "amazon.de", "amazon.co.jp",200 "amazon.fr", "amazon.ca", "amazon.in", "amazon.com.br", "amzn.to", "microsoft.com", "apple.com", "google.com", "naver.com", "daum.net",201 "qq.com", "163.com", "sina.com.cn", "baidu.com", "sohu.com", "douyin.com", "kuaishou.com", "zhihu.com", "xiaohongshu.com",202}203# Registrable domains that are themselves seed companies: only the bare/www host counts as that company's site (not sub-brands/store pages).204PLATFORM_ROOTS = {"google.com": "www.google.com", "apple.com": "www.apple.com", "amazon.com": "www.amazon.com", "microsoft.com": "www.microsoft.com",205 "naver.com": "www.naver.com", "yahoo.com": "www.yahoo.com", "qq.com": "www.qq.com", "baidu.com": "www.baidu.com",206 "163.com": "www.163.com", "sohu.com": "www.sohu.com", "sina.com.cn": "www.sina.com.cn", "daum.net": "www.daum.net",207 "kakao.com": "www.kakao.com", "shopify.com": "www.shopify.com", "spotify.com": "www.spotify.com", "reddit.com": "www.reddit.com",208 "ebay.com": "www.ebay.com", "etsy.com": "www.etsy.com", "yelp.com": "www.yelp.com", "tripadvisor.com": "www.tripadvisor.com",209 "linkedin.com": "www.linkedin.com", "facebook.com": "www.facebook.com", "instagram.com": "www.instagram.com",210 "youtube.com": "www.youtube.com", "twitter.com": "twitter.com", "x.com": "x.com", "tiktok.com": "www.tiktok.com",211 "github.com": "github.com", "gitlab.com": "gitlab.com", "medium.com": "medium.com", "substack.com": "substack.com",212 "patreon.com": "www.patreon.com", "twitch.tv": "www.twitch.tv", "pinterest.com": "www.pinterest.com", "vimeo.com": "vimeo.com",213 "soundcloud.com": "soundcloud.com", "bandcamp.com": "bandcamp.com", "imdb.com": "www.imdb.com", "crunchbase.com": "www.crunchbase.com",214 "bloomberg.com": "www.bloomberg.com", "wix.com": "www.wix.com", "squarespace.com": "www.squarespace.com", "weebly.com": "www.weebly.com",215 "godaddy.com": "www.godaddy.com", "wordpress.com": "wordpress.com", "tumblr.com": "www.tumblr.com", "flickr.com": "www.flickr.com",216 "quora.com": "www.quora.com", "scribd.com": "www.scribd.com", "issuu.com": "issuu.com", "discord.com": "discord.com",217 "telegram.org": "telegram.org", "whatsapp.com": "www.whatsapp.com", "line.me": "line.me", "bilibili.com": "www.bilibili.com",218 "weibo.com": "weibo.com", "vk.com": "vk.com", "zhihu.com": "www.zhihu.com", "aliexpress.com": "www.aliexpress.com",219 "taobao.com": "www.taobao.com", "tmall.com": "www.tmall.com", "rakuten.co.jp": "www.rakuten.co.jp", "shopee.com": "shopee.com",220 "mercadolibre.com": "www.mercadolibre.com", "archive.org": "archive.org", "sourceforge.net": "sourceforge.net", "itch.io": "itch.io",221 "steampowered.com": "store.steampowered.com", "deezer.com": "www.deezer.com", "dailymotion.com": "www.dailymotion.com",222 "behance.net": "www.behance.net", "dribbble.com": "dribbble.com", "notion.so": "www.notion.so", "gumroad.com": "gumroad.com",223 "onlyfans.com": "onlyfans.com", "yahoo.co.jp": "www.yahoo.co.jp", "aol.com": "www.aol.com", "free.fr": "www.free.fr",224 "orange.fr": "www.orange.fr", "t-online.de": "www.t-online.de", "web.de": "web.de", "gmx.de": "www.gmx.de", "uol.com.br": "www.uol.com.br",225 "terra.com.br": "www.terra.com.br", "sapo.pt": "www.sapo.pt", "libero.it": "www.libero.it", "douyin.com": "www.douyin.com",226 "kuaishou.com": "www.kuaishou.com", "xiaohongshu.com": "www.xiaohongshu.com", "note.com": "note.com", "cafe24.com": "www.cafe24.com",227 "hatenablog.com": "hatenablog.com", "mixi.jp": "mixi.jp", "myspace.com": "myspace.com", "livejournal.com": "www.livejournal.com",228 "webflow.com": "webflow.com", "carrd.co": "carrd.co", "linktr.ee": "linktr.ee", "about.me": "about.me", "netlify.com": "www.netlify.com",229 "vercel.com": "vercel.com", "heroku.com": "www.heroku.com", "glitch.com": "glitch.com", "readthedocs.org": "readthedocs.org",230 "gitbook.com": "www.gitbook.com", "ko-fi.com": "ko-fi.com", "foursquare.com": "foursquare.com", "slideshare.net": "www.slideshare.net"}231232233# ------------------------------------------------------------------------------------------------------------ SPARQL client234class Sparql:235 def __init__(self, *, refresh: bool = False) -> None:236 self.client = httpx.Client(headers={"User-Agent": USER_AGENT, "Accept": "application/sparql-results+json"}, timeout=TIMEOUT_S + 15,237 follow_redirects=True)238 self.last_call = 0.0239 self.refresh = refresh240 self.queries = 0241 self.cached = 0242 CACHE_DIR.mkdir(parents=True, exist_ok=True)243244 def query(self, sparql: str, *, label: str = "") -> list[dict[str, str]]:245 key = hashlib.sha256(sparql.encode("utf-8")).hexdigest()246 path = CACHE_DIR / f"{key}.json"247 if path.exists() and not self.refresh:248 self.cached += 1249 return json.loads(path.read_text(encoding="utf-8"))["bindings"]250 delay = 5.0251 last_err = ""252 gateway_timeouts = 0253 for attempt in range(1, MAX_TRIES + 1):254 wait = MIN_INTERVAL_S - (time.monotonic() - self.last_call)255 if wait > 0:256 time.sleep(wait)257 t0 = time.monotonic()258 try:259 r = self.client.get(SPARQL_URL, params={"query": sparql, "format": "json"})260 self.last_call = time.monotonic()261 self.queries += 1262 if r.status_code == 200:263 try:264 # strict=False: a handful of Wikidata literals contain raw control characters.265 payload = json.loads(r.text, strict=False)266 except json.JSONDecodeError:267 # The endpoint streams results and, on a server-side timeout, appends a Java stack trace to a *200* body268 # (which the gateway may even cache). Treat a truncated body as a timeout so callers can split or skip.269 if "TimeoutException" in r.text or "SPARQL-QUERY" in r.text:270 raise TimeoutError(f"server timeout (truncated body) {label}") from None271 last_err = "truncated/invalid JSON body"272 gateway_timeouts += 1273 if gateway_timeouts >= 2:274 raise TimeoutError(f"truncated body twice {label}") from None275 continue276 rows = [{k: v["value"] for k, v in b.items()} for b in payload["results"]["bindings"]]277 path.write_text(json.dumps({"label": label, "fetched_at": datetime.now(UTC).isoformat(), "query": sparql, "bindings": rows},278 ensure_ascii=False), encoding="utf-8")279 log.info("sparql ok %s rows=%d %.1fs", label, len(rows), self.last_call - t0)280 return rows281 last_err = f"HTTP {r.status_code}: {r.text[:160]!r}"282 if r.status_code == 504:283 # Gateway timeout = the query ran past the 60 s server limit. One retry (load varies), then let the caller split it.284 gateway_timeouts += 1285 if gateway_timeouts >= 2:286 raise TimeoutError(f"gateway timeout {label}")287 if r.status_code == 429:288 retry_after = r.headers.get("Retry-After")289 delay = max(delay, float(retry_after)) if retry_after and retry_after.isdigit() else max(delay, 30.0)290 if r.status_code == 400:291 raise RuntimeError(f"bad query {label}: {r.text[:500]}")292 if r.status_code == 500 and "TimeoutException" in r.text:293 raise TimeoutError(f"server timeout {label}")294 except (httpx.TimeoutException, httpx.TransportError) as e:295 self.last_call = time.monotonic()296 last_err = f"{type(e).__name__}: {e}"297 if isinstance(e, httpx.RemoteProtocolError):298 # "incomplete chunked read": the server cut the stream at its time limit — same as a timeout, fail fast.299 gateway_timeouts += 1300 if gateway_timeouts >= 2:301 raise TimeoutError(f"stream cut by server {label}") from None302 log.warning("sparql retry %d/%d %s (%s) sleeping %.0fs", attempt, MAX_TRIES, label, last_err, delay)303 time.sleep(delay)304 delay = min(delay * 2, 120.0)305 raise RuntimeError(f"sparql failed {label}: {last_err}")306307308def values_clause(var: str, qids: list[str]) -> str:309 return f"VALUES ?{var} {{ {' '.join('wd:' + q for q in qids)} }}"310311312def class_values() -> str:313 return values_clause("cls", list(CLASSES))314315316# Truthy website (best rank, deprecated excluded). Dissolution is fetched as an OPTIONAL and filtered client-side: `FILTER NOT EXISTS`317# sub-selects over tens of thousands of bindings are what pushed the big class scans past the 60 s server limit.318WEBSITE_BLOCK = """319 ?item wdt:P856 ?web .320 OPTIONAL { ?item wdt:P576 ?dissolved }321 OPTIONAL { ?item wdt:P17 ?c . ?c wdt:P297 ?iso }322"""323324325def q_class_band(cls: str, lo: int, hi: int | None, extra: str = "") -> str:326 band = f"?sl >= {lo}" + (f" && ?sl < {hi}" if hi else "")327 return f"""SELECT ?item ?sl ?web ?iso WHERE {{328 ?item wdt:P31 wd:{cls} ; wikibase:sitelinks ?sl .329 FILTER({band})330 {WEBSITE_BLOCK}{extra}331}}"""332333334def suffix_filter(suffix: str) -> str:335 """Split filter for queries that time out as a whole: keep the items whose QID ends with `suffix` (standard SPARQL, no modulo)."""336 return f' FILTER(STRENDS(STR(?item), "{suffix}"))\n' if suffix else ""337338339def q_exchanges() -> str:340 """Every stock exchange used as a P414 value with its statement count — drives the per-exchange listed-company scans."""341 return """SELECT ?ex ?exLabel (COUNT(?item) AS ?n) WHERE {342 ?item wdt:P414 ?ex .343 SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }344} GROUP BY ?ex ?exLabel ORDER BY DESC(?n)"""345346347def q_listed(exchange_qids: list[str], extra: str = "") -> str:348 """Companies listed on the given exchanges (any P31 class): the listing's end time (P582) is fetched so delisted-only items are349 treated as not listed client-side (they remain ordinary candidates through their other bands)."""350 return f"""SELECT ?item ?sl ?web ?iso ?ex ?end WHERE {{351 {values_clause("ex", exchange_qids)}352 ?item p:P414 ?st . ?st ps:P414 ?ex .353 OPTIONAL {{ ?st pq:P582 ?end }}354 ?item wikibase:sitelinks ?sl .355 {WEBSITE_BLOCK}{extra}356}}"""357358359def q_ticker_only(extra: str = "") -> str:360 """Items with a ticker symbol as a main statement (P249 is normally a qualifier of P414; a few dozen items carry it directly)."""361 return f"""SELECT ?item ?sl ?web ?iso ?ticker WHERE {{362 ?item wdt:P249 ?ticker ; wikibase:sitelinks ?sl .363 {WEBSITE_BLOCK}{extra}364}}"""365366367def q_class_measure(cls: str, prop: str, minimum: int | None, extra: str = "") -> str:368 """Companies of one class with a P1128 (employees ≥ minimum) or P2139 (revenue, any value) statement, any sitelink count. Class-369 restricted on purpose: 70 % of the items with a revenue on Wikidata are nonprofits, universities, hospitals or municipalities."""370 cond = f" FILTER(?val >= {minimum})\n" if minimum is not None else ""371 return f"""SELECT ?item ?sl ?web ?iso ?val WHERE {{372 ?item wdt:P31 wd:{cls} ; wdt:{prop} ?val ; wikibase:sitelinks ?sl .373{cond} {WEBSITE_BLOCK}{extra}374}}"""375376377def q_country_qids(isos: list[str]) -> str:378 vals = " ".join(json.dumps(x) for x in isos)379 return f"""SELECT ?iso ?country WHERE {{ VALUES ?iso {{ {vals} }} ?country wdt:P297 ?iso . FILTER NOT EXISTS {{ ?country wdt:P576 [] }} }}"""380381382def q_country_boost(iso: str, country_qid: str) -> str:383 """Country-first scan (P17 → website → class) with the optimizer pinned. Cheap for countries with few items in Wikidata; the384 caller only issues it for countries still short of their minimum after the class stage and tolerates a timeout."""385 return f"""SELECT ?item ?sl ?web ?iso WHERE {{386 hint:Query hint:optimizer "None" .387 ?item wdt:P17 wd:{country_qid} .388 ?item wdt:P856 ?web .389 ?item wdt:P31 ?cls .390 {class_values()}391 ?item wikibase:sitelinks ?sl . FILTER(?sl >= {BOOST_MIN_SITELINKS})392 OPTIONAL {{ ?item wdt:P576 ?dissolved }}393 BIND("{iso}" AS ?iso)394}}"""395396397def q_industry_boost(labels: list[str]) -> str:398 vals = " ".join(json.dumps(x) + "@en" for x in labels)399 return f"""SELECT ?item ?sl ?web ?iso ?indLabel WHERE {{400 hint:Query hint:optimizer "None" .401 VALUES ?indLabel {{ {vals} }}402 ?ind rdfs:label ?indLabel .403 ?item wdt:P452 ?ind .404 ?item wikibase:sitelinks ?sl .405 FILTER(?sl >= {BOOST_MIN_SITELINKS})406 ?item wdt:P31 ?cls .407 {class_values()}408 {WEBSITE_BLOCK}409}} ORDER BY DESC(?sl) LIMIT {INDUSTRY_BOOST_LIMIT}"""410411412def q_dissolved(qids: list[str]) -> str:413 """Only items that have a dissolution / abolition date (P576) or an end time (P582) as a main statement come back."""414 return f"""SELECT ?item ?dissolved WHERE {{ {values_clause("item", qids)} ?item wdt:P576|wdt:P582 ?dissolved . }}"""415416417def q_classes(qids: list[str]) -> str:418 """Every P31 class of the items — the non-company exclusion in `assemble` needs more than the class a candidate was matched on."""419 return f"""SELECT ?item ?cls WHERE {{ {values_clause("item", qids)} ?item wdt:P31 ?cls . }}"""420421422def q_labels(qids: list[str]) -> str:423 return f"""SELECT ?item ?itemLabel ?itemDescription ?itemAltLabel WHERE {{424 {values_clause("item", qids)}425 SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en". }}426}}"""427428429# Plain (non-aggregated) detail queries, reduced client-side: a GROUP BY with a dozen SAMPLE() aggregates over nested OPTIONALs makes430# Blazegraph throw StackOverflowError.431def q_misc(qids: list[str]) -> str:432 return f"""SELECT ?item ?inception ?coord ?lei ?cik ?logo WHERE {{433 {values_clause("item", qids)}434 OPTIONAL {{ ?item wdt:P571 ?inception }}435 OPTIONAL {{ ?item wdt:P625 ?coord }}436 OPTIONAL {{ ?item wdt:P1278 ?lei }}437 OPTIONAL {{ ?item wdt:P5531 ?cik }}438 OPTIONAL {{ ?item wdt:P154 ?logo }}439}}"""440441442def q_hq_parent(qids: list[str]) -> str:443 return f"""SELECT ?item ?hq ?hqEnd ?hqLabel ?hqcoord ?hqRegionLabel ?hqiso ?parent ?parentEnd ?parentLabel WHERE {{444 {values_clause("item", qids)}445 OPTIONAL {{446 ?item p:P159 ?hqs . ?hqs ps:P159 ?hq .447 OPTIONAL {{ ?hqs pq:P582 ?hqEnd }}448 OPTIONAL {{ ?hq rdfs:label ?hqLabel FILTER(LANG(?hqLabel) = "en") }}449 OPTIONAL {{ ?hq wdt:P625 ?hqcoord }}450 OPTIONAL {{ ?hq wdt:P131 ?hqRegion . ?hqRegion rdfs:label ?hqRegionLabel FILTER(LANG(?hqRegionLabel) = "en") }}451 OPTIONAL {{ ?hq wdt:P17 ?hqc . ?hqc wdt:P297 ?hqiso }}452 }}453 OPTIONAL {{454 ?item p:P749 ?ps . ?ps ps:P749 ?parent .455 OPTIONAL {{ ?ps pq:P582 ?parentEnd }}456 OPTIONAL {{ ?parent rdfs:label ?parentLabel FILTER(LANG(?parentLabel) = "en") }}457 }}458}}"""459460461def q_names_industries(qids: list[str]) -> str:462 return f"""SELECT ?item ?legal ?indLabel WHERE {{463 {values_clause("item", qids)}464 OPTIONAL {{ ?item wdt:P1448 ?legal }}465 OPTIONAL {{ ?item wdt:P452 ?ind . ?ind rdfs:label ?indLabel FILTER(LANG(?indLabel) = "en") }}466}}"""467468469def q_tickers_employees(qids: list[str]) -> str:470 return f"""SELECT ?item ?ticker ?exchangeLabel ?ticker2 ?employees ?empDate WHERE {{471 {values_clause("item", qids)}472 OPTIONAL {{473 ?item p:P414 ?exs . ?exs ps:P414 ?exchange . FILTER NOT EXISTS {{ ?exs pq:P582 [] }}474 OPTIONAL {{ ?exs pq:P249 ?ticker }}475 OPTIONAL {{ ?exchange rdfs:label ?exchangeLabel FILTER(LANG(?exchangeLabel) = "en") }}476 }}477 OPTIONAL {{ ?item wdt:P249 ?ticker2 }}478 OPTIONAL {{ ?item p:P1128 ?es . ?es ps:P1128 ?employees . OPTIONAL {{ ?es pq:P585 ?empDate }} }}479}}"""480481482# ------------------------------------------------------------------------------------------------------------ helpers483def qid_of(uri: str) -> str:484 return uri.rsplit("/", 1)[-1]485486487def load_countries() -> dict[str, dict[str, str]]:488 with COUNTRIES_CSV.open(encoding="utf-8") as f:489 return {r["code"]: r for r in csv.DictReader(f)}490491492def normalise_website(url: str) -> tuple[str, str] | None:493 """→ (website `https://host`, registrable domain) or None when unusable/generic."""494 url = (url or "").strip()495 if not url:496 return None497 if "://" not in url:498 url = "https://" + url499 try:500 p = urlparse(url)501 except ValueError:502 return None503 host = (p.hostname or "").lower().strip().rstrip(".")504 if not host or "." not in host or re.fullmatch(r"[\d.]+", host) or ":" in host:505 return None506 if p.scheme not in ("http", "https"):507 return None508 try:509 host.encode("idna")510 except UnicodeError:511 return None512 dom = registrable_domain(host)513 if not dom or "." not in dom:514 return None515 path = (p.path or "/").rstrip("/")516 if dom in PLATFORM_ROOTS:517 canonical_host = PLATFORM_ROOTS[dom]518 if host not in (canonical_host, dom, "www." + dom) or path:519 return None520 return f"https://{canonical_host}", dom521 if dom in GENERIC_DOMAINS or host in GENERIC_DOMAINS:522 return None523 if host.endswith((".blogspot.com", ".wordpress.com", ".github.io", ".wixsite.com", ".weebly.com", ".tumblr.com")):524 return None525 return f"https://{host}", dom526527528def parse_point(value: str | None) -> tuple[float, float] | None:529 if not value or not value.startswith("Point("):530 return None531 try:532 lon, lat = value[6:-1].split()533 return round(float(lat), 5), round(float(lon), 5)534 except ValueError:535 return None536537538def parse_year(value: str | None) -> int | None:539 if not value:540 return None541 m = re.match(r"^(-?\d{1,4})-", value)542 if not m:543 return None544 y = int(m.group(1))545 return y if 1000 <= y <= datetime.now(UTC).year else None546547548def parse_int(value: str | None) -> int | None:549 try:550 return int(float(value)) if value not in (None, "") else None551 except ValueError:552 return None553554555def commons_url(filename: str | None) -> str | None:556 """P154 arrives as an already-encoded Special:FilePath URL (or a bare file name): decode, then encode once."""557 if not filename:558 return None559 name = unquote(filename.rsplit("/", 1)[-1]).replace(" ", "_")560 return f"https://commons.wikimedia.org/wiki/Special:FilePath/{quote(name)}"561562563# Home exchanges per country: preferred listing when a company has several tickers (Wikidata order is arbitrary).564HOME_EXCHANGES: dict[str, tuple[str, ...]] = {565 "US": ("NASDAQ", "Nasdaq", "New York Stock Exchange", "NYSE"), "GB": ("London Stock Exchange",), "JP": ("Tokyo Stock Exchange",),566 "DE": ("Frankfurt Stock Exchange", "Xetra", "Börse"), "FR": ("Euronext Paris", "Paris"), "CA": ("Toronto Stock Exchange", "TSX Venture"),567 "KR": ("Korea Exchange", "KOSDAQ"), "IN": ("National Stock Exchange of India", "Bombay Stock Exchange"), "AU": ("Australian Securities Exchange",),568 "CN": ("Shanghai Stock Exchange", "Shenzhen Stock Exchange"), "HK": ("Hong Kong Stock Exchange",), "TW": ("Taiwan Stock Exchange", "Taipei"),569 "CH": ("SIX Swiss Exchange",), "NL": ("Euronext Amsterdam",), "BE": ("Euronext Brussels",), "SE": ("Nasdaq Stockholm", "Stockholm"),570 "FI": ("Nasdaq Helsinki", "Helsinki"), "DK": ("Nasdaq Copenhagen", "Copenhagen"), "NO": ("Oslo",), "IT": ("Borsa Italiana", "Euronext Milan"),571 "ES": ("Bolsa de Madrid", "Madrid"), "PT": ("Euronext Lisbon",), "BR": ("B3", "São Paulo"), "MX": ("Mexican Stock Exchange", "Bolsa Mexicana"),572 "SG": ("Singapore Exchange",), "ID": ("Indonesia Stock Exchange",), "MY": ("Bursa Malaysia",), "TH": ("Stock Exchange of Thailand",),573 "ZA": ("Johannesburg Stock Exchange", "JSE"), "SA": ("Saudi Stock Exchange", "Tadawul"), "AE": ("Dubai Financial Market", "Abu Dhabi"),574 "IL": ("Tel Aviv Stock Exchange",), "RU": ("Moscow Exchange",), "PL": ("Warsaw Stock Exchange",), "AT": ("Vienna Stock Exchange", "Wiener Börse"),575 "IE": ("Euronext Dublin", "Irish Stock Exchange"), "NZ": ("New Zealand Exchange", "NZX"), "AR": ("Buenos Aires",), "CL": ("Santiago",),576 "TR": ("Borsa Istanbul", "Istanbul"), "EG": ("Egyptian Exchange",), "NG": ("Nigerian", "Nigeria"), "PH": ("Philippine Stock Exchange",),577 "VN": ("Ho Chi Minh", "Hanoi"), "PK": ("Pakistan Stock Exchange",), "GR": ("Athens Stock Exchange",), "CZ": ("Prague Stock Exchange",),578 "HU": ("Budapest Stock Exchange",), "LU": ("Luxembourg Stock Exchange",), "KZ": ("Kazakhstan",), "QA": ("Qatar Stock Exchange",),579 "KW": ("Boursa Kuwait", "Kuwait"), "CO": ("Colombia",), "PE": ("Lima",),580}581582583def pick_ticker(tickers: list[list[str | None]], country: str | None) -> tuple[str | None, str | None]:584 """(ticker, exchange): home exchange of the company's country first, then any listing with an alphabetic ticker, then the first."""585 if not tickers:586 return None, None587 home = HOME_EXCHANGES.get(country or "", ())588 for t, ex in tickers:589 if ex and any(h.lower() in ex.lower() for h in home):590 return t, ex591 for t, ex in tickers:592 if t and not t.isdigit():593 return t, ex594 return tickers[0][0], tickers[0][1]595596597DOMAIN_LIKE_RE = re.compile(r"^(https?://)?[\w-]+(\.[\w-]+)+(/.*)?$")598JUNK_LABEL_RE = re.compile(r"classification|category|list of|wikimedia", re.IGNORECASE)599600601def dump_json(path: Path, data: Any) -> None:602 path.parent.mkdir(parents=True, exist_ok=True)603 path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")604605606def load_json(path: Path) -> Any:607 return json.loads(path.read_text(encoding="utf-8"))608609610STATS_PATH = DATA_DIR / "harvest-stats.json"611612613def update_stats(**sections: Any) -> dict[str, Any]:614 """Per-stage counters kept across runs (each stage overwrites its own section); rendered as the README harvest report."""615 stats = load_json(STATS_PATH) if STATS_PATH.exists() else {}616 stats.update(sections)617 dump_json(STATS_PATH, stats)618 return stats619620621# ------------------------------------------------------------------------------------------------------------ stage: candidates622def new_candidate() -> dict[str, Any]:623 return {"sitelinks": 0, "websites": [], "isos": [], "classes": [], "boost_countries": [], "boost_industries": [], "bands": [],624 "listed": False, "exchanges": [], "employees_hint": None, "has_revenue": False}625626627def stage_candidates(sp: Sparql, *, refresh: bool = False) -> dict[str, dict[str, Any]]:628 out_path = DATA_DIR / "candidates.json"629 cands: dict[str, dict[str, Any]] = {}630 dissolved: set[str] = set()631 band_rows: Counter[str] = Counter() # rows returned per band family (for the harvest report)632 band_new: Counter[str] = Counter() # candidates first seen in each band family633 exchange_labels: dict[str, str] = {}634635 def add(row: dict[str, str], *, band: str, cls: str | None = None, boost_country: str | None = None,636 boost_industry: str | None = None) -> None:637 band_rows[band] += 1638 if row.get("dissolved"):639 dissolved.add(qid_of(row["item"]))640 return641 qid = qid_of(row["item"])642 if qid not in cands:643 band_new[band] += 1644 c = cands.setdefault(qid, new_candidate())645 c["sitelinks"] = max(c["sitelinks"], parse_int(row.get("sl")) or 0)646 if row.get("web") and row["web"] not in c["websites"]:647 c["websites"].append(row["web"])648 iso = (row.get("iso") or "").upper()649 if iso and iso not in c["isos"]:650 c["isos"].append(iso)651 if cls and cls not in c["classes"]:652 c["classes"].append(cls)653 if boost_country and boost_country not in c["boost_countries"]:654 c["boost_countries"].append(boost_country)655 if boost_industry and boost_industry not in c["boost_industries"]:656 c["boost_industries"].append(boost_industry)657 if band not in c["bands"]:658 c["bands"].append(band)659 if band == "listed" and row.get("ex") and not row.get("end"):660 c["listed"] = True661 ex_label = exchange_labels.get(qid_of(row["ex"]), qid_of(row["ex"]))662 if ex_label not in c["exchanges"]:663 c["exchanges"].append(ex_label)664 if band == "ticker" and row.get("ticker"):665 c["listed"] = True666 if band == "employees" and parse_int(row.get("val")):667 c["employees_hint"] = max(c["employees_hint"] or 0, parse_int(row["val"]) or 0)668 if band == "revenue":669 c["has_revenue"] = True670671 def run_split(build: Any, label: str, handle: Any, suffix: str = "") -> None:672 """Run `build(extra)`; when the endpoint times out, split the query ten ways on the last QID digit (then the last two)."""673 try:674 rows = sp.query(build(suffix_filter(suffix)), label=label + (f" qid…{suffix}" if suffix else ""))675 except TimeoutError:676 if len(suffix) >= SPLIT_DEPTH_MAX:677 raise678 log.warning("splitting %s by QID suffix", label + (f" …{suffix}" if suffix else ""))679 for d in "0123456789":680 run_split(build, label, handle, d + suffix)681 return682 for r in rows:683 handle(r)684685 def run_band(cls: str, lo: int, hi: int | None) -> None:686 label = f"class {cls} {CLASSES[cls]} sl[{lo},{hi or '∞'})"687 try:688 rows = sp.query(q_class_band(cls, lo, hi), label=label)689 except TimeoutError:690 if hi is None:691 hi = 400692 if hi - lo <= 1:693 log.warning("band %s too large even at width 1 — splitting by QID suffix", label)694 run_split(lambda extra: q_class_band(cls, lo, hi, extra), label, lambda r: add(r, band="class", cls=cls))695 return696 mid = (lo + hi) // 2697 log.warning("splitting band %s → [%d,%d) [%d,%d)", label, lo, mid, mid, hi)698 run_band(cls, lo, mid)699 run_band(cls, mid, hi)700 return701 for r in rows:702 add(r, band="class", cls=cls)703704 # 1. Company classes × sitelink bands (≥ 5 as in the first harvest, then the (3, 5) band for every class).705 for cls in CLASSES:706 if cls in BANDED_CLASSES:707 for lo, hi in SITELINK_BANDS:708 run_band(cls, lo, hi)709 else:710 run_band(cls, MIN_SITELINKS, None)711 run_band(cls, *LOW_BAND)712 log.info("candidates so far: %d", len(cands))713 # 2. Every listed company: one scan per large exchange, small exchanges grouped (≈ EXCHANGE_GROUP_MAX statements per query).714 exchanges = sp.query(q_exchanges(), label="exchanges")715 exchange_labels.update({qid_of(r["ex"]): r.get("exLabel") or qid_of(r["ex"]) for r in exchanges})716 groups: list[list[str]] = []717 current: list[str] = []718 current_n = 0719 for r in exchanges:720 n = parse_int(r.get("n")) or 0721 if n >= EXCHANGE_GROUP_MAX:722 groups.append([qid_of(r["ex"])])723 continue724 if current and current_n + n > EXCHANGE_GROUP_MAX:725 groups.append(current)726 current, current_n = [], 0727 current.append(qid_of(r["ex"]))728 current_n += n729 if current:730 groups.append(current)731 for i, group in enumerate(groups, 1):732 names = ", ".join(exchange_labels.get(q, q) for q in group[:3]) + (f" +{len(group) - 3}" if len(group) > 3 else "")733 run_split(lambda extra, g=group: q_listed(g, extra), f"listed {i}/{len(groups)} {names}", lambda r: add(r, band="listed"))734 run_split(q_ticker_only, "ticker-only P249", lambda r: add(r, band="ticker"))735 log.info("candidates after listed companies: %d (%d listed)", len(cands), sum(1 for c in cands.values() if c["listed"]))736 # 3. Large companies by employees (P1128 ≥ EMPLOYEES_MIN) or with a revenue statement (P2139), per class, any sitelink count.737 for cls, cls_name in CLASSES.items():738 run_split(lambda extra, c=cls: q_class_measure(c, "P1128", EMPLOYEES_MIN, extra), f"employees≥{EMPLOYEES_MIN} {cls} {cls_name}",739 lambda r, c=cls: add(r, band="employees", cls=c))740 run_split(lambda extra, c=cls: q_class_measure(c, "P2139", None, extra), f"revenue {cls} {cls_name}",741 lambda r, c=cls: add(r, band="revenue", cls=c))742 log.info("candidates after employees/revenue bands: %d", len(cands))743 # 4. Boosts (country-first scans only where a minimum-coverage country is still short; industry label scans always).744 iso_to_qid = {r["iso"]: qid_of(r["country"]) for r in sp.query(q_country_qids(list(COUNTRY_MINIMUMS)), label="country qids")}745 per_country: Counter[str] = Counter(c["isos"][0] for c in cands.values() if c["isos"])746 boost_failures: list[str] = []747 for iso, minimum in COUNTRY_MINIMUMS.items():748 if per_country[iso] >= minimum * BOOST_MARGIN or iso not in iso_to_qid:749 log.info("country %s: %d candidates (min %d) — no boost", iso, per_country[iso], minimum)750 continue751 try:752 rows = sp.query(q_country_boost(iso, iso_to_qid[iso]), label=f"country boost {iso}")753 except (RuntimeError, TimeoutError) as e:754 log.warning("country boost %s failed (%s) — known gap, see README", iso, e)755 boost_failures.append(iso)756 continue757 for r in rows:758 add(r, band="country-boost", boost_country=iso)759 for slug, labels in INDUSTRY_BOOST_LABELS.items():760 for r in sp.query(q_industry_boost(labels), label=f"industry boost {slug}"):761 add(r, band="industry-boost", boost_industry=slug)762 for qid in dissolved:763 cands.pop(qid, None)764 dump_json(out_path, cands)765 update_stats(candidates={766 "total": len(cands), "dissolved_inline": len(dissolved), "listed": sum(1 for c in cands.values() if c["listed"]),767 "classes": len(CLASSES), "exchanges": len(exchanges), "exchange_queries": len(groups), "band_rows": dict(band_rows),768 "band_new": dict(band_new), "country_boost_failures": boost_failures,769 "live_queries": sp.queries, "cached_queries": sp.cached, "at": datetime.now(UTC).replace(microsecond=0).isoformat(),770 })771 log.info("candidates: %d (%d dissolved dropped; bands %s) → %s", len(cands), len(dissolved), dict(band_new), out_path)772 return cands773774775# ------------------------------------------------------------------------------------------------------------ stage: dissolved776def stage_dissolved(sp: Sparql, *, batch: int = 300) -> set[str]:777 """P576 (dissolved/abolished) check over every candidate — the candidate queries cannot afford a NOT EXISTS filter."""778 cands = load_json(DATA_DIR / "candidates.json")779 qids = sorted(cands, key=lambda q: int(q[1:]))780 dissolved: dict[str, str] = {}781 for i in range(0, len(qids), batch):782 chunk = qids[i:i + batch]783 for r in sp.query(q_dissolved(chunk), label=f"dissolved {i + len(chunk)}/{len(qids)}"):784 dissolved.setdefault(qid_of(r["item"]), r.get("dissolved") or "")785 dump_json(DATA_DIR / "dissolved.json", dissolved)786 update_stats(dissolved={"candidates": len(qids), "dissolved": len(dissolved), "queries": (len(qids) + batch - 1) // batch})787 log.info("dissolved: %d of %d candidates → %s", len(dissolved), len(qids), DATA_DIR / "dissolved.json")788 return set(dissolved)789790791# ------------------------------------------------------------------------------------------------------------ stage: select792def prefilter(cands: dict[str, dict[str, Any]], countries: dict[str, dict[str, str]]) -> tuple[list[dict[str, Any]], dict[str, int]]:793 """Normalise websites, pick one country, drop generic hosts and duplicate registrable domains (highest sitelinks wins)."""794 stats: Counter[str] = Counter()795 rows: list[dict[str, Any]] = []796 for qid, c in cands.items():797 picked = None798 for w in c["websites"]:799 picked = normalise_website(w)800 if picked:801 break802 if not picked:803 stats["dropped_generic_or_invalid_website"] += 1804 continue805 website, dom = picked806 isos = [i for i in c["isos"] if i in countries]807 country = isos[0] if isos else None808 rows.append({"wikidata_id": qid, "sitelinks": c["sitelinks"], "website": website, "canonical_domain": dom, "country": country,809 "country_candidates": isos, "classes": c["classes"], "boost_industries": c["boost_industries"],810 "bands": c.get("bands") or [], "listed": bool(c.get("listed")), "exchanges": c.get("exchanges") or [],811 "employees_hint": c.get("employees_hint"), "has_revenue": bool(c.get("has_revenue"))})812 rows.sort(key=lambda r: (-r["sitelinks"], int(r["wikidata_id"][1:])))813 by_domain: dict[str, dict[str, Any]] = {}814 for r in rows:815 keeper = by_domain.get(r["canonical_domain"])816 if keeper is None:817 by_domain[r["canonical_domain"]] = r818 else:819 keeper.setdefault("domain_conflicts", []).append(r["wikidata_id"])820 stats["dropped_duplicate_domain"] += 1821 kept = list(by_domain.values())822 stats["kept"] = len(kept)823 return kept, dict(stats)824825826def eligible(r: dict[str, Any], min_sitelinks: int) -> bool:827 """Listed companies and large companies (employees / revenue bands) qualify at any sitelink count; class-band and boost828 candidates need `min_sitelinks`."""829 if r.get("listed") or r.get("employees_hint") or r.get("has_revenue"):830 return True831 return r["sitelinks"] >= min_sitelinks832833834def select_diversified(rows: list[dict[str, Any]], *, target: int, countries: dict[str, dict[str, str]]) -> tuple[list[dict[str, Any]], dict[str, int]]:835 """Diversified selection: country minimums first, then every listed company, then the rest in sitelink order (≥ 3, then ≥ 2 if the836 target is still short) — all under the country and class caps. Rows must be sorted by sitelinks desc."""837 caps: dict[str | None, int] = defaultdict(lambda: int(target * OTHER_CAP_SHARE))838 caps["US"] = int(target * US_CAP_SHARE)839 caps[None] = int(target * UNKNOWN_COUNTRY_SHARE)840 chosen: dict[str, dict[str, Any]] = {}841 per_country: Counter[str | None] = Counter()842 per_class: Counter[str] = Counter()843 passes: dict[str, int] = {}844845 def take(r: dict[str, Any]) -> None:846 chosen[r["wikidata_id"]] = r847 per_country[r["country"]] += 1848 for c in r.get("classes", []):849 per_class[c] += 1850851 def class_capped(r: dict[str, Any]) -> bool:852 return any(per_class[c] >= int(target * CLASS_CAPS[c]) for c in r.get("classes", []) if c in CLASS_CAPS)853854 def sweep(name: str, pool: list[dict[str, Any]], min_sitelinks: int) -> None:855 before = len(chosen)856 for r in pool:857 if len(chosen) >= target:858 break859 if r["wikidata_id"] in chosen or not eligible(r, min_sitelinks):860 continue861 if per_country[r["country"]] >= caps[r["country"]] or class_capped(r):862 continue863 take(r)864 passes[name] = len(chosen) - before865866 by_country: dict[str | None, list[dict[str, Any]]] = defaultdict(list)867 for r in rows:868 by_country[r["country"]].append(r)869 for iso, minimum in COUNTRY_MINIMUMS.items():870 for r in [x for x in by_country.get(iso, []) if eligible(x, FALLBACK_MIN_SITELINKS)][:minimum]:871 take(r)872 passes["country_minimums"] = len(chosen)873 sweep("listed", [r for r in rows if r.get("listed")], SELECT_MIN_SITELINKS)874 sweep(f"sitelinks_ge_{SELECT_MIN_SITELINKS}", rows, SELECT_MIN_SITELINKS)875 if len(chosen) < target:876 sweep(f"sitelinks_ge_{FALLBACK_MIN_SITELINKS}", rows, FALLBACK_MIN_SITELINKS)877 return list(chosen.values()), passes878879880def stage_select(*, target: int) -> list[dict[str, Any]]:881 countries = load_countries()882 cands = load_json(DATA_DIR / "candidates.json")883 dissolved_path = DATA_DIR / "dissolved.json"884 if dissolved_path.exists():885 dissolved = set(load_json(dissolved_path))886 cands = {q: c for q, c in cands.items() if q not in dissolved}887 log.info("select: %d dissolved candidates excluded", len(dissolved))888 else:889 log.warning("select: no dissolved.json — run the `dissolved` stage first to exclude defunct companies")890 kept, stats = prefilter(cands, countries)891 log.info("prefilter: %s", stats)892 selected, passes = select_diversified(kept, target=target, countries=countries)893 log.info("selection passes: %s", passes)894 stats["listed_after_prefilter"] = sum(1 for r in kept if r["listed"])895 stats["listed_selected"] = sum(1 for r in selected if r["listed"])896 stats["passes"] = passes897 stats["target"] = target898 # Pool for the industry guarantee: best boost candidates per industry, fetched in the details stage too (≤ INDUSTRY_RESERVE each).899 chosen_ids = {r["wikidata_id"] for r in selected}900 reserve: list[dict[str, Any]] = []901 per_ind: Counter[str] = Counter()902 for r in kept:903 if r["wikidata_id"] in chosen_ids:904 continue905 for slug in r["boost_industries"]:906 if per_ind[slug] < INDUSTRY_RESERVE:907 per_ind[slug] += 1908 reserve.append(r)909 break910 dump_json(DATA_DIR / "selected.json", {"selected": selected, "reserve": reserve, "stats": stats, "target": target})911 update_stats(select=stats)912 log.info("selected %d (+%d reserve for industry minimums) → %s", len(selected), len(reserve), DATA_DIR / "selected.json")913 return selected914915916# ------------------------------------------------------------------------------------------------------------ stage: details917def stage_details(sp: Sparql, *, batch: int = 100) -> dict[str, dict[str, Any]]:918 sel = load_json(DATA_DIR / "selected.json")919 qids = [r["wikidata_id"] for r in sel["selected"]] + [r["wikidata_id"] for r in sel["reserve"]]920 out_path = DATA_DIR / "details.json"921 details: dict[str, dict[str, Any]] = load_json(out_path) if out_path.exists() else {}922 todo = [q for q in qids if q not in details]923 log.info("details: %d to fetch (%d cached)", len(todo), len(qids) - len(todo))924 for i in range(0, len(todo), batch):925 chunk = todo[i:i + batch]926 tag = f"details {i + len(chunk)}/{len(todo)}"927 d: dict[str, dict[str, Any]] = {q: {"legal_names": [], "industry_labels": [], "tickers": [], "employees_obs": []} for q in chunk}928 for r in sp.query(q_labels(chunk), label=f"{tag} labels"):929 q = qid_of(r["item"])930 d[q]["label"] = r.get("itemLabel")931 d[q]["description"] = r.get("itemDescription")932 d[q]["alt_labels"] = [a.strip() for a in (r.get("itemAltLabel") or "").split(",") if a.strip()]933 for r in sp.query(q_misc(chunk), label=f"{tag} misc"):934 m = d[qid_of(r["item"])]935 for key in ("inception", "coord", "lei", "cik", "logo"):936 if r.get(key) and (not m.get(key) or (key == "inception" and r[key] < m[key])):937 m[key] = r[key]938 grouped: dict[str, list[dict[str, str]]] = defaultdict(list)939 for r in sp.query(q_hq_parent(chunk), label=f"{tag} hq/parent"):940 grouped[qid_of(r["item"])].append(r)941 for q, rs in grouped.items():942 hqs = [r for r in rs if r.get("hq")]943 current = [r for r in hqs if not r.get("hqEnd")] or hqs944 if current:945 r = current[0]946 d[q].update({"hq": qid_of(r["hq"]), "hq_label": r.get("hqLabel"), "hq_coord": r.get("hqcoord"),947 "hq_region": r.get("hqRegionLabel"), "hq_iso": r.get("hqiso")})948 parents = [r for r in rs if r.get("parent") and not r.get("parentEnd")]949 if parents:950 d[q].update({"parent": qid_of(parents[0]["parent"]), "parent_label": parents[0].get("parentLabel")})951 for r in sp.query(q_names_industries(chunk), label=f"{tag} names/industries"):952 q = qid_of(r["item"])953 if r.get("legal") and r["legal"] not in d[q]["legal_names"]:954 d[q]["legal_names"].append(r["legal"])955 if r.get("indLabel") and r["indLabel"] not in d[q]["industry_labels"]:956 d[q]["industry_labels"].append(r["indLabel"])957 for r in sp.query(q_tickers_employees(chunk), label=f"{tag} tickers/employees"):958 q = qid_of(r["item"])959 t = r.get("ticker") or r.get("ticker2")960 if t:961 pair = [t, r.get("exchangeLabel")]962 if pair not in d[q]["tickers"]:963 d[q]["tickers"].append(pair)964 if r.get("employees"):965 obs = [r["employees"], r.get("empDate")]966 if obs not in d[q]["employees_obs"]:967 d[q]["employees_obs"].append(obs)968 details.update(d)969 dump_json(out_path, details)970 # Full P31 class lists (300 QIDs per query) for every item that lacks them — feeds the non-company exclusion in `assemble`.971 todo_cls = [q for q in qids if q in details and "p31" not in details[q]]972 log.info("details: P31 classes to fetch for %d items", len(todo_cls))973 for i in range(0, len(todo_cls), CLASS_BATCH):974 chunk = todo_cls[i:i + CLASS_BATCH]975 for q in chunk:976 details[q]["p31"] = []977 for r in sp.query(q_classes(chunk), label=f"classes {i + len(chunk)}/{len(todo_cls)}"):978 q, c = qid_of(r["item"]), qid_of(r["cls"])979 if c not in details[q]["p31"]:980 details[q]["p31"].append(c)981 if (i // CLASS_BATCH) % 10 == 9:982 dump_json(out_path, details)983 dump_json(out_path, details)984 log.info("details: %d entries → %s", len(details), out_path)985 return details986987988# ------------------------------------------------------------------------------------------------------------ stage: assemble989def importance_score(sitelinks: int, employees: int | None, public: bool, ticker: str | None) -> float:990 s_sl = min(1.0, math.log1p(max(sitelinks, 0)) / math.log1p(300))991 s_emp = min(1.0, math.log10((employees or 0) + 1) / 6) if employees else 0.0992 score = 0.6 * s_sl + 0.2 * s_emp + 0.1 * (1 if public else 0) + 0.1 * (1 if ticker else 0)993 return round(min(1.0, max(0.02, score)), 4)994995996def tier_cutoffs(n: int) -> tuple[int, int, int]:997 """Cumulative rank cut-offs of tiers 1–3 for a universe of `n` companies (TIER_SHARES quantiles; ≈ 300 / 2,300 / 10,300 at 30 k)."""998 t1 = round(n * TIER_SHARES[0])999 t2 = t1 + round(n * TIER_SHARES[1])1000 t3 = t2 + round(n * TIER_SHARES[2])1001 return t1, t2, t3100210031004def assign_tiers(rows: list[dict[str, Any]]) -> None:1005 rows.sort(key=lambda r: (-r["importance"], -r["sitelinks"], r["wikidata_id"]))1006 t1, t2, t3 = tier_cutoffs(len(rows))1007 for i, r in enumerate(rows):1008 r["tier"] = 1 if i < t1 else 2 if i < t2 else 3 if i < t3 else 4100910101011def build_row(sel: dict[str, Any], d: dict[str, Any], harvested_at: str) -> dict[str, Any]:1012 label = d.get("label") or ""1013 if not label or re.fullmatch(r"Q\d+", label):1014 label = (d.get("legal_names") or [sel["canonical_domain"]])[0]1015 legal = next((x for x in d.get("legal_names", []) if re.search(r"[A-Za-z]", x)), None) or (d.get("legal_names") or [None])[0]1016 aliases = [a for a in d.get("alt_labels", []) if a and a not in (label, legal) and len(a) <= 80 and not DOMAIN_LIKE_RE.match(a)][:8]1017 emp = None1018 obs = d.get("employees_obs") or []1019 if obs:1020 obs = sorted(obs, key=lambda o: (o[1] or ""), reverse=True)1021 emp = parse_int(obs[0][0])1022 if emp is None and sel.get("employees_hint"):1023 emp = parse_int(str(sel["employees_hint"]))1024 coord = parse_point(d.get("coord")) or parse_point(d.get("hq_coord"))1025 country = sel["country"]1026 if d.get("hq_iso") and d["hq_iso"] in (sel.get("country_candidates") or []):1027 country = d["hq_iso"]1028 ticker, exchange = pick_ticker(d.get("tickers") or [], country)1029 if not exchange and sel.get("listed") and sel.get("exchanges"):1030 exchange = sel["exchanges"][0] # listing without a ticker qualifier on Wikidata: keep the exchange, ticker stays null1031 public = "Q891723" in sel.get("classes", []) or bool(sel.get("listed")) or bool(ticker) or bool(exchange)1032 industry_labels = [x for x in (d.get("industry_labels") or []) if not JUNK_LABEL_RE.search(x)]1033 slugs = map_industry(industry_labels)1034 if not slugs and sel.get("boost_industries"):1035 slugs = [sel["boost_industries"][0]]1036 if not slugs:1037 slugs = map_industry([label, d.get("description") or ""] + [CLASSES.get(c, "") for c in sel.get("classes", [])], limit=2)1038 return {1039 "wikidata_id": sel["wikidata_id"], "display_name": label.strip(), "legal_name": legal, "aliases": aliases, "website": sel["website"],1040 "canonical_domain": sel["canonical_domain"], "country": country, "hq_city": d.get("hq_label"), "hq_region": d.get("hq_region"),1041 "lat": coord[0] if coord else None, "lon": coord[1] if coord else None, "industries": slugs, "industry_labels": industry_labels[:8],1042 "founded_year": parse_year(d.get("inception")), "employees": emp, "public_company": public, "ticker": ticker, "exchange": exchange,1043 "lei": d.get("lei"), "sec_cik": str(d["cik"]).lstrip("0") or None if d.get("cik") else None,1044 "parent": {"wikidata_id": d["parent"], "name": d.get("parent_label")} if d.get("parent") else None,1045 "logo_url": commons_url(d.get("logo")), "description": (d.get("description") or None), "sitelinks": sel["sitelinks"],1046 "importance": importance_score(sel["sitelinks"], emp, public, ticker), "tier": 4, "source": "wikidata", "harvested_at": harvested_at,1047 "domain_conflicts": sel.get("domain_conflicts") or [],1048 }104910501051def non_company_reason(sel: dict[str, Any], d: dict[str, Any], row: dict[str, Any]) -> str | None:1052 """Why an item is not a company (None when it is one): a NON_COMPANY_CLASSES P31 class, or a museum/university/club/… description1053 without any ticker, exchange, employee count or revenue. A current stock-exchange listing or a ticker always keeps the item."""1054 if sel.get("listed") or row.get("ticker") or row.get("exchange"):1055 return None1056 bad = [c for c in (d.get("p31") or []) if c in NON_COMPANY_CLASSES]1057 if bad:1058 return f"non_company_class:{bad[0]}"1059 desc = row.get("description") or ""1060 if NON_COMPANY_DESC_RE.search(desc) and not COMPANY_DESC_RE.search(desc) and not row.get("employees") and not sel.get("has_revenue"):1061 return "non_company_description"1062 return None106310641065def stage_assemble() -> list[dict[str, Any]]:1066 countries = load_countries()1067 sel = load_json(DATA_DIR / "selected.json")1068 details = load_json(DATA_DIR / "details.json")1069 harvested_at = datetime.now(UTC).replace(microsecond=0).isoformat()1070 selected = [r for r in sel["selected"] if r["wikidata_id"] in details]1071 reserve = [r for r in sel["reserve"] if r["wikidata_id"] in details]1072 dropped: list[dict[str, Any]] = []1073 non_company: Counter[str] = Counter()10741075 def build_companies(sels: list[dict[str, Any]]) -> list[dict[str, Any]]:1076 out = []1077 for s in sels:1078 row = build_row(s, details[s["wikidata_id"]], harvested_at)1079 reason = non_company_reason(s, details[s["wikidata_id"]], row)1080 if reason:1081 non_company[reason] += 11082 dropped.append({"wikidata_id": s["wikidata_id"], "reason": reason, "name": row["display_name"], "description": row["description"]})1083 continue1084 out.append(row)1085 return out10861087 rows = build_companies(selected)1088 selected_kept = len(rows)1089 # Industry minimums: top up from the reserve where a top-level industry is below the floor.1090 counts: Counter[str] = Counter()1091 for r in rows:1092 for top in {top_level_of(s) for s in r["industries"]}:1093 counts[top] += 11094 chosen = {r["wikidata_id"] for r in rows}1095 reserve_rows = build_companies(reserve)1096 reserve_rows.sort(key=lambda r: -r["sitelinks"])1097 for top in top_level_slugs():1098 for rr in reserve_rows:1099 if counts[top] >= INDUSTRY_MINIMUM:1100 break1101 if rr["wikidata_id"] in chosen or top not in {top_level_of(s) for s in rr["industries"]}:1102 continue1103 rows.append(rr)1104 chosen.add(rr["wikidata_id"])1105 for t in {top_level_of(s) for s in rr["industries"]}:1106 counts[t] += 11107 # Parent/child sharing a domain: parent wins (regardless of sitelinks); duplicate domains (should not happen after prefilter) drop the lower.1108 by_domain: dict[str, dict[str, Any]] = {}1109 for r in sorted(rows, key=lambda r: -r["sitelinks"]):1110 by_domain.setdefault(r["canonical_domain"], r)1111 ids = {r["wikidata_id"] for r in by_domain.values()}1112 for r in list(by_domain.values()):1113 p = r.get("parent")1114 if p and p["wikidata_id"] in ids and p["wikidata_id"] != r["wikidata_id"]:1115 parent_row = next((x for x in by_domain.values() if x["wikidata_id"] == p["wikidata_id"]), None)1116 if parent_row and parent_row["canonical_domain"] == r["canonical_domain"]:1117 dropped.append({"wikidata_id": r["wikidata_id"], "reason": "shares_parent_domain", "parent": p["wikidata_id"]})1118 for r in rows:1119 if r["wikidata_id"] not in ids:1120 dropped.append({"wikidata_id": r["wikidata_id"], "reason": "duplicate_domain", "domain": r["canonical_domain"]})1121 drop_ids = {x["wikidata_id"] for x in dropped}1122 final = [r for r in by_domain.values() if r["wikidata_id"] not in drop_ids]1123 for r in final:1124 r["notes"] = [{"related_domain_conflict": q} for q in r.pop("domain_conflicts", [])] or []1125 if not r["notes"]:1126 r.pop("notes")1127 assign_tiers(final)1128 # Write per region.1129 OUT_DIR.mkdir(parents=True, exist_ok=True)1130 for old in OUT_DIR.glob("wikidata-*.ndjson"):1131 old.unlink()1132 by_region: dict[str, list[dict[str, Any]]] = defaultdict(list)1133 for r in final:1134 region = (countries.get(r["country"] or "", {}).get("region") or "other").lower().replace(" ", "-")1135 by_region[region].append(r)1136 for region, items in sorted(by_region.items()):1137 items.sort(key=lambda r: (r["tier"], -r["importance"], r["wikidata_id"]))1138 with (OUT_DIR / f"wikidata-{region}.ndjson").open("w", encoding="utf-8") as f:1139 for r in items:1140 f.write(json.dumps(r, ensure_ascii=False) + "\n")1141 dump_json(DATA_DIR / "dropped.json", dropped)1142 by_bad_class = Counter(NON_COMPANY_CLASSES.get(k.split(":", 1)[1], k) for k in non_company.elements() if k.startswith("non_company_class:"))1143 stats = update_stats(assemble={1144 "companies": len(final), "selected_with_details": len(selected), "industry_top_up": len(rows) - selected_kept,1145 "non_company_excluded": sum(non_company.values()), "non_company_by_class": sum(v for k, v in non_company.items() if k != "non_company_description"),1146 "non_company_by_description": non_company.get("non_company_description", 0), "non_company_top_classes": dict(by_bad_class.most_common(12)),1147 "dropped_shares_parent_domain": sum(1 for x in dropped if x["reason"] == "shares_parent_domain"),1148 "dropped_duplicate_domain": sum(1 for x in dropped if x["reason"] == "duplicate_domain"),1149 "domain_conflict_notes": sum(len(r.get("notes") or []) for r in final), "tier_cutoffs": tier_cutoffs(len(final)),1150 "harvested_at": harvested_at,1151 })1152 write_readme(final, countries, stats)1153 log.info("assembled %d companies into %d region files (%d dropped) → %s", len(final), len(by_region), len(dropped), OUT_DIR)1154 return final115511561157def write_readme(rows: list[dict[str, Any]], countries: dict[str, dict[str, str]], stats: dict[str, Any] | None = None) -> None:1158 names = {i.slug: i.name for i in load_industries()}1159 by_country = Counter(r["country"] or "??" for r in rows)1160 by_tier = Counter(r["tier"] for r in rows)1161 by_top: Counter[str] = Counter()1162 by_ind: Counter[str] = Counter()1163 for r in rows:1164 for s in r["industries"]:1165 by_ind[s] += 11166 for t in {top_level_of(s) for s in r["industries"]}:1167 by_top[t] += 11168 n = len(rows)1169 no_ind = sum(1 for r in rows if not r["industries"])1170 public = sum(1 for r in rows if r["public_company"])1171 with_ticker = sum(1 for r in rows if r["ticker"])1172 with_exchange = sum(1 for r in rows if r["exchange"])1173 with_employees = sum(1 for r in rows if r["employees"])1174 region = Counter((countries.get(r["country"] or "", {}).get("region") or "other") for r in rows)1175 by_exchange = Counter(r["exchange"] for r in rows if r["exchange"])1176 generated = rows[0]["harvested_at"] if rows else ""1177 intro = (f"Generated by `scripts/seed_wikidata.py` on {generated}. **{n} companies**, {public} public ({100 * public / n:.1f} %), "1178 f"{with_exchange} with a stock exchange, {with_ticker} with a ticker, {with_employees} with an employee count, {no_ind} without an "1179 f"industry mapping ({100 * no_ind / n:.1f} %), {len(by_country)} countries.")1180 files_note = ("Files: one NDJSON per UN region (`wikidata-<region>.ndjson`), one JSON object per line — see `docs/SEEDS.md` for the "1181 "schema, the diversification rules and how to add companies. `scripts/seed_edgar.py` rewrites the files in place with "1182 "SEC data; run it after every `assemble`.")1183 lines = [1184 "# Seed registry — Wikidata harvest", "", intro, "", files_note, "",1185 "## Tiers", "", "| Tier | Companies |", "|---|---|",1186 *[f"| {t} ({ {1: 'global', 2: 'major', 3: 'notable', 4: 'long tail'}[t] }) | {by_tier[t]} |" for t in sorted(by_tier)], "",1187 "## Regions", "", "| Region | Companies |", "|---|---|", *[f"| {k} | {v} |" for k, v in region.most_common()], "",1188 "## Countries", "", "| Code | Country | Companies | Share |", "|---|---|---|---|",1189 *[f"| {c} | {countries.get(c, {}).get('name', 'unknown')} | {cnt} | {100 * cnt / n:.1f} % |" for c, cnt in by_country.most_common()], "",1190 "## Stock exchanges (primary listing kept per company)", "", "| Exchange | Companies |", "|---|---|",1191 *[f"| {k} | {v} |" for k, v in by_exchange.most_common(40)], "",1192 "## Top-level industries (a company counts once per top-level sector)", "", "| Industry | Companies |", "|---|---|",1193 *[f"| {names.get(k, k)} | {v} |" for k, v in by_top.most_common()], "",1194 "## All industries", "", "| Slug | Companies |", "|---|---|", *[f"| {k} | {v} |" for k, v in by_ind.most_common()], "",1195 ]1196 lines += harvest_report(stats or {}, rows)1197 (OUT_DIR / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8")119811991200def harvest_report(stats: dict[str, Any], rows: list[dict[str, Any]]) -> list[str]:1201 """Human-readable summary of `data/seed/harvest-stats.json` (candidate bands, exclusions, selection passes, tiers)."""1202 c = stats.get("candidates") or {}1203 d = stats.get("dissolved") or {}1204 s = stats.get("select") or {}1205 a = stats.get("assemble") or {}1206 if not (c or s or a):1207 return []1208 day = (a.get("harvested_at") or "")[:10]1209 lines = [f"## Harvest report — {day}", "",1210 ("* Pipeline: `scripts/seed_wikidata.py all --target N` (candidates → dissolved → select → details → assemble), then "1211 "`scripts/seed_edgar.py`, then `catlas seed`. Every SPARQL result is cached under `data/seed/wikidata/sparql/`; per-stage "1212 "counters in `data/seed/harvest-stats.json`.")]1213 if c:1214 bands = ", ".join(f"{k} {v:,}" for k, v in sorted((c.get("band_new") or {}).items(), key=lambda kv: -kv[1]))1215 lines.append(f"* Candidates: **{c.get('total', 0):,}** unique items with a website from {c.get('classes')} P31 classes × sitelink bands "1216 f"(≥ 3), {c.get('exchanges')} stock exchanges scanned in {c.get('exchange_queries')} P414 queries ({c.get('listed', 0):,} "1217 f"currently listed), employees ≥ {EMPLOYEES_MIN} / revenue bands per class, plus country and industry boosts. "1218 f"First seen per band: {bands}.")1219 if c.get("country_boost_failures"):1220 lines.append(f"* Country-first boosts cut by the endpoint time limit (tolerated): {', '.join(c['country_boost_failures'])}.")1221 if d:1222 lines.append(f"* Dissolved check (P576 / P582, {d.get('queries')} queries of 300 QIDs): {d.get('dissolved', 0):,} of "1223 f"{d.get('candidates', 0):,} candidates excluded.")1224 if s:1225 passes = s.get("passes") or {}1226 lines.append(f"* Prefilter: {s.get('dropped_generic_or_invalid_website', 0):,} generic / invalid websites dropped, "1227 f"{s.get('dropped_duplicate_domain', 0):,} duplicate registrable domains (highest sitelinks kept), {s.get('kept', 0):,} kept "1228 f"({s.get('listed_after_prefilter', 0):,} listed).")1229 lines.append(f"* Selection (target {s.get('target', 0):,}; caps US ≤ {US_CAP_SHARE:.0%}, other countries ≤ {OTHER_CAP_SHARE:.0%}, "1230 f"unknown country ≤ {UNKNOWN_COUNTRY_SHARE:.0%}, narrow classes 2–5 %): country minimums {passes.get('country_minimums', 0):,}, "1231 f"then {passes.get('listed', 0):,} listed companies, then {passes.get(f'sitelinks_ge_{SELECT_MIN_SITELINKS}', 0):,} by sitelinks "1232 f"≥ {SELECT_MIN_SITELINKS}" + (f", then {passes[f'sitelinks_ge_{FALLBACK_MIN_SITELINKS}']:,} by sitelinks ≥ {FALLBACK_MIN_SITELINKS}"1233 if f"sitelinks_ge_{FALLBACK_MIN_SITELINKS}" in passes else "") +1234 f"; {s.get('listed_selected', 0):,} listed companies selected in total.")1235 if a:1236 t1, t2, t3 = a.get("tier_cutoffs") or (0, 0, 0)1237 top = ", ".join(f"{k} {v}" for k, v in (a.get("non_company_top_classes") or {}).items())1238 lines.append(f"* Non-company exclusion: **{a.get('non_company_excluded', 0):,} items removed** — {a.get('non_company_by_class', 0):,} for a "1239 f"non-company P31 class (museums, libraries, universities / schools, government agencies, NGOs / charities / foundations / "1240 f"nonprofits, religious organisations, political parties, trade unions, hospitals, sports clubs, YouTube channels; top: {top}) "1241 f"and {a.get('non_company_by_description', 0):,} for a description matching museum / university / school / ministry / agency / "1242 f"charity / foundation / association / club / church / channel with no ticker, exchange, employee count or revenue. "1243 f"Items with a current listing or a ticker are always kept (listed football clubs). List: `data/seed/dropped.json`.")1244 lines.append(f"* Assemble: {a.get('selected_with_details', 0):,} selected companies with details + {a.get('industry_top_up', 0):,} from the "1245 f"industry top-up (every top-level industry ≥ {INDUSTRY_MINIMUM}); {a.get('dropped_shares_parent_domain', 0)} subsidiaries "1246 f"sharing their parent's domain and {a.get('dropped_duplicate_domain', 0)} duplicate domains dropped; "1247 f"{a.get('domain_conflict_notes', 0):,} `related_domain_conflict` notes → **{a.get('companies', 0):,} companies**.")1248 lines.append(f"* Tiers by importance quantile ({TIER_SHARES[0]:.0%} / {TIER_SHARES[1]:.2%} / {TIER_SHARES[2]:.2%}): tier 1 = ranks 1–{t1:,}, "1249 f"tier 2 → {t2:,}, tier 3 → {t3:,}, tier 4 = rest.")1250 mins = {iso: sum(1 for r in rows if r["country"] == iso) for iso in COUNTRY_MINIMUMS}1251 short = {iso: n for iso, n in mins.items() if n < COUNTRY_MINIMUMS[iso]}1252 lines.append("* Country minimums: " + ("all met" if not short else "short: " + ", ".join(f"{k} {v}/{COUNTRY_MINIMUMS[k]}" for k, v in short.items()))1253 + " (" + ", ".join(f"{k} {v}" for k, v in sorted(mins.items(), key=lambda kv: -kv[1])[:8]) + " …).")1254 return lines + [""]125512561257# ------------------------------------------------------------------------------------------------------------ main1258def main() -> int:1259 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)1260 ap.add_argument("stage", nargs="?", default="all", choices=["candidates", "dissolved", "select", "details", "assemble", "all"])1261 ap.add_argument("--target", type=int, default=30000, help="companies to select before detail fetching (default 30000)")1262 ap.add_argument("--batch", type=int, default=100, help="QIDs per detail query")1263 ap.add_argument("--refresh", action="store_true", help="ignore the SPARQL cache")1264 ap.add_argument("-v", "--verbose", action="store_true")1265 args = ap.parse_args()1266 logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(asctime)s %(levelname)s %(message)s")1267 logging.getLogger("httpx").setLevel(logging.WARNING)1268 DATA_DIR.mkdir(parents=True, exist_ok=True)1269 sp = Sparql(refresh=args.refresh)1270 t0 = time.monotonic()1271 if args.stage in ("candidates", "all"):1272 stage_candidates(sp)1273 if args.stage in ("dissolved", "all"):1274 stage_dissolved(sp)1275 if args.stage in ("select", "all"):1276 stage_select(target=args.target)1277 if args.stage in ("details", "all"):1278 stage_details(sp, batch=args.batch)1279 if args.stage in ("assemble", "all"):1280 stage_assemble()1281 log.info("done in %.0fs (%d live queries, %d cached)", time.monotonic() - t0, sp.queries, sp.cached)1282 return 0128312841285if __name__ == "__main__":1286 sys.exit(main())1287