spb/company-atlas

1"""Industry taxonomy (registry/industries.yaml) and deterministic mapping of free-text industry labels to our slugs.23`map_industry(labels)` turns Wikidata P452 labels ("software industry", "banking"), SIC descriptions or any free text into an ordered,4de-duplicated list of taxonomy slugs. Matching is lexical only (no LLM): exact keyword match first, then the longest keyword phrase found5inside the label on word boundaries. `map_sic(code)` maps a US SIC code (SEC EDGAR) to a slug.6"""7from __future__ import annotations89import re10from dataclasses import dataclass, field11from functools import lru_cache12from pathlib import Path1314import yaml1516REGISTRY_DIR = Path(__file__).resolve().parents[3] / "registry"17INDUSTRIES_FILE = REGISTRY_DIR / "industries.yaml"181920@dataclass(frozen=True)21class Industry:22    slug: str23    name: str24    parent: str | None25    description: str26    keywords: tuple[str, ...]27    sort_order: int = 10028    children: tuple[str, ...] = field(default_factory=tuple)293031def _norm(text: str) -> str:32    text = text.lower().replace("_", " ").replace("&", " and ").replace("/", " ")33    text = re.sub(r"[^\w\s\-']", " ", text)34    return re.sub(r"\s+", " ", text).strip()353637@lru_cache38def load_industries(path: Path = INDUSTRIES_FILE) -> tuple[Industry, ...]:39    raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}40    rows = raw.get("industries") or []41    children: dict[str, list[str]] = {}42    for r in rows:43        if r.get("parent"):44            children.setdefault(r["parent"], []).append(r["slug"])45    out: list[Industry] = []46    for i, r in enumerate(rows):47        out.append(Industry(slug=r["slug"], name=r["name"], parent=r.get("parent"), description=(r.get("description") or "").strip(),48                            keywords=tuple(_norm(k) for k in (r.get("keywords") or []) if k), sort_order=(i + 1) * 10,49                            children=tuple(children.get(r["slug"], ()))))50    return tuple(out)515253@lru_cache54def industry_index() -> dict[str, Industry]:55    return {i.slug: i for i in load_industries()}565758def top_level_slugs() -> list[str]:59    return [i.slug for i in load_industries() if i.parent is None]606162def top_level_of(slug: str) -> str:63    ind = industry_index().get(slug)64    if ind is None:65        return slug66    return ind.parent or ind.slug676869def is_valid_slug(slug: str) -> bool:70    return slug in industry_index()717273@lru_cache74def _keyword_table() -> tuple[dict[str, tuple[str, ...]], tuple[tuple[str, re.Pattern[str], tuple[str, ...]], ...]]:75    """(exact keyword → slugs, [(keyword, boundary regex, slugs)] sorted by keyword length desc)."""76    exact: dict[str, list[str]] = {}77    for ind in load_industries():78        for kw in ind.keywords:79            exact.setdefault(kw, [])80            if ind.slug not in exact[kw]:81                exact[kw].append(ind.slug)82    phrases = []83    for kw, slugs in sorted(exact.items(), key=lambda kv: (-len(kv[0]), kv[0])):84        pat = re.compile(r"(?<![\w\-])" + re.escape(kw) + r"(?![\w\-])")85        phrases.append((kw, pat, tuple(slugs)))86    return {k: tuple(v) for k, v in exact.items()}, tuple(phrases)878889def map_label(label: str) -> list[str]:90    """Slugs for a single label. Exact keyword match, else the longest keyword phrase(s) contained in the label."""91    text = _norm(label)92    if not text:93        return []94    exact, phrases = _keyword_table()95    if text in exact:96        return list(exact[text])97    for variant in (text.removesuffix(" industry"), text.removesuffix(" company"), text.removesuffix(" sector"), text.removesuffix("s")):98        if variant != text and variant in exact:99            return list(exact[variant])100    best_len = 0101    found: list[str] = []102    for kw, pat, slugs in phrases:103        if best_len and len(kw) < best_len:104            break105        if pat.search(text):106            best_len = len(kw)107            for s in slugs:108                if s not in found:109                    found.append(s)110    return found111112113def map_industry(labels: str | list[str] | tuple[str, ...] | None, *, limit: int = 4) -> list[str]:114    """Ordered, de-duplicated taxonomy slugs for one or many labels (first label's matches come first)."""115    if not labels:116        return []117    if isinstance(labels, str):118        labels = [labels]119    out: list[str] = []120    for label in labels:121        for slug in map_label(label):122            if slug not in out:123                out.append(slug)124    return out[:limit]125126127# SIC (Standard Industrial Classification, as used by SEC EDGAR) → slug. Specific codes first, then ranges (inclusive).128_SIC_EXACT: dict[int, str] = {129    1311: "oil-gas", 1381: "oil-gas", 1382: "oil-gas", 1389: "oil-gas", 2111: "consumer-goods", 2834: "pharmaceuticals", 2835: "pharmaceuticals",130    2836: "biotechnology", 2833: "pharmaceuticals", 2911: "oil-gas", 3571: "technology", 3572: "cloud-infrastructure", 3576: "cloud-infrastructure",131    3577: "technology", 3578: "technology", 3661: "telecommunications", 3663: "telecommunications", 3669: "telecommunications",132    3674: "semiconductors", 3672: "semiconductors", 3679: "technology", 3630: "consumer-goods", 3634: "consumer-goods", 3651: "consumer-goods",133    3711: "automotive", 3713: "automotive", 3714: "automotive", 3715: "automotive", 3716: "automotive", 3751: "automotive",134    3720: "aerospace-defense", 3721: "aerospace-defense", 3724: "aerospace-defense", 3728: "aerospace-defense", 3760: "aerospace-defense",135    3812: "aerospace-defense", 3730: "manufacturing", 3743: "transportation", 3841: "medical-devices", 3842: "medical-devices",136    3843: "medical-devices", 3844: "medical-devices", 3845: "medical-devices", 3851: "medical-devices", 3942: "consumer-goods",137    3944: "consumer-goods", 3949: "consumer-goods", 4011: "transportation", 4013: "transportation", 4210: "logistics", 4213: "logistics",138    4400: "shipping", 4412: "shipping", 4512: "airlines", 4513: "logistics", 4522: "airlines", 4610: "oil-gas", 4700: "logistics",139    4731: "logistics", 4812: "telecommunications", 4813: "telecommunications", 4822: "telecommunications", 4832: "media", 4833: "media",140    4841: "media", 4899: "telecommunications", 4911: "utilities", 4922: "utilities", 4923: "utilities", 4924: "utilities", 4931: "utilities",141    4932: "utilities", 4941: "utilities", 4950: "utilities", 4953: "utilities", 4955: "utilities", 4991: "renewables", 5812: "food-beverage",142    5912: "retail", 5961: "e-commerce", 6021: "banking", 6022: "banking", 6029: "banking", 6035: "banking", 6036: "banking",143    6099: "payments", 6111: "financial-services", 6141: "financial-services", 6153: "financial-services", 6159: "financial-services",144    6162: "financial-services", 6163: "financial-services", 6172: "financial-services", 6189: "financial-services", 6199: "financial-services",145    6200: "financial-services", 6211: "financial-services", 6221: "financial-services", 6282: "asset-management", 6311: "insurance",146    6321: "insurance", 6324: "insurance", 6331: "insurance", 6351: "insurance", 6361: "insurance", 6399: "insurance", 6411: "insurance",147    6500: "real-estate", 6510: "real-estate", 6512: "real-estate", 6513: "real-estate", 6519: "real-estate", 6531: "real-estate",148    6552: "real-estate", 6770: "financial-services", 6792: "financial-services", 6794: "professional-services", 6795: "mining",149    6798: "real-estate", 6799: "asset-management", 7011: "hospitality", 7200: "professional-services", 7310: "media", 7311: "media",150    7320: "professional-services", 7330: "professional-services", 7331: "media", 7350: "professional-services", 7359: "professional-services",151    7361: "professional-services", 7363: "professional-services", 7370: "software", 7371: "software", 7372: "software", 7373: "software",152    7374: "cloud-infrastructure", 7377: "cloud-infrastructure", 7380: "professional-services", 7381: "professional-services",153    7384: "professional-services", 7385: "telecommunications", 7389: "professional-services", 7500: "automotive", 7510: "automotive",154    7812: "entertainment", 7819: "entertainment", 7822: "entertainment", 7829: "entertainment", 7830: "entertainment", 7841: "entertainment",155    7900: "entertainment", 7948: "entertainment", 7990: "entertainment", 7997: "hospitality", 8000: "healthcare", 8011: "healthcare",156    8050: "healthcare", 8051: "healthcare", 8060: "healthcare", 8062: "healthcare", 8071: "healthcare", 8082: "healthcare", 8090: "healthcare",157    8093: "healthcare", 8111: "professional-services", 8200: "education", 8300: "healthcare", 8351: "education", 8600: "professional-services",158    8700: "professional-services", 8711: "professional-services", 8721: "professional-services", 8731: "biotechnology",159    8734: "professional-services", 8741: "professional-services", 8742: "consulting", 8744: "professional-services", 8748: "consulting",160    8880: "financial-services", 8888: "financial-services", 8900: "professional-services", 9995: "financial-services",161}162_SIC_RANGES: tuple[tuple[int, int, str], ...] = (163    (100, 999, "agriculture"), (1000, 1299, "mining"), (1300, 1399, "oil-gas"), (1400, 1499, "mining"), (1500, 1799, "construction"),164    (2000, 2099, "food-beverage"), (2100, 2199, "consumer-goods"), (2200, 2399, "apparel"), (2400, 2499, "materials"), (2500, 2599, "consumer-goods"),165    (2600, 2699, "materials"), (2700, 2799, "media"), (2800, 2829, "chemicals"), (2830, 2839, "pharmaceuticals"), (2840, 2899, "chemicals"),166    (2900, 2999, "oil-gas"), (3000, 3099, "chemicals"), (3100, 3199, "apparel"), (3200, 3299, "materials"), (3300, 3399, "mining"),167    (3400, 3499, "manufacturing"), (3500, 3569, "industrial-machinery"), (3570, 3579, "technology"), (3580, 3599, "industrial-machinery"),168    (3600, 3629, "manufacturing"), (3630, 3639, "consumer-goods"), (3640, 3659, "manufacturing"), (3660, 3669, "telecommunications"),169    (3670, 3679, "semiconductors"), (3680, 3699, "technology"), (3700, 3719, "automotive"), (3720, 3729, "aerospace-defense"),170    (3730, 3739, "manufacturing"), (3740, 3749, "transportation"), (3750, 3759, "automotive"), (3760, 3769, "aerospace-defense"),171    (3770, 3799, "manufacturing"), (3800, 3839, "technology"), (3840, 3859, "medical-devices"), (3860, 3899, "technology"),172    (3900, 3999, "consumer-goods"), (4000, 4099, "transportation"), (4100, 4199, "transportation"), (4200, 4299, "logistics"),173    (4300, 4399, "logistics"), (4400, 4499, "shipping"), (4500, 4599, "airlines"), (4600, 4699, "oil-gas"), (4700, 4799, "logistics"),174    (4800, 4829, "telecommunications"), (4830, 4849, "media"), (4850, 4899, "telecommunications"), (4900, 4999, "utilities"),175    (5000, 5199, "retail"), (5200, 5799, "retail"), (5800, 5899, "food-beverage"), (5900, 5999, "retail"), (6000, 6099, "banking"),176    (6100, 6199, "financial-services"), (6200, 6299, "financial-services"), (6300, 6499, "insurance"), (6500, 6599, "real-estate"),177    (6700, 6799, "asset-management"), (7000, 7099, "hospitality"), (7200, 7299, "professional-services"), (7300, 7369, "professional-services"),178    (7370, 7379, "software"), (7380, 7399, "professional-services"), (7500, 7599, "automotive"), (7600, 7699, "professional-services"),179    (7800, 7899, "entertainment"), (7900, 7999, "entertainment"), (8000, 8099, "healthcare"), (8100, 8199, "professional-services"),180    (8200, 8299, "education"), (8300, 8399, "healthcare"), (8400, 8499, "entertainment"), (8600, 8699, "professional-services"),181    (8700, 8799, "professional-services"), (8800, 8999, "professional-services"),182)183184185def map_sic(code: int | str | None) -> str | None:186    """US SIC code → taxonomy slug (None when unknown/blank)."""187    if code in (None, ""):188        return None189    try:190        n = int(str(code).strip())191    except ValueError:192        return None193    if n in _SIC_EXACT:194        return _SIC_EXACT[n]195    for lo, hi, slug in _SIC_RANGES:196        if lo <= n <= hi:197            return slug198    return None199200201__all__ = ["INDUSTRIES_FILE", "REGISTRY_DIR", "Industry", "industry_index", "is_valid_slug", "load_industries", "map_industry",202           "map_label", "map_sic", "top_level_of", "top_level_slugs"]203