"""Industry taxonomy (registry/industries.yaml) and deterministic mapping of free-text industry labels to our slugs. `map_industry(labels)` turns Wikidata P452 labels ("software industry", "banking"), SIC descriptions or any free text into an ordered, de-duplicated list of taxonomy slugs. Matching is lexical only (no LLM): exact keyword match first, then the longest keyword phrase found inside the label on word boundaries. `map_sic(code)` maps a US SIC code (SEC EDGAR) to a slug. """ from __future__ import annotations import re from dataclasses import dataclass, field from functools import lru_cache from pathlib import Path import yaml REGISTRY_DIR = Path(__file__).resolve().parents[3] / "registry" INDUSTRIES_FILE = REGISTRY_DIR / "industries.yaml" @dataclass(frozen=True) class Industry: slug: str name: str parent: str | None description: str keywords: tuple[str, ...] sort_order: int = 100 children: tuple[str, ...] = field(default_factory=tuple) def _norm(text: str) -> str: text = text.lower().replace("_", " ").replace("&", " and ").replace("/", " ") text = re.sub(r"[^\w\s\-']", " ", text) return re.sub(r"\s+", " ", text).strip() @lru_cache def load_industries(path: Path = INDUSTRIES_FILE) -> tuple[Industry, ...]: raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} rows = raw.get("industries") or [] children: dict[str, list[str]] = {} for r in rows: if r.get("parent"): children.setdefault(r["parent"], []).append(r["slug"]) out: list[Industry] = [] for i, r in enumerate(rows): out.append(Industry(slug=r["slug"], name=r["name"], parent=r.get("parent"), description=(r.get("description") or "").strip(), keywords=tuple(_norm(k) for k in (r.get("keywords") or []) if k), sort_order=(i + 1) * 10, children=tuple(children.get(r["slug"], ())))) return tuple(out) @lru_cache def industry_index() -> dict[str, Industry]: return {i.slug: i for i in load_industries()} def top_level_slugs() -> list[str]: return [i.slug for i in load_industries() if i.parent is None] def top_level_of(slug: str) -> str: ind = industry_index().get(slug) if ind is None: return slug return ind.parent or ind.slug def is_valid_slug(slug: str) -> bool: return slug in industry_index() @lru_cache def _keyword_table() -> tuple[dict[str, tuple[str, ...]], tuple[tuple[str, re.Pattern[str], tuple[str, ...]], ...]]: """(exact keyword → slugs, [(keyword, boundary regex, slugs)] sorted by keyword length desc).""" exact: dict[str, list[str]] = {} for ind in load_industries(): for kw in ind.keywords: exact.setdefault(kw, []) if ind.slug not in exact[kw]: exact[kw].append(ind.slug) phrases = [] for kw, slugs in sorted(exact.items(), key=lambda kv: (-len(kv[0]), kv[0])): pat = re.compile(r"(? list[str]: """Slugs for a single label. Exact keyword match, else the longest keyword phrase(s) contained in the label.""" text = _norm(label) if not text: return [] exact, phrases = _keyword_table() if text in exact: return list(exact[text]) for variant in (text.removesuffix(" industry"), text.removesuffix(" company"), text.removesuffix(" sector"), text.removesuffix("s")): if variant != text and variant in exact: return list(exact[variant]) best_len = 0 found: list[str] = [] for kw, pat, slugs in phrases: if best_len and len(kw) < best_len: break if pat.search(text): best_len = len(kw) for s in slugs: if s not in found: found.append(s) return found def map_industry(labels: str | list[str] | tuple[str, ...] | None, *, limit: int = 4) -> list[str]: """Ordered, de-duplicated taxonomy slugs for one or many labels (first label's matches come first).""" if not labels: return [] if isinstance(labels, str): labels = [labels] out: list[str] = [] for label in labels: for slug in map_label(label): if slug not in out: out.append(slug) return out[:limit] # SIC (Standard Industrial Classification, as used by SEC EDGAR) → slug. Specific codes first, then ranges (inclusive). _SIC_EXACT: dict[int, str] = { 1311: "oil-gas", 1381: "oil-gas", 1382: "oil-gas", 1389: "oil-gas", 2111: "consumer-goods", 2834: "pharmaceuticals", 2835: "pharmaceuticals", 2836: "biotechnology", 2833: "pharmaceuticals", 2911: "oil-gas", 3571: "technology", 3572: "cloud-infrastructure", 3576: "cloud-infrastructure", 3577: "technology", 3578: "technology", 3661: "telecommunications", 3663: "telecommunications", 3669: "telecommunications", 3674: "semiconductors", 3672: "semiconductors", 3679: "technology", 3630: "consumer-goods", 3634: "consumer-goods", 3651: "consumer-goods", 3711: "automotive", 3713: "automotive", 3714: "automotive", 3715: "automotive", 3716: "automotive", 3751: "automotive", 3720: "aerospace-defense", 3721: "aerospace-defense", 3724: "aerospace-defense", 3728: "aerospace-defense", 3760: "aerospace-defense", 3812: "aerospace-defense", 3730: "manufacturing", 3743: "transportation", 3841: "medical-devices", 3842: "medical-devices", 3843: "medical-devices", 3844: "medical-devices", 3845: "medical-devices", 3851: "medical-devices", 3942: "consumer-goods", 3944: "consumer-goods", 3949: "consumer-goods", 4011: "transportation", 4013: "transportation", 4210: "logistics", 4213: "logistics", 4400: "shipping", 4412: "shipping", 4512: "airlines", 4513: "logistics", 4522: "airlines", 4610: "oil-gas", 4700: "logistics", 4731: "logistics", 4812: "telecommunications", 4813: "telecommunications", 4822: "telecommunications", 4832: "media", 4833: "media", 4841: "media", 4899: "telecommunications", 4911: "utilities", 4922: "utilities", 4923: "utilities", 4924: "utilities", 4931: "utilities", 4932: "utilities", 4941: "utilities", 4950: "utilities", 4953: "utilities", 4955: "utilities", 4991: "renewables", 5812: "food-beverage", 5912: "retail", 5961: "e-commerce", 6021: "banking", 6022: "banking", 6029: "banking", 6035: "banking", 6036: "banking", 6099: "payments", 6111: "financial-services", 6141: "financial-services", 6153: "financial-services", 6159: "financial-services", 6162: "financial-services", 6163: "financial-services", 6172: "financial-services", 6189: "financial-services", 6199: "financial-services", 6200: "financial-services", 6211: "financial-services", 6221: "financial-services", 6282: "asset-management", 6311: "insurance", 6321: "insurance", 6324: "insurance", 6331: "insurance", 6351: "insurance", 6361: "insurance", 6399: "insurance", 6411: "insurance", 6500: "real-estate", 6510: "real-estate", 6512: "real-estate", 6513: "real-estate", 6519: "real-estate", 6531: "real-estate", 6552: "real-estate", 6770: "financial-services", 6792: "financial-services", 6794: "professional-services", 6795: "mining", 6798: "real-estate", 6799: "asset-management", 7011: "hospitality", 7200: "professional-services", 7310: "media", 7311: "media", 7320: "professional-services", 7330: "professional-services", 7331: "media", 7350: "professional-services", 7359: "professional-services", 7361: "professional-services", 7363: "professional-services", 7370: "software", 7371: "software", 7372: "software", 7373: "software", 7374: "cloud-infrastructure", 7377: "cloud-infrastructure", 7380: "professional-services", 7381: "professional-services", 7384: "professional-services", 7385: "telecommunications", 7389: "professional-services", 7500: "automotive", 7510: "automotive", 7812: "entertainment", 7819: "entertainment", 7822: "entertainment", 7829: "entertainment", 7830: "entertainment", 7841: "entertainment", 7900: "entertainment", 7948: "entertainment", 7990: "entertainment", 7997: "hospitality", 8000: "healthcare", 8011: "healthcare", 8050: "healthcare", 8051: "healthcare", 8060: "healthcare", 8062: "healthcare", 8071: "healthcare", 8082: "healthcare", 8090: "healthcare", 8093: "healthcare", 8111: "professional-services", 8200: "education", 8300: "healthcare", 8351: "education", 8600: "professional-services", 8700: "professional-services", 8711: "professional-services", 8721: "professional-services", 8731: "biotechnology", 8734: "professional-services", 8741: "professional-services", 8742: "consulting", 8744: "professional-services", 8748: "consulting", 8880: "financial-services", 8888: "financial-services", 8900: "professional-services", 9995: "financial-services", } _SIC_RANGES: tuple[tuple[int, int, str], ...] = ( (100, 999, "agriculture"), (1000, 1299, "mining"), (1300, 1399, "oil-gas"), (1400, 1499, "mining"), (1500, 1799, "construction"), (2000, 2099, "food-beverage"), (2100, 2199, "consumer-goods"), (2200, 2399, "apparel"), (2400, 2499, "materials"), (2500, 2599, "consumer-goods"), (2600, 2699, "materials"), (2700, 2799, "media"), (2800, 2829, "chemicals"), (2830, 2839, "pharmaceuticals"), (2840, 2899, "chemicals"), (2900, 2999, "oil-gas"), (3000, 3099, "chemicals"), (3100, 3199, "apparel"), (3200, 3299, "materials"), (3300, 3399, "mining"), (3400, 3499, "manufacturing"), (3500, 3569, "industrial-machinery"), (3570, 3579, "technology"), (3580, 3599, "industrial-machinery"), (3600, 3629, "manufacturing"), (3630, 3639, "consumer-goods"), (3640, 3659, "manufacturing"), (3660, 3669, "telecommunications"), (3670, 3679, "semiconductors"), (3680, 3699, "technology"), (3700, 3719, "automotive"), (3720, 3729, "aerospace-defense"), (3730, 3739, "manufacturing"), (3740, 3749, "transportation"), (3750, 3759, "automotive"), (3760, 3769, "aerospace-defense"), (3770, 3799, "manufacturing"), (3800, 3839, "technology"), (3840, 3859, "medical-devices"), (3860, 3899, "technology"), (3900, 3999, "consumer-goods"), (4000, 4099, "transportation"), (4100, 4199, "transportation"), (4200, 4299, "logistics"), (4300, 4399, "logistics"), (4400, 4499, "shipping"), (4500, 4599, "airlines"), (4600, 4699, "oil-gas"), (4700, 4799, "logistics"), (4800, 4829, "telecommunications"), (4830, 4849, "media"), (4850, 4899, "telecommunications"), (4900, 4999, "utilities"), (5000, 5199, "retail"), (5200, 5799, "retail"), (5800, 5899, "food-beverage"), (5900, 5999, "retail"), (6000, 6099, "banking"), (6100, 6199, "financial-services"), (6200, 6299, "financial-services"), (6300, 6499, "insurance"), (6500, 6599, "real-estate"), (6700, 6799, "asset-management"), (7000, 7099, "hospitality"), (7200, 7299, "professional-services"), (7300, 7369, "professional-services"), (7370, 7379, "software"), (7380, 7399, "professional-services"), (7500, 7599, "automotive"), (7600, 7699, "professional-services"), (7800, 7899, "entertainment"), (7900, 7999, "entertainment"), (8000, 8099, "healthcare"), (8100, 8199, "professional-services"), (8200, 8299, "education"), (8300, 8399, "healthcare"), (8400, 8499, "entertainment"), (8600, 8699, "professional-services"), (8700, 8799, "professional-services"), (8800, 8999, "professional-services"), ) def map_sic(code: int | str | None) -> str | None: """US SIC code → taxonomy slug (None when unknown/blank).""" if code in (None, ""): return None try: n = int(str(code).strip()) except ValueError: return None if n in _SIC_EXACT: return _SIC_EXACT[n] for lo, hi, slug in _SIC_RANGES: if lo <= n <= hi: return slug return None __all__ = ["INDUSTRIES_FILE", "REGISTRY_DIR", "Industry", "industry_index", "is_valid_slug", "load_industries", "map_industry", "map_label", "map_sic", "top_level_of", "top_level_slugs"]