spb/company-atlas
Public
Python 66.3%
TypeScript 22.7%
JavaScript 8.6%
HTML 1.4%
CSS 0.7%
1"""HTML → semantic building blocks (spec §17–19, §107).23 html ──parse()──▶ NormalizedPage(title, meta, lang, blocks, text, links, jsonld, microdata) ──to_extraction()──▶ Extraction45Two layers are kept deliberately separate:67* **stored text** — whitespace-normalised, but otherwise the original (what a human reads in the historical viewer);8* **hash / diff layer** — `normalized_text()` replaces classic noise (dates, times, "3 minutes ago", counters, csrf/nonce/session9 tokens, cache-busting query params) with stable tokens so that `text_hash`, block hashes and simhashes ignore it.1011Blocks carry a stable `key` = kind + heading path + simhash bucket (+ occurrence index) — never DOM position — so a reordered12section is a *move*, not an add/remove pair. Deterministic; no network; no LLM.13"""14from __future__ import annotations1516import hashlib17import html as html_lib18import json19import logging20import re21from collections import Counter22from dataclasses import dataclass, field23from typing import Any2425from selectolax.lexbor import LexborHTMLParser, LexborNode2627from companyatlas.sdk.models import Block, Extraction2829log = logging.getLogger(__name__)3031NORMALIZE_VERSION = "normalize-v1"3233# ------------------------------------------------------------------------------------------------------------ element sets3435DROP_TAGS = frozenset({"script", "style", "noscript", "svg", "iframe", "template", "canvas", "video", "audio", "source", "track", "object",36 "embed", "map", "area", "input", "select", "textarea", "option", "optgroup", "datalist", "meter", "progress", "link",37 "meta", "base", "head", "picture", "img", "dialog", "math"})38BLOCK_TAGS = frozenset({"address", "article", "aside", "blockquote", "body", "dd", "details", "div", "dl", "dt", "fieldset", "figcaption",39 "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre",40 "section", "summary", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul", "caption", "legend", "menu"})41HEADING_TAGS = ("h1", "h2", "h3", "h4", "h5", "h6")4243COOKIE_RE = re.compile(r"(cookie|consent|gdpr|onetrust|cookiebot|truste|didomi|usercentrics|osano|cc-banner|cc-window|privacy-banner|"44 r"cmp-container|qc-cmp|sp_message|termly|iubenda|klaro|cookieyes|axeptio|tarteaucitron)", re.IGNORECASE)45NAV_CLASS_RE = re.compile(r"(^|[\s_-])(nav|navbar|navigation|menu|breadcrumbs?|topbar|masthead|site-header|global-header)([\s_-]|$)", re.IGNORECASE)46FOOTER_CLASS_RE = re.compile(r"(^|[\s_-])(footer|site-footer|global-footer|colophon|legal-links)([\s_-]|$)", re.IGNORECASE)47HERO_CLASS_RE = re.compile(r"(^|[\s_-])(hero|jumbotron|banner|masthead|splash|intro|cover|landing-hero)([\s_-]|$)", re.IGNORECASE)48CARD_PATTERNS: list[tuple[str, re.Pattern[str]]] = [49 ("pricing_plan", re.compile(r"(^|[\s_-])(pricing[-_ ]?(plan|card|tier|column|table|box|option)|plan[-_ ]?(card|box|column|tier|item)|tier[-_ ]?card|"50 r"price[-_ ]?(card|box|column|plan)|package[-_ ]?(card|box))([\s_-]|$)", re.IGNORECASE)),51 ("job_listing", re.compile(r"(^|[\s_-])(job|jobs|opening|position|vacancy|posting|role|career)[-_ ]?(item|card|listing|row|link|entry|tile|post)?([\s_-]|$)", re.IGNORECASE)),52 ("person", re.compile(r"(^|[\s_-])(team[-_ ]?member|member[-_ ]?card|person|people[-_ ]?card|bio|executive|leader|profile[-_ ]?card|staff|founder|"53 r"board[-_ ]?member|management[-_ ]?member|employee[-_ ]?card|director)([\s_-]|$)", re.IGNORECASE)),54 ("location", re.compile(r"(^|[\s_-])(office|location|store|branch|address|showroom|site[-_ ]?card|headquarters|hq)([\s_-]|$)", re.IGNORECASE)),55 ("news_item", re.compile(r"(^|[\s_-])(news|press|article|post|release|story|blog|announcement|update|publication)[-_ ]?(item|card|teaser|tile|preview|summary|link|list-item|entry)?([\s_-]|$)", re.IGNORECASE)),56 ("product_card", re.compile(r"(^|[\s_-])((product|products|catalog|sku|offering|solution)[-_ ]?(card|tile|item|box|grid-item|teaser|entry)?|item[-_ ]?card|grid[-_ ]item)([\s_-]|$)", re.IGNORECASE)),57 ("faq", re.compile(r"(^|[\s_-])(faq|accordion|question|collapsible)([\s_-]|$)", re.IGNORECASE)),58]59CARD_MAX_TEXT = 18006061# Weights per block kind (spec §19–20): what matters for significance. Nav/footer/cookie are kept for discovery, not for change value.62BLOCK_WEIGHTS: dict[str, float] = {63 "hero": 1.5, "pricing_plan": 1.6, "job_listing": 1.4, "person": 1.4, "product_card": 1.3, "location": 1.2, "news_item": 1.2,64 "heading": 1.0, "paragraph": 1.0, "section": 1.0, "table": 1.1, "list": 0.9, "faq": 0.8, "code": 0.7, "quote": 0.6, "other": 0.5,65 "header": 0.3, "nav": 0.2, "footer": 0.15,66}67LOW_VALUE_KINDS = frozenset({"nav", "footer", "header"})6869# ------------------------------------------------------------------------------------------------------------ noise normalisation7071_MONTH = r"(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?|" \72 r"janvier|février|fevrier|mars|avril|mai|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre|" \73 r"januar|februar|märz|maerz|april|juni|juli|august|oktober|dezember|enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)"74NOISE_RULES: list[tuple[re.Pattern[str], str]] = [75 (re.compile(r"\b\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?\b"), "<date>"),76 (re.compile(r"\b(?:\d{1,2}(?:st|nd|rd|th)?\s+)?" + _MONTH + r"\.?\s+\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\b", re.IGNORECASE), "<date>"),77 (re.compile(r"\b\d{1,2}(?:st|nd|rd|th)?\s+" + _MONTH + r"\.?,?\s+\d{4}\b", re.IGNORECASE), "<date>"),78 (re.compile(r"\b" + _MONTH + r"\.?\s+\d{4}\b", re.IGNORECASE), "<date>"),79 (re.compile(r"\b\d{1,2}[/.-]\d{1,2}[/.-]\d{2,4}\b"), "<date>"),80 (re.compile(r"\b\d{1,2}:\d{2}(?::\d{2})?\s*(?:am|pm|a\.m\.|p\.m\.|utc|gmt|est|pst|cet|z)?\b", re.IGNORECASE), "<time>"),81 (re.compile(r"\b(?:\d+|a|an|one|few|several)\s+(?:sec(?:ond)?s?|min(?:ute)?s?|hours?|hrs?|days?|weeks?|months?|years?)\s+ago\b", re.IGNORECASE), "<rel>"),82 (re.compile(r"\b(?:il y a|hace|vor)\s+\d+\s+\w+\b", re.IGNORECASE), "<rel>"),83 (re.compile(r"\b(?:yesterday|today|just now|hier|aujourd'hui|heute|gestern)\b", re.IGNORECASE), "<rel>"),84 (re.compile(r"\b\d[\d,.]*\s*(?:k|m)?\s*(?:views?|comments?|likes?|shares?|followers?|visitors?|reads?|replies|upvotes?|downloads?|stars?|members?|online)\b", re.IGNORECASE), "<count>"),85 (re.compile(r"(?:©|\(c\)|\bcopyright)\s*(?:\d{4}\s*[-–]\s*)?\d{4}\b", re.IGNORECASE), "<copyright>"),86 (re.compile(r"\b(?:\d{4}\s*[-–]\s*)?\d{4}\s*(?:©|\(c\))", re.IGNORECASE), "<copyright>"),87 (re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.IGNORECASE), "<uuid>"),88 (re.compile(r"\b[0-9a-f]{24,}\b", re.IGNORECASE), "<hex>"),89 (re.compile(r"\b(?=[A-Za-z0-9_-]*\d)(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9_-]{32,}\b"), "<token>"),90 (re.compile(r"([?&](?:v|ver|version|rev|build|cb|cache|nocache|_|t|ts|timestamp|hash|nonce|csrf|token|_token|csrfmiddlewaretoken|"91 r"authenticity_token|utm_[a-z]+|fbclid|gclid|sessionid|session_id|phpsessid|jsessionid)=)[^&\s#\"']+", re.IGNORECASE), r"\1<q>"),92 (re.compile(r"\b(?:nonce|csrf[-_]?token|xsrf[-_]?token|session[-_]?id|request[-_]?id|trace[-_]?id|build[-_]?id)\s*[:=]\s*['\"]?[A-Za-z0-9_\-+/=.]{8,}", re.IGNORECASE), "<token>"),93]94WS_RE = re.compile(r"[ \t\r\f\v\u00a0\u1680\u2000-\u200b\u2028\u2029\u202f\u205f\u3000\ufeff]+")95NL_RE = re.compile(r"\s*\n\s*")96MULTI_NL_RE = re.compile(r"\n{3,}")979899def normalize_whitespace(text: str) -> str:100 text = text.replace("\u00ad", "").replace("\ufeff", "")101 text = WS_RE.sub(" ", text)102 text = NL_RE.sub("\n", text)103 text = MULTI_NL_RE.sub("\n\n", text)104 return text.strip()105106107def normalized_text(text: str) -> str:108 """Noise-normalised, lower-cased text for hashing/diffing only. The stored text keeps the originals."""109 out = normalize_whitespace(text)110 for pat, repl in NOISE_RULES:111 out = pat.sub(repl, out)112 return WS_RE.sub(" ", out).lower().strip()113114115def text_hash(text: str) -> str:116 return hashlib.sha256(normalized_text(text).encode("utf-8")).hexdigest()117118119def structural_hash(blocks: list[Block]) -> str:120 seq = "\n".join(f"{b.kind}|{b.key}" for b in blocks)121 return hashlib.sha256(seq.encode("utf-8")).hexdigest()122123124# ------------------------------------------------------------------------------------------------------------ simhash125126_TOKEN_RE = re.compile(r"[\w<>]+", re.UNICODE)127128129def _features(text: str) -> Counter[str]:130 toks = _TOKEN_RE.findall(text)131 feats: Counter[str] = Counter(toks)132 for i in range(len(toks) - 1):133 feats[toks[i] + " " + toks[i + 1]] += 1134 return feats135136137def simhash(text: str) -> int:138 """64-bit simhash over word uni+bigrams of the noise-normalised text (0 for empty)."""139 feats = _features(normalized_text(text))140 if not feats:141 return 0142 v = [0] * 64143 for feat, w in feats.items():144 h = int.from_bytes(hashlib.blake2b(feat.encode("utf-8"), digest_size=8).digest(), "big")145 for i in range(64):146 v[i] += w if (h >> i) & 1 else -w147 out = 0148 for i in range(64):149 if v[i] > 0:150 out |= 1 << i151 return out152153154def hamming(a: int, b: int) -> int:155 return (a ^ b).bit_count()156157158def simhash_bucket(h: int, bits: int = 12) -> str:159 return format(h >> (64 - bits), "x") if h else "0"160161162# ------------------------------------------------------------------------------------------------------------ language guess163164_STOPWORDS: dict[str, frozenset[str]] = {165 "en": frozenset(["the", "and", "of", "to", "in", "for", "with", "on", "is", "are", "our", "we", "you", "your", "this", "that", "from", "by", "at", "as", "be"]),166 "fr": frozenset(["le", "la", "les", "et", "de", "des", "du", "en", "pour", "avec", "sur", "est", "sont", "nous", "vous", "votre", "notre", "une", "un", "dans", "par", "au", "aux", "ce", "cette"]),167 "de": frozenset(["der", "die", "das", "und", "von", "zu", "in", "für", "mit", "auf", "ist", "sind", "wir", "sie", "ihre", "unsere", "eine", "ein", "im", "den", "dem", "nicht"]),168 "es": frozenset(["el", "la", "los", "las", "y", "de", "del", "en", "para", "con", "sobre", "es", "son", "nosotros", "su", "nuestra", "una", "un", "por", "al", "como", "más"]),169 "it": frozenset(["il", "la", "gli", "le", "e", "di", "del", "della", "in", "per", "con", "su", "è", "sono", "noi", "nostro", "una", "un", "dal", "nel", "che"]),170 "pt": frozenset(["o", "a", "os", "as", "e", "de", "do", "da", "em", "para", "com", "sobre", "é", "são", "nós", "nosso", "uma", "um", "pelo", "na", "no", "que"]),171 "nl": frozenset(["de", "het", "een", "en", "van", "voor", "met", "op", "is", "zijn", "wij", "onze", "je", "jouw", "dat", "dit", "niet", "ook"]),172}173_CJK_RE = re.compile(r"[-ヿ]")174_HAN_RE = re.compile(r"[一-鿿]")175_HANGUL_RE = re.compile(r"[가-]")176_CYR_RE = re.compile(r"[Ѐ-ӿ]")177_ARAB_RE = re.compile(r"[-ۿ]")178179180def language_guess(text: str, html_lang: str | None = None) -> str | None:181 if html_lang:182 code = html_lang.strip().lower().split("-")[0].split("_")[0]183 if 2 <= len(code) <= 3 and code.isalpha():184 return code185 sample = text[:6000]186 if not sample.strip():187 return None188 if _CJK_RE.search(sample):189 return "ja"190 if _HANGUL_RE.search(sample):191 return "ko"192 if _HAN_RE.search(sample) and len(_HAN_RE.findall(sample)) > 20:193 return "zh"194 if len(_CYR_RE.findall(sample)) > 40:195 return "ru"196 if len(_ARAB_RE.findall(sample)) > 40:197 return "ar"198 words = re.findall(r"[a-zà-ÿ']+", sample.lower())199 if len(words) < 8:200 return None201 best, best_n = None, 0202 for lang, sw in _STOPWORDS.items():203 n = sum(1 for w in words if w in sw)204 if n > best_n:205 best, best_n = lang, n206 if best_n / max(1, len(words)) < 0.04:207 return None208 return best209210211# ------------------------------------------------------------------------------------------------------------ page model212213214@dataclass(slots=True)215class Link:216 url: str217 anchor: str218 region: str = "main" # nav | header | footer | main219 rel: str = ""220 title: str = ""221222223@dataclass(slots=True)224class NormalizedPage:225 url: str226 title: str | None227 lang: str | None228 meta: dict[str, Any]229 blocks: list[Block]230 text: str # stored main text (original wording, whitespace-normalised)231 full_text: str # including nav/header/footer232 links: list[Link]233 jsonld: dict[str, list[dict[str, Any]]]234 microdata: dict[str, list[dict[str, Any]]]235 headings: list[tuple[int, str]]236 feeds: list[str] = field(default_factory=list)237 main_selector: str = "body"238239 @property240 def normalized(self) -> str:241 return normalized_text(self.text)242243 def blocks_of(self, *kinds: str) -> list[Block]:244 return [b for b in self.blocks if b.kind in kinds]245246 def to_extraction(self) -> Extraction:247 return Extraction(text=self.text, blocks=self.blocks, title=self.title, language=self.lang, meta=dict(self.meta))248249250# ------------------------------------------------------------------------------------------------------------ parsing251252253def _attr(node: LexborNode, name: str) -> str:254 try:255 v = node.attributes.get(name)256 except Exception: # noqa: BLE001257 return ""258 return (v or "").strip() if v is not None else ("" if name not in node.attributes else "true")259260261def _classes(node: LexborNode) -> str:262 return f"{_attr(node, 'id')} {_attr(node, 'class')} {_attr(node, 'role')} {_attr(node, 'data-testid')} {_attr(node, 'itemtype')}".strip()263264265def _is_hidden(node: LexborNode) -> bool:266 if node.tag in ("html", "body"):267 return False268 attrs = node.attributes269 if "hidden" in attrs and attrs.get("hidden") != "false":270 return True271 if (attrs.get("aria-hidden") or "").strip().lower() == "true":272 return True273 style = (attrs.get("style") or "").replace(" ", "").lower()274 return "display:none" in style or "visibility:hidden" in style275276277def _is_cookie(node: LexborNode) -> bool:278 cls = _classes(node)279 return bool(cls) and bool(COOKIE_RE.search(cls)) and node.tag in ("div", "section", "aside", "dialog", "footer", "header", "nav", "form")280281282def _region_of(node: LexborNode) -> str | None:283 tag = node.tag284 role = _attr(node, "role").lower()285 cls = _classes(node)286 if tag == "nav" or role == "navigation" or NAV_CLASS_RE.search(cls):287 return "nav"288 if tag == "footer" or role == "contentinfo" or FOOTER_CLASS_RE.search(cls):289 return "footer"290 if tag == "header" or role == "banner":291 return "header"292 return None293294295def _node_text(node: LexborNode) -> str:296 """Text with block boundaries as newlines and inline elements joined by spaces."""297 parts: list[str] = []298299 def walk(n: LexborNode, depth: int) -> None:300 if depth > 400:301 return302 for c in n.iter(include_text=True):303 tag = c.tag304 if tag == "-text":305 t = c.text_content306 if t:307 parts.append(t)308 continue309 if tag in DROP_TAGS or tag == "-comment" or _is_hidden(c) or _is_cookie(c):310 continue311 if tag == "br":312 parts.append("\n")313 continue314 block = tag in BLOCK_TAGS315 if block:316 parts.append("\n")317 walk(c, depth + 1)318 if block:319 parts.append("\n")320 else:321 parts.append(" ")322323 walk(node, 0)324 return normalize_whitespace(html_lib.unescape("".join(parts)))325326327def _first_text(node: LexborNode, limit: int = 300) -> str:328 return _node_text(node)[:limit]329330331def _card_kind(node: LexborNode, *, allow_container: bool = False) -> str | None:332 cls = _classes(node)333 if not cls:334 return None335 kind = next((k for k, pat in CARD_PATTERNS if pat.search(cls)), None)336 if kind is None:337 return None338 if not allow_container and _is_card_container(node):339 return None340 return kind341342343LISTING_LINK_SHARE = 0.6 # a list whose items mostly carry links is a listing (jobs, news, products), not a bullet list344LISTING_MIN_ITEMS = 2345346347def _list_items(node: LexborNode) -> list[LexborNode]:348 items: list[LexborNode] = []349 for c in node.iter():350 if c.tag in ("li", "tr"):351 items.append(c)352 elif c.tag in ("tbody", "thead", "tfoot"):353 items.extend(x for x in c.iter() if x.tag == "tr")354 return items355356357def _is_listing(node: LexborNode) -> bool:358 """`ul`/`ol`/`table` whose items are card-like or mostly linked — the rows are entities, the wrapper is a container."""359 items = _list_items(node)360 if len(items) < LISTING_MIN_ITEMS:361 return False362 if any(_card_kind(i, allow_container=True) for i in items):363 return True364 linked = sum(1 for i in items if i.css_first("a[href]") is not None)365 return linked >= len(items) * LISTING_LINK_SHARE366367368def _card_text(node: LexborNode) -> str:369 """Card text with one line per direct child element, so inline cells (`<a>title</a><span>city</span>`) do not merge into one line."""370 parts: list[str] = []371 for c in node.iter(include_text=True):372 if c.tag == "-text":373 t = c.text_content374 if t and t.strip():375 parts.append(t)376 elif c.tag in DROP_TAGS or c.tag == "-comment" or c.tag == "br" or _is_hidden(c) or _is_cookie(c):377 continue378 else:379 parts.append(_node_text(c))380 return normalize_whitespace(html_lib.unescape("\n".join(p for p in parts if p)))381382383def _is_card_container(node: LexborNode) -> bool:384 """A grid/list wrapper (`.products`, `.news-list`, `.team-grid`, `.job-list`) holds ≥ 2 card-like children — or a listing-like list/table —385 so the children are segmented instead of the wrapper becoming a single card. A feature `<ul>` inside a pricing card is not a listing."""386 n = 0387 for child in node.iter():388 if child.tag not in ("div", "section", "article", "li", "a", "figure", "tr", "ul", "ol", "table"):389 continue390 if child.tag in ("ul", "ol", "table") and _is_listing(child):391 return True392 is_card = child.tag in ("ul", "ol", "table") or _card_kind(child, allow_container=True) is not None393 if not is_card and child.tag == "div":394 is_card = any(_card_kind(g, allow_container=True) for g in child.iter() if g.tag in ("div", "article", "li", "a"))395 if is_card:396 n += 1397 if n >= 2:398 return True399 return False400401402class _Segmenter:403 def __init__(self, base_url: str):404 self.base_url = base_url405 self.blocks: list[Block] = []406 self.heading_stack: list[tuple[int, str]] = []407 self.headings: list[tuple[int, str]] = []408 self.order = 0409 self.hero_done = False410 self.text_parts: list[str] = []411412 # ---------------------------------------------------------------- helpers413 def path(self) -> str:414 return " > ".join(h[1] for h in self.heading_stack)[:200]415416 def push_heading(self, level: int, text: str) -> None:417 while self.heading_stack and self.heading_stack[-1][0] >= level:418 self.heading_stack.pop()419 self.heading_stack.append((level, text[:80]))420 self.headings.append((level, text))421422 def add(self, kind: str, text: str, *, path: str | None = None, attrs: dict[str, Any] | None = None, weight: float | None = None) -> None:423 text = normalize_whitespace(text)424 if not text:425 return426 if len(text) > 6000:427 text = text[:6000]428 p = self.path() if path is None else path429 self.blocks.append(Block(key="", kind=kind, text=text, path=p, order=self.order, weight=weight or BLOCK_WEIGHTS.get(kind, 1.0),430 attrs=attrs or {}))431 self.order += 1432433 # ---------------------------------------------------------------- traversal434 def walk(self, node: LexborNode, region: str, depth: int = 0, hint: str | None = None) -> None:435 """`hint` is the card kind of an enclosing container (`.job-list`), inherited by plain list rows / table rows underneath it."""436 if depth > 300:437 return438 for c in node.iter(include_text=True):439 tag = c.tag440 if tag == "-text":441 t = normalize_whitespace(c.text_content or "")442 if t and region == "main":443 self.add("paragraph" if len(t) > 40 else "other", t)444 elif t:445 self.add(region, t)446 continue447 if tag in DROP_TAGS or tag == "-comment" or _is_hidden(c) or _is_cookie(c):448 continue449 sub = _region_of(c) if region == "main" else None450 if sub is not None:451 txt = _node_text(c)452 if txt:453 kind = "nav" if sub == "nav" else ("footer" if sub == "footer" else "header")454 self.add(kind, txt[:1500], path="", attrs={"region": sub})455 continue456 if region != "main":457 # inside nav/header/footer: flatten into one block per top-level child458 txt = _node_text(c)459 if txt:460 self.add(region, txt[:1500], path="", attrs={"region": region})461 continue462 if tag in HEADING_TAGS:463 txt = _node_text(c)464 if txt:465 level = int(tag[1])466 self.push_heading(level, txt)467 self.add("heading", txt, path=" > ".join(h[1] for h in self.heading_stack[:-1])[:200],468 attrs={"level": level}, weight=BLOCK_WEIGHTS["heading"] * (1.3 if level == 1 else 1.0))469 continue470 if tag == "p":471 self.add("paragraph", _node_text(c))472 continue473 if tag == "blockquote":474 self.add("quote", _node_text(c))475 continue476 if tag == "pre" or (tag == "code" and len(_node_text(c)) > 80):477 self.add("code", _node_text(c))478 continue479 if tag in ("details",):480 self.add("faq", _node_text(c))481 continue482 if tag in ("ul", "ol", "dl", "menu"):483 self.list_block(c, hint)484 continue485 if tag == "table":486 self.table_block(c, hint)487 continue488 if tag in ("hr", "br", "wbr"):489 continue490 card = _card_kind(c) if tag in ("div", "section", "article", "li", "a", "figure", "aside", "tr", "td") else None491 if card is not None:492 txt = _card_text(c)493 if txt and len(txt) <= CARD_MAX_TEXT:494 attrs: dict[str, Any] = {}495 href = self._first_href(c)496 if href:497 attrs["href"] = href498 self.add(card, txt, attrs=attrs)499 continue500 if not self.hero_done and tag in ("section", "div", "header", "article") and (HERO_CLASS_RE.search(_classes(c)) or self._has_h1(c)):501 txt = _node_text(c)502 if txt and len(txt) <= 2500:503 self.hero_done = True504 for h in c.css("h1"):505 ht = _node_text(h)506 if ht:507 self.push_heading(1, ht)508 break509 self.add("hero", txt, path="", attrs={"href": self._first_href(c)} if self._first_href(c) else {})510 continue511 container_kind = _card_kind(c, allow_container=True) if tag in ("div", "section", "article", "ul", "ol") else None512 self.walk(c, region, depth + 1, container_kind or hint)513514 def _has_h1(self, node: LexborNode) -> bool:515 return node.css_first("h1") is not None and len(_node_text(node)) < 2500516517 def _first_href(self, node: LexborNode) -> str | None:518 a = node if node.tag == "a" else node.css_first("a[href]")519 if a is None:520 return None521 from companyatlas.urls import absolutize522523 return absolutize(self.base_url, _attr(a, "href"))524525 def list_block(self, node: LexborNode, hint: str | None = None) -> None:526 items = [li for li in node.iter() if li.tag in ("li", "dt", "dd")]527 if not items:528 txt = _node_text(node)529 if txt:530 self.add("list", txt)531 return532 texts = [_card_text(li) for li in items]533 texts_nonempty = [t for t in texts if t]534 if not texts_nonempty:535 return536 avg = sum(len(t) for t in texts_nonempty) / len(texts_nonempty)537 has_links = sum(1 for li in items if li.css_first("a[href]") is not None)538 listing_like = len(items) >= 3 and (has_links >= len(items) * LISTING_LINK_SHARE or avg > 60)539 if not listing_like or len(items) > 400:540 self.add("list", "\n".join(texts_nonempty)[:4000])541 return542 for li, t in zip(items, texts, strict=False):543 if not t:544 continue545 kind = _card_kind(li, allow_container=True) or _card_kind(node, allow_container=True) or hint or "list"546 attrs: dict[str, Any] = {}547 href = self._first_href(li)548 if href:549 attrs["href"] = href550 self.add(kind, t[:1500], attrs=attrs)551552 def table_block(self, node: LexborNode, hint: str | None = None) -> None:553 rows = [tr for tr in node.css("tr")]554 caption = node.css_first("caption")555 cap = _node_text(caption) if caption is not None else ""556 if len(rows) <= 1 or len(rows) > 500:557 txt = _node_text(node)558 if txt:559 self.add("table", txt[:4000])560 return561 header_cells = [_node_text(th) for th in rows[0].css("th")]562 header = " | ".join(x for x in header_cells if x)563 base_path = self.path()564 table_path = " > ".join(x for x in (base_path, cap or header[:80]) if x)[:200]565 for i, tr in enumerate(rows):566 cells = [_node_text(td) for td in tr.iter() if td.tag in ("td", "th")]567 txt = " | ".join(c for c in cells if c)568 if not txt:569 continue570 kind = _card_kind(tr, allow_container=True) or hint or "table"571 attrs: dict[str, Any] = {"row": i}572 href = self._first_href(tr)573 if href:574 attrs["href"] = href575 self.add(kind, txt[:1500], path=table_path, attrs=attrs)576577578def _finalize_blocks(blocks: list[Block]) -> list[Block]:579 total = len(blocks) or 1580 seen: Counter[str] = Counter()581 for b in blocks:582 norm = normalized_text(b.text)583 b.hash = hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]584 b.simhash = simhash(b.text)585 path_h = hashlib.blake2b(b.path.lower().encode("utf-8"), digest_size=3).hexdigest() if b.path else "root"586 base = f"{b.kind}:{path_h}:{simhash_bucket(b.simhash)}"587 n = seen[base]588 seen[base] += 1589 b.key = base if n == 0 else f"{base}#{n + 1}"590 rel = b.order / total591 pos = 1.1 if rel < 0.1 else (0.9 if rel > 0.9 else 1.0)592 if b.kind in LOW_VALUE_KINDS:593 pos = 1.0594 b.weight = round(b.weight * pos, 3)595 return blocks596597598def _collect_links(tree: LexborHTMLParser, base_url: str) -> list[Link]:599 from companyatlas.urls import absolutize600601 links: list[Link] = []602 seen: set[tuple[str, str]] = set()603 for a in tree.css("a[href]"):604 href = _attr(a, "href")605 url = absolutize(base_url, href)606 if not url:607 continue608 anchor = _node_text(a)[:160] or _attr(a, "aria-label")[:160] or _attr(a, "title")[:160]609 region = "main"610 p = a.parent611 hops = 0612 while p is not None and hops < 40:613 r = _region_of(p) if p.tag not in ("html", "body") else None614 if r:615 region = r616 break617 p = p.parent618 hops += 1619 k = (url, anchor.lower())620 if k in seen:621 continue622 seen.add(k)623 links.append(Link(url=url, anchor=anchor, region=region, rel=_attr(a, "rel"), title=_attr(a, "title")))624 if len(links) >= 3000:625 break626 return links627628629def _meta(tree: LexborHTMLParser, base_url: str) -> tuple[str | None, str | None, dict[str, Any], list[str]]:630 from companyatlas.urls import absolutize631632 meta: dict[str, Any] = {}633 title_node = tree.css_first("title")634 title = normalize_whitespace(title_node.text()) if title_node is not None else None635 lang = _attr(tree.root, "lang") if tree.root is not None else ""636 if not lang:637 html_node = tree.css_first("html")638 lang = _attr(html_node, "lang") if html_node is not None else ""639 feeds: list[str] = []640 for m in tree.css("meta"):641 name = (_attr(m, "name") or _attr(m, "property") or _attr(m, "http-equiv")).lower()642 content = _attr(m, "content")643 if not name or not content:644 continue645 if name in ("description", "generator", "robots", "author", "keywords", "twitter:card", "theme-color") or name.startswith("og:") and name in ("og:title", "og:description", "og:type", "og:site_name", "og:locale", "og:url"):646 meta[name.replace(":", "_")] = content[:500]647 elif name in ("content-language",) and not lang:648 lang = content649 for ln in tree.css("link[rel]"):650 rel = _attr(ln, "rel").lower()651 href = _attr(ln, "href")652 if not href:653 continue654 if "canonical" in rel:655 meta["canonical"] = absolutize(base_url, href) or href656 elif "alternate" in rel:657 typ = _attr(ln, "type").lower()658 if "rss" in typ or "atom" in typ or "feed" in typ or "json" in typ and "feed" in href:659 u = absolutize(base_url, href)660 if u and u not in feeds:661 feeds.append(u)662 elif _attr(ln, "hreflang"):663 meta["hreflang_count"] = int(meta.get("hreflang_count", 0)) + 1664 return title or None, (lang or None), meta, feeds665666667# ------------------------------------------------------------------------------------------------------------ JSON-LD / microdata668669JSONLD_BUCKETS: dict[str, str] = {670 "organization": "organizations", "corporation": "organizations", "localbusiness": "organizations", "ngo": "organizations",671 "jobposting": "job_postings", "product": "products", "offer": "offers", "aggregateoffer": "offers", "service": "products",672 "softwareapplication": "products", "newsarticle": "articles", "blogposting": "articles", "article": "articles", "pressrelease": "articles",673 "techarticle": "articles", "report": "articles", "person": "persons", "postaladdress": "addresses", "place": "places",674 "breadcrumblist": "breadcrumbs", "faqpage": "faqs", "website": "websites", "webpage": "webpages", "event": "events",675}676677678def _iter_jsonld(obj: Any, out: list[dict[str, Any]], depth: int = 0) -> None:679 if depth > 8 or len(out) > 500:680 return681 if isinstance(obj, list):682 for x in obj:683 _iter_jsonld(x, out, depth + 1)684 elif isinstance(obj, dict):685 if "@graph" in obj and isinstance(obj["@graph"], list):686 _iter_jsonld(obj["@graph"], out, depth + 1)687 if "@type" in obj:688 out.append(obj)689 for k in ("mainEntity", "itemListElement", "hasPart", "member", "employee", "founder", "address", "location", "offers", "item"):690 if k in obj:691 _iter_jsonld(obj[k], out, depth + 1)692693694def extract_jsonld(tree: LexborHTMLParser) -> dict[str, list[dict[str, Any]]]:695 buckets: dict[str, list[dict[str, Any]]] = {}696 for s in tree.css('script[type="application/ld+json"], script[type="application/json+ld"]'):697 raw = (s.text() or "").strip()698 if not raw or len(raw) > 2_000_000:699 continue700 try:701 data = json.loads(raw)702 except json.JSONDecodeError:703 try:704 data = json.loads(html_lib.unescape(raw).replace("\n", " "))705 except json.JSONDecodeError:706 continue707 found: list[dict[str, Any]] = []708 _iter_jsonld(data, found)709 for item in found:710 types = item.get("@type")711 for t in (types if isinstance(types, list) else [types]):712 if not isinstance(t, str):713 continue714 key = JSONLD_BUCKETS.get(t.rsplit("/", 1)[-1].lower())715 if key:716 buckets.setdefault(key, []).append(item)717 return buckets718719720def extract_microdata(tree: LexborHTMLParser) -> dict[str, list[dict[str, Any]]]:721 buckets: dict[str, list[dict[str, Any]]] = {}722 for scope in tree.css("[itemscope][itemtype]"):723 itype = _attr(scope, "itemtype").rsplit("/", 1)[-1].lower()724 key = JSONLD_BUCKETS.get(itype)725 if not key:726 continue727 item: dict[str, Any] = {"@type": itype}728 for prop in scope.css("[itemprop]"):729 name = _attr(prop, "itemprop")730 if not name or "itemscope" in prop.attributes:731 if name and "itemscope" in prop.attributes:732 item[name] = _first_text(prop, 200)733 continue734 if prop.tag == "a" or prop.tag == "link":735 val = _attr(prop, "href")736 elif prop.tag in ("meta",):737 val = _attr(prop, "content")738 elif prop.tag == "time":739 val = _attr(prop, "datetime") or _first_text(prop, 100)740 elif prop.tag == "img":741 val = _attr(prop, "src")742 else:743 val = _first_text(prop, 300)744 if val and name not in item:745 item[name] = val746 buckets.setdefault(key, []).append(item)747 if sum(len(v) for v in buckets.values()) > 500:748 break749 return buckets750751752# ------------------------------------------------------------------------------------------------------------ main-content detection753754MAIN_SELECTORS = ("main", "[role=main]", "article", "#main", "#main-content", "#content", ".main-content", "#primary", ".site-main", ".content")755756757def _prune(tree: LexborHTMLParser) -> None:758 for tag in ("script", "style", "noscript", "svg", "iframe", "template", "canvas", "video", "audio", "object", "embed", "input", "select",759 "textarea", "option", "datalist"):760 for n in tree.css(tag):761 n.decompose()762763764def _find_main(tree: LexborHTMLParser) -> tuple[LexborNode | None, str]:765 body = tree.body766 if body is None:767 return None, "body"768 for sel in MAIN_SELECTORS:769 try:770 n = tree.css_first(sel)771 except Exception as exc: # noqa: BLE001772 log.debug("main selector failed", extra={"selector": sel, "error": str(exc)})773 continue774 if n is not None and not _is_hidden(n) and len(_node_text(n)) >= 80:775 return n, sel776 # largest text region among body's block descendants (depth ≤ 3) that is not nav/header/footer777 best, best_len = None, 0778 candidates: list[LexborNode] = []779780 def collect(n: LexborNode, d: int) -> None:781 for c in n.iter():782 if c.tag in ("div", "section", "article", "td") and not _is_hidden(c) and _region_of(c) is None:783 candidates.append(c)784 if d < 3:785 collect(c, d + 1)786787 collect(body, 0)788 body_len = len(_node_text(body)) or 1789 for c in candidates[:400]:790 ln = len(_node_text(c))791 if ln > best_len and ln >= body_len * 0.35:792 best, best_len = c, ln793 return (best or body), ("largest" if best is not None else "body")794795796def parse(html_text: str, *, url: str = "", surface: str | None = None) -> NormalizedPage:797 """Parse HTML into blocks + text + metadata. `surface` is a hint only (kept in meta)."""798 tree = LexborHTMLParser(html_text or "")799 title, lang, meta, feeds = _meta(tree, url)800 jsonld = extract_jsonld(tree)801 microdata = extract_microdata(tree)802 links = _collect_links(tree, url)803 _prune(tree)804 body = tree.body805 seg = _Segmenter(url)806 main_node, main_sel = _find_main(tree)807 if body is not None:808 # nav/header/footer first as low-weight blocks (in document order they usually wrap main; we walk body and let the segmenter route)809 seg.walk(body, "main")810 blocks = _finalize_blocks(seg.blocks)811 main_text = _node_text(main_node) if main_node is not None else ""812 full_text = _node_text(body) if body is not None else ""813 if not main_text:814 main_text = "\n".join(b.text for b in blocks if b.kind not in LOW_VALUE_KINDS)815 meta = {**meta, "lang": lang, "main_selector": main_sel, "jsonld_types": sorted({k for k in jsonld}), "normalize_version": NORMALIZE_VERSION}816 if surface:817 meta["surface_hint"] = surface818 if feeds:819 meta["feeds"] = feeds[:10]820 orgs = jsonld.get("organizations") or []821 if orgs:822 same_as = [s for o in orgs for s in (o.get("sameAs") or []) if isinstance(s, str)] if isinstance(orgs[0].get("sameAs", []), list) else []823 if same_as:824 meta["same_as"] = same_as[:20]825 language = language_guess(main_text, lang)826 return NormalizedPage(url=url, title=title, lang=language, meta=meta, blocks=blocks, text=main_text, full_text=full_text, links=links,827 jsonld=jsonld, microdata=microdata, headings=seg.headings, feeds=feeds, main_selector=main_sel)828829830__all__ = ["BLOCK_WEIGHTS", "LOW_VALUE_KINDS", "NORMALIZE_VERSION", "Link", "NormalizedPage", "extract_jsonld", "extract_microdata", "hamming",831 "language_guess", "normalize_whitespace", "normalized_text", "parse", "simhash", "simhash_bucket", "structural_hash", "text_hash"]832