"""HTML → semantic building blocks (spec §17–19, §107).

    html  ──parse()──▶  NormalizedPage(title, meta, lang, blocks, text, links, jsonld, microdata)  ──to_extraction()──▶  Extraction

Two layers are kept deliberately separate:

* **stored text** — whitespace-normalised, but otherwise the original (what a human reads in the historical viewer);
* **hash / diff layer** — `normalized_text()` replaces classic noise (dates, times, "3 minutes ago", counters, csrf/nonce/session
  tokens, cache-busting query params) with stable tokens so that `text_hash`, block hashes and simhashes ignore it.

Blocks carry a stable `key` = kind + heading path + simhash bucket (+ occurrence index) — never DOM position — so a reordered
section is a *move*, not an add/remove pair. Deterministic; no network; no LLM.
"""
from __future__ import annotations

import hashlib
import html as html_lib
import json
import logging
import re
from collections import Counter
from dataclasses import dataclass, field
from typing import Any

from selectolax.lexbor import LexborHTMLParser, LexborNode

from companyatlas.sdk.models import Block, Extraction

log = logging.getLogger(__name__)

NORMALIZE_VERSION = "normalize-v1"

# ------------------------------------------------------------------------------------------------------------ element sets

DROP_TAGS = frozenset({"script", "style", "noscript", "svg", "iframe", "template", "canvas", "video", "audio", "source", "track", "object",
                       "embed", "map", "area", "input", "select", "textarea", "option", "optgroup", "datalist", "meter", "progress", "link",
                       "meta", "base", "head", "picture", "img", "dialog", "math"})
BLOCK_TAGS = frozenset({"address", "article", "aside", "blockquote", "body", "dd", "details", "div", "dl", "dt", "fieldset", "figcaption",
                        "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre",
                        "section", "summary", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul", "caption", "legend", "menu"})
HEADING_TAGS = ("h1", "h2", "h3", "h4", "h5", "h6")

COOKIE_RE = re.compile(r"(cookie|consent|gdpr|onetrust|cookiebot|truste|didomi|usercentrics|osano|cc-banner|cc-window|privacy-banner|"
                       r"cmp-container|qc-cmp|sp_message|termly|iubenda|klaro|cookieyes|axeptio|tarteaucitron)", re.IGNORECASE)
NAV_CLASS_RE = re.compile(r"(^|[\s_-])(nav|navbar|navigation|menu|breadcrumbs?|topbar|masthead|site-header|global-header)([\s_-]|$)", re.IGNORECASE)
FOOTER_CLASS_RE = re.compile(r"(^|[\s_-])(footer|site-footer|global-footer|colophon|legal-links)([\s_-]|$)", re.IGNORECASE)
HERO_CLASS_RE = re.compile(r"(^|[\s_-])(hero|jumbotron|banner|masthead|splash|intro|cover|landing-hero)([\s_-]|$)", re.IGNORECASE)
CARD_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
    ("pricing_plan", re.compile(r"(^|[\s_-])(pricing[-_ ]?(plan|card|tier|column|table|box|option)|plan[-_ ]?(card|box|column|tier|item)|tier[-_ ]?card|"
                                r"price[-_ ]?(card|box|column|plan)|package[-_ ]?(card|box))([\s_-]|$)", re.IGNORECASE)),
    ("job_listing", re.compile(r"(^|[\s_-])(job|jobs|opening|position|vacancy|posting|role|career)[-_ ]?(item|card|listing|row|link|entry|tile|post)?([\s_-]|$)", re.IGNORECASE)),
    ("person", re.compile(r"(^|[\s_-])(team[-_ ]?member|member[-_ ]?card|person|people[-_ ]?card|bio|executive|leader|profile[-_ ]?card|staff|founder|"
                          r"board[-_ ]?member|management[-_ ]?member|employee[-_ ]?card|director)([\s_-]|$)", re.IGNORECASE)),
    ("location", re.compile(r"(^|[\s_-])(office|location|store|branch|address|showroom|site[-_ ]?card|headquarters|hq)([\s_-]|$)", re.IGNORECASE)),
    ("news_item", re.compile(r"(^|[\s_-])(news|press|article|post|release|story|blog|announcement|update|publication)[-_ ]?(item|card|teaser|tile|preview|summary|link|list-item|entry)?([\s_-]|$)", re.IGNORECASE)),
    ("product_card", re.compile(r"(^|[\s_-])((product|products|catalog|sku|offering|solution)[-_ ]?(card|tile|item|box|grid-item|teaser|entry)?|item[-_ ]?card|grid[-_ ]item)([\s_-]|$)", re.IGNORECASE)),
    ("faq", re.compile(r"(^|[\s_-])(faq|accordion|question|collapsible)([\s_-]|$)", re.IGNORECASE)),
]
CARD_MAX_TEXT = 1800

# Weights per block kind (spec §19–20): what matters for significance. Nav/footer/cookie are kept for discovery, not for change value.
BLOCK_WEIGHTS: dict[str, float] = {
    "hero": 1.5, "pricing_plan": 1.6, "job_listing": 1.4, "person": 1.4, "product_card": 1.3, "location": 1.2, "news_item": 1.2,
    "heading": 1.0, "paragraph": 1.0, "section": 1.0, "table": 1.1, "list": 0.9, "faq": 0.8, "code": 0.7, "quote": 0.6, "other": 0.5,
    "header": 0.3, "nav": 0.2, "footer": 0.15,
}
LOW_VALUE_KINDS = frozenset({"nav", "footer", "header"})

# ------------------------------------------------------------------------------------------------------------ noise normalisation

_MONTH = r"(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?|" \
         r"janvier|février|fevrier|mars|avril|mai|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre|" \
         r"januar|februar|märz|maerz|april|juni|juli|august|oktober|dezember|enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)"
NOISE_RULES: list[tuple[re.Pattern[str], str]] = [
    (re.compile(r"\b\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?\b"), "<date>"),
    (re.compile(r"\b(?:\d{1,2}(?:st|nd|rd|th)?\s+)?" + _MONTH + r"\.?\s+\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\b", re.IGNORECASE), "<date>"),
    (re.compile(r"\b\d{1,2}(?:st|nd|rd|th)?\s+" + _MONTH + r"\.?,?\s+\d{4}\b", re.IGNORECASE), "<date>"),
    (re.compile(r"\b" + _MONTH + r"\.?\s+\d{4}\b", re.IGNORECASE), "<date>"),
    (re.compile(r"\b\d{1,2}[/.-]\d{1,2}[/.-]\d{2,4}\b"), "<date>"),
    (re.compile(r"\b\d{1,2}:\d{2}(?::\d{2})?\s*(?:am|pm|a\.m\.|p\.m\.|utc|gmt|est|pst|cet|z)?\b", re.IGNORECASE), "<time>"),
    (re.compile(r"\b(?:\d+|a|an|one|few|several)\s+(?:sec(?:ond)?s?|min(?:ute)?s?|hours?|hrs?|days?|weeks?|months?|years?)\s+ago\b", re.IGNORECASE), "<rel>"),
    (re.compile(r"\b(?:il y a|hace|vor)\s+\d+\s+\w+\b", re.IGNORECASE), "<rel>"),
    (re.compile(r"\b(?:yesterday|today|just now|hier|aujourd'hui|heute|gestern)\b", re.IGNORECASE), "<rel>"),
    (re.compile(r"\b\d[\d,.]*\s*(?:k|m)?\s*(?:views?|comments?|likes?|shares?|followers?|visitors?|reads?|replies|upvotes?|downloads?|stars?|members?|online)\b", re.IGNORECASE), "<count>"),
    (re.compile(r"(?:©|\(c\)|\bcopyright)\s*(?:\d{4}\s*[-–]\s*)?\d{4}\b", re.IGNORECASE), "<copyright>"),
    (re.compile(r"\b(?:\d{4}\s*[-–]\s*)?\d{4}\s*(?:©|\(c\))", re.IGNORECASE), "<copyright>"),
    (re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.IGNORECASE), "<uuid>"),
    (re.compile(r"\b[0-9a-f]{24,}\b", re.IGNORECASE), "<hex>"),
    (re.compile(r"\b(?=[A-Za-z0-9_-]*\d)(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9_-]{32,}\b"), "<token>"),
    (re.compile(r"([?&](?:v|ver|version|rev|build|cb|cache|nocache|_|t|ts|timestamp|hash|nonce|csrf|token|_token|csrfmiddlewaretoken|"
                r"authenticity_token|utm_[a-z]+|fbclid|gclid|sessionid|session_id|phpsessid|jsessionid)=)[^&\s#\"']+", re.IGNORECASE), r"\1<q>"),
    (re.compile(r"\b(?:nonce|csrf[-_]?token|xsrf[-_]?token|session[-_]?id|request[-_]?id|trace[-_]?id|build[-_]?id)\s*[:=]\s*['\"]?[A-Za-z0-9_\-+/=.]{8,}", re.IGNORECASE), "<token>"),
]
WS_RE = re.compile(r"[ \t\r\f\v\u00a0\u1680\u2000-\u200b\u2028\u2029\u202f\u205f\u3000\ufeff]+")
NL_RE = re.compile(r"\s*\n\s*")
MULTI_NL_RE = re.compile(r"\n{3,}")


def normalize_whitespace(text: str) -> str:
    text = text.replace("\u00ad", "").replace("\ufeff", "")
    text = WS_RE.sub(" ", text)
    text = NL_RE.sub("\n", text)
    text = MULTI_NL_RE.sub("\n\n", text)
    return text.strip()


def normalized_text(text: str) -> str:
    """Noise-normalised, lower-cased text for hashing/diffing only. The stored text keeps the originals."""
    out = normalize_whitespace(text)
    for pat, repl in NOISE_RULES:
        out = pat.sub(repl, out)
    return WS_RE.sub(" ", out).lower().strip()


def text_hash(text: str) -> str:
    return hashlib.sha256(normalized_text(text).encode("utf-8")).hexdigest()


def structural_hash(blocks: list[Block]) -> str:
    seq = "\n".join(f"{b.kind}|{b.key}" for b in blocks)
    return hashlib.sha256(seq.encode("utf-8")).hexdigest()


# ------------------------------------------------------------------------------------------------------------ simhash

_TOKEN_RE = re.compile(r"[\w<>]+", re.UNICODE)


def _features(text: str) -> Counter[str]:
    toks = _TOKEN_RE.findall(text)
    feats: Counter[str] = Counter(toks)
    for i in range(len(toks) - 1):
        feats[toks[i] + " " + toks[i + 1]] += 1
    return feats


def simhash(text: str) -> int:
    """64-bit simhash over word uni+bigrams of the noise-normalised text (0 for empty)."""
    feats = _features(normalized_text(text))
    if not feats:
        return 0
    v = [0] * 64
    for feat, w in feats.items():
        h = int.from_bytes(hashlib.blake2b(feat.encode("utf-8"), digest_size=8).digest(), "big")
        for i in range(64):
            v[i] += w if (h >> i) & 1 else -w
    out = 0
    for i in range(64):
        if v[i] > 0:
            out |= 1 << i
    return out


def hamming(a: int, b: int) -> int:
    return (a ^ b).bit_count()


def simhash_bucket(h: int, bits: int = 12) -> str:
    return format(h >> (64 - bits), "x") if h else "0"


# ------------------------------------------------------------------------------------------------------------ language guess

_STOPWORDS: dict[str, frozenset[str]] = {
    "en": frozenset(["the", "and", "of", "to", "in", "for", "with", "on", "is", "are", "our", "we", "you", "your", "this", "that", "from", "by", "at", "as", "be"]),
    "fr": frozenset(["le", "la", "les", "et", "de", "des", "du", "en", "pour", "avec", "sur", "est", "sont", "nous", "vous", "votre", "notre", "une", "un", "dans", "par", "au", "aux", "ce", "cette"]),
    "de": frozenset(["der", "die", "das", "und", "von", "zu", "in", "für", "mit", "auf", "ist", "sind", "wir", "sie", "ihre", "unsere", "eine", "ein", "im", "den", "dem", "nicht"]),
    "es": frozenset(["el", "la", "los", "las", "y", "de", "del", "en", "para", "con", "sobre", "es", "son", "nosotros", "su", "nuestra", "una", "un", "por", "al", "como", "más"]),
    "it": frozenset(["il", "la", "gli", "le", "e", "di", "del", "della", "in", "per", "con", "su", "è", "sono", "noi", "nostro", "una", "un", "dal", "nel", "che"]),
    "pt": frozenset(["o", "a", "os", "as", "e", "de", "do", "da", "em", "para", "com", "sobre", "é", "são", "nós", "nosso", "uma", "um", "pelo", "na", "no", "que"]),
    "nl": frozenset(["de", "het", "een", "en", "van", "voor", "met", "op", "is", "zijn", "wij", "onze", "je", "jouw", "dat", "dit", "niet", "ook"]),
}
_CJK_RE = re.compile(r"[぀-ヿ]")
_HAN_RE = re.compile(r"[一-鿿]")
_HANGUL_RE = re.compile(r"[가-힯]")
_CYR_RE = re.compile(r"[Ѐ-ӿ]")
_ARAB_RE = re.compile(r"[؀-ۿ]")


def language_guess(text: str, html_lang: str | None = None) -> str | None:
    if html_lang:
        code = html_lang.strip().lower().split("-")[0].split("_")[0]
        if 2 <= len(code) <= 3 and code.isalpha():
            return code
    sample = text[:6000]
    if not sample.strip():
        return None
    if _CJK_RE.search(sample):
        return "ja"
    if _HANGUL_RE.search(sample):
        return "ko"
    if _HAN_RE.search(sample) and len(_HAN_RE.findall(sample)) > 20:
        return "zh"
    if len(_CYR_RE.findall(sample)) > 40:
        return "ru"
    if len(_ARAB_RE.findall(sample)) > 40:
        return "ar"
    words = re.findall(r"[a-zà-ÿ']+", sample.lower())
    if len(words) < 8:
        return None
    best, best_n = None, 0
    for lang, sw in _STOPWORDS.items():
        n = sum(1 for w in words if w in sw)
        if n > best_n:
            best, best_n = lang, n
    if best_n / max(1, len(words)) < 0.04:
        return None
    return best


# ------------------------------------------------------------------------------------------------------------ page model


@dataclass(slots=True)
class Link:
    url: str
    anchor: str
    region: str = "main"                 # nav | header | footer | main
    rel: str = ""
    title: str = ""


@dataclass(slots=True)
class NormalizedPage:
    url: str
    title: str | None
    lang: str | None
    meta: dict[str, Any]
    blocks: list[Block]
    text: str                            # stored main text (original wording, whitespace-normalised)
    full_text: str                       # including nav/header/footer
    links: list[Link]
    jsonld: dict[str, list[dict[str, Any]]]
    microdata: dict[str, list[dict[str, Any]]]
    headings: list[tuple[int, str]]
    feeds: list[str] = field(default_factory=list)
    main_selector: str = "body"

    @property
    def normalized(self) -> str:
        return normalized_text(self.text)

    def blocks_of(self, *kinds: str) -> list[Block]:
        return [b for b in self.blocks if b.kind in kinds]

    def to_extraction(self) -> Extraction:
        return Extraction(text=self.text, blocks=self.blocks, title=self.title, language=self.lang, meta=dict(self.meta))


# ------------------------------------------------------------------------------------------------------------ parsing


def _attr(node: LexborNode, name: str) -> str:
    try:
        v = node.attributes.get(name)
    except Exception:  # noqa: BLE001
        return ""
    return (v or "").strip() if v is not None else ("" if name not in node.attributes else "true")


def _classes(node: LexborNode) -> str:
    return f"{_attr(node, 'id')} {_attr(node, 'class')} {_attr(node, 'role')} {_attr(node, 'data-testid')} {_attr(node, 'itemtype')}".strip()


def _is_hidden(node: LexborNode) -> bool:
    if node.tag in ("html", "body"):
        return False
    attrs = node.attributes
    if "hidden" in attrs and attrs.get("hidden") != "false":
        return True
    if (attrs.get("aria-hidden") or "").strip().lower() == "true":
        return True
    style = (attrs.get("style") or "").replace(" ", "").lower()
    return "display:none" in style or "visibility:hidden" in style


def _is_cookie(node: LexborNode) -> bool:
    cls = _classes(node)
    return bool(cls) and bool(COOKIE_RE.search(cls)) and node.tag in ("div", "section", "aside", "dialog", "footer", "header", "nav", "form")


def _region_of(node: LexborNode) -> str | None:
    tag = node.tag
    role = _attr(node, "role").lower()
    cls = _classes(node)
    if tag == "nav" or role == "navigation" or NAV_CLASS_RE.search(cls):
        return "nav"
    if tag == "footer" or role == "contentinfo" or FOOTER_CLASS_RE.search(cls):
        return "footer"
    if tag == "header" or role == "banner":
        return "header"
    return None


def _node_text(node: LexborNode) -> str:
    """Text with block boundaries as newlines and inline elements joined by spaces."""
    parts: list[str] = []

    def walk(n: LexborNode, depth: int) -> None:
        if depth > 400:
            return
        for c in n.iter(include_text=True):
            tag = c.tag
            if tag == "-text":
                t = c.text_content
                if t:
                    parts.append(t)
                continue
            if tag in DROP_TAGS or tag == "-comment" or _is_hidden(c) or _is_cookie(c):
                continue
            if tag == "br":
                parts.append("\n")
                continue
            block = tag in BLOCK_TAGS
            if block:
                parts.append("\n")
            walk(c, depth + 1)
            if block:
                parts.append("\n")
            else:
                parts.append(" ")

    walk(node, 0)
    return normalize_whitespace(html_lib.unescape("".join(parts)))


def _first_text(node: LexborNode, limit: int = 300) -> str:
    return _node_text(node)[:limit]


def _card_kind(node: LexborNode, *, allow_container: bool = False) -> str | None:
    cls = _classes(node)
    if not cls:
        return None
    kind = next((k for k, pat in CARD_PATTERNS if pat.search(cls)), None)
    if kind is None:
        return None
    if not allow_container and _is_card_container(node):
        return None
    return kind


LISTING_LINK_SHARE = 0.6            # a list whose items mostly carry links is a listing (jobs, news, products), not a bullet list
LISTING_MIN_ITEMS = 2


def _list_items(node: LexborNode) -> list[LexborNode]:
    items: list[LexborNode] = []
    for c in node.iter():
        if c.tag in ("li", "tr"):
            items.append(c)
        elif c.tag in ("tbody", "thead", "tfoot"):
            items.extend(x for x in c.iter() if x.tag == "tr")
    return items


def _is_listing(node: LexborNode) -> bool:
    """`ul`/`ol`/`table` whose items are card-like or mostly linked — the rows are entities, the wrapper is a container."""
    items = _list_items(node)
    if len(items) < LISTING_MIN_ITEMS:
        return False
    if any(_card_kind(i, allow_container=True) for i in items):
        return True
    linked = sum(1 for i in items if i.css_first("a[href]") is not None)
    return linked >= len(items) * LISTING_LINK_SHARE


def _card_text(node: LexborNode) -> str:
    """Card text with one line per direct child element, so inline cells (`<a>title</a><span>city</span>`) do not merge into one line."""
    parts: list[str] = []
    for c in node.iter(include_text=True):
        if c.tag == "-text":
            t = c.text_content
            if t and t.strip():
                parts.append(t)
        elif c.tag in DROP_TAGS or c.tag == "-comment" or c.tag == "br" or _is_hidden(c) or _is_cookie(c):
            continue
        else:
            parts.append(_node_text(c))
    return normalize_whitespace(html_lib.unescape("\n".join(p for p in parts if p)))


def _is_card_container(node: LexborNode) -> bool:
    """A grid/list wrapper (`.products`, `.news-list`, `.team-grid`, `.job-list`) holds ≥ 2 card-like children — or a listing-like list/table —
    so the children are segmented instead of the wrapper becoming a single card. A feature `<ul>` inside a pricing card is not a listing."""
    n = 0
    for child in node.iter():
        if child.tag not in ("div", "section", "article", "li", "a", "figure", "tr", "ul", "ol", "table"):
            continue
        if child.tag in ("ul", "ol", "table") and _is_listing(child):
            return True
        is_card = child.tag in ("ul", "ol", "table") or _card_kind(child, allow_container=True) is not None
        if not is_card and child.tag == "div":
            is_card = any(_card_kind(g, allow_container=True) for g in child.iter() if g.tag in ("div", "article", "li", "a"))
        if is_card:
            n += 1
            if n >= 2:
                return True
    return False


class _Segmenter:
    def __init__(self, base_url: str):
        self.base_url = base_url
        self.blocks: list[Block] = []
        self.heading_stack: list[tuple[int, str]] = []
        self.headings: list[tuple[int, str]] = []
        self.order = 0
        self.hero_done = False
        self.text_parts: list[str] = []

    # ---------------------------------------------------------------- helpers
    def path(self) -> str:
        return " > ".join(h[1] for h in self.heading_stack)[:200]

    def push_heading(self, level: int, text: str) -> None:
        while self.heading_stack and self.heading_stack[-1][0] >= level:
            self.heading_stack.pop()
        self.heading_stack.append((level, text[:80]))
        self.headings.append((level, text))

    def add(self, kind: str, text: str, *, path: str | None = None, attrs: dict[str, Any] | None = None, weight: float | None = None) -> None:
        text = normalize_whitespace(text)
        if not text:
            return
        if len(text) > 6000:
            text = text[:6000]
        p = self.path() if path is None else path
        self.blocks.append(Block(key="", kind=kind, text=text, path=p, order=self.order, weight=weight or BLOCK_WEIGHTS.get(kind, 1.0),
                                 attrs=attrs or {}))
        self.order += 1

    # ---------------------------------------------------------------- traversal
    def walk(self, node: LexborNode, region: str, depth: int = 0, hint: str | None = None) -> None:
        """`hint` is the card kind of an enclosing container (`.job-list`), inherited by plain list rows / table rows underneath it."""
        if depth > 300:
            return
        for c in node.iter(include_text=True):
            tag = c.tag
            if tag == "-text":
                t = normalize_whitespace(c.text_content or "")
                if t and region == "main":
                    self.add("paragraph" if len(t) > 40 else "other", t)
                elif t:
                    self.add(region, t)
                continue
            if tag in DROP_TAGS or tag == "-comment" or _is_hidden(c) or _is_cookie(c):
                continue
            sub = _region_of(c) if region == "main" else None
            if sub is not None:
                txt = _node_text(c)
                if txt:
                    kind = "nav" if sub == "nav" else ("footer" if sub == "footer" else "header")
                    self.add(kind, txt[:1500], path="", attrs={"region": sub})
                continue
            if region != "main":
                # inside nav/header/footer: flatten into one block per top-level child
                txt = _node_text(c)
                if txt:
                    self.add(region, txt[:1500], path="", attrs={"region": region})
                continue
            if tag in HEADING_TAGS:
                txt = _node_text(c)
                if txt:
                    level = int(tag[1])
                    self.push_heading(level, txt)
                    self.add("heading", txt, path=" > ".join(h[1] for h in self.heading_stack[:-1])[:200],
                             attrs={"level": level}, weight=BLOCK_WEIGHTS["heading"] * (1.3 if level == 1 else 1.0))
                continue
            if tag == "p":
                self.add("paragraph", _node_text(c))
                continue
            if tag == "blockquote":
                self.add("quote", _node_text(c))
                continue
            if tag == "pre" or (tag == "code" and len(_node_text(c)) > 80):
                self.add("code", _node_text(c))
                continue
            if tag in ("details",):
                self.add("faq", _node_text(c))
                continue
            if tag in ("ul", "ol", "dl", "menu"):
                self.list_block(c, hint)
                continue
            if tag == "table":
                self.table_block(c, hint)
                continue
            if tag in ("hr", "br", "wbr"):
                continue
            card = _card_kind(c) if tag in ("div", "section", "article", "li", "a", "figure", "aside", "tr", "td") else None
            if card is not None:
                txt = _card_text(c)
                if txt and len(txt) <= CARD_MAX_TEXT:
                    attrs: dict[str, Any] = {}
                    href = self._first_href(c)
                    if href:
                        attrs["href"] = href
                    self.add(card, txt, attrs=attrs)
                    continue
            if not self.hero_done and tag in ("section", "div", "header", "article") and (HERO_CLASS_RE.search(_classes(c)) or self._has_h1(c)):
                txt = _node_text(c)
                if txt and len(txt) <= 2500:
                    self.hero_done = True
                    for h in c.css("h1"):
                        ht = _node_text(h)
                        if ht:
                            self.push_heading(1, ht)
                            break
                    self.add("hero", txt, path="", attrs={"href": self._first_href(c)} if self._first_href(c) else {})
                    continue
            container_kind = _card_kind(c, allow_container=True) if tag in ("div", "section", "article", "ul", "ol") else None
            self.walk(c, region, depth + 1, container_kind or hint)

    def _has_h1(self, node: LexborNode) -> bool:
        return node.css_first("h1") is not None and len(_node_text(node)) < 2500

    def _first_href(self, node: LexborNode) -> str | None:
        a = node if node.tag == "a" else node.css_first("a[href]")
        if a is None:
            return None
        from companyatlas.urls import absolutize

        return absolutize(self.base_url, _attr(a, "href"))

    def list_block(self, node: LexborNode, hint: str | None = None) -> None:
        items = [li for li in node.iter() if li.tag in ("li", "dt", "dd")]
        if not items:
            txt = _node_text(node)
            if txt:
                self.add("list", txt)
            return
        texts = [_card_text(li) for li in items]
        texts_nonempty = [t for t in texts if t]
        if not texts_nonempty:
            return
        avg = sum(len(t) for t in texts_nonempty) / len(texts_nonempty)
        has_links = sum(1 for li in items if li.css_first("a[href]") is not None)
        listing_like = len(items) >= 3 and (has_links >= len(items) * LISTING_LINK_SHARE or avg > 60)
        if not listing_like or len(items) > 400:
            self.add("list", "\n".join(texts_nonempty)[:4000])
            return
        for li, t in zip(items, texts, strict=False):
            if not t:
                continue
            kind = _card_kind(li, allow_container=True) or _card_kind(node, allow_container=True) or hint or "list"
            attrs: dict[str, Any] = {}
            href = self._first_href(li)
            if href:
                attrs["href"] = href
            self.add(kind, t[:1500], attrs=attrs)

    def table_block(self, node: LexborNode, hint: str | None = None) -> None:
        rows = [tr for tr in node.css("tr")]
        caption = node.css_first("caption")
        cap = _node_text(caption) if caption is not None else ""
        if len(rows) <= 1 or len(rows) > 500:
            txt = _node_text(node)
            if txt:
                self.add("table", txt[:4000])
            return
        header_cells = [_node_text(th) for th in rows[0].css("th")]
        header = " | ".join(x for x in header_cells if x)
        base_path = self.path()
        table_path = " > ".join(x for x in (base_path, cap or header[:80]) if x)[:200]
        for i, tr in enumerate(rows):
            cells = [_node_text(td) for td in tr.iter() if td.tag in ("td", "th")]
            txt = " | ".join(c for c in cells if c)
            if not txt:
                continue
            kind = _card_kind(tr, allow_container=True) or hint or "table"
            attrs: dict[str, Any] = {"row": i}
            href = self._first_href(tr)
            if href:
                attrs["href"] = href
            self.add(kind, txt[:1500], path=table_path, attrs=attrs)


def _finalize_blocks(blocks: list[Block]) -> list[Block]:
    total = len(blocks) or 1
    seen: Counter[str] = Counter()
    for b in blocks:
        norm = normalized_text(b.text)
        b.hash = hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
        b.simhash = simhash(b.text)
        path_h = hashlib.blake2b(b.path.lower().encode("utf-8"), digest_size=3).hexdigest() if b.path else "root"
        base = f"{b.kind}:{path_h}:{simhash_bucket(b.simhash)}"
        n = seen[base]
        seen[base] += 1
        b.key = base if n == 0 else f"{base}#{n + 1}"
        rel = b.order / total
        pos = 1.1 if rel < 0.1 else (0.9 if rel > 0.9 else 1.0)
        if b.kind in LOW_VALUE_KINDS:
            pos = 1.0
        b.weight = round(b.weight * pos, 3)
    return blocks


def _collect_links(tree: LexborHTMLParser, base_url: str) -> list[Link]:
    from companyatlas.urls import absolutize

    links: list[Link] = []
    seen: set[tuple[str, str]] = set()
    for a in tree.css("a[href]"):
        href = _attr(a, "href")
        url = absolutize(base_url, href)
        if not url:
            continue
        anchor = _node_text(a)[:160] or _attr(a, "aria-label")[:160] or _attr(a, "title")[:160]
        region = "main"
        p = a.parent
        hops = 0
        while p is not None and hops < 40:
            r = _region_of(p) if p.tag not in ("html", "body") else None
            if r:
                region = r
                break
            p = p.parent
            hops += 1
        k = (url, anchor.lower())
        if k in seen:
            continue
        seen.add(k)
        links.append(Link(url=url, anchor=anchor, region=region, rel=_attr(a, "rel"), title=_attr(a, "title")))
        if len(links) >= 3000:
            break
    return links


def _meta(tree: LexborHTMLParser, base_url: str) -> tuple[str | None, str | None, dict[str, Any], list[str]]:
    from companyatlas.urls import absolutize

    meta: dict[str, Any] = {}
    title_node = tree.css_first("title")
    title = normalize_whitespace(title_node.text()) if title_node is not None else None
    lang = _attr(tree.root, "lang") if tree.root is not None else ""
    if not lang:
        html_node = tree.css_first("html")
        lang = _attr(html_node, "lang") if html_node is not None else ""
    feeds: list[str] = []
    for m in tree.css("meta"):
        name = (_attr(m, "name") or _attr(m, "property") or _attr(m, "http-equiv")).lower()
        content = _attr(m, "content")
        if not name or not content:
            continue
        if name in ("description", "generator", "robots", "author", "keywords", "twitter:card", "theme-color") or name.startswith("og:") and name in ("og:title", "og:description", "og:type", "og:site_name", "og:locale", "og:url"):
            meta[name.replace(":", "_")] = content[:500]
        elif name in ("content-language",) and not lang:
            lang = content
    for ln in tree.css("link[rel]"):
        rel = _attr(ln, "rel").lower()
        href = _attr(ln, "href")
        if not href:
            continue
        if "canonical" in rel:
            meta["canonical"] = absolutize(base_url, href) or href
        elif "alternate" in rel:
            typ = _attr(ln, "type").lower()
            if "rss" in typ or "atom" in typ or "feed" in typ or "json" in typ and "feed" in href:
                u = absolutize(base_url, href)
                if u and u not in feeds:
                    feeds.append(u)
            elif _attr(ln, "hreflang"):
                meta["hreflang_count"] = int(meta.get("hreflang_count", 0)) + 1
    return title or None, (lang or None), meta, feeds


# ------------------------------------------------------------------------------------------------------------ JSON-LD / microdata

JSONLD_BUCKETS: dict[str, str] = {
    "organization": "organizations", "corporation": "organizations", "localbusiness": "organizations", "ngo": "organizations",
    "jobposting": "job_postings", "product": "products", "offer": "offers", "aggregateoffer": "offers", "service": "products",
    "softwareapplication": "products", "newsarticle": "articles", "blogposting": "articles", "article": "articles", "pressrelease": "articles",
    "techarticle": "articles", "report": "articles", "person": "persons", "postaladdress": "addresses", "place": "places",
    "breadcrumblist": "breadcrumbs", "faqpage": "faqs", "website": "websites", "webpage": "webpages", "event": "events",
}


def _iter_jsonld(obj: Any, out: list[dict[str, Any]], depth: int = 0) -> None:
    if depth > 8 or len(out) > 500:
        return
    if isinstance(obj, list):
        for x in obj:
            _iter_jsonld(x, out, depth + 1)
    elif isinstance(obj, dict):
        if "@graph" in obj and isinstance(obj["@graph"], list):
            _iter_jsonld(obj["@graph"], out, depth + 1)
        if "@type" in obj:
            out.append(obj)
        for k in ("mainEntity", "itemListElement", "hasPart", "member", "employee", "founder", "address", "location", "offers", "item"):
            if k in obj:
                _iter_jsonld(obj[k], out, depth + 1)


def extract_jsonld(tree: LexborHTMLParser) -> dict[str, list[dict[str, Any]]]:
    buckets: dict[str, list[dict[str, Any]]] = {}
    for s in tree.css('script[type="application/ld+json"], script[type="application/json+ld"]'):
        raw = (s.text() or "").strip()
        if not raw or len(raw) > 2_000_000:
            continue
        try:
            data = json.loads(raw)
        except json.JSONDecodeError:
            try:
                data = json.loads(html_lib.unescape(raw).replace("\n", " "))
            except json.JSONDecodeError:
                continue
        found: list[dict[str, Any]] = []
        _iter_jsonld(data, found)
        for item in found:
            types = item.get("@type")
            for t in (types if isinstance(types, list) else [types]):
                if not isinstance(t, str):
                    continue
                key = JSONLD_BUCKETS.get(t.rsplit("/", 1)[-1].lower())
                if key:
                    buckets.setdefault(key, []).append(item)
    return buckets


def extract_microdata(tree: LexborHTMLParser) -> dict[str, list[dict[str, Any]]]:
    buckets: dict[str, list[dict[str, Any]]] = {}
    for scope in tree.css("[itemscope][itemtype]"):
        itype = _attr(scope, "itemtype").rsplit("/", 1)[-1].lower()
        key = JSONLD_BUCKETS.get(itype)
        if not key:
            continue
        item: dict[str, Any] = {"@type": itype}
        for prop in scope.css("[itemprop]"):
            name = _attr(prop, "itemprop")
            if not name or "itemscope" in prop.attributes:
                if name and "itemscope" in prop.attributes:
                    item[name] = _first_text(prop, 200)
                continue
            if prop.tag == "a" or prop.tag == "link":
                val = _attr(prop, "href")
            elif prop.tag in ("meta",):
                val = _attr(prop, "content")
            elif prop.tag == "time":
                val = _attr(prop, "datetime") or _first_text(prop, 100)
            elif prop.tag == "img":
                val = _attr(prop, "src")
            else:
                val = _first_text(prop, 300)
            if val and name not in item:
                item[name] = val
        buckets.setdefault(key, []).append(item)
        if sum(len(v) for v in buckets.values()) > 500:
            break
    return buckets


# ------------------------------------------------------------------------------------------------------------ main-content detection

MAIN_SELECTORS = ("main", "[role=main]", "article", "#main", "#main-content", "#content", ".main-content", "#primary", ".site-main", ".content")


def _prune(tree: LexborHTMLParser) -> None:
    for tag in ("script", "style", "noscript", "svg", "iframe", "template", "canvas", "video", "audio", "object", "embed", "input", "select",
                "textarea", "option", "datalist"):
        for n in tree.css(tag):
            n.decompose()


def _find_main(tree: LexborHTMLParser) -> tuple[LexborNode | None, str]:
    body = tree.body
    if body is None:
        return None, "body"
    for sel in MAIN_SELECTORS:
        try:
            n = tree.css_first(sel)
        except Exception as exc:  # noqa: BLE001
            log.debug("main selector failed", extra={"selector": sel, "error": str(exc)})
            continue
        if n is not None and not _is_hidden(n) and len(_node_text(n)) >= 80:
            return n, sel
    # largest text region among body's block descendants (depth ≤ 3) that is not nav/header/footer
    best, best_len = None, 0
    candidates: list[LexborNode] = []

    def collect(n: LexborNode, d: int) -> None:
        for c in n.iter():
            if c.tag in ("div", "section", "article", "td") and not _is_hidden(c) and _region_of(c) is None:
                candidates.append(c)
                if d < 3:
                    collect(c, d + 1)

    collect(body, 0)
    body_len = len(_node_text(body)) or 1
    for c in candidates[:400]:
        ln = len(_node_text(c))
        if ln > best_len and ln >= body_len * 0.35:
            best, best_len = c, ln
    return (best or body), ("largest" if best is not None else "body")


def parse(html_text: str, *, url: str = "", surface: str | None = None) -> NormalizedPage:
    """Parse HTML into blocks + text + metadata. `surface` is a hint only (kept in meta)."""
    tree = LexborHTMLParser(html_text or "")
    title, lang, meta, feeds = _meta(tree, url)
    jsonld = extract_jsonld(tree)
    microdata = extract_microdata(tree)
    links = _collect_links(tree, url)
    _prune(tree)
    body = tree.body
    seg = _Segmenter(url)
    main_node, main_sel = _find_main(tree)
    if body is not None:
        # nav/header/footer first as low-weight blocks (in document order they usually wrap main; we walk body and let the segmenter route)
        seg.walk(body, "main")
    blocks = _finalize_blocks(seg.blocks)
    main_text = _node_text(main_node) if main_node is not None else ""
    full_text = _node_text(body) if body is not None else ""
    if not main_text:
        main_text = "\n".join(b.text for b in blocks if b.kind not in LOW_VALUE_KINDS)
    meta = {**meta, "lang": lang, "main_selector": main_sel, "jsonld_types": sorted({k for k in jsonld}), "normalize_version": NORMALIZE_VERSION}
    if surface:
        meta["surface_hint"] = surface
    if feeds:
        meta["feeds"] = feeds[:10]
    orgs = jsonld.get("organizations") or []
    if orgs:
        same_as = [s for o in orgs for s in (o.get("sameAs") or []) if isinstance(s, str)] if isinstance(orgs[0].get("sameAs", []), list) else []
        if same_as:
            meta["same_as"] = same_as[:20]
    language = language_guess(main_text, lang)
    return NormalizedPage(url=url, title=title, lang=language, meta=meta, blocks=blocks, text=main_text, full_text=full_text, links=links,
                          jsonld=jsonld, microdata=microdata, headings=seg.headings, feeds=feeds, main_selector=main_sel)


__all__ = ["BLOCK_WEIGHTS", "LOW_VALUE_KINDS", "NORMALIZE_VERSION", "Link", "NormalizedPage", "extract_jsonld", "extract_microdata", "hamming",
           "language_guess", "normalize_whitespace", "normalized_text", "parse", "simhash", "simhash_bucket", "structural_hash", "text_hash"]