"""Generic HTML connector (spec §9.1, §107): one connector for every corporate web surface. `sdk.normalize` supplies text, blocks,
metadata, JSON-LD and links; this module adds *surface-aware* typed extraction:

    leadership → people (role category, is_executive)      pricing  → plans (currency, period, unit, contact-sales, features)
    products/services/solutions → product cards            locations → offices/stores (city / country only when stated)
    newsroom/blog/press/changelog/research/IR → news items  careers → job listings (list / table / card patterns with job-like anchors)
    legal/docs/homepage/about/… → text + blocks only

Structured data (JSON-LD / microdata) is used first, DOM heuristics second, and nothing is ever invented: no address, no country, no
date that the page does not state. Every surface also yields `discovered` URLs (classified links) for the discovery feedback loop.
Precision rules (`connectors/_precision.py`) reject navigation / CTA / cookie-consent / marketing noise at the point of extraction.
"""
from __future__ import annotations

import re
from collections.abc import Mapping
from typing import Any
from urllib.parse import urlparse

from selectolax.lexbor import LexborHTMLParser

from companyatlas.config import settings
from companyatlas.connectors._precision import (
    STREET_RE,
    is_known_city,
    looks_like_person_name,
    looks_like_role_title,
    news_verdict,
    normalize_location,
    normalize_person,
    plan_name_ok,
    plan_verdict,
    price_text_from,
    product_verdict,
    refine_job,
    role_category,
    states_job_location,
)
from companyatlas.connectors._util import (
    country_code,
    date_from_text,
    date_from_url,
    finish_job,
    job_blocks,
    norm_name,
    parse_date,
    parse_location,
    text_of,
)
from companyatlas.connectors.jsonld_jobs import jobs_from_jsonld
from companyatlas.fetch import FetchResult
from companyatlas.sdk import normalize
from companyatlas.sdk.connector import Connector, ConnectorMeta, register
from companyatlas.sdk.models import (
    Block,
    DiscoveredUrl,
    ExtractedJob,
    ExtractedLocation,
    ExtractedNewsItem,
    ExtractedPerson,
    ExtractedPlan,
    ExtractedProduct,
    Extraction,
)
from companyatlas.sdk.normalize import NormalizedPage, normalize_whitespace
from companyatlas.taxonomy import FetchMode, Surface
from companyatlas.urls import absolutize, canonicalize_url, classify_url, is_static_asset, looks_like_trap, registrable_domain

MAX_PEOPLE, MAX_PLANS, MAX_LOCATIONS, MAX_NEWS, MAX_JOBS, MAX_PRODUCTS, MAX_DISCOVERED = 200, 16, 300, 80, 300, 120, 120
NEWS_SURFACES = {Surface.NEWSROOM, Surface.BLOG, Surface.CHANGELOG, Surface.RESEARCH, Surface.INVESTOR_RELATIONS}
NEWS_CATEGORY = {Surface.NEWSROOM: "press", Surface.BLOG: "blog", Surface.CHANGELOG: "changelog", Surface.RESEARCH: "research",
                 Surface.INVESTOR_RELATIONS: "ir"}
PRODUCT_SURFACES = {Surface.PRODUCTS, Surface.SERVICES, Surface.SOLUTIONS}

# ------------------------------------------------------------------------------------------------------------ people

# Role vocabulary, name shape and title cleaning live in `_precision` (shared with the pipeline); re-exported here for callers/tests.
looks_like_name = looks_like_person_name
looks_like_title = looks_like_role_title
PERSON_CARD_SCAN_LINES = 4           # a person card states the name within its first lines
PERSON_TITLE_WINDOW = 3              # …and the title within the next few


def _card_person(lines: list[str]) -> tuple[str, str | None] | None:
    """(name, title) from a card's lines: the title may precede the name (swapped cards) or follow it; role-like lines win over prose."""
    name_idx = next((i for i, ln in enumerate(lines[:PERSON_CARD_SCAN_LINES]) if looks_like_name(ln)), None)
    if name_idx is None:
        return None
    before = [x for x in lines[:name_idx] if looks_like_title(x)]
    after = [x for x in lines[name_idx + 1:name_idx + 1 + PERSON_TITLE_WINDOW] if not looks_like_name(x)]
    title = (before[-1] if before else None) or next((x for x in after if looks_like_title(x)), None) or (after[0] if after else None)
    return lines[name_idx], title


def extract_people(page: NormalizedPage) -> list[ExtractedPerson]:
    out: list[ExtractedPerson] = []
    seen: set[str] = set()

    def add(name: str, title: str | None, url: str | None = None) -> None:
        fixed = normalize_person(name, title)
        if fixed is None:
            return
        name, title = fixed
        key = norm_name(name)
        if not key or key in seen or len(out) >= MAX_PEOPLE:
            return
        seen.add(key)
        cat, is_exec = role_category(title)
        out.append(ExtractedPerson(name=name[:120], title=(title[:160] if title else None), role_category=cat, is_executive=is_exec, url=url))

    for p in page.jsonld.get("persons", []) + page.microdata.get("persons", []):
        name = text_of(p.get("name"))
        if name:
            add(name, text_of(p.get("jobTitle")) or text_of(p.get("title")), text_of(p.get("url")) if isinstance(p.get("url"), str) else None)
    for b in page.blocks:
        if b.kind != "person":
            continue
        lines = [ln.strip() for ln in b.text.split("\n") if ln.strip()]
        found = _card_person(lines)
        if found is not None:
            add(found[0], found[1], b.attrs.get("href"))
    if len(out) < 2:  # fallback: heading = name, next short block = title
        blocks = page.blocks
        for i, b in enumerate(blocks):
            if b.kind == "heading" and int(b.attrs.get("level", 3)) >= 2 and looks_like_name(b.text):
                nxt = next((x for x in blocks[i + 1:i + 3] if x.kind in ("paragraph", "other", "list", "section")), None)
                if nxt is not None and looks_like_title(nxt.text.split("\n")[0]):
                    add(b.text, nxt.text.split("\n")[0])
    return out


# ------------------------------------------------------------------------------------------------------------ pricing

CURRENCY_SYMBOLS = {"$": "USD", "US$": "USD", "USD": "USD", "€": "EUR", "EUR": "EUR", "£": "GBP", "GBP": "GBP", "¥": "JPY", "JPY": "JPY", "₹": "INR",
                    "INR": "INR", "C$": "CAD", "CA$": "CAD", "CAD": "CAD", "A$": "AUD", "AUD": "AUD", "CHF": "CHF", "SEK": "SEK", "NOK": "NOK",
                    "DKK": "DKK", "kr": "SEK", "zł": "PLN", "PLN": "PLN", "R$": "BRL", "BRL": "BRL", "MX$": "MXN", "₩": "KRW", "SGD": "SGD", "S$": "SGD",
                    "HK$": "HKD", "NZ$": "NZD", "₺": "TRY", "元": "CNY", "CNY": "CNY", "RMB": "CNY"}
PRICE_RE = re.compile(r"(?:(?P<cur>US\$|CA\$|C\$|A\$|NZ\$|HK\$|S\$|MX\$|R\$|USD|EUR|GBP|CAD|AUD|CHF|JPY|INR|SEK|NOK|DKK|PLN|BRL|CNY|RMB|SGD|[$€£¥₹₩₺])\s?"
                      r"(?P<amt>\d{1,3}(?:[,.\s]\d{3})*(?:[.,]\d{1,2})?|\d+(?:[.,]\d{1,2})?))|"
                      r"(?:(?P<amt2>\d{1,3}(?:[,.\s]\d{3})*(?:[.,]\d{1,2})?|\d+(?:[.,]\d{1,2})?)\s?(?P<cur2>€|£|USD|EUR|GBP|CHF|kr|zł|元|\$))")
PERIOD_RE = re.compile(r"(?:(?:per|/|a|each|every)\s*(?P<unit>month|mo|year|yr|annum|week|wk|day|hour|hr|user|seat|member|license|licence|agent|editor|"
                       r"contact|1,?000|1k|GB|TB|request|transaction|call|minute|mois|an|année|monat|jahr)\b)|(?P<word>monthly|annually|yearly|per annum|"
                       r"billed (?:monthly|annually|yearly)|mensuel|annuel|monatlich|jährlich|one[- ]time|lifetime)", re.IGNORECASE)
CONTACT_RE = re.compile(r"\b(contact (us|sales)|talk to (us|sales|an expert)|custom pricing|custom quote|get a quote|request (a )?(quote|demo)|let'?s talk|"
                        r"on request|upon request|sur devis|nous contacter|contactez-nous|auf anfrage|individuell|tailored|bespoke|call us)\b", re.IGNORECASE)
FREE_RE = re.compile(r"^(free|gratuit|kostenlos|gratis|\$\s?0(\.00)?|0\s?€|€\s?0)\b", re.IGNORECASE)
FROM_RE = re.compile(r"\b(from|starting at|starts at|as low as|à partir de|ab)\b", re.IGNORECASE)
MONTH_UNITS = {"month", "mo", "monthly", "mensuel", "monatlich", "mois", "monat"}
YEAR_UNITS = {"year", "yr", "annum", "annually", "yearly", "per annum", "annuel", "jährlich", "an", "année", "jahr"}
SEAT_UNITS = {"user", "seat", "member", "license", "licence", "agent", "editor", "contact"}
USAGE_UNITS = {"1,000", "1000", "1k", "gb", "tb", "request", "transaction", "call", "minute", "hour", "hr", "day", "week", "wk"}


def parse_price(text: str) -> dict[str, Any] | None:
    """→ {price, currency, billing_period, unit, price_text, contact_sales} or None when the text states no price."""
    t = normalize_whitespace(text)
    if CONTACT_RE.search(t) and not PRICE_RE.search(t):
        return {"price": None, "currency": None, "billing_period": "contact", "unit": None, "price_text": price_text_from(t) or t[:80], "contact_sales": True}
    m = PRICE_RE.search(t)
    if m is None:
        if FREE_RE.match(t):
            return {"price": 0.0, "currency": None, "billing_period": None, "unit": None, "price_text": price_text_from(t) or t[:80], "contact_sales": False}
        return None
    cur = m.group("cur") or m.group("cur2")
    amt = m.group("amt") or m.group("amt2")
    amt_clean = amt.replace(" ", "")
    if re.fullmatch(r"\d{1,3}(,\d{3})+(\.\d{1,2})?", amt_clean):
        amt_clean = amt_clean.replace(",", "")
    elif re.fullmatch(r"\d{1,3}(\.\d{3})+(,\d{1,2})?", amt_clean):
        amt_clean = amt_clean.replace(".", "").replace(",", ".")
    elif re.fullmatch(r"\d+,\d{1,2}", amt_clean):
        amt_clean = amt_clean.replace(",", ".")
    else:
        amt_clean = amt_clean.replace(",", "")
    try:
        price = float(amt_clean)
    except ValueError:
        return None
    tail = t[m.end():m.end() + 60]
    pm = PERIOD_RE.search(tail) or PERIOD_RE.search(t)
    period, unit = None, None
    if pm:
        u = (pm.group("unit") or pm.group("word") or "").lower()
        if u in MONTH_UNITS or u.startswith("billed monthly"):
            period = "month"
        elif u in YEAR_UNITS or "annual" in u or "yearly" in u:
            period = "year"
        elif u in ("one-time", "one time", "lifetime"):
            period = "one_time"
        elif u in SEAT_UNITS:
            unit = u
        elif u in USAGE_UNITS:
            period, unit = "usage", u
        # a second unit ("per user per month")
        for pm2 in PERIOD_RE.finditer(tail):
            u2 = (pm2.group("unit") or pm2.group("word") or "").lower()
            if u2 in SEAT_UNITS and not unit:
                unit = u2
            elif (u2 in MONTH_UNITS) and not period:
                period = "month"
            elif (u2 in YEAR_UNITS) and not period:
                period = "year"
    return {"price": price, "currency": CURRENCY_SYMBOLS.get(cur, cur.upper() if cur and cur.isalpha() else None), "billing_period": period, "unit": unit,
            "price_text": price_text_from(t) or t[m.start():m.end()].strip()[:80], "contact_sales": bool(CONTACT_RE.search(t))}


PLAN_NAME_SCAN_LINES = 3             # the plan name is one of the first lines before the price (eyebrows / marketing headings are skipped)


def _plan_name_from(lines: list[str], path: str) -> tuple[str | None, list[str]]:
    """(plan name, remaining lines): the first valid short label before the price line (eyebrows such as "Most popular" are skipped).
    Otherwise the heading just above the card (its path tail) names the tier — unless that heading is generic ("Plans", "Pricing") or a
    marketing sentence, in which case the card yields no plan."""
    for i, ln in enumerate(lines[:PLAN_NAME_SCAN_LINES]):
        if plan_name_ok(ln):                         # "Free" is a valid tier name even though it also reads as a price
            return ln, lines[i + 1:]
        if PRICE_RE.search(ln) or parse_price(ln) is not None:
            break
    tail = path.split(" > ")[-1] if path else ""
    return (tail if plan_name_ok(tail) else None), lines


def _plan_from_lines(name: str | None, lines: list[str]) -> ExtractedPlan | None:
    if not name:
        return None
    price_info = None
    features: list[str] = []
    for ln in lines:
        if price_info is None:
            info = parse_price(ln)
            if info is not None:
                price_info = info
                continue
        if 2 <= len(ln) <= 140 and not parse_price(ln) and len(features) < 25:
            features.append(ln)
    if price_info is None and FREE_RE.match(name):
        price_info = parse_price(name)               # a tier literally called "Free" with no separate price line
    if price_info is None:
        return None
    return ExtractedPlan(plan_name=name[:80], price=price_info["price"], price_text=price_info["price_text"], currency=price_info["currency"],
                         billing_period=price_info["billing_period"], unit=price_info["unit"], features=features, contact_sales=price_info["contact_sales"])


def extract_plans(page: NormalizedPage) -> list[ExtractedPlan]:
    out: list[ExtractedPlan] = []
    seen: set[str] = set()

    def add(plan: ExtractedPlan | None) -> None:
        if plan is None or not plan_verdict(plan).ok:
            return
        key = norm_name(plan.plan_name)
        if not key or key in seen or len(out) >= MAX_PLANS:
            return
        seen.add(key)
        out.append(plan)

    for b in page.blocks:
        if b.kind != "pricing_plan":
            continue
        lines = [ln.strip() for ln in b.text.split("\n") if ln.strip()]
        if not lines:
            continue
        name, rest = _plan_name_from(lines, b.path)
        add(_plan_from_lines(name, rest))
    if not out:  # fallback: heading followed by a price before the next heading (common in hand-rolled pricing tables)
        blocks = page.blocks
        i = 0
        while i < len(blocks):
            b = blocks[i]
            if b.kind == "heading" and int(b.attrs.get("level", 2)) >= 2 and plan_name_ok(b.text):
                j = i + 1
                lines: list[str] = []
                while j < len(blocks) and blocks[j].kind != "heading" and j - i <= 10:
                    lines.extend(x.strip() for x in blocks[j].text.split("\n") if x.strip())
                    j += 1
                add(_plan_from_lines(b.text, lines))
                i = j
                continue
            i += 1
    if not out:  # tables: first cell = plan, another cell = price
        for b in page.blocks:
            if b.kind == "table" and "|" in b.text:
                cells = [c.strip() for c in b.text.split("|")]
                if len(cells) >= 2 and parse_price(cells[0]) is None and any(parse_price(c) for c in cells[1:]):
                    add(_plan_from_lines(cells[0], cells[1:]))
    return out


# ------------------------------------------------------------------------------------------------------------ locations

LOCATION_KIND_RULES: list[tuple[str, re.Pattern[str]]] = [
    ("headquarters", re.compile(r"\b(headquarters|head office|hq|global office|corporate office|siège|hauptsitz|sede central)\b", re.IGNORECASE)),
    ("factory", re.compile(r"\b(factory|plant|manufacturing|production site|mill|foundry|usine|werk|fabrik)\b", re.IGNORECASE)),
    ("warehouse", re.compile(r"\b(warehouse|distribution cent(er|re)|fulfil?lment|logistics hub|entrepôt|lager)\b", re.IGNORECASE)),
    ("lab", re.compile(r"\b(lab|laboratory|research cent(er|re)|r&d|innovation cent(er|re))\b", re.IGNORECASE)),
    ("data_center", re.compile(r"\b(data ?cent(er|re)|datacenter|server farm)\b", re.IGNORECASE)),
    ("store", re.compile(r"\b(store|shop|boutique|showroom|outlet|dealer(ship)?|branch|agence|filiale)\b", re.IGNORECASE)),
]
CITY_COUNTRY_RE = re.compile(r"^([A-ZÀ-Ý][\w'’.\- ]{1,40}),\s*([A-Za-zÀ-ÿ .]{2,40})$")
MAX_VENUE_LINE_CHARS = 60           # "Hormuz Grand Hotel" — a venue line under a country heading


def _location_from_lines(name: str, lines: list[str], href: str | None = None) -> ExtractedLocation | None:
    text = " \n ".join(lines)
    kind = next((k for k, pat in LOCATION_KIND_RULES if pat.search(name) or pat.search(text[:200])), "office")
    city = region = country = address = None
    for ln in lines:
        street = STREET_RE.search(ln)
        if street and address is None and len(ln) <= 120:
            address = ln
        probe = (ln[:street.start()] + " " + ln[street.end():]).strip(" ,") if street else ln
        if country is None and probe:
            loc = parse_location(probe)
            if loc["country"]:                       # the line that states the country is the authoritative "City, Country" line
                city, region, country = loc["city"] or city, loc["region"], loc["country"]
            elif loc["region"]:
                city, region = city or loc["city"], loc["region"]
    if country is None:
        c = country_code(name)
        if c:
            country = c
        else:
            m = CITY_COUNTRY_RE.match(name)
            if m and country_code(m.group(2)):
                city, country = m.group(1).strip(), country_code(m.group(2))
    if country_code(name) and city is None:
        # heading is a country ("Oman") — the first short line that is neither an address nor a place is the venue / office name
        venue = next((ln for ln in lines if ln != address and len(ln) <= MAX_VENUE_LINE_CHARS and not parse_location(ln)["country"]
                      and not country_code(ln) and not any(ch.isdigit() for ch in ln)), None)
        if venue:
            name = venue
        elif is_known_city(name):
            city = name                                                              # city-states: Singapore, Monaco, Hong Kong
    elif city is None and country is not None and looks_like_city(name):
        city = name
    if city is None and country is None and address is None and kind == "office":
        return None
    return ExtractedLocation(name=name[:120], kind=kind, city=city, region=region, country=country, address_text=address)


def looks_like_city(text: str) -> bool:
    t = normalize_whitespace(text)
    return 2 <= len(t) <= 40 and not any(ch.isdigit() for ch in t) and t[0].isupper() and not LOCATION_KIND_RULES[0][1].search(t)


def extract_locations(page: NormalizedPage) -> list[ExtractedLocation]:
    out: list[ExtractedLocation] = []
    seen: set[str] = set()

    def add(loc: ExtractedLocation | None) -> None:
        loc = normalize_location(loc) if loc is not None else None
        if loc is None:
            return
        key = norm_name(loc.name)
        if not key or key in seen or len(out) >= MAX_LOCATIONS:
            return
        seen.add(key)
        out.append(loc)

    for a in page.jsonld.get("addresses", []) + page.jsonld.get("places", []) + page.microdata.get("addresses", []):
        addr = a.get("address") if isinstance(a.get("address"), dict) else a
        city = text_of(addr.get("addressLocality")) if isinstance(addr, dict) else None
        country = country_code(text_of(addr.get("addressCountry"))) if isinstance(addr, dict) else None
        name = text_of(a.get("name")) or ", ".join(x for x in (city, country) if x)
        if name and (city or country):
            street = text_of(addr.get("streetAddress")) if isinstance(addr, dict) else None
            add(ExtractedLocation(name=name[:120], kind="office", city=city, region=text_of(addr.get("addressRegion")) if isinstance(addr, dict) else None,
                                  country=country, address_text=street))
    for b in page.blocks:
        if b.kind != "location":
            continue
        lines = [ln.strip() for ln in b.text.split("\n") if ln.strip()]
        if lines:
            add(_location_from_lines(lines[0], lines[1:], b.attrs.get("href")))
    if len(out) < 2:  # fallback: headings naming a city/country followed by address-like lines
        blocks = page.blocks
        for i, b in enumerate(blocks):
            if b.kind == "heading" and int(b.attrs.get("level", 3)) >= 2 and (looks_like_city(b.text) or country_code(b.text)):
                lines: list[str] = []
                for nb in blocks[i + 1:i + 4]:
                    if nb.kind == "heading":
                        break
                    lines.extend(x.strip() for x in nb.text.split("\n") if x.strip())
                add(_location_from_lines(b.text, lines))
    return out


# ------------------------------------------------------------------------------------------------------------ news

NEWS_NOISE_ANCHOR = re.compile(r"^(read more|learn more|more|continue reading|view all|see all|all news|all posts|next|previous|older|newer|\d+)$", re.IGNORECASE)
MIN_NEWS_TITLE_CHARS = 3             # anything shorter is a glyph or a counter; the precision rule decides the rest (words / date)


def _time_index(html: str, base_url: str) -> dict[str, Any]:
    """canonical href → datetime for <time datetime> elements near a link (dates that the page states)."""
    out: dict[str, Any] = {}
    try:
        tree = LexborHTMLParser(html)
    except Exception:  # noqa: BLE001
        return out
    for t in tree.css("time[datetime]")[:400]:
        dt = parse_date(t.attributes.get("datetime"))
        if dt is None:
            continue
        node = t
        for _ in range(7):
            if node is None or node.tag in ("body", "html"):
                break
            a = node.css_first("a[href]") if node.tag != "a" else node
            if a is not None:
                url = absolutize(base_url, a.attributes.get("href") or "")
                if url:
                    out.setdefault(canonicalize_url(url), dt)
                    break
            node = node.parent
    return out


def extract_news(page: NormalizedPage, html: str, *, surface: str, base_url: str) -> list[ExtractedNewsItem]:
    category = NEWS_CATEGORY.get(surface, "other")  # type: ignore[call-overload]
    times = _time_index(html, base_url)
    out: list[ExtractedNewsItem] = []
    seen: set[str] = set()
    site = registrable_domain(base_url)

    def add(title: str, url: str, published: Any, summary: str | None = None) -> None:
        title = normalize_whitespace(title)
        if len(title) < MIN_NEWS_TITLE_CHARS or NEWS_NOISE_ANCHOR.match(title) or len(out) >= MAX_NEWS:
            return
        canon = canonicalize_url(url)
        if canon in seen or canon == canonicalize_url(base_url) or is_static_asset(url):
            return
        published = published or times.get(canon)
        if not news_verdict(title, url=url, published_at=published).ok:
            return
        seen.add(canon)
        out.append(ExtractedNewsItem(title=title[:300], url=url, published_at=published, summary=(summary or None), category=category, language=page.lang))

    for a in page.jsonld.get("articles", []) + page.microdata.get("articles", []):
        title, url = text_of(a.get("headline")) or text_of(a.get("name")), a.get("url") or (a.get("mainEntityOfPage") if isinstance(a.get("mainEntityOfPage"), str) else None)
        if title and isinstance(url, str):
            add(title, url, parse_date(a.get("datePublished")), text_of(a.get("description")))
    for b in page.blocks:
        href = b.attrs.get("href")
        if b.kind != "news_item" or not href:
            continue
        lines = [ln.strip() for ln in b.text.split("\n") if ln.strip()]
        undated = [ln for ln in lines if not (len(ln) <= 32 and date_from_text(ln))]
        title = next((ln for ln in undated if len(ln) >= 12), undated[0] if undated else "")
        rest = [ln for ln in undated if ln != title]
        add(title, href, date_from_text(b.text) or times.get(canonicalize_url(href)) or date_from_url(href), " ".join(rest[:2])[:300] or None)
    if len(out) < 3:  # fallback: article-like links in the main region (same site, deeper path, long anchor)
        base_depth = urlparse(base_url).path.strip("/").count("/")
        for ln in page.links:
            if ln.region != "main" or registrable_domain(ln.url) != site or looks_like_trap(ln.url):
                continue
            depth = urlparse(ln.url).path.strip("/").count("/")
            dated = date_from_url(ln.url) or times.get(canonicalize_url(ln.url))
            if (len(ln.anchor) >= 25 and depth >= base_depth) or (dated and len(ln.anchor) >= 12):
                add(ln.anchor, ln.url, dated)
    out.sort(key=lambda n: (n.published_at is None, -(n.published_at.timestamp() if n.published_at else 0)))
    return out


# ------------------------------------------------------------------------------------------------------------ careers (HTML)

JOB_URL_RE = re.compile(r"(/jobs?/|/careers?/[^/]+|/positions?/|/openings?/|/vacanc|/opportunit|/stellen|/emplois?/|/offres?/|gh_jid=|/job-|lever\.co/|"
                        r"greenhouse\.io/|ashbyhq\.com/|myworkdayjobs\.com/.+/job/|smartrecruiters\.com/|workable\.com/j/|recruitee\.com/o/|"
                        r"personio\.de/job/|teamtailor\.com/jobs/|bamboohr\.com/careers/\d|jobvite\.com/|icims\.com/jobs/|taleo\.net/.+job)", re.IGNORECASE)
JOB_NOISE_ANCHOR = re.compile(r"^(apply( now)?|view( all)?( jobs| openings| roles| positions)?|see (all|more|open)( jobs| roles| positions)?|learn more|read more|"
                              r"careers?|jobs?|join (us|the team|our team)|open (roles|positions)|all (jobs|roles|positions|openings)|search jobs|"
                              r"browse jobs|explore|more|back|home|filter|next|previous|\d+)$", re.IGNORECASE)


_states_location = states_job_location


def extract_jobs_html(page: NormalizedPage, *, base_url: str) -> list[ExtractedJob]:
    jobs: list[ExtractedJob] = []
    seen: set[str] = set()

    def add(title: str, url: str | None, location: str | None, department: str | None = None) -> None:
        title = normalize_whitespace(title).strip(" -–—|·")
        if not (4 <= len(title) <= 140) or JOB_NOISE_ANCHOR.match(title) or len(jobs) >= MAX_JOBS:
            return
        job = refine_job(ExtractedJob(title=title, url=url, location_text=location, department=department))
        if job is None:
            return
        key = (canonicalize_url(url) if url else "") + "|" + job.title.lower()
        if key in seen:
            return
        seen.add(key)
        jobs.append(finish_job(job))

    jl_jobs = jobs_from_jsonld(list(page.jsonld.get("job_postings", [])) + list(page.microdata.get("job_postings", [])), page_url=base_url)
    for j in jl_jobs:
        add(j.title, j.url, j.location_text, j.department)
        if jobs and jobs[-1].title == normalize_whitespace(j.title):
            jobs[-1] = j
    block_jobs: list[ExtractedJob] = []
    for b in page.blocks:
        href = b.attrs.get("href")
        if b.kind != "job_listing" or not href or not JOB_URL_RE.search(href) and registrable_domain(href) == registrable_domain(base_url) and urlparse(href).path.count("/") < 2:
            continue
        lines = [ln.strip() for ln in b.text.split("\n") if ln.strip()]
        if lines and " | " in lines[0]:                      # table row: cells joined by " | "
            lines = [c.strip() for c in lines[0].split(" | ") if c.strip()] + lines[1:]
        if not lines:
            continue
        title = lines[0]
        loc = next((ln for ln in lines[1:6] if _states_location(ln) or is_known_city(ln)), None)
        dept = next((ln for ln in lines[1:6] if ln != loc and 2 <= len(ln) <= 40 and not _states_location(ln) and not date_from_text(ln)), None)
        before = len(jobs)
        add(title, href, loc, dept)
        if len(jobs) > before:
            block_jobs.append(jobs[-1])
    if len(jobs) < 3:  # fallback: job-like anchors anywhere in the main region
        cands = [ln for ln in page.links if ln.region == "main" and JOB_URL_RE.search(ln.url) and 4 <= len(ln.anchor) <= 140
                 and not JOB_NOISE_ANCHOR.match(ln.anchor) and not is_static_asset(ln.url)]
        if len(cands) >= 3:
            for ln in cands:
                add(ln.anchor, ln.url, None)
    if len(jobs) < 3 and not jl_jobs:
        return []                      # not a trustworthy listing — the pipeline must not mark jobs removed on this
    return jobs


# ------------------------------------------------------------------------------------------------------------ products


def extract_products(page: NormalizedPage) -> list[ExtractedProduct]:
    out: list[ExtractedProduct] = []
    seen: set[str] = set()

    def add(name: str, url: str | None, description: str | None, category: str | None = None) -> None:
        name = normalize_whitespace(name).strip(" -–—|·")
        key = norm_name(name)
        if not (2 <= len(name) <= 90) or not key or key in seen or len(out) >= MAX_PRODUCTS or NEWS_NOISE_ANCHOR.match(name):
            return
        if not product_verdict(name).ok:
            return
        seen.add(key)
        out.append(ExtractedProduct(name=name, url=url, category=category, description=(description or None)))

    for p in page.jsonld.get("products", []) + page.microdata.get("products", []):
        name = text_of(p.get("name"))
        if name:
            add(name, p.get("url") if isinstance(p.get("url"), str) else None, text_of(p.get("description")), text_of(p.get("category")))
    for b in page.blocks:
        if b.kind != "product_card":
            continue
        lines = [ln.strip() for ln in b.text.split("\n") if ln.strip()]
        if lines:
            add(lines[0], b.attrs.get("href"), " ".join(lines[1:4])[:300], b.path.split(" > ")[-1] if b.path else None)
    return out


# ------------------------------------------------------------------------------------------------------------ discovery links


def discovered_links(page: NormalizedPage, *, base_url: str, canonical_domain: str | None) -> list[DiscoveredUrl]:
    site = registrable_domain(canonical_domain or base_url)
    out: list[DiscoveredUrl] = []
    seen: set[str] = set()
    for ln in page.links:
        if registrable_domain(ln.url) != site or is_static_asset(ln.url) or looks_like_trap(ln.url):
            continue
        canon = canonicalize_url(ln.url)
        if canon in seen:
            continue
        surface, conf = classify_url(ln.url, anchor=ln.anchor, canonical_domain=canonical_domain or site)
        if surface in (Surface.OTHER, Surface.HOMEPAGE) or conf < settings.discovery_min_confidence:
            continue
        seen.add(canon)
        out.append(DiscoveredUrl(url=ln.url, surface=surface, confidence=conf, anchor=ln.anchor[:120] or None, method="nav" if ln.region in ("nav", "header", "footer") else "link"))
        if len(out) >= MAX_DISCOVERED:
            break
    for feed in page.feeds:
        out.append(DiscoveredUrl(url=feed, surface=Surface.FEED, confidence=0.9, method="feed"))
    return out


# ------------------------------------------------------------------------------------------------------------ connector


@register
class GenericHtmlConnector(Connector):
    meta = ConnectorMeta(connector_id="generic-html-v1", name="Generic HTML surface", version="1", category=Surface.OTHER, fetch_mode=FetchMode.HTTP,
                         supports_discovery=True, default_interval_s=24 * 3600, surfaces=("*",), priority=1,
                         description="Surface-aware extraction for any corporate web page")

    def extract(self, sensor: Mapping[str, Any], result: FetchResult) -> Extraction:
        surface = str(sensor.get("surface") or Surface.OTHER)
        html = result.text
        if not result.is_html and result.is_json:
            raise ValueError("generic HTML connector received JSON")
        page = normalize.parse(html, url=result.final_url, surface=surface)
        ex = page.to_extraction()
        cfg = sensor.get("config") or {}
        canonical_domain = str(cfg.get("canonical_domain") or "") or None
        if surface == Surface.LEADERSHIP or surface == Surface.ABOUT:
            ex.people = extract_people(page)
        if surface == Surface.PRICING:
            ex.plans = extract_plans(page)
        if surface == Surface.LOCATIONS or surface == Surface.CONTACT:
            ex.locations = extract_locations(page)
        if surface in NEWS_SURFACES:
            ex.news = extract_news(page, html, surface=surface, base_url=result.final_url)
        if surface in (Surface.CAREERS, Surface.JOBS_BOARD):
            ex.jobs = extract_jobs_html(page, base_url=result.final_url)
            if ex.jobs:
                ex.blocks = [b for b in ex.blocks if b.kind != "job_listing"] + job_blocks(ex.jobs, path="Jobs")
        if surface in PRODUCT_SURFACES:
            ex.products = extract_products(page)
        ex.discovered = discovered_links(page, base_url=result.final_url, canonical_domain=canonical_domain)
        ex.meta.update({"surface": surface, "title": page.title, "headings": [h[1][:80] for h in page.headings[:30]], "link_count": len(page.links),
                        "structured": bool(ex.jobs or ex.plans or ex.people or ex.locations or ex.news or ex.products)})
        return ex


def blocks_summary(blocks: list[Block]) -> dict[str, int]:
    out: dict[str, int] = {}
    for b in blocks:
        out[b.kind] = out.get(b.kind, 0) + 1
    return out


__all__ = ["GenericHtmlConnector", "blocks_summary", "discovered_links", "extract_jobs_html", "extract_locations", "extract_news", "extract_people",
           "extract_plans", "extract_products", "looks_like_name", "parse_price", "role_category"]