"""Sitemap connector (spec §9.2): `urlset` and `sitemapindex` (plain or .gz), `lastmod`, bounded by `settings.discovery_max_sitemap_urls`.
The observable change is the *set of URLs* (new / gone since the previous snapshot — the pipeline diffs `extracted.urls`) plus `lastmod`
moves; every URL is classified with `urls.classify_url` and emitted as `DiscoveredUrl` so the discovery feedback loop can add sensors.
Child sitemaps of an index are fetched breadth-first (bounded `MAX_CHILD_SITEMAPS`)."""
from __future__ import annotations

import gzip
import hashlib
import logging
import re
from collections.abc import Mapping
from typing import Any
from urllib.parse import urlparse

from companyatlas.config import settings
from companyatlas.connectors._util import merged_result
from companyatlas.fetch import Fetcher, FetchResult
from companyatlas.sdk.connector import Connector, ConnectorContext, ConnectorMeta, register
from companyatlas.sdk.models import Block, DiscoveredUrl, Extraction
from companyatlas.sdk.normalize import simhash
from companyatlas.taxonomy import FetchMode, Surface
from companyatlas.urls import canonicalize_url, classify_url, is_static_asset, looks_like_trap

log = logging.getLogger(__name__)

MAX_CHILD_SITEMAPS = 25
LOC_RE = re.compile(r"<loc>\s*(?:<!\[CDATA\[)?\s*([^<\]\s]+)\s*(?:\]\]>)?\s*</loc>", re.IGNORECASE)
ENTRY_RE = re.compile(r"<(url|sitemap)>(.*?)</\1>", re.IGNORECASE | re.DOTALL)
LASTMOD_RE = re.compile(r"<lastmod>\s*([^<\s]+)\s*</lastmod>", re.IGNORECASE)
# Sitemap children whose names suggest low-value bulk content (products/tags/images) are visited last.
LOW_VALUE_CHILD_RE = re.compile(r"(image|video|tag|category|author|product|shop|store|collection|attachment|page-\d+|post-\d+)", re.IGNORECASE)
HIGH_VALUE_CHILD_RE = re.compile(r"(page|pages|static|main|site|news|press|blog|careers?|jobs?|misc|general)", re.IGNORECASE)


def _decode(content: bytes) -> str:
    if content[:2] == b"\x1f\x8b":
        try:
            content = gzip.decompress(content)
        except (OSError, EOFError):
            return ""
    return content.decode("utf-8", errors="replace")


def parse_sitemap(text: str) -> tuple[list[tuple[str, str | None]], list[tuple[str, str | None]]]:
    """→ (page_urls[(loc, lastmod)], child_sitemaps[(loc, lastmod)])."""
    pages: list[tuple[str, str | None]] = []
    children: list[tuple[str, str | None]] = []
    for m in ENTRY_RE.finditer(text):
        body = m.group(2)
        loc = LOC_RE.search(body)
        if not loc:
            continue
        lm = LASTMOD_RE.search(body)
        item = (loc.group(1).strip(), lm.group(1).strip() if lm else None)
        (children if m.group(1).lower() == "sitemap" else pages).append(item)
    if not pages and not children:      # tolerate sitemaps without <url> wrappers / text sitemaps
        for m in LOC_RE.finditer(text):
            pages.append((m.group(1).strip(), None))
        if not pages:
            for line in text.splitlines():
                line = line.strip()
                if line.startswith(("http://", "https://")):
                    pages.append((line, None))
    return pages, children


@register
class SitemapConnector(Connector):
    meta = ConnectorMeta(connector_id="sitemap-v1", name="XML sitemap", version="1", category=Surface.SITEMAP, fetch_mode=FetchMode.SITEMAP,
                         supports_discovery=True, default_interval_s=24 * 3600, url_pattern=r"sitemap[^/]*\.xml(\.gz)?$|/sitemap_index\.xml$|/sitemaps?/",
                         priority=40, accept="application/xml,text/xml,*/*;q=0.5", description="urlset / sitemapindex with lastmod, bounded")

    async def fetch(self, ctx: ConnectorContext, sensor: Mapping[str, Any], fetcher: Fetcher) -> FetchResult:
        first = await fetcher.get(str(sensor["url"]), etag=sensor.get("etag"), last_modified=sensor.get("last_modified"), accept=self.meta.accept)
        pages, children = parse_sitemap(_decode(first.content))
        if not children:
            return first
        limit = settings.discovery_max_sitemap_urls
        ordered = sorted(children, key=lambda c: (0 if HIGH_VALUE_CHILD_RE.search(c[0]) else (2 if LOW_VALUE_CHILD_RE.search(c[0]) else 1), c[0]))
        merged: list[dict[str, Any]] = [{"loc": u, "lastmod": lm} for u, lm in pages]
        visited: list[str] = []
        for child_url, _lm in ordered[:MAX_CHILD_SITEMAPS]:
            if len(merged) >= limit:
                break
            try:
                res = await fetcher.get(child_url, accept=self.meta.accept)
            except Exception as exc:  # noqa: BLE001 - one broken child must not fail the whole sitemap
                log.info("child sitemap skipped", extra={"child": child_url, "error": str(exc)[:200]})
                continue
            p, _c = parse_sitemap(_decode(res.content))
            merged.extend({"loc": u, "lastmod": lm} for u, lm in p[: max(0, limit - len(merged))])
            visited.append(child_url)
        payload = {"index": first.final_url, "children": [c[0] for c in ordered], "visited": visited, "urls": merged[:limit]}
        return merged_result(first, payload, pages=1 + len(visited))

    def extract(self, sensor: Mapping[str, Any], result: FetchResult) -> Extraction:
        limit = settings.discovery_max_sitemap_urls
        if result.headers.get("x-companyatlas-pages"):
            data = result.json()
            entries = [(e.get("loc"), e.get("lastmod")) for e in data.get("urls", []) if e.get("loc")]
            children = list(data.get("children", []))
        else:
            entries, kids = parse_sitemap(_decode(result.content))
            children = [c[0] for c in kids]
        entries = entries[:limit]
        canonical_domain = str((sensor.get("config") or {}).get("canonical_domain") or urlparse(str(sensor["url"])).hostname or "")
        seen: set[str] = set()
        blocks: list[Block] = []
        discovered: list[DiscoveredUrl] = []
        by_surface: dict[str, int] = {}
        urls_out: list[dict[str, Any]] = []
        for loc, lastmod in entries:
            if not loc or is_static_asset(loc):
                continue
            canon = canonicalize_url(loc)
            if canon in seen:
                continue
            seen.add(canon)
            surface, conf = classify_url(loc, canonical_domain=canonical_domain)
            by_surface[str(surface)] = by_surface.get(str(surface), 0) + 1
            urls_out.append({"url": canon, "lastmod": lastmod, "surface": str(surface), "confidence": conf})
            key = f"url:{hashlib.blake2b(canon.encode('utf-8'), digest_size=8).hexdigest()}"
            blocks.append(Block(key=key, kind="list", text=canon, path="Sitemap", hash=hashlib.sha256(canon.encode()).hexdigest()[:16],
                                simhash=simhash(canon), weight=0.6, order=len(blocks), attrs={"lastmod": lastmod}))
            if conf >= settings.discovery_min_confidence and surface not in (Surface.OTHER, Surface.SITEMAP, Surface.HOMEPAGE) and not looks_like_trap(loc):
                discovered.append(DiscoveredUrl(url=loc, surface=surface, confidence=conf, method="sitemap"))
        text = "\n".join(u["url"] for u in urls_out)
        meta = {"url_count": len(urls_out), "child_sitemaps": children[:50], "by_surface": by_surface, "truncated": len(entries) >= limit,
                "urls": urls_out, "structured": True}
        return Extraction(text=text, blocks=blocks, title=f"Sitemap — {len(urls_out)} URLs", meta=meta, discovered=discovered[:500])


__all__ = ["SitemapConnector", "parse_sitemap"]