#!/usr/bin/env python3
"""Quick pre-validation probe for candidate sensor URLs (registry authoring helper).

Usage: python3 scripts/gen-probe-urls.py <urls.txt> [--out report.tsv] [--parallel 12]
One URL per line (blank lines and # comments ignored). For each URL a curl GET is issued (20 s timeout,
browser-like User-Agent, redirects followed) and the body is classified:
  FEED  n=<items>      RSS 2.0 / Atom / RDF with <item>/<entry> count
  JSON  <shape>        JSON document — reports top-level keys / array length / guessed items path
  HTML  <bytes>        server-rendered HTML (size of visible text)
  ERR   <code|reason>  HTTP error, timeout, TLS failure, WAF challenge…
The result is a TSV: url, class, detail, http code, final url, content-type. It is only a triage aid — the real
gate is apps/engine/src/validate.ts.
"""
import concurrent.futures
import json
import re
import subprocess
import sys

UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15"


def guess_items(obj, path=""):
    """Find the first array of dicts inside a JSON object (depth ≤ 3)."""
    if isinstance(obj, list):
        if obj and isinstance(obj[0], dict):
            return path or "", len(obj), sorted(obj[0].keys())[:12]
        return None
    if isinstance(obj, dict):
        for k, v in obj.items():
            r = guess_items(v, f"{path}.{k}" if path else k)
            if r:
                return r
    return None


def probe(url: str):
    try:
        p = subprocess.run(
            ["curl", "-sSL", "--max-time", "25", "--connect-timeout", "10", "-A", UA,
             "-H", "Accept: application/rss+xml, application/atom+xml, application/json, text/html;q=0.9, */*;q=0.8",
             "-w", "\n@@@%{http_code}@@@%{url_effective}@@@%{content_type}", url],
            capture_output=True, timeout=40)
        out = p.stdout.decode("utf-8", "replace")
        if "@@@" not in out:
            err = p.stderr.decode("utf-8", "replace").strip().splitlines()
            return url, "ERR", (err[-1] if err else "no response")[:80], "", "", ""
        body, meta = out.rsplit("\n@@@", 1)
        code, final, ctype = (meta.split("@@@") + ["", ""])[:3]
    except subprocess.TimeoutExpired:
        return url, "ERR", "timeout", "", "", ""
    if code and code != "200":
        return url, "ERR", f"http {code}", code, final, ctype
    b = body.lstrip(" \r\n\t")
    if b.startswith("{") or b.startswith("["):
        try:
            obj = json.loads(b)
        except Exception as e:  # noqa: BLE001
            return url, "ERR", f"bad json {str(e)[:40]}", code, final, ctype
        g = guess_items(obj)
        if g:
            return url, "JSON", f"items@{g[0] or '(root)'} n={g[1]} keys={','.join(g[2])}", code, final, ctype
        keys = list(obj.keys())[:10] if isinstance(obj, dict) else f"array[{len(obj)}]"
        return url, "JSON", f"no list; keys={keys}", code, final, ctype
    if re.search(r"<(rss|feed|rdf:RDF)[\s>]", b[:4000]):
        n = len(re.findall(r"<(item|entry)[\s>]", b))
        return url, "FEED", f"n={n}", code, final, ctype
    if re.search(r"<(urlset|sitemapindex)[\s>]", b[:2000]):
        n = len(re.findall(r"<loc>", b))
        return url, "SITEMAP", f"n={n}", code, final, ctype
    low = b[:6000].lower()
    if "cf-chl" in low or "just a moment" in low or "incapsula" in low or "_incapsula_resource" in low or "captcha" in low or "access denied" in low:
        return url, "ERR", "waf challenge", code, final, ctype
    text = re.sub(r"<script.*?</script>|<style.*?</style>", " ", b, flags=re.S | re.I)
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"\s+", " ", text).strip()
    title = re.search(r"<title[^>]*>(.*?)</title>", b, flags=re.S | re.I)
    return url, "HTML", f"text={len(text)} title={(title.group(1).strip()[:60] if title else '')!r}", code, final, ctype


def main() -> None:
    src = sys.argv[1]
    out = sys.argv[sys.argv.index("--out") + 1] if "--out" in sys.argv else None
    par = int(sys.argv[sys.argv.index("--parallel") + 1]) if "--parallel" in sys.argv else 12
    urls = [l.strip() for l in open(src) if l.strip() and not l.startswith("#")]
    rows = []
    with concurrent.futures.ThreadPoolExecutor(max_workers=par) as ex:
        for r in ex.map(probe, urls):
            rows.append(r)
            print("\t".join(r), flush=True)
    if out:
        with open(out, "w") as f:
            for r in rows:
                f.write("\t".join(r) + "\n")


if __name__ == "__main__":
    main()