#!/usr/bin/env python3
"""Generate config/sources.d/51-world-governments-regulators.yaml from a probed candidate list.

Usage: python3 scripts/gen-world-gov.py probe      # curl every candidate → /tmp/wg-probe.json
       python3 scripts/gen-world-gov.py write      # write the fragment from surviving candidates
Candidates live in scripts/gen_world_gov_data.py (SOURCES + CANDS). A candidate survives the probe when
the response is 200 and looks like what the connector expects (feed with items / JSON array / substantive HTML).
For every (source, sensor-name) only the first surviving alternative is kept.
"""
import json, os, re, subprocess, sys, glob
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlparse

sys.path.insert(0, os.path.dirname(__file__))
from gen_world_gov_data import SOURCES, CANDS  # noqa: E402
import gen_world_gov_data2, gen_world_gov_data3, gen_world_gov_data4  # noqa: E402,F401  (register candidates)

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "config/sources.d/51-world-governments-regulators.yaml")
PROBE = "/tmp/wg-probe.json"
UA = "WebSensorBot/0.1 (+https://www.websensor.io/bot; contact@websensor.io)"


def expand(url):
    import datetime
    def rep(m):
        n, u = m.group(1), m.group(2)
        d = datetime.datetime.utcnow()
        if n:
            d -= datetime.timedelta(**{{"h": "hours", "d": "days", "m": "minutes"}[u]: int(n)})
        return d.strftime("%Y-%m-%dT%H:%M:%S.000")
    return re.sub(r"\{now(?:-(\d+)([hmd]))?\}", rep, url)


def probe_one(c):
    url = expand(c["url"])
    body = f"/tmp/wg-body-{abs(hash(url))}.bin"
    try:
        r = subprocess.run(["curl", "-sSL", "--compressed", "--max-time", "30", "-A", UA, "-o", body,
                            "-w", "%{http_code}\t%{content_type}\t%{url_effective}", url],
                           capture_output=True, text=True, timeout=45)
        parts = r.stdout.strip().split("\t")
        code = int(parts[0] or 0); ctype = parts[1] if len(parts) > 1 else ""; final = parts[2] if len(parts) > 2 else ""
    except Exception as e:  # noqa: BLE001
        return {**c, "code": 0, "note": str(e)[:80], "ok": False}
    data = b""
    if os.path.exists(body):
        data = open(body, "rb").read(); os.remove(body)
    txt = data[:400000].decode("utf-8", "ignore")
    kind = c["kind"]; ok = False; note = ""
    if code == 200:
        low = txt[:2000].lower()
        if kind == "rss":
            if "<rss" in low or "<feed" in low or "<rdf:rdf" in low or low.lstrip().startswith("{"):
                n = txt.count("<item") + txt.count("<entry")
                ok = n > 0; note = f"items={n}"
            else:
                note = "not-a-feed"
        elif kind == "json":
            try:
                j = json.loads(txt)
                arr = j
                for p in (c.get("cfg", {}).get("itemsPath") or "").split("."):
                    if p: arr = arr[int(p)] if isinstance(arr, list) else arr[p]
                ok = isinstance(arr, list) and len(arr) > 0; note = f"items={len(arr) if isinstance(arr, list) else '?'}"
            except Exception as e:  # noqa: BLE001
                note = f"json-error {str(e)[:40]}"
        else:  # http (html)
            text = re.sub(r"<script.*?</script>|<style.*?</style>", " ", txt, flags=re.S | re.I)
            text = re.sub(r"<[^>]+>", " ", text); text = re.sub(r"\s+", " ", text)
            ok = len(text) > 1500 and "<html" in low; note = f"text={len(text)}"
    return {**c, "code": code, "ctype": ctype[:40], "final": final, "ok": ok, "note": note}


def do_probe():
    with ThreadPoolExecutor(max_workers=16) as ex:
        res = list(ex.map(probe_one, CANDS))
    json.dump(res, open(PROBE, "w"), indent=1)
    okn = sum(r["ok"] for r in res)
    print(f"probed {len(res)} → ok {okn}")
    for r in res:
        print(f"{'OK ' if r['ok'] else '-- '} {r['code']:>3} {r['id']:28s} {r['name'][:28]:28s} {r['url'][:90]} {r['note']}")


def registry_index():
    import yaml
    ids, urls, hosts = {}, set(), {}
    for f in [os.path.join(ROOT, "config/sources.yaml")] + sorted(glob.glob(os.path.join(ROOT, "config/sources.d/*.yaml"))):
        if f.endswith("51-world-governments-regulators.yaml"):
            continue
        d = yaml.safe_load(open(f))
        for s in d.get("sources", []):
            if not s.get("extend"):
                ids[s["id"]] = f
            for sen in s.get("sensors", []) or []:
                urls.add(sen["url"]); hosts.setdefault(urlparse(sen["url"]).hostname, set()).add(s["id"])
    return ids, urls, hosts


def q(s):
    return json.dumps(s, ensure_ascii=False)


# Sensor ids derive from ASCII letters of the name — give non-Latin names an English equivalent.
NAMES = {
    "новини": "news (Ukrainian)", "новости": "news (Russian)", "новини (VGP)": "news", "документи": "documents", "документы": "documents (Russian)",
    "постанови та розпорядження": "resolutions and orders", "рішення": "decisions", "експрес-випуски": "express releases",
    "оперативна інформація": "operational information", "оперативная информация": "operational information (Russian)",
    "пресс-релизы": "press releases (Russian)", "новые документы": "new documents (Russian)",
    "прессъобщения": "press releases (Bulgarian)", "новини (Bulgarian)": "news", "законопроекти": "bills (Bulgarian)", "последен брой": "latest issue (Bulgarian)",
    "ανακοινώσεις": "announcements (Greek)", "δελτία τύπου": "press releases (Greek)", "νομοσχέδια": "bills (Greek)", "τελευταία ΦΕΚ": "latest gazette issues (Greek)",
    "ข่าวรัฐบาล": "government news (Thai)", "ข่าวประชาสัมพันธ์": "news (Thai)", "ข่าวสาร": "news (Thai)", "ประกาศล่าสุด": "latest announcements (Thai)",
    "ข่าว ก.ล.ต.": "news (Thai)", "ข่าวกระทรวง": "ministry news (Thai)",
    "新聞稿": "press releases (Chinese)", "新聞": "news (Chinese)", "最新消息": "latest news (Chinese)", "最新公報": "latest gazette (Chinese)",
    "新聞資料": "news (Chinese)", "工作动态": "news (Chinese)", "政策法规": "policies and regulations (Chinese)", "总局动态": "news (Chinese)",
    "新闻发布": "press releases (Chinese)", "药品监管动态": "drug regulation news (Chinese)", "最新地震": "latest earthquakes (Chinese)",
    "tin tức": "news (Vietnamese)", "tin tức (VGP)": "news (Vietnamese, VGP)", "văn bản mới": "new legal documents", "công báo mới": "latest gazette issues",
    "hírek": "news (Hungarian)", "irományok": "parliamentary documents", "legfrissebb lapszámok": "latest issues", "gyorstájékoztatók": "first releases (Hungarian)",
    "határozatok": "decisions", "sajtóközlemények": "press releases (Hungarian)",
}


def ascii_name(name, lang):
    if name in NAMES:
        return NAMES[name]
    if not re.search(r"[a-z0-9]", name.lower()):
        return f"news ({lang or 'local'})"
    return name


def do_write():
    res = json.load(open(PROBE))
    ids, urls, hosts = registry_index()
    bad = set()
    if os.path.exists("/tmp/wg-bad.txt"):
        bad = {l.strip() for l in open("/tmp/wg-bad.txt") if l.strip()}
    keep = {}
    for r in res:
        if not r["ok"] or r["url"] in bad:
            continue
        if r["url"] in urls:
            print("DUP URL skipped:", r["url"]); continue
        k = (r["id"], r["name"])
        if k in keep:
            continue
        keep[k] = r
    by_src = {}
    for (sid, _), r in keep.items():
        by_src.setdefault(sid, []).append(r)
    lines = [
        "# config/sources.d/51-world-governments-regulators.yaml — world governments & regulators (authored 2026-09-11):",
        "# governments, parliaments, official gazettes, statistics offices, data-protection / competition / telecom / financial /",
        "# energy / medicines regulators, transport-safety boards and civil-protection alert feeds for the countries not covered by",
        "# 44-governments.yaml (Benelux, Ireland, Switzerland, Austria, Portugal, the Nordics, Central & Eastern Europe, the Baltics,",
        "# Ukraine, Türkiye, Israel, the Gulf, Egypt, Africa, Latin America, New Zealand, Singapore, Hong Kong, Taiwan, South-East Asia,",
        "# South Asia, China's English portals, Russia's official feeds) plus international bodies missing from 17. Every entry carries",
        "# `country:` (INT for international bodies) and `language:` when the feed is not in English. Every sensor was fetched and",
        "# parsed by apps/engine/src/validate.ts; blocked or client-rendered endpoints are recorded in `notes:`.",
        "sources:",
    ]
    cur_country = None
    secmap = {}
    for m in SOURCES.values():
        if m.get("section") and m["country"] not in secmap:
            secmap[m["country"]] = m["section"]
    for sid, meta in SOURCES.items():
        sens = by_src.get(sid, [])
        if not sens and not meta.get("keep_empty"):
            continue
        if meta.get("extend"):
            if sid not in ids:
                print("EXTEND of unknown id:", sid); continue
        elif sid in ids:
            print("ID COLLISION:", sid, ids[sid]); continue
        if meta["country"] != cur_country:
            cur_country = meta["country"]
            lines.append(f"  # ───────────────────────────── {secmap.get(cur_country, cur_country)} ─────────────────────────────")
        lines.append(f"  - id: {sid}")
        if meta.get("extend"):
            lines.append("    extend: true")
        else:
            lines.append(f"    name: {meta['name']}")
            lines.append(f"    domain: {meta['domain']}")
            if meta.get("homepage"):
                lines.append(f"    homepage: {meta['homepage']}")
            lines.append(f"    categories: [{', '.join(meta['categories'])}]")
            lines.append(f"    tier: {meta.get('tier', 'B')}")
            if meta.get("weight"):
                lines.append(f"    weight: {meta['weight']}")
            lines.append(f"    aliases: [{', '.join(meta['aliases'])}]")
            if meta.get("llm") is False:
                lines.append("    llm: false")
        lines.append(f"    country: {q(meta['country']) if meta['country'] in ('NO', 'ON', 'OFF', 'YES') else meta['country']}")
        if meta.get("language"):
            lines.append(f"    language: {q(meta['language']) if meta['language'] in ('no', 'on', 'off', 'yes', 'y', 'n') else meta['language']}")
        if meta.get("notes"):
            lines.append(f"    notes: {q(meta['notes'])}")
        if sens:
            lines.append("    sensors:")
            for r in sens:
                cfg = r.get("cfg")
                typ = {"rss": "RSS", "json": "REST_API", "http": "HTML"}[r["kind"]]
                if r["kind"] == "rss" and ("atom" in r["url"].lower() or "/feed" in r["url"].lower() and r["url"].endswith("feed")):
                    typ = "ATOM" if "atom" in r["url"].lower() else "RSS"
                conn = {"rss": "rss", "json": "jsonlist", "http": "http"}[r["kind"]]
                nm = ascii_name(r["name"], meta.get("language"))
                s = f'      - {{ name: {nm}, url: {q(r["url"])}, type: {typ}, connector: {conn}, tier: {r.get("tier") or meta.get("tier", "B")}'
                if cfg:
                    s += ", config: " + yaml_flow(cfg)
                s += " }"
                lines.append(s)
    open(OUT, "w").write("\n".join(lines) + "\n")
    n = sum(len(v) for v in by_src.values())
    print(f"wrote {OUT}: {sum(1 for s in SOURCES if by_src.get(s) or SOURCES[s].get('keep_empty'))} sources, {n} sensors")


def yaml_flow(o):
    if isinstance(o, dict):
        return "{ " + ", ".join(f"{k}: {yaml_flow(v)}" for k, v in o.items()) + " }"
    if isinstance(o, list):
        return "[" + ", ".join(yaml_flow(v) for v in o) + "]"
    if isinstance(o, bool):
        return "true" if o else "false"
    if isinstance(o, (int, float)):
        return str(o)
    return json.dumps(o, ensure_ascii=False)


if __name__ == "__main__":
    {"probe": do_probe, "write": do_write}[sys.argv[1]]()