#!/usr/bin/env python3
# -----------------------------------------------------------------------------
# Home-Ka — probe_sources.py : turn discovered brokerages into SOURCES.
#
# For every brokerage in the registry (not yet a source), test the cheapest
# no-proxy path: sitemap.xml → listing URLs → one page → JSON-LD listing node.
# When a site qualifies, register a `jsonld` source (config inferred: sitemap,
# url_include, external_id_regex, state) and stamp the brokerage
# (possible_feed_type=jsonld-ready, evidence.jsonld_probe).
#
#   .venv/bin/python scripts/probe_sources.py [limit] [--register]
#
# Direct requests only (2-3 per site, 0.6s politeness) — probing must stay
# cheap; the resilient/proxy chain is reserved for registered sources.
# -----------------------------------------------------------------------------
from __future__ import annotations

import json
import re
import sys
import time
from pathlib import Path

import requests

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from homeka import db                                    # noqa: E402
from homeka.connectors.json.jsonld_site import iter_ld, _LISTING_TYPES  # noqa: E402

UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/126 Safari/537.36 "
      "HomeKaBot/1.0 (+https://www.home-ka.com/bot; contact@spboucher.ai)")
LOC_RE = re.compile(r"<loc>\s*(.*?)\s*</loc>", re.S | re.I)
LISTING_URL_RE = re.compile(
    r"/(listing|property|properties|homedetails|homes-for-sale|home-for-sale|idx)/", re.I)


def probe(website: str, s: requests.Session) -> dict | None:
    base = website if website.startswith("http") else f"https://{website}"
    base = base.rstrip("/")
    try:
        r = s.get(f"{base}/sitemap.xml", timeout=15, allow_redirects=True)
        if r.status_code != 200 or "<" not in r.text[:200]:
            return None
    except requests.RequestException:
        return None
    locs = LOC_RE.findall(r.text)
    nested = [u for u in locs if u.endswith(".xml")][:8]
    listing_urls = [u for u in locs if LISTING_URL_RE.search(u)]
    for sm in nested:
        if listing_urls:
            break
        try:
            time.sleep(0.6)
            body = s.get(sm, timeout=15).text
        except requests.RequestException:
            continue
        listing_urls = [u for u in LOC_RE.findall(body)
                        if LISTING_URL_RE.search(u)]
    if not listing_urls:
        return None
    sample = listing_urls[len(listing_urls) // 2]
    try:
        time.sleep(0.6)
        page = s.get(sample, timeout=20).text
    except requests.RequestException:
        return None
    nodes = [n for n in iter_ld(page)
             if str(n.get("@type") or "").lower() in _LISTING_TYPES]
    if not nodes:
        return None
    has_addr = any(isinstance(n.get("address"), dict) and
                   n["address"].get("streetAddress") for n in nodes)
    has_price = any(n.get("offers") for n in nodes)
    if not (has_addr or has_price):
        return None
    # infer url_include + external_id_regex from the sample URL
    m = re.search(r"/(listing|property|properties|homedetails)/", sample, re.I)
    seg = m.group(1).lower() if m else "listing"
    include = f"/{seg}/"
    ext_re = ""
    mid = re.search(rf"/{seg}/([A-Za-z0-9_-]+)/", sample, re.I)
    if mid and len(mid.group(1)) <= 24:
        ext_re = rf"/{seg}/([A-Za-z0-9_-]+)/"
    return {"sample": sample, "n_urls": len(listing_urls),
            "url_include": include, "external_id_regex": ext_re,
            "has_addr": has_addr, "has_price": has_price,
            "types": sorted({str(n.get("@type")) for n in nodes})}


def main() -> None:
    limit = next((int(a) for a in sys.argv[1:] if a.isdigit()), 100)
    register = "--register" in sys.argv
    con = db.connect()
    existing_sites = set()
    for src in db.get_sources(con):
        u = (src["config"].get("base_url") or "").replace("https://", "")
        u = u.replace("http://", "").rstrip("/").lower()
        existing_sites.add(u.removeprefix("www."))
    rows = con.execute(
        "SELECT id, name, website, states FROM brokerages"
        " WHERE website<>'' AND (inspect_error IS NULL OR inspect_error='')"
        " AND (json_extract(COALESCE(evidence,'{}'),'$.jsonld_probe') IS NULL)"
        " ORDER BY priority_score DESC LIMIT ?", (limit,)).fetchall()
    s = requests.Session()
    s.headers["User-Agent"] = UA
    found = 0
    for r in rows:
        site = r["website"].lower().lstrip("www.")
        if r["website"].lower().replace("www.", "") in {e.replace("www.", "") for e in existing_sites}:
            continue
        time.sleep(0.6)
        try:
            res = probe(r["website"], s)
        except Exception:
            res = None
        ev_row = con.execute("SELECT evidence FROM brokerages WHERE id=?",
                             (r["id"],)).fetchone()
        ev = json.loads(ev_row["evidence"] or "{}")
        ev["jsonld_probe"] = {"ok": bool(res), "ts": int(time.time()),
                              **({k: res[k] for k in ("n_urls", "types")} if res else {})}
        sets = ["evidence=?", "updated=?"]
        args: list = [json.dumps(ev, ensure_ascii=False), time.time()]
        if res:
            found += 1
            sets += ["possible_feed_type='jsonld-ready'",
                     "feed_probability_score=MAX(COALESCE(feed_probability_score,0), 55)"]
            print(f"[OK] {r['name']} — {res['n_urls']} listing URLs, "
                  f"types={res['types']}, sample={res['sample'][:80]}")
            if register:
                states = json.loads(r["states"] or "[]")
                sid = re.sub(r"[^a-z0-9]+", "_",
                             r["name"].lower()).strip("_")[:40]
                db.upsert_source(
                    con, sid, r["name"], "jsonld",
                    config={"base_url": f"https://{r['website']}",
                            "url_include": res["url_include"],
                            **({"external_id_regex": res["external_id_regex"]}
                               if res["external_id_regex"] else {}),
                            "max_pages": 500,
                            **({"static": {"state": states[0]}}
                               if len(states) == 1 else {})},
                    authority=2, states=states, brokerage_id=r["id"],
                    notes="auto-registered by probe_sources.py (JSON-LD ready)")
                con.execute("UPDATE brokerages SET source_id=?,"
                            " partnership_status='prospect' WHERE id=?",
                            (sid, r["id"]))
        con.execute(f"UPDATE brokerages SET {', '.join(sets)} WHERE id=?",
                    args + [r["id"]])
        con.commit()
    con.close()
    print(f"probed {len(rows)} brokerage(s), {found} JSON-LD ready"
          + (" (registered)" if register else ""))


if __name__ == "__main__":
    main()