#!/usr/bin/env python3
# -----------------------------------------------------------------------------
# House-Ka — scout RealtyPress reste-du-Canada (2026-08-27)
# scripts/scout_canada_rp.py : découverte de sites d'agences RealtyPress/CREA
#   DDF hors Ontario/Québec via Serper (Google), puis validation live par
#   empreinte (`rps-property-result`) et estimation de volume par sondage de
#   pagination. Sortie : /tmp/rp_canada_candidates.json
# Usage : SERPER_API_KEY dans ~/.claude/.env — .venv/bin/python scripts/scout_canada_rp.py
# -----------------------------------------------------------------------------
import json
import os
import re
import time
from pathlib import Path
from urllib.parse import urlparse

import requests

# clé Serper (~/.claude/.env du nœud)
env = Path.home() / ".claude" / ".env"
for line in env.read_text().splitlines():
    if line.startswith("SERPER_API_KEY="):
        os.environ.setdefault("SERPER_API_KEY", line.split("=", 1)[1].strip())
KEY = os.environ["SERPER_API_KEY"]

UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/126.0 Safari/537.36")

PROVINCES = {
    "British Columbia": ["Vancouver", "Victoria", "Kelowna", "Kamloops", "Nanaimo", "Prince George"],
    "Alberta": ["Calgary", "Edmonton", "Red Deer", "Lethbridge", "Medicine Hat", "Grande Prairie"],
    "Saskatchewan": ["Saskatoon", "Regina", "Prince Albert", "Moose Jaw"],
    "Manitoba": ["Winnipeg", "Brandon", "Steinbach"],
    "New Brunswick": ["Moncton", "Fredericton", "Saint John", "Miramichi"],
    "Nova Scotia": ["Halifax", "Sydney", "Truro", "Yarmouth"],
    "Prince Edward Island": ["Charlottetown", "Summerside"],
    "Newfoundland and Labrador": ["St. John's", "Corner Brook", "Gander"],
    "Yukon": ["Whitehorse"],
    "Northwest Territories": ["Yellowknife"],
}

SKIP_DOMAINS = re.compile(
    r"realtor\.ca|realtypress\.ca|zillow|point2homes|remax\.ca|royallepage\.ca|"
    r"centris|duproprio|kijiji|facebook|youtube|instagram|linkedin|wikipedia|"
    r"realtylink|housesigma|zolo\.ca|honestdoor|wowa\.ca|estately|redfin|"
    r"strata\.ca|rew\.ca|ovlix|homefinder|propertyguys")

sess = requests.Session()
sess.headers["User-Agent"] = UA
sess.cookies.set("disclaimer", "accepted")


def serper(q: str) -> list[str]:
    try:
        r = requests.post("https://google.serper.dev/search",
                          headers={"X-API-KEY": KEY, "Content-Type": "application/json"},
                          json={"q": q, "gl": "ca", "hl": "en", "num": 30}, timeout=20)
        r.raise_for_status()
        out = []
        for it in r.json().get("organic", []):
            host = urlparse(it.get("link", "")).netloc.lower().removeprefix("www.")
            if host and not SKIP_DOMAINS.search(host):
                out.append(host)
        return out
    except Exception as e:
        print(f"  serper KO ({q[:50]}…): {e}")
        return []


def fetch(url: str) -> str:
    try:
        r = sess.get(url, timeout=18, allow_redirects=True)
        return r.text if r.status_code == 200 else ""
    except Exception:
        return ""


CARD = "rps-property-result"
PROV_RE = re.compile(r",\s*(British Columbia|Alberta|Saskatchewan|Manitoba|"
                     r"New Brunswick|Nova Scotia|Prince Edward Island|"
                     r"Newfoundland and Labrador|Ontario|Qu[ée]bec|Yukon|"
                     r"Northwest Territories|Nunavut)\b", re.I)


def probe(domain: str) -> dict | None:
    """Valide l'empreinte RealtyPress et estime le volume par sondage de pages."""
    for archive in ("listing", "listings", "all-regional-listings"):
        base = f"https://{domain}/{archive}"
        body = fetch(f"{base}/?posts_per_page=100")
        n = body.count(CARD)
        if n < 3:
            continue
        # provinces observées sur la 1re page
        provs: dict[str, int] = {}
        for m in PROV_RE.finditer(body):
            p = m.group(1).title().replace("Nd", "nd").replace(" And ", " and ")
            provs[p] = provs.get(p, 0) + 1
        # volume : dernière page non vide parmi des jalons croissants
        last_ok, per_page = 1, n
        for pg in (5, 20, 50, 100, 200, 400, 800, 1200):
            b = fetch(f"{base}/page/{pg}/?posts_per_page=100")
            c = b.count(CARD)
            if c >= 3:
                last_ok, per_page = pg, max(per_page, c)
            else:
                break
            time.sleep(0.4)
        return {"domain": domain, "archive": archive, "cards_page1": n,
                "provinces": provs, "approx_pages_gte": last_ok,
                "approx_volume_gte": last_ok * per_page}
    return None


def main() -> None:
    domains: dict[str, str] = {}
    for prov, cities in PROVINCES.items():
        queries = [
            f'"MLS® Number" "{prov}" inurl:listing',
            f'"Powered by RealtyPress" "{prov}"',
        ] + [f'inurl:/listing/ "MLS®" "{c}" "{prov}"' for c in cities[:3]]
        for q in queries:
            for d in serper(q):
                domains.setdefault(d, prov)
            time.sleep(0.4)
        print(f"[scout] {prov}: {sum(1 for v in domains.values() if v == prov)} candidats cumulés")

    print(f"[scout] {len(domains)} domaines candidats — validation live…")
    results = []
    for i, (d, prov_hint) in enumerate(sorted(domains.items()), 1):
        r = probe(d)
        if r:
            r["province_hint"] = prov_hint
            results.append(r)
            print(f"  ✅ {d} ({r['archive']}) ~≥{r['approx_volume_gte']} — {r['provinces']}")
        if i % 20 == 0:
            print(f"  … {i}/{len(domains)}")
        time.sleep(0.3)

    out = Path("/tmp/rp_canada_candidates.json")
    out.write_text(json.dumps(results, indent=1, ensure_ascii=False))
    print(f"[scout] {len(results)} sites RealtyPress validés → {out}")


if __name__ == "__main__":
    main()