#!/usr/bin/env python3
# -----------------------------------------------------------------------------
# House-Ka — scout RealtyPress reste-du-Canada, PASSE 2 (2026-08-27)
# Dorks plus discriminants : les fiches RealtyPress/DDF portent « MLS® Number »
# + la queue de description « (id:NNNN) » — combo quasi unique. Ciblage par
# grande ville des provinces encore mal couvertes.
# Sortie : /tmp/rp_canada_candidates2.json
# -----------------------------------------------------------------------------
import json
import os
import re
import time
from pathlib import Path
from urllib.parse import urlparse

import requests

env = Path.home() / ".claude" / ".env"
for line in env.read_text().splitlines():
    if line.startswith("SERPER_API_KEY="):
        os.environ.setdefault("SERPER_API_KEY", line.split("=", 1)[1].strip())
KEY = os.environ["SERPER_API_KEY"]

UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/126.0 Safari/537.36")

CITIES = {
    "British Columbia": ["Vancouver", "Surrey", "Victoria", "Kelowna", "Kamloops",
                         "Nanaimo", "Abbotsford", "Prince George", "Chilliwack", "Vernon"],
    "Alberta": ["Calgary", "Edmonton", "Red Deer", "Lethbridge", "Fort McMurray",
                "Airdrie", "Medicine Hat", "Grande Prairie"],
    "Saskatchewan": ["Saskatoon", "Regina", "Prince Albert", "Moose Jaw", "Swift Current"],
    "Manitoba": ["Winnipeg", "Brandon", "Steinbach", "Portage la Prairie"],
    "New Brunswick": ["Moncton", "Fredericton", "Saint John", "Dieppe", "Bathurst"],
    "Nova Scotia": ["Halifax", "Dartmouth", "Sydney", "Truro", "Bridgewater"],
    "Prince Edward Island": ["Charlottetown", "Summerside", "Stratford"],
    "Newfoundland and Labrador": ["St. John's", "Mount Pearl", "Corner Brook"],
    "Yukon": ["Whitehorse"],
    "Northwest Territories": ["Yellowknife"],
}

SKIP_DOMAINS = re.compile(
    r"realtor\.ca|realtypress\.ca|zillow|point2homes|remax\.ca|royallepage\.ca|"
    r"centris|duproprio|kijiji|facebook|youtube|instagram|linkedin|wikipedia|"
    r"realtylink|housesigma|zolo\.ca|honestdoor|wowa\.ca|estately|redfin|"
    r"strata\.ca|rew\.ca|ovlix|homefinder|propertyguys|fandom|reddit")

sess = requests.Session()
sess.headers["User-Agent"] = UA
sess.cookies.set("disclaimer", "accepted")


def serper(q: str) -> list[str]:
    try:
        r = requests.post("https://google.serper.dev/search",
                          headers={"X-API-KEY": KEY, "Content-Type": "application/json"},
                          json={"q": q, "gl": "ca", "hl": "en", "num": 30}, timeout=20)
        r.raise_for_status()
        out = []
        for it in r.json().get("organic", []):
            host = urlparse(it.get("link", "")).netloc.lower().removeprefix("www.")
            if host and not SKIP_DOMAINS.search(host):
                out.append(host)
        return out
    except Exception as e:
        print(f"  serper KO ({q[:60]}…): {e}", flush=True)
        return []


def fetch(url: str) -> str:
    try:
        r = sess.get(url, timeout=18, allow_redirects=True)
        return r.text if r.status_code == 200 else ""
    except Exception:
        return ""


CARD = "rps-property-result"
PROV_RE = re.compile(r",\s*(British Columbia|Alberta|Saskatchewan|Manitoba|"
                     r"New Brunswick|Nova Scotia|Prince Edward Island|"
                     r"Newfoundland (?:and|&) Labrador|Ontario|Qu[ée]bec|Yukon|"
                     r"Northwest Territories|Nunavut)\b", re.I)


def probe(domain: str) -> dict | None:
    for archive in ("listing", "listings", "all-regional-listings", "property-listings"):
        base = f"https://{domain}/{archive}"
        body = fetch(f"{base}/?posts_per_page=100")
        n = body.count(CARD)
        if n < 3:
            continue
        provs: dict[str, int] = {}
        for m in PROV_RE.finditer(body):
            provs[m.group(1).title()] = provs.get(m.group(1).title(), 0) + 1
        last_ok, per_page = 1, n
        for pg in (5, 20, 50, 100, 200, 400, 800):
            b = fetch(f"{base}/page/{pg}/?posts_per_page=100")
            c = b.count(CARD)
            if c >= 3:
                last_ok, per_page = pg, max(per_page, c)
            else:
                break
            time.sleep(0.4)
        return {"domain": domain, "archive": archive, "cards_page1": n,
                "provinces": provs, "approx_pages_gte": last_ok,
                "approx_volume_gte": last_ok * per_page}
    return None


def main() -> None:
    domains: dict[str, str] = {}
    for prov, cities in CITIES.items():
        for c in cities:
            for q in (f'"MLS® Number" "(id:" "{c}"',
                      f'"MLS® Number" "Ownership Type" "{c}, {prov}"'):
                for d in serper(q):
                    domains.setdefault(d, prov)
                time.sleep(0.35)
        print(f"[scout2] {prov}: {sum(1 for v in domains.values() if v == prov)} candidats",
              flush=True)

    print(f"[scout2] {len(domains)} domaines — validation live…", flush=True)
    results = []
    for i, (d, hint) in enumerate(sorted(domains.items()), 1):
        r = probe(d)
        if r:
            r["province_hint"] = hint
            results.append(r)
            print(f"  ✅ {d} ({r['archive']}) ~≥{r['approx_volume_gte']} — {r['provinces']}",
                  flush=True)
        if i % 25 == 0:
            print(f"  … {i}/{len(domains)}", flush=True)
        time.sleep(0.3)

    Path("/tmp/rp_canada_candidates2.json").write_text(
        json.dumps(results, indent=1, ensure_ascii=False))
    print(f"[scout2] {len(results)} sites validés → /tmp/rp_canada_candidates2.json",
          flush=True)


if __name__ == "__main__":
    main()