spb/house-ka

1#!/usr/bin/env python32# -----------------------------------------------------------------------------3# House-Ka — scout RealtyPress reste-du-Canada (2026-08-27)4# scripts/scout_canada_rp.py : découverte de sites d'agences RealtyPress/CREA5#   DDF hors Ontario/Québec via Serper (Google), puis validation live par6#   empreinte (`rps-property-result`) et estimation de volume par sondage de7#   pagination. Sortie : /tmp/rp_canada_candidates.json8# Usage : SERPER_API_KEY dans ~/.claude/.env — .venv/bin/python scripts/scout_canada_rp.py9# -----------------------------------------------------------------------------10import json11import os12import re13import time14from pathlib import Path15from urllib.parse import urlparse1617import requests1819# clé Serper (~/.claude/.env du nœud)20env = Path.home() / ".claude" / ".env"21for line in env.read_text().splitlines():22    if line.startswith("SERPER_API_KEY="):23        os.environ.setdefault("SERPER_API_KEY", line.split("=", 1)[1].strip())24KEY = os.environ["SERPER_API_KEY"]2526UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "27      "(KHTML, like Gecko) Chrome/126.0 Safari/537.36")2829PROVINCES = {30    "British Columbia": ["Vancouver", "Victoria", "Kelowna", "Kamloops", "Nanaimo", "Prince George"],31    "Alberta": ["Calgary", "Edmonton", "Red Deer", "Lethbridge", "Medicine Hat", "Grande Prairie"],32    "Saskatchewan": ["Saskatoon", "Regina", "Prince Albert", "Moose Jaw"],33    "Manitoba": ["Winnipeg", "Brandon", "Steinbach"],34    "New Brunswick": ["Moncton", "Fredericton", "Saint John", "Miramichi"],35    "Nova Scotia": ["Halifax", "Sydney", "Truro", "Yarmouth"],36    "Prince Edward Island": ["Charlottetown", "Summerside"],37    "Newfoundland and Labrador": ["St. John's", "Corner Brook", "Gander"],38    "Yukon": ["Whitehorse"],39    "Northwest Territories": ["Yellowknife"],40}4142SKIP_DOMAINS = re.compile(43    r"realtor\.ca|realtypress\.ca|zillow|point2homes|remax\.ca|royallepage\.ca|"44    r"centris|duproprio|kijiji|facebook|youtube|instagram|linkedin|wikipedia|"45    r"realtylink|housesigma|zolo\.ca|honestdoor|wowa\.ca|estately|redfin|"46    r"strata\.ca|rew\.ca|ovlix|homefinder|propertyguys")4748sess = requests.Session()49sess.headers["User-Agent"] = UA50sess.cookies.set("disclaimer", "accepted")515253def serper(q: str) -> list[str]:54    try:55        r = requests.post("https://google.serper.dev/search",56                          headers={"X-API-KEY": KEY, "Content-Type": "application/json"},57                          json={"q": q, "gl": "ca", "hl": "en", "num": 30}, timeout=20)58        r.raise_for_status()59        out = []60        for it in r.json().get("organic", []):61            host = urlparse(it.get("link", "")).netloc.lower().removeprefix("www.")62            if host and not SKIP_DOMAINS.search(host):63                out.append(host)64        return out65    except Exception as e:66        print(f"  serper KO ({q[:50]}…): {e}")67        return []686970def fetch(url: str) -> str:71    try:72        r = sess.get(url, timeout=18, allow_redirects=True)73        return r.text if r.status_code == 200 else ""74    except Exception:75        return ""767778CARD = "rps-property-result"79PROV_RE = re.compile(r",\s*(British Columbia|Alberta|Saskatchewan|Manitoba|"80                     r"New Brunswick|Nova Scotia|Prince Edward Island|"81                     r"Newfoundland and Labrador|Ontario|Qu[ée]bec|Yukon|"82                     r"Northwest Territories|Nunavut)\b", re.I)838485def probe(domain: str) -> dict | None:86    """Valide l'empreinte RealtyPress et estime le volume par sondage de pages."""87    for archive in ("listing", "listings", "all-regional-listings"):88        base = f"https://{domain}/{archive}"89        body = fetch(f"{base}/?posts_per_page=100")90        n = body.count(CARD)91        if n < 3:92            continue93        # provinces observées sur la 1re page94        provs: dict[str, int] = {}95        for m in PROV_RE.finditer(body):96            p = m.group(1).title().replace("Nd", "nd").replace(" And ", " and ")97            provs[p] = provs.get(p, 0) + 198        # volume : dernière page non vide parmi des jalons croissants99        last_ok, per_page = 1, n100        for pg in (5, 20, 50, 100, 200, 400, 800, 1200):101            b = fetch(f"{base}/page/{pg}/?posts_per_page=100")102            c = b.count(CARD)103            if c >= 3:104                last_ok, per_page = pg, max(per_page, c)105            else:106                break107            time.sleep(0.4)108        return {"domain": domain, "archive": archive, "cards_page1": n,109                "provinces": provs, "approx_pages_gte": last_ok,110                "approx_volume_gte": last_ok * per_page}111    return None112113114def main() -> None:115    domains: dict[str, str] = {}116    for prov, cities in PROVINCES.items():117        queries = [118            f'"MLS® Number" "{prov}" inurl:listing',119            f'"Powered by RealtyPress" "{prov}"',120        ] + [f'inurl:/listing/ "MLS®" "{c}" "{prov}"' for c in cities[:3]]121        for q in queries:122            for d in serper(q):123                domains.setdefault(d, prov)124            time.sleep(0.4)125        print(f"[scout] {prov}: {sum(1 for v in domains.values() if v == prov)} candidats cumulés")126127    print(f"[scout] {len(domains)} domaines candidats — validation live…")128    results = []129    for i, (d, prov_hint) in enumerate(sorted(domains.items()), 1):130        r = probe(d)131        if r:132            r["province_hint"] = prov_hint133            results.append(r)134            print(f"  ✅ {d} ({r['archive']}) ~≥{r['approx_volume_gte']} — {r['provinces']}")135        if i % 20 == 0:136            print(f"  … {i}/{len(domains)}")137        time.sleep(0.3)138139    out = Path("/tmp/rp_canada_candidates.json")140    out.write_text(json.dumps(results, indent=1, ensure_ascii=False))141    print(f"[scout] {len(results)} sites RealtyPress validés → {out}")142143144if __name__ == "__main__":145    main()146