spb/house-ka

1#!/usr/bin/env python32# -----------------------------------------------------------------------------3# House-Ka — scout RealtyPress reste-du-Canada, PASSE 2 (2026-08-27)4# Dorks plus discriminants : les fiches RealtyPress/DDF portent « MLS® Number »5# + la queue de description « (id:NNNN) » — combo quasi unique. Ciblage par6# grande ville des provinces encore mal couvertes.7# Sortie : /tmp/rp_canada_candidates2.json8# -----------------------------------------------------------------------------9import json10import os11import re12import time13from pathlib import Path14from urllib.parse import urlparse1516import requests1718env = Path.home() / ".claude" / ".env"19for line in env.read_text().splitlines():20    if line.startswith("SERPER_API_KEY="):21        os.environ.setdefault("SERPER_API_KEY", line.split("=", 1)[1].strip())22KEY = os.environ["SERPER_API_KEY"]2324UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "25      "(KHTML, like Gecko) Chrome/126.0 Safari/537.36")2627CITIES = {28    "British Columbia": ["Vancouver", "Surrey", "Victoria", "Kelowna", "Kamloops",29                         "Nanaimo", "Abbotsford", "Prince George", "Chilliwack", "Vernon"],30    "Alberta": ["Calgary", "Edmonton", "Red Deer", "Lethbridge", "Fort McMurray",31                "Airdrie", "Medicine Hat", "Grande Prairie"],32    "Saskatchewan": ["Saskatoon", "Regina", "Prince Albert", "Moose Jaw", "Swift Current"],33    "Manitoba": ["Winnipeg", "Brandon", "Steinbach", "Portage la Prairie"],34    "New Brunswick": ["Moncton", "Fredericton", "Saint John", "Dieppe", "Bathurst"],35    "Nova Scotia": ["Halifax", "Dartmouth", "Sydney", "Truro", "Bridgewater"],36    "Prince Edward Island": ["Charlottetown", "Summerside", "Stratford"],37    "Newfoundland and Labrador": ["St. John's", "Mount Pearl", "Corner Brook"],38    "Yukon": ["Whitehorse"],39    "Northwest Territories": ["Yellowknife"],40}4142SKIP_DOMAINS = re.compile(43    r"realtor\.ca|realtypress\.ca|zillow|point2homes|remax\.ca|royallepage\.ca|"44    r"centris|duproprio|kijiji|facebook|youtube|instagram|linkedin|wikipedia|"45    r"realtylink|housesigma|zolo\.ca|honestdoor|wowa\.ca|estately|redfin|"46    r"strata\.ca|rew\.ca|ovlix|homefinder|propertyguys|fandom|reddit")4748sess = requests.Session()49sess.headers["User-Agent"] = UA50sess.cookies.set("disclaimer", "accepted")515253def serper(q: str) -> list[str]:54    try:55        r = requests.post("https://google.serper.dev/search",56                          headers={"X-API-KEY": KEY, "Content-Type": "application/json"},57                          json={"q": q, "gl": "ca", "hl": "en", "num": 30}, timeout=20)58        r.raise_for_status()59        out = []60        for it in r.json().get("organic", []):61            host = urlparse(it.get("link", "")).netloc.lower().removeprefix("www.")62            if host and not SKIP_DOMAINS.search(host):63                out.append(host)64        return out65    except Exception as e:66        print(f"  serper KO ({q[:60]}…): {e}", flush=True)67        return []686970def fetch(url: str) -> str:71    try:72        r = sess.get(url, timeout=18, allow_redirects=True)73        return r.text if r.status_code == 200 else ""74    except Exception:75        return ""767778CARD = "rps-property-result"79PROV_RE = re.compile(r",\s*(British Columbia|Alberta|Saskatchewan|Manitoba|"80                     r"New Brunswick|Nova Scotia|Prince Edward Island|"81                     r"Newfoundland (?:and|&) Labrador|Ontario|Qu[ée]bec|Yukon|"82                     r"Northwest Territories|Nunavut)\b", re.I)838485def probe(domain: str) -> dict | None:86    for archive in ("listing", "listings", "all-regional-listings", "property-listings"):87        base = f"https://{domain}/{archive}"88        body = fetch(f"{base}/?posts_per_page=100")89        n = body.count(CARD)90        if n < 3:91            continue92        provs: dict[str, int] = {}93        for m in PROV_RE.finditer(body):94            provs[m.group(1).title()] = provs.get(m.group(1).title(), 0) + 195        last_ok, per_page = 1, n96        for pg in (5, 20, 50, 100, 200, 400, 800):97            b = fetch(f"{base}/page/{pg}/?posts_per_page=100")98            c = b.count(CARD)99            if c >= 3:100                last_ok, per_page = pg, max(per_page, c)101            else:102                break103            time.sleep(0.4)104        return {"domain": domain, "archive": archive, "cards_page1": n,105                "provinces": provs, "approx_pages_gte": last_ok,106                "approx_volume_gte": last_ok * per_page}107    return None108109110def main() -> None:111    domains: dict[str, str] = {}112    for prov, cities in CITIES.items():113        for c in cities:114            for q in (f'"MLS® Number" "(id:" "{c}"',115                      f'"MLS® Number" "Ownership Type" "{c}, {prov}"'):116                for d in serper(q):117                    domains.setdefault(d, prov)118                time.sleep(0.35)119        print(f"[scout2] {prov}: {sum(1 for v in domains.values() if v == prov)} candidats",120              flush=True)121122    print(f"[scout2] {len(domains)} domaines — validation live…", flush=True)123    results = []124    for i, (d, hint) in enumerate(sorted(domains.items()), 1):125        r = probe(d)126        if r:127            r["province_hint"] = hint128            results.append(r)129            print(f"  ✅ {d} ({r['archive']}) ~≥{r['approx_volume_gte']} — {r['provinces']}",130                  flush=True)131        if i % 25 == 0:132            print(f"  … {i}/{len(domains)}", flush=True)133        time.sleep(0.3)134135    Path("/tmp/rp_canada_candidates2.json").write_text(136        json.dumps(results, indent=1, ensure_ascii=False))137    print(f"[scout2] {len(results)} sites validés → /tmp/rp_canada_candidates2.json",138          flush=True)139140141if __name__ == "__main__":142    main()143