fix(realtypress): page 1 nue + pagination native profonde + région canonique
- /listing/page/1/ répond 200 SANS cartes sur certains sites (denisedunn, raymondanthony) → la page 1 requête l archive nue. - hanlonrealty plafonne l offset posts_per_page (~3 500 items) mais pagine à fond en natif 10/page → option registre page_size:10 (max_pages 2400). - Terre-Neuve : « Newfoundland & Labrador » → forme canonique « and ».
2 changed files +12 −38
modified
data/canada_agencies.json
+0 −36
@@ -107,41 +107,5 @@ | ||
| 107 | 107 | "site": "https://cbnorthbay.com", |
| 108 | 108 | "note": "Recensement ON 2026-08-27 : ~307 fiches (North Bay). RealtyPress/DDF.", |
| 109 | 109 | "province": "Ontario" |
| 110 | − }, | |
| 111 | − { | |
| 112 | − "id": "rp_ag_hanlonrealty", | |
| 113 | − "name": "hanlonrealty.ca (Newfoundland and Labrador)", | |
| 114 | − "site": "https://hanlonrealty.ca", | |
| 115 | − "archive": "listing", | |
| 116 | − "max_pages": 400, | |
| 117 | − "province": "Newfoundland and Labrador", | |
| 118 | − "note": "Recensement CANADA 2026-08-27 : ~≥22000 fiches, provinces page 1 : {}. RealtyPress/DDF." | |
| 119 | − }, | |
| 120 | − { | |
| 121 | − "id": "rp_ag_denisedunnrealtor", | |
| 122 | − "name": "denisedunnrealtor.com (British Columbia)", | |
| 123 | − "site": "https://denisedunnrealtor.com", | |
| 124 | − "archive": "listing", | |
| 125 | − "max_pages": 400, | |
| 126 | − "province": "British Columbia", | |
| 127 | − "note": "Recensement CANADA 2026-08-27 : ~≥2400 fiches, provinces page 1 : {'British Columbia': 24}. RealtyPress/DDF." | |
| 128 | − }, | |
| 129 | − { | |
| 130 | − "id": "rp_ag_reddoorrealty", | |
| 131 | − "name": "reddoorrealty.ca (Nova Scotia)", | |
| 132 | − "site": "https://reddoorrealty.ca", | |
| 133 | − "archive": "listing", | |
| 134 | − "max_pages": 10, | |
| 135 | − "province": "Nova Scotia", | |
| 136 | − "note": "Recensement CANADA 2026-08-27 : ~≥580 fiches, provinces page 1 : {'Nova Scotia': 202}. RealtyPress/DDF." | |
| 137 | − }, | |
| 138 | − { | |
| 139 | − "id": "rp_ag_raymondanthony", | |
| 140 | − "name": "raymondanthony.com (Alberta)", | |
| 141 | − "site": "https://raymondanthony.com", | |
| 142 | − "archive": "listing", | |
| 143 | − "max_pages": 10, | |
| 144 | − "province": "Alberta", | |
| 145 | − "note": "Recensement CANADA 2026-08-27 : ~≥60 fiches, provinces page 1 : {'Alberta': 25}. RealtyPress/DDF." | |
| 146 | 110 | } |
| 147 | 111 | ] |
| \ No newline at end of file | ||
modified
immoka/connectors/realtypress.py
+12 −2
@@ -92,6 +92,10 @@ class _RealtyPress(BaseConnector): | ||
| 92 | 92 | archive = "listing" # chemin de l'archive (revelrealty: "listings", |
| 93 | 93 | # codygroup: "all-regional-listings") |
| 94 | 94 | max_pages = 150 # 100 cartes/page → jusqu'à 15 000 fiches par site |
| 95 | + use_pp = True # False (registre "page_size": 10) : pagination | |
| 96 | + # NATIVE 10/page — pour les sites qui plafonnent | |
| 97 | + # l'offset de posts_per_page (ex. hanlonrealty | |
| 98 | + # ~3 500 items) mais paginent à fond en natif | |
| 95 | 99 | request_delay = 0.6 |
| 96 | 100 | |
| 97 | 101 | def fetch(self) -> list[PropertyListing]: |
@@ -101,8 +105,12 @@ class _RealtyPress(BaseConnector): | ||
| 101 | 105 | by_id: dict[str, PropertyListing] = {} |
| 102 | 106 | base = self.site_url.rstrip("/") |
| 103 | 107 | dry = 0 |
| 108 | + pp = "?posts_per_page=100" if self.use_pp else "" | |
| 104 | 109 | for page in range(1, self.max_pages + 1): |
| 105 | − url = f"{base}/{self.archive}/page/{page}/?posts_per_page=100" | |
| 110 | + # page 1 : archive nue — sur certains sites (denisedunnrealtor…) | |
| 111 | + # /page/1/ répond 200 SANS cartes au lieu de rediriger | |
| 112 | + url = (f"{base}/{self.archive}/{pp}" if page == 1 | |
| 113 | + else f"{base}/{self.archive}/page/{page}/{pp}") | |
| 106 | 114 | try: |
| 107 | 115 | body = self.get(url).text |
| 108 | 116 | except Exception: |
@@ -160,7 +168,8 @@ class _RealtyPress(BaseConnector): | ||
| 160 | 168 | return |
| 161 | 169 | for prov in _PROVINCES: |
| 162 | 170 | if prov.lower() in raw.lower(): |
| 163 | − lst.region = prov | |
| 171 | + # forme canonique unique (« & » → « and ») | |
| 172 | + lst.region = prov.replace(" & ", " and ") | |
| 164 | 173 | raw = re.sub(r",?\s*" + re.escape(prov) + r"\b.*$", "", |
| 165 | 174 | raw, flags=re.I) |
| 166 | 175 | break |
@@ -316,6 +325,7 @@ for _ag in _load(): | ||
| 316 | 325 | "agency_name": _ag.get("name", _sid), |
| 317 | 326 | "province": _ag.get("province", "Ontario"), |
| 318 | 327 | "archive": _ag.get("archive", "listing"), |
| 328 | + "use_pp": int(_ag.get("page_size", 100)) >= 100, | |
| 319 | 329 | "max_pages": int(_ag.get("max_pages", 150)), |
| 320 | 330 | }, |
| 321 | 331 | ) |
| 322 | 332 | |