fix(realtypress): page 1 nue + pagination native profonde + région canonique
- /listing/page/1/ répond 200 SANS cartes sur certains sites (denisedunn, raymondanthony) → la page 1 requête l archive nue. - hanlonrealty plafonne l offset posts_per_page (~3 500 items) mais pagine à fond en natif 10/page → option registre page_size:10 (max_pages 2400). - Terre-Neuve : « Newfoundland & Labrador » → forme canonique « and ».
2 changed files +12 −38
modified
data/canada_agencies.json
+0 −36
@@ -107,41 +107,5 @@ | |||||
| 107 | "site": "https://cbnorthbay.com", | 107 | "site": "https://cbnorthbay.com", | ||
| 108 | "note": "Recensement ON 2026-08-27 : ~307 fiches (North Bay). RealtyPress/DDF.", | 108 | "note": "Recensement ON 2026-08-27 : ~307 fiches (North Bay). RealtyPress/DDF.", | ||
| 109 | "province": "Ontario" | 109 | "province": "Ontario" | ||
| 110 | − }, | ||||
| 111 | − { | ||||
| 112 | − "id": "rp_ag_hanlonrealty", | ||||
| 113 | − "name": "hanlonrealty.ca (Newfoundland and Labrador)", | ||||
| 114 | − "site": "https://hanlonrealty.ca", | ||||
| 115 | − "archive": "listing", | ||||
| 116 | − "max_pages": 400, | ||||
| 117 | − "province": "Newfoundland and Labrador", | ||||
| 118 | − "note": "Recensement CANADA 2026-08-27 : ~≥22000 fiches, provinces page 1 : {}. RealtyPress/DDF." | ||||
| 119 | − }, | ||||
| 120 | − { | ||||
| 121 | − "id": "rp_ag_denisedunnrealtor", | ||||
| 122 | − "name": "denisedunnrealtor.com (British Columbia)", | ||||
| 123 | − "site": "https://denisedunnrealtor.com", | ||||
| 124 | − "archive": "listing", | ||||
| 125 | − "max_pages": 400, | ||||
| 126 | − "province": "British Columbia", | ||||
| 127 | − "note": "Recensement CANADA 2026-08-27 : ~≥2400 fiches, provinces page 1 : {'British Columbia': 24}. RealtyPress/DDF." | ||||
| 128 | − }, | ||||
| 129 | − { | ||||
| 130 | − "id": "rp_ag_reddoorrealty", | ||||
| 131 | − "name": "reddoorrealty.ca (Nova Scotia)", | ||||
| 132 | − "site": "https://reddoorrealty.ca", | ||||
| 133 | − "archive": "listing", | ||||
| 134 | − "max_pages": 10, | ||||
| 135 | − "province": "Nova Scotia", | ||||
| 136 | − "note": "Recensement CANADA 2026-08-27 : ~≥580 fiches, provinces page 1 : {'Nova Scotia': 202}. RealtyPress/DDF." | ||||
| 137 | − }, | ||||
| 138 | − { | ||||
| 139 | − "id": "rp_ag_raymondanthony", | ||||
| 140 | − "name": "raymondanthony.com (Alberta)", | ||||
| 141 | − "site": "https://raymondanthony.com", | ||||
| 142 | − "archive": "listing", | ||||
| 143 | − "max_pages": 10, | ||||
| 144 | − "province": "Alberta", | ||||
| 145 | − "note": "Recensement CANADA 2026-08-27 : ~≥60 fiches, provinces page 1 : {'Alberta': 25}. RealtyPress/DDF." | ||||
| 146 | } | 110 | } | ||
| 147 | ] | 111 | ] | ||
| \ No newline at end of file | |||||
modified
immoka/connectors/realtypress.py
+12 −2
@@ -92,6 +92,10 @@ class _RealtyPress(BaseConnector): | |||||
| 92 | archive = "listing" # chemin de l'archive (revelrealty: "listings", | 92 | archive = "listing" # chemin de l'archive (revelrealty: "listings", | ||
| 93 | # codygroup: "all-regional-listings") | 93 | # codygroup: "all-regional-listings") | ||
| 94 | max_pages = 150 # 100 cartes/page → jusqu'à 15 000 fiches par site | 94 | max_pages = 150 # 100 cartes/page → jusqu'à 15 000 fiches par site | ||
| 95 | + use_pp = True # False (registre "page_size": 10) : pagination | ||||
| 96 | + # NATIVE 10/page — pour les sites qui plafonnent | ||||
| 97 | + # l'offset de posts_per_page (ex. hanlonrealty | ||||
| 98 | + # ~3 500 items) mais paginent à fond en natif | ||||
| 95 | request_delay = 0.6 | 99 | request_delay = 0.6 | ||
| 96 | 100 | ||||
| 97 | def fetch(self) -> list[PropertyListing]: | 101 | def fetch(self) -> list[PropertyListing]: | ||
@@ -101,8 +105,12 @@ class _RealtyPress(BaseConnector): | |||||
| 101 | by_id: dict[str, PropertyListing] = {} | 105 | by_id: dict[str, PropertyListing] = {} | ||
| 102 | base = self.site_url.rstrip("/") | 106 | base = self.site_url.rstrip("/") | ||
| 103 | dry = 0 | 107 | dry = 0 | ||
| 108 | + pp = "?posts_per_page=100" if self.use_pp else "" | ||||
| 104 | for page in range(1, self.max_pages + 1): | 109 | for page in range(1, self.max_pages + 1): | ||
| 105 | − url = f"{base}/{self.archive}/page/{page}/?posts_per_page=100" | 110 | + # page 1 : archive nue — sur certains sites (denisedunnrealtor…) | ||
| 111 | + # /page/1/ répond 200 SANS cartes au lieu de rediriger | ||||
| 112 | + url = (f"{base}/{self.archive}/{pp}" if page == 1 | ||||
| 113 | + else f"{base}/{self.archive}/page/{page}/{pp}") | ||||
| 106 | try: | 114 | try: | ||
| 107 | body = self.get(url).text | 115 | body = self.get(url).text | ||
| 108 | except Exception: | 116 | except Exception: | ||
@@ -160,7 +168,8 @@ class _RealtyPress(BaseConnector): | |||||
| 160 | return | 168 | return | ||
| 161 | for prov in _PROVINCES: | 169 | for prov in _PROVINCES: | ||
| 162 | if prov.lower() in raw.lower(): | 170 | if prov.lower() in raw.lower(): | ||
| 163 | − lst.region = prov | 171 | + # forme canonique unique (« & » → « and ») | ||
| 172 | + lst.region = prov.replace(" & ", " and ") | ||||
| 164 | raw = re.sub(r",?\s*" + re.escape(prov) + r"\b.*$", "", | 173 | raw = re.sub(r",?\s*" + re.escape(prov) + r"\b.*$", "", | ||
| 165 | raw, flags=re.I) | 174 | raw, flags=re.I) | ||
| 166 | break | 175 | break | ||
@@ -316,6 +325,7 @@ for _ag in _load(): | |||||
| 316 | "agency_name": _ag.get("name", _sid), | 325 | "agency_name": _ag.get("name", _sid), | ||
| 317 | "province": _ag.get("province", "Ontario"), | 326 | "province": _ag.get("province", "Ontario"), | ||
| 318 | "archive": _ag.get("archive", "listing"), | 327 | "archive": _ag.get("archive", "listing"), | ||
| 328 | + "use_pp": int(_ag.get("page_size", 100)) >= 100, | ||||
| 319 | "max_pages": int(_ag.get("max_pages", 150)), | 329 | "max_pages": int(_ag.get("max_pages", 150)), | ||
| 320 | }, | 330 | }, | ||
| 321 | ) | 331 | ) | ||
| 322 | 332 | ||||