[ka2] fix connecteur duquesimms: SiteGround sert un challenge sgcaptcha en HTTP 202 (aucune exception levée → sitemap parsé vide, 0 fiche) — is_blocked reconnaît sgcaptcha/202 et le connecteur passe par get_resilient (sitemap + détails), escalade Oxylabs OK (0 → 25 fiches, médiane 25)
2 changed files +10 −4
modified
immoka/connectors/_resilient.py
+5 −2
@@ -60,6 +60,9 @@ _CHALLENGE_MARKERS = ( | ||
| 60 | 60 | "px-captcha", "perimeterx", "incapsula", "_incapsula_", "datadome", |
| 61 | 61 | "captcha-delivery", "please enable javascript and cookies", |
| 62 | 62 | "checking your browser", "ddos protection by", |
| 63 | + # SiteGround : challenge JS servi en HTTP 202 (meta-refresh vers | |
| 64 | + # /.well-known/sgcaptcha/ puis page « Robot Challenge Screen ») | |
| 65 | + "sgcaptcha", "robot challenge screen", | |
| 63 | 66 | ) |
| 64 | 67 | |
| 65 | 68 | _UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " |
@@ -169,8 +172,8 @@ def is_blocked(resp) -> bool: | ||
| 169 | 172 | code = getattr(resp, "status_code", 0) or 0 |
| 170 | 173 | if code in BLOCK_STATUS: |
| 171 | 174 | return True |
| 172 | − # 200 mais page-challenge servie | |
| 173 | − if code == 200: | |
| 175 | + # 200 (ou 202 SiteGround) mais page-challenge servie | |
| 176 | + if code in (200, 202): | |
| 174 | 177 | try: |
| 175 | 178 | body = (resp.text or "")[:4000].lower() |
| 176 | 179 | except Exception: # noqa: BLE001 |
modified
immoka/connectors/duquesimms.py
+5 −2
@@ -54,7 +54,9 @@ class DuqueSimmsConnector(BaseConnector): | ||
| 54 | 54 | request_delay = 0.4 |
| 55 | 55 | |
| 56 | 56 | def fetch(self) -> list[PropertyListing]: |
| 57 | − xml = self.get(SITEMAP).text | |
| 57 | + # SiteGround sert un challenge sgcaptcha en HTTP 202 (sans exception) : | |
| 58 | + # get_resilient escalade (Oxylabs → Scrapfly ASP → Bright Data) | |
| 59 | + xml = self.get_resilient(SITEMAP).text | |
| 58 | 60 | by_id: dict[str, PropertyListing] = {} |
| 59 | 61 | for url, mls in _LOC_RE.findall(xml): |
| 60 | 62 | by_id.setdefault(mls, PropertyListing( |
@@ -63,7 +65,8 @@ class DuqueSimmsConnector(BaseConnector): | ||
| 63 | 65 | title="", mls=mls, agency=AGENCY, broker_name=AGENCY, |
| 64 | 66 | )) |
| 65 | 67 | listings = list(by_id.values()) |
| 66 | − du.enrich(self, listings, DETAIL_LIMIT, _parse_detail, key="v1") | |
| 68 | + du.enrich(self, listings, DETAIL_LIMIT, _parse_detail, key="v1", | |
| 69 | + fetch_html=lambda u: self.get_resilient(u).text) | |
| 67 | 70 | out = [] |
| 68 | 71 | for lst in listings: |
| 69 | 72 | if not lst.price or lst.price < MIN_SALE_PRICE: |
| 70 | 73 | |