Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 98.9%
Python 0.6%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/nexliving.py : connecteur NexLiving Communities5# (residential.nexliving.ca) — REIT pancanadien ; côté Québec : Gatineau6# (Pointe-Gatineau, Place du Golf, Saint-James, Prado, Nelson-Morin,7# Progrès, Futaie, Lanctot, JGR), Thurso (Fraser) et Val-d'Or8# (Roland-Audet). Site Yardi RentCafe derrière Cloudflare (403 direct) :9# tout passe par Scrapfly (ASP), SANS rendu JS — RentCafe embarque un bloc10# SEO rendu serveur (#list-view-cards-hidden, li.property-box-hidden) avec11# nom, adresse complète, lits/sdb/pi², prix « à partir de », téléphone et12# vignette. Une annonce par propriété (uid = slug RentCafe stable).13# Pages parcourues : /qc/gatineau/apartments, /qc/thurso/apartments,14# /qc/val-d'or/apartments — SEULES les fiches /apartments/qc/ passent15# (les parcs ON et NB du REIT sont exclus), avec la ville réelle par16# annonce. La fiche propriété (galerie, description) passe par17# self.detail() (cache BD) avec budget Scrapfly par synchronisation.18# -----------------------------------------------------------------------------19from __future__ import annotations2021import hashlib22import re2324from bs4 import BeautifulSoup2526from ..schema import Listing, parse_price27from .base import BaseConnector2829BASE = "https://residential.nexliving.ca"30# pages liste par ville québécoise (l'apostrophe de val-d'or est bien dans31# l'URL du site)32SEARCH_PATHS = ["/qc/gatineau/apartments", "/qc/thurso/apartments",33 "/qc/val-d'or/apartments"]3435# segment ville de /apartments/qc/<ville>/<slug> -> ville affichée36_QC_CITIES = {37 "gatineau": "Gatineau",38 "thurso": "Thurso",39 "val-d-or": "Val-d'Or",40}41_PROP_RE = re.compile(r"/apartments/qc/([a-z0-9\-.']+)/([a-z0-9\-]+)/")42_BED_TYPES = {"0": "Studio", "1": "3½", "2": "4½", "3": "5½", "4": "6½+"}43_PRICE_RE = re.compile(r"\$[\d,]+(?:\.\d{2})?(?:(?:\s*(?:-|to))+\s*"44 r"\$[\d,]+(?:\.\d{2})?)?")45_IMG_RE = re.compile(46 r"https://resource\.rentcafe\.com/image/upload/[^\"'\s\\]+?"47 r"\.(?:jpg|jpeg|png|webp)", re.I)48_SKIP_IMG = re.compile(r"logo|icon|favicon|placeholder|lockup", re.I)495051class _BudgetReached(Exception):52 """Plafond d'appels Scrapfly atteint pour cette synchronisation."""535455class NexLivingConnector(BaseConnector):56 source_id = "nexliving"57 request_delay = 1.058 max_images = 2059 max_details = 12 # budget de fiches propriété Scrapfly (hors cache)6061 # -- page derrière Cloudflare (bloc SEO rendu serveur, pas de JS) ----------62 def _page(self, url: str) -> str:63 html = self.get_scrapfly(url, render_js=False)64 if html and "property-box-hidden" in html:65 return html66 # seconde chance avec rendu JS complet (bloc SEO absent/incomplet)67 return self.get_scrapfly(url, rendering_wait=5000)6869 def fetch(self) -> list[Listing]:70 self._fetched = 071 listings: list[Listing] = []72 seen: set[str] = set()73 pages_ok = 074 for path in SEARCH_PATHS:75 try:76 html = self._page(BASE + path)77 soup = BeautifulSoup(html, "html.parser")78 cards = soup.select("#list-view-cards-hidden "79 "li.property-box-hidden")80 if not cards:81 continue82 pages_ok += 183 except Exception:84 continue85 for card in cards:86 try:87 lst = self._card_listing(card)88 except Exception:89 continue90 if lst and lst.external_id not in seen:91 seen.add(lst.external_id)92 listings.append(lst)93 if pages_ok == 0:94 # échec transitoire -> journalisé comme ÉCHEC par ingest (ok=0),95 # pas comme un passage « ok » à 0 annonce96 raise RuntimeError(97 "aucune page liste NexLiving accessible via Scrapfly")98 return listings99100 # -- une annonce par propriété -----------------------------------------------101 def _card_listing(self, card) -> Listing | None:102 a = card.select_one(".property-name a[href]")103 if not a:104 return None105 url = (a.get("href") or "").split("?")[0].replace("http://", "https://")106 m = _PROP_RE.search(url)107 if not m:108 return None # fiche hors Québec (ON/NB) : exclue109 city = _QC_CITIES.get(m.group(1).replace("'", "-"))110 slug = m.group(2)111 addr_el = card.select_one(".card-prop-address")112 address = addr_el.get_text(" ", strip=True) if addr_el else ""113 if not city:114 # ville QC inattendue : la lire dans l'adresse (« …, Ville, QC … »)115 am = re.search(r",\s*([^,]+),\s*QC\b", address)116 if not am:117 return None # jamais de ville inventée118 city = am.group(1).strip()119120 name = re.sub(r"\s*opens in a new tab\s*", "",121 a.get_text(" ", strip=True)).strip()122 name = name or slug.replace("-", " ").title()123124 # lits/sdb/pi² (le type d'unité seulement si la gamme est sans125 # ambiguïté : « 3.0Beds - 3.0Beds »)126 beds = baths = sqft = ""127 for li in card.select(".card-bed-bath-rent li"):128 it = li.get_text(" ", strip=True)129 if "Bed" in it:130 beds = it131 elif "Bath" in it:132 baths = it133 elif "Sq" in it:134 sqft = re.sub(r"\s*to\s*-\s*", " - ", it)135 unit_type = ""136 bm = re.match(r"^(?:(\d)\.0Beds\s*-\s*(\d)\.0Beds|Studio\s*-\s*Studio)$",137 beds or "")138 if bm:139 if bm.group(1) is None:140 unit_type = "Studio"141 elif bm.group(1) == bm.group(2):142 unit_type = _BED_TYPES.get(bm.group(1), "")143144 # prix « $999.00 to - $1,899.00 » ou « Call for Details »145 price = None146 price_label = ""147 pm = _PRICE_RE.search(card.get_text(" ", strip=True))148 if pm:149 price_label = re.sub(r"\s*to\s*-\s*", " - ", pm.group(0))150 price = parse_price(price_label)151 if "-" in price_label:152 price_label = "À partir de " + price_label153154 # téléphone du bureau de location (lien tel: structuré)155 phone = ""156 tel = card.select_one("a[href^='tel:']")157 if tel:158 tm = re.search(r"\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})",159 tel.get("href", ""))160 if tm:161 phone = f"{tm.group(1)}-{tm.group(2)}-{tm.group(3)}"162163 images: list[str] = []164 img = card.select_one("img[src*='rentcafe']")165 if img and img.get("src"):166 images.append(img["src"])167168 # fiche propriété (galerie + description) via cache BD : Scrapfly169 # n'est rappelé que si la carte liste a changé170 key = hashlib.sha1(f"{name}|{address}|{beds}|{baths}|{sqft}|"171 f"{price_label}".encode("utf-8")).hexdigest()172 try:173 payload = self.detail(slug, key, lambda: self._fetch_detail(url))174 except Exception:175 payload = {}176 for im in (payload.get("images") or []):177 if im not in images:178 images.append(im)179180 prop_type = ""181 pt = card.select_one(".unit-types")182 if pt:183 prop_type = pt.get_text(" ", strip=True)184185 desc_parts = ([payload["description"]]186 if payload.get("description") else [])187 desc_parts += [b for b in (prop_type, beds, baths, sqft) if b]188189 details: dict = {}190 if phone:191 details["contact"] = {"phone": phone}192193 return Listing(194 source=self.source_id,195 external_id=slug,196 url=url,197 title=name,198 address=address,199 sector="",200 city=city,201 unit_type=unit_type,202 price=price,203 price_label=price_label,204 description=" — ".join(desc_parts)[:800],205 details=details,206 images=images[: self.max_images],207 )208209 # -- fiche propriété (Scrapfly sans JS, budget par sync) ---------------------210 def _fetch_detail(self, url: str) -> dict:211 if self._fetched >= self.max_details:212 raise _BudgetReached()213 self._fetched += 1214 payload: dict = {"description": "", "images": []}215 html = self.get_scrapfly(url, render_js=False)216 if not html:217 return payload218 soup = BeautifulSoup(html, "html.parser")219 for u in dict.fromkeys(_IMG_RE.findall(html)):220 if not _SKIP_IMG.search(u) and u not in payload["images"]:221 payload["images"].append(u)222 payload["images"] = payload["images"][: self.max_images]223 paras = [p.get_text(" ", strip=True) for p in soup.find_all("p")]224 paras = [p for p in paras if len(p) > 80]225 if paras:226 payload["description"] = " ".join(paras[:2])[:600]227 return payload228