spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/centurion.py : connecteur Centurion / CP Living (cpliving.com)5# REIT national (Centurion Property Associates) — région de Québec : le6# Quartier QB, 2551 chemin des Quatre-Bourgeois (Sainte-Foy), 684 unités.7# Le site est propulsé par Rentsync : la page de recherche référence un8# proxy legacy `widgets/search/api_proxy.php?client_id=21&city_id=2310`9# (404 aujourd'hui) mais son JS (scripts/main.js) appelle en réalité le10# flux JSON officiel `https://api.theliftsystem.com/v2/search` avec un11# auth_token embarqué. On lit client_id/city_id sur la page, le token dans12# main.js (repli : constantes), puis on interroge le flux directement —13# aucune exécution JavaScript nécessaire, Cloudflare accepte le UA de base.14# Filtre région : city_ids 2310 (Ville de Québec) + 1039 (Gatineau, ajouté15# à l'expansion provinciale 2026-08) + garde-fou province/ville.16# La fiche propriété (rendu serveur) fournit la galerie photo, via le cache17# BD self.detail() (revisitée seulement quand le flux change).18# -----------------------------------------------------------------------------19from __future__ import annotations2021import hashlib22import re2324from bs4 import BeautifulSoup2526from ..schema import Listing, infer_city, normalize_unit_type, strip_accents27from .base import BaseConnector2829BASE = "https://www.cpliving.com"30SEARCH_PAGE = f"{BASE}/apartments-for-rent/quebec-city"31LIFT_API = "https://api.theliftsystem.com/v2/search"3233# Valeurs observées sur la page/main.js — repli si l'extraction dynamique casse34DEFAULT_CLIENT_ID = "21"35DEFAULT_CITY_ID = "2310" # Ville de Québec dans la base Lift/Rentsync36GATINEAU_CITY_ID = "1039" # Gatineau (expansion provinciale 2026-08)37DEFAULT_AUTH_TOKEN = "sswpREkUtyeYjeoahA2i"3839# Garde-fou région : villes couvertes — agglomération Québec/Lévis + Gatineau40# (clé sans accents, suffixe « QC » du flux retiré par _city_key)41_QC_CITIES = {42 "quebec", "ville de quebec", "quebec city", "sainte-foy", "ste-foy",43 "sillery", "beauport", "charlesbourg", "cap-rouge", "levis",44 "l-ancienne-lorette", "saint-augustin-de-desmaures",45 "gatineau", "hull", "aylmer",46}4748# jeton d'authentification dans main.js : `s="&client_id=21",e="&auth_token=…"`49_TOKEN_RE = r'client_id={cid}",\w+="&auth_token=([A-Za-z0-9]+)"'50_MAINJS_RE = re.compile(r'src="(/scripts/main\.js[^"]*)"')51# galerie de la fiche propriété (img + backgrounds CSS)52_IMG_RE = re.compile(53 r"https://assets\.rentsync\.com/[^\"'\\)\s]+\.(?:jpg|jpeg|png|webp)", re.I)54_SKIP_IMG = re.compile(r"logo|icon|favicon|badge|/thumb", re.I)555657def _city_key(city: str) -> str:58 key = strip_accents((city or "").strip().lower())59 return re.sub(r"\s+qc$", "", key) # le flux accole parfois la province606162class CenturionConnector(BaseConnector):63 source_id = "centurion"64 request_delay = 0.765 max_properties = 30 # garde-fou (1 propriété à Québec aujourd'hui)66 max_images = 206768 # -- paramètres du flux (page + main.js, avec replis) ----------------------69 def _feed_params(self) -> tuple[str, str, str]:70 """(client_id, city_id, auth_token) lus sur le site, replis constants."""71 client_id, city_id, token = (DEFAULT_CLIENT_ID, DEFAULT_CITY_ID,72 DEFAULT_AUTH_TOKEN)73 try:74 page = self.get(SEARCH_PAGE).text75 soup = BeautifulSoup(page, "html.parser")76 data = soup.find("div", class_="search-data")77 if data:78 client_id = (data.get("data-client-id") or client_id).strip()79 city_id = (data.get("data-city-id") or city_id).strip()80 m = _MAINJS_RE.search(page)81 if m:82 js = self.get(BASE + m.group(1)).text83 mt = re.search(_TOKEN_RE.format(cid=re.escape(client_id)), js)84 if mt:85 token = mt.group(1)86 except Exception:87 pass # replis : les constantes observées88 return client_id, city_id, token8990 def fetch(self) -> list[Listing]:91 client_id, city_id, token = self._feed_params()92 props = self.get(LIFT_API, params={93 "client_id": client_id,94 "auth_token": token,95 "city_ids": f"{city_id},{GATINEAU_CITY_ID}",96 "show_all_properties": "true",97 "show_custom_fields": "true",98 "show_amenities": "true",99 "show_promotions": "true",100 "limit": "1000",101 }, headers={"Accept": "application/json",102 "Referer": BASE + "/"}).json()103104 listings: list[Listing] = []105 for p in props[: self.max_properties]:106 try:107 addr = p.get("address") or {}108 if (addr.get("province_code") or "").upper() != "QC":109 continue # REIT national : rester dans la province110 if _city_key(addr.get("city", "")) not in _QC_CITIES:111 continue # garde-fou région Québec/Lévis112 listings.append(self._listing(p))113 except Exception:114 continue115 return listings116117 # -- une annonce par propriété ---------------------------------------------118 def _listing(self, p: dict) -> Listing:119 pid = str(p.get("id"))120 addr = p.get("address") or {}121 url = p.get("permalink") or SEARCH_PAGE122 name = (p.get("name") or "").strip()123124 # adresse complète : rue + ville + code postal (tous fournis au flux)125 ck = _city_key(addr.get("city", ""))126 if ck in ("quebec", "ville de quebec", "quebec city"):127 city = "Québec"128 else: # retirer le suffixe province du flux (« Gatineau QC »)129 city = re.sub(r"\s+QC$", "", (addr.get("city") or "").strip(),130 flags=re.I)131 street = (addr.get("address") or "").strip()132 postal = (addr.get("postal_code") or "").strip()133 full_addr = ", ".join(x for x in (street, city) if x)134 if postal:135 full_addr += f", QC {postal}"136 sector = (addr.get("neighbourhood") or "").strip()137138 # coordonnées GPS structurées du flux139 geo = p.get("geocode") or {}140 try:141 lat = float(geo["latitude"]) if geo.get("latitude") else None142 lng = float(geo["longitude"]) if geo.get("longitude") else None143 except (TypeError, ValueError):144 lat = lng = None145146 # sommaire des unités disponibles (rempli seulement s'il y a vacance)147 stats = ((p.get("statistics") or {}).get("suites") or {})148 rates = stats.get("rates") or {}149 beds = stats.get("bedrooms") or {}150 baths = stats.get("bathrooms") or {}151 sqft = stats.get("square_feet") or {}152 price = float(rates["min"]) if rates.get("min") else None153 price_label = ""154 if price is not None:155 price_label = (f"À partir de {price:.0f} $"156 if rates.get("max") and rates["max"] != rates["min"]157 else f"{price:.0f} $ /mois")158 # type d'unité : seulement si la gamme est sans ambiguïté159 unit_type = ""160 if beds.get("min") is not None and beds.get("min") == beds.get("max"):161 n = int(beds["min"])162 unit_type = "Studio" if n == 0 else normalize_unit_type(163 f"{n} chambres")164 # superficie : le flux publie parfois « 0.0 » (Gatineau) — ignorer165 area = None166 try:167 v = float(sqft.get("min") or 0)168 if 80 <= v <= 20000:169 area = v170 except (TypeError, ValueError):171 pass172173 # disponibilité : libellé du flux (« No Vacancy », « X Vacancies »…)174 availability = (p.get("availability_status_label") or "").strip()175 avail_date = None176 mad = str(p.get("min_availability_date") or "").strip()177 if re.fullmatch(r"20\d{2}-\d{2}-\d{2}", mad[:10]):178 avail_date = mad[:10]179180 # description : aperçu HTML du flux (rendu texte)181 details_src = p.get("details") or {}182 desc = BeautifulSoup(details_src.get("overview") or "",183 "html.parser").get_text(" ", strip=True)184 promo = p.get("promotion") or {}185 promo_txt = (promo.get("title") or promo.get("name") or "").strip() \186 if isinstance(promo, dict) else ""187 if promo_txt:188 desc = f"Promotion : {promo_txt}. {desc}".strip()189190 # commodités : liste du flux + champ personnalisé Rentsync (CSV)191 amenities: list[str] = []192 for a in p.get("amenities") or []:193 t = (a.get("name") if isinstance(a, dict) else str(a) or "").strip()194 if t and t not in amenities:195 amenities.append(t)196 cf = p.get("custom_fields") or {}197 for t in (cf.get("amenities") or "").split(","):198 t = t.strip()199 if t and t not in amenities:200 amenities.append(t)201202 # champs structurés du flux203 details: dict = {}204 contact = p.get("contact") or {}205 if contact.get("phone"):206 details["contact"] = {"phone": contact["phone"]}207 if contact.get("email"):208 details.setdefault("contact", {})["email"] = contact["email"]209 # pet_friendly=false ne distingue pas « interdit » de « non renseigné »210 pets = "oui" if p.get("pet_friendly") is True else None211212 # galerie photo de la fiche propriété — via le cache BD : revisitée213 # seulement quand la ligne du flux change214 feed_key = hashlib.sha1("|".join(str(x) for x in (215 p.get("availability_count"), p.get("availability_status"),216 rates.get("min"), rates.get("max"), mad, p.get("photo"),217 )).encode("utf-8")).hexdigest()218 d = self.detail(pid, feed_key, lambda: self._fetch_gallery(url))219 images = list(d.get("images") or [])220 photo = (p.get("photo_path") or "").strip()221 if photo and photo not in images:222 images.insert(0, photo)223224 return Listing(225 source=self.source_id,226 external_id=pid,227 url=url,228 title=name,229 address=full_addr,230 sector=sector,231 city=infer_city(sector, default=city or "Québec"),232 unit_type=unit_type,233 price=price,234 price_label=price_label,235 availability=availability,236 availability_date=avail_date,237 area_sqft=area,238 pets=pets,239 description=desc[:600] + (240 f" Salles de bain : {baths['min']:g}+."241 if baths.get("min") else ""),242 amenities=amenities[:25],243 details=details,244 images=images[: self.max_images],245 lat=lat,246 lng=lng,247 )248249 def _fetch_gallery(self, url: str) -> dict:250 """Scrape la galerie photo (assets.rentsync.com) de la fiche propriété."""251 out: dict = {"images": []}252 try:253 page = self.get(url).text254 except Exception:255 return out256 images: list[str] = []257 for u in _IMG_RE.findall(page):258 if _SKIP_IMG.search(u):259 continue260 # variante pleine résolution de la galerie (…/gallery/full/…)261 u = re.sub(r"/gallery/\d{3,4}/", "/gallery/full/", u)262 if u not in images:263 images.append(u)264 out["images"] = images[: self.max_images]265 return out266