Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 98.9%
Python 0.6%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/fb_marketplace.py : Facebook Marketplace — catégorie location5# (propertyrentals), TOUTE la province. Annonces de particuliers, fort6# complément aux gestionnaires. Depuis 2026-08-22 le scraping est délégué à7# l'ACTEUR APIFY MAISON gorgeous_thistle/ka-fb-marketplace (source :8# actors/ka-fb-marketplace de ce repo) : HTML public déconnecté SANS rendu9# JS (les JSON de recherche, la fiche PDP et la galerie MediaViewer10# préchargée vivent dans le HTML brut) via proxy résidentiel CA — ~100×11# moins cher que l'ancienne chaîne Scrapfly ASP+render_js, et couverture12# élargie de 10 à 33 villes (IDs Marketplace des villes régionales validés ;13# les slugs inventés retombent sur des villes aléatoires, ne pas en ajouter14# sans les valider).15# Le scroll infini reste bloqué hors connexion (~25 annonces/URL) : on16# pagine par TRANCHES DE PRIX et on accumule sur plusieurs synchronisations.17# Le flux public TOURNE : une phase de RATTRAPAGE (extraDetailIds) visite18# les fiches actives jamais enrichies (budget LOUKA_FBMP_BACKFILL).19# Le connecteur garde le cache détail BD (clé v2, payloads compatibles) et20# passe à l'acteur la liste des fiches fraîches à NE PAS revisiter.21# ⚠️ Données personnelles (Loi 25) : ne jamais republier nom/téléphone du22# vendeur, garder le lien sortant.23# -----------------------------------------------------------------------------24from __future__ import annotations2526import json27import os28import re29import time3031import requests as _requests3233from ..schema import Listing, normalize_unit_type34from .base import BaseConnector35from . import _detailutil as du3637BASE = "https://www.facebook.com/marketplace"38APIFY_API = "https://api.apify.com/v2"39ACTOR = os.environ.get("LOUKA_FBMP_ACTOR", "gorgeous_thistle~ka-fb-marketplace")4041# villes québécoises — gros parcs : slug vanity Marketplace -> nom d'affichage42CITIES = {43 "montreal": "Montréal", "laval": "Laval", "longueuil": "Longueuil",44 "quebec": "Québec", "gatineau": "Gatineau", "sherbrooke": "Sherbrooke",45 "trois-rivieres": "Trois-Rivières", "levis": "Lévis",46 "saguenay": "Saguenay", "drummondville": "Drummondville",47}48# villes régionales : ID numérique Marketplace -> nom (découverts via Serper,49# lat/lng validés live 2026-08-22 — rayon ~65 km chacun)50REGIONS = {51 "105790762794728": "Rimouski", "110925022261358": "Rivière-du-Loup",52 "106528256049537": "Matane", "109502305742889": "Gaspé",53 "100246746683723": "Sept-Îles", "105615529471535": "Baie-Comeau",54 "106016152771864": "Rouyn-Noranda", "110594102301488": "Val-d'Or",55 "105598479473872": "Amos", "109699385723610": "Saint-Georges",56 "110304502331480": "Thetford Mines", "111938355489908": "Granby",57 "108126699220609": "Victoriaville", "109753249050671": "Joliette",58 "109429165750459": "Saint-Hyacinthe",59 "108645709159963": "Saint-Jean-sur-Richelieu",60 "112946182053396": "Saint-Jérôme",61 "113011352052118": "Salaberry-de-Valleyfield",62 "107992819229760": "Magog", "109514615734434": "Alma",63 "109664672393669": "Mont-Laurier", "108557812502323": "La Tuque",64 "108296932538245": "Dolbeau-Mistassini",65}66# tranches de prix (bornes en $) : le flux déconnecté sert un sous-ensemble67# tournant, chaque tranche renvoie un lot quasi disjoint68PRICE_BANDS = [(0, 800), (800, 1100), (1100, 1400), (1400, 1700),69 (1700, 2100), (2100, 2800), (2800, 6000)]70REGIONAL_BANDS = [(0, 1200), (1200, 6000)] # parcs plus petits : 2 tranches7172# garde-fous de prix mensuel (rejette « $90 » = /nuit, et les valeurs à vendre)73PRICE_MIN, PRICE_MAX = 300, 120007475DETAIL_LIMIT = int(os.environ.get("LOUKA_FBMP_DETAIL_LIMIT", "120"))76# rattrapage : fiches ACTIVES en BD encore pauvres (sans description, sans77# GPS ou avec ≤ 1 image) qui ne repassent plus dans la recherche publique78BACKFILL_LIMIT = int(os.environ.get("LOUKA_FBMP_BACKFILL", "80"))79# les annonces FB changent peu après publication : TTL long = le budget détail80# sert surtout aux NOUVELLES annonces plutôt qu'à re-visiter les connues81TTL_DAYS = float(os.environ.get("LOUKA_FBMP_TTL_DAYS", "30"))82CITY_LIMIT = os.environ.get("LOUKA_FBMP_CITIES", "") # ex. "montreal,laval"83CONCURRENCY = int(os.environ.get("LOUKA_FBMP_CONCURRENCY", "8"))84RUN_TIMEOUT = int(os.environ.get("LOUKA_FBMP_RUN_TIMEOUT", "2400")) # s8586# clé du cache détail — v2 : payloads identiques à l'ère Scrapfly (galerie,87# ville/province, statut) ; l'acteur produit la même forme, cache réutilisé88DETAIL_KEY = "v2"8990# provinces acceptées : le flux public glisse parfois des annonces hors Québec91# (ex. « Norfolk, ON ») dans les résultats d'une ville québécoise92_QC_STATES = {"qc", "quebec", "québec"}939495def _unit_type(title: str, desc: str = "") -> str:96 blob = f"{title} {desc}"97 ut = normalize_unit_type(title)98 if re.match(r"^\d½$|^6½\+$|^Studio$|^Loft$", ut or ""):99 return ut100 m = re.search(r"(\d+)\s*(?:bed|bedroom|chambre|cc|br)\b", blob, re.I)101 if m:102 n = int(m.group(1)) + 2103 return "6½+" if n >= 6 else f"{n}½"104 if re.search(r"\bstudio|bachelor\b", blob, re.I):105 return "Studio"106 if re.search(r"\b(?:private\s+)?room\b|chambre", blob, re.I):107 return "Chambre"108 return ""109110111def _usable(payload: dict | None) -> bool:112 """Payload détail exploitable (ni vide, ni marqueur « fiche sans objet »)."""113 return bool(payload) and not payload.get("nopdp")114115116def _non_qc(payload: dict) -> bool:117 state = (payload.get("state") or "").strip().lower()118 if state:119 return state not in _QC_STATES120 # sans état : filet géographique — certains PDP ne livrent que le GPS121 lat, lng = payload.get("lat"), payload.get("lng")122 if lat is not None and lng is not None:123 return not (44.9 <= lat <= 62.8 and -79.6 <= lng <= -56.9)124 return False125126127# version des payloads détail : bump quand parse_detail (acteur) apprend de128# nouveaux champs, pour re-visiter progressivement les fiches déjà en cache129PAYLOAD_VERSION = 3130131_TYPE_LABELS = {"apartment", "house", "townhouse", "condo", "room", "flat",132 "appartement", "maison", "maison de ville", "chambre", "loft"}133_BEDS_RX = re.compile(r"(\d+(?:[.,]\d+)?)\s*(?:beds?\b|chambres?\b|lits?\b)",134 re.I)135_BATHS_RX = re.compile(r"(\d+(?:[.,]\d+)?)\s*(?:baths?\b|salles?\b)", re.I)136137138def _std_detail(d: dict) -> dict:139 """Payload acteur -> dict pour du.apply_detail (labels PDP en/fr)."""140 std = {k: d[k] for k in ("description", "images", "lat", "lng",141 "city", "address") if d.get(k)}142 amen: list[str] = []143 for lbl in d.get("unit_fields") or []:144 mb = _BEDS_RX.search(lbl)145 if mb: # « 2 beds · 1 bath »146 std["bedrooms"] = float(mb.group(1).replace(",", "."))147 ms = _BATHS_RX.search(lbl)148 if ms:149 std["bathrooms"] = float(ms.group(1).replace(",", "."))150 continue151 low = lbl.casefold()152 if low in _TYPE_LABELS:153 continue # type générique : _unit_type (titre/desc) fait mieux154 if "furnish" in low or "meublé" in low:155 std["furnished"] = not ("unfurnish" in low or "non meublé" in low)156 if "pet" in low or "animaux" in low:157 std["pets"] = ("non" if "no pet" in low or "pas d" in low158 else "oui")159 amen.append(lbl)160 if amen:161 std["amenities"] = amen162 for part in (d.get("listed_text") or "").split("·"):163 p = part.strip()164 if p.casefold().startswith(("available", "disponible")):165 std["availability"] = p166 extras = {k: d[k] for k in ("walk_score", "transit_score", "bike_score")167 if d.get(k) is not None}168 if d.get("virtual_tour_url"):169 extras["virtual_tour"] = d["virtual_tour_url"]170 if extras:171 std["details"] = extras172 return std173174175class FacebookMarketplaceConnector(BaseConnector):176 source_id = "fb_marketplace"177 request_delay = 1.0178179 # -- orchestration de l'acteur Apify --------------------------------------180 def _search_urls(self) -> tuple[list[str], dict[str, str]]:181 """URLs de recherche (ville × tranche) + mapping URL -> nom de ville."""182 cities, regions = dict(CITIES), dict(REGIONS)183 if CITY_LIMIT:184 wanted = {c.strip().casefold() for c in CITY_LIMIT.split(",")}185 cities = {k: v for k, v in cities.items()186 if k in wanted or v.casefold() in wanted}187 regions = {k: v for k, v in regions.items()188 if v.casefold() in wanted}189 urls: list[str] = []190 by_url: dict[str, str] = {}191 for slug, city in cities.items():192 for lo, hi in PRICE_BANDS:193 u = (f"{BASE}/{slug}/propertyrentals"194 f"?minPrice={lo}&maxPrice={hi}&sortBy=creation_time_descend")195 urls.append(u)196 by_url[u] = city197 for cid, city in regions.items():198 for lo, hi in REGIONAL_BANDS:199 u = (f"{BASE}/{cid}/propertyrentals"200 f"?minPrice={lo}&maxPrice={hi}&sortBy=creation_time_descend")201 urls.append(u)202 by_url[u] = city203 return urls, by_url204205 def _run_actor(self, payload: dict, token: str) -> list[dict]:206 """Lance l'acteur, attend la fin, retourne les items du dataset."""207 r = _requests.post(208 f"{APIFY_API}/acts/{ACTOR}/runs?waitForFinish=120",209 json=payload, timeout=180,210 headers={"Authorization": f"Bearer {token}"})211 r.raise_for_status()212 run = r.json()["data"]213 deadline = time.time() + RUN_TIMEOUT214 while run["status"] in ("READY", "RUNNING") and time.time() < deadline:215 time.sleep(10)216 run = _requests.get(217 f"{APIFY_API}/actor-runs/{run['id']}", timeout=60,218 headers={"Authorization": f"Bearer {token}"}).json()["data"]219 if run["status"] != "SUCCEEDED":220 raise RuntimeError(f"acteur {ACTOR} : run {run['id']} "221 f"terminé en {run['status']}")222 items: list[dict] = []223 offset = 0224 while True:225 batch = _requests.get(226 f"{APIFY_API}/datasets/{run['defaultDatasetId']}/items"227 f"?limit=1000&offset={offset}", timeout=120,228 headers={"Authorization": f"Bearer {token}"}).json()229 items.extend(batch)230 if len(batch) < 1000:231 return items232 offset += 1000233234 # -- caches / BD -----------------------------------------------------------235 def _fresh_ids(self, cache: du.TtlDetailCache) -> list[str]:236 """IDs dont le payload détail en cache est encore frais (clé + TTL) :237 l'acteur ne les revisitera pas."""238 rows = cache.con.execute(239 "SELECT external_id FROM detail_cache"240 " WHERE source=? AND key=? AND fetched_at > ?"241 " AND (json_extract(payload,'$.pv') >= ?"242 " OR json_extract(payload,'$.nopdp') IS NOT NULL"243 " OR json_extract(payload,'$.gone') IS NOT NULL)",244 (self.source_id, DETAIL_KEY,245 time.time() - TTL_DAYS * 86400, PAYLOAD_VERSION)).fetchall()246 return [r["external_id"] for r in rows]247248 def _poor_active_rows(self, cache: du.TtlDetailCache) -> list:249 """Fiches ACTIVES en BD encore pauvres (candidates au rattrapage)."""250 rows = cache.con.execute(251 "SELECT external_id, url, title, address, sector, city, unit_type,"252 " bedrooms, bathrooms, price, price_label, availability,"253 " availability_date, area_sqft, pets, furnished, description,"254 " amenities, details, images, lat, lng"255 " FROM listings WHERE source=? AND active=1"256 " ORDER BY last_seen DESC", (self.source_id,)).fetchall()257258 def poor(r) -> bool:259 try:260 n_img = len(json.loads(r["images"] or "[]"))261 except ValueError:262 n_img = 0263 return (not (r["description"] or "").strip()264 or r["lat"] is None or n_img <= 1265 or not (r["address"] or "").strip()266 or r["bedrooms"] is None)267 return [r for r in rows if poor(r)]268269 def _listing_from_row(self, r) -> Listing:270 """Reconstruit le Listing depuis sa ligne BD (phase de rattrapage)."""271 def js(s, default):272 try:273 return json.loads(s) if s else default274 except ValueError:275 return default276 return Listing(277 source=self.source_id, external_id=r["external_id"],278 url=r["url"], title=r["title"] or "", address=r["address"] or "",279 sector=r["sector"] or "", city=r["city"] or "",280 unit_type=r["unit_type"] or "", bedrooms=r["bedrooms"],281 bathrooms=r["bathrooms"], price=r["price"],282 price_label=r["price_label"] or "",283 availability=r["availability"] or "",284 availability_date=r["availability_date"],285 area_sqft=r["area_sqft"], pets=r["pets"],286 furnished=(None if r["furnished"] is None else bool(r["furnished"])),287 description=r["description"] or "",288 amenities=js(r["amenities"], []), details=js(r["details"], {}),289 images=js(r["images"], []), lat=r["lat"], lng=r["lng"],290 )291292 def _backfill(self, cache: du.TtlDetailCache, poor_rows: list,293 done: set[str]) -> list[Listing]:294 """Rattrapage : ré-émet les fiches actives pauvres dont le cache295 (rempli par l'acteur via extraDetailIds) apporte du neuf. Les296 annonces mortes suivent le cycle normal miss_count → retrait."""297 def adds(r, s: dict) -> bool:298 if s.get("description") and len(s["description"]) > \299 len(r["description"] or ""):300 return True301 if s.get("lat") is not None and r["lat"] is None:302 return True303 try:304 n_img = len(json.loads(r["images"] or "[]"))305 except ValueError:306 n_img = 0307 if len(s.get("images") or []) > n_img:308 return True309 if s.get("address") and not (r["address"] or "").strip():310 return True311 if s.get("bedrooms") is not None and r["bedrooms"] is None:312 return True313 return bool(s.get("city")) and not (r["city"] or "")314315 out: list[Listing] = []316 for r in poor_rows:317 if r["external_id"] in done:318 continue319 d, _fresh = cache.peek(r["external_id"])320 if not _usable(d) or d.get("gone") or _non_qc(d):321 continue # morte / hors QC / rien de neuf322 std = _std_detail(d)323 if not adds(r, std):324 continue # rien à apporter : laisser vivre son cycle325 lst = self._listing_from_row(r)326 du.apply_detail(lst, std)327 if d.get("city"): # reverse geocode FB : autoritaire328 lst.city = d["city"]329 if not lst.unit_type:330 lst.unit_type = _unit_type(lst.title, lst.description)331 out.append(lst)332 return out333334 # -- pipeline principal ----------------------------------------------------335 def fetch(self) -> list[Listing]:336 token = os.environ.get("APIFY_TOKEN")337 if not token:338 raise RuntimeError("APIFY_TOKEN manquant (voir .env)")339 urls, city_by_url = self._search_urls()340341 cache = du.TtlDetailCache(self, budget=0, ttl_days=TTL_DAYS,342 key=DETAIL_KEY, fetch_html=lambda _u: "")343 try:344 fresh = set(self._fresh_ids(cache))345 poor_rows = self._poor_active_rows(cache)346 extra = [r["external_id"] for r in poor_rows347 if r["external_id"] not in fresh][:BACKFILL_LIMIT]348349 actor_input: dict = {350 "searchUrls": urls,351 "getDetails": True,352 "maxDetails": DETAIL_LIMIT + BACKFILL_LIMIT,353 "skipDetailIds": sorted(fresh),354 "extraDetailIds": extra,355 "concurrency": CONCURRENCY,356 "requestDelay": 1,357 }358 # abonnement résidentiel Oxylabs (déjà payé) plutôt que le proxy359 # Apify facturé au Go ; l'acteur retombe sur Apify RESIDENTIAL360 # si le gabarit est absent361 ox_u = os.environ.get("OXYLABS_PROXY_USER")362 ox_p = os.environ.get("OXYLABS_PROXY_PASS")363 if ox_u and ox_p:364 actor_input["proxyUrlTemplate"] = (365 f"http://{ox_u}-cc-CA-sessid-{{session}}:{ox_p}"366 f"@pr.oxylabs.io:7777")367368 items = self._run_actor(actor_input, token)369370 # payloads détail -> cache BD (mêmes formes/clés que l'ère Scrapfly)371 for it in items:372 if it.get("kind") != "detail":373 continue374 if not it.get("ok"):375 continue # échec réseau acteur : pas de cache376 payload = {k: v for k, v in it.items()377 if k not in ("kind", "id", "ok") and v is not None}378 if payload and not payload.get("nopdp"):379 payload["pv"] = PAYLOAD_VERSION380 cache.put(str(it["id"]), payload or {"nopdp": True})381382 out: dict[str, Listing] = {}383 for it in items:384 if it.get("kind") != "listing":385 continue386 lid = str(it["id"])387 price = it.get("price")388 if price is None or not (PRICE_MIN <= price <= PRICE_MAX):389 continue # sans prix mensuel valable : ignorer390 if it.get("is_sold") or it.get("is_pending"):391 continue # déjà loué / en attente392 if _non_qc(it):393 continue # suggestion FB hors Québec dès la recherche394 detail, _f = cache.peek(lid)395 detail = detail if _usable(detail) else {}396 if detail.get("gone"):397 continue # la fiche dit : loué / retiré398 if detail and _non_qc(detail):399 continue # annonce hors Québec glissée dans le flux400 title = it.get("title") or ""401 desc = detail.get("description") or ""402 photo = it.get("primary_photo") or ""403 images = detail.get("images") or ([photo] if photo else [])404 city = (detail.get("city") or it.get("city")405 or city_by_url.get(it.get("source_url") or "", ""))406 lst = Listing(407 source=self.source_id,408 external_id=lid,409 url=f"{BASE}/item/{lid}/",410 title=title[:200],411 city=city,412 unit_type=_unit_type(title, desc),413 price=float(price),414 price_label=f"{price:,.0f} $/mois".replace(",", " "),415 description=desc,416 images=images,417 lat=detail.get("lat"),418 lng=detail.get("lng"),419 )420 # champs structurés du PDP : adresse, chambres/sdb, commodités,421 # disponibilité, scores de marche/transport422 du.apply_detail(lst, _std_detail(detail))423 out[lid] = lst424425 listings = list(out.values())426 # rattrapage des fiches actives pauvres sorties de la recherche427 listings.extend(self._backfill(cache, poor_rows, set(out)))428 finally:429 cache.close()430 return listings431