Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/fb_marketplace.py : Facebook Marketplace — catégorie location5# (propertyrentals), TOUTE la province. Annonces de particuliers, fort6# complément aux gestionnaires. Depuis 2026-08-22 le scraping est délégué à7# l'ACTEUR APIFY MAISON gorgeous_thistle/ka-fb-marketplace (source :8# actors/ka-fb-marketplace de ce repo) : HTML public déconnecté SANS rendu9# JS (les JSON de recherche, la fiche PDP et la galerie MediaViewer10# préchargée vivent dans le HTML brut) via proxy résidentiel CA — ~100×11# moins cher que l'ancienne chaîne Scrapfly ASP+render_js, et couverture12# élargie de 10 à 33 villes (IDs Marketplace des villes régionales validés ;13# les slugs inventés retombent sur des villes aléatoires, ne pas en ajouter14# sans les valider).15# Le scroll infini reste bloqué hors connexion (~25 annonces/URL) : on16# pagine par TRANCHES DE PRIX et on accumule sur plusieurs synchronisations.17# Le flux public TOURNE : une phase de RATTRAPAGE (extraDetailIds) visite18# les fiches actives jamais enrichies (budget RENTKA_FBMP_BACKFILL).19# Le connecteur garde le cache détail BD (clé v2, payloads compatibles) et20# passe à l'acteur la liste des fiches fraîches à NE PAS revisiter.21# ⚠️ Données personnelles (Loi 25) : ne jamais republier nom/téléphone du22# vendeur, garder le lien sortant.23# -----------------------------------------------------------------------------24from __future__ import annotations2526import json27import os28import re29import time3031import requests as _requests3233from ..schema import Listing, normalize_unit_type34from .base import BaseConnector35from . import _detailutil as du3637BASE = "https://www.facebook.com/marketplace"38APIFY_API = "https://api.apify.com/v2"39ACTOR = os.environ.get("RENTKA_FBMP_ACTOR", "gorgeous_thistle~ka-fb-marketplace")4041# Canadian cities outside Québec — large inventories: Marketplace vanity42# slug -> display name. Only UNAMBIGUOUS slugs (e.g. «london» resolves to43# London UK on Facebook — use a verified numeric REGIONS id instead).44CITIES = {45 "toronto": "Toronto", "ottawa": "Ottawa", "mississauga": "Mississauga",46 "brampton": "Brampton", "vancouver": "Vancouver", "calgary": "Calgary",47 "edmonton": "Edmonton", "winnipeg": "Winnipeg", "saskatoon": "Saskatoon",48 "regina": "Regina", "halifax": "Halifax",49}50# city -> province for the searched cities (province of each listing)51_CITY_PROV = {52 "Toronto": "ON", "Ottawa": "ON", "Mississauga": "ON", "Brampton": "ON",53 "Vancouver": "BC", "Calgary": "AB", "Edmonton": "AB", "Winnipeg": "MB",54 "Saskatoon": "SK", "Regina": "SK", "Halifax": "NS",55}56# Facebook state labels -> province code57_STATE_PROV = {58 "on": "ON", "ontario": "ON", "bc": "BC", "british columbia": "BC",59 "ab": "AB", "alberta": "AB", "sk": "SK", "saskatchewan": "SK",60 "mb": "MB", "manitoba": "MB", "ns": "NS", "nova scotia": "NS",61 "nb": "NB", "new brunswick": "NB", "pe": "PE",62 "prince edward island": "PE", "nl": "NL", "newfoundland": "NL",63 "newfoundland and labrador": "NL", "yt": "YT", "nt": "NT", "nu": "NU",64}65# regional cities: Marketplace numeric ID -> name (to be filled with66# VERIFIED ids for London ON, Kitchener, Hamilton, Victoria, Moncton…67# — vanity slugs are ambiguous for those)68REGIONS: dict[str, str] = {}69# tranches de prix (bornes en $) : le flux déconnecté sert un sous-ensemble70# tournant, chaque tranche renvoie un lot quasi disjoint71PRICE_BANDS = [(0, 800), (800, 1100), (1100, 1400), (1400, 1700),72 (1700, 2100), (2100, 2800), (2800, 6000)]73REGIONAL_BANDS = [(0, 1200), (1200, 6000)] # parcs plus petits : 2 tranches7475# garde-fous de prix mensuel (rejette « $90 » = /nuit, et les valeurs à vendre)76PRICE_MIN, PRICE_MAX = 300, 120007778DETAIL_LIMIT = int(os.environ.get("RENTKA_FBMP_DETAIL_LIMIT", "120"))79# rattrapage : fiches ACTIVES en BD encore pauvres (sans description, sans80# GPS ou avec ≤ 1 image) qui ne repassent plus dans la recherche publique81BACKFILL_LIMIT = int(os.environ.get("RENTKA_FBMP_BACKFILL", "80"))82# les annonces FB changent peu après publication : TTL long = le budget détail83# sert surtout aux NOUVELLES annonces plutôt qu'à re-visiter les connues84TTL_DAYS = float(os.environ.get("RENTKA_FBMP_TTL_DAYS", "30"))85CITY_LIMIT = os.environ.get("RENTKA_FBMP_CITIES", "") # ex. "montreal,laval"86CONCURRENCY = int(os.environ.get("RENTKA_FBMP_CONCURRENCY", "8"))87RUN_TIMEOUT = int(os.environ.get("RENTKA_FBMP_RUN_TIMEOUT", "2400")) # s8889# clé du cache détail — v2 : payloads identiques à l'ère Scrapfly (galerie,90# ville/province, statut) ; l'acteur produit la même forme, cache réutilisé91DETAIL_KEY = "v2"9293# accepted scope: Canada outside Québec — the public feed sometimes slips94# out-of-area ads (US border cities, Québec) into a city's results95_QC_STATES = {"qc", "quebec", "québec"}96_CANADA_BBOX = (41.6, 83.2, -141.0, -52.5)97_QC_BBOX = (44.9, 62.8, -79.6, -56.9)9899100def _unit_type(title: str, desc: str = "") -> str:101 blob = f"{title} {desc}"102 ut = normalize_unit_type(title)103 if re.match(r"^\d bedrooms?$|^5\+ bedrooms$|^Studio$|^Loft$", ut or ""):104 return ut105 m = re.search(r"(\d+)\s*(?:bed|bedroom|chambre|cc|br)\b", blob, re.I)106 if m:107 n = int(m.group(1))108 if n >= 5:109 return "5+ bedrooms"110 return f"{n} bedroom" + ("s" if n > 1 else "")111 if re.search(r"\bstudio|bachelor\b", blob, re.I):112 return "Studio"113 if re.search(r"\b(?:private\s+)?room\b|chambre", blob, re.I):114 return "Room"115 return ""116117118def _usable(payload: dict | None) -> bool:119 """Payload détail exploitable (ni vide, ni marqueur « fiche sans objet »)."""120 return bool(payload) and not payload.get("nopdp")121122123def _non_qc(payload: dict) -> bool:124 """True when the ad is OUT of Rent-Ka's scope (Québec or outside125 Canada). Name kept for the call sites."""126 state = (payload.get("state") or "").strip().lower()127 if state:128 if state in _QC_STATES:129 return True130 return state not in _STATE_PROV # US states etc.131 # no state: geographic net — some PDPs only provide GPS132 lat, lng = payload.get("lat"), payload.get("lng")133 if lat is not None and lng is not None:134 in_canada = (_CANADA_BBOX[0] <= lat <= _CANADA_BBOX[1]135 and _CANADA_BBOX[2] <= lng <= _CANADA_BBOX[3])136 in_qc = (_QC_BBOX[0] <= lat <= _QC_BBOX[1]137 and _QC_BBOX[2] <= lng <= _QC_BBOX[3])138 return (not in_canada) or in_qc139 return False140141142# version des payloads détail : bump quand parse_detail (acteur) apprend de143# nouveaux champs, pour re-visiter progressivement les fiches déjà en cache144PAYLOAD_VERSION = 3145146_TYPE_LABELS = {"apartment", "house", "townhouse", "condo", "room", "flat",147 "appartement", "maison", "maison de ville", "chambre", "loft"}148_BEDS_RX = re.compile(r"(\d+(?:[.,]\d+)?)\s*(?:beds?\b|chambres?\b|lits?\b)",149 re.I)150_BATHS_RX = re.compile(r"(\d+(?:[.,]\d+)?)\s*(?:baths?\b|salles?\b)", re.I)151152153def _std_detail(d: dict) -> dict:154 """Payload acteur -> dict pour du.apply_detail (labels PDP en/fr)."""155 std = {k: d[k] for k in ("description", "images", "lat", "lng",156 "city", "address") if d.get(k)}157 amen: list[str] = []158 for lbl in d.get("unit_fields") or []:159 mb = _BEDS_RX.search(lbl)160 if mb: # « 2 beds · 1 bath »161 std["bedrooms"] = float(mb.group(1).replace(",", "."))162 ms = _BATHS_RX.search(lbl)163 if ms:164 std["bathrooms"] = float(ms.group(1).replace(",", "."))165 continue166 low = lbl.casefold()167 if low in _TYPE_LABELS:168 continue # type générique : _unit_type (titre/desc) fait mieux169 if "furnish" in low or "meublé" in low:170 std["furnished"] = not ("unfurnish" in low or "non meublé" in low)171 if "pet" in low or "animaux" in low:172 std["pets"] = ("non" if "no pet" in low or "pas d" in low173 else "oui")174 amen.append(lbl)175 if amen:176 std["amenities"] = amen177 for part in (d.get("listed_text") or "").split("·"):178 p = part.strip()179 if p.casefold().startswith(("available", "disponible")):180 std["availability"] = p181 extras = {k: d[k] for k in ("walk_score", "transit_score", "bike_score")182 if d.get(k) is not None}183 if d.get("virtual_tour_url"):184 extras["virtual_tour"] = d["virtual_tour_url"]185 if extras:186 std["details"] = extras187 return std188189190class FacebookMarketplaceConnector(BaseConnector):191 source_id = "fb_marketplace"192 request_delay = 1.0193194 # -- orchestration de l'acteur Apify --------------------------------------195 def _search_urls(self) -> tuple[list[str], dict[str, str]]:196 """URLs de recherche (ville × tranche) + mapping URL -> nom de ville."""197 cities, regions = dict(CITIES), dict(REGIONS)198 if CITY_LIMIT:199 wanted = {c.strip().casefold() for c in CITY_LIMIT.split(",")}200 cities = {k: v for k, v in cities.items()201 if k in wanted or v.casefold() in wanted}202 regions = {k: v for k, v in regions.items()203 if v.casefold() in wanted}204 urls: list[str] = []205 by_url: dict[str, str] = {}206 for slug, city in cities.items():207 for lo, hi in PRICE_BANDS:208 u = (f"{BASE}/{slug}/propertyrentals"209 f"?minPrice={lo}&maxPrice={hi}&sortBy=creation_time_descend")210 urls.append(u)211 by_url[u] = city212 for cid, city in regions.items():213 for lo, hi in REGIONAL_BANDS:214 u = (f"{BASE}/{cid}/propertyrentals"215 f"?minPrice={lo}&maxPrice={hi}&sortBy=creation_time_descend")216 urls.append(u)217 by_url[u] = city218 return urls, by_url219220 def _run_actor(self, payload: dict, token: str) -> list[dict]:221 """Lance l'acteur, attend la fin, retourne les items du dataset."""222 r = _requests.post(223 f"{APIFY_API}/acts/{ACTOR}/runs?waitForFinish=120",224 json=payload, timeout=180,225 headers={"Authorization": f"Bearer {token}"})226 r.raise_for_status()227 run = r.json()["data"]228 deadline = time.time() + RUN_TIMEOUT229 while run["status"] in ("READY", "RUNNING") and time.time() < deadline:230 time.sleep(10)231 run = _requests.get(232 f"{APIFY_API}/actor-runs/{run['id']}", timeout=60,233 headers={"Authorization": f"Bearer {token}"}).json()["data"]234 if run["status"] != "SUCCEEDED":235 raise RuntimeError(f"acteur {ACTOR} : run {run['id']} "236 f"terminé en {run['status']}")237 items: list[dict] = []238 offset = 0239 while True:240 batch = _requests.get(241 f"{APIFY_API}/datasets/{run['defaultDatasetId']}/items"242 f"?limit=1000&offset={offset}", timeout=120,243 headers={"Authorization": f"Bearer {token}"}).json()244 items.extend(batch)245 if len(batch) < 1000:246 return items247 offset += 1000248249 # -- caches / BD -----------------------------------------------------------250 def _fresh_ids(self, cache: du.TtlDetailCache) -> list[str]:251 """IDs dont le payload détail en cache est encore frais (clé + TTL) :252 l'acteur ne les revisitera pas."""253 rows = cache.con.execute(254 "SELECT external_id FROM detail_cache"255 " WHERE source=? AND key=? AND fetched_at > ?"256 " AND (json_extract(payload,'$.pv') >= ?"257 " OR json_extract(payload,'$.nopdp') IS NOT NULL"258 " OR json_extract(payload,'$.gone') IS NOT NULL)",259 (self.source_id, DETAIL_KEY,260 time.time() - TTL_DAYS * 86400, PAYLOAD_VERSION)).fetchall()261 return [r["external_id"] for r in rows]262263 def _poor_active_rows(self, cache: du.TtlDetailCache) -> list:264 """Fiches ACTIVES en BD encore pauvres (candidates au rattrapage)."""265 rows = cache.con.execute(266 "SELECT external_id, url, title, address, sector, city, unit_type,"267 " bedrooms, bathrooms, price, price_label, availability,"268 " availability_date, area_sqft, pets, furnished, description,"269 " amenities, details, images, lat, lng"270 " FROM listings WHERE source=? AND active=1"271 " ORDER BY last_seen DESC", (self.source_id,)).fetchall()272273 def poor(r) -> bool:274 try:275 n_img = len(json.loads(r["images"] or "[]"))276 except ValueError:277 n_img = 0278 return (not (r["description"] or "").strip()279 or r["lat"] is None or n_img <= 1280 or not (r["address"] or "").strip()281 or r["bedrooms"] is None)282 return [r for r in rows if poor(r)]283284 def _listing_from_row(self, r) -> Listing:285 """Reconstruit le Listing depuis sa ligne BD (phase de rattrapage)."""286 def js(s, default):287 try:288 return json.loads(s) if s else default289 except ValueError:290 return default291 return Listing(292 source=self.source_id, external_id=r["external_id"],293 url=r["url"], title=r["title"] or "", address=r["address"] or "",294 sector=r["sector"] or "", city=r["city"] or "",295 unit_type=r["unit_type"] or "", bedrooms=r["bedrooms"],296 bathrooms=r["bathrooms"], price=r["price"],297 price_label=r["price_label"] or "",298 availability=r["availability"] or "",299 availability_date=r["availability_date"],300 area_sqft=r["area_sqft"], pets=r["pets"],301 furnished=(None if r["furnished"] is None else bool(r["furnished"])),302 description=r["description"] or "",303 amenities=js(r["amenities"], []), details=js(r["details"], {}),304 images=js(r["images"], []), lat=r["lat"], lng=r["lng"],305 )306307 def _backfill(self, cache: du.TtlDetailCache, poor_rows: list,308 done: set[str]) -> list[Listing]:309 """Rattrapage : ré-émet les fiches actives pauvres dont le cache310 (rempli par l'acteur via extraDetailIds) apporte du neuf. Les311 annonces mortes suivent le cycle normal miss_count → retrait."""312 def adds(r, s: dict) -> bool:313 if s.get("description") and len(s["description"]) > \314 len(r["description"] or ""):315 return True316 if s.get("lat") is not None and r["lat"] is None:317 return True318 try:319 n_img = len(json.loads(r["images"] or "[]"))320 except ValueError:321 n_img = 0322 if len(s.get("images") or []) > n_img:323 return True324 if s.get("address") and not (r["address"] or "").strip():325 return True326 if s.get("bedrooms") is not None and r["bedrooms"] is None:327 return True328 return bool(s.get("city")) and not (r["city"] or "")329330 out: list[Listing] = []331 for r in poor_rows:332 if r["external_id"] in done:333 continue334 d, _fresh = cache.peek(r["external_id"])335 if not _usable(d) or d.get("gone") or _non_qc(d):336 continue # morte / hors QC / rien de neuf337 std = _std_detail(d)338 if not adds(r, std):339 continue # rien à apporter : laisser vivre son cycle340 lst = self._listing_from_row(r)341 du.apply_detail(lst, std)342 if d.get("city"): # reverse geocode FB : autoritaire343 lst.city = d["city"]344 if not lst.unit_type:345 lst.unit_type = _unit_type(lst.title, lst.description)346 out.append(lst)347 return out348349 # -- pipeline principal ----------------------------------------------------350 def fetch(self) -> list[Listing]:351 token = os.environ.get("APIFY_TOKEN")352 if not token:353 raise RuntimeError("APIFY_TOKEN manquant (voir .env)")354 urls, city_by_url = self._search_urls()355356 cache = du.TtlDetailCache(self, budget=0, ttl_days=TTL_DAYS,357 key=DETAIL_KEY, fetch_html=lambda _u: "")358 try:359 fresh = set(self._fresh_ids(cache))360 poor_rows = self._poor_active_rows(cache)361 extra = [r["external_id"] for r in poor_rows362 if r["external_id"] not in fresh][:BACKFILL_LIMIT]363364 actor_input: dict = {365 "searchUrls": urls,366 "getDetails": True,367 "maxDetails": DETAIL_LIMIT + BACKFILL_LIMIT,368 "skipDetailIds": sorted(fresh),369 "extraDetailIds": extra,370 "concurrency": CONCURRENCY,371 "requestDelay": 1,372 }373 # abonnement résidentiel Oxylabs (déjà payé) plutôt que le proxy374 # Apify facturé au Go ; l'acteur retombe sur Apify RESIDENTIAL375 # si le gabarit est absent376 ox_u = os.environ.get("OXYLABS_PROXY_USER")377 ox_p = os.environ.get("OXYLABS_PROXY_PASS")378 if ox_u and ox_p:379 actor_input["proxyUrlTemplate"] = (380 f"http://{ox_u}-cc-CA-sessid-{{session}}:{ox_p}"381 f"@pr.oxylabs.io:7777")382383 items = self._run_actor(actor_input, token)384385 # payloads détail -> cache BD (mêmes formes/clés que l'ère Scrapfly)386 for it in items:387 if it.get("kind") != "detail":388 continue389 if not it.get("ok"):390 continue # échec réseau acteur : pas de cache391 payload = {k: v for k, v in it.items()392 if k not in ("kind", "id", "ok") and v is not None}393 if payload and not payload.get("nopdp"):394 payload["pv"] = PAYLOAD_VERSION395 cache.put(str(it["id"]), payload or {"nopdp": True})396397 out: dict[str, Listing] = {}398 for it in items:399 if it.get("kind") != "listing":400 continue401 lid = str(it["id"])402 price = it.get("price")403 if price is None or not (PRICE_MIN <= price <= PRICE_MAX):404 continue # sans prix mensuel valable : ignorer405 if it.get("is_sold") or it.get("is_pending"):406 continue # déjà loué / en attente407 if _non_qc(it):408 continue # suggestion FB hors Québec dès la recherche409 detail, _f = cache.peek(lid)410 detail = detail if _usable(detail) else {}411 if detail.get("gone"):412 continue # la fiche dit : loué / retiré413 if detail and _non_qc(detail):414 continue # annonce hors Québec glissée dans le flux415 title = it.get("title") or ""416 desc = detail.get("description") or ""417 photo = it.get("primary_photo") or ""418 images = detail.get("images") or ([photo] if photo else [])419 city = (detail.get("city") or it.get("city")420 or city_by_url.get(it.get("source_url") or "", ""))421 prov = (_STATE_PROV.get(422 (detail.get("state") or "").strip().lower())423 or _CITY_PROV.get(city, "ON"))424 lst = Listing(425 source=self.source_id,426 external_id=lid,427 url=f"{BASE}/item/{lid}/",428 title=title[:200],429 city=city,430 province=prov,431 unit_type=_unit_type(title, desc),432 price=float(price),433 price_label=f"${price:,.0f}/month",434 description=desc,435 images=images,436 lat=detail.get("lat"),437 lng=detail.get("lng"),438 )439 # champs structurés du PDP : adresse, chambres/sdb, commodités,440 # disponibilité, scores de marche/transport441 du.apply_detail(lst, _std_detail(detail))442 out[lid] = lst443444 listings = list(out.values())445 # rattrapage des fiches actives pauvres sorties de la recherche446 listings.extend(self._backfill(cache, poor_rows, set(out)))447 finally:448 cache.close()449 return listings450