Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File: src/main.py (ka-fb-marketplace)4# Desc: Facebook Marketplace — logements à louer (propertyrentals), Québec.5# Scraping public DÉCONNECTÉ, AUCUN rendu JS : les données vivent dans6# les <script type="application/json"> du HTML brut.7# - Recherche : objets marketplace_listing_title (~25/URL, on multiplie8# les URLs ville × tranche de prix pour élargir).9# - Fiche /marketplace/item/<id> : le target GraphQL préchargé10# (marketplace_product_details_page) expose GPS, ville11# (reverse_geocode_detailed), description, statut is_live/is_sold ;12# la GALERIE complète vient du target MediaViewer préchargé13# (listing_photos du même id) — validé live, aucun rendu requis.14# Sortie dataset : {kind:"listing"} (recherche) et {kind:"detail"}.15# ⚠️ Données personnelles (Loi 25) : ne jamais republier nom/téléphone16# du vendeur ; le champ seller n'est PAS extrait.17# ==============================================================================18from __future__ import annotations1920import asyncio21import html as _html22import json23import re2425from apify import Actor2627from .net import Fetcher2829BASE = "https://www.facebook.com/marketplace"3031# Couverture Québec par défaut (mode autonome, sans searchUrls) :32# slug vanity OU id numérique Marketplace (validés live 2026-08-22 — les slugs33# inventés retombent sur des villes aléatoires, seuls ceux-ci sont sûrs).34# Chaque recherche couvre un rayon d'environ 65 km.35BIG_CITIES = { # gros parcs : tranches de prix fines (flux ~25 annonces/URL)36 "montreal": "Montréal", "laval": "Laval", "longueuil": "Longueuil",37 "quebec": "Québec", "gatineau": "Gatineau", "sherbrooke": "Sherbrooke",38 "trois-rivieres": "Trois-Rivières", "levis": "Lévis",39 "saguenay": "Saguenay", "drummondville": "Drummondville",40}41REGIONAL_CITIES = { # couverture régions : ids numériques (Serper + validation)42 "105790762794728": "Rimouski", "110925022261358": "Rivière-du-Loup",43 "106528256049537": "Matane", "109502305742889": "Gaspé",44 "100246746683723": "Sept-Îles", "105615529471535": "Baie-Comeau",45 "106016152771864": "Rouyn-Noranda", "110594102301488": "Val-d'Or",46 "105598479473872": "Amos", "109699385723610": "Saint-Georges",47 "110304502331480": "Thetford Mines", "111938355489908": "Granby",48 "108126699220609": "Victoriaville", "109753249050671": "Joliette",49 "109429165750459": "Saint-Hyacinthe",50 "108645709159963": "Saint-Jean-sur-Richelieu",51 "112946182053396": "Saint-Jérôme", "113011352052118": "Salaberry-de-Valleyfield",52 "107992819229760": "Magog", "109514615734434": "Alma",53 "109664672393669": "Mont-Laurier", "108557812502323": "La Tuque",54 "108296932538245": "Dolbeau-Mistassini",55}56PRICE_BANDS = [(0, 800), (800, 1100), (1100, 1400), (1400, 1700),57 (1700, 2100), (2100, 2800), (2800, 6000)]58REGIONAL_BANDS = [(0, 1200), (1200, 6000)] # parcs plus petits : 2 tranches5960_SCRIPT_RE = re.compile(61 r'<script type="application/json"[^>]*>(.*?)</script>', re.S)626364def default_search_urls() -> list[str]:65 urls = []66 for slug in BIG_CITIES:67 for lo, hi in PRICE_BANDS:68 urls.append(f"{BASE}/{slug}/propertyrentals"69 f"?minPrice={lo}&maxPrice={hi}"70 f"&sortBy=creation_time_descend")71 for cid in REGIONAL_CITIES:72 for lo, hi in REGIONAL_BANDS:73 urls.append(f"{BASE}/{cid}/propertyrentals"74 f"?minPrice={lo}&maxPrice={hi}"75 f"&sortBy=creation_time_descend")76 return urls777879def _iter_json_scripts(html: str):80 for m in _SCRIPT_RE.finditer(html):81 try:82 yield json.loads(m.group(1))83 except ValueError:84 continue858687def walk_listings(html: str) -> dict:88 """Objets d'annonce (marketplace_listing_title) des <script> JSON."""89 out: dict = {}9091 def walk(o):92 if isinstance(o, dict):93 if "marketplace_listing_title" in o and o.get("id"):94 out.setdefault(str(o["id"]), o)95 for v in o.values():96 walk(v)97 elif isinstance(o, list):98 for v in o:99 walk(v)100101 for data in _iter_json_scripts(html):102 walk(data)103 return out104105106def price_of(obj: dict) -> float | None:107 """Montant du loyer depuis listing_price (offset en cents géré)."""108 pr = obj.get("listing_price") or {}109 for k in ("amount", "amount_with_offset_in_currency"):110 v = pr.get(k)111 if v is None:112 continue113 try:114 val = float(v)115 except (TypeError, ValueError):116 continue117 if k == "amount_with_offset_in_currency" and val > 100000:118 val /= 100.0119 if val > 0:120 return round(val, 0)121 return None122123124def _pdp_target(html: str) -> dict | None:125 """Nœud cible le plus RICHE (description/GPS/statut) parmi les targets126 marketplace_product_details_page préchargés (stubs inclus)."""127 found: list[dict] = []128129 def walk(o):130 if isinstance(o, dict):131 pdp = o.get("marketplace_product_details_page")132 if isinstance(pdp, dict) and isinstance(pdp.get("target"), dict):133 found.append(pdp["target"])134 for v in o.values():135 walk(v)136 elif isinstance(o, list):137 for v in o:138 walk(v)139140 for data in _iter_json_scripts(html):141 walk(data)142143 def score(t: dict) -> tuple:144 desc = (t.get("redacted_description") or {}).get("text") or ""145 loc = t.get("location") or {}146 return (1 if desc else 0) + (1 if loc.get("latitude") else 0), len(t)147 return max(found, key=score) if found else None148149150def _gallery(html: str, lid: str) -> list[str]:151 """Galerie complète : plus grand tableau listing_photos porté par un objet152 du MÊME id (le target MediaViewer préchargé la contient hors connexion)."""153 best: list[str] = []154155 def walk(o):156 nonlocal best157 if isinstance(o, dict):158 lp = o.get("listing_photos")159 if (isinstance(lp, list) and lp160 and str(o.get("id") or "") == lid):161 uris = []162 for p in lp:163 uri = ((p.get("image") or {}).get("uri")164 if isinstance(p, dict) else "") or ""165 if uri and uri not in uris:166 uris.append(uri)167 if len(uris) > len(best):168 best = uris169 for v in o.values():170 walk(v)171 elif isinstance(o, list):172 for v in o:173 walk(v)174175 for data in _iter_json_scripts(html):176 walk(data)177 return best178179180def _clean_desc(text: str) -> str:181 t = text.replace("\r", "\n")182 t = re.sub(r"[ \t ]+", " ", t)183 t = re.sub(r" ?\n ?", "\n", t)184 t = re.sub(r"\n{3,}", "\n\n", t)185 return t.strip()186187188def parse_detail(html: str, lid: str) -> dict:189 """Champs riches d'une fiche /marketplace/item/<id> hors connexion.190191 {} si la page n'a pas pu être rendue (échec réseau : l'appelant ne doit192 PAS mettre en cache), {"nopdp": True} si rendue mais sans objet cible193 (fiche supprimée / mur de login : cacher pour ne pas boucler)."""194 if not html:195 return {}196 o = _pdp_target(html)197 if not o:198 return {"nopdp": True}199 out: dict = {}200 if o.get("is_sold") or o.get("is_pending") or o.get("is_live") is False:201 out["gone"] = True # loué / en attente / retiré202 loc = o.get("location") or {}203 if loc.get("latitude") and loc.get("longitude"):204 out["lat"] = loc["latitude"]205 out["lng"] = loc["longitude"]206 geo = loc.get("reverse_geocode_detailed") or {}207 if geo.get("city"):208 out["city"] = geo["city"]209 if geo.get("state"):210 out["state"] = geo["state"]211 lt = (o.get("location_text") or {}).get("text") or ""212 if lt: # ex. « Québec, QC » / « Norfolk, ON »213 out["location_text"] = lt214 parts = [p.strip() for p in lt.split(",") if p.strip()]215 if parts and not out.get("city"):216 out["city"] = parts[0]217 if len(parts) > 1 and not out.get("state"):218 out["state"] = parts[1]219 desc = (o.get("redacted_description") or {}).get("text") or ""220 if desc:221 out["description"] = _clean_desc(desc)[:6000]222 addr = ((o.get("home_address") or {}).get("street") or "").strip()223 unit_fields: list[str] = []224 for sec in (o.get("pdp_display_sections") or []):225 stype = sec.get("section_type") or ""226 for f in (sec.get("pdp_fields") or []):227 lbl = (f.get("display_label") or "").strip()228 if not lbl:229 continue230 if stype == "UNIT_INCLUDES":231 unit_fields.append(lbl)232 elif stype == "UNIT_SUBTITLE":233 if (f.get("icon_name") or "") == "pin" and not addr:234 addr = lbl.split(",")[0].strip()235 elif (f.get("icon_name") or "") == "clock":236 out["listed_text"] = lbl237 if addr:238 out["address"] = addr239 if unit_fields:240 out["unit_fields"] = unit_fields241 for src_k, dst_k in (("walk_score_info", "walk_score"),242 ("transit_score_info", "transit_score"),243 ("bike_score_info", "bike_score")):244 sc = (o.get(src_k) or {}).get("score")245 if sc is not None:246 out[dst_k] = sc247 if o.get("virtual_tour_url"):248 out["virtual_tour_url"] = o["virtual_tour_url"]249 photos = _gallery(html, lid)250 if photos:251 out["images"] = photos252 return out253254255async def main() -> None:256 async with Actor:257 inp = await Actor.get_input() or {}258 search_urls = [u for u in (inp.get("searchUrls") or []) if u]259 if not search_urls:260 search_urls = default_search_urls()261 get_details = inp.get("getDetails", True)262 max_details = int(inp.get("maxDetails") or 200)263 skip_ids = {str(i) for i in (inp.get("skipDetailIds") or [])}264 extra_ids = [str(i) for i in (inp.get("extraDetailIds") or [])]265 template = (inp.get("proxyUrlTemplate") or "").strip() or None266 proxy = None267 if not template:268 proxy = await Actor.create_proxy_configuration(269 actor_proxy_input=inp.get("proxyConfiguration"))270 fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1),271 proxy_template=template)272 sem = asyncio.Semaphore(int(inp.get("concurrency") or 8))273274 # -- phase recherche ---------------------------------------------------275 found: dict[str, tuple[dict, str]] = {} # id -> (objet, source_url)276 fails = 0277278 async def search(url: str) -> None:279 nonlocal fails280 async with sem:281 try:282 resp = await fetcher.get(url, session_id=f"q{hash(url)}")283 except Exception as exc:284 fails += 1285 Actor.log.warning(f"recherche KO {url} : {exc}")286 return287 if resp.status_code != 200:288 fails += 1289 return290 for lid, obj in walk_listings(resp.text).items():291 found.setdefault(lid, (obj, url))292293 await asyncio.gather(*[search(u) for u in search_urls])294 Actor.log.info(f"recherche : {len(found)} annonces uniques "295 f"({len(search_urls)} URLs, {fails} échecs)")296297 for lid, (obj, src) in found.items():298 photo = ((obj.get("primary_listing_photo") or {})299 .get("image") or {}).get("uri") or ""300 loc = obj.get("location") or {}301 geo = loc.get("reverse_geocode") or {}302 await Actor.push_data({303 "kind": "listing",304 "id": lid,305 "title": (obj.get("marketplace_listing_title") or "")[:300],306 "price": price_of(obj),307 "city": (geo.get("city")308 or (geo.get("city_page") or {}).get("display_name")309 or ""),310 "state": geo.get("state") or "",311 "primary_photo": photo,312 "is_sold": bool(obj.get("is_sold")),313 "is_pending": bool(obj.get("is_pending")),314 "source_url": src,315 "url": f"{BASE}/item/{lid}/",316 })317318 # -- phase détail ------------------------------------------------------319 if not get_details or max_details <= 0:320 return321 todo: list[str] = []322 for lid, (obj, _src) in found.items():323 if lid in skip_ids or obj.get("is_sold") or obj.get("is_pending"):324 continue325 if price_of(obj) is None:326 continue # sans prix : jamais publiable, épargner327 # le flux public glisse des annonces d'autres provinces (suggestions)328 geo = (obj.get("location") or {}).get("reverse_geocode") or {}329 st = (geo.get("state") or "").strip().lower()330 if st and st not in ("qc", "quebec", "québec"):331 continue332 todo.append(lid)333 for lid in extra_ids: # rattrapage : après les nouveautés334 if lid not in todo and lid not in skip_ids:335 todo.append(lid)336 todo = todo[:max_details]337338 async def detail(lid: str) -> None:339 async with sem:340 try:341 resp = await fetcher.get(f"{BASE}/item/{lid}/",342 session_id=f"d{lid}")343 payload = parse_detail(344 resp.text if resp.status_code == 200 else "", lid)345 except Exception as exc:346 Actor.log.warning(f"détail KO {lid} : {exc}")347 payload = {}348 await Actor.push_data({"kind": "detail", "id": lid,349 "ok": bool(payload), **payload})350351 await asyncio.gather(*[detail(lid) for lid in todo])352 Actor.log.info(f"détail : {len(todo)} fiches visitées")353