Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File: src/main.py (ka-apartments-com)4# Desc: Apartments.com — logements à louer, province de Québec.5# Phase 1 : pagination des pages de recherche (/qc/ = toute la6# province, HTML server-rendered ; arrêt quand le <title> ne porte7# plus « - Page N », le site retombant silencieusement sur la page 1).8# Phase 2 : fiches détail en français (/fr/<slug>/<id>/) : GPS,9# description, plans d'unités (pricingGridItem), commodités, galerie.10# Items poussés : kind="listing" (placard) et kind="detail".11# ==============================================================================12from __future__ import annotations1314import asyncio15import html as htmllib16import re1718from apify import Actor1920from .net import BrightData2122# --- recherche (placards) -----------------------------------------------------2324_ARTICLE_RX = re.compile(25 r"<article\b(?P<attrs>[^>]*\bdata-listingid=\"[^\"]+\"[^>]*)>"26 r"(?P<body>.*?)</article>", re.S)27_ATTR_RX = re.compile(r"data-(listingid|url|streetaddress)=\"([^\"]*)\"")28_TITLE_TAG_RX = re.compile(r"<title>([^<]*)</title>")29_PLACARD_TITLE_RX = re.compile(r"js-placardTitle[^\"]*\"[^>]*>([^<]+)<")30_ADDRESS_RX = re.compile(r"property-address[^\"]*\"[^>]*>([^<]+)<")31_BED_RENT_RX = re.compile(32 r"bedTextBox\">([^<]*)<.{0,400}?priceTextBox\">\s*<span[^>]*>([^<]*)<",33 re.S)34_IMG_COUNT_RX = re.compile(r"js-spnImgCount\">(\d+)")35_PHONE_RX = re.compile(r"phone-data=\"(\d{7,15})\"")36_AMENITIES_BLOCK_RX = re.compile(37 r"property-amenities\"[^>]*>(.*?)</p>", re.S)38_SPAN_RX = re.compile(r"<span[^>]*>([^<]+)</span>")39_IMG_RX = re.compile(r"https://images1\.apartments\.com/i2/[^\"'\s?]+")40_QC_SLUG_RX = re.compile(r"apartments\.com/[^\"]*-qc[-/]")414243def _clean(text: str) -> str:44 return re.sub(r"\s+", " ", htmllib.unescape(text or "")).strip()454647def parse_search(page_html: str) -> list[dict]:48 """Placards d'une page de recherche → dicts (propriétés QC seulement)."""49 out: list[dict] = []50 for m in _ARTICLE_RX.finditer(page_html):51 attrs = dict(_ATTR_RX.findall(m.group("attrs")))52 pid, url = attrs.get("listingid", ""), attrs.get("url", "")53 if not pid or not url:54 continue55 body = m.group("body")56 title_m = _PLACARD_TITLE_RX.search(body)57 addr_m = _ADDRESS_RX.search(body)58 title = _clean(title_m.group(1)) if title_m else ""59 address = _clean(addr_m.group(1)) if addr_m else ""60 if not address and ", QC" in title:61 address = title # gabarit silverpropres : adresse dans le titre62 # nearby hors-Québec glissé en bas de page → on filtre63 if ", QC" not in address and not _QC_SLUG_RX.search(url):64 continue65 rents = [{"beds": _clean(b), "price": _clean(p)}66 for b, p in _BED_RENT_RX.findall(body)]67 img_count_m = _IMG_COUNT_RX.search(body)68 phone_m = _PHONE_RX.search(body)69 amen_m = _AMENITIES_BLOCK_RX.search(body)70 amenities = [_clean(s) for s in _SPAN_RX.findall(amen_m.group(1))] \71 if amen_m else []72 img_m = _IMG_RX.search(body)73 out.append({74 "id": pid,75 "url": url,76 "title": title,77 "address": address,78 "street": _clean(attrs.get("streetaddress", "")),79 "rents": rents,80 "img_count": int(img_count_m.group(1)) if img_count_m else 0,81 "phone": phone_m.group(1) if phone_m else "",82 "amenities": [a for a in amenities if a],83 "image": img_m.group(0) if img_m else "",84 })85 return out868788def page_title(page_html: str) -> str:89 m = _TITLE_TAG_RX.search(page_html)90 return _clean(m.group(1)) if m else ""919293# --- fiche détail (version française) ------------------------------------------9495_META_LAT_RX = re.compile(96 r"<meta property=\"place:location:latitude\" content=\"([^\"]+)\"")97_META_LNG_RX = re.compile(98 r"<meta property=\"place:location:longitude\" content=\"([^\"]+)\"")99_PROP_NAME_RX = re.compile(r"id=\"propertyName\"[^>]*>([^<]+)")100_ADDR_BLOCK_RX = re.compile(101 r"delivery-address\"><span>([^<]+)</span>.*?<span>([^<]+)</span>,\s*"102 r"<span class=\"stateZipContainer\">\s*<span>([^<]+)</span>\s*"103 r"<span>([^<]*)</span>", re.S)104_DESC_SECTION_RX = re.compile(105 r"<section[^>]*id=\"descriptionSection\"[^>]*>(.*?)</section>", re.S)106_MODEL_WRAPPER_RX = re.compile(107 r"priceGridModelWrapper[^\"]*\"\s*data-rentalkey=\"([^\"]*)\"(.*?)"108 r"(?=priceGridModelWrapper|$)", re.S)109_MODEL_NAME_RX = re.compile(r"modelName\">([^<]+)<")110_RENT_LABEL_RX = re.compile(r"rentLabel\">\s*([^<]+)")111_DETAILS_WRAPPER_RX = re.compile(112 r"detailsTextWrapper\">(.*?)</span>\s*</span>", re.S)113_AVAIL_RX = re.compile(r"availabilityInfo\">([^<]+)<")114_BEDS_BATHS_RX = re.compile(r"data-beds=\"([^\"]*)\"\s*data-baths=\"([^\"]*)\"")115_SPEC_INFO_RX = re.compile(r"specInfo\">\s*<span>([^<]+)</span>")116_TAG_RX = re.compile(r"<[^>]+>")117118119def _to_float(text: str) -> float | None:120 try:121 return float(text.strip())122 except (TypeError, ValueError):123 return None124125126def parse_detail(page_html: str) -> dict:127 out: dict = {}128 name_m = _PROP_NAME_RX.search(page_html)129 if name_m:130 out["name"] = _clean(name_m.group(1))131 addr_m = _ADDR_BLOCK_RX.search(page_html)132 if addr_m:133 out["street"] = _clean(addr_m.group(1))134 out["city"] = _clean(addr_m.group(2))135 out["state"] = _clean(addr_m.group(3))136 out["zip"] = _clean(addr_m.group(4))137 lat_m, lng_m = _META_LAT_RX.search(page_html), _META_LNG_RX.search(page_html)138 if lat_m and lng_m:139 out["lat"] = _to_float(lat_m.group(1))140 out["lng"] = _to_float(lng_m.group(1))141 desc_m = _DESC_SECTION_RX.search(page_html)142 if desc_m:143 lines = [_clean(t) for t in144 _TAG_RX.sub("\n", desc_m.group(1)).split("\n")]145 lines = [t for t in lines if t146 and not t.lower().startswith(("à propos", "about"))]147 if lines:148 out["description"] = "\n\n".join(dict.fromkeys(lines))149 plans: list[dict] = []150 for key, body in _MODEL_WRAPPER_RX.findall(page_html):151 plan: dict = {"key": key}152 nm = _MODEL_NAME_RX.search(body)153 if nm:154 plan["name"] = _clean(nm.group(1))155 rl = _RENT_LABEL_RX.search(body)156 if rl:157 plan["rent"] = _clean(rl.group(1))158 dw = _DETAILS_WRAPPER_RX.search(body)159 if dw:160 plan["details"] = [_clean(s) for s in161 _SPAN_RX.findall(dw.group(1) + "</span>")]162 bb = _BEDS_BATHS_RX.search(body)163 if bb:164 plan["beds"] = _to_float(bb.group(1))165 plan["baths"] = _to_float(bb.group(2))166 av = _AVAIL_RX.search(body)167 if av:168 plan["availability"] = _clean(av.group(1))169 if plan.get("name") or plan.get("rent"):170 # la grille est présente deux fois dans la page (desktop/mobile)171 if not any(p.get("key") == plan.get("key")172 and p.get("name") == plan.get("name")173 and p.get("rent") == plan.get("rent") for p in plans):174 plans.append(plan)175 if plans:176 out["plans"] = plans177 amenities = [_clean(s) for s in _SPEC_INFO_RX.findall(page_html)]178 if amenities:179 out["amenities"] = list(dict.fromkeys(a for a in amenities if a))180 images = list(dict.fromkeys(181 u for u in _IMG_RX.findall(page_html) if "-logo" not in u))182 if images:183 out["images"] = images184 return out185186187def fr_url(url: str) -> str:188 if "apartments.com/fr/" in url:189 return url190 return url.replace("www.apartments.com/", "www.apartments.com/fr/", 1)191192193# --- orchestration --------------------------------------------------------------194195async def _crawl_search(bd: BrightData, base: str, max_pages: int,196 props: dict[str, dict]) -> None:197 base = base if base.endswith("/") else base + "/"198 for page in range(1, max_pages + 1):199 url = base if page == 1 else f"{base}{page}/"200 page_html = await bd.get(url)201 if page_html is None:202 Actor.log.warning(f"recherche : échec fetch {url}, arrêt")203 break204 if page > 1 and f"- Page {page}" not in page_title(page_html):205 Actor.log.info(f"recherche : fin de pagination à la page {page} "206 f"({base})")207 break208 placards = parse_search(page_html)209 if not placards:210 Actor.log.info(f"recherche : page {page} vide ({base})")211 break212 fresh = [p for p in placards if p["id"] not in props]213 for p in fresh:214 props[p["id"]] = p215 await Actor.push_data({"kind": "listing", **p})216 Actor.log.info(f"recherche p.{page} : {len(placards)} placards, "217 f"{len(fresh)} nouveaux ({base})")218219220async def _fetch_detail(bd: BrightData, pid: str, url: str) -> None:221 page_html = await bd.get(fr_url(url))222 if page_html is None:223 Actor.log.warning(f"détail {pid} : échec fetch")224 return225 detail = parse_detail(page_html)226 if not detail:227 Actor.log.warning(f"détail {pid} : page sans données")228 return229 await Actor.push_data({"kind": "detail", "id": pid, "url": url, **detail})230231232async def main() -> None:233 async with Actor:234 inp = await Actor.get_input() or {}235 token = (inp.get("brightdataToken") or "").strip()236 if not token:237 raise ValueError("brightdataToken requis (Web Unlocker)")238 bd = BrightData(239 token=token,240 zone=(inp.get("brightdataZone") or "web_unlocker1").strip(),241 concurrency=int(inp.get("concurrency") or 4),242 delay=float(inp.get("requestDelay") or 0))243 try:244 search_urls = [u.strip() for u in245 (inp.get("searchUrls")246 or ["https://www.apartments.com/qc/"])247 if u and u.strip()]248 max_pages = int(inp.get("maxPages") or 25)249 props: dict[str, dict] = {}250 for base in search_urls:251 await _crawl_search(bd, base, max_pages, props)252 Actor.log.info(f"recherche terminée : {len(props)} propriétés")253254 if not inp.get("getDetails", True):255 return256 skip = set(inp.get("skipDetailIds") or [])257 targets: list[tuple[str, str]] = [258 (pid, p["url"]) for pid, p in props.items() if pid not in skip]259 for enc in inp.get("extraDetailIds") or []:260 pid, _, url = enc.partition("|")261 if pid and url and pid not in props and pid not in skip:262 targets.append((pid, url))263 max_details = int(inp.get("maxDetails") or 0)264 if max_details >= 0:265 targets = targets[:max_details]266 Actor.log.info(f"détails : {len(targets)} fiches à visiter")267 await asyncio.gather(268 *(_fetch_detail(bd, pid, url) for pid, url in targets))269 finally:270 await bd.close()271