spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/app_urbains.py : connecteur Appartements Urbains5# (appartementsurbains.ca — Ste-Foy, Montcalm, Limoilou, Loretteville, Lévis)6# Pages projets (Oxygen/WP) : tableau d'unités disponibles avec liens7# « /unites/?_unite=NNN ». Une annonce par unité disponible.8# -----------------------------------------------------------------------------9from __future__ import annotations1011import re12from urllib.parse import unquote1314from bs4 import BeautifulSoup1516from ..schema import Listing, infer_city, normalize_unit_type, parse_price17from .base import BaseConnector1819BASE = "https://www.appartementsurbains.ca"20LIST_URL = f"{BASE}/projets-residentiels/"2122KNOWN_SECTORS = ("Montcalm", "Limoilou", "Lévis", "Ste-Foy", "Loretteville",23 "Sainte-Foy", "Saint-Roch", "Lebourgneuf")2425ADDR_RE = re.compile(26 r"\d{1,5}[^,<>|]{2,60},\s*(?:Lévis|Québec)\s*,?\s*(?:QC|Qu[ée]bec)?"27 r"[^.<>|]{0,15}[A-Z]\d[A-Z]\s?\d[A-Z]\d"28)29# adresse plus permissive (sans code postal) pour la carte-projet wpgb30ADDR_LOOSE_RE = re.compile(31 r"\d{1,5}[^,<>|]{2,60},\s*(?:Lévis|Québec)\b[^.<>|]{0,25}")32AREA_RE = re.compile(r"(\d[\d\s]{1,6})\s*pi[²2]", re.I)33IMG_RE = re.compile(r'https?://[^"\s\\)]+/wp-content/uploads/[^"\s\\)]+'34 r"\.(?:jpe?g|png|webp)", re.I)35BAD_IMG_RE = re.compile(r"favicon|logo|icon|cropped-|social|plugins|hqdefault|"36 r"pdf|texture", re.I)37TYPE_RE = re.compile(r"\d\s*½(?:\s*\+\s*\w+)?|\d\s*1/2|Studio|Loft", re.I)383940class AppartementsUrbainsConnector(BaseConnector):41 source_id = "app_urbains"42 request_delay = 0.643 max_projects = 20 # garde-fou de crawl4445 def _discover_projects(self) -> dict[str, dict]:46 """Menu de navigation : « Nom du projet » + « Quartier »."""47 projects: dict[str, dict] = {}48 html = self.get(LIST_URL).text49 soup = BeautifulSoup(html, "html.parser")50 for a in soup.select(f'a[href^="{BASE}/"], a[href^="/"]'):51 href = a.get("href") or ""52 if href.startswith("/"):53 href = BASE + href54 href = href.split("?")[0].rstrip("/")55 slug = href.replace(BASE, "").strip("/")56 # les pages projets sont à la racine (un seul segment, pas de section connue)57 if (not slug or "/" in slug or slug in58 ("projets-residentiels", "nouveaux-projets", "quartiers",59 "a-propos", "actualites", "carrieres", "nous-joindre",60 "html-sitemap", "projets-commerciaux")):61 continue62 text = re.sub(r"\s+", " ", a.get_text(" ", strip=True))63 sector = ""64 for s in KNOWN_SECTORS:65 if re.search(rf"{re.escape(s)}\s*$", text):66 sector = s67 break68 if not sector: # lien sans étiquette de quartier -> ignorer69 continue70 name = text[: -len(sector)].strip()71 if slug not in projects and name:72 projects[slug] = {"name": name, "sector": sector}73 return projects7475 def fetch(self) -> list[Listing]:76 listings: list[Listing] = []77 try:78 projects = self._discover_projects()79 except Exception:80 return listings8182 for i, (slug, meta) in enumerate(projects.items()):83 if i >= self.max_projects:84 break85 url = f"{BASE}/{slug}/"86 try:87 html = self.get(url).text88 except Exception:89 continue90 soup = BeautifulSoup(html, "html.parser")91 text = re.sub(r"\s+", " ", soup.get_text(" ", strip=True))9293 # Adresse : de préférence la carte wpgb du projet lui-même94 # (la 1re adresse de la page peut être un point d'intérêt voisin)95 address = ""96 for card in soup.select(".wpgb-card"):97 card_text = re.sub(r"\s+", " ", card.get_text(" ", strip=True))98 if card_text.lower().startswith(meta["name"].lower()):99 am_ = ADDR_RE.search(card_text) or \100 ADDR_LOOSE_RE.search(card_text)101 if am_:102 address = am_.group(0).strip(" ,")103 break104 if not address:105 m = ADDR_RE.search(text)106 address = m.group(0).strip() if m else ""107 sector = meta["sector"]108 city = infer_city(sector)109 if "Lévis" in address:110 city = "Lévis"111112 # Images du projet (souvent servies depuis le domaine du projet)113 images = [u for u in dict.fromkeys(114 IMG_RE.findall(html) +115 re.findall(r'data-lazy-src="([^"]+\.(?:jpe?g|png|webp))"', html, re.I))116 if not BAD_IMG_RE.search(u)]117 images = [u if u.startswith("http") else BASE + u for u in images][:25]118119 og = soup.find("meta", attrs={"property": "og:description"})120 desc = og["content"].strip()[:600] if og and og.get("content") else ""121122 # Commodités (blocs « avantages » — deux générations de gabarits)123 amenities = list(dict.fromkeys(124 [re.sub(r"\s+", " ", h.get_text(" ", strip=True))125 for h in soup.select(".listing-avantages h4")126 if h.get_text(strip=True)] +127 [s.get_text(strip=True)128 for s in soup.select("div.ct-div-block.c-center span.ct-span")129 if s.get_text(strip=True)]))[:15]130131 # Téléphone du projet (lien tel: structuré) -> details.contact132 details: dict = {}133 tel = soup.select_one('a[href^="tel:"]')134 if tel:135 digits = re.sub(r"\D", "", unquote(tel.get("href", "")))[-10:]136 if len(digits) == 10:137 details["contact"] = {"phone": "-".join(138 (digits[:3], digits[3:6], digits[6:]))}139140 # Tableau des unités disponibles141 seen: set[str] = set()142 for a in soup.select('a[href*="_unite="]'):143 try:144 num = a.get_text(strip=True)145 if not num or num in seen:146 continue147 seen.add(num)148 row = a149 row_text = ""150 for _ in range(4):151 row = row.parent152 if row is None:153 break154 cls = " ".join(row.get("class") or [])155 if "c-full-width" in cls and "flex-row" in cls:156 row_text = re.sub(r"\s+", " ",157 row.get_text(" ", strip=True))158 break159 tm = TYPE_RE.search(row_text)160 unit_type = tm.group(0) if tm else ""161 pm = re.search(r"Prix\s*([\d\s]+\$)", row_text)162 price_label = pm.group(1).strip() if pm else ""163 am = re.search(164 r"Disponibilité\s+(.*?)(?:\s+Superficie|\s+Prix|$)",165 row_text)166 availability = am.group(1).strip() if am else ""167 # Superficie de l'unité (colonne « Superficie (pi²) ») :168 # texte brut dans amenities — finalize() dérive area_sqft169 unit_amenities = amenities170 sqm = AREA_RE.search(row_text)171 if sqm:172 unit_amenities = amenities + [173 f"{sqm.group(1).strip()} pi²"]174 listings.append(Listing(175 source=self.source_id,176 external_id=f"{slug}-{num}",177 url=url,178 title=f"{meta['name']} — unité {num}",179 address=address,180 sector=sector,181 city=city,182 unit_type=normalize_unit_type(unit_type),183 price=parse_price(price_label),184 price_label=price_label,185 availability=availability,186 description=desc,187 amenities=unit_amenities,188 details=dict(details),189 images=images,190 ))191 except Exception:192 continue193194 return listings195