# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/app_urbains.py : connecteur Appartements Urbains # (appartementsurbains.ca — Ste-Foy, Montcalm, Limoilou, Loretteville, Lévis) # Pages projets (Oxygen/WP) : tableau d'unités disponibles avec liens # « /unites/?_unite=NNN ». Une annonce par unité disponible. # ----------------------------------------------------------------------------- from __future__ import annotations import re from urllib.parse import unquote from bs4 import BeautifulSoup from ..schema import Listing, infer_city, normalize_unit_type, parse_price from .base import BaseConnector BASE = "https://www.appartementsurbains.ca" LIST_URL = f"{BASE}/projets-residentiels/" KNOWN_SECTORS = ("Montcalm", "Limoilou", "Lévis", "Ste-Foy", "Loretteville", "Sainte-Foy", "Saint-Roch", "Lebourgneuf") ADDR_RE = re.compile( r"\d{1,5}[^,<>|]{2,60},\s*(?:Lévis|Québec)\s*,?\s*(?:QC|Qu[ée]bec)?" r"[^.<>|]{0,15}[A-Z]\d[A-Z]\s?\d[A-Z]\d" ) # adresse plus permissive (sans code postal) pour la carte-projet wpgb ADDR_LOOSE_RE = re.compile( r"\d{1,5}[^,<>|]{2,60},\s*(?:Lévis|Québec)\b[^.<>|]{0,25}") AREA_RE = re.compile(r"(\d[\d\s]{1,6})\s*pi[²2]", re.I) IMG_RE = re.compile(r'https?://[^"\s\\)]+/wp-content/uploads/[^"\s\\)]+' r"\.(?:jpe?g|png|webp)", re.I) BAD_IMG_RE = re.compile(r"favicon|logo|icon|cropped-|social|plugins|hqdefault|" r"pdf|texture", re.I) TYPE_RE = re.compile(r"\d\s*½(?:\s*\+\s*\w+)?|\d\s*1/2|Studio|Loft", re.I) class AppartementsUrbainsConnector(BaseConnector): source_id = "app_urbains" request_delay = 0.6 max_projects = 20 # garde-fou de crawl def _discover_projects(self) -> dict[str, dict]: """Menu de navigation : « Nom du projet » + « Quartier ».""" projects: dict[str, dict] = {} html = self.get(LIST_URL).text soup = BeautifulSoup(html, "html.parser") for a in soup.select(f'a[href^="{BASE}/"], a[href^="/"]'): href = a.get("href") or "" if href.startswith("/"): href = BASE + href href = href.split("?")[0].rstrip("/") slug = href.replace(BASE, "").strip("/") # les pages projets sont à la racine (un seul segment, pas de section connue) if (not slug or "/" in slug or slug in ("projets-residentiels", "nouveaux-projets", "quartiers", "a-propos", "actualites", "carrieres", "nous-joindre", "html-sitemap", "projets-commerciaux")): continue text = re.sub(r"\s+", " ", a.get_text(" ", strip=True)) sector = "" for s in KNOWN_SECTORS: if re.search(rf"{re.escape(s)}\s*$", text): sector = s break if not sector: # lien sans étiquette de quartier -> ignorer continue name = text[: -len(sector)].strip() if slug not in projects and name: projects[slug] = {"name": name, "sector": sector} return projects def fetch(self) -> list[Listing]: listings: list[Listing] = [] try: projects = self._discover_projects() except Exception: return listings for i, (slug, meta) in enumerate(projects.items()): if i >= self.max_projects: break url = f"{BASE}/{slug}/" try: html = self.get(url).text except Exception: continue soup = BeautifulSoup(html, "html.parser") text = re.sub(r"\s+", " ", soup.get_text(" ", strip=True)) # Adresse : de préférence la carte wpgb du projet lui-même # (la 1re adresse de la page peut être un point d'intérêt voisin) address = "" for card in soup.select(".wpgb-card"): card_text = re.sub(r"\s+", " ", card.get_text(" ", strip=True)) if card_text.lower().startswith(meta["name"].lower()): am_ = ADDR_RE.search(card_text) or \ ADDR_LOOSE_RE.search(card_text) if am_: address = am_.group(0).strip(" ,") break if not address: m = ADDR_RE.search(text) address = m.group(0).strip() if m else "" sector = meta["sector"] city = infer_city(sector) if "Lévis" in address: city = "Lévis" # Images du projet (souvent servies depuis le domaine du projet) images = [u for u in dict.fromkeys( IMG_RE.findall(html) + re.findall(r'data-lazy-src="([^"]+\.(?:jpe?g|png|webp))"', html, re.I)) if not BAD_IMG_RE.search(u)] images = [u if u.startswith("http") else BASE + u for u in images][:25] og = soup.find("meta", attrs={"property": "og:description"}) desc = og["content"].strip()[:600] if og and og.get("content") else "" # Commodités (blocs « avantages » — deux générations de gabarits) amenities = list(dict.fromkeys( [re.sub(r"\s+", " ", h.get_text(" ", strip=True)) for h in soup.select(".listing-avantages h4") if h.get_text(strip=True)] + [s.get_text(strip=True) for s in soup.select("div.ct-div-block.c-center span.ct-span") if s.get_text(strip=True)]))[:15] # Téléphone du projet (lien tel: structuré) -> details.contact details: dict = {} tel = soup.select_one('a[href^="tel:"]') if tel: digits = re.sub(r"\D", "", unquote(tel.get("href", "")))[-10:] if len(digits) == 10: details["contact"] = {"phone": "-".join( (digits[:3], digits[3:6], digits[6:]))} # Tableau des unités disponibles seen: set[str] = set() for a in soup.select('a[href*="_unite="]'): try: num = a.get_text(strip=True) if not num or num in seen: continue seen.add(num) row = a row_text = "" for _ in range(4): row = row.parent if row is None: break cls = " ".join(row.get("class") or []) if "c-full-width" in cls and "flex-row" in cls: row_text = re.sub(r"\s+", " ", row.get_text(" ", strip=True)) break tm = TYPE_RE.search(row_text) unit_type = tm.group(0) if tm else "" pm = re.search(r"Prix\s*([\d\s]+\$)", row_text) price_label = pm.group(1).strip() if pm else "" am = re.search( r"Disponibilité\s+(.*?)(?:\s+Superficie|\s+Prix|$)", row_text) availability = am.group(1).strip() if am else "" # Superficie de l'unité (colonne « Superficie (pi²) ») : # texte brut dans amenities — finalize() dérive area_sqft unit_amenities = amenities sqm = AREA_RE.search(row_text) if sqm: unit_amenities = amenities + [ f"{sqm.group(1).strip()} pi²"] listings.append(Listing( source=self.source_id, external_id=f"{slug}-{num}", url=url, title=f"{meta['name']} — unité {num}", address=address, sector=sector, city=city, unit_type=normalize_unit_type(unit_type), price=parse_price(price_label), price_label=price_label, availability=availability, description=desc, amenities=unit_amenities, details=dict(details), images=images, )) except Exception: continue return listings