spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/firma.py : connecteur Groupe Firma (groupefirma.ca)5# Condos locatifs en Montérégie / banlieue sud-ouest de Montréal6# (Salaberry-de-Valleyfield, Saint-Zotique, Sainte-Barbe, L'Île-Perrot...).7# Découverte des immeubles via l'API REST WordPress (post type "housing"),8# puis parsing des fiches /logement/<slug>/ : unités <li id="unit-NNN">9# avec type, disponibilité, étage, superficie, commodités (icônes bjm-active)10# et images (plan + galerie de l'unité). Le prix est présent dans un11# commentaire HTML "<!-- - 1380$ / mois -->" (souvent vide).12# Lachute (Laurentides, hors Grand Montréal) est exclue.13# -----------------------------------------------------------------------------14from __future__ import annotations1516import html as htmllib17import re1819from bs4 import BeautifulSoup2021from ..schema import Listing, normalize_unit_type, parse_price, strip_accents22from .base import BaseConnector2324BASE = "https://groupefirma.ca"25API_URL = f"{BASE}/wp-json/wp/v2/housing?per_page=100"2627# Municipalités admissibles (Montérégie sud-ouest / Grand Montréal).28_ALLOWED_CITY_KEYS = (29 "valleyfield", "saint-zotique", "st-zotique", "sainte-barbe", "ste-barbe",30 "saint-louis-de-gonzague", "st-louis-de-gonzague", "ile-perrot",31 "saint-timothee", "st-timothee", "les-coteaux", "coteau-du-lac",32 "vaudreuil", "pincourt", "beauharnois", "chateauguay",33)3435# Normalisation des noms de villes rencontrés dans les adresses.36_CITY_CANON = {37 "valleyfield": "Salaberry-de-Valleyfield",38 "salaberry-de-valleyfield": "Salaberry-de-Valleyfield",39 "saint-zotique": "Saint-Zotique",40 "st-zotique": "Saint-Zotique",41 "ste-barbe": "Sainte-Barbe",42 "sainte-barbe": "Sainte-Barbe",43 "st-louis-de-gonzague": "Saint-Louis-de-Gonzague",44 "saint-louis-de-gonzague": "Saint-Louis-de-Gonzague",45 "st-timothee": "Saint-Timothée",46 "saint-timothee": "Saint-Timothée",47 "l'ile-perrot": "L'Île-Perrot",48 "lile-perrot": "L'Île-Perrot",49 "ile-perrot": "L'Île-Perrot",50}5152_PRICE_COMMENT_RE = re.compile(r"<!--\s*-?\s*([\d\s,.]*)\$\s*/\s*mois\s*-->")53_IMG_RE = re.compile(r"\.(?:jpg|jpeg|png|webp)(?:$|\?)", re.I)54_ICON_RE = re.compile(r"-ico\.|icon|logo|favicon", re.I)555657def _canon_city(raw: str) -> str:58 key = strip_accents(raw.strip().lower()).replace("’", "'")59 key = re.sub(r"\s+qc\.?$", "", key).strip(" ,")60 return _CITY_CANON.get(key, raw.strip(" ,"))616263def _city_allowed(raw: str) -> bool:64 key = strip_accents(raw.strip().lower()).replace(" ", "-").replace("’", "'")65 return any(tok in key for tok in _ALLOWED_CITY_KEYS)666768class FirmaConnector(BaseConnector):69 source_id = "firma"70 request_delay = 0.671 max_buildings = 40 # garde-fou de crawl7273 def fetch(self) -> list[Listing]:74 listings: list[Listing] = []75 try:76 buildings = self.get(API_URL).json()77 except Exception:78 return listings79 if not isinstance(buildings, list):80 return listings8182 for i, b in enumerate(buildings):83 if i >= self.max_buildings:84 break85 try:86 listings.extend(self._parse_building(b))87 except Exception:88 continue89 return listings9091 # -- une fiche d'immeuble --------------------------------------------------92 def _parse_building(self, b: dict) -> list[Listing]:93 slug = b.get("slug") or ""94 url = b.get("link") or f"{BASE}/logement/{slug}/"95 name = htmllib.unescape((b.get("title") or {}).get("rendered") or slug)96 name = re.sub(r"\s+", " ", name).strip()9798 html = self.get(url).text99 soup = BeautifulSoup(html, "html.parser")100101 # Bloc d'en-tête : nom + adresse civique de l'immeuble102 address = ""103 addr_el = soup.select_one(".bjm-singlehousing-details-bluebox-address")104 if addr_el:105 address = addr_el.get_text(" ", strip=True)106 city_raw = address.split(",")[-1].strip() if "," in address else ""107 if not city_raw or not _city_allowed(city_raw):108 return [] # hors Grand Montréal (ex. Lachute)109 city = _canon_city(city_raw)110111 desc_el = soup.select_one(".bjm-singlehousing-details-description")112 description = (desc_el.get_text(" ", strip=True)[:600] if desc_el else "")113114 # Commodités de l'immeuble (icônes actives du bloc de détails)115 building_amenities: list[str] = []116 for item in soup.select(".bjm-singlehousing-details-icons-item"):117 img = item.select_one("img.bjm-active")118 if img:119 label = item.get_text(" ", strip=True)120 if label:121 building_amenities.append(label)122123 # Images de secours : galerie « photos du projet »124 fallback_imgs = self._section_images(soup.find(id="bjm-singlehousing-photos"))125126 results: list[Listing] = []127 for li in soup.select('li[id^="unit-"]'):128 lst = self._parse_unit(li, slug, url, name, address, city,129 description, building_amenities,130 fallback_imgs)131 if lst:132 results.append(lst)133 return results134135 # -- une unité (li id="unit-NNN") -----------------------------------------136 def _parse_unit(self, li, slug: str, url: str, name: str, address: str,137 city: str, description: str, building_amenities: list[str],138 fallback_imgs: list[str]) -> Listing | None:139 unit_no = (li.get("id") or "").replace("unit-", "").strip()140 if not unit_no:141 return None142143 num_el = li.select_one(".bjm-unit-number")144 label = num_el.get_text(strip=True).lstrip("#") if num_el else unit_no145146 h3 = li.find("h3")147 type_raw = h3.get_text(" ", strip=True) if h3 else ""148 unit_type = normalize_unit_type(type_raw)149150 # Prix : masqué dans un commentaire HTML "<!-- - 1380$ / mois -->"151 price = None152 price_label = ""153 m = _PRICE_COMMENT_RE.search(str(li))154 if m and re.search(r"\d", m.group(1)):155 price_label = f"{m.group(1).strip()}$ / mois"156 price = parse_price(price_label)157158 # Disponibilité : badge + date du bloc de texte159 avail_el = li.select_one(".bjm-det-label")160 availability = avail_el.get_text(" ", strip=True) if avail_el else ""161 details_txt = li.get_text(" ", strip=True)162 dm = re.search(r"Disponibilit[ée]\s*:\s*([\d/]+)", details_txt)163 if dm:164 availability = (f"{availability} ({dm.group(1)})"165 if availability else dm.group(1))166167 # Champs structurés de la fiche d'unité : étage, superficies (pi²)168 details: dict = {}169 area_sqft = None170 extras = []171 fm = re.search(r"[ÉE]tage\s*:\s*([^\s].{0,30}?)(?:\s{2,}|Superficie)",172 details_txt)173 if fm:174 floor_txt = fm.group(1).strip()175 extras.append(f"Étage : {floor_txt}")176 if floor_txt.isdigit() and 0 < int(floor_txt) <= 60:177 details["floor"] = int(floor_txt)178 sm = re.search(r"Superficie\s*:\s*(\d[\d\s.,]*)", details_txt)179 if sm:180 extras.append(f"Superficie : {sm.group(1).strip()} pi²")181 try:182 v = float(sm.group(1).strip().replace(" ", "")183 .replace(",", "."))184 if 80 <= v <= 20000:185 # explicite : sinon le parsing générique prendrait le min186 # avec la superficie du balcon ci-dessous187 area_sqft = v188 except ValueError:189 pass190 bm = re.search(r"Superficie du balcon\s*:\s*(\d[\d\s.,]*)", details_txt)191 if bm:192 extras.append(f"Superficie du balcon : {bm.group(1).strip()} pi²")193 unit_desc = " | ".join(extras)194 full_desc = " — ".join(p for p in (unit_desc, description) if p)[:600]195196 # Commodités de l'unité : icônes actives seulement197 amenities = [img.get("alt", "").strip()198 for img in li.select("img.bjm-active[alt]")199 if img.get("alt", "").strip()]200 amenities = list(dict.fromkeys(amenities + building_amenities))201202 # Images : plan + galerie propres à l'unité, sinon photos du projet203 images = self._section_images(li)204 if not images:205 images = fallback_imgs206207 return Listing(208 source=self.source_id,209 external_id=f"{slug}-{unit_no}",210 url=url,211 title=f"{name} — {unit_type or 'unité'} #{label}".strip(),212 address=address,213 sector="",214 city=city,215 unit_type=unit_type,216 price=price,217 price_label=price_label,218 availability=availability,219 area_sqft=area_sqft,220 description=full_desc,221 amenities=amenities,222 details=details,223 images=images,224 )225226 # -- images d'une section (liens lightbox + <img>) -------------------------227 @staticmethod228 def _section_images(node) -> list[str]:229 if node is None:230 return []231 urls: list[str] = []232 for a in node.select("a[href]"):233 href = a.get("href") or ""234 if _IMG_RE.search(href):235 urls.append(href)236 for img in node.select("img[src]"):237 src = img.get("src") or ""238 if _IMG_RE.search(src) and "uploads" in src:239 urls.append(src)240 out: list[str] = []241 for u in urls:242 if not u.startswith("http"):243 u = BASE + u244 if not _ICON_RE.search(u):245 out.append(u)246 return list(dict.fromkeys(out))[:25]247