# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/groupe_mathieu.py : connecteur Groupe Mathieu (groupemathieu.com) # Promoteur Rive-Nord : condos locatifs à Blainville (Station 54/56, # Le Siège Social, Place Notre Dame), Sainte-Thérèse (Ste-Thé, Quartier # Belmont), Terrebonne (Noüvo District) et Laval. WordPress/Elementor rendu # serveur : le méga-menu « PROJETS À LOUER » liste les projets par ville ; # chaque page projet affiche l'adresse, « Occupation … » et des blocs # typologie (« 3 ½ », description pi²/chambres, « À partir de N$ par mois »). # Granularité : une annonce par TYPOLOGIE de projet (sinon une par projet # quand la page ne détaille pas les typologies). Les projets « 100 % vendu/ # loué » sont exclus. # ----------------------------------------------------------------------------- from __future__ import annotations import re from bs4 import BeautifulSoup from ..schema import Listing, normalize_unit_type from .base import BaseConnector BASE = "https://groupemathieu.com" UNIT_RE = re.compile(r"^\s*(\d)\s*½\s*$") FROM_RE = re.compile(r"[ÀA]\s*partir\s*de\s*([\d\s ,]+)\s*\$\s*(?:/|par)?" r"\s*mois", re.I) AREA_RE = re.compile(r"de\s*([\d\s ]+)\s*(?:à\s*[\d\s ]+\s*)?pi", re.I) BED_RE = re.compile(r"(\d)\s*chambres?\b", re.I) ADDR_RE = re.compile( r"\d{1,5}[^,<>\n]{3,60},?\s*(?:)?\s*" r"[A-Za-zÀ-ÿ\- ]{3,30},?\s*\(Qu[ée]bec\)\s*[A-Z]\d[A-Z]\s?\d[A-Z]\d") OCC_RE = re.compile(r"Occupation\s+[^<\n]{2,40}", re.I) SOLD_RE = re.compile(r"100\s*%\s*(?:vendu|lou[ée])", re.I) IMG_RE = re.compile(r"https://groupemathieu\.com/wp-content/uploads/" r'[^"\s\\)]+\.(?:jpe?g|png|webp)', re.I) class GroupeMathieuConnector(BaseConnector): source_id = "groupe_mathieu" request_delay = 0.7 def fetch(self) -> list[Listing]: listings: list[Listing] = [] projects: list[tuple[str, str, str]] = [] # (nom, url, ville) # Découverte primaire : la page /condos-locatifs/ (rendue serveur, # stable) liste tous les projets locatifs via des cartes # « En savoir plus ». Le méga-menu Jet de l'accueil n'est servi que # par certaines variantes de cache Elementor → repli seulement. try: lhtml = self.get(BASE + "/condos-locatifs/").text lsoup = BeautifulSoup(lhtml, "html.parser") for a in lsoup.find_all("a", href=True): if "en savoir plus" not in a.get_text(" ", strip=True).lower(): continue href = a["href"].split("?")[0] if href.startswith("/"): href = BASE + href if "groupemathieu.com" not in href or "promo" in href: continue projects.append(("", href, "")) except Exception: pass if not projects: projects = self._projects_from_menu() seen: set[str] = set() for name, url, ville in projects: slug = url.rstrip("/").split("/")[-1] if not slug or slug in seen: continue seen.add(slug) try: listings.extend(self._fetch_project(name, url, slug, ville)) except Exception: continue return listings def _projects_from_menu(self) -> list[tuple[str, str, str]]: # méga-menu « PROJETS À LOUER » : colonnes ville (h5) + items projets try: html = self.get(BASE + "/").text except Exception: return [] soup = BeautifulSoup(html, "html.parser") menu = None for title in soup.select(".jet-menu-title, .jet-menu-item"): if "PROJETS À LOUER" in title.get_text(" ", strip=True).upper(): menu = title if title.name == "li" else \ title.find_parent("li") break if menu is None: return [] sub = menu.select_one(".jet-sub-mega-menu") if sub is None: return [] projects: list[tuple[str, str, str]] = [] city = "" for el in sub.select("h5.elementor-heading-title," " li.elementor-icon-list-item a"): if el.name == "h5": city = el.get_text(" ", strip=True).title() continue label_el = el.select_one(".elementor-icon-list-text") label = label_el.get_text(" ", strip=True) if label_el else "" if "locatif" not in label.lower(): continue href = el.get("href", "") if href.startswith("/"): href = BASE + href strong = el.find("strong") name = label.replace(strong.get_text(strip=True), "").strip() \ if strong else label if href: projects.append((name or href, href.split("?")[0], city)) return projects seen: set[str] = set() for name, url, ville in projects: slug = url.rstrip("/").split("/")[-1] if not slug or slug in seen: continue seen.add(slug) try: listings.extend(self._fetch_project(name, url, slug, ville)) except Exception: continue return listings def _fetch_project(self, name: str, url: str, slug: str, city: str) -> list[Listing]: html = self.get(url).text if SOLD_RE.search(html): return [] # projet complet (ex. Vëe « 100% vendu ») soup = BeautifulSoup(html, "html.parser") text = soup.get_text("\n", strip=True) address = "" m = ADDR_RE.search(text.replace("\n", ", ")) if m: address = re.sub(r"\s+", " ", m.group(0)).replace(" ,", ",") if not city and address: # découverte via /condos-locatifs/ : ville m = re.search(r",\s*([A-Za-zÀ-ÿ\- ']{3,30}),?\s*\(Qu", address) if m: city = m.group(1).strip().title() if not name: h1 = soup.find("h1") t = h1.get_text(" ", strip=True) if h1 else "" if not t and soup.title: t = soup.title.get_text(strip=True).split("|")[0].strip() t = re.sub(r"\s*[-–|]\s*Groupe Mathieu\s*$", "", t, flags=re.I) name = re.sub(r"\s+", " ", t) or slug availability = "" m = OCC_RE.search(text) if m: availability = re.sub(r"\s+", " ", m.group(0)).strip() rented = "" m = re.search(r"LOU[ÉE] À\s*\n?\s*(\d{1,3}\s*%)", text, re.I) if m: rented = m.group(1).replace(" ", "") images = [u for u in dict.fromkeys(IMG_RE.findall(html)) if not re.search(r"logo|icon|favicon|plan|-\d{2,3}x\d{2,3}\.", u, re.I)][:20] # blocs typologie, en ordre du document (dédoublés desktop/mobile) units: dict[str, dict] = {} current: str | None = None for el in soup.select("h2.elementor-heading-title," " .elementor-widget-text-editor p"): t = re.sub(r"\s+", " ", el.get_text(" ", strip=True)) if not t: continue m = UNIT_RE.match(t) if m: ut = f"{m.group(1)}½" if ut in units: current = None # doublon (version mobile) else: units[ut] = {} current = ut continue if current is None: continue u = units[current] m = FROM_RE.search(t) if m and "price" not in u: try: u["price"] = float(m.group(1).replace(" ", "") .replace(" ", "").replace(" ", "") .replace(",", "")) u["price_label"] = re.sub(r"\s+", " ", m.group(0)) except ValueError: pass current = None # bloc complet continue if el.name == "p" and len(t) > 40 and "desc" not in u: u["desc"] = t[:400] m = AREA_RE.search(t) if m: try: u["area"] = float(m.group(1).replace(" ", "") .replace(" ", "").replace(" ", "")) except ValueError: pass m = BED_RE.search(t) if m: u["beds"] = float(m.group(1)) details: dict = {} if rented: details["taux_loue"] = rented out: list[Listing] = [] if units: for ut, u in units.items(): out.append(Listing( source=self.source_id, external_id=f"{slug}-{ut.replace('½', '.5')}", url=url, title=f"{ut} — {name} ({city})", address=address, city=city, unit_type=normalize_unit_type(ut), bedrooms=u.get("beds"), price=u.get("price"), price_label=u.get("price_label", ""), availability=availability, area_sqft=u.get("area"), description=u.get("desc", ""), details=dict(details), images=images, )) elif availability: # page sans blocs typologie (ex. Place Notre Dame) desc = " ".join(p.get_text(" ", strip=True) for p in soup.find_all("p") if len(p.get_text(strip=True)) > 80)[:900] out.append(Listing( source=self.source_id, external_id=slug, url=url, title=f"{name} — condos locatifs ({city})", address=address, city=city, availability=availability, description=re.sub(r"\s+", " ", desc), details=dict(details), images=images, )) return out