spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/dupin_despres.py : connecteur Dupin Després (dupindespres.com)5#   Gestionnaire de Repentigny / Rive-Nord (Les Cimes Repentigny, Quartier du6#   Moulin, immeubles rue Notre-Dame…). WordPress (thème d'agence « Voyou »,7#   classes vy_*) rendu serveur : /appartements-a-louer + /page/N liste des8#   cartes par UNITÉ (titre, typologie, ville, prix, bandeau de disponibilité,9#   badge 55 ans+). La page détail fournit description, photos et l'adresse10#   civique (fil d'Ariane JSON-LD, ex. « #118-325 rue Notre-Dame Repentigny »).11#   Granularité : unité.12# -----------------------------------------------------------------------------13from __future__ import annotations1415import json16import re1718from bs4 import BeautifulSoup1920from ..schema import Listing, normalize_unit_type21from .base import BaseConnector2223BASE = "https://dupindespres.com"24LIST_URL = f"{BASE}/appartements-a-louer"25MAX_PAGES = 102627IMG_RE = re.compile(28    r"https://dupindespres\.com/app/uploads/[^\"'\s\\]+\.(?:jpe?g|png|webp)", re.I)29SKIP_IMG_RE = re.compile(r"logo|favicon|icon|-\d{2,4}x\d{2,4}\.", re.I)30PHONE_RE = re.compile(r"(\d{3})\s*(\d{3})[\s-]*(\d{4})")31# dernier maillon du fil d'Ariane JSON-LD : « #118-325 rue Notre-Dame Repentigny »32CRUMB_RE = re.compile(r'"name"\s*:\s*"(#[^"]+)"')333435class DupinDespresConnector(BaseConnector):36    source_id = "dupin_despres"37    request_delay = 0.838    max_images = 123940    # -- page détail (cachée en BD via self.detail) ---------------------------41    def _detail(self, url: str) -> dict:42        try:43            html = self.get(url).text44        except Exception:45            return {}46        soup = BeautifulSoup(html, "html.parser")47        payload: dict = {}4849        # adresse : dernier maillon du fil d'Ariane JSON-LD (« #118-325 rue … »)50        for script in soup.find_all("script", type="application/ld+json"):51            m = CRUMB_RE.findall(script.string or "")52            if m:53                addr = json.loads(f'"{m[-1]}"') if "\\" in m[-1] else m[-1]54                # « #118-325 rue Notre-Dame Repentigny » -> « 325 rue Notre-Dame »55                addr = re.sub(r"^#?\d+\s*-\s*", "", addr).strip()56                payload["address"] = addr57                break5859        # description : paragraphes du corps de la fiche60        body = soup.select_one(".vy_main_body")61        if body is not None:62            paras = [re.sub(r"\s+", " ", p.get_text(" ", strip=True))63                     for p in body.find_all("p")]64            desc = " ".join(t for t in paras if len(t) > 40)65            if desc:66                payload["description"] = desc[:1500]6768        # contact (téléphone affiché sur la fiche)69        m = re.search(r'href="tel:\+?1?(\d{10})"', html)70        if m:71            d = m.group(1)72            payload["phone"] = f"{d[:3]}-{d[3:6]}-{d[6:]}"7374        # photos de la fiche75        imgs = [u for u in dict.fromkeys(IMG_RE.findall(html))76                if not SKIP_IMG_RE.search(u)]77        payload["images"] = imgs[: self.max_images]78        return payload7980    # -- fetch -----------------------------------------------------------------81    def fetch(self) -> list[Listing]:82        listings: list[Listing] = []83        seen: set[str] = set()8485        for page in range(1, MAX_PAGES + 1):86            url = LIST_URL if page == 1 else f"{LIST_URL}/page/{page}"87            try:88                html = self.get(url).text89            except Exception:90                break91            soup = BeautifulSoup(html, "html.parser")92            items = soup.select(".vy_rentals_listing_item")93            if not items:94                break9596            new_on_page = 097            for it in items:98                link = it.select_one("a.vy_link_cover[href]")99                if link is None:100                    continue101                href = link["href"].split("?")[0].rstrip("/")102                slug = href.rsplit("/", 1)[-1]103                if not slug or "/page/" in href or slug in seen:104                    continue105                seen.add(slug)106                new_on_page += 1107108                title = ""109                el = it.select_one(".vy_rentals_listing_item_title")110                if el is not None:111                    title = re.sub(r"\s+", " ", el.get_text(" ", strip=True))112113                # <span>4 ½</span> | <span>Repentigny</span> — certaines cartes114                # n'ont pas de span typologie (ex. Les Cimes Saint-Sulpice)115                unit_type = city = ""116                info = it.select_one(".vy_rentals_listing_item_info")117                if info is not None:118                    spans = [s.get_text(" ", strip=True)119                             for s in info.find_all("span")]120                    for s in (s for s in spans if s):121                        if re.search(r"½|1/2|studio|loft|chambre", s, re.I):122                            unit_type = normalize_unit_type(s)123                        else:124                            city = s125126                price = None127                price_label = ""128                el = it.select_one(".vy_rentals_listing_item_price")129                if el is not None:130                    price_label = el.get_text(" ", strip=True)131                    m = re.search(r"(\d[\d\s,]*)\s*\$", price_label)132                    if m:133                        try:134                            val = float(m.group(1).replace(" ", "")135                                        .replace(" ", "").replace(",", ""))136                            if 300 <= val <= 20000:137                                price = val138                        except ValueError:139                            pass140141                availability = ""142                el = it.select_one(".vy_bannerinfo_text")143                if el is not None:144                    availability = el.get_text(" ", strip=True)145146                amenities: list[str] = []147                if it.select_one(".vy_badge.--fiftyfive") is not None:148                    amenities.append("55 ans et plus")149150                card_img = ""151                img = it.select_one("img[data-src]")152                if img is not None:153                    card_img = img.get("data-src") or ""154155                key = f"{unit_type}|{city}|{price_label}|{availability}"156                det = self.detail(slug, key, lambda u=href: self._detail(u))157158                images = list(det.get("images") or [])159                if card_img and card_img not in images:160                    images.insert(0, card_img)161                details: dict = {}162                if det.get("phone"):163                    details["contact"] = {"phone": det["phone"]}164165                # adresse : fiche détail, sinon dérivée du slug de l'URL166                address = det.get("address") or ""167                if not address:168                    m = re.match(r"^\d+-(.+?)(?:-repentigny|-terrebonne)?$", slug)169                    if m and re.search(r"[a-z]{3}", m.group(1)):170                        address = m.group(1).replace("-", " ")171172                listings.append(Listing(173                    source=self.source_id,174                    external_id=slug,175                    url=href,176                    title=title or f"Appartement {unit_type}",177                    address=address,178                    city=city,179                    unit_type=unit_type,180                    price=price,181                    price_label=price_label,182                    availability=availability,183                    description=det.get("description", ""),184                    amenities=amenities,185                    details=details,186                    images=images[: self.max_images],187                ))188            if new_on_page == 0:189                break190        return listings191