spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/roomies.py : Roomies.ca — chambres et colocation (segment que les5#   gestionnaires ne couvrent pas). Site Laravel/Livewire server-rendered,6#   robots.txt entièrement ouvert, aucun anti-bot (curl simple = 200).7#   Découverte par les pages villes `/{ville}-qc?page=N` (les sitemaps8#   `sitemap/rooms.xml` répondent 504 par intermittence — non fiables).9#   Fiche `/rooms/<id>` : JSON-LD `Room` complet (offers.price mensuel,10#   availabilityStarts, lat/lng, petsAllowed, chambres/SDB de l'appartement,11#   PostalAddress, amenityFeature) + description intégrale dans un div12#   `prose` + galerie Cloudinary. On ignore les profils de chercheurs de13#   colocataires (/profiles) : seules les chambres offertes sont des annonces.14# -----------------------------------------------------------------------------15from __future__ import annotations1617import datetime18import os19import re2021from bs4 import BeautifulSoup2223from ..schema import Listing24from .base import BaseConnector25from . import _detailutil as du2627BASE = "https://www.roomies.ca"2829# villes québécoises servies (slug -qc) ; une page absente (404) est ignorée30CITIES = {31    "montreal-qc": "Montréal", "quebec-qc": "Québec", "laval-qc": "Laval",32    "gatineau-qc": "Gatineau", "longueuil-qc": "Longueuil",33    "sherbrooke-qc": "Sherbrooke", "trois-rivieres-qc": "Trois-Rivières",34    "levis-qc": "Lévis", "brossard-qc": "Brossard",35    "terrebonne-qc": "Terrebonne", "repentigny-qc": "Repentigny",36    "saint-jean-sur-richelieu-qc": "Saint-Jean-sur-Richelieu",37    "drummondville-qc": "Drummondville", "granby-qc": "Granby",38    "saguenay-qc": "Saguenay",39}40MAX_PAGES = int(os.environ.get("LOUKA_ROOMIES_MAX_PAGES", "20"))41DETAIL_LIMIT = int(os.environ.get("LOUKA_ROOMIES_DETAIL_LIMIT", "250"))42TTL_DAYS = float(os.environ.get("LOUKA_ROOMIES_TTL_DAYS", "5"))4344_ROOM_LINK_RE = re.compile(r'href="(?:https://www\.roomies\.ca)?(/rooms/(\d+))"')45_PHOTO_RE = re.compile(46    r'https://cloudinary\.roomies\.pics/image/upload/[^"\\\s]+?/(\w+)')474849def _parse_fiche(html: str) -> dict:50    out: dict = {}51    for node in du.ld_nodes(html):52        if node.get("@type") != "Room":53            continue54        name = str(node.get("name") or "")55        out["type_label"] = name.split("|")[0].strip()   # « Furnished room in… »56        addr = node.get("address") or {}57        out["address"] = (addr.get("streetAddress") or "").strip()58        out["postal_code"] = (addr.get("postalCode") or "").strip()59        out["lat"] = node.get("latitude")60        out["lng"] = node.get("longitude")61        if node.get("petsAllowed") is not None:62            out["pets"] = "oui" if node["petsAllowed"] else "non"63        details = {}64        if node.get("numberOfBedrooms"):65            details["Chambres (logement)"] = str(node["numberOfBedrooms"])66        if node.get("numberOfBathroomsTotal"):67            details["Salles de bain"] = str(node["numberOfBathroomsTotal"])68        if details:69            out["details"] = details70        feats = node.get("amenityFeature") or []71        amenities = []72        for f in feats:73            if isinstance(f, dict) and f.get("name") and f.get("value") is not False:74                amenities.append(str(f["name"]))75                if f["name"] == "Furnished":76                    out["furnished"] = True77        if amenities:78            out["amenities"] = amenities79        offers = node.get("offers") or {}80        try:81            out["price"] = float(offers.get("price"))82        except (TypeError, ValueError):83            pass84        starts = str(offers.get("availabilityStarts") or "")[:10]85        if starts:86            out["availability_date"] = ("now" if starts87                                        <= datetime.date.today().isoformat()88                                        else starts)89        if node.get("description"):90            out["description"] = str(node["description"]).strip()[:6000]9192    # description intégrale (le JSON-LD est tronqué à ~250 caractères)93    soup = BeautifulSoup(html, "html.parser")94    prose = soup.find("div", class_=lambda c: c and "prose" in c95                      and "break-words" in c)96    if prose:97        full = prose.get_text("\n", strip=True)98        if len(full) > len(out.get("description") or ""):99            out["description"] = full[:6000]100101    # galerie Cloudinary : dédupliquée par identifiant public, format standard102    photos, seen = [], set()103    for pid in _PHOTO_RE.findall(html):104        if pid in seen or len(pid) < 12:105            continue106        seen.add(pid)107        photos.append("https://cloudinary.roomies.pics/image/upload/"108                      f"c_fill,f_jpg,fl_lossy,g_auto,h_512,q_auto:good,w_896/{pid}")109    if photos:110        out["images"] = photos111    return out112113114class RoomiesConnector(BaseConnector):115    source_id = "roomies"116    request_delay = 0.8117118    def _city_room_ids(self, slug: str) -> list[str]:119        ids, page = [], 1120        while page <= MAX_PAGES:121            url = f"{BASE}/{slug}" + (f"?page={page}" if page > 1 else "")122            try:123                html = self.get(url).text124            except Exception:125                break126            found = [rid for _, rid in _ROOM_LINK_RE.findall(html)]127            fresh = [r for r in found if r not in ids]128            if not fresh:129                break130            ids.extend(fresh)131            page += 1132        return ids133134    def fetch(self) -> list[Listing]:135        cache = du.TtlDetailCache(self, budget=DETAIL_LIMIT, ttl_days=TTL_DAYS)136        out: dict[str, Listing] = {}137        try:138            for slug, city in CITIES.items():139                for rid in self._city_room_ids(slug):140                    if rid in out:141                        continue142                    url = f"{BASE}/rooms/{rid}"143                    payload = cache.get(rid, url, _parse_fiche)144                    if not payload or payload.get("gone"):145                        continue146                    price = payload.get("price")147                    avail = payload.get("availability_date")148                    type_label = (payload.get("type_label") or "").lower()149                    unit_type = ("Studio" if "studio" in type_label150                                 and "shared" not in type_label else "Chambre")151                    lst = Listing(152                        source=self.source_id,153                        external_id=rid,154                        url=url,155                        title=(payload.get("type_label") or "Chambre")156                        + (f" — {payload['address']}" if payload.get("address") else ""),157                        address=payload.get("address") or "",158                        city=city,159                        unit_type=unit_type,160                        price=price,161                        price_label=(f"{price:,.0f} $/mois".replace(",", " ")162                                     if price else ""),163                        availability_date=avail,164                        availability=("Libre immédiatement" if avail == "now"165                                      else f"Libre le {avail}" if avail else ""),166                        description=payload.get("description") or "",167                        amenities=payload.get("amenities") or [],168                        details=payload.get("details") or {},169                        images=payload.get("images") or [],170                        lat=payload.get("lat"),171                        lng=payload.get("lng"),172                    )173                    if payload.get("postal_code"):174                        lst.details["Code postal"] = payload["postal_code"]175                    if payload.get("pets"):176                        lst.pets = payload["pets"]177                    if payload.get("furnished"):178                        lst.furnished = True179                    out[rid] = lst180        finally:181            cache.close()182        return list(out.values())183