spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/werkliv.py : connecteur Werkliv (logement étudiant)5#   werkliv.com est le site corporatif ; la location de ses immeubles passe6#   par sa plateforme University Apartments (universityapartments.ca —7#   WordPress + FacetWP rendu serveur). Immeubles montréalais : Palay8#   (2025 rue Peel, centre-ville) et Le Mojave (3476 rue Saint-Dominique,9#   Plateau/Milton-Parc). Une annonce par typologie (1-BEDROOM, 4-BEDROOM...),10#   loyer par personne (colocation étudiante meublée).11# -----------------------------------------------------------------------------12from __future__ import annotations1314import hashlib15import re1617from bs4 import BeautifulSoup1819from ..schema import Listing20from .base import BaseConnector2122BASE = "https://universityapartments.ca"23LIST_URL = f"{BASE}/apartment-listings/?_listings_city_en=montreal"2425# Immeuble (nom affiché) -> (slug fiche immeuble, secteur)26BUILDINGS = {27    "palay": ("palay", "Centre-ville (Ville-Marie)"),28    "le mojave": ("le-mojave", "Milton-Parc / Plateau-Mont-Royal"),29    "mojave": ("le-mojave", "Milton-Parc / Plateau-Mont-Royal"),30}3132ADDR_RE = re.compile(33    r"\d{2,5}[^<>\"|]{2,60}?(?:Montr[ée]al)[,\s]+QC(?:\s+[A-Z]\d[A-Z]\s?\d[A-Z]\d)?")3435# Émojis/puces en tête des items de commodités des fiches36_EMOJI_PREFIX_RE = re.compile(r"^[\W_]+", re.UNICODE)373839def _unit_type(label: str) -> str:40    """'1-BEDROOM' -> 3½, '4-BEDROOM' -> 6½, 'STUDIO' -> Studio."""41    s = (label or "").lower()42    if "studio" in s:43        return "Studio"44    m = re.search(r"(\d+)", s)45    if m:46        n = int(m.group(1))47        return "Studio" if n == 0 else f"{n + 2}½"48    return label.strip()495051class WerklivConnector(BaseConnector):52    source_id = "werkliv"53    request_delay = 0.654    max_details = 20         # garde-fou5556    def fetch(self) -> list[Listing]:57        listings: list[Listing] = []58        try:59            html = self.get(LIST_URL).text60        except Exception:61            return listings62        soup = BeautifulSoup(html, "html.parser")6364        addresses: dict[str, str] = {}     # slug immeuble -> adresse65        seen: set[str] = set()66        for card in soup.select(".lcl-card"):67            try:68                a = card.select_one('a.lcl-link[href*="/listings/"]') or \69                    card.select_one('a[href*="/listings/"]')70                if not a:71                    continue72                url = (a.get("href") or "").split("?")[0]73                m = re.search(r"/listings/([^/]+)/?$", url)74                if not m or m.group(1) in seen:75                    continue76                slug = m.group(1)77                seen.add(slug)7879                city_el = card.select_one(".lcl-city")80                city_raw = city_el.get_text(" ", strip=True) if city_el else ""81                if "montreal" not in city_raw.lower():82                    continue         # hors Montréal (Halifax, PEI...)8384                typo_el = card.select_one(".lcl-title .h4, .lcl-title")85                typology = typo_el.get_text(" ", strip=True) if typo_el else ""86                prop_el = card.select_one(".lcl-property")87                building = prop_el.get_text(" ", strip=True) if prop_el else ""88                price_el = card.select_one(".lcl-price")89                price_label = price_el.get_text(" ", strip=True) \90                    if price_el else ""91                price = None92                pm = re.search(r"\$\s*([\d,]+)", price_label)93                if pm:94                    try:95                        v = float(pm.group(1).replace(",", ""))96                        price = v if 100 <= v <= 20000 else None97                    except ValueError:98                        pass99100                avail = ""101                av_el = card.select_one(".lcl-available span")102                if av_el:103                    avail = re.sub(r"\s+", " ",104                                   av_el.get_text(" ", strip=True))105                amenities = ["Logement étudiant"]106                for chip in card.select(".lcl-chip"):107                    amenities.append(re.sub(r"\s+", " ",108                                            chip.get_text(" ", strip=True)))109110                bslug, sector = BUILDINGS.get(building.strip().lower(),111                                              ("", ""))112                # adresse depuis la fiche de l'immeuble (mise en cache)113                address = ""114                if bslug:115                    if bslug not in addresses:116                        addresses[bslug] = self._building_address(bslug)117                    address = addresses[bslug]118119                images = []120                header = card.select_one("[data-bg]")121                if header and header.get("data-bg", "").startswith("http"):122                    images.append(header["data-bg"])123124                listings.append(Listing(125                    source=self.source_id,126                    external_id=slug,127                    url=url,128                    title=f"{building} — {typology} (par chambre)",129                    address=address,130                    sector=sector,131                    city="Montréal",132                    unit_type=_unit_type(typology),133                    price=price,134                    price_label=f"{price_label} (par personne)"135                                if price_label else "",136                    availability=avail,137                    amenities=list(dict.fromkeys(amenities)),138                    images=images,139                ))140            except Exception:141                continue142143        # Fiches détaillées (photos, description, dispo ACF, bail, commodités)144        # via le cache BD : 1 vraie requête par annonce et par changement.145        self._detail_requests = 0146        for i, lst in enumerate(listings):147            if i >= self.max_details:148                break149            key = hashlib.sha1(150                f"{lst.title}|{lst.price_label}|{lst.availability}"151                f"|{'|'.join(lst.amenities)}".encode("utf-8")).hexdigest()152153            def _fetch(url=lst.url) -> dict:154                if self._detail_requests >= self.max_details:155                    return {}156                self._detail_requests += 1157                return self._fetch_detail(url)158159            try:160                payload = self.detail(lst.external_id, key, _fetch) or {}161            except Exception:162                payload = {}163            self._apply_detail(lst, payload)164        return listings165166    # -- fiche immeuble (adresse) --------------------------------------------------167    def _building_address(self, slug: str) -> str:168        try:169            html = self.get(f"{BASE}/properties/{slug}/").text170        except Exception:171            return ""172        m = ADDR_RE.search(html.replace("+", " "))173        if not m:174            return ""175        addr = re.sub(r"\s+", " ", m.group(0)).strip()176        return addr.replace("Montreal", "Montréal")177178    # -- fiche annonce ---------------------------------------------------------------179    def _fetch_detail(self, url: str) -> dict:180        """Fiche d'une annonce : photos, description, sidebar ACF structuré181        (Available, Lease Terms) et commodités (BUILDING AMENITIES, cuisine).182        """183        html = self.get(url).text184        soup = BeautifulSoup(html, "html.parser")185186        imgs = re.findall(187            r'https://universityapartments\.ca/wp-content/uploads/'188            r'[^"\s\\\)]+\.(?:jpg|jpeg|png|webp)', html)189        imgs = [u for u in dict.fromkeys(imgs)190                if not re.search(r"logo|icon|favicon|chrome|-\d{2,3}x\d{2,3}\.",191                                 u, re.I)]192193        description = ""194        og = soup.find("meta", attrs={"property": "og:description"})195        if og and og.get("content"):196            description = og["content"].strip()[:600]197        else:198            p = soup.select_one(".fl-rich-text p, article p")199            if p:200                description = p.get_text(" ", strip=True)[:600]201202        # Sidebar ACF : <dl class="acf-data"><dt>Available</dt><dd>...</dd></dl>203        available = ""204        for dl in soup.select("dl.acf-data"):205            dts = [d.get_text(" ", strip=True) for d in dl.select("dt")]206            dds = [d.get_text(" ", strip=True) for d in dl.select("dd")]207            for k, v in zip(dts, dds):208                if k.lower().startswith("available") and v:209                    available = v210        lease_terms = [t.get_text(" ", strip=True)211                       for t in soup.select(".rental-term")]212213        # Commodités de l'immeuble : items <p> à émoji après le titre214        # « BUILDING AMENITIES » (même bloc rich-text).215        amenities: list[str] = []216        head = soup.find(string=re.compile(r"BUILDING AMENITIES", re.I))217        if head:218            h = head.find_parent(["h1", "h2", "h3", "h4", "strong"]) \219                or head.parent220            node = h.find_parent(["h1", "h2", "h3", "h4"]) or h221            for sib in node.find_next_siblings():222                if sib.name in ("h1", "h2", "h3"):223                    break224                text = _EMOJI_PREFIX_RE.sub("", sib.get_text(" ", strip=True))225                text = re.sub(r"\s+", " ", text).strip()226                if not text or text.startswith("***") or "Disclaimer" in text:227                    break228                if len(text) > 90:229                    continue230                # « Wi-Fi ($) » = payant : ne pas laisser la normalisation231                # le classer « internet inclus » (lacune générique notée)232                if text.endswith("($)"):233                    continue234                # stationnement à vélo ≠ stationnement auto235                if re.search(r"\bbike (?:parking|storage)\b", text, re.I):236                    text = "Espace vélos (sous-sol)"237                if text not in amenities:238                    amenities.append(text)239240        # Électroménagers de la cuisine partagée (ligne explicite de la fiche)241        kitchen = soup.find(string=re.compile(242            r"appliances,? including a fridge", re.I))243        if kitchen:244            amenities.append(_EMOJI_PREFIX_RE.sub(245                "", re.sub(r"\s+", " ", str(kitchen)).strip()))246247        return {"images": imgs, "description": description,248                "available": available, "lease_terms": lease_terms,249                "amenities": amenities}250251    def _apply_detail(self, lst: Listing, payload: dict) -> None:252        if not payload:253            return254        if payload.get("images"):255            lst.images = list(dict.fromkeys(256                lst.images + payload["images"]))[:40]257        if payload.get("description"):258            lst.description = payload["description"]259        if not lst.availability and payload.get("available"):260            lst.availability = f"Available {payload['available']}"261        if payload.get("lease_terms"):262            lst.amenities.append(263                "Lease terms: " + ", ".join(payload["lease_terms"]))264        for a in payload.get("amenities") or []:265            if a not in lst.amenities:266                lst.amenities.append(a)267