spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/kass.py : connecteur KASS Property Management (kassproperties.com)5#   Gestionnaire Ottawa-Gatineau. WordPress + thème immobilier Houzez (même6#   famille que gimcote.py) : l'archive /city/gatineau/ liste les cartes du7#   parc québécois — prix, ville, lits/sdb/pi², galerie (data-images),8#   étiquettes de statut. Les cartes « Rented » sont sautées, de même que le9#   widget « propriétés similaires » d'Ottawa (cartes SANS étiquette de10#   statut) : les villes ontariennes sont exclues du périmètre Lou-Ka.11#   La fiche détail (cache BD) ajoute la description, le bloc « Details »12#   structuré (Move-in Date, Pet Friendly, Smoking, superficie, type), les13#   commodités et l'adresse civique complète.14# -----------------------------------------------------------------------------15from __future__ import annotations1617import hashlib18import html as htmllib19import json20import re2122from bs4 import BeautifulSoup2324from ..schema import (Listing, normalize_unit_type, parse_area_sqft,25                      parse_price, strip_accents)26from .base import BaseConnector2728BASE = "https://kassproperties.com"29LIST_URL = f"{BASE}/city/gatineau/"3031_SIZE_SUFFIX = re.compile(r"-\d{2,4}x\d{2,4}(?=\.(?:jpg|jpeg|png|webp)$)", re.I)32# secteurs de Gatineau repérables dans le titre ou l'adresse33_SECTORS = ["Hull", "Aylmer", "Buckingham", "Masson-Angers", "Plateau"]343536def _pets_value(raw: str) -> str | None:37    k = strip_accents((raw or "").strip().lower())38    if not k:39        return None40    if k.startswith(("no", "non")):41        return "non"42    if k.startswith(("yes", "oui")):43        return "oui"44    return "conditions"454647class KassConnector(BaseConnector):48    source_id = "kass"49    request_delay = 1.050    max_pages = 551    max_details = 2052    max_images = 205354    def fetch(self) -> list[Listing]:55        listings: dict[str, Listing] = {}56        for page in range(1, self.max_pages + 1):57            url = LIST_URL if page == 1 else f"{LIST_URL}page/{page}/"58            try:59                html = self.get(url).text60            except Exception:61                break62            soup = BeautifulSoup(html, "html.parser")63            before = len(listings)64            for card in soup.select("div.item-listing-wrap"):65                try:66                    self._parse_card(card, listings)67                except Exception:68                    continue69            if len(listings) == before:   # plus de résultats d'archive70                break7172        # fiches détail Houzez (cache BD)73        self._fetched = 074        for lst in listings.values():75            key = hashlib.sha1(76                f"{lst.title}|{lst.price_label}|{lst.url}"77                .encode("utf-8")).hexdigest()[:20]78            try:79                payload = self.detail(lst.external_id, key,80                                      lambda u=lst.url: self._fetch_detail(u))81            except Exception:82                continue83            self._apply_detail(lst, payload)84        return list(listings.values())8586    # -- carte Houzez ------------------------------------------------------------87    def _parse_card(self, card, listings: dict[str, Listing]) -> None:88        # cartes d'archive seulement : le widget « similaires » (Ottawa) n'a89        # pas d'étiquette de statut90        status = [a.get_text(strip=True)91                  for a in card.select("a[href*='/status/']")]92        if not any(re.search(r"(?i)for rent", s) for s in status):93            return94        labels = [a.get_text(strip=True)95                  for a in card.select("a[href*='/label/']")]96        if any(re.search(r"(?i)rented|lou[ée]", s) for s in labels + status):97            return   # déjà loué9899        addr_el = card.select_one("address.item-address")100        card_city = addr_el.get_text(" ", strip=True) if addr_el else ""101        if not re.search(r"(?i)gatineau|hull|aylmer|buckingham", card_city):102            return   # villes ontariennes exclues103104        link = card.select_one("h2.item-title a[href]")105        if not link:106            return107        url = link["href"]108        title = link.get_text(" ", strip=True)109        ext = str(card.get("data-hz-id") or "")110        if not ext:111            m = re.search(r"/property/([^/]+)/?", url)112            ext = m.group(1) if m else ""113        if not ext or ext in listings:114            return115        if re.search(r"(?i)parking|storage|commercial|office", title):116            return   # non résidentiel117118        price_el = card.select_one("li.item-price")119        price_label = price_el.get_text(" ", strip=True) if price_el else ""120121        # lits/sdb/pi² de la carte122        beds = sqft = ""123        amen_bits: list[str] = []124        for li in card.select("ul.item-amenities li"):125            t = re.sub(r"\s+", " ", li.get_text(" ", strip=True))126            if re.match(r"(?i)^bed", t):127                beds = t128            elif "sqft" in t.lower():129                sqft = t130            if t:131                amen_bits.append(t)132        unit_type = ""133        m = re.search(r"(\d+)", beds)134        if m:135            unit_type = normalize_unit_type(f"{m.group(1)} chambres")136137        # secteur si l'agence le nomme dans le titre138        sector = ""139        for s in _SECTORS:140            if re.search(rf"(?i)\b{s}\b", title):141                sector = s142                break143144        # galerie complète (attribut data-images, JSON Houzez)145        images: list[str] = []146        raw = card.get("data-images") or ""147        if raw:148            try:149                items = json.loads(htmllib.unescape(raw))150                urls = [it.get("image", "") if isinstance(it, dict) else str(it)151                        for it in items]152            except Exception:153                urls = re.findall(r"https?:[^\"',\\]+", htmllib.unescape(raw))154            for u in urls:155                u = u.replace("\\/", "/").strip()156                if u.startswith("http"):157                    u = _SIZE_SUFFIX.sub("", u)158                    if u not in images:159                        images.append(u)160        if not images:161            thumb = card.select_one("img.wp-post-image[src]")162            if thumb:163                images = [_SIZE_SUFFIX.sub("", thumb["src"])]164165        listings[ext] = Listing(166            source=self.source_id,167            external_id=ext,168            url=url,169            title=title,170            sector=sector,171            city="Gatineau",172            unit_type=unit_type,173            price=parse_price(price_label.replace(",", "")),174            price_label=price_label,175            description=" — ".join(amen_bits),176            images=images[: self.max_images],177        )178179    # -- fiche détail Houzez -------------------------------------------------------180    def _fetch_detail(self, url: str) -> dict:181        if self._fetched >= self.max_details:182            raise RuntimeError("budget de fiches détail atteint")183        self._fetched += 1184        soup = BeautifulSoup(self.get(url).text, "html.parser")185        out: dict = {}186187        desc_el = soup.select_one("#property-description-wrap")188        if desc_el:189            txt = desc_el.get_text("\n", strip=True)190            txt = re.sub(r"^Description\n", "", txt)191            out["description"] = re.sub(r"[ \t]+", " ", txt).strip()[:1500]192193        out["amenities"] = [a.get_text(" ", strip=True)194                            for a in soup.select("#property-features-wrap li")195                            if a.get_text(strip=True)][:25]196197        # bloc « Details » : Move-in Date, Pet Friendly, Smoking, Size, Type198        for li in soup.select("#property-detail-wrap li"):199            t = re.sub(r"\s+", " ", li.get_text(" ", strip=True))200            lab = strip_accents(t.lower())201            val = re.sub(r"^[^ ]+( [^ ]+)? ", "", t).strip()202            if lab.startswith("move-in date"):203                out["availability"] = t.replace("Move-in Date", "").strip()204            elif lab.startswith("pet friendly"):205                out["pets_raw"] = t.replace("Pet Friendly", "").strip()206            elif lab.startswith("smoking"):207                out["smoking_raw"] = t.replace("Smoking", "").strip()208            elif lab.startswith("property size"):209                out["size_raw"] = val210            elif lab.startswith("property type"):211                out["type_raw"] = t.replace("Property Type", "").strip()212213        for li in soup.select("#property-address-wrap li"):214            t = re.sub(r"\s+", " ", li.get_text(" ", strip=True))215            if t.lower().startswith("address:"):216                out["address"] = t.split(":", 1)[1].strip()217        return out218219    def _apply_detail(self, lst: Listing, d: dict) -> None:220        if not d:221            return222        if d.get("description"):223            lst.description = d["description"]224        if d.get("amenities"):225            lst.amenities = list(dict.fromkeys(lst.amenities + d["amenities"]))226        if d.get("availability"):227            lst.availability = d["availability"]228            # format Houzez « 1-Sep-24 » (année sur 2 chiffres) : la229            # normalisation commune ignorerait l'année et projetterait une230            # date future — on la résout ici (date passée -> « now »)231            m = re.match(r"^(\d{1,2})-([A-Za-z]{3})-(\d{2})$",232                         d["availability"].strip())233            if m:234                from datetime import date235                months = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5,236                          "jun": 6, "jul": 7, "aug": 8, "sep": 9, "oct": 10,237                          "nov": 11, "dec": 12}238                mo = months.get(m.group(2).lower())239                if mo:240                    dt = date(2000 + int(m.group(3)), mo, int(m.group(1)))241                    lst.availability_date = ("now" if dt <= date.today()242                                             else dt.isoformat())243        if d.get("address"):244            lst.address = d["address"]245            if not lst.sector:246                for s in _SECTORS:247                    if re.search(rf"(?i)\b{s}\b", d["address"]):248                        lst.sector = s249                        break250        if lst.area_sqft is None and d.get("size_raw"):251            lst.area_sqft = parse_area_sqft(d["size_raw"])252        pets = _pets_value(d.get("pets_raw", ""))253        if pets:254            lst.pets = pets255        details: dict = {}256        if d.get("type_raw"):257            details["building_type"] = d["type_raw"]258        smoking = strip_accents(d.get("smoking_raw", "").lower())259        if smoking.startswith("no"):260            details["smoking"] = False261        elif smoking:262            details["smoking_raw"] = d["smoking_raw"]263        if details:264            lst.details = details265