spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/elk.py : connecteur ELK Property Management (elkproperty.com)5#   Gestionnaire du Plateau/Hull à Gatineau. Très vieux site PHP (PinchCMS),6#   HTTP SEULEMENT (pas de HTTPS) : la page residential_new.php?typeID=17#   (Ottawa/Gatineau) rend côté serveur un bloc par complexe :8#     - div.results-in : adresse (h2 + code postal), galerie lightbox,9#       description à puces (secteur « Hull District »), contact, note de10#       loyer « Starting from $1050.00/month | Hydro/Gas not included »,11#       listes « Building Amenities » / « Apartment Features » ;12#     - div.record-bttm#units_<id> : colonnes parallèles BEDROOMS / FLOORPLAN13#       (PDF) / RENT alignées par index -> une annonce par TYPOLOGIE affichée14#       sous « NOW RENTING / AVAILABLE APARTMENTS ».15#   Les complexes hors Québec (Halifax sous typeID=2, adresses ON) sont16#   exclus : seules les adresses « QC » passent.17# -----------------------------------------------------------------------------18from __future__ import annotations1920import re2122from bs4 import BeautifulSoup2324from ..schema import Listing, normalize_unit_type25from .base import BaseConnector2627BASE = "http://www.elkproperty.com"28LIST_URL = f"{BASE}/residential_new.php?typeID=1"2930_SECTORS = ["Hull", "Aylmer", "Buckingham", "Plateau"]313233class ElkConnector(BaseConnector):34    source_id = "elk"35    request_delay = 1.036    max_images = 123738    def fetch(self) -> list[Listing]:39        soup = BeautifulSoup(self.get(LIST_URL).text, "html.parser")4041        # tables d'unités par complexe : record-bttm id="units_<id>"42        units_by_id: dict[str, list[dict]] = {}43        for block in soup.select("div.record-bttm[id^='units_']"):44            pid = block["id"].split("_", 1)[1]45            cols: dict[str, list] = {}46            for ul in block.find_all("ul"):47                h2 = ul.find("h2")48                if not h2:49                    continue50                head = h2.get_text(" ", strip=True).upper()51                cells = ul.find_all("li")[1:]     # après l'en-tête52                cols[head] = cells53            rows: list[dict] = []54            beds = cols.get("BEDROOMS", [])55            rents = cols.get("RENT", [])56            plans = cols.get("FLOORPLAN", [])57            for i, bcell in enumerate(beds):58                b = re.sub(r"\s+", " ", bcell.get_text(" ", strip=True))59                if not b:60                    continue61                row: dict = {"beds": b}62                if i < len(rents):63                    row["rent"] = re.sub(r"\s+", " ",64                                         rents[i].get_text(" ", strip=True))65                if i < len(plans):66                    a = plans[i].find("a", href=True)67                    if a:68                        row["plan"] = a["href"]69                rows.append(row)70            units_by_id[pid] = rows7172        listings: dict[str, Listing] = {}73        for res in soup.select("div.results-in"):74            try:75                self._parse_complex(res, units_by_id, listings)76            except Exception:77                continue78        return list(listings.values())7980    def _parse_complex(self, res, units_by_id: dict,81                       listings: dict[str, Listing]) -> None:82        right = res.select_one(".gallary-right")83        h2 = right.find("h2") if right else None84        if not h2:85            return86        span = h2.find("span")87        locality = span.get_text(" ", strip=True) if span else ""88        if span:89            span.extract()90        street = re.sub(r"\s+", " ", h2.get_text(" ", strip=True)).strip()91        # « Gatineau QC, J9A 3J2 » : Québec seulement (Halifax/Ottawa exclus)92        if not re.search(r"\bQC\b", locality):93            return94        m = re.match(r"^([A-Za-zÀ-ÿ' .-]+?)\s+QC", locality)95        city = (m.group(1).strip() if m else "Gatineau")9697        # id du complexe via la galerie lightbox « apt_6 » -> table units_698        pid = ""99        gal = res.select_one("[data-lightbox]")100        if gal:101            mm = re.search(r"(\d+)$", gal.get("data-lightbox", ""))102            if mm:103                pid = mm.group(1)104105        images = []106        for a in res.select("a[data-lightbox][href]"):107            u = a["href"]108            if not u.startswith("http"):109                u = BASE + (u if u.startswith("/") else "/" + u)110            if u not in images:111                images.append(u)112113        text = right.get_text("\n", strip=True)114        # description à puces + note de loyer, texte fidèle de l'agence115        desc_lines = [re.sub(r"\s+", " ", l).strip() for l in text.split("\n")]116        desc_lines = [l for l in desc_lines117                      if l and not re.match(r"(?i)^(contact us today|rent:$)", l)118                      and "@" not in l and not re.match(r"^\d{3}-\d{3}-\d{4}$", l)]119        rent_note = ""120        for l in desc_lines:121            if re.search(r"(?i)starting from \$", l):122                rent_note = l123                break124125        sector = ""126        for s in _SECTORS:127            if re.search(rf"(?i)\b{s}\b", text):128                sector = s129                break130131        amenities: list[str] = []132        for h4 in right.find_all("h4"):133            sec = h4.get_text(" ", strip=True)134            if not re.search(r"(?i)amenities|features", sec):135                continue136            ul = h4.find_next("ul")137            for li in (ul.find_all("li") if ul else []):138                t = re.sub(r"\s+", " ", li.get_text(" ", strip=True))139                if t and t not in amenities:140                    amenities.append(t)141142        contact = {}143        mm = re.search(r"([\w.+-]+@elkproperty\.com)", text)144        if mm:145            contact["email"] = mm.group(1)146        mm = re.search(r"\b(\d{3})[-. ](\d{3})[-. ](\d{4})\b", text)147        if mm:148            contact["phone"] = f"{mm.group(1)}-{mm.group(2)}-{mm.group(3)}"149150        address = f"{street}, {city}"151        rows = units_by_id.get(pid, [])152        for row in rows:153            beds = row["beds"]154            mm = re.match(r"^(\d+)", beds)155            unit_type = (normalize_unit_type(f"{mm.group(1)} chambres")156                         if mm else "")157            rent = row.get("rent", "")158            price = None159            pm = re.search(r"\$\s*([\d,]+)", rent)160            if pm:161                price = float(pm.group(1).replace(",", ""))162163            details: dict = {}164            if contact:165                details["contact"] = dict(contact)166            plan = row.get("plan", "")167            if plan:168                if not plan.startswith("http"):169                    plan = BASE + (plan if plan.startswith("/")170                                   else "/" + plan)171                details["floorplan_pdf"] = plan172173            ext = f"{pid}-{mm.group(1) if mm else beds}"174            if ext in listings:175                continue176            desc = " — ".join(x for x in [177                " ".join(desc_lines[:6]), rent_note] if x)178            listings[ext] = Listing(179                source=self.source_id,180                external_id=ext,181                url=f"{LIST_URL}#units_{pid}",182                title=f"{street} — {beds} bedroom(s)",183                address=address,184                sector=sector,185                city=city,186                unit_type=unit_type,187                price=price,188                price_label=rent,189                availability="Now renting",   # bandeau de la section source190                description=desc[:900],191                amenities=amenities,192                details=details,193                images=images[: self.max_images],194            )195