spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/groupe_dallaire.py : connecteur Groupe Dallaire — Faubourg du5#   Moulin (faubourgdumoulin.ca). Complexes Alizé, Noroît (I-II), Sirocco,6#   Sonora, Zéphyr (I-II) — rue Lionel-Audet, Québec (Beauport).7#   La page /disponibilite charge les unités via JetEngine (admin-ajax.php) :8#   on rejoue l'appel `get_listing` (lazy-load) puis la pagination9#   `jet_smart_filters` pour récupérer toutes les unités disponibles10#   (numéro, étage, prix, date de disponibilité, plan). Les fiches d'unité11#   (via cache BD) ajoutent type d'unité, superficie nette (« Pieds carrés »)12#   et inclusions réelles (« Incluant »).13# -----------------------------------------------------------------------------14from __future__ import annotations1516import hashlib17import html as htmllib18import json19import re20import time2122from bs4 import BeautifulSoup2324from ..schema import Listing, normalize_unit_type, parse_price25from .base import BaseConnector2627BASE = "https://faubourgdumoulin.ca"28AJAX = f"{BASE}/wp-admin/admin-ajax.php"29DISPO_URL = f"{BASE}/disponibilite/"30SECTOR = "Beauport"31CITY = "Québec"32ADDRESS = "rue Lionel-Audet, Québec (Faubourg du Moulin)"3334COMPLEXES = {35    "alize": "Alizé", "alize-2": "Alizé (Phase II)",36    "noroit": "Noroît", "noroit-2": "Noroît II",37    "sirocco": "Sirocco", "sonora": "Sonora",38    "zephyr": "Zéphyr", "zephyr-2": "Zéphyr II",39}4041# paramètres du widget « dispo » de la page /disponibilite42PAGE_SETTINGS = {43    "page_settings[post_id]": "105",44    "page_settings[queried_id]": "105|WP_Post",45    "page_settings[element_id]": "cb310a3",46    "page_settings[page]": "1",47    "listing_type": "elementor",48    "isEditMode": "false",49}505152def _flatten(prefix: str, obj, out: dict) -> None:53    """Aplati un dict/list en champs de formulaire PHP (a[b][0]=x)."""54    if isinstance(obj, dict):55        for k, v in obj.items():56            _flatten(f"{prefix}[{k}]", v, out)57    elif isinstance(obj, list):58        for i, v in enumerate(obj):59            _flatten(f"{prefix}[{i}]", v, out)60    elif isinstance(obj, bool):61        out[prefix] = "true" if obj else "false"62    else:63        out[prefix] = "" if obj is None else str(obj)646566class GroupeDallaireConnector(BaseConnector):67    source_id = "groupe_dallaire"68    request_delay = 0.669    max_pages = 25            # garde-fou pagination70    max_details = 150         # garde-fou fiches d'unité (vraies requêtes)7172    # -- POST throttlé ---------------------------------------------------73    def _post(self, url: str, data: dict):74        wait = self.request_delay - (time.time() - self._last_request)75        if wait > 0:76            time.sleep(wait)77        resp = self.session.post(url, data=data, timeout=self.timeout)78        self._last_request = time.time()79        resp.raise_for_status()80        return resp8182    # ---------------------------------------------------------------------83    def fetch(self) -> list[Listing]:84        # 1) Premier appel lazy-load : items page 1 + query/widget_settings85        first = self._post(AJAX, {86            "action": "jet_engine_ajax",87            "handler": "get_listing",88            **PAGE_SETTINGS,89        }).json()90        html1 = (first.get("data") or {}).get("html", "")9192        nav = None93        m = re.search(r'data-nav="([^"]*)"', html1)94        if m:95            try:96                nav = json.loads(htmllib.unescape(m.group(1)))97            except Exception:98                nav = None99100        listings: dict[str, Listing] = {}101        for lst in self._parse_items(html1):102            listings.setdefault(lst.external_id, lst)103104        # 2) Pagination via jet_smart_filters (si le data-nav est disponible)105        if nav and nav.get("query") and nav.get("widget_settings"):106            page = 2107            while page <= self.max_pages:108                data = {109                    "action": "jet_smart_filters",110                    "provider": "jet-engine/dispo",111                    "paged": str(page),112                }113                _flatten("defaults", nav["query"], data)114                _flatten("settings", nav["widget_settings"], data)115                try:116                    resp = self._post(AJAX, data).json()117                except Exception:118                    break119                content = resp.get("content") or ""120                new = 0121                for lst in self._parse_items(content):122                    if lst.external_id not in listings:123                        listings[lst.external_id] = lst124                        new += 1125                pag = resp.get("pagination") or {}126                max_pages = int(pag.get("max_num_pages") or 0)127                if new == 0 or (max_pages and page >= max_pages):128                    break129                page += 1130131        # 3) Fiche de l'unité (cache BD) : type d'unité (« 3½ TYPE D'UNITÉ »),132        #    superficie nette (« Pieds carrés »), inclusions réelles133        #    (« Incluant ») et plans (unité + niveau)134        self._fetched = 0135        for lst in listings.values():136            card_key = hashlib.sha1(137                f"{lst.title}|{lst.price_label}|{lst.availability}"138                .encode("utf-8")).hexdigest()139            try:140                payload = self.detail(lst.external_id, card_key,141                                      lambda u=lst.url: self._fetch_detail(u))142            except Exception:143                continue144            self._apply_detail(lst, payload)145146        return list(listings.values())147148    # ---------------------------------------------------------------------149    def _fetch_detail(self, url: str) -> dict:150        """Fiche d'unité : type, superficie nette, inclusions, plans.151        Retour JSON-sérialisable (cache BD)."""152        if self._fetched >= self.max_details:153            raise RuntimeError("budget de fiches d'unité atteint")154        self._fetched += 1155        dhtml = self.get(url).text156        out: dict = {}157158        dtext = re.sub(r"<script.*?</script>", "", dhtml, flags=re.S)159        dtext = re.sub(r"<style.*?</style>", "", dtext, flags=re.S)160        dtext = re.sub(r"<[^>]+>", "\n", dtext)161        tm = re.search(r"(\d\s*½\s*\+?|Studio)\s*\n+\s*TYPE D['’]UNITÉ", dtext)162        if tm:163            out["unit_type"] = tm.group(1)164        sm = re.search(r"(\d{3,4})\s*\n+\s*Pieds carrés", dtext)165        if sm:166            out["area_sqft"] = float(sm.group(1))   # champ « Pieds carrés »167168        # Inclusions réelles (« Incluant » -> répéteur JetEngine)169        dsoup = BeautifulSoup(dhtml, "html.parser")170        amen = [sp.get_text(" ", strip=True)171                for sp in dsoup.select(".jet-listing-dynamic-repeater__item span")]172        out["amenities"] = [a for a in dict.fromkeys(amen) if a][:15]173174        # Plans : plan de l'unité (jpg) + plan du niveau (NIVEAU_x.png)175        plans = []176        for u in dict.fromkeys(re.findall(177                r"https://faubourgdumoulin\.ca/wp-content/uploads/"178                r"[^\"'\s\\]+\.(?:jpe?g|webp|png)", dhtml, re.I)):179            if re.search(r"logo|_rgb_|descripteur|favicon|cropped-|dalcon|"180                         r"alpha-|-\d+x\d+\.", u, re.I):181                continue182            if u.lower().endswith(".png") and "niveau" not in u.lower():183                continue184            plans.append(u)185        out["images"] = plans[:10]186        return out187188    def _apply_detail(self, lst: Listing, d: dict) -> None:189        """Reporte le payload (frais/cache) sur l'annonce."""190        if not d:191            return192        if d.get("unit_type") and not lst.unit_type:193            lst.unit_type = normalize_unit_type(d["unit_type"])194            if lst.unit_type and "½" not in lst.title \195                    and "Studio" not in lst.title:196                lst.title += f" ({lst.unit_type})"197        if d.get("area_sqft"):198            lst.area_sqft = d["area_sqft"]199        if d.get("amenities"):200            # inclusions réelles de la fiche (remplacent le repli générique)201            extra = [a for a in lst.amenities if a == "En promotion"]202            lst.amenities = d["amenities"] + extra203        if d.get("images"):204            lst.images = list(dict.fromkeys(lst.images + d["images"]))[:15]205206    # ---------------------------------------------------------------------207    def _parse_items(self, content: str) -> list[Listing]:208        out: list[Listing] = []209        if not content:210            return out211        soup = BeautifulSoup(content, "html.parser")212        for item in soup.select(".jet-listing-grid__item[data-post-id]"):213            try:214                a = item.select_one('a[href*="/unites/"]')215                if not a:216                    continue217                url = a["href"]218                slug = url.rstrip("/").split("/")[-1]219220                for st in item.select("style"):221                    st.decompose()222                text = item.get_text(" ", strip=True)223224                # Complexe depuis le slug (ex. sonora-101, noroit-2-151)225                cslug = re.match(r"([a-z]+(?:-2)?)-\d+", slug)226                complexe = COMPLEXES.get(cslug.group(1) if cslug else "", "")227                if not complexe and cslug:228                    complexe = cslug.group(1).replace("-", " ").title()229230                num_m = re.search(r"Unité\s+(\w+)\s*-?", text)231                unit_no = num_m.group(1) if num_m else slug.split("-")[-1]232                floor_m = re.search(r"Étage\s+(\d+)", text)233234                price = None235                price_label = ""236                pm = re.search(r"(\d{3,5})\s*\$\s*(/mois\*?)", text)237                if pm:238                    price = parse_price(pm.group(1) + "$")239                    price_label = (pm.group(1) + "$" + pm.group(2))240                    if "*" in pm.group(2):241                        price_label += (" (chauffé, éclairé, climatisé, "242                                        "stationnement int.)")243244                avail_m = re.search(r"Disponibilité\s*:\s*([^|]+?)(?:Unité|$)",245                                    text)246                availability = (f"Disponibilité : {avail_m.group(1).strip()}"247                                if avail_m else "Disponible")248249                promo = "En promotion" in text250251                # Images : logo du complexe exclu, plan de l'unité conservé252                images: list[str] = []253                unit_type = ""254                for img in item.select("img[src]"):255                    src = img["src"]256                    if not src.startswith("http"):257                        continue258                    if re.search(r"logo|_rgb_|descripteur|\.svg$", src, re.I):259                        continue260                    full = re.sub(r"-\d+x\d+(\.(?:jpg|jpeg|png|webp))$",261                                  r"\1", src)262                    images.append(full)263                    # type d'unité encodé dans le nom du plan264                    tm = (re.search(r"pieces[-_](\d)\.5", src, re.I)265                          or re.search(r"_(\d)\.5\b", src))266                    if tm and not unit_type:267                        unit_type = normalize_unit_type(f"{tm.group(1)} 1/2")268                images = list(dict.fromkeys(images))269270                desc_parts = [f"Complexe {complexe}" if complexe else ""]271                if floor_m:272                    desc_parts.append(f"Étage {floor_m.group(1)}")273                if promo:274                    desc_parts.append("En promotion")275                desc_parts = [d for d in desc_parts if d]276277                # repli générique (remplacé par la liste « Incluant » réelle278                # de la fiche d'unité quand elle est disponible)279                amenities = ["Eau chaude incluse",280                             "Internet haute vitesse inclus",281                             "Stationnement extérieur ou intérieur",282                             "Accès aux aires communes"]283                if promo:284                    amenities.append("En promotion")285286                details: dict = {}287                if floor_m:288                    details["floor"] = int(floor_m.group(1))289290                out.append(Listing(291                    source=self.source_id,292                    external_id=slug,293                    url=url,294                    title=f"Faubourg du Moulin ({complexe or 'complexe'}) — "295                          f"Unité {unit_no}"296                          f"{' (' + unit_type + ')' if unit_type else ''}",297                    address=ADDRESS,298                    sector=SECTOR,299                    city=CITY,300                    unit_type=unit_type,301                    price=price,302                    price_label=price_label,303                    availability=availability,304                    description=" | ".join(desc_parts),305                    amenities=amenities,306                    details=details,307                    images=images,308                ))309            except Exception:310                continue311        return out312