spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Location court terme3# connectors/chaletsauquebec.py : ChaletsAuQuebec.com — « le plus grand4# répertoire de chalets à louer par le propriétaire au Québec » (~3 900).5#6# Méthode (site ASP.NET/Telerik, pagination des listes en postback → évitée) :7#   1. la vue carte de chaque région (`/chalets-a-louer/quebec/<région>/8#      default.aspx?ar=c`) embarque TOUT l'inventaire régional dans la9#      variable JS `gAddressPoints = [[lat, lng, "id"], …]` → inventaire10#      complet + géolocalisation en 17 requêtes ;11#   2. page détail `/<id>` (cache self.detail) : microdonnées schema.org12#      (priceRange, geo), h1, Région/Ville, icônes (capacité, chambres,13#      salles de bain, animaux), commodités par groupe, no CITQ, photos.14# -----------------------------------------------------------------------------15from __future__ import annotations1617import re18import sys1920from ..schema import StListing21from .base import StConnector2223BASE = "https://www.chaletsauquebec.com"24INDEX = f"{BASE}/chalets-a-louer/quebec"2526_POINTS_RE = re.compile(r"gAddressPoints\s*=\s*(\[\[.*?\]\])\s*;", re.S)27_REGION_HREF_RE = re.compile(r'href="(/chalets-a-louer/quebec/[a-z0-9-]+)"')28# dernier mot avant « à louer » dans <title> : « … - Chalet à louer Ville | … »29_TITLE_TYPE_RE = re.compile(r"([\w’'-]+)\s+à louer\s", re.UNICODE)30_REGION_RE = re.compile(r"Région\s*:\s*<a[^>]*>([^<]+)</a>")31_VILLE_RE = re.compile(r"Ville\s*:\s*<a[^>]*>([^<]+)</a>")32_PHOTO_RE = re.compile(r"/_photos/grand/[^\"'\s]+", re.I)3334# groupes de commodités = équipements de la propriété (le reste = contexte)35_AMEN_GROUPS = ("cuisine", "communications", "général", "general",36                "équipements de loisir", "equipements de loisir",37                "équipements extérieur", "equipements exterieur")383940class ChaletsAuQuebec(StConnector):41    source_id = "chaletsauquebec"42    request_delay = 0.35   # vieux site costaud ; ~3 900 pages détail (cache)4344    # -- inventaire (vues carte régionales) -----------------------------------45    def _region_slugs(self) -> list[str]:46        html = self.get(INDEX).text47        slugs = []48        for href in _REGION_HREF_RE.findall(html):49            slug = href.rsplit("/", 1)[-1]50            if slug not in slugs:51                slugs.append(slug)52        return slugs5354    def _region_points(self, slug: str) -> list[tuple[str, float, float]]:55        """[(id, lat, lng), …] de la vue carte d'une région (inventaire complet)."""56        import json57        url = f"{BASE}/chalets-a-louer/quebec/{slug}/default.aspx?ar=c"58        m = _POINTS_RE.search(self.get(url).text)59        if not m:60            return []61        out = []62        for pt in json.loads(m.group(1)):63            try:64                lat, lng, ext = float(pt[0]), float(pt[1]), str(pt[2])65            except (TypeError, ValueError, IndexError):66                continue67            out.append((ext, lat, lng))68        return out6970    # -- page détail -----------------------------------------------------------71    def _fetch_detail(self, ext: str) -> dict:72        from bs4 import BeautifulSoup73        html = self.get(f"{BASE}/{ext}").text74        soup = BeautifulSoup(html, "html.parser")75        d: dict = {}7677        h1 = soup.find("h1")78        d["title"] = h1.get_text(" ", strip=True) if h1 else ""79        tt = soup.find("title")80        types = _TITLE_TYPE_RE.findall(tt.get_text()) if tt else []81        d["property_type"] = types[-1].strip() if types else ""8283        pr = soup.find("meta", attrs={"itemprop": "priceRange"})84        d["price_label"] = (pr.get("content") or "").strip() if pr else ""85        for prop, key in (("latitude", "lat"), ("longitude", "lng")):86            tag = soup.find("meta", attrs={"itemprop": prop})87            if tag and tag.get("content"):88                try:89                    d[key] = float(tag["content"])90                except ValueError:91                    pass9293        # évaluations (microdonnées AggregateRating, note sur 5)94        agg = soup.find(attrs={"itemprop": "aggregateRating"})95        if agg is not None:96            for prop, key, cast in (("ratingValue", "rating", float),97                                    ("reviewCount", "reviews", int)):98                tag = agg.find("meta", attrs={"itemprop": prop})99                if tag and tag.get("content"):100                    try:101                        d[key] = cast(tag["content"])102                    except ValueError:103                        pass104105        m = _REGION_RE.search(html)106        d["region"] = m.group(1).strip() if m else ""107        m = _VILLE_RE.search(html)108        d["city"] = m.group(1).strip() if m else ""109110        # icônes du résumé : capacité, chambres, sdb, animaux + drapeaux111        amenities: list[str] = []112        for td in soup.select("td.styleTxtIcon"):113            txt = td.get_text(" ", strip=True)114            low = txt.lower()115            mn = re.search(r"(\d+)", txt)116            if "pers" in low and mn:117                d["capacity"] = float(mn.group(1))118            elif "chambre" in low and mn:119                if int(mn.group(1)) > 0:       # « 0 chambre » = non renseigné120                    d["bedrooms"] = float(mn.group(1))121            elif "salle" in low and "bain" in low and mn:122                d["bathrooms"] = float(mn.group(1))123            elif low.startswith("animaux"):124                d["pets"] = ("non" if "non permis" in low125                             else "conditions" if "restriction" in low126                             else "oui" if "permis" in low else None)127            elif low.startswith("fumeur"):128                d.setdefault("details", {})["fumeurs"] = txt129            elif txt:130                amenities.append(txt)          # Foyer, Internet, Spa, Bord de l'eau…131132        # commodités par groupe + no CITQ + lits133        details = d.setdefault("details", {})134        for td in soup.select("td.titreItemComm"):135            name = td.get_text(" ", strip=True)136            val_td = td.find_next_sibling("td")137            if not name or val_td is None:138                continue139            vals = [v.get_text(" ", strip=True)140                    for v in val_td.select(".paddingValComm")]141            vals = [v for v in vals if v]142            low = name.lower()143            if "citq" in low:144                d["citq"] = vals[0] if vals else val_td.get_text(strip=True)145            elif low == "chambres":            # « - 3 lits simples - 6 lits doubles … »146                blob = " ".join(vals)147                n = sum(int(x) for x in re.findall(r"(\d+)\s+lits?", blob))148                if n:149                    d["beds"] = float(n)150                details["lits"] = blob151            elif low.startswith(_AMEN_GROUPS):152                amenities.extend(vals)153            elif vals:                          # activités été/hiver à proximité…154                details[name] = vals155        seen: set[str] = set()156        d["amenities"] = [a for a in amenities157                          if not (a in seen or seen.add(a))]158159        # description (retirer les tables d'icônes/commodités, garder le texte)160        sec = soup.find(id="idSectionDescription")161        if sec is not None:162            for t in sec.find_all("table"):163                t.decompose()164            lines = [ln for ln in sec.get_text("\n", strip=True).split("\n")165                     if ln and not ln.isupper()]      # titres de section en CAPS166            d["description"] = "\n".join(lines).strip()167168        imgs: list[str] = []169        seen_img: set[str] = set()              # même fichier en /Grand/ et /grand/170        for u in _PHOTO_RE.findall(html):171            u = BASE + u if u.startswith("/") else u172            if u.lower() not in seen_img:173                seen_img.add(u.lower())174                imgs.append(u)175        d["images"] = imgs176        return d177178    # -- contrat ----------------------------------------------------------------179    def fetch(self) -> list[StListing]:180        points: dict[str, tuple[float, float]] = {}181        for slug in self._region_slugs():182            for ext, lat, lng in self._region_points(slug):183                points.setdefault(ext, (lat, lng))184185        listings, errors = [], 0186        for ext, (lat, lng) in points.items():187            try:188                d = self.detail(ext, "pdp1", lambda e=ext: self._fetch_detail(e))189            except Exception as exc:  # noqa: BLE001 — annonce retirée / réseau190                errors += 1191                if errors <= 5:192                    print(f"[chaletsauquebec] détail {ext} : {exc}",193                          file=sys.stderr)194                continue195            if not d.get("title"):196                continue                        # page vide / annonce retirée197            listings.append(StListing(198                source=self.source_id,199                external_id=ext,200                url=f"{BASE}/{ext}",201                title=d.get("title", ""),202                property_type=d.get("property_type", ""),203                city=d.get("city", ""),204                region=d.get("region", ""),205                price_label=d.get("price_label", ""),206                capacity=d.get("capacity"),207                bedrooms=d.get("bedrooms"),208                beds=d.get("beds"),209                bathrooms=d.get("bathrooms"),210                pets=d.get("pets"),211                citq=str(d.get("citq", "") or ""),212                rating=d.get("rating"),213                reviews=d.get("reviews"),214                description=d.get("description", ""),215                amenities=d.get("amenities", []),216                details=d.get("details", {}),217                images=d.get("images", []),218                lat=d.get("lat", lat),219                lng=d.get("lng", lng),220            ))221        if errors:222            print(f"[chaletsauquebec] {errors} pages détail en erreur (ignorées)",223                  file=sys.stderr)224        return listings225