court terme : 4 connecteurs niche QC — Tremblant Living (sitemap + JSON-LD Streamline), Gîtes du Passant (annuaire Yapla Terroir et Saveurs), Kijiji locations de vacances c814 (Bright Data), Hébergement Québec (site mort, disabled)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
4 changed files +734 −0
added
louka/shortterm/connectors/gitespassant.py
+224 −0
@@ -0,0 +1,224 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/gitespassant.py : Gîtes et Auberges du Passant certifiés | |
| 4 | +# (réseau officiel de la Fédération des Agricotours du Québec — marques | |
| 5 | +# déposées « Gîte du Passant », « Auberge du Passant », « Maison de | |
| 6 | +# Campagne »). L'ancien giteetaubergedupassant.com ne répond plus : le | |
| 7 | +# répertoire vit maintenant sur membres.terroiretsaveurs.com (plateforme | |
| 8 | +# Yapla/MemboGo de Terroir et Saveurs du Québec). | |
| 9 | +# | |
| 10 | +# Méthode (annuaire Yapla, aucun anti-bot) : | |
| 11 | +# 1. LISTE : POST /fr/repertoire/pagination/pageNumber/1 avec | |
| 12 | +# categorie_detablissement[]=36680 (« Hébergement ») → le script | |
| 13 | +# memboGo.directory.scroll.list["organization"] contient TOUS les ids de | |
| 14 | +# membres (une vingtaine). Le même appel filtré par regionadministrative | |
| 15 | +# (19 régions touristiques) donne la région de chaque membre. | |
| 16 | +# 2. FICHE (cache self.detail, rafraîchie chaque mois) : | |
| 17 | +# /fr/repertoire/detailorganization/id/<id> — HTML serveur : nom (h1), | |
| 18 | +# description, « Types d'hébergement » (Gîte/Auberge du Passant, Maison | |
| 19 | +# de Campagne…), numéro CITQ (#attestation), classification (soleils), | |
| 20 | +# adresse postale (spans), téléphone, site web, photos du carrousel. | |
| 21 | +# Pas de prix affiché (les tarifs sont chez chaque établissement). | |
| 22 | +# ----------------------------------------------------------------------------- | |
| 23 | +from __future__ import annotations | |
| 24 | + | |
| 25 | +import html as _html | |
| 26 | +import json | |
| 27 | +import os | |
| 28 | +import re | |
| 29 | +import time | |
| 30 | + | |
| 31 | +from ..schema import StListing | |
| 32 | +from .base import StConnector | |
| 33 | + | |
| 34 | +BASE = "https://membres.terroiretsaveurs.com" | |
| 35 | +PAGINATION = BASE + "/fr/repertoire/pagination/pageNumber/1" | |
| 36 | +FICHE = BASE + "/fr/repertoire/detailorganization/id/{id}" | |
| 37 | + | |
| 38 | +CAT_HEBERGEMENT = "36680" # option « Hébergement » du formulaire | |
| 39 | + | |
| 40 | +# valeur de l'option regionadministrative → région touristique canonique | |
| 41 | +_REGIONS = { | |
| 42 | + "316581": "Abitibi-Témiscamingue", | |
| 43 | + "316582": "Bas-Saint-Laurent", | |
| 44 | + "316584": "Cantons-de-l'Est", | |
| 45 | + "316585": "Centre-du-Québec", | |
| 46 | + "316586": "Charlevoix", | |
| 47 | + "316583": "Chaudière-Appalaches", | |
| 48 | + "316587": "Côte-Nord", | |
| 49 | + "591978": "Eeyou Istchee Baie-James", | |
| 50 | + "316589": "Gaspésie", | |
| 51 | + "316590": "Îles-de-la-Madeleine", | |
| 52 | + "316588": "Lanaudière", | |
| 53 | + "316591": "Laurentides", | |
| 54 | + "316592": "Laval", | |
| 55 | + "316593": "Mauricie", | |
| 56 | + "316594": "Montérégie", | |
| 57 | + "316595": "Outaouais", | |
| 58 | + "316596": "Montréal", | |
| 59 | + "316597": "Québec", | |
| 60 | + "316598": "Saguenay–Lac-Saint-Jean", | |
| 61 | +} | |
| 62 | + | |
| 63 | +# « Types d'hébergement » de la fiche → type canonique Lou-Ka | |
| 64 | +_TYPES = [ | |
| 65 | + ("gite du passant", "Gîte"), ("gite", "Gîte"), | |
| 66 | + ("auberge du passant", "Auberge"), ("auberge", "Auberge"), | |
| 67 | + ("maison de campagne", "Maison"), ("maison de ville", "Maison"), | |
| 68 | + ("chalet", "Chalet"), | |
| 69 | +] | |
| 70 | + | |
| 71 | +_TAG_RE = re.compile(r"<[^>]+>") | |
| 72 | + | |
| 73 | + | |
| 74 | +def _text(fragment: str) -> str: | |
| 75 | + return _html.unescape(re.sub(r"\s+", " ", _TAG_RE.sub(" ", fragment))).strip() | |
| 76 | + | |
| 77 | + | |
| 78 | +class GitesPassant(StConnector): | |
| 79 | + source_id = "gites_passant" | |
| 80 | + | |
| 81 | + # -- liste (ids de membres) -------------------------------------------- | |
| 82 | + def _ids(self, region_value: str = "") -> list[str]: | |
| 83 | + data = {"activetab": "organization", "search_type": "advanced", | |
| 84 | + "categorie_detablissement[]": CAT_HEBERGEMENT} | |
| 85 | + if region_value: | |
| 86 | + data["regionadministrative"] = region_value | |
| 87 | + resp = self.post(PAGINATION, data=data, | |
| 88 | + headers={"X-Requested-With": "XMLHttpRequest"}) | |
| 89 | + m = re.search(r'scroll\.list\["organization"\]\s*=\s*(\[.*?\]);', | |
| 90 | + resp.text, re.S) | |
| 91 | + if not m: | |
| 92 | + return [] | |
| 93 | + try: | |
| 94 | + groups = json.loads(m.group(1)) | |
| 95 | + except ValueError: | |
| 96 | + return [] | |
| 97 | + out: list[str] = [] | |
| 98 | + for g in groups: | |
| 99 | + out.extend(str(i) for i in (g or {}).get("items") or []) | |
| 100 | + return out | |
| 101 | + | |
| 102 | + # -- fiche --------------------------------------------------------------- | |
| 103 | + def _fiche(self, member_id: str) -> dict: | |
| 104 | + brut = self.get(FICHE.format(id=member_id)).text | |
| 105 | + # les fiches truffent le HTML de <script> inline (carrousel, vidéo…) | |
| 106 | + h = re.sub(r"(?s)<script[^>]*>.*?</script>", " ", brut) | |
| 107 | + d: dict = {} | |
| 108 | + | |
| 109 | + h1s = re.findall(r"<h1[^>]*>(.*?)</h1>", h, re.S) | |
| 110 | + titres = [_text(x) for x in h1s if _text(x) and _text(x) != "Répertoire"] | |
| 111 | + if titres: | |
| 112 | + d["title"] = titres[-1] | |
| 113 | + | |
| 114 | + m = re.search(r"(?s)<h2>\s*Description\s*</h2>(.*?)" | |
| 115 | + r'<(?:h2|div class="carousel|/section)', h) | |
| 116 | + if m: | |
| 117 | + d["description"] = _text(m.group(1))[:5000] | |
| 118 | + | |
| 119 | + m = re.search(r'(?s)id="hebergement"[^>]*>(.*?)</div>', h) | |
| 120 | + if m: | |
| 121 | + d["types"] = _text(m.group(1)).replace("Types d'hébergement :", | |
| 122 | + "").strip() | |
| 123 | + m = re.search(r'(?s)id="attestation"[^>]*>.*?<p>\s*(\d{4,8})\s*</p>', h) | |
| 124 | + if m: | |
| 125 | + d["citq"] = m.group(1) | |
| 126 | + m = re.search(r'(?s)id="classification"[^>]*>.*?<p[^>]*>(.*?)</p>', h) | |
| 127 | + if m: | |
| 128 | + classement = _text(m.group(1)) | |
| 129 | + if classement not in ("", "0"): | |
| 130 | + d["classification"] = classement | |
| 131 | + | |
| 132 | + # adresse : <p><span>rue</span><br/><span>ville</span> <span>(Québec)… | |
| 133 | + m = re.search(r"(?s)Coordonnées</h3>(.*?)</p>", h) | |
| 134 | + if m: | |
| 135 | + spans = [_text(s) for s in | |
| 136 | + re.findall(r"<span[^>]*>(.*?)</span>", m.group(1))] | |
| 137 | + spans = [s for s in spans if s and not s.startswith("Site Web")] | |
| 138 | + if spans: | |
| 139 | + d["street"] = spans[0] | |
| 140 | + if len(spans) > 1: | |
| 141 | + d["city"] = spans[1] | |
| 142 | + for s in spans[2:]: | |
| 143 | + if re.match(r"^[A-Z]\d[A-Z]\s?\d[A-Z]\d$", s): | |
| 144 | + d["postal"] = s | |
| 145 | + mm = re.search(r"Téléphone\s*:\s*([\d ().+-]{7,20})", m.group(1)) | |
| 146 | + if mm: | |
| 147 | + d["phone"] = mm.group(1).strip() | |
| 148 | + m = re.search(r'id="site"[^>]*>.*?href="([^"]+)"', h, re.S) | |
| 149 | + if m: | |
| 150 | + d["website"] = m.group(1) | |
| 151 | + | |
| 152 | + # photos : le carrousel de la fiche seulement (ailleurs sur la page, | |
| 153 | + # les « Suggestions d'articles » ont aussi des images membres) | |
| 154 | + logo = "" | |
| 155 | + m = re.search(r'(?s)id="logo">\s*<img[^>]+src="([^"]+)"', brut) | |
| 156 | + if m: | |
| 157 | + logo = m.group(1) | |
| 158 | + imgs: list[str] = [] | |
| 159 | + debut = brut.find('id="myCarousel"') | |
| 160 | + if debut >= 0: | |
| 161 | + fin = brut.find("carousel-control", debut) | |
| 162 | + zone = brut[debut:fin if fin > debut else debut + 40000] | |
| 163 | + for u in re.findall(r'(https://cdn\.ca\.yapla\.com/[^\s"\'{}]*' | |
| 164 | + r'member/organization/[^\s"\'{}]+' | |
| 165 | + r'\.(?:jpe?g|png|webp))', zone): | |
| 166 | + if u != logo and u not in imgs: | |
| 167 | + imgs.append(u) | |
| 168 | + if imgs: | |
| 169 | + d["images"] = imgs[:15] | |
| 170 | + return d | |
| 171 | + | |
| 172 | + # -- contrat ------------------------------------------------------------ | |
| 173 | + def fetch(self) -> list[StListing]: | |
| 174 | + limit = int(os.environ.get("LOUKA_GITES_LIMIT", "0") or 0) | |
| 175 | + | |
| 176 | + ids = self._ids() | |
| 177 | + regions: dict[str, str] = {} | |
| 178 | + for value, canon in _REGIONS.items(): | |
| 179 | + for mid in self._ids(value): | |
| 180 | + regions.setdefault(mid, canon) | |
| 181 | + if limit and len(regions) >= len(ids): | |
| 182 | + break | |
| 183 | + | |
| 184 | + listings: list[StListing] = [] | |
| 185 | + for mid in ids: | |
| 186 | + # fiche rafraîchie une fois par mois (pas de lastmod côté liste) | |
| 187 | + det = self.detail(mid, time.strftime("%Y-%m"), | |
| 188 | + lambda i=mid: self._fiche(i)) | |
| 189 | + if not det.get("title"): | |
| 190 | + continue | |
| 191 | + | |
| 192 | + types = det.get("types") or "" | |
| 193 | + tl = types.lower() | |
| 194 | + ptype = next((canon for needle, canon in _TYPES if needle in tl), | |
| 195 | + "Gîte") | |
| 196 | + | |
| 197 | + details = {k: v for k, v in { | |
| 198 | + "types_hebergement": types, | |
| 199 | + "classification_citq": det.get("classification"), | |
| 200 | + "telephone": det.get("phone"), | |
| 201 | + "site_web": det.get("website"), | |
| 202 | + "reseau": "Gîtes et Auberges du Passant certifiés " | |
| 203 | + "(Terroir et Saveurs du Québec)", | |
| 204 | + }.items() if v} | |
| 205 | + | |
| 206 | + street = det.get("street") or "" | |
| 207 | + city = det.get("city") or "" | |
| 208 | + listings.append(StListing( | |
| 209 | + source=self.source_id, | |
| 210 | + external_id=mid, | |
| 211 | + url=FICHE.format(id=mid), | |
| 212 | + title=det["title"], | |
| 213 | + property_type=ptype, | |
| 214 | + address=", ".join(p for p in (street, city) if p), | |
| 215 | + city=city, | |
| 216 | + region=regions.get(mid, ""), | |
| 217 | + citq=det.get("citq") or "", | |
| 218 | + description=det.get("description") or "", | |
| 219 | + details=details, | |
| 220 | + images=det.get("images") or [], | |
| 221 | + )) | |
| 222 | + if limit and len(listings) >= limit: | |
| 223 | + break | |
| 224 | + return listings | |
added
louka/shortterm/connectors/hebergementquebec.py
+30 −0
@@ -0,0 +1,30 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/hebergementquebec.py : Hébergement Québec — SOURCE MORTE. | |
| 4 | +# | |
| 5 | +# Vérifié le 2026-08-23 : | |
| 6 | +# - hebergementquebec.net (l'ancien répertoire d'hébergements de la CITQ) | |
| 7 | +# redirige en 301 vers https://citq.qc.ca/fr/index.php — le site | |
| 8 | +# d'information de la Corporation de l'industrie touristique du Québec, | |
| 9 | +# qui n'affiche aucune annonce ni répertoire d'établissements scrapable. | |
| 10 | +# - hebergementquebec.com est aujourd'hui un hébergeur WEB (« Hébergement | |
| 11 | +# Web Québécois — à partir de 15 $/mois », Alice Média) : rien à voir | |
| 12 | +# avec l'hébergement touristique. | |
| 13 | +# - Aucun autre domaine équivalent trouvé (Serper, gl=ca hl=fr). | |
| 14 | +# | |
| 15 | +# Le module est conservé (avec disabled = True, donc exclu du registre) au | |
| 16 | +# cas où le répertoire renaîtrait ; l'inventaire CITQ officiel reste couvert | |
| 17 | +# par le connecteur bonjourquebec. | |
| 18 | +# ----------------------------------------------------------------------------- | |
| 19 | +from __future__ import annotations | |
| 20 | + | |
| 21 | +from ..schema import StListing | |
| 22 | +from .base import StConnector | |
| 23 | + | |
| 24 | + | |
| 25 | +class HebergementQuebec(StConnector): | |
| 26 | + source_id = "hebergementquebec" | |
| 27 | + disabled = True # site disparu (voir en-tête) | |
| 28 | + | |
| 29 | + def fetch(self) -> list[StListing]: | |
| 30 | + return [] | |
added
louka/shortterm/connectors/kijiji.py
+317 −0
@@ -0,0 +1,317 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/kijiji.py : Kijiji (kijiji.ca) — petites annonces, catégorie | |
| 4 | +# « Locations de vacances » à destination du Québec (c814 : les annonces y | |
| 5 | +# sont classées par province de LA PROPRIÉTÉ, pas de l'annonceur — la | |
| 6 | +# recherche « quebec/c800 » retournait des condos en Floride affichés | |
| 7 | +# depuis Québec). | |
| 8 | +# | |
| 9 | +# Méthode (Kijiji est un Next.js derrière un anti-bot : HTML via Bright Data | |
| 10 | +# Web Unlocker, Scrapfly ASP en secours — même recette que le connecteur | |
| 11 | +# Airbnb) : | |
| 12 | +# 1. LISTE : /b-vacation-rentals-quebec/canada/c814l0 (+ /page-N/) — | |
| 13 | +# __NEXT_DATA__ → __APOLLO_STATE__ → searchResultsPageByUrl → results | |
| 14 | +# (topListings + mainListings, ~40/page, totalCount ≈ 75). Chaque entité | |
| 15 | +# StandardListing donne titre, prix (cents), photos, attributs canoniques. | |
| 16 | +# 2. DÉTAIL (cache self.detail) : la page /v-…/<id> embarque le même | |
| 17 | +# APOLLO_STATE avec en plus la description complète, les coordonnées | |
| 18 | +# GPS, toutes les photos et les attributs en clair (« 2 bedrooms and | |
| 19 | +# den », région touristique dans l'attribut « city », animaux…). | |
| 20 | +# | |
| 21 | +# Filtres court terme : on ne garde que les annonces OFFER qui ressemblent à | |
| 22 | +# un hébergement (attributs chambres/personnes/type de vacances présents — | |
| 23 | +# la catégorie contient aussi maillots de bain, machines à espresso…) et on | |
| 24 | +# écarte les locations au mois (« 31 jours et plus », monthly…). Le prix | |
| 25 | +# Kijiji est un montant sans période : price_night n'est rempli que si le | |
| 26 | +# texte précise « /nuit » ou « /semaine », sinon le montant affiché va dans | |
| 27 | +# details.prix_affiche. | |
| 28 | +# | |
| 29 | +# Réglage env : LOUKA_KIJIJI_LIMIT (nb max d'annonces, pour tester petit). | |
| 30 | +# ----------------------------------------------------------------------------- | |
| 31 | +from __future__ import annotations | |
| 32 | + | |
| 33 | +import json | |
| 34 | +import os | |
| 35 | +import re | |
| 36 | +import time | |
| 37 | + | |
| 38 | +import requests | |
| 39 | + | |
| 40 | +from ..schema import StListing, normalize_region, parse_price_night, REGIONS | |
| 41 | +from .airbnb import _region_from_latlng | |
| 42 | +from .base import StConnector | |
| 43 | + | |
| 44 | +BRIGHTDATA_API = "https://api.brightdata.com/request" | |
| 45 | +BASE = "https://www.kijiji.ca" | |
| 46 | +LISTE = BASE + "/b-vacation-rentals-quebec/canada/{page}c814l0" | |
| 47 | + | |
| 48 | +# attributs canoniques qui signent un vrai hébergement | |
| 49 | +_ATTRS_HEBERGEMENT = {"numberbedrooms", "maxpeople", "vacationtype", | |
| 50 | + "numberbathrooms", "minnights"} | |
| 51 | + | |
| 52 | +# location au mois (ou plus) : hors mandat court terme | |
| 53 | +_MENSUEL_RE = re.compile( | |
| 54 | + r"au mois|par mois|/\s*mois|mensuel|monthly|per\s+month|/\s*month" | |
| 55 | + r"|3[01]\s*jours\s*(?:et plus|minimum|min)|month(?:ly)?\s+rental", re.I) | |
| 56 | + | |
| 57 | +_NUIT_RE = re.compile(r"(\d[\d\s,.]{0,9})\s*\$\s*(?:/|par|la|per)?\s*" | |
| 58 | + r"(?:nuit|night)", re.I) | |
| 59 | +_SEMAINE_RE = re.compile(r"(\d[\d\s,.]{0,9})\s*\$\s*(?:/|par|la|per)?\s*" | |
| 60 | + r"(?:sem(?:aine)?|week)", re.I) | |
| 61 | + | |
| 62 | +_TYPE_HINTS = [ | |
| 63 | + ("chalet", "Chalet"), ("cottage", "Chalet"), ("cabin", "Chalet"), | |
| 64 | + ("chaumière", "Chalet"), ("condo", "Condo"), ("appartement", "Appartement"), | |
| 65 | + ("apartment", "Appartement"), ("loft", "Loft"), ("studio", "Studio"), | |
| 66 | + ("maison", "Maison"), ("house", "Maison"), ("gîte", "Gîte"), | |
| 67 | + ("gite", "Gîte"), ("auberge", "Auberge"), ("yourte", "Yourte"), | |
| 68 | + ("yurt", "Yourte"), ("dôme", "Dôme"), ("dome", "Dôme"), | |
| 69 | + ("chambre", "Chambre"), ("room", "Chambre"), ("camping", "Camping"), | |
| 70 | + ("roulotte", "Prêt-à-camper"), ("trailer", "Prêt-à-camper"), | |
| 71 | +] | |
| 72 | + | |
| 73 | + | |
| 74 | +def _num(texts: list[str]) -> float | None: | |
| 75 | + """Premier nombre d'une liste de valeurs Kijiji (« 2 bedrooms and den »).""" | |
| 76 | + for t in texts or []: | |
| 77 | + m = re.search(r"(\d+(?:[.,]5)?)", str(t)) | |
| 78 | + if m: | |
| 79 | + return float(m.group(1).replace(",", ".")) | |
| 80 | + return None | |
| 81 | + | |
| 82 | + | |
| 83 | +def _attrs(entity: dict) -> dict[str, list[str]]: | |
| 84 | + """{canonicalName: values (humaines si présentes, sinon canoniques)}.""" | |
| 85 | + out: dict[str, list[str]] = {} | |
| 86 | + for a in ((entity.get("attributes") or {}).get("all") or []): | |
| 87 | + name = (a or {}).get("canonicalName") or "" | |
| 88 | + vals = a.get("values") or a.get("canonicalValues") or [] | |
| 89 | + if name: | |
| 90 | + out[name] = [str(v) for v in vals] | |
| 91 | + return out | |
| 92 | + | |
| 93 | + | |
| 94 | +class KijijiCt(StConnector): | |
| 95 | + source_id = "kijiji_ct" | |
| 96 | + request_delay = 0.5 | |
| 97 | + | |
| 98 | + # -- fetch HTML (anti-bot) ---------------------------------------------- | |
| 99 | + def _brightdata(self, url: str) -> str: | |
| 100 | + key = os.environ.get("BRIGHTDATA_API_KEY") | |
| 101 | + if not key: | |
| 102 | + return "" | |
| 103 | + wait = self.request_delay - (time.time() - self._last_request) | |
| 104 | + if wait > 0: | |
| 105 | + time.sleep(wait) | |
| 106 | + try: | |
| 107 | + resp = requests.post( | |
| 108 | + BRIGHTDATA_API, | |
| 109 | + headers={"Authorization": f"Bearer {key}", | |
| 110 | + "Content-Type": "application/json"}, | |
| 111 | + json={"zone": os.environ.get("BRIGHTDATA_ZONE", "web_unlocker1"), | |
| 112 | + "url": url, "format": "raw"}, | |
| 113 | + timeout=150) | |
| 114 | + except requests.RequestException: | |
| 115 | + return "" | |
| 116 | + finally: | |
| 117 | + self._last_request = time.time() | |
| 118 | + return resp.text if resp.status_code == 200 else "" | |
| 119 | + | |
| 120 | + def _html(self, url: str) -> str: | |
| 121 | + html = self._brightdata(url) | |
| 122 | + if "__NEXT_DATA__" in html: | |
| 123 | + return html | |
| 124 | + return self.get_scrapfly(url, render_js=False, asp=True) | |
| 125 | + | |
| 126 | + # -- parse APOLLO_STATE --------------------------------------------------- | |
| 127 | + @staticmethod | |
| 128 | + def _apollo(html: str) -> dict: | |
| 129 | + m = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', | |
| 130 | + html, re.S) | |
| 131 | + if not m: | |
| 132 | + return {} | |
| 133 | + try: | |
| 134 | + data = json.loads(m.group(1)) | |
| 135 | + except ValueError: | |
| 136 | + return {} | |
| 137 | + return (data.get("props") or {}).get("pageProps", {}) \ | |
| 138 | + .get("__APOLLO_STATE__") or {} | |
| 139 | + | |
| 140 | + @staticmethod | |
| 141 | + def _search_page(apollo: dict) -> tuple[list[dict], int]: | |
| 142 | + """(entités StandardListing de la page, totalCount).""" | |
| 143 | + root = apollo.get("ROOT_QUERY") or {} | |
| 144 | + for key, srp in root.items(): | |
| 145 | + if not key.startswith("searchResultsPageByUrl"): | |
| 146 | + continue | |
| 147 | + res = (srp or {}).get("results") or {} | |
| 148 | + total = ((srp or {}).get("pagination") or {}).get("totalCount") or 0 | |
| 149 | + refs: list[str] = [] | |
| 150 | + for rk, rv in res.items(): | |
| 151 | + if rk.startswith(("mainListings", "topListings")) \ | |
| 152 | + and isinstance(rv, list): | |
| 153 | + refs.extend(x.get("__ref") for x in rv | |
| 154 | + if isinstance(x, dict) and x.get("__ref")) | |
| 155 | + return [apollo[r] for r in refs if r in apollo], int(total) | |
| 156 | + return [], 0 | |
| 157 | + | |
| 158 | + # -- détail ---------------------------------------------------------------- | |
| 159 | + def _detail(self, url: str, eid: str) -> dict: | |
| 160 | + apollo = self._apollo(self._html(url)) | |
| 161 | + e = apollo.get(f"StandardListing:{eid}") or {} | |
| 162 | + if not e: | |
| 163 | + return {} | |
| 164 | + attrs = _attrs(e) | |
| 165 | + loc = e.get("location") or {} | |
| 166 | + coords = loc.get("coordinates") or {} | |
| 167 | + return { | |
| 168 | + "description": (e.get("description") or "")[:5000], | |
| 169 | + "images": [u for u in (e.get("imageUrls") or []) | |
| 170 | + if isinstance(u, str) and u.startswith("https://")][:20], | |
| 171 | + "address": loc.get("address") or "", | |
| 172 | + "lat": coords.get("latitude"), | |
| 173 | + "lng": coords.get("longitude"), | |
| 174 | + "attrs": attrs, | |
| 175 | + "region": (attrs.get("city") or [""])[0], # région touristique QC | |
| 176 | + } | |
| 177 | + | |
| 178 | + # -- contrat ------------------------------------------------------------ | |
| 179 | + def fetch(self) -> list[StListing]: | |
| 180 | + limit = int(os.environ.get("LOUKA_KIJIJI_LIMIT", "0") or 0) | |
| 181 | + | |
| 182 | + entities: list[dict] = [] | |
| 183 | + vus: set[str] = set() | |
| 184 | + page, total = 1, None | |
| 185 | + while page <= 10: | |
| 186 | + seg = "" if page == 1 else f"page-{page}/" | |
| 187 | + ents, tot = self._search_page(self._apollo( | |
| 188 | + self._html(LISTE.format(page=seg)))) | |
| 189 | + if not ents: | |
| 190 | + break | |
| 191 | + total = tot or total | |
| 192 | + nouveaux = 0 | |
| 193 | + for e in ents: | |
| 194 | + eid = str(e.get("id") or "") | |
| 195 | + if eid and eid not in vus: | |
| 196 | + vus.add(eid) | |
| 197 | + entities.append(e) | |
| 198 | + nouveaux += 1 | |
| 199 | + if nouveaux == 0 or (total and len(vus) >= total): | |
| 200 | + break | |
| 201 | + if limit and len(entities) >= limit * 3: # marge pour les filtres | |
| 202 | + break | |
| 203 | + page += 1 | |
| 204 | + | |
| 205 | + listings: list[StListing] = [] | |
| 206 | + for e in entities: | |
| 207 | + eid = str(e.get("id") or "") | |
| 208 | + url = e.get("url") or "" | |
| 209 | + title = (e.get("title") or "").strip() | |
| 210 | + if not eid or not url or not title: | |
| 211 | + continue | |
| 212 | + if (e.get("type") or "OFFER") != "OFFER": | |
| 213 | + continue | |
| 214 | + attrs = _attrs(e) | |
| 215 | + if not (_ATTRS_HEBERGEMENT & set(attrs)): | |
| 216 | + continue # maillots de bain, cafetières, vans… | |
| 217 | + texte = f"{title}\n{e.get('description') or ''}" | |
| 218 | + if _MENSUEL_RE.search(texte): | |
| 219 | + continue # location au mois : hors mandat | |
| 220 | + | |
| 221 | + key = json.dumps([title, e.get("imageCount"), | |
| 222 | + (e.get("price") or {}).get("amount")], | |
| 223 | + ensure_ascii=False) | |
| 224 | + try: | |
| 225 | + det = self.detail(eid, key, | |
| 226 | + lambda u=url, i=eid: self._detail(u, i)) | |
| 227 | + except Exception: # une fiche cassée ≠ annonce perdue | |
| 228 | + det = {} | |
| 229 | + if det.get("attrs"): | |
| 230 | + attrs = det["attrs"] | |
| 231 | + texte = (f"{title}\n" | |
| 232 | + f"{det.get('description') or e.get('description') or ''}") | |
| 233 | + if _MENSUEL_RE.search(texte): | |
| 234 | + continue | |
| 235 | + | |
| 236 | + # prix : montant en cents, période seulement si le texte la donne | |
| 237 | + price_night = None | |
| 238 | + price_label = "" | |
| 239 | + amount = (e.get("price") or {}).get("amount") | |
| 240 | + montant = round(amount / 100, 2) if isinstance( | |
| 241 | + amount, (int, float)) and amount else None | |
| 242 | + m = _NUIT_RE.search(texte) | |
| 243 | + if m: | |
| 244 | + price_label = f"{m.group(1).strip()} $ / nuit" | |
| 245 | + elif _SEMAINE_RE.search(texte): | |
| 246 | + price_label = f"{_SEMAINE_RE.search(texte).group(1).strip()}" \ | |
| 247 | + " $ / semaine" | |
| 248 | + elif montant: | |
| 249 | + nuit = (attrs.get("minnights") or ["1"])[0] | |
| 250 | + if str(nuit) in ("", "1"): # 1 nuit min : montant ≈ par nuit | |
| 251 | + price_night = montant | |
| 252 | + price_label = f"{montant:g} $" | |
| 253 | + if price_label and price_night is None: | |
| 254 | + price_night = parse_price_night(price_label) | |
| 255 | + | |
| 256 | + hay = title.lower() | |
| 257 | + ptype = next((canon for needle, canon in _TYPE_HINTS | |
| 258 | + if needle in hay), "") | |
| 259 | + | |
| 260 | + pets = None | |
| 261 | + if attrs.get("petsallowed"): | |
| 262 | + v = attrs["petsallowed"][0].lower() | |
| 263 | + pets = "oui" if v in ("1", "yes", "oui") else "non" | |
| 264 | + | |
| 265 | + address = det.get("address") or (e.get("location") or {}).get( | |
| 266 | + "address") or "" | |
| 267 | + # « 60 Rue Quaile, Otter Lake, QC J0X 2P0 » → ville = Otter Lake | |
| 268 | + m = re.search(r"([^,]+),\s*(?:QC|Qu[ée]bec)\b", address) | |
| 269 | + city = m.group(1).strip() if m else "" | |
| 270 | + coords = ((e.get("location") or {}).get("coordinates") or {}) | |
| 271 | + lat = det.get("lat") if det.get("lat") is not None \ | |
| 272 | + else coords.get("latitude") | |
| 273 | + lng = det.get("lng") if det.get("lng") is not None \ | |
| 274 | + else coords.get("longitude") | |
| 275 | + | |
| 276 | + # région : attribut « city » de Kijiji (souvent la région | |
| 277 | + # touristique), sinon le point GPS (centroïde le plus proche) | |
| 278 | + region = normalize_region(det.get("region") or "") | |
| 279 | + if region not in REGIONS: | |
| 280 | + region = _region_from_latlng(lat, lng) | |
| 281 | + | |
| 282 | + details = {k: v for k, v in { | |
| 283 | + "prix_affiche": montant if price_night is None else None, | |
| 284 | + "min_nights": (attrs.get("minnights") or [None])[0], | |
| 285 | + "vacation_type": (attrs.get("vacationtype") or [None])[0], | |
| 286 | + "disponible_du": (attrs.get("availabilitystartdate") | |
| 287 | + or [None])[0], | |
| 288 | + "disponible_au": (attrs.get("availabilityenddate") | |
| 289 | + or [None])[0], | |
| 290 | + }.items() if v} | |
| 291 | + | |
| 292 | + listings.append(StListing( | |
| 293 | + source=self.source_id, | |
| 294 | + external_id=eid, | |
| 295 | + url=url, | |
| 296 | + title=title, | |
| 297 | + property_type=ptype, | |
| 298 | + address=address, | |
| 299 | + city=city, | |
| 300 | + region=region, | |
| 301 | + price_night=price_night, | |
| 302 | + price_label=price_label, | |
| 303 | + capacity=_num(attrs.get("maxpeople")), | |
| 304 | + bedrooms=_num(attrs.get("numberbedrooms")), | |
| 305 | + bathrooms=_num(attrs.get("numberbathrooms")), | |
| 306 | + pets=pets, | |
| 307 | + description=det.get("description") or "", | |
| 308 | + details=details, | |
| 309 | + images=det.get("images") | |
| 310 | + or [u for u in (e.get("imageUrls") or []) | |
| 311 | + if isinstance(u, str)][:20], | |
| 312 | + lat=lat, | |
| 313 | + lng=lng, | |
| 314 | + )) | |
| 315 | + if limit and len(listings) >= limit: | |
| 316 | + break | |
| 317 | + return listings | |
added
louka/shortterm/connectors/tremblantliving.py
+163 −0
@@ -0,0 +1,163 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/tremblantliving.py : Tremblant Living (tremblantliving.com → | |
| 4 | +# tremblantliving.ca) — agence de chalets et condos de luxe à Mont-Tremblant | |
| 5 | +# (~35 propriétés, moteur Streamline VRS sur WordPress). | |
| 6 | +# | |
| 7 | +# Méthode : property-sitemap.xml (~37 fiches /property/ et /rental/, lastmod | |
| 8 | +# = clé du cache détail). Chaque page détail embarque un JSON-LD schema.org | |
| 9 | +# VacationRental complet : identifiant Streamline (unit_id), chambres, | |
| 10 | +# salles de bain, capacité, note/avis, adresse, lat/lng, photos (galerie | |
| 11 | +# streamlinevrs.com). La description longue vient du bloc | |
| 12 | +# <div class="description block">, les commodités des <li class="amenity_item">. | |
| 13 | +# Pas de prix statique : les tarifs passent par l'API Streamline | |
| 14 | +# (admin-ajax.php), bloquée par Cloudflare en POST — price_night reste vide. | |
| 15 | +# Les /monthly-rentals/ (units-sitemap.xml) sont du long terme : ignorés. | |
| 16 | +# ----------------------------------------------------------------------------- | |
| 17 | +from __future__ import annotations | |
| 18 | + | |
| 19 | +import html as _html | |
| 20 | +import json | |
| 21 | +import os | |
| 22 | +import re | |
| 23 | + | |
| 24 | +from ..schema import StListing | |
| 25 | +from .base import StConnector | |
| 26 | + | |
| 27 | +SITE = "https://www.tremblantliving.ca" | |
| 28 | +SITEMAP = SITE + "/property-sitemap.xml" | |
| 29 | + | |
| 30 | +# type déduit du nom de la fiche (agence ~100 % chalets et condos) | |
| 31 | +_TYPE_HINTS = [ | |
| 32 | + ("penthouse", "Condo"), ("condo", "Condo"), ("appartement", "Appartement"), | |
| 33 | + ("apartment", "Appartement"), ("loft", "Loft"), ("studio", "Studio"), | |
| 34 | + ("estate", "Maison"), ("maison", "Maison"), ("house", "Maison"), | |
| 35 | + ("villa", "Maison"), ("chalet", "Chalet"), ("cottage", "Chalet"), | |
| 36 | + ("cabin", "Chalet"), ("lodge", "Chalet"), | |
| 37 | +] | |
| 38 | + | |
| 39 | +_TAG_RE = re.compile(r"<[^>]+>") | |
| 40 | + | |
| 41 | + | |
| 42 | +def _text(fragment: str) -> str: | |
| 43 | + return _html.unescape(re.sub(r"\s+", " ", _TAG_RE.sub(" ", fragment))).strip() | |
| 44 | + | |
| 45 | + | |
| 46 | +def _f(v) -> float | None: | |
| 47 | + try: | |
| 48 | + return float(v) if v is not None else None | |
| 49 | + except (TypeError, ValueError): | |
| 50 | + return None | |
| 51 | + | |
| 52 | + | |
| 53 | +class TremblantLiving(StConnector): | |
| 54 | + source_id = "tremblant_living" | |
| 55 | + | |
| 56 | + # -- page détail -------------------------------------------------------- | |
| 57 | + def _detail(self, url: str) -> dict: | |
| 58 | + h = self.get(url).text | |
| 59 | + d: dict = {} | |
| 60 | + | |
| 61 | + for block in re.findall(r'<script type="application/ld\+json"[^>]*>' | |
| 62 | + r"(.*?)</script>", h, re.S): | |
| 63 | + try: | |
| 64 | + ld = json.loads(block) | |
| 65 | + except ValueError: | |
| 66 | + continue | |
| 67 | + if ld.get("@type") == "VacationRental": | |
| 68 | + d["ld"] = ld | |
| 69 | + break | |
| 70 | + | |
| 71 | + # description longue : <div class="description block"><article>… | |
| 72 | + m = re.search(r'(?s)<div class="description block[^"]*"[^>]*>.*?' | |
| 73 | + r"<article[^>]*>(.*?)</article>", h) | |
| 74 | + if m: | |
| 75 | + texte = re.sub(r"<br\s*/?>", "\n", m.group(1)) | |
| 76 | + texte = _html.unescape(_TAG_RE.sub(" ", texte)) | |
| 77 | + texte = re.sub(r"[ \t]+", " ", texte) | |
| 78 | + texte = re.sub(r"\n\s+", "\n", texte).strip() | |
| 79 | + d["description"] = texte[:5000] | |
| 80 | + | |
| 81 | + # commodités : <li class="amenity_item"> avec coche (les entêtes de | |
| 82 | + # catégorie sont des <li> en gras sans icône fa-check) | |
| 83 | + amen: list[str] = [] | |
| 84 | + for li in re.findall(r'(?s)<li class="amenity_item"[^>]*>(.*?)</li>', h): | |
| 85 | + if "fa-check" not in li: | |
| 86 | + continue | |
| 87 | + t = _text(li) | |
| 88 | + if t and t not in amen: | |
| 89 | + amen.append(t) | |
| 90 | + if amen: | |
| 91 | + d["amenities"] = amen | |
| 92 | + return d | |
| 93 | + | |
| 94 | + # -- contrat ------------------------------------------------------------ | |
| 95 | + def fetch(self) -> list[StListing]: | |
| 96 | + limit = int(os.environ.get("LOUKA_TREMBLANT_LIMIT", "0") or 0) | |
| 97 | + xml = self.get(SITEMAP).text | |
| 98 | + entries = re.findall(r"(?s)<url>\s*<loc>([^<]+)</loc>" | |
| 99 | + r"(?:\s*<lastmod>([^<]*)</lastmod>)?", xml) | |
| 100 | + | |
| 101 | + listings: list[StListing] = [] | |
| 102 | + vus: set[str] = set() | |
| 103 | + for url, lastmod in entries: | |
| 104 | + parts = [p for p in url.split("/") if p] | |
| 105 | + # …/property/<slug>/ ou …/rental/<slug>/ (les /monthly-rentals/ | |
| 106 | + # sont dans units-sitemap.xml : long terme, hors mandat) | |
| 107 | + if len(parts) < 4 or parts[-2] not in ("property", "rental"): | |
| 108 | + continue | |
| 109 | + slug = parts[-1] | |
| 110 | + if slug in vus: | |
| 111 | + continue | |
| 112 | + vus.add(slug) | |
| 113 | + | |
| 114 | + det = self.detail(slug, lastmod or "v1", | |
| 115 | + lambda u=url: self._detail(u)) | |
| 116 | + ld = det.get("ld") or {} | |
| 117 | + if not ld: | |
| 118 | + continue | |
| 119 | + place = ld.get("containsPlace") or {} | |
| 120 | + addr = ld.get("address") or {} | |
| 121 | + agg = ld.get("aggregateRating") or {} | |
| 122 | + occupancy = (place.get("occupancy") or {}).get("value") | |
| 123 | + | |
| 124 | + title = _text(str(ld.get("name") or slug)) | |
| 125 | + hay = f"{title} {slug}".lower() | |
| 126 | + ptype = next((canon for needle, canon in _TYPE_HINTS | |
| 127 | + if needle in hay), "Chalet") | |
| 128 | + | |
| 129 | + amen = det.get("amenities") or [] | |
| 130 | + pets = "oui" if any("pet friendly" in a.lower() | |
| 131 | + for a in amen) else None | |
| 132 | + | |
| 133 | + imgs = ld.get("image") or [] | |
| 134 | + if isinstance(imgs, str): | |
| 135 | + imgs = [imgs] | |
| 136 | + reviews = agg.get("reviewCount") | |
| 137 | + listings.append(StListing( | |
| 138 | + source=self.source_id, | |
| 139 | + external_id=str(ld.get("identifier") or slug), | |
| 140 | + url=url, | |
| 141 | + title=title, | |
| 142 | + property_type=ptype, | |
| 143 | + address=_text(str(addr.get("streetAddress") or "")), | |
| 144 | + city=_text(str(addr.get("addressLocality") or "Mont-Tremblant")), | |
| 145 | + region="Laurentides", | |
| 146 | + capacity=_f(occupancy), | |
| 147 | + bedrooms=_f(place.get("numberOfBedrooms")), | |
| 148 | + bathrooms=_f(place.get("numberOfBathroomsTotal")), | |
| 149 | + pets=pets, | |
| 150 | + rating=_f(agg.get("ratingValue")), | |
| 151 | + reviews=int(reviews) if reviews else None, | |
| 152 | + description=det.get("description") | |
| 153 | + or _text(str(ld.get("description") or "")), | |
| 154 | + amenities=amen, | |
| 155 | + details={"postal_code": addr.get("postalCode") or ""}, | |
| 156 | + images=[u for u in imgs if isinstance(u, str) | |
| 157 | + and u.startswith("https://")][:20], | |
| 158 | + lat=_f(ld.get("latitude")), | |
| 159 | + lng=_f(ld.get("longitude")), | |
| 160 | + )) | |
| 161 | + if limit and len(listings) >= limit: | |
| 162 | + break | |
| 163 | + return listings | |
| 164 | ||