court terme : 2e vague — 5 connecteurs (Expedia, Hipcamp, Glamping Hub, Parcs Canada, Sinistar, Corporate Stays) + 3 sources fermées documentées
- expedia : SSR + rendu Scrapfly ASP (pile Vrbo), 13 destinations QC × 2 tris - hipcamp : GraphQL public (LandsSearch bbox QC + détail camper), ~350 terrains - glampinghub : endpoint AJAX ouvert, 208 fiches QC, prix CAD seulement - parcscanada : API GoingToCamp (reservation.pc.gc.ca), oTENTik/Ôasis/MicrOcube des 12 terrains québécois, prix via feeDetails - sinistar : index Algolia public prod_housings (state:QC + quadtree bbox), ~5 600 hébergements de relogement, pas de prix public (assumé) - corporatestays : API REST WordPress properties, 47 meublés MTL/QC - tripadvisor (vertical locations fermé nov. 2024), sonder (liquidée nov. 2025, devenue affilié Booking), blueground (aucun marché QC) : disabled documentés Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
9 changed files +1,396 −0
added
louka/shortterm/connectors/blueground.py
+28 −0
@@ -0,0 +1,28 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/blueground.py : Blueground (theblueground.com) — SOURCE NON | |
| 4 | +# VIABLE (2026-08) : aucun inventaire au Québec. | |
| 5 | +# | |
| 6 | +# Constat au 2026-08-23 : le sitemap officiel (theblueground.com/sitemap.xml) | |
| 7 | +# énumère les 37 marchés de la plateforme par code ville (ath, atl, …, tor, | |
| 8 | +# tyo, vie, wdc, zrh) — le SEUL marché canadien est Toronto (« tor ») ; | |
| 9 | +# ni Montréal ni aucune ville québécoise n'existe (l'URL | |
| 10 | +# /furnished-apartments-montreal-canada renvoie un 404 applicatif). | |
| 11 | +# Blueground n'opère donc pas au Québec ; rien à agréger. | |
| 12 | +# | |
| 13 | +# À réévaluer si la plateforme ouvre Montréal : la mécanique serait alors | |
| 14 | +# simple (pages ville server-rendered + sitemaps par code ville, ex. | |
| 15 | +# sitemap.tor.xml, aucune protection anti-bot rencontrée). | |
| 16 | +# ----------------------------------------------------------------------------- | |
| 17 | +from __future__ import annotations | |
| 18 | + | |
| 19 | +from ..schema import StListing | |
| 20 | +from .base import StConnector | |
| 21 | + | |
| 22 | + | |
| 23 | +class Blueground(StConnector): | |
| 24 | + source_id = "blueground" | |
| 25 | + disabled = True # aucun marché québécois (Toronto = seul marché canadien) | |
| 26 | + | |
| 27 | + def fetch(self) -> list[StListing]: | |
| 28 | + return [] | |
added
louka/shortterm/connectors/corporatestays.py
+201 −0
@@ -0,0 +1,201 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/corporatestays.py : Corporate Stays (corporatestays.com) | |
| 4 | +# | |
| 5 | +# Meublés corporatifs (siège à Montréal), ~450 unités dans les Amériques dont | |
| 6 | +# ~47 au Québec (Montréal, Québec, Laval). Site WordPress (plugin HomeRunner) ; | |
| 7 | +# l'API REST WP expose tout, aucun anti-bot : | |
| 8 | +# 1. TAXONOMIE : GET /wp-json/wp/v2/property-states?per_page=100 → | |
| 9 | +# ids des états québécois (slugs « pq » et « qc », les deux coexistent). | |
| 10 | +# 2. LISTE : GET /wp-json/wp/v2/properties?property-states=<ids>& | |
| 11 | +# per_page=100&page=N — chaque fiche est complète : titre, description | |
| 12 | +# (content.rendered), acf.{bedrooms, bathrooms, maximum_guests, | |
| 13 | +# minimum_stay, geolocation lat/lng/adresse, currency, base_daily_rate}, | |
| 14 | +# lowest_rent/average_rent ($/nuit CAD), photos (acf.photos[].sizes), | |
| 15 | +# taxonomies embarquées cities/types, subtitle = nom de l'immeuble. | |
| 16 | +# 3. COMMODITÉS : GET /wp-json/wp/v2/property_amenities?include=… pour | |
| 17 | +# traduire les ids en libellés (une passe, mise en cache mémoire). | |
| 18 | +# Séjours de 30 nuits minimum en général, mais le prix affiché est bien un | |
| 19 | +# tarif À LA NUIT en CAD (base_daily_rate) → price_night renseigné. | |
| 20 | +# | |
| 21 | +# Réglage env : LOUKA_CORPORATESTAYS_LIMIT (nb max d'annonces, 0 = tout). | |
| 22 | +# ----------------------------------------------------------------------------- | |
| 23 | +from __future__ import annotations | |
| 24 | + | |
| 25 | +import html as _html | |
| 26 | +import os | |
| 27 | +import re | |
| 28 | + | |
| 29 | +from ..schema import StListing | |
| 30 | +from .airbnb import _region_from_latlng | |
| 31 | +from .base import StConnector | |
| 32 | + | |
| 33 | +API = "https://corporatestays.com/wp-json/wp/v2" | |
| 34 | + | |
| 35 | +# slugs de la taxonomie property-states considérés québécois | |
| 36 | +_QC_STATE_SLUGS = {"qc", "pq", "quebec"} | |
| 37 | + | |
| 38 | +# ville → (ville affichée, région touristique) | |
| 39 | +_CITIES = { | |
| 40 | + "montreal": ("Montréal", "Montréal"), | |
| 41 | + "quebec city": ("Québec", "Québec"), | |
| 42 | + "quebec": ("Québec", "Québec"), | |
| 43 | + "laval": ("Laval", "Laval"), | |
| 44 | + "gatineau": ("Gatineau", "Outaouais"), | |
| 45 | +} | |
| 46 | + | |
| 47 | +_TYPES = { | |
| 48 | + "apartment": "Appartement", "condo": "Condo", "studio": "Studio", | |
| 49 | + "loft": "Loft", "house": "Maison", "townhouse": "Maison", | |
| 50 | + "penthouse": "Appartement", "suite": "Appartement", | |
| 51 | +} | |
| 52 | + | |
| 53 | +_TAG_RE = re.compile(r"<[^>]+>") | |
| 54 | + | |
| 55 | + | |
| 56 | +def _num(v) -> float | None: | |
| 57 | + try: | |
| 58 | + return float(v) if v not in (None, "") else None | |
| 59 | + except (TypeError, ValueError): | |
| 60 | + return None | |
| 61 | + | |
| 62 | + | |
| 63 | +def _strip_html(txt: str) -> str: | |
| 64 | + return re.sub(r"\s+", " ", _TAG_RE.sub(" ", _html.unescape(txt or ""))).strip() | |
| 65 | + | |
| 66 | + | |
| 67 | +class CorporateStays(StConnector): | |
| 68 | + source_id = "corporatestays" | |
| 69 | + request_delay = 0.6 | |
| 70 | + | |
| 71 | + def _get_json(self, url: str): | |
| 72 | + return self.get(url, headers={"Accept": "application/json"}).json() | |
| 73 | + | |
| 74 | + # -- taxonomies ----------------------------------------------------------- | |
| 75 | + def _qc_state_ids(self) -> list[int]: | |
| 76 | + states = self._get_json( | |
| 77 | + f"{API}/property-states?per_page=100&_fields=id,slug,count") | |
| 78 | + return [s["id"] for s in states | |
| 79 | + if (s.get("slug") or "").lower() in _QC_STATE_SLUGS | |
| 80 | + and s.get("count")] | |
| 81 | + | |
| 82 | + def _amenity_names(self, ids: set[int]) -> dict[int, str]: | |
| 83 | + names: dict[int, str] = {} | |
| 84 | + todo = sorted(ids) | |
| 85 | + for i in range(0, len(todo), 100): | |
| 86 | + chunk = ",".join(str(x) for x in todo[i:i + 100]) | |
| 87 | + try: | |
| 88 | + for a in self._get_json(f"{API}/property_amenities?per_page=100" | |
| 89 | + f"&_fields=id,name&include={chunk}"): | |
| 90 | + names[a["id"]] = _html.unescape(a.get("name") or "").strip() | |
| 91 | + except Exception: # noqa: BLE001 — libellés manquants ≠ blocage | |
| 92 | + break | |
| 93 | + return names | |
| 94 | + | |
| 95 | + # -- liste ---------------------------------------------------------------- | |
| 96 | + def _all_items(self, state_ids: list[int]) -> list[dict]: | |
| 97 | + items: list[dict] = [] | |
| 98 | + states = ",".join(str(i) for i in state_ids) | |
| 99 | + page = 1 | |
| 100 | + while page <= 20: | |
| 101 | + batch = self._get_json(f"{API}/properties?property-states={states}" | |
| 102 | + f"&per_page=100&page={page}") | |
| 103 | + if not isinstance(batch, list) or not batch: | |
| 104 | + break | |
| 105 | + items.extend(batch) | |
| 106 | + if len(batch) < 100: | |
| 107 | + break | |
| 108 | + page += 1 | |
| 109 | + return items | |
| 110 | + | |
| 111 | + # -- contrat -------------------------------------------------------------- | |
| 112 | + def fetch(self) -> list[StListing]: | |
| 113 | + limit = int(os.environ.get("LOUKA_CORPORATESTAYS_LIMIT", "0") or 0) | |
| 114 | + state_ids = self._qc_state_ids() | |
| 115 | + if not state_ids: | |
| 116 | + return [] | |
| 117 | + items = self._all_items(state_ids) | |
| 118 | + if limit: | |
| 119 | + items = items[:limit] | |
| 120 | + | |
| 121 | + amen_ids = {a for it in items for a in (it.get("property_amenities") | |
| 122 | + or []) if isinstance(a, int)} | |
| 123 | + amen_names = self._amenity_names(amen_ids) if amen_ids else {} | |
| 124 | + | |
| 125 | + listings: list[StListing] = [] | |
| 126 | + seen: set[str] = set() | |
| 127 | + for it in items: | |
| 128 | + pid = str(it.get("id") or "").strip() | |
| 129 | + url = (it.get("link") or "").strip() | |
| 130 | + title = _strip_html((it.get("title") or {}).get("rendered") or "") | |
| 131 | + if not pid or pid in seen or not url or not title: | |
| 132 | + continue | |
| 133 | + seen.add(pid) | |
| 134 | + | |
| 135 | + acf = it.get("acf") or {} | |
| 136 | + geo = acf.get("geolocation") or {} | |
| 137 | + cities = it.get("cities") or [] | |
| 138 | + raw_city = (cities[0].get("name") or "").strip() if cities else "" | |
| 139 | + city, region = _CITIES.get(raw_city.lower(), (raw_city, "")) | |
| 140 | + lat, lng = _num(geo.get("lat")), _num(geo.get("lng")) | |
| 141 | + if not region: | |
| 142 | + region = _region_from_latlng(lat, lng) | |
| 143 | + | |
| 144 | + types = it.get("types") or [] | |
| 145 | + raw_type = (types[0].get("name") or "").strip() if types else "" | |
| 146 | + ptype = _TYPES.get(raw_type.lower(), raw_type) | |
| 147 | + | |
| 148 | + # prix à la nuit (CAD) : tarif de base, sinon plus bas tarif affiché | |
| 149 | + price = None | |
| 150 | + if (acf.get("currency") or "CAD").upper() == "CAD": | |
| 151 | + price = _num(acf.get("base_daily_rate")) \ | |
| 152 | + or _num(it.get("lowest_rent")) | |
| 153 | + price_label = (f"À partir de {price:g} $/nuit" | |
| 154 | + if price is not None else "") | |
| 155 | + | |
| 156 | + images = [] | |
| 157 | + for ph in acf.get("photos") or []: | |
| 158 | + sizes = (ph or {}).get("sizes") or {} | |
| 159 | + u = (sizes.get("large") or sizes.get("homelocal-medium") | |
| 160 | + or sizes.get("medium") or "") | |
| 161 | + if u.startswith("https://") and u not in images: | |
| 162 | + images.append(u) | |
| 163 | + if len(images) >= 15: | |
| 164 | + break | |
| 165 | + | |
| 166 | + amenities = [amen_names[a] for a in (it.get("property_amenities") | |
| 167 | + or []) if amen_names.get(a)] | |
| 168 | + rating = _num(it.get("average_rating")) | |
| 169 | + reviews = it.get("total_reviews") | |
| 170 | + details = {k: v for k, v in { | |
| 171 | + "building": _strip_html(it.get("subtitle") or ""), | |
| 172 | + "min_stay": _num(acf.get("minimum_stay")), | |
| 173 | + "avg_rate_night": _num(it.get("average_rent")), | |
| 174 | + "corporate": True, | |
| 175 | + }.items() if v not in (None, "", 0)} | |
| 176 | + | |
| 177 | + listings.append(StListing( | |
| 178 | + source=self.source_id, | |
| 179 | + external_id=pid, | |
| 180 | + url=url, | |
| 181 | + title=title, | |
| 182 | + property_type=ptype, | |
| 183 | + address=(geo.get("address") or "").strip(), | |
| 184 | + city=city, | |
| 185 | + region=region, | |
| 186 | + price_night=price, | |
| 187 | + price_label=price_label, | |
| 188 | + capacity=_num(acf.get("maximum_guests")), | |
| 189 | + bedrooms=_num(acf.get("bedrooms")), | |
| 190 | + bathrooms=_num(acf.get("bathrooms")), | |
| 191 | + rating=rating if rating else None, | |
| 192 | + reviews=int(reviews) if reviews else None, | |
| 193 | + description=_strip_html((it.get("content") or {}) | |
| 194 | + .get("rendered") or "")[:3000], | |
| 195 | + amenities=amenities, | |
| 196 | + details=details, | |
| 197 | + images=images, | |
| 198 | + lat=lat, | |
| 199 | + lng=lng, | |
| 200 | + ).finalize()) | |
| 201 | + return listings | |
added
louka/shortterm/connectors/expedia.py
+253 −0
@@ -0,0 +1,253 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/expedia.py : Expedia (expedia.ca) — locations de vacances au Québec | |
| 4 | +# (onglet « Homes » : maisons, chalets, condos… — PAS les hôtels). | |
| 5 | +# | |
| 6 | +# Méthode : même pile que Vrbo (groupe Expedia) — anti-bot Akamai + page | |
| 7 | +# client-side. Le SSR (__PLUGIN_STATE__ → controllers.stores.searchResults) | |
| 8 | +# ne porte que les 3 premières cartes (vérifié 2026-08-23), le reste arrive | |
| 9 | +# par GraphQL après hydratation → Scrapfly ASP avec rendu JS + js_scenario de | |
| 10 | +# scrolls, puis parsing DOM des cartes `[data-stid="lodging-card-responsive"]`. | |
| 11 | +# Le filtre « Homes » de la recherche = paramètre d'URL | |
| 12 | +# `categorySearch=vacation_rentals_option` (exclut les hôtels classiques). | |
| 13 | +# | |
| 14 | +# Limites assumées : ~19 cartes rendues par requête (liste virtualisée, comme | |
| 15 | +# Vrbo) → on multiplie destinations × tris (RECOMMENDED + PRICE_LOW_TO_HIGH) | |
| 16 | +# pour élargir la couverture ; pas de lat/lng ni d'adresse sur les cartes. | |
| 17 | +# Recherche SANS dates : Expedia auto-assigne des dates (~2 semaines) et | |
| 18 | +# affiche un prix par nuit avant taxes → price_label « à partir de … ». | |
| 19 | +# (AVEC dates explicites la disponibilité ralentit le rendu : 3 cartes et | |
| 20 | +# aucun prix dans le snapshot — vérifié 2026-08-23, ne pas en remettre.) | |
| 21 | +# L'inventaire recoupe en partie Vrbo (même groupe) mais avec ses propres ids | |
| 22 | +# et des exclusivités hôtelières-résidentielles (apparts-hôtels, glamping). | |
| 23 | +# | |
| 24 | +# Réglage env : LOUKA_EXPEDIA_LIMIT (nb max d'annonces, pour tester petit). | |
| 25 | +# ----------------------------------------------------------------------------- | |
| 26 | +from __future__ import annotations | |
| 27 | + | |
| 28 | +import os | |
| 29 | +import re | |
| 30 | +import sys | |
| 31 | +from urllib.parse import quote | |
| 32 | + | |
| 33 | +from bs4 import BeautifulSoup | |
| 34 | + | |
| 35 | +from ..schema import StListing | |
| 36 | +from .base import StConnector | |
| 37 | + | |
| 38 | +# (destination Expedia, ville affichée par défaut, région touristique QC) | |
| 39 | +DESTINATIONS = [ | |
| 40 | + ("Mont-Tremblant, Quebec, Canada", "Mont-Tremblant", "Laurentides"), | |
| 41 | + ("Saint-Sauveur, Quebec, Canada", "Saint-Sauveur", "Laurentides"), | |
| 42 | + ("Magog, Quebec, Canada", "Magog", "Cantons-de-l'Est"), | |
| 43 | + ("Bromont, Quebec, Canada", "Bromont", "Cantons-de-l'Est"), | |
| 44 | + ("Baie-Saint-Paul, Quebec, Canada", "Baie-Saint-Paul", "Charlevoix"), | |
| 45 | + ("La Malbaie, Quebec, Canada", "La Malbaie", "Charlevoix"), | |
| 46 | + ("Quebec City, Quebec, Canada", "Québec", "Québec"), | |
| 47 | + ("Montreal, Quebec, Canada", "Montréal", "Montréal"), | |
| 48 | + ("Perce, Quebec, Canada", "Percé", "Gaspésie"), | |
| 49 | + ("Rimouski, Quebec, Canada", "Rimouski", "Bas-Saint-Laurent"), | |
| 50 | + ("Saguenay, Quebec, Canada", "Saguenay", "Saguenay–Lac-Saint-Jean"), | |
| 51 | + ("Shawinigan, Quebec, Canada", "Shawinigan", "Mauricie"), | |
| 52 | + ("Gatineau, Quebec, Canada", "Gatineau", "Outaouais"), | |
| 53 | +] | |
| 54 | + | |
| 55 | +# Deux tris par destination pour dépasser la vingtaine de cartes rendues | |
| 56 | +SORTS = ["RECOMMENDED", "PRICE_LOW_TO_HIGH"] | |
| 57 | + | |
| 58 | +# Mot-clé du libellé type Expedia (en) → type canonique Lou-Ka | |
| 59 | +TYPE_MAP = { | |
| 60 | + "apartment": "Appartement", "apart-hotel": "Appartement", | |
| 61 | + "aparthotel": "Appartement", "condo": "Condo", "chalet": "Chalet", | |
| 62 | + "cabin": "Chalet", "cottage": "Chalet", "house": "Maison", | |
| 63 | + "home": "Maison", "villa": "Maison", "townhouse": "Maison", | |
| 64 | + "bungalow": "Maison", "studio": "Studio", "loft": "Loft", | |
| 65 | + "room": "Chambre", "guesthouse": "Gîte", "bed & breakfast": "Gîte", | |
| 66 | + "lodge": "Auberge", "hostel": "Auberge", "yurt": "Yourte", | |
| 67 | + "tiny house": "Mini-maison", "houseboat": "Autre", | |
| 68 | +} | |
| 69 | + | |
| 70 | +_ID_RE = re.compile(r"\.h(\d+)\.Hotel-Information") | |
| 71 | +_TYPELINE_RE = re.compile(r"^(?:Entire|Private|Shared)\s+(.+?)(?:\s+by\s+Vrbo)?$", | |
| 72 | + re.I) | |
| 73 | +_SLEEPS_RE = re.compile(r"Sleeps\s+(\d+)") | |
| 74 | +_BEDROOMS_RE = re.compile(r"(\d+)\s*bedrooms?") | |
| 75 | +_BATHROOMS_RE = re.compile(r"([\d.]+)\s*bathrooms?") | |
| 76 | +_RATING_RE = re.compile(r"([\d.]+)\s*out of 10") | |
| 77 | +_REVIEWS_RE = re.compile(r"\((\d[\d,\s]*)\s*reviews?\)") | |
| 78 | +_PRICE_RE = re.compile(r"The current price is CA\s*\$([\d,]+)") # \s : | |
| 79 | + | |
| 80 | +# Segments de carte qui ne sont PAS une localité (drapeaux, commodités…) | |
| 81 | +_NOT_A_PLACE = re.compile( | |
| 82 | + r"refundable|reserve now|sign in|member price|pool|hot tub|washer|dryer|" | |
| 83 | + r"kitchen|parking|wifi|out of 10|review|current price|previous price|" | |
| 84 | + r"total|includes|off\b|ad\b|sleeps|photo gallery|show (previous|next)|" | |
| 85 | + r"more information|opens", re.I) | |
| 86 | +# Hors Québec possible dans les résultats frontaliers (Gatineau → Ottawa…) | |
| 87 | +_OUT_OF_QC = re.compile(r"ottawa|ontario|vermont|new hampshire|new york|maine", | |
| 88 | + re.I) | |
| 89 | + | |
| 90 | + | |
| 91 | +class Expedia(StConnector): | |
| 92 | + source_id = "expedia" | |
| 93 | + request_delay = 1.0 | |
| 94 | + | |
| 95 | + # -- parsing d'une carte ---------------------------------------------------- | |
| 96 | + def _parse_card(self, card, city: str, region: str) -> StListing | None: | |
| 97 | + link = card.select_one('a[data-stid="open-product-information"]') \ | |
| 98 | + or card.select_one('a[href*=".Hotel-Information"]') | |
| 99 | + href = (link.get("href") if link else "") or "" | |
| 100 | + m = _ID_RE.search(href) | |
| 101 | + if not m: | |
| 102 | + return None # carte commanditée / lien de connexion | |
| 103 | + external_id = m.group(1) | |
| 104 | + url = "https://www.expedia.ca" + href.split("?")[0].lstrip() | |
| 105 | + if not url.startswith("https://www.expedia.ca/"): | |
| 106 | + url = f"https://www.expedia.ca/h{external_id}.Hotel-Information" | |
| 107 | + | |
| 108 | + title = "" | |
| 109 | + for h in card.find_all("h3"): | |
| 110 | + if "is-visually-hidden" not in " ".join(h.get("class") or []): | |
| 111 | + title = h.get_text(strip=True) | |
| 112 | + break | |
| 113 | + if not title: | |
| 114 | + return None | |
| 115 | + | |
| 116 | + segs = list(card.stripped_strings) | |
| 117 | + blob = " | ".join(segs) | |
| 118 | + if _OUT_OF_QC.search(blob): | |
| 119 | + return None | |
| 120 | + | |
| 121 | + # ligne type (« Entire home by Vrbo ») + ligne config (« Sleeps 4 · … ») | |
| 122 | + property_type = "" | |
| 123 | + capacity = bedrooms = bathrooms = None | |
| 124 | + try: | |
| 125 | + i_title = segs.index(title) | |
| 126 | + except ValueError: | |
| 127 | + i_title = 0 | |
| 128 | + place = "" | |
| 129 | + for seg in segs[i_title + 1:]: | |
| 130 | + tm = _TYPELINE_RE.match(seg) | |
| 131 | + if tm and len(seg) < 60: | |
| 132 | + kind = tm.group(1).strip().lower() | |
| 133 | + property_type = next( | |
| 134 | + (v for k, v in TYPE_MAP.items() if k in kind), "Autre") | |
| 135 | + continue | |
| 136 | + if _SLEEPS_RE.search(seg): | |
| 137 | + sm = _SLEEPS_RE.search(seg) | |
| 138 | + capacity = float(sm.group(1)) | |
| 139 | + bm = _BEDROOMS_RE.search(seg) | |
| 140 | + if bm: | |
| 141 | + bedrooms = float(bm.group(1)) | |
| 142 | + elif "studio" in seg.lower(): | |
| 143 | + bedrooms = 0.0 | |
| 144 | + am = _BATHROOMS_RE.search(seg) | |
| 145 | + if am: | |
| 146 | + bathrooms = float(am.group(1)) | |
| 147 | + continue | |
| 148 | + # première ligne « libre » après titre/type/config = localité | |
| 149 | + if (not place and seg != title and len(seg) < 60 | |
| 150 | + and not _NOT_A_PLACE.search(seg) | |
| 151 | + and not re.match(r"^[\d($]", seg)): | |
| 152 | + place = seg | |
| 153 | + if place: | |
| 154 | + city = place | |
| 155 | + | |
| 156 | + rating = reviews = None | |
| 157 | + rm = _RATING_RE.search(blob) | |
| 158 | + if rm: | |
| 159 | + try: | |
| 160 | + rating = round(float(rm.group(1)) / 2, 2) # /10 → /5 | |
| 161 | + except ValueError: | |
| 162 | + pass | |
| 163 | + vm = _REVIEWS_RE.search(blob) | |
| 164 | + if vm: | |
| 165 | + reviews = int(re.sub(r"[\s,]", "", vm.group(1))) | |
| 166 | + | |
| 167 | + # prix par nuit avant taxes (dates indicatives passées dans l'URL) | |
| 168 | + price_night, price_label = None, "" | |
| 169 | + pm = _PRICE_RE.search(blob) | |
| 170 | + if pm: | |
| 171 | + try: | |
| 172 | + price_night = float(pm.group(1).replace(",", "")) | |
| 173 | + except ValueError: | |
| 174 | + price_night = None | |
| 175 | + if price_night: | |
| 176 | + price_label = (f"à partir de {price_night:.0f} $ / nuit " | |
| 177 | + "(prochaines dates, avant taxes)") | |
| 178 | + | |
| 179 | + images = [] | |
| 180 | + for img in card.select("img[src]"): | |
| 181 | + src = img.get("src") or "" | |
| 182 | + if src.startswith("https://images.trvl-media.com/") \ | |
| 183 | + and src not in images: | |
| 184 | + images.append(src) | |
| 185 | + if len(images) >= 5: | |
| 186 | + break | |
| 187 | + | |
| 188 | + return StListing( | |
| 189 | + source=self.source_id, | |
| 190 | + external_id=external_id, | |
| 191 | + url=url, | |
| 192 | + title=title, | |
| 193 | + property_type=property_type, | |
| 194 | + city=city, | |
| 195 | + region=region, | |
| 196 | + price_night=price_night, | |
| 197 | + price_label=price_label, | |
| 198 | + capacity=capacity, | |
| 199 | + bedrooms=bedrooms, | |
| 200 | + bathrooms=bathrooms, | |
| 201 | + rating=rating, | |
| 202 | + reviews=reviews, | |
| 203 | + images=images, | |
| 204 | + ) | |
| 205 | + | |
| 206 | + # -- contrat ----------------------------------------------------------------- | |
| 207 | + def fetch(self) -> list[StListing]: | |
| 208 | + limit = int(os.environ.get("LOUKA_EXPEDIA_LIMIT", "0") or 0) | |
| 209 | + | |
| 210 | + # scrolls progressifs : déclenche le fetch client + rend les cartes | |
| 211 | + scenario = [{"wait": 1500}] | |
| 212 | + for y in (4000, 9000, 15000): | |
| 213 | + scenario += [{"scroll": {"y": y}}, {"wait": 1500}] | |
| 214 | + | |
| 215 | + listings: dict[str, StListing] = {} | |
| 216 | + for dest, city, region in DESTINATIONS: | |
| 217 | + for sort in SORTS: | |
| 218 | + if limit and len(listings) >= limit: | |
| 219 | + return list(listings.values()) | |
| 220 | + url = ("https://www.expedia.ca/Hotel-Search?destination=" | |
| 221 | + + quote(dest) | |
| 222 | + + "&adults=2&categorySearch=vacation_rentals_option") | |
| 223 | + if sort != "RECOMMENDED": | |
| 224 | + url += f"&sort={sort}" | |
| 225 | + cards = [] | |
| 226 | + for attempt in (1, 2): # le rendu revient parfois vide | |
| 227 | + try: | |
| 228 | + html = self.get_scrapfly( | |
| 229 | + url, render_js=True, asp=True, rendering_wait=2000, | |
| 230 | + wait_for_selector='[data-stid="lodging-card-responsive"]', | |
| 231 | + js_scenario=scenario) | |
| 232 | + except Exception as exc: # noqa: BLE001 — une requête ratée ≠ sync ratée | |
| 233 | + print(f"[expedia] {city} ({sort}) : {exc}", | |
| 234 | + file=sys.stderr) | |
| 235 | + continue | |
| 236 | + soup = BeautifulSoup(html or "", "html.parser") | |
| 237 | + cards = soup.select('[data-stid="lodging-card-responsive"]') | |
| 238 | + if cards: | |
| 239 | + break | |
| 240 | + print(f"[expedia] {city} ({sort}) : 0 carte rendue" | |
| 241 | + f" (essai {attempt})", file=sys.stderr) | |
| 242 | + n_before = len(listings) | |
| 243 | + for card in cards: | |
| 244 | + try: | |
| 245 | + lst = self._parse_card(card, city, region) | |
| 246 | + except Exception: # noqa: BLE001 | |
| 247 | + continue | |
| 248 | + if lst and lst.external_id not in listings: | |
| 249 | + listings[lst.external_id] = lst | |
| 250 | + print(f"[expedia] {city} ({sort}) :" | |
| 251 | + f" {len(listings) - n_before} nouvelles" | |
| 252 | + f" (total {len(listings)})", file=sys.stderr) | |
| 253 | + return list(listings.values()) | |
added
louka/shortterm/connectors/glampinghub.py
+177 −0
@@ -0,0 +1,177 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/glampinghub.py : Glamping Hub (glampinghub.com) — hébergements | |
| 4 | +# insolites (dômes, yourtes, pods, cabanes…), ~200 fiches au Québec. | |
| 5 | +# | |
| 6 | +# Méthode : l'endpoint AJAX de la page de recherche est ouvert (aucun anti-bot) : | |
| 7 | +# GET https://glampinghub.com/search-accommodations/?lang=en&page=N | |
| 8 | +# &q=Quebec, Canada&location={"state": "Quebec", "country": "Canada"} | |
| 9 | +# &adults=2&…&numberOfResultsPerPage=24 | |
| 10 | +# → search_results[…] + total_results. Chaque fiche est TRÈS riche : nom, | |
| 11 | +# catégorie, ville + coordonnées, prix (estimated_rate.daily_rate en devise | |
| 12 | +# originale, CAD au Québec), chambres/lits, capacité par unité | |
| 13 | +# (units_distribution), note sur 5, commodités (nested_features), photos. | |
| 14 | +# Une seule requête paginée suffit — pas de page détail nécessaire. | |
| 15 | +# | |
| 16 | +# URL publique : https://glampinghub.com<absolute_url_en>. Région touristique | |
| 17 | +# déduite des coordonnées (centroïdes partagés avec airbnb.py). | |
| 18 | +# Réglage env : LOUKA_GLAMPINGHUB_LIMIT (nb max de fiches, 0 = tout). | |
| 19 | +# ----------------------------------------------------------------------------- | |
| 20 | +from __future__ import annotations | |
| 21 | + | |
| 22 | +import os | |
| 23 | + | |
| 24 | +from ..schema import StListing | |
| 25 | +from .base import StConnector | |
| 26 | +from .airbnb import _region_from_latlng | |
| 27 | + | |
| 28 | +SITE = "https://glampinghub.com" | |
| 29 | +API = f"{SITE}/search-accommodations/" | |
| 30 | +PAGE_SIZE = 24 | |
| 31 | + | |
| 32 | +# Catégorie Glamping Hub → type canonique Lou-Ka | |
| 33 | +_CATEGORIES = { | |
| 34 | + "cabins": "Chalet", "cottages": "Chalet", "log cabins": "Chalet", | |
| 35 | + "vacation rentals": "Maison", "villas": "Maison", | |
| 36 | + "designer rentals": "Maison", "unique stays": "Autre", | |
| 37 | + "tiny houses": "Mini-maison", "hobbit houses": "Mini-maison", | |
| 38 | + "domes": "Dôme", "yurts": "Yourte", "tipis": "Prêt-à-camper", | |
| 39 | + "pods": "Prêt-à-camper", "bell tents": "Prêt-à-camper", | |
| 40 | + "safari tents": "Prêt-à-camper", "tented cabins": "Prêt-à-camper", | |
| 41 | + "canvas tents": "Prêt-à-camper", "airstreams": "Prêt-à-camper", | |
| 42 | + "caravans": "Prêt-à-camper", "campervans": "Prêt-à-camper", | |
| 43 | + "tree houses": "Autre", "barns": "Autre", "boats": "Autre", | |
| 44 | + "islands": "Autre", "castles": "Autre", "condos": "Condo", | |
| 45 | + "apartments": "Appartement", "lodges": "Auberge", | |
| 46 | + "bed and breakfasts": "Gîte", | |
| 47 | +} | |
| 48 | + | |
| 49 | + | |
| 50 | +def _num(v) -> float | None: | |
| 51 | + try: | |
| 52 | + return float(v) if v not in (None, "") else None | |
| 53 | + except (TypeError, ValueError): | |
| 54 | + return None | |
| 55 | + | |
| 56 | + | |
| 57 | +class GlampingHub(StConnector): | |
| 58 | + source_id = "glampinghub" | |
| 59 | + request_delay = 0.8 | |
| 60 | + | |
| 61 | + # -- liste -------------------------------------------------------------- | |
| 62 | + def _search_page(self, page: int) -> dict: | |
| 63 | + resp = self.get(API, params={ | |
| 64 | + "lang": "en", | |
| 65 | + "page": page, | |
| 66 | + "q": "Quebec, Canada", | |
| 67 | + "location": '{"state": "Quebec", "country": "Canada"}', | |
| 68 | + "adults": 2, "children": 0, "infants": 0, | |
| 69 | + "sort": "-ranking_engine_boost", | |
| 70 | + "source": "rentalsearch", | |
| 71 | + "numberOfResultsPerPage": PAGE_SIZE, | |
| 72 | + }, headers={"Accept": "application/json", | |
| 73 | + "X-Requested-With": "XMLHttpRequest", | |
| 74 | + "Referer": f"{SITE}/rentalsearch/"}) | |
| 75 | + return resp.json() | |
| 76 | + | |
| 77 | + def _all_items(self) -> list[dict]: | |
| 78 | + items: list[dict] = [] | |
| 79 | + page, total = 0, 1 | |
| 80 | + while len(items) < min(total, 2000) and page < 100: | |
| 81 | + data = self._search_page(page) | |
| 82 | + batch = data.get("search_results") or [] | |
| 83 | + total = data.get("total_results") or 0 | |
| 84 | + if not batch: | |
| 85 | + break | |
| 86 | + items.extend(batch) | |
| 87 | + page += 1 | |
| 88 | + return items | |
| 89 | + | |
| 90 | + # -- contrat -------------------------------------------------------------- | |
| 91 | + def fetch(self) -> list[StListing]: | |
| 92 | + limit = int(os.environ.get("LOUKA_GLAMPINGHUB_LIMIT", "0") or 0) | |
| 93 | + listings: list[StListing] = [] | |
| 94 | + seen: set[str] = set() | |
| 95 | + for it in self._all_items(): | |
| 96 | + gid = str(it.get("id") or "").strip() | |
| 97 | + title = (it.get("name_en") or "").strip() | |
| 98 | + loc = it.get("location") or {} | |
| 99 | + if not gid or gid in seen or not title: | |
| 100 | + continue | |
| 101 | + if (loc.get("state_en") or "").strip().lower() != "quebec": | |
| 102 | + continue | |
| 103 | + seen.add(gid) | |
| 104 | + | |
| 105 | + lat = lng = None | |
| 106 | + coords = (loc.get("coords") or "").split(",") | |
| 107 | + if len(coords) == 2: | |
| 108 | + lat, lng = _num(coords[0]), _num(coords[1]) | |
| 109 | + | |
| 110 | + path = it.get("absolute_url_en") or "" | |
| 111 | + if not path: | |
| 112 | + for u in it.get("absolute_url") or []: | |
| 113 | + if (u or {}).get("lang") == "en": | |
| 114 | + path = u.get("url") or "" | |
| 115 | + break | |
| 116 | + | |
| 117 | + images = [] | |
| 118 | + for ph in (it.get("images") or [])[:15]: | |
| 119 | + u = (ph or {}).get("url") or "" | |
| 120 | + if u.startswith("//"): | |
| 121 | + u = "https:" + u | |
| 122 | + if u.startswith("https://") and u not in images: | |
| 123 | + images.append(u) | |
| 124 | + | |
| 125 | + # prix : tarif nuit de base en devise originale (CAD au Québec) | |
| 126 | + price_night, price_label = None, "" | |
| 127 | + rate = it.get("estimated_rate") or {} | |
| 128 | + daily = _num(rate.get("daily_rate")) | |
| 129 | + if daily and (it.get("original_currency") or "") == "CAD": | |
| 130 | + price_night = daily | |
| 131 | + price_label = f"à partir de {daily:.0f} $ CAD / nuit" | |
| 132 | + | |
| 133 | + units = it.get("units_distribution") or [] | |
| 134 | + capacity = max((_num(u.get("capacity")) or 0 for u in units), | |
| 135 | + default=0) or None | |
| 136 | + | |
| 137 | + amenities = [] | |
| 138 | + for f in it.get("nested_features") or []: | |
| 139 | + name = (f or {}).get("name_en") or "" | |
| 140 | + if f.get("active") and name and name not in amenities: | |
| 141 | + amenities.append(name) | |
| 142 | + | |
| 143 | + category = (it.get("category_en") or "").strip() | |
| 144 | + details = {k: v for k, v in { | |
| 145 | + "category": category, | |
| 146 | + "units": len(units) or None, | |
| 147 | + "min_stay": rate.get("min_stay"), | |
| 148 | + "verified": bool(it.get("verified_accommodation")) or None, | |
| 149 | + "remarkable": [f.get("name_en") for f in | |
| 150 | + (it.get("remarkable_features") or []) | |
| 151 | + if (f or {}).get("name_en")] or None, | |
| 152 | + }.items() if v} | |
| 153 | + | |
| 154 | + rating = _num(it.get("average_rate")) | |
| 155 | + listings.append(StListing( | |
| 156 | + source=self.source_id, | |
| 157 | + external_id=gid, | |
| 158 | + url=SITE + path if path else SITE, | |
| 159 | + title=title, | |
| 160 | + property_type=_CATEGORIES.get(category.lower(), "Autre"), | |
| 161 | + city=loc.get("city_en") or "", | |
| 162 | + region=_region_from_latlng(lat, lng), | |
| 163 | + price_night=price_night, | |
| 164 | + price_label=price_label, | |
| 165 | + capacity=capacity, | |
| 166 | + bedrooms=_num(it.get("bedrooms_number")), | |
| 167 | + beds=_num(it.get("beds_number")), | |
| 168 | + rating=rating if rating else None, | |
| 169 | + amenities=amenities, | |
| 170 | + details=details, | |
| 171 | + images=images, | |
| 172 | + lat=lat, | |
| 173 | + lng=lng, | |
| 174 | + )) | |
| 175 | + if limit and len(listings) >= limit: | |
| 176 | + break | |
| 177 | + return listings | |
added
louka/shortterm/connectors/hipcamp.py
+256 −0
@@ -0,0 +1,256 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/hipcamp.py : Hipcamp (hipcamp.com) — camping et prêt-à-camper | |
| 4 | +# chez des hôtes privés (terres agricoles, boisés…), ~350 terrains au Québec. | |
| 5 | +# | |
| 6 | +# Méthode : le site (Next.js) parle à un GraphQL public, accessible sans | |
| 7 | +# session avec les en-têtes HIPCAMP-API-KEY (clé publique embarquée dans le | |
| 8 | +# bundle JS) et HIPCAMP-PLATFORM: Web. | |
| 9 | +# 1. LISTE : POST https://www.hipcamp.com/graphql/search — requête | |
| 10 | +# LandsSearch(landFilter: {boundingBox: <bbox Québec>}) paginée par | |
| 11 | +# offset/limit (50 par page). Chaque « land » (terrain, souvent | |
| 12 | +# multi-emplacements) : nom, ville, coordonnées, prix/nuit (« CA$54.00 »), | |
| 13 | +# types d'hébergement (tent/rv/house), photos, % de recommandations. | |
| 14 | +# Le bbox mord sur l'Ontario et le Maine : on garde stateAbbrvName == QC. | |
| 15 | +# 2. DÉTAIL (cache self.detail) : POST /graphql/camper — requête Land par | |
| 16 | +# maskedId : description (overview, souvent bilingue avec no CITQ), | |
| 17 | +# capacité max, nb d'emplacements par type, commodités et activités. | |
| 18 | +# 3. Photos : CDN Cloudinary (https://hipcamp-res.cloudinary.com/…). | |
| 19 | +# | |
| 20 | +# Pas de note sur 5 chez Hipcamp : % de recommandations → details. Région | |
| 21 | +# touristique déduite des coordonnées (centroïdes partagés avec airbnb.py). | |
| 22 | +# Réglage env : LOUKA_HIPCAMP_LIMIT (nb max de terrains, 0 = tout). | |
| 23 | +# ----------------------------------------------------------------------------- | |
| 24 | +from __future__ import annotations | |
| 25 | + | |
| 26 | +import os | |
| 27 | +import re | |
| 28 | +import sys | |
| 29 | + | |
| 30 | +from ..schema import StListing | |
| 31 | +from .base import StConnector | |
| 32 | +from .airbnb import _in_quebec, _region_from_latlng | |
| 33 | + | |
| 34 | +SITE = "https://www.hipcamp.com" | |
| 35 | +GRAPHQL_SEARCH = f"{SITE}/graphql/search" | |
| 36 | +GRAPHQL_CAMPER = f"{SITE}/graphql/camper" | |
| 37 | +CDN = "https://hipcamp-res.cloudinary.com" | |
| 38 | + | |
| 39 | +# Clé API publique (embarquée dans le bundle JS du site, module 79018) | |
| 40 | +HEADERS = { | |
| 41 | + "Content-Type": "application/json", | |
| 42 | + "Accept": "application/json", | |
| 43 | + "HIPCAMP-API-KEY": "Dp7qfhE8y8cTx73qSYu8b6M2", | |
| 44 | + "HIPCAMP-PLATFORM": "Web", | |
| 45 | +} | |
| 46 | + | |
| 47 | +# Zone habitée du Québec (sud, ouest, nord, est) — même bbox qu'airbnb.py | |
| 48 | +QC_BBOX = (44.95, -79.80, 52.20, -56.90) | |
| 49 | +PAGE_SIZE = 50 | |
| 50 | + | |
| 51 | +LIST_QUERY = """query LandsSearch($landFilter: LandFilterInput!, $privateOffset: Int, $privateLimit: Int) { | |
| 52 | + lands(landFilter: $landFilter) { | |
| 53 | + privateLands(offset: $privateOffset, limit: $privateLimit) { | |
| 54 | + total | |
| 55 | + edges { | |
| 56 | + availableAccommodationKeys | |
| 57 | + availableCampsitesCount | |
| 58 | + node { | |
| 59 | + allAccommodationKeys | |
| 60 | + cityName | |
| 61 | + coordinate { latitude longitude } | |
| 62 | + countryCode | |
| 63 | + id | |
| 64 | + maskedId | |
| 65 | + name | |
| 66 | + stateAbbrvName | |
| 67 | + locationSummary | |
| 68 | + topPhotos { filename } | |
| 69 | + url | |
| 70 | + } | |
| 71 | + pricePerNight { symbol format minorAmount } | |
| 72 | + } | |
| 73 | + } | |
| 74 | + } | |
| 75 | +}""" | |
| 76 | + | |
| 77 | +DETAIL_QUERY = """query Land($landId: ID!, $landIdType: LandIdTypeEnum!) { | |
| 78 | + land(landId: $landId, landIdType: $landIdType) { | |
| 79 | + maskedId fullName cityName countyName overview subheader | |
| 80 | + maxSiteCapacity campsiteCount structureCount rvCount tentCount | |
| 81 | + recommendsPercentage recommendsCount bookingsCount | |
| 82 | + coordinate { lat lng } | |
| 83 | + minPricePerNight { amount format isoCode } | |
| 84 | + coreAmenities: landCampFeatures(type: CORE_AMENITY) { name } | |
| 85 | + basicAmenities: landCampFeatures(type: BASIC_AMENITY) { name } | |
| 86 | + activities: landCampFeatures(type: ACTIVITY) { name } | |
| 87 | + } | |
| 88 | +}""" | |
| 89 | + | |
| 90 | +_PRICE_FMT_RE = re.compile(r"CA\$([\d,]+(?:\.\d{2})?)") | |
| 91 | + | |
| 92 | + | |
| 93 | +def _photo_url(filename: str) -> str: | |
| 94 | + """URL CDN d'une photo — deux formats de filename coexistent.""" | |
| 95 | + if not filename: | |
| 96 | + return "" | |
| 97 | + if filename.startswith("images/"): # chemin complet : pas de transform | |
| 98 | + return f"{CDN}/{filename}" | |
| 99 | + return f"{CDN}/f_auto,c_limit,w_1120,q_auto/{filename}" | |
| 100 | + | |
| 101 | + | |
| 102 | +def _property_type(keys: list[str]) -> str: | |
| 103 | + """tent/rv = emplacements nus, house = unités bâties (cabane, dôme…).""" | |
| 104 | + ks = {str(k).lower() for k in (keys or [])} | |
| 105 | + if "house" in ks: | |
| 106 | + return "Chalet" if not (ks & {"tent", "rv"}) else "Prêt-à-camper" | |
| 107 | + return "Camping" | |
| 108 | + | |
| 109 | + | |
| 110 | +def _price_cad(price: dict) -> tuple[float | None, str]: | |
| 111 | + fmt = (price or {}).get("format") or "" | |
| 112 | + m = _PRICE_FMT_RE.search(fmt) | |
| 113 | + if not m: | |
| 114 | + return None, fmt | |
| 115 | + return float(m.group(1).replace(",", "")), f"{fmt} / nuit" | |
| 116 | + | |
| 117 | + | |
| 118 | +class Hipcamp(StConnector): | |
| 119 | + source_id = "hipcamp" | |
| 120 | + request_delay = 0.5 | |
| 121 | + | |
| 122 | + def _graphql(self, url: str, query: str, variables: dict) -> dict: | |
| 123 | + resp = self.post(url, json={"query": query, "variables": variables}, | |
| 124 | + headers=HEADERS) | |
| 125 | + data = resp.json() | |
| 126 | + if data.get("errors"): | |
| 127 | + raise RuntimeError(str(data["errors"])[:200]) | |
| 128 | + return data.get("data") or {} | |
| 129 | + | |
| 130 | + # -- liste ------------------------------------------------------------------ | |
| 131 | + def _all_edges(self) -> list[dict]: | |
| 132 | + s, w, n, e = QC_BBOX | |
| 133 | + land_filter = {"boundingBox": { | |
| 134 | + "northeastLatitude": n, "northeastLongitude": e, | |
| 135 | + "southwestLatitude": s, "southwestLongitude": w}} | |
| 136 | + edges: list[dict] = [] | |
| 137 | + offset, total = 0, 1 | |
| 138 | + while offset < min(total, 3000): | |
| 139 | + data = self._graphql(GRAPHQL_SEARCH, LIST_QUERY, { | |
| 140 | + "landFilter": land_filter, | |
| 141 | + "privateOffset": offset, "privateLimit": PAGE_SIZE}) | |
| 142 | + page = ((data.get("lands") or {}).get("privateLands") or {}) | |
| 143 | + batch = page.get("edges") or [] | |
| 144 | + total = page.get("total") or 0 | |
| 145 | + if not batch: | |
| 146 | + break | |
| 147 | + edges.extend(batch) | |
| 148 | + offset += PAGE_SIZE | |
| 149 | + return edges | |
| 150 | + | |
| 151 | + # -- détail (cache BD) -------------------------------------------------------- | |
| 152 | + def _fetch_detail(self, masked_id: str) -> dict: | |
| 153 | + data = self._graphql(GRAPHQL_CAMPER, DETAIL_QUERY, | |
| 154 | + {"landId": masked_id, "landIdType": "MASKED"}) | |
| 155 | + land = data.get("land") or {} | |
| 156 | + if not land: | |
| 157 | + return {} | |
| 158 | + amenities = [] | |
| 159 | + for grp in ("coreAmenities", "basicAmenities"): | |
| 160 | + for it in land.get(grp) or []: | |
| 161 | + name = (it or {}).get("name") or "" | |
| 162 | + if name and name not in amenities: | |
| 163 | + amenities.append(name) | |
| 164 | + activities = [a.get("name") for a in (land.get("activities") or []) | |
| 165 | + if (a or {}).get("name")] | |
| 166 | + overview = re.sub(r"\s+", " ", land.get("overview") or "").strip() | |
| 167 | + price = land.get("minPricePerNight") or {} | |
| 168 | + return { | |
| 169 | + "description": overview[:4000], | |
| 170 | + "subheader": land.get("subheader") or "", | |
| 171 | + "county": land.get("countyName") or "", | |
| 172 | + "capacity": land.get("maxSiteCapacity"), | |
| 173 | + "campsites": land.get("campsiteCount"), | |
| 174 | + "structures": land.get("structureCount"), | |
| 175 | + "amenities": amenities, | |
| 176 | + "activities": activities, | |
| 177 | + "recommends_pct": land.get("recommendsPercentage"), | |
| 178 | + "recommends_count": land.get("recommendsCount"), | |
| 179 | + "min_price": (price.get("amount") | |
| 180 | + if price.get("isoCode") == "CAD" else None), | |
| 181 | + } | |
| 182 | + | |
| 183 | + # -- contrat -------------------------------------------------------------- | |
| 184 | + def fetch(self) -> list[StListing]: | |
| 185 | + limit = int(os.environ.get("LOUKA_HIPCAMP_LIMIT", "0") or 0) | |
| 186 | + listings: list[StListing] = [] | |
| 187 | + seen: set[str] = set() | |
| 188 | + for edge in self._all_edges(): | |
| 189 | + node = edge.get("node") or {} | |
| 190 | + mid = str(node.get("maskedId") or "").strip() | |
| 191 | + title = (node.get("name") or "").strip() | |
| 192 | + if not mid or mid in seen or not title: | |
| 193 | + continue | |
| 194 | + if (node.get("stateAbbrvName") or "").upper() != "QC": | |
| 195 | + continue | |
| 196 | + seen.add(mid) | |
| 197 | + | |
| 198 | + coord = node.get("coordinate") or {} | |
| 199 | + lat, lng = coord.get("latitude"), coord.get("longitude") | |
| 200 | + if lat is not None and lng is not None and not _in_quebec(lat, lng): | |
| 201 | + continue | |
| 202 | + | |
| 203 | + images = [] | |
| 204 | + for ph in (node.get("topPhotos") or [])[:15]: | |
| 205 | + u = _photo_url((ph or {}).get("filename") or "") | |
| 206 | + if u and u not in images: | |
| 207 | + images.append(u) | |
| 208 | + | |
| 209 | + keys = node.get("allAccommodationKeys") or [] | |
| 210 | + price_night, price_label = _price_cad(edge.get("pricePerNight")) | |
| 211 | + | |
| 212 | + # clé de cache détail : sous-ensemble stable de la carte liste | |
| 213 | + key = f"{title}|{','.join(sorted(keys))}|{len(images)}" | |
| 214 | + try: | |
| 215 | + det = self.detail(mid, key, | |
| 216 | + lambda m=mid: self._fetch_detail(m)) | |
| 217 | + except Exception as exc: # noqa: BLE001 — détail cassé ≠ perdu | |
| 218 | + print(f"[hipcamp] détail {mid} : {exc}", file=sys.stderr) | |
| 219 | + det = {} | |
| 220 | + | |
| 221 | + if price_night is None and det.get("min_price"): | |
| 222 | + price_night = float(det["min_price"]) | |
| 223 | + price_label = f"à partir de {price_night:.0f} $ / nuit" | |
| 224 | + | |
| 225 | + details = {k: v for k, v in { | |
| 226 | + "accommodation_keys": ", ".join(keys) or None, | |
| 227 | + "campsites": det.get("campsites"), | |
| 228 | + "structures": det.get("structures"), | |
| 229 | + "county": det.get("county"), | |
| 230 | + "recommends_pct": det.get("recommends_pct"), | |
| 231 | + "recommends_count": det.get("recommends_count"), | |
| 232 | + "activities": det.get("activities") or None, | |
| 233 | + }.items() if v} | |
| 234 | + | |
| 235 | + cap = det.get("capacity") | |
| 236 | + listings.append(StListing( | |
| 237 | + source=self.source_id, | |
| 238 | + external_id=mid, | |
| 239 | + url=SITE + (node.get("url") or f"/en-CA/land/{mid}"), | |
| 240 | + title=title, | |
| 241 | + property_type=_property_type(keys), | |
| 242 | + city=node.get("cityName") or "", | |
| 243 | + region=_region_from_latlng(lat, lng), | |
| 244 | + price_night=price_night, | |
| 245 | + price_label=price_label, | |
| 246 | + capacity=float(cap) if cap else None, | |
| 247 | + description=det.get("description") or "", | |
| 248 | + amenities=det.get("amenities") or [], | |
| 249 | + details=details, | |
| 250 | + images=images, | |
| 251 | + lat=lat, | |
| 252 | + lng=lng, | |
| 253 | + )) | |
| 254 | + if limit and len(listings) >= limit: | |
| 255 | + break | |
| 256 | + return listings | |
added
louka/shortterm/connectors/parcscanada.py
+194 −0
@@ -0,0 +1,194 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/parcscanada.py : Parcs Canada (reservation.pc.gc.ca) — | |
| 4 | +# hébergements « prêt-à-camper » des parcs nationaux et lieux historiques | |
| 5 | +# fédéraux AU QUÉBEC : oTENTik, Ôasis, MicrOcube, tentes de prospecteur… | |
| 6 | +# (Forillon, La Mauricie, Archipel-de-Mingan, canaux de Lachine et | |
| 7 | +# de Saint-Ours). ~60 unités. | |
| 8 | +# | |
| 9 | +# Méthode : le système de réservation (GoingToCamp/Aspira, derrière un WAF | |
| 10 | +# Azure qui laisse passer les requêtes au User-Agent de navigateur) expose | |
| 11 | +# une API JSON publique : | |
| 12 | +# 1. GET /api/resourcecategory → id de catégorie → nom fr (oTENTik, Ôasis, | |
| 13 | +# Yourte, Chalet rustique, Tente de prospecteur, MicrOcube…) ; | |
| 14 | +# 2. GET /api/resourcelocation/resources?resourceLocationId=<id> → TOUTES | |
| 15 | +# les ressources d'un terrain (emplacements + hébergements) : nom, | |
| 16 | +# description fr, capacité, photos, catégorie. On ne garde que les | |
| 17 | +# catégories « hébergement » (table KEEP ci-dessous) ; | |
| 18 | +# 3. prix (cache self.detail, clé mensuelle) : POST /api/resource/feeDetails | |
| 19 | +# ?resourceId=…&startDate=<J+35> → feeTotal = tarif d'une nuit. | |
| 20 | +# | |
| 21 | +# Les ids sont des entiers négatifs (int32 min + n). Aucune géoloc par unité | |
| 22 | +# dans l'API : lat/lng et région touristique viennent de la table statique | |
| 23 | +# des terrains québécois (QC_LOCATIONS) — fait géographique stable. | |
| 24 | +# Pas de page publique par unité : l'URL pointe la recherche du terrain. | |
| 25 | +# Réglage env : LOUKA_PARCSCANADA_LIMIT (nb max d'unités, 0 = tout). | |
| 26 | +# ----------------------------------------------------------------------------- | |
| 27 | +from __future__ import annotations | |
| 28 | + | |
| 29 | +import datetime | |
| 30 | +import html as _html | |
| 31 | +import os | |
| 32 | +import re | |
| 33 | +import sys | |
| 34 | + | |
| 35 | +from ..schema import StListing | |
| 36 | +from .base import StConnector | |
| 37 | + | |
| 38 | +BASE = "https://reservation.pc.gc.ca" | |
| 39 | + | |
| 40 | +HEADERS = {"Accept": "application/json, text/plain, */*", | |
| 41 | + "Accept-Language": "fr", | |
| 42 | + "Referer": BASE + "/"} | |
| 43 | + | |
| 44 | +# Terrains de Parcs Canada au Québec : id → (nom court, région touristique, | |
| 45 | +# lat, lng approximatifs du terrain). Les autres provinces sont ignorées. | |
| 46 | +QC_LOCATIONS = { | |
| 47 | + -2147483625: ("Forillon – Des-Rosiers", "Gaspésie", 48.879, -64.295), | |
| 48 | + -2147483626: ("Forillon – Petit-Gaspé", "Gaspésie", 48.812, -64.279), | |
| 49 | + -2147483627: ("Forillon – Cap-Bon-Ami", "Gaspésie", 48.868, -64.235), | |
| 50 | + -2147483534: ("Forillon – Arrière-pays", "Gaspésie", 48.850, -64.350), | |
| 51 | + -2147483575: ("La Mauricie – Mistagance", "Mauricie", 46.790, -72.940), | |
| 52 | + -2147483573: ("La Mauricie – Rivière-à-la-Pêche", "Mauricie", | |
| 53 | + 46.672, -72.920), | |
| 54 | + -2147483524: ("La Mauricie – Saint-Jean-des-Piles", "Mauricie", | |
| 55 | + 46.669, -72.752), | |
| 56 | + -2147483578: ("La Mauricie – Wapizagonke", "Mauricie", 46.757, -73.005), | |
| 57 | + -2147483570: ("Archipel-de-Mingan", "Côte-Nord", 50.222, -63.600), | |
| 58 | + -2147483571: ("Canal-de-Lachine", "Montréal", 45.430, -73.670), | |
| 59 | + -2147483548: ("Canal-de-Saint-Ours", "Montérégie", 45.869, -73.150), | |
| 60 | + -2147483513: ("Canal-de-Chambly", "Montérégie", 45.441, -73.283), | |
| 61 | +} | |
| 62 | + | |
| 63 | +# Catégories « hébergement » (les emplacements de camping nus sont exclus) | |
| 64 | +KEEP = { | |
| 65 | + -2147483647: "Yourte", # Yourte | |
| 66 | + -2147483646: "Chalet", # Chalet | |
| 67 | + -2147483645: "Chalet", # Chalet rustique | |
| 68 | + -2147483644: "Prêt-à-camper", # Ôasis (cocon suspendu) | |
| 69 | + -2147483643: "Prêt-à-camper", # oTENTik | |
| 70 | + -2147483642: "Mini-maison", # MicrOcube | |
| 71 | + -2147483635: "Prêt-à-camper", # Camping tout équipé | |
| 72 | + -2147483634: "Chalet", # Chalet en arrière-pays | |
| 73 | + -2147483633: "Yourte", # Yourte en arrière-pays | |
| 74 | + -2147483631: "Prêt-à-camper", # Tipi | |
| 75 | + -2147483630: "Prêt-à-camper", # Tente de prospecteur | |
| 76 | + -2147483629: "Refuge", # Abri en zone d'arrière-pays | |
| 77 | +} | |
| 78 | + | |
| 79 | +_TAG_RE = re.compile(r"<[^>]+>") | |
| 80 | + | |
| 81 | + | |
| 82 | +def _fr(localized: list[dict], *keys: str) -> dict: | |
| 83 | + by_culture = {v.get("cultureName"): v for v in (localized or [])} | |
| 84 | + return by_culture.get("fr-CA") or by_culture.get("en-CA") or {} | |
| 85 | + | |
| 86 | + | |
| 87 | +def _pets(description: str) -> str | None: | |
| 88 | + low = description.lower() | |
| 89 | + if re.search(r"accueille les animaux|animaux .{0,20}(admis|accept|bienvenu)", | |
| 90 | + low): | |
| 91 | + return "oui" | |
| 92 | + if re.search(r"n'accueille pas les animaux|animaux .{0,20}(interdits|" | |
| 93 | + r"pas admis|non admis)|aucun animal", low): | |
| 94 | + return "non" | |
| 95 | + return None | |
| 96 | + | |
| 97 | + | |
| 98 | +class ParcsCanada(StConnector): | |
| 99 | + source_id = "parcscanada" | |
| 100 | + request_delay = 0.5 | |
| 101 | + | |
| 102 | + def _api(self, path: str, **params) -> object: | |
| 103 | + resp = self.get(BASE + path, params=params or None, headers=HEADERS) | |
| 104 | + return resp.json() | |
| 105 | + | |
| 106 | + # -- prix d'une nuit (cache BD, clé mensuelle) ------------------------------- | |
| 107 | + def _fetch_fee(self, resource_id: int) -> dict: | |
| 108 | + start = (datetime.date.today() | |
| 109 | + + datetime.timedelta(days=35)).isoformat() | |
| 110 | + resp = self.post( | |
| 111 | + BASE + "/api/resource/feeDetails", | |
| 112 | + params={"resourceId": resource_id, "startDate": start, | |
| 113 | + "boatLength": 0, "bookingCategoryId": 0, | |
| 114 | + "entryPointResourceId": 0, "exitPointResourceId": 0}, | |
| 115 | + json=[], headers=HEADERS) | |
| 116 | + fees = (resp.json() or {}).get("resourceFeeDetails") or [] | |
| 117 | + total = sum(f.get("feeTotal") or 0 for f in fees) | |
| 118 | + return {"fee": round(total, 2)} if total > 0 else {} | |
| 119 | + | |
| 120 | + # -- contrat -------------------------------------------------------------- | |
| 121 | + def fetch(self) -> list[StListing]: | |
| 122 | + limit = int(os.environ.get("LOUKA_PARCSCANADA_LIMIT", "0") or 0) | |
| 123 | + | |
| 124 | + cat_names: dict[int, str] = {} | |
| 125 | + for c in self._api("/api/resourcecategory") or []: | |
| 126 | + name = _fr(c.get("localizedValues")).get("name") or "" | |
| 127 | + if name: | |
| 128 | + cat_names[c.get("resourceCategoryId")] = name | |
| 129 | + | |
| 130 | + month = datetime.date.today().strftime("%Y-%m") # prix revus au mois | |
| 131 | + listings: list[StListing] = [] | |
| 132 | + for loc_id, (loc_name, region, lat, lng) in QC_LOCATIONS.items(): | |
| 133 | + try: | |
| 134 | + resources = self._api("/api/resourcelocation/resources", | |
| 135 | + resourceLocationId=loc_id) or {} | |
| 136 | + except Exception as exc: # noqa: BLE001 — un terrain HS ≠ tout perdu | |
| 137 | + print(f"[parcscanada] terrain {loc_name} : {exc}", | |
| 138 | + file=sys.stderr) | |
| 139 | + continue | |
| 140 | + | |
| 141 | + for res in resources.values(): | |
| 142 | + cat_id = res.get("resourceCategoryId") | |
| 143 | + if cat_id not in KEEP: | |
| 144 | + continue | |
| 145 | + rid = res.get("resourceId") | |
| 146 | + fr = _fr(res.get("localizedValues")) | |
| 147 | + name = (fr.get("name") or "").strip() | |
| 148 | + if rid is None or not name: | |
| 149 | + continue | |
| 150 | + | |
| 151 | + desc = _html.unescape(_TAG_RE.sub( | |
| 152 | + " ", (fr.get("description") or "").replace("<br>", "\n"))) | |
| 153 | + desc = re.sub(r"[ \t]+", " ", desc).strip() | |
| 154 | + | |
| 155 | + images = [] | |
| 156 | + for ph in (res.get("photos") or [])[:12]: | |
| 157 | + u = ((ph or {}).get("photoUrlResult") or {}).get("url") or "" | |
| 158 | + if u.startswith("https://") and u not in images: | |
| 159 | + images.append(u) | |
| 160 | + | |
| 161 | + cat_fr = cat_names.get(cat_id, KEEP[cat_id]) | |
| 162 | + title = f"{cat_fr} {name} – {loc_name}" | |
| 163 | + | |
| 164 | + try: | |
| 165 | + fee = self.detail(str(rid), month, | |
| 166 | + lambda r=rid: self._fetch_fee(r)) | |
| 167 | + except Exception as exc: # noqa: BLE001 — le prix est optionnel | |
| 168 | + print(f"[parcscanada] tarif {rid} : {exc}", file=sys.stderr) | |
| 169 | + fee = {} | |
| 170 | + price = fee.get("fee") | |
| 171 | + | |
| 172 | + cap = res.get("maxCapacity") | |
| 173 | + listings.append(StListing( | |
| 174 | + source=self.source_id, | |
| 175 | + external_id=str(rid), | |
| 176 | + url=(f"{BASE}/create-booking/results?resourceLocationId=" | |
| 177 | + f"{loc_id}&searchTabGroupId=0&bookingCategoryId=0"), | |
| 178 | + title=title, | |
| 179 | + property_type=KEEP[cat_id], | |
| 180 | + address=loc_name, | |
| 181 | + region=region, | |
| 182 | + price_night=float(price) if price else None, | |
| 183 | + price_label=(f"{price:.2f} $ / nuit" if price else ""), | |
| 184 | + capacity=float(cap) if cap else None, | |
| 185 | + pets=_pets(desc), | |
| 186 | + description=desc[:4000], | |
| 187 | + details={"parc": loc_name, "categorie": cat_fr}, | |
| 188 | + images=images, | |
| 189 | + lat=lat, | |
| 190 | + lng=lng, | |
| 191 | + ).finalize()) | |
| 192 | + if limit and len(listings) >= limit: | |
| 193 | + return listings | |
| 194 | + return listings | |
added
louka/shortterm/connectors/sinistar.py
+226 −0
@@ -0,0 +1,226 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/sinistar.py : Sinistar (sinistar.ca) | |
| 4 | +# | |
| 5 | +# Plateforme québécoise de relogement temporaire pour sinistrés (assurance | |
| 6 | +# habitation) : ~5 600 logements meublés au Québec, partout en province. | |
| 7 | +# SPA Next.js, mais la recherche passe par un index Algolia public : | |
| 8 | +# 1. LISTE : POST https://RKSJN2W5I1-dsn.algolia.net/1/indexes/prod_housings/ | |
| 9 | +# query (clé search-only embarquée dans les bundles du site) avec | |
| 10 | +# facetFilters state:QC. Chaque hit : reference, ville, type, chambres, | |
| 11 | +# lits, sdb, photos, _geoloc. L'index plafonne à 1 000 enregistrements | |
| 12 | +# par requête (paginationLimitedTo) → couverture par quadrillage | |
| 13 | +# insideBoundingBox : une cellule qui dépasse 1 000 hits est subdivisée | |
| 14 | +# en 4 (quadtree, même esprit que le connecteur Airbnb). | |
| 15 | +# 2. DÉTAIL (cache self.detail) : GET https://sinistar.ca/fr/housing/<ref> | |
| 16 | +# — la page embarque le JSON du logement dans le flux React Flight | |
| 17 | +# (self.__next_f.push) : description, commodités, capacité. | |
| 18 | +# 3. AUCUN PRIX PUBLIC : le tarif est négocié entre l'hôte et l'assureur | |
| 19 | +# (« couvert par l'assurance ») → price_night=None, prix absent assumé. | |
| 20 | +# La région touristique est déduite des coordonnées (centroïdes Airbnb). | |
| 21 | +# | |
| 22 | +# Réglage env : LOUKA_SINISTAR_LIMIT (nb max d'annonces, 0 = tout ; utile | |
| 23 | +# pour tester petit sans payer les ~5 600 pages détail du premier run). | |
| 24 | +# ----------------------------------------------------------------------------- | |
| 25 | +from __future__ import annotations | |
| 26 | + | |
| 27 | +import json | |
| 28 | +import os | |
| 29 | +import re | |
| 30 | +import sys | |
| 31 | + | |
| 32 | +from ..schema import StListing | |
| 33 | +from .airbnb import _region_from_latlng | |
| 34 | +from .base import StConnector | |
| 35 | + | |
| 36 | +SITE = "https://sinistar.ca" | |
| 37 | +ALGOLIA_URL = ("https://RKSJN2W5I1-dsn.algolia.net/1/indexes/" | |
| 38 | + "prod_housings/query") | |
| 39 | +ALGOLIA_HEADERS = { | |
| 40 | + "x-algolia-application-id": "RKSJN2W5I1", | |
| 41 | + # clé search-only publique (extraite des bundles JS de sinistar.ca) | |
| 42 | + "x-algolia-api-key": "4dbe623a9ffc1181003e9d98cad4c05b", | |
| 43 | + "Content-Type": "application/json", | |
| 44 | +} | |
| 45 | + | |
| 46 | +# Zone habitée du Québec (sud, ouest, nord, est) pour le quadrillage géo. | |
| 47 | +QC_BBOX = (44.95, -79.80, 62.00, -56.90) | |
| 48 | + | |
| 49 | +# type Sinistar → (type canonique Lou-Ka, libellé français pour le titre) | |
| 50 | +_TYPES = { | |
| 51 | + "house": ("Maison", "Maison meublée"), | |
| 52 | + "cityhouse": ("Maison", "Maison de ville meublée"), | |
| 53 | + "semidetached": ("Maison", "Maison jumelée meublée"), | |
| 54 | + "appartment": ("Appartement", "Appartement meublé"), | |
| 55 | + "apartment": ("Appartement", "Appartement meublé"), | |
| 56 | + "condo": ("Condo", "Condo meublé"), | |
| 57 | + "cottage": ("Chalet", "Chalet meublé"), | |
| 58 | + "loft": ("Loft", "Loft meublé"), | |
| 59 | + "hotel": ("Auberge", "Hébergement hôtelier"), | |
| 60 | +} | |
| 61 | + | |
| 62 | +_FLIGHT_RE = re.compile(r'self\.__next_f\.push\(\[1,"(.*?)"\]\)', re.S) | |
| 63 | + | |
| 64 | + | |
| 65 | +def _num(v) -> float | None: | |
| 66 | + try: | |
| 67 | + return float(v) if v not in (None, "") else None | |
| 68 | + except (TypeError, ValueError): | |
| 69 | + return None | |
| 70 | + | |
| 71 | + | |
| 72 | +def _decode_flight(html: str) -> str: | |
| 73 | + """Concatène les segments React Flight (chaînes JS échappées → UTF-8).""" | |
| 74 | + out = [] | |
| 75 | + for chunk in _FLIGHT_RE.findall(html): | |
| 76 | + try: | |
| 77 | + s = chunk.encode("utf-8").decode("unicode_escape") | |
| 78 | + out.append(s.encode("latin-1", "replace").decode("utf-8", "replace")) | |
| 79 | + except (UnicodeDecodeError, UnicodeEncodeError): | |
| 80 | + continue | |
| 81 | + return "".join(out) | |
| 82 | + | |
| 83 | + | |
| 84 | +class Sinistar(StConnector): | |
| 85 | + source_id = "sinistar" | |
| 86 | + request_delay = 0.4 | |
| 87 | + | |
| 88 | + # -- liste (Algolia, quadtree géo) ---------------------------------------- | |
| 89 | + def _query(self, params: str) -> dict: | |
| 90 | + resp = self.post(ALGOLIA_URL, headers=ALGOLIA_HEADERS, | |
| 91 | + json={"params": params}) | |
| 92 | + return resp.json() | |
| 93 | + | |
| 94 | + def _all_hits(self) -> list[dict]: | |
| 95 | + hits: list[dict] = [] | |
| 96 | + seen: set[str] = set() | |
| 97 | + stack = [QC_BBOX] | |
| 98 | + while stack: | |
| 99 | + s, w, n, e = stack.pop() | |
| 100 | + box = f"{s:.4f},{w:.4f},{n:.4f},{e:.4f}" | |
| 101 | + data = self._query( | |
| 102 | + "hitsPerPage=1000" | |
| 103 | + "&facetFilters=%5B%5B%22state%3AQC%22%5D%5D" | |
| 104 | + f"&insideBoundingBox={box}") | |
| 105 | + nb = data.get("nbHits") or 0 | |
| 106 | + if nb > 1000 and (n - s) > 0.002: | |
| 107 | + # cellule saturée (limite Algolia) → subdivision en 4 | |
| 108 | + mlat, mlng = (s + n) / 2, (w + e) / 2 | |
| 109 | + stack.extend([(s, w, mlat, mlng), (s, mlng, mlat, e), | |
| 110 | + (mlat, w, n, mlng), (mlat, mlng, n, e)]) | |
| 111 | + continue | |
| 112 | + for h in data.get("hits") or []: | |
| 113 | + oid = h.get("objectID") or "" | |
| 114 | + if oid and oid not in seen: | |
| 115 | + seen.add(oid) | |
| 116 | + hits.append(h) | |
| 117 | + return hits | |
| 118 | + | |
| 119 | + # -- détail (page housing, cache BD) --------------------------------------- | |
| 120 | + def _fetch_detail(self, reference: str) -> dict: | |
| 121 | + resp = self.get(f"{SITE}/fr/housing/{reference}") | |
| 122 | + blob = _decode_flight(resp.text) | |
| 123 | + i = blob.find(f'"reference":{reference}') | |
| 124 | + if i < 0: | |
| 125 | + return {} | |
| 126 | + line = blob[blob.rfind("\n", 0, i) + 1:] | |
| 127 | + line = line[:line.find("\n")] if "\n" in line else line | |
| 128 | + try: | |
| 129 | + payload = json.loads(line.split(":", 1)[1]) | |
| 130 | + except (ValueError, IndexError): | |
| 131 | + return {} | |
| 132 | + housing = {} | |
| 133 | + | |
| 134 | + def walk(o): | |
| 135 | + nonlocal housing | |
| 136 | + if housing: | |
| 137 | + return | |
| 138 | + if isinstance(o, dict): | |
| 139 | + if str(o.get("reference")) == reference and "amenities" in o: | |
| 140 | + housing = o | |
| 141 | + return | |
| 142 | + for v in o.values(): | |
| 143 | + walk(v) | |
| 144 | + elif isinstance(o, list): | |
| 145 | + for v in o: | |
| 146 | + walk(v) | |
| 147 | + | |
| 148 | + walk(payload) | |
| 149 | + if not housing: | |
| 150 | + return {} | |
| 151 | + amenities = [] | |
| 152 | + for cat in (housing.get("amenities") or {}).values(): | |
| 153 | + for a in (cat if isinstance(cat, list) else []): | |
| 154 | + if isinstance(a, str): | |
| 155 | + amenities.append(a.replace("_", " ").strip()) | |
| 156 | + return { | |
| 157 | + "description": (housing.get("description") or "").strip(), | |
| 158 | + "amenities": amenities, | |
| 159 | + "capacity": _num(housing.get("capacity")), | |
| 160 | + } | |
| 161 | + | |
| 162 | + # -- contrat -------------------------------------------------------------- | |
| 163 | + def fetch(self) -> list[StListing]: | |
| 164 | + limit = int(os.environ.get("LOUKA_SINISTAR_LIMIT", "0") or 0) | |
| 165 | + listings: list[StListing] = [] | |
| 166 | + seen: set[str] = set() | |
| 167 | + for h in self._all_hits(): | |
| 168 | + ref = str(h.get("reference") or "").strip() | |
| 169 | + city = (h.get("city") or "").strip() | |
| 170 | + if not ref or ref in seen or not city: | |
| 171 | + continue | |
| 172 | + seen.add(ref) | |
| 173 | + | |
| 174 | + geo = h.get("_geoloc") or {} | |
| 175 | + lat, lng = _num(geo.get("lat")), _num(geo.get("lng")) | |
| 176 | + ptype, label = _TYPES.get((h.get("type") or "").strip().lower(), | |
| 177 | + ("Autre", "Logement meublé")) | |
| 178 | + bedrooms = _num(h.get("bedrooms")) | |
| 179 | + title = label | |
| 180 | + if bedrooms: | |
| 181 | + title += f" {int(bedrooms)} chambre{'s' if bedrooms > 1 else ''}" | |
| 182 | + title += f" à {city}" | |
| 183 | + | |
| 184 | + images = [p.get("url") for p in (h.get("pictures") or [])[:15] | |
| 185 | + if isinstance(p.get("url"), str) | |
| 186 | + and p["url"].startswith("https://")] | |
| 187 | + if not images and isinstance( | |
| 188 | + (h.get("coverPicture") or {}).get("url"), str): | |
| 189 | + images = [h["coverPicture"]["url"]] | |
| 190 | + | |
| 191 | + # clé de cache détail : sous-ensemble stable du hit liste | |
| 192 | + key = json.dumps([ref, h.get("type"), h.get("bedrooms"), | |
| 193 | + h.get("beds"), h.get("bathrooms"), len(images)], | |
| 194 | + ensure_ascii=False) | |
| 195 | + try: | |
| 196 | + det = self.detail(ref, key, lambda r=ref: self._fetch_detail(r)) | |
| 197 | + except Exception as exc: # une fiche cassée ≠ annonce perdue | |
| 198 | + print(f"[sinistar] détail {ref} en échec : {exc}", | |
| 199 | + file=sys.stderr) | |
| 200 | + det = {} | |
| 201 | + | |
| 202 | + listings.append(StListing( | |
| 203 | + source=self.source_id, | |
| 204 | + external_id=ref, | |
| 205 | + url=f"{SITE}/fr/housing/{ref}", | |
| 206 | + title=title, | |
| 207 | + property_type=ptype, | |
| 208 | + city=city, | |
| 209 | + region=_region_from_latlng(lat, lng), | |
| 210 | + price_night=None, # tarif négocié avec l'assureur | |
| 211 | + price_label="", | |
| 212 | + capacity=det.get("capacity"), | |
| 213 | + bedrooms=bedrooms, | |
| 214 | + beds=_num(h.get("beds")), | |
| 215 | + bathrooms=_num(h.get("bathrooms")), | |
| 216 | + description=det.get("description") or "", | |
| 217 | + amenities=det.get("amenities") or [], | |
| 218 | + details={"relocation": True, | |
| 219 | + "sinistar_type": h.get("type") or ""}, | |
| 220 | + images=images, | |
| 221 | + lat=lat, | |
| 222 | + lng=lng, | |
| 223 | + ).finalize()) | |
| 224 | + if limit and len(listings) >= limit: | |
| 225 | + break | |
| 226 | + return listings | |
added
louka/shortterm/connectors/sonder.py
+30 −0
@@ -0,0 +1,30 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/sonder.py : Sonder (sonder.com) — SOURCE NON VIABLE (2026-08) | |
| 4 | +# | |
| 5 | +# Sonder Holdings (appartements-hôtels, plusieurs immeubles à Montréal) a | |
| 6 | +# annoncé sa liquidation le 9 novembre 2025 après la rupture de son entente | |
| 7 | +# de licence avec Marriott ; les opérations ont cessé immédiatement | |
| 8 | +# (faillite, clients évincés — actions désormais cotées SONDQ en OTC). | |
| 9 | +# | |
| 10 | +# Constat au 2026-08-23 : le domaine sonder.com est devenu un AGRÉGATEUR | |
| 11 | +# AFFILIÉ sans inventaire propre — les pages « /all/canada/quebec/montreal » | |
| 12 | +# (CloudFront, contournable via Bright Data) listent des hôtels et annonces | |
| 13 | +# tiers avec ids « BC-… » et liens /redirect-partner vers booking.com | |
| 14 | +# (campagne d'affiliation bc_cid=SONDER, collection « formerly-sonder »). | |
| 15 | +# Scraper ce site ne produirait que des doublons de notre connecteur Booking, | |
| 16 | +# attribués à la mauvaise source. On garde le module en éteignoir au cas où | |
| 17 | +# la marque renaîtrait avec un vrai inventaire. | |
| 18 | +# ----------------------------------------------------------------------------- | |
| 19 | +from __future__ import annotations | |
| 20 | + | |
| 21 | +from ..schema import StListing | |
| 22 | +from .base import StConnector | |
| 23 | + | |
| 24 | + | |
| 25 | +class Sonder(StConnector): | |
| 26 | + source_id = "sonder" | |
| 27 | + disabled = True # liquidation nov. 2025 ; sonder.com = affilié Booking | |
| 28 | + | |
| 29 | + def fetch(self) -> list[StListing]: | |
| 30 | + return [] | |
added
louka/shortterm/connectors/tripadvisor.py
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Lou-Ka — Location court terme | |
| 3 | +# connectors/tripadvisor.py : TripAdvisor Vacation Rentals — SOURCE NON VIABLE | |
| 4 | +# | |
| 5 | +# TripAdvisor a FERMÉ son produit de location de vacances le 1er novembre 2024 | |
| 6 | +# (fin des réservations directes, fermeture de FlipKey, Holiday Lettings et | |
| 7 | +# Niumba — annonce Skift/communiqué de septembre 2024). Vérifié au 2026-08-23 : | |
| 8 | +# toutes les URL /VacationRentals-g<geo>-Reviews-… (tripadvisor.ca ET .com, | |
| 9 | +# testées via Scrapfly ASP : Québec g155033, Montréal g155032, Mont-Tremblant | |
| 10 | +# g155059, New York g28953) répondent 301 → la page /Hotels-g<geo>-…-Hotels.html | |
| 11 | +# ou 404. Il n'existe plus ni vertical « locations de vacances », ni API/JSON | |
| 12 | +# embarqué correspondant ; les hébergements « autres » affichés dans la | |
| 13 | +# recherche hôtels sont des fiches d'avis renvoyant vers des OTA tierces | |
| 14 | +# (Booking, Vrbo, Expedia) que nos autres connecteurs couvrent déjà — les | |
| 15 | +# scraper ne produirait que des doublons mal attribués. | |
| 16 | +# | |
| 17 | +# On garde le module en éteignoir au cas où TripAdvisor relancerait un vrai | |
| 18 | +# inventaire de locations (l'auto-registre l'exclut tant que disabled = True). | |
| 19 | +# ----------------------------------------------------------------------------- | |
| 20 | +from __future__ import annotations | |
| 21 | + | |
| 22 | +from ..schema import StListing | |
| 23 | +from .base import StConnector | |
| 24 | + | |
| 25 | + | |
| 26 | +class TripAdvisor(StConnector): | |
| 27 | + source_id = "tripadvisor" | |
| 28 | + disabled = True # produit locations de vacances fermé le 2024-11-01 | |
| 29 | + | |
| 30 | + def fetch(self) -> list[StListing]: | |
| 31 | + return [] | |
| 32 | ||