spb/auto-ka Public
Python 81.9%
TypeScript 12.4%
CSS 5.5%
1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/sm360_dealers.py : connecteurs des concessionnaires sur plateforme5# SM360 / 360.agency (fournisseur de sites très répandu chez les6# concessionnaires franchisés au Québec).7#8# Stratégie : le sitemap (/fr/sitemap-xml) recense chaque page véhicule9# d'occasion (/fr/inventaire-occasion/<marque>/<modèle>/<slug>-id<digits>).10# Chaque page détail est server-rendered et expose :11# 1. un JSON-LD schema.org (@type Car) : prix, km, VIN, transmission,12# carrosserie, couleurs, portes/places, moteur, succursale vendeuse ;13# 2. un dataLayer JS : stockNo, isNew/isDemo, catégorie de véhicule,14# carburant, succursale (utile pour les groupes multi-succursales) ;15# 3. la galerie photo complète (img.sm360.ca/…/images/inventory/…) ;16# 4. le lien Carfax le cas échéant.17# Les pages détail sont mises en cache BD (clé hebdomadaire) : seuls les18# nouveaux véhicules — et une revalidation par semaine pour les prix —19# génèrent de vraies requêtes.20#21# En complément, l'API GraphQL publique que le site utilise lui-même22# (webauto-supplier-api.sm360.ca — celle des widgets VDP) donne pour chaque23# id : la liste d'équipements (options.labels), la galerie complète24# (multimedia.pictures — indispensable pour les gabarits dont la galerie25# n'est PAS server-rendered : touteslesmarques, spinelli, rivesudchrysler,26# toyotagatineau) et une description/tagline de repli. Un POST par véhicule,27# mis en cache avec le même payload hebdomadaire.28#29# Le listing lui-même est un widget JS (GraphQL webauto-supplier-api) —30# inutile ici, le sitemap suffit. robots.txt : Allow /*?page=* et /*/api/*,31# Crawl-delay 10 → délai généreux + cache détail.32# -----------------------------------------------------------------------------33from __future__ import annotations3435import datetime36import html as htmllib37import json38import os39import re40import time4142import requests4344from ..schema import Vehicle45from .base import BaseConnector4647# URLs de véhicules d'occasion dans le sitemap (jamais catalogue-neuf /48# inventaire-neuf, donc jamais de véhicules neufs)49_VEH_URL_RE = re.compile(r"/fr/inventaire-occasion/.+-id(\d+)/?$")5051# segments marque/modèle qui ne sont PAS des automobiles (récréatif)52_EXCLUDE_SEGMENTS = re.compile(53 r"ski-?doo|sea-?doo|can-?am|polaris|arctic-?cat|lynx|brp|harley|"54 r"kawasaki|suzuki-moto|ktm|husqvarna|triumph|ducati|vespa|"55 r"motoneige|vtt|moto(?:s)?\b|motomarine|bateau|ponton|roulotte|"56 r"remorque|\bvr\b|campeur|spyder|ryker|side-?by-?side|atv|snow|"57 r"powersport|nautique", re.I)5859# catégories dataLayer non automobiles (par prudence — normalement le60# sitemap /fr/inventaire-occasion ne contient que des autos)61_EXCLUDE_CATEGORIES = {62 "motorcycle", "atv", "snowmobile", "watercraft", "boat", "rv",63 "trailer", "sxs", "utv", "scooter", "powersport",64}6566_LD_RE = re.compile(67 r'<script type=["\']application/ld\+json["\']>(.*?)</script>', re.S)6869# galerie photo : URLs img.sm360.ca, avec ou sans préfixe resizer /ir/wNNN/70_IMG_RE = re.compile(71 r'https://img\.sm360\.ca/(?:ir/[a-zA-Z0-9]+/)?'72 r'(images/inventory/[^"\'\s<>\\]+\.(?:jpe?g|png|webp))', re.I)7374_CARFAX_RE = re.compile(r'https?://(?:vhr\.)?carfax\.ca/[^\s"\'<>]+', re.I)7576# API GraphQL publique des widgets SM360 (celle que la page détail utilise77# elle-même pour afficher équipements et galerie) — id = id inventaire global78_GRAPHQL_API = "https://webauto-supplier-api.sm360.ca/webauto/graphql"79_GQL_VEHICLE = ("{ vehicle(id: %d) { description tagline "80 "options { labels } multimedia { pictures { url } } } }")8182# clés extraites du dataLayer JS de la page détail (valeurs 'entre quotes')83_DL_KEYS = (84 "salePrice", "regularPrice", "mileage", "stockNo", "vehicleVin",85 "isNew", "isDemo", "vehicleCategory", "condition", "vehicleLocation",86 "vehicleFuelType", "transmission", "trim", "make", "model", "year",87)8889_DRIVE_MAP = {90 "FrontWheelDriveConfiguration": "Traction",91 "RearWheelDriveConfiguration": "Propulsion",92 "AllWheelDriveConfiguration": "Intégrale / 4x4",93 "FourWheelDriveConfiguration": "Intégrale / 4x4",94}959697def _clean(text: str | None) -> str:98 """Désencode les entités HTML et retire balises / « s/o » SM360."""99 if not text:100 return ""101 out = htmllib.unescape(htmllib.unescape(str(text)))102 out = re.sub(r"<[^>]+>", " ", out)103 out = re.sub(r"\s{2,}", " ", out).strip()104 return "" if out.lower() in ("s/o", "n/d", "n/a") else out105106107def _num(value) -> float | None:108 """float() tolérant ; 0 et négatif → None (SM360 met 0 = prix caché)."""109 try:110 f = float(value)111 except (TypeError, ValueError):112 return None113 return f if f > 0 else None114115116class SM360Connector(BaseConnector):117 """Base commune aux sites SM360 / 360.agency. Sous-classes : base_url…"""118119 base_url: str = ""120 dealer_name: str = ""121 city: str = ""122 request_delay: float = 1.5 # robots.txt SM360 demande de la patience123 use_firecrawl: bool = False # au cas où un site passe sous Cloudflare124125 # plafond de vraies requêtes détail par sync (AUTOKA_MAX_DETAILS le réduit126 # pour les tests) — le reste sera complété aux syncs suivants127 max_details = 600128129 def _get_text(self, url: str) -> str:130 if self.use_firecrawl:131 return self.get_rendered(url)132 try:133 return self.get(url).text134 except requests.exceptions.ConnectionError:135 # le serveur SM360 ferme parfois la connexion keep-alive : on136 # repart sur une connexion neuve et on réessaie une fois137 self.session.close()138 time.sleep(2.0)139 return self.get(url).text140141 # -- sitemap ---------------------------------------------------------------142 def _vehicle_urls(self) -> list[str]:143 xml = self._get_text(f"{self.base_url}/fr/sitemap-xml")144 urls: list[str] = []145 for loc in re.findall(r"<loc>([^<]+)</loc>", xml):146 loc = loc.strip()147 if not _VEH_URL_RE.search(loc):148 continue149 # segments /fr/inventaire-occasion/<marque>/<modèle>/<slug>150 path = loc.split("/fr/inventaire-occasion/", 1)[1]151 segments = path.split("/")[:-1]152 if any(_EXCLUDE_SEGMENTS.search(s) for s in segments):153 continue154 urls.append(loc)155 return urls156157 # -- page détail -> payload (JSON-LD + dataLayer + galerie + carfax) --------158 def _fetch_detail(self, url: str, ext_id: str) -> dict:159 html = self._get_text(url)160 payload: dict = {"ld": {}, "dl": {}, "images": [], "carfax": ""}161162 for block in _LD_RE.findall(html):163 try:164 data = json.loads(block.strip(), strict=False)165 except ValueError:166 continue167 items = data if isinstance(data, list) else [data]168 for item in items:169 if not isinstance(item, dict):170 continue171 types = item.get("@type")172 types = types if isinstance(types, list) else [types]173 if "Car" in types or "Vehicle" in types:174 payload["ld"] = item175 break176 if payload["ld"]:177 break178179 for key in _DL_KEYS:180 m = re.search(r"\b" + key + r"\s*:\s*'((?:[^'\\]|\\.)*)'", html)181 if m:182 payload["dl"][key] = m.group(1).replace("\\'", "'").strip()183184 # galerie : uniquement les photos de CE véhicule, sans préfixe resizer,185 # triées par index _NNN de fin de nom de fichier186 seen: set[str] = set()187 images: list[str] = []188 for path in _IMG_RE.findall(html):189 if f"/{ext_id}/" not in path:190 continue191 full = "https://img.sm360.ca/" + path192 if full not in seen:193 seen.add(full)194 images.append(full)195196 def _idx(u: str) -> int:197 m = re.search(r"_(\d+)\.\w+$", u)198 return int(m.group(1)) if m else 999199200 payload["images"] = sorted(images, key=_idx)[:20]201202 m = _CARFAX_RE.search(html)203 if m:204 payload["carfax"] = htmllib.unescape(m.group(0))205206 payload["gql"] = self._fetch_gql(ext_id)207 return payload208209 # -- API GraphQL : équipements + galerie + description de repli -------------210 def _fetch_gql(self, ext_id: str) -> dict:211 """options.labels, multimedia.pictures et description/tagline du212 véhicule via l'API publique des widgets. Tolérant : {} en cas d'échec213 (le payload HTML reste utilisable seul)."""214 wait = self.request_delay - (time.time() - self._last_request)215 if wait > 0:216 time.sleep(wait)217 try:218 resp = self.session.post(219 _GRAPHQL_API,220 json={"query": _GQL_VEHICLE % int(ext_id)},221 timeout=self.timeout)222 self._last_request = time.time()223 resp.raise_for_status()224 vehicle = (resp.json().get("data") or {}).get("vehicle") or {}225 except Exception:226 self._last_request = time.time()227 return {}228 if not isinstance(vehicle, dict):229 return {}230 labels = (vehicle.get("options") or {}).get("labels") or []231 options = [_clean(lb) for lb in labels if _clean(lb)]232233 images: list[str] = []234 pictures = (vehicle.get("multimedia") or {}).get("pictures") or []235 for pic in pictures:236 u = str((pic or {}).get("url") or "").strip()237 if not u:238 continue239 if not u.startswith("http"): # chemin relatif au serveur d'images240 u = "https://img.sm360.ca/images/inventory" + u241 images.append(u)242243 return {244 "options": options[:60],245 "images": images[:20],246 "description": _clean(vehicle.get("description")),247 "tagline": _clean(vehicle.get("tagline")),248 }249250 # -- multi-succursales (hook, surchargé par TLM) -----------------------------251 def _dealer_city(self, dl: dict, ld: dict) -> tuple[str, str]:252 return self.dealer_name, self.city253254 # -- contrat ---------------------------------------------------------------255 def fetch(self) -> list[Vehicle]:256 urls = self._vehicle_urls()257 week = datetime.date.today().isocalendar()258 # v2 : + équipements/galerie GraphQL (bump = re-crawl complet forcé)259 cache_key = f"v2:{week.year}w{week.week}" # revalidation hebdomadaire260 cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))261262 vehicles: list[Vehicle] = []263 real_fetches = 0264 for url in urls:265 ext_id = _VEH_URL_RE.search(url).group(1)266267 def _fetch(u=url, e=ext_id):268 return self._fetch_detail(u, e)269270 # plafond atteint : ne servir que le cache (fetch réel interdit)271 if real_fetches >= cap:272 from .. import db273 if self._detail_con is None:274 self._detail_con = db.connect()275 data = db.get_cached_detail(self._detail_con, self.source_id,276 ext_id, cache_key)277 if data is None:278 continue279 else:280 before = self._last_request281 try:282 data = self.detail(ext_id, cache_key, _fetch)283 except Exception:284 # page disparue (vendu) ou erreur ponctuelle : ne jamais285 # bloquer la source entière286 if self._last_request != before:287 real_fetches += 1288 continue289 if self._last_request != before: # une vraie requête a eu lieu290 real_fetches += 1291 if not data:292 continue293 veh = self._to_vehicle(ext_id, url, data)294 if veh is not None:295 vehicles.append(veh)296 return vehicles297298 def _to_vehicle(self, ext_id: str, url: str, data: dict) -> Vehicle | None:299 ld: dict = data.get("ld") or {}300 dl: dict = data.get("dl") or {}301 if not ld and not dl:302 return None303304 # filtres de sécurité : jamais de neuf, jamais de récréatif305 if dl.get("isNew", "").lower() == "true":306 return None307 if dl.get("vehicleCategory", "").lower() in _EXCLUDE_CATEGORIES:308 return None309310 brand = ld.get("brand") or {}311 make = _clean(brand.get("name") if isinstance(brand, dict) else brand)312 model = _clean(ld.get("model")) or _clean(dl.get("model")).title()313314 year = None315 for raw in (ld.get("vehicleModelDate"), ld.get("modelDate"),316 dl.get("year")):317 if raw:318 try:319 year = int(str(raw)[:4])320 break321 except ValueError:322 pass323324 offers = ld.get("offers") or {}325 price = _num(offers.get("price")) or _num(dl.get("salePrice")) \326 or _num(dl.get("regularPrice"))327328 odo = ld.get("mileageFromOdometer") or {}329 km = _num(odo.get("value") if isinstance(odo, dict) else odo) \330 or _num(dl.get("mileage"))331332 # titre : le "name" JSON-LD varie (parfois "CR-V", parfois333 # "Chevrolet Colorado 4WD Trail Boss 2025") — on complète au besoin334 title = _clean(ld.get("name")) or f"{make} {model}".strip()335 if make and make.lower() not in title.lower():336 title = f"{make} {title}"337 if year and str(year) not in title:338 title = f"{title} {year}"339340 drive_raw = str(ld.get("driveWheelConfiguration") or "")341 drivetrain = ""342 for key, canon in _DRIVE_MAP.items():343 if key in drive_raw:344 drivetrain = canon345 break346347 engines = ld.get("vehicleEngine") or []348 if isinstance(engines, dict):349 engines = [engines]350 fuel, engine_txt = "", ""351 if engines and isinstance(engines[0], dict):352 fuel = _clean(engines[0].get("fuelType"))353 e_type = _clean(engines[0].get("engineType"))354 if len(e_type) > 3: # "4" = nb cylindres, pas descriptif355 engine_txt = e_type356 fuel = fuel or _clean(dl.get("vehicleFuelType"))357358 transmission = _clean(ld.get("vehicleTransmission")) \359 or _clean(dl.get("transmission"))360361 trim = _clean(ld.get("vehicleConfiguration"))362 if not trim:363 trim = _clean(dl.get("trim")).upper()364365 gql: dict = data.get("gql") or {}366367 images = list(data.get("images") or [])368 # la galerie GraphQL est complète même quand le gabarit ne la rend pas369 # côté serveur (touteslesmarques, spinelli, rive-sud, toyotagatineau)370 gql_images = list(gql.get("images") or [])371 if len(gql_images) > len(images):372 images = gql_images373 if not images:374 ld_img = ld.get("image") or []375 images = [ld_img] if isinstance(ld_img, str) else list(ld_img)376 # écarter les stubs sans nom de fichier (véhicule sans photo)377 images = [u for u in images if isinstance(u, str)378 and re.search(r"\.(?:jpe?g|png|webp)$", u, re.I)]379380 def _int(v):381 try:382 return int(str(v))383 except (TypeError, ValueError):384 return None385386 dealer_name, city = self._dealer_city(dl, ld)387388 details: dict = {}389 if dl.get("isDemo", "").lower() == "true":390 details["demo"] = True391 if dl.get("condition"):392 details["condition"] = dl["condition"]393394 return Vehicle(395 source=self.source_id,396 external_id=ext_id,397 url=url,398 title=title,399 make=make,400 model=model,401 trim=trim,402 year=year,403 price=price,404 price_label=f"{price:,.0f} $".replace(",", " ") if price else "",405 mileage_km=km,406 mileage_label=f"{km:,.0f} km".replace(",", " ") if km else "",407 transmission=transmission,408 fuel=fuel,409 drivetrain=drivetrain,410 body_type=_clean(ld.get("bodyType")),411 exterior_color=_clean(ld.get("color")),412 interior_color=_clean(ld.get("vehicleInteriorColor")),413 engine=engine_txt,414 doors=_int(ld.get("numberOfDoors")),415 seats=_int(ld.get("vehicleSeatingCapacity")416 or ld.get("seatingCapacity")),417 vin=(_clean(ld.get("vehicleIdentificationNumber"))418 or _clean(dl.get("vehicleVin"))).upper(),419 stock_number=_clean(dl.get("stockNo")) or _clean(ld.get("sku")),420 dealer_name=dealer_name,421 city=city,422 description=(_clean(ld.get("description"))423 or gql.get("description", "")424 or gql.get("tagline", ""))[:4000],425 features=list(gql.get("options") or []),426 details=details,427 images=images,428 carfax_url=data.get("carfax") or "",429 )430431432# ---------------------------------------------------------------------------433# Concessionnaires (1 sous-classe = 1 source)434# ---------------------------------------------------------------------------435436class ToutesLesMarques(SM360Connector):437 """Groupe multi-succursales (Chicoutimi, Baie-Comeau, Sept-Îles).438439 Le dataLayer `vehicleLocation` donne la succursale qui détient le440 véhicule ; la ville est déduite du nom de la succursale.441 """442 source_id = "touteslesmarques"443 base_url = "https://www.touteslesmarques.ca"444 dealer_name = "Toutes les marques"445 city = "Chicoutimi"446447 def _dealer_city(self, dl: dict, ld: dict) -> tuple[str, str]:448 branch = _clean(dl.get("vehicleLocation"))449 if not branch or branch.lower() == "touteslesmarques.ca":450 return self.dealer_name, self.city451 low = branch.lower()452 if "baie" in low and "comeau" in low:453 city = "Baie-Comeau"454 elif "sept" in low and ("iles" in low or "îles" in low):455 city = "Sept-Îles"456 else: # Arnold, L'Ami Junior, Intégral Subaru…457 city = "Chicoutimi"458 return branch, city459460461class StJeromeAutoDepot(SM360Connector):462 source_id = "stjeromeautodepot"463 base_url = "https://www.stjeromeautodepot.com"464 dealer_name = "St-Jérôme Auto Dépôt"465 city = "Saint-Jérôme"466467468class LamontagneAuto(SM360Connector):469 # concessionnaire Honda — le sitemap /fr/inventaire-occasion ne contient470 # que l'occasion (le neuf vit sous /fr/catalogue-neuf, jamais ramassé)471 source_id = "lamontagneauto"472 base_url = "https://www.lamontagneauto.com"473 dealer_name = "Lamontagne Auto"474 city = "Rimouski"475476477# ---------------------------------------------------------------------------478# Expansion 2 — concessionnaires SM360 supplémentaires479# ---------------------------------------------------------------------------480481class Spinelli(SM360Connector):482 """Groupe multi-concessions de l'île de Montréal (Pointe-Claire, Lachine…)."""483 source_id = "spinelli"484 base_url = "https://www.spinelli.com"485 dealer_name = "Groupe Spinelli"486 city = "Montréal"487488489class AutoDesjardins(SM360Connector):490 source_id = "autodesjardins"491 base_url = "https://www.autodesjardins.com"492 dealer_name = "Automobiles Desjardins"493 city = "Québec"494495496class LallierHondaHull(SM360Connector):497 source_id = "lallierhondahull"498 base_url = "https://www.lallierhondahull.com"499 dealer_name = "Lallier Honda Hull"500 city = "Gatineau"501502503class RiveSudChrysler(SM360Connector):504 source_id = "rivesudchrysler"505 base_url = "https://www.rivesudchrysler.com"506 dealer_name = "Rive-Sud Chrysler"507 city = "Brossard"508509510class ToyotaGatineau(SM360Connector):511 source_id = "toyotagatineau"512 base_url = "https://www.toyotagatineau.ca"513 dealer_name = "Toyota Gatineau"514 city = "Gatineau"515516517# ---------------------------------------------------------------------------518# Expansion 3 — Côte-Nord, Bas-Saint-Laurent (Matane) et île de Montréal519# ---------------------------------------------------------------------------520521class SeptIlesHonda(SM360Connector):522 source_id = "septileshonda"523 base_url = "https://www.septileshonda.com"524 dealer_name = "Sept-Îles Honda"525 city = "Sept-Îles"526527528class SeptIlesGM(SM360Connector):529 source_id = "septilesgm"530 base_url = "https://www.sept-ilesgm.com"531 dealer_name = "Sept-Îles Chevrolet Buick GMC"532 city = "Sept-Îles"533534535class BaieComeauGM(SM360Connector):536 source_id = "baiecomeaugm"537 base_url = "https://www.baiecomeaugm.com"538 dealer_name = "Baie-Comeau Chevrolet Buick GMC"539 city = "Baie-Comeau"540541542class VolvoCarsSaintLeonard(SM360Connector):543 source_id = "volvosaintleonard"544 base_url = "https://www.volvocarssaintleonard.com"545 dealer_name = "Volvo Cars Saint-Léonard"546 city = "Montréal"547548549class KiaMatane(SM360Connector):550 source_id = "kiamatane"551 base_url = "https://www.kiamatane.com"552 dealer_name = "Kia Matane"553 city = "Matane"554555556class MarquisAutomobiles(SM360Connector):557 source_id = "marquisautomobiles"558 base_url = "https://www.marquisautomobiles.com"559 dealer_name = "Marquis Automobiles"560 city = "Matane"561562563class VilleneuveFord(SM360Connector):564 source_id = "villeneuveford"565 base_url = "https://www.villeneuveford.com"566 dealer_name = "Villeneuve Ford"567 city = "Matane"568