spb/auto-ka Public
Python 81.9%
TypeScript 12.4%
CSS 5.5%
1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/central_dealers.py : connecteurs des concessionnaires sur la5# plateforme AMVOQ « Focus 360 / Instavid360 » (thème WordPress6# `theme_central`, inventaire Vue.js alimenté par un index Solr).7#8# Stratégie : le thème expose un endpoint JSON non authentifié9# /wp-content/themes/theme_central/include/functions/RechercheVoitures.php10# (exige seulement un en-tête Referer du même site) qui renvoie les11# documents Solr complets de l'inventaire : gvo_id, année, marque, modèle,12# sous-modèle, prix, km, transmission, carburant, traction, carrosserie,13# couleurs, portes, VIN, no d'unité, municipalité/région et 1re photo.14#15# La page détail (/vehicule-usage/<slug>/<gvo_id>/, rendue côté serveur,16# le slug est ignoré par le serveur) complète : galerie photos complète,17# description, options/équipements, nb de passagers, cylindrée. Ces pages18# sont mises en cache BD (clé statique v1 : le prix/km frais viennent19# toujours de l'API liste, le contenu détail est quasi immuable).20#21# vehicule_type_i discrimine le type d'inventaire (1 = automobile,22# 2 = moto, 3 = VR) ; on garde le type 1 + un garde-fou regex sur la23# catégorie/carrosserie (motoneige, VTT, Classe A/B/C, récréatif…).24# -----------------------------------------------------------------------------25from __future__ import annotations2627import html as htmllib28import os29import re3031from ..schema import Vehicle, strip_accents32from .base import BaseConnector3334IMG_HOST = "https://image.autousagee.ca"35_SEARCH_PATH = ("/wp-content/themes/theme_central/include/functions/"36 "RechercheVoitures.php")3738# catégories / carrosseries qui ne sont PAS des automobiles usagées39_EXCLUDE_RE = re.compile(40 r"moto(?:cyclette|neige|marine)?\b|\bvtt\b|\batv\b|scooter|spyder|"41 r"classe\s*[abc]\b|recreatif|roulotte|remorque|bateau|ponton|"42 r"tracteur|souffleuse|habitation|\bvr\b|campeur", re.I)434445def _slugify(text: str) -> str:46 """Slug d'URL à la façon du thème (le serveur ne le valide pas)."""47 t = strip_accents(text or "").lower()48 t = re.sub(r"[^a-z0-9]+", "-", t)49 return t.strip("-") or "vehicule"505152def _clean(text: str | None) -> str:53 if not text:54 return ""55 out = htmllib.unescape(str(text))56 return re.sub(r"\s+", " ", out).strip()575859def _num(value, cast=float):60 try:61 v = cast(value)62 except (TypeError, ValueError):63 return None64 return v if v else None656667class CentralConnector(BaseConnector):68 """Base commune aux sites `theme_central` (AMVOQ / Instavid360)."""6970 base_url: str = ""71 dealer_name: str = ""72 city: str = ""73 request_delay: float = 1.074 page_size: int = 2007576 # plafond de vraies requêtes détail par sync (AUTOKA_MAX_DETAILS le réduit77 # pour les tests) — au-delà, le véhicule est émis avec les données liste78 # seulement et sera enrichi aux syncs suivants.79 max_details = 4008081 # -- API liste (Solr) -------------------------------------------------------82 def _api_get(self, params: dict) -> dict:83 url = self.base_url + _SEARCH_PATH84 headers = {85 "Referer": f"{self.base_url}/inventaire/",86 "Accept": "application/json, text/plain, */*",87 }88 return self.get(url, params=params, headers=headers).json()8990 def _api_docs(self) -> list[dict]:91 docs: list[dict] = []92 page = 193 while page <= 50:94 data = self._api_get({95 "order": "alphaASC",96 "type": 0,97 "nbCarsPerPage": self.page_size,98 "currentPage": page,99 })100 batch = data.get("data") or []101 docs.extend(batch)102 total = int((data.get("meta") or {}).get("numFound") or 0)103 if not batch or len(docs) >= total:104 break105 page += 1106 return docs107108 # -- page détail ------------------------------------------------------------109 def _fetch_detail(self, url: str) -> dict:110 from bs4 import BeautifulSoup111 soup = BeautifulSoup(self.get(url).text, "html.parser")112 payload: dict = {}113114 # galerie photos complète115 images = []116 for img in soup.select(".vdp__slider-image img[src]"):117 src = img["src"]118 if "autousagee.ca" in src and src not in images:119 images.append(src)120 payload["images"] = images[:30]121122 # description libre123 desc = soup.select_one('div[name="description-content"]')124 if desc:125 text = _clean(desc.get_text(" "))126 text = re.sub(r"^DESCRIPTION DU V[ÉE]HICULE\s*", "", text, flags=re.I)127 payload["description"] = text[:4000]128129 # spécifications (label -> valeur)130 specs: dict[str, str] = {}131 info = soup.select_one('div[name="informations-content"]')132 if info:133 for li in info.select("li"):134 spans = [_clean(s.get_text()) for s in li.find_all("span")]135 if len(spans) >= 2 and spans[0] and "--" not in spans[1]:136 specs[spans[0]] = spans[1]137 payload["specs"] = specs138139 # options / équipements140 feats = []141 opts = soup.select_one('div[name="options-content"]')142 if opts:143 for li in opts.select("li"):144 t = _clean(li.get_text(" "))145 if t and t not in feats:146 feats.append(t)147 payload["features"] = feats[:80]148 return payload149150 # -- doc Solr -> Vehicle ------------------------------------------------------151 def _to_vehicle(self, doc: dict) -> Vehicle | None:152 gvo_id = doc.get("vehicule_gvo_id_i")153 make = _clean(doc.get("vehicule_marque_nom_s"))154 model = _clean(doc.get("vehicule_modele_s"))155 if not gvo_id or not (make or model):156 return None157158 year = _num(doc.get("vehicule_annee_i"), int)159 trim = _clean(doc.get("vehicule_sous_modele_valide_s"))160 slug = _slugify(f"{make} {model} {year or ''}")161 url = f"{self.base_url}/vehicule-usage/{slug}/{gvo_id}/"162163 price = _num(doc.get("vehicule_prix_d"))164 km = _num(doc.get("vehicule_km_i"))165 city = _clean(doc.get("vehicule_municipalite_s")) or self.city166 body = _clean(doc.get("vehicule_carrosserie_type_s")).split(",")[0]167168 photos = doc.get("vehicule_photo_t") or []169 if isinstance(photos, str):170 photos = [photos]171 images = [f"{IMG_HOST}/1024/{p}" for p in photos if isinstance(p, str)]172173 details: dict = {}174 for src_key, out_key in (("vehicule_rabais_d", "rabais"),175 ("vehicule_categorie_s", "categorie"),176 ("vehicule_garantie_legal_texte_s", "garantie")):177 if doc.get(src_key):178 details[out_key] = doc[src_key]179180 title = " ".join(x for x in (make, model, trim, str(year or "")) if x)181 return Vehicle(182 source=self.source_id,183 external_id=str(gvo_id),184 url=url,185 title=title,186 make=make,187 model=model,188 trim=trim,189 year=year,190 price=price,191 price_label=f"{price:,.0f} $".replace(",", " ") if price else "",192 mileage_km=km,193 mileage_label=f"{km:,.0f} km".replace(",", " ") if km else "",194 transmission=_clean(doc.get("vehicule_transmission_s")),195 fuel=_clean(doc.get("vehicule_carburant_nom_s")),196 drivetrain=_clean(doc.get("vehicule_traction_s")),197 body_type=body,198 exterior_color=_clean(doc.get("vehicule_couleur_exterieur_s")),199 interior_color=_clean(doc.get("vehicule_couleur_interieur_s")),200 engine=_clean(doc.get("vehicule_moteur_s")),201 doors=_num(doc.get("vehicule_porte_nom_s"), int),202 vin=_clean(doc.get("vehicule_numero_serie_s")),203 stock_number=_clean(doc.get("vehicule_numero_unite_s")),204 dealer_name=self.dealer_name,205 city=city.title() if city.isupper() else city,206 region=_clean(doc.get("vehicule_region_s")),207 details=details,208 images=images,209 )210211 def _enrich(self, veh: Vehicle, payload: dict) -> None:212 if payload.get("images"):213 veh.images = payload["images"][:30]214 if payload.get("description"):215 veh.description = payload["description"]216 if payload.get("features"):217 veh.features = payload["features"]218 specs = payload.get("specs") or {}219 seats = specs.get("Nombre de passagers")220 if seats and veh.seats is None:221 veh.seats = _num(seats, int)222 litres = specs.get("Moteur")223 cyl = specs.get("Cylindre")224 if litres and re.fullmatch(r"\d+(\.\d+)?", litres):225 veh.engine = f"{litres}L" + (f" {cyl} cyl." if cyl else "")226227 # -- contrat ------------------------------------------------------------------228 def fetch(self) -> list[Vehicle]:229 cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))230 vehicles: list[Vehicle] = []231 real_fetches = 0232 for doc in self._api_docs():233 type_i = doc.get("vehicule_type_i")234 if type_i not in (None, 1): # 2 = moto, 3 = VR235 continue236 cat = f"{doc.get('vehicule_categorie_s') or ''} " \237 f"{doc.get('vehicule_carrosserie_type_s') or ''}"238 if _EXCLUDE_RE.search(strip_accents(cat)):239 continue240 veh = self._to_vehicle(doc)241 if veh is None:242 continue243244 # enrichissement page détail (galerie, description, options) —245 # servi du cache BD sauf pour les nouveaux véhicules, dans la246 # limite du budget de vraies requêtes.247 if real_fetches < cap:248 def _fetch(u=veh.url):249 return self._fetch_detail(u)250 before = self._last_request251 try:252 payload = self.detail(veh.external_id, "v1", _fetch)253 except Exception:254 payload = {}255 if self._last_request != before:256 real_fetches += 1257 self._enrich(veh, payload)258 vehicles.append(veh)259 return vehicles260261262# ---------------------------------------------------------------------------263# Concessionnaires (1 sous-classe = 1 source)264# ---------------------------------------------------------------------------265266class DenisAuto(CentralConnector):267 source_id = "denisauto"268 base_url = "https://www.denisauto.ca"269 dealer_name = "Denis Auto"270 city = "Gatineau"271272273class JRAuto(CentralConnector):274 source_id = "jrautotr"275 base_url = "https://www.jrautotr.com"276 dealer_name = "J.R. Auto"277 city = "Trois-Rivières"278279280class Autosag(CentralConnector):281 source_id = "autosag"282 base_url = "https://www.autosag.com"283 dealer_name = "Autosag"284 city = "Chicoutimi"285286287class LeCentreAutoQuebec(CentralConnector):288 source_id = "lecentreautoquebec"289 base_url = "https://www.lecentreautoquebec.com"290 dealer_name = "Le Centre Auto Québec Plus"291 city = "Québec"292293294class AutoCreditPlus(CentralConnector):295 source_id = "autocreditplus"296 base_url = "https://www.autocreditplus.ca"297 dealer_name = "Auto Crédit Plus"298 city = "Trois-Rivières"299300301class FleuryAutoGroupe(CentralConnector):302 source_id = "fleuryautogroupe"303 base_url = "https://www.fleuryautogroupe.com"304 dealer_name = "Fleury Auto Groupe"305 city = "Gatineau"306307308# ---------------------------------------------------------------------------309# Expansion 3 — Bas-Saint-Laurent et île de Montréal310# ---------------------------------------------------------------------------311312class AutoBSL(CentralConnector):313 source_id = "autobsl"314 base_url = "https://www.autobsl.com"315 dealer_name = "Auto BSL"316 city = "Rimouski"317318319class AutoXtreme(CentralConnector):320 source_id = "autoxtreme"321 base_url = "https://www.autoxtreme.ca"322 dealer_name = "Auto Xtrême"323 city = "Montréal"324325326class DAstousAuto(CentralConnector):327 source_id = "dastousauto"328 base_url = "https://www.dastousauto.com"329 dealer_name = "D'Astous Auto"330 city = "Montréal"331