Python 61.6%
TypeScript 20.9%
CSS 11.4%
JavaScript 5.1%
HTML 1.1%
1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/autousagee.py : PORTAIL — AutoUsagee.ca (AMVOQ)5#6# AutoUsagee.ca est la place de marché officielle de l'AMVOQ (Association7# des marchands de véhicules d'occasion du Québec) : ~12 000 annonces de8# centaines de petits marchands indépendants dont la plupart n'ont AUCUN9# site propre connectable — c'est ici que vit leur inventaire (la même10# plateforme alimente les sites `theme_central` déjà connectés et les11# images image.autousagee.ca, voir central_dealers.py / yannicklaberge.py).12#13# Stratégie (HTML rendu serveur ; rate-limit IP agressif : 429 après14# ~150 requêtes rapprochées — la chaîne de résilience commune escalade15# alors vers Oxylabs résidentiel -cc-CA, vérifié 2026-08-23) :16# Liste : GET /recherche?type_id=1&page=N — 15 cartes/page, ~800 pages.17# Aucun jeton requis pour le GET (le _token CSRF ne sert qu'aux POST18# AJAX). Chaque carte : URL canonique (/vehicule-usagee/<marchand>/19# <id_marchand>/<slug>/<id_annonce>), photo 300px (→ 1024px), prix,20# marque/modèle FR (h2), année (<time datetime>), sous-modèle (h3),21# km, NOM DU MARCHAND et VILLE — l'annonce est complète sans détail.22# Détail (plafonné + cache BD, clé statique v1 — prix/km frais viennent23# de la liste) : specs label→valeur (Marque, Modèle, Sous-modèle,24# Odomètre, Transmission, Traction, Moteur/Cylindre, Carrosserie,25# Portières, Couleurs, Numéro d'unité, Numéro de série = VIN, PDSF,26# Consommation, Energie), description libre, galerie 1024px complète,27# site web du marchand. Fallback : bloc Contact@Once (data-cao-lpvars)28# porte year/make/model/VIN/StockID en JSON.29#30# type_id=1 = « voiture-camion-vus » seulement (2,3,4 = VR/loisirs/lourd,31# volontairement exclus — motoneiges, roulottes, bateaux mêlés aux motos).32#33# Recouvrement : quelques marchands AMVOQ déjà connectés en direct (Okaze,34# theme_central…) republient ici — la dédup VIN inter-sources masque les35# doublons, priorité au site du marchand.36#37# Plafonds : AUTOKA_AUTOUSAGEE_MAX (annonces, défaut 0 = tout),38# AUTOKA_MAX_PAGES (pages liste, tests), AUTOKA_MAX_DETAILS (vraies39# requêtes détail/sync, défaut 500 — inventaire enrichi progressivement).40# -----------------------------------------------------------------------------41from __future__ import annotations4243import json44import os45import re4647from ..schema import Vehicle, parse_mileage, parse_price48from .base import BaseConnector4950BASE = "https://www.autousagee.ca"51SEARCH = BASE + "/recherche"52IMG_HOST = "image.autousagee.ca"5354PAGE_SIZE = 1555MAX_PAGES_HARD = 1000 # garde-fou dur (~800 pages réelles)5657_URL_RE = re.compile(58 r"/vehicule-usagee/(?P<dealer_slug>[^/]+)/(?P<dealer_id>\d+)/"59 r"(?P<slug>[^/]+)/(?P<id>\d+)/?$")60_VIN_RE = re.compile(r"^[A-HJ-NPR-Z0-9]{17}$")61_TOTAL_RE = re.compile(r"([\d\s]{1,12})résultats")62_KM_RE = re.compile(r"^[\d\s,.]+\s*km$", re.I)63_CONSO_RE = re.compile(r"Ville\s*:\s*([\d.,]+).*?Autoroute\s*:\s*([\d.,]+)",64 re.I | re.S)65_CAO_RE = re.compile(r'data-cao-lpvars="([^"]+)"')6667# clé de cache détail (statique : specs/galerie quasi immuables ;68# le prix et le km frais viennent toujours de la liste)69_DETAIL_KEY = "v1"707172def _clean(text: str | None) -> str:73 return re.sub(r"\s+", " ", str(text or "")).strip()747576def _titlecase(text: str) -> str:77 """SALABERRY-DE-VALLEYFIELD -> Salaberry-De-Valleyfield (villes/marchands78 livrés en MAJUSCULES par la plateforme — les accents restent parfois en79 minuscules, ex. « ESTHéTIQUE », d'où le seuil plutôt que isupper())."""80 t = _clean(text)81 letters = [c for c in t if c.isalpha()]82 if letters and sum(c.isupper() for c in letters) / len(letters) >= 0.8:83 return t.title()84 return t858687def _max_listings() -> int:88 try:89 return max(0, int(os.environ.get("AUTOKA_AUTOUSAGEE_MAX", "0")))90 except ValueError:91 return 0929394class AutoUsageeQuebec(BaseConnector):95 """Portail AutoUsagee.ca — inventaire des marchands AMVOQ du Québec."""9697 source_id = "autousagee"98 request_delay = 1.0 # politesse : portail associatif99 max_details = 500 # plafond de vraies requêtes détail / sync100101 def __init__(self) -> None:102 super().__init__()103 self.session.headers.update({104 "Accept": "text/html,application/xhtml+xml",105 "Accept-Language": "fr-CA,fr;q=0.9",106 })107108 # -- liste ------------------------------------------------------------------109 def _list_page(self, page: int) -> tuple[list[dict], int | None]:110 """Cartes d'une page de résultats + total annoncé par le portail."""111 from bs4 import BeautifulSoup112 html = self.get(SEARCH, params={"type_id": 1, "page": page}).text113 soup = BeautifulSoup(html, "html.parser")114115 total = None116 m = _TOTAL_RE.search(soup.get_text(" "))117 if m:118 try:119 total = int(re.sub(r"\s", "", m.group(1)))120 except ValueError:121 pass122123 rows: list[dict] = []124 for art in soup.select("article.content_item"):125 link = art.select_one("a.link_img[href]")126 if link is None:127 continue128 um = _URL_RE.search(link["href"].strip())129 if um is None:130 continue131 row: dict = {132 "url": link["href"].strip(),133 "id": um.group("id"),134 "dealer_slug": um.group("dealer_slug"),135 "dealer_id": um.group("dealer_id"),136 }137 img = art.select_one(".img_vehicule img[src]")138 if img and IMG_HOST in img["src"]:139 src = img["src"]140 if src.startswith("//"):141 src = "https:" + src142 row["image"] = src.replace(f"{IMG_HOST}/300/",143 f"{IMG_HOST}/1024/")144 price_el = art.select_one(".prix_voiture span")145 row["price_label"] = _clean(price_el.get_text()) if price_el else ""146 h2 = art.select_one("h2.marque_modele")147 row["title"] = _clean(h2.get_text()) if h2 else ""148 h3 = art.select_one("h3.marque_modele")149 row["trim"] = _clean(h3.get_text()) if h3 else ""150 time_el = art.find("time")151 row["year"] = _clean(time_el.get("datetime")) if time_el else ""152153 # specs de carte : 1er span « NNN KM », l'autre = nom du marchand154 for span in art.select(".infos_voiture .spec .info span"):155 text = _clean(span.get_text())156 if not text:157 continue158 if _KM_RE.match(text):159 row["km_label"] = text160 elif "dealer" not in row or not row.get("dealer"):161 row["dealer"] = text162 loc = art.select_one(".infos_voiture_location .info span")163 row["city"] = _clean(loc.get_text()) if loc else ""164 rows.append(row)165 return rows, total166167 # -- page détail --------------------------------------------------------------168 def _fetch_detail(self, url: str) -> dict:169 from bs4 import BeautifulSoup170 html = self.get(url).text171 soup = BeautifulSoup(html, "html.parser")172 payload: dict = {}173174 # spécifications label -> valeur (Marque, Odomètre, Transmission,175 # Numéro de série…) — apostrophes typographiques normalisées176 specs: dict[str, str] = {}177 for info in soup.select("div.info"):178 lab, span = info.find("label"), info.find("span")179 if lab is None or span is None:180 continue181 key = _clean(lab.get_text()).rstrip(" |:").replace("’", "'")182 val = _clean(span.get_text(" "))183 if key and val and val != "--" and key not in specs:184 specs[key] = val185 payload["specs"] = specs186187 # description libre188 desc = soup.select_one("div.desc_container")189 if desc:190 payload["description"] = _clean(desc.get_text(" "))[:4000]191192 # galerie complète 1024px (propre à cette annonce)193 listing_id = _URL_RE.search(url)194 lid = listing_id.group("id") if listing_id else ""195 images: list[str] = []196 for m in re.finditer(197 rf"//{IMG_HOST}/1024/{lid}/[^\"'\s\\)]+", html):198 u = "https:" + m.group(0)199 if u not in images:200 images.append(u)201 payload["images"] = images[:30]202203 # site web du marchand (logo cliquable de la colonne coordonnées)204 site = soup.select_one(".logo_marchand a[href]")205 if site and site["href"].startswith("http"):206 payload["dealer_site"] = site["href"].strip()207208 # fallback Contact@Once : year/make/model/VIN/StockID en JSON209 cao = _CAO_RE.search(html)210 if cao:211 try:212 import html as htmllib213 pairs = json.loads(htmllib.unescape(cao.group(1)))214 payload["cao"] = {p["name"]: str(p.get("value") or "")215 for p in pairs if isinstance(p, dict)}216 except (ValueError, KeyError, TypeError):217 pass218 return payload219220 # -- contrat --------------------------------------------------------------221 def fetch(self) -> list[Vehicle]:222 cap_listings = _max_listings()223 page_cap = int(os.environ.get("AUTOKA_MAX_PAGES", MAX_PAGES_HARD))224 detail_cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))225226 rows: list[dict] = []227 seen: set[str] = set()228 page, total = 1, None229 while page <= min(page_cap, MAX_PAGES_HARD):230 batch, page_total = self._list_page(page)231 if not batch:232 break233 total = page_total or total234 for row in batch:235 if row["id"] not in seen:236 seen.add(row["id"])237 rows.append(row)238 if cap_listings and len(rows) >= cap_listings:239 rows = rows[:cap_listings]240 break241 if total is not None and page * PAGE_SIZE >= total:242 break243 page += 1244245 vehicles: list[Vehicle] = []246 real_fetches = 0247 for row in rows:248 extra: dict = {}249 if real_fetches >= detail_cap:250 # budget épuisé : on sert quand même le cache BD existant251 from .. import db252 if self._detail_con is None:253 self._detail_con = db.connect()254 extra = db.get_cached_detail(self._detail_con, self.source_id,255 row["id"], _DETAIL_KEY) or {}256 else:257 before = self._last_request258 try:259 extra = self.detail(row["id"], _DETAIL_KEY,260 lambda u=row["url"]:261 self._fetch_detail(u))262 except Exception:263 extra = {} # détail indisponible : données liste suffisent264 if self._last_request != before:265 real_fetches += 1266 veh = self._to_vehicle(row, extra)267 if veh is not None:268 vehicles.append(veh)269 return vehicles270271 # -- carte + détail -> Vehicle ---------------------------------------------272 def _to_vehicle(self, row: dict, extra: dict) -> Vehicle | None:273 title = row.get("title") or ""274 if not title:275 return None276 specs: dict = extra.get("specs") or {}277 cao: dict = extra.get("cao") or {}278279 year = None280 for raw in (row.get("year"), cao.get("year")):281 try:282 year = int(str(raw)[:4])283 break284 except (TypeError, ValueError):285 continue286287 price = parse_price(row.get("price_label") or "")288 km = parse_mileage(row.get("km_label") or "") \289 or parse_mileage(specs.get("Odomètre") or "")290291 make = _clean(specs.get("Marque") or cao.get("make") or "")292 model = _clean(specs.get("Modèle") or cao.get("model") or "")293 trim = _clean(specs.get("Sous-modèle") or row.get("trim") or "")294295 vin = _clean(specs.get("Numéro de série") or cao.get("VIN") or "").upper()296 if not _VIN_RE.match(vin):297 vin = ""298299 engine = ""300 litres = specs.get("Moteur") or ""301 if re.fullmatch(r"\d+(\.\d+)?", litres):302 cyl = specs.get("Cylindre") or ""303 engine = f"{litres}L" + (f" {cyl} cyl." if cyl.isdigit() else "")304305 doors = None306 try:307 doors = int(specs["Nombre de portières"])308 except (KeyError, ValueError):309 pass310311 images = [u for u in (extra.get("images") or []) if isinstance(u, str)]312 if not images and row.get("image"):313 images = [row["image"]]314315 details: dict = {}316 msrp = parse_price(specs.get("Prix neuf (PDSF)") or "")317 if msrp and (not price or msrp > price):318 details["msrp"] = msrp319 conso = _CONSO_RE.search(specs.get("Consommation (Litre / 100 Km)")320 or "")321 if conso:322 details["fuel_city_l_100km"] = conso.group(1).replace(",", ".")323 details["fuel_hwy_l_100km"] = conso.group(2).replace(",", ".")324 if extra.get("dealer_site"):325 details["dealer_site"] = extra["dealer_site"]326 details["dealer_slug"] = row["dealer_slug"]327 details["dealer_amvoq_id"] = row["dealer_id"]328329 return Vehicle(330 source=self.source_id,331 external_id=row["id"],332 url=row["url"],333 title=f"{title} {year or ''}".strip(),334 make=make,335 model=model,336 trim=trim,337 year=year,338 price=price,339 price_label=row.get("price_label") or "",340 mileage_km=km,341 mileage_label=row.get("km_label") or "",342 transmission=_clean(specs.get("Transmission") or ""),343 fuel=_clean(specs.get("Energie") or specs.get("Énergie") or ""),344 drivetrain=_clean(specs.get("Traction") or ""),345 body_type=_clean(specs.get("Carrosserie") or ""),346 exterior_color=_clean(specs.get("Couleur extérieur") or ""),347 interior_color=_clean(specs.get("Couleur intérieur") or ""),348 engine=engine,349 doors=doors,350 vin=vin,351 stock_number=_clean(specs.get("Numéro d'unité")352 or cao.get("StockID") or ""),353 dealer_name=_titlecase(row.get("dealer") or ""),354 city=_titlecase(row.get("city") or ""),355 description=extra.get("description") or "",356 details=details,357 images=images[:30],358 )359