spb/auto-ka Public
Python 81.9%
TypeScript 12.4%
CSS 5.5%
1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/albioccasion.py : connecteur ALBI Occasion (albioccasion.com),5# la division occasion d'ALBI le Géant — très grosse chaîne multi-6# succursales (Mascouche, Repentigny, Blainville, Laval, Rive-Nord…),7# ~1 100+ véhicules d'occasion.8#9# Plateforme : ASP.NET WebForms maison. La liste /fr/inventaire/?page=N10# (12 véhicules/page, « Page X de Y » dans le HTML) référence chaque page11# détail /fr/vehicule/<id>/<slug>/. La page détail expose un microdata12# schema.org complet (itemprop) : prix, km, VIN, année, version,13# transmission, portes, places, couleurs, carrosserie + des lignes de14# spécifications (SpecLine : entraînement, catégorie…), la galerie photos15# (S3 enzo-production …/vehicules_medias/…) et la succursale réelle du16# véhicule (lien Google Maps « ALBI Occasion <Ville> »).17#18# Pages détail en cache BD (clé hebdomadaire — le prix est revalidé chaque19# semaine). AUTOKA_MAX_PAGES limite la pagination liste et20# AUTOKA_MAX_DETAILS plafonne les vraies requêtes détail par sync.21# -----------------------------------------------------------------------------22from __future__ import annotations2324import datetime25import html as htmllib26import os27import re28import urllib.parse2930from ..schema import Vehicle31from .base import BaseConnector3233_LINK_RE = re.compile(r'href="/fr/vehicule/(\d+)/([^"/]+)/?"')34_TOTAL_PAGES_RE = re.compile(r'TotalPagesLabel">\s*(\d+)')3536_H1_RE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S)37_ITEMPROP_CONTENT_RE = r'itemprop="{name}"[^>]*content="([^"]*)"'38_ITEMPROP_TEXT_RE = r'itemprop="{name}"[^>]*>\s*([^<]+)'39_SPEC_RE = re.compile(r'class="SpecLine">\s*([^:<]+):\s*(?:<span[^>]*>)?'40 r"\s*([^<]*)", re.S)41_STOCK_RE = re.compile(r'data\.stock\s*=\s*"([^"]+)"')42_BRANCH_RE = re.compile(r"maps/dir//([^,\"]+),")43_IMG_RE = re.compile(r'https://[^"\'\s\\]+/vehicules_medias/[^"\'\s\\]+'44 r"\.(?:jpe?g|png|webp)", re.I)4546_EXCLUDE_RE = re.compile(47 r"moto(?:cyclette|neige|marine)?\b|\bvtt\b|scooter|spyder|ryker|"48 r"can-?am|ski-?doo|sea-?doo|roulotte|remorque|\bvr\b|campeur", re.I)495051def _clean(text) -> str:52 if not text:53 return ""54 out = htmllib.unescape(str(text))55 out = re.sub(r"<[^>]+>", " ", out)56 return re.sub(r"\s{2,}", " ", out).strip()575859def _prop(html: str, name: str) -> str:60 m = re.search(_ITEMPROP_CONTENT_RE.format(name=name), html)61 if m and m.group(1).strip():62 return _clean(m.group(1))63 m = re.search(_ITEMPROP_TEXT_RE.format(name=name), html)64 return _clean(m.group(1)) if m else ""656667def _to_float(text) -> float | None:68 if text in (None, ""):69 return None70 t = re.sub(r"[^\d.]", "", str(text))71 try:72 return float(t) if t else None73 except ValueError:74 return None757677class AlbiOccasion(BaseConnector):78 """ALBI Occasion — liste ASP.NET paginée + pages détail (microdata)."""7980 source_id = "albioccasion"81 base_url = "https://www.albioccasion.com"82 dealer_name = "ALBI Occasion"83 city = "Mascouche"84 request_delay = 2.5 # le site retourne 429 sous ~1 s/requête85 max_pages = 150 # garde-fou dur (~95 pages réelles)86 max_details = 400 # plafond de vraies requêtes par sync8788 def get(self, url, **kw):89 """GET avec backoff sur 429 (rate-limit ALBI)."""90 import time as _t9192 import requests as _rq93 for attempt in range(4):94 try:95 return super().get(url, **kw)96 except _rq.exceptions.HTTPError as exc:97 if exc.response is not None and exc.response.status_code == 429 \98 and attempt < 3:99 _t.sleep(20 * (attempt + 1)) # 20 s, 40 s, 60 s100 continue101 raise102103 # -- pages liste -> [(id, slug)] --------------------------------------------104 def _vehicle_refs(self) -> list[tuple[str, str]]:105 cap = int(os.environ.get("AUTOKA_MAX_PAGES", self.max_pages))106 refs: list[tuple[str, str]] = []107 seen: set[str] = set()108109 page, last_page = 1, 1110 while page <= min(last_page, cap, self.max_pages):111 url = f"{self.base_url}/fr/inventaire/"112 if page > 1:113 url += f"?page={page}"114 html = self.get(url).text115 if page == 1:116 m = _TOTAL_PAGES_RE.search(html)117 if m:118 last_page = int(m.group(1))119 new = 0120 for veh_id, slug in _LINK_RE.findall(html):121 if veh_id not in seen:122 seen.add(veh_id)123 refs.append((veh_id, slug))124 new += 1125 if new == 0: # page recyclée / fin atteinte126 break127 page += 1128 return refs129130 # -- page détail -> payload ---------------------------------------------------131 def _fetch_detail(self, url: str) -> dict:132 html = self.get(url).text133 payload: dict = {}134135 m = _H1_RE.search(html)136 if m:137 payload["title"] = _clean(m.group(1))138139 for name in ("price", "vehicleModelDate", "vehicleConfiguration",140 "vehicleIdentificationNumber", "mileageFromOdometer",141 "vehicleTransmission", "color", "vehicleInteriorColor",142 "bodyType", "numberOfDoors", "vehicleSeatingCapacity",143 "description", "name"):144 value = _prop(html, name)145 if value:146 payload[name] = value147148 specs: dict[str, str] = {}149 for label, value in _SPEC_RE.findall(html):150 label, value = _clean(label), _clean(value)151 if label and value:152 specs.setdefault(label, value)153 payload["specs"] = specs154155 m = _STOCK_RE.search(html)156 if m:157 payload["stock"] = m.group(1).strip()158 m = re.search(r'data\.make\s*=\s*"([^"]*)"', html)159 if m:160 payload["make"] = m.group(1).strip()161 m = re.search(r'data\.model\s*=\s*"([^"]*)"', html)162 if m:163 payload["model"] = m.group(1).strip()164165 m = _BRANCH_RE.search(html)166 if m:167 payload["branch"] = urllib.parse.unquote_plus(m.group(1)).strip()168169 images, seen = [], set()170 for u in _IMG_RE.findall(html):171 if u not in seen:172 seen.add(u)173 images.append(u)174 payload["images"] = images[:20]175 return payload176177 # -- contrat ---------------------------------------------------------------178 def fetch(self) -> list[Vehicle]:179 refs = self._vehicle_refs()180 week = datetime.date.today().isocalendar()181 cache_key = f"v1:{week.year}w{week.week}" # revalidation hebdomadaire182 cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))183184 vehicles: list[Vehicle] = []185 real_fetches = 0186 for veh_id, slug in refs:187 url = f"{self.base_url}/fr/vehicule/{veh_id}/{slug}/"188189 def _fetch(u=url):190 return self._fetch_detail(u)191192 if real_fetches >= cap: # plafond : cache seulement193 from .. import db194 if self._detail_con is None:195 self._detail_con = db.connect()196 data = db.get_cached_detail(self._detail_con, self.source_id,197 veh_id, cache_key)198 if data is None:199 continue200 else:201 before = self._last_request202 try:203 data = self.detail(veh_id, cache_key, _fetch)204 except Exception:205 # page disparue (vendu) ou erreur ponctuelle : continuer206 if self._last_request != before:207 real_fetches += 1208 continue209 if self._last_request != before:210 real_fetches += 1211 if not data:212 continue213 veh = self._to_vehicle(veh_id, url, data)214 if veh is not None:215 vehicles.append(veh)216 return vehicles217218 # -- payload -> Vehicle ------------------------------------------------------219 def _to_vehicle(self, veh_id: str, url: str, d: dict) -> Vehicle | None:220 specs = d.get("specs") or {}221 make = _clean(d.get("make"))222 model = _clean(d.get("model"))223 title = _clean(d.get("name") or d.get("title"))224 if not title and not (make or model):225 return None226 body = _clean(d.get("bodyType") or specs.get("Catégorie"))227 if _EXCLUDE_RE.search(f"{make} {model} {body} {title}"):228 return None229230 year = None231 m = re.search(r"(19|20)\d{2}", str(d.get("vehicleModelDate") or ""))232 if m:233 year = int(m.group(0))234235 branch = _clean(d.get("branch"))236 city = self.city237 if branch:238 # « ALBI Occasion Mascouche » -> ville ; les succursales de239 # marque (« ALBI Le Géant Mazda »…) restent sur la ville de repli240 m = re.match(r"ALBI\s+Occasion\s+(.+)$", branch, re.I)241 if m:242 city = m.group(1).strip()243244 details: dict = {}245 if branch:246 details["branch"] = branch247 if specs.get("Type"):248 details["size_class"] = specs["Type"]249 if specs.get("Garantie manufacturier"):250 details["manufacturer_warranty"] = specs["Garantie manufacturier"]251252 price = _to_float(d.get("price"))253 km = _to_float(d.get("mileageFromOdometer"))254255 return Vehicle(256 source=self.source_id,257 external_id=veh_id,258 url=url,259 title=title or " ".join(x for x in (make, model, year or "") if x),260 make=make,261 model=model,262 trim=_clean(d.get("vehicleConfiguration")),263 year=year,264 price=price,265 price_label=f"{price:,.0f} $".replace(",", " ") if price else "",266 mileage_km=km,267 mileage_label=f"{km:,.0f} km".replace(",", " ") if km else "",268 transmission=_clean(d.get("vehicleTransmission")269 or specs.get("Transmission")),270 drivetrain=_clean(specs.get("Entraînement")),271 body_type=body,272 exterior_color=_clean(d.get("color")273 or specs.get("Couleur extérieure")),274 interior_color=_clean(d.get("vehicleInteriorColor")275 or specs.get("Couleur intérieure")),276 doors=int(_to_float(d.get("numberOfDoors")277 or specs.get("Portes")) or 0) or None,278 seats=int(_to_float(d.get("vehicleSeatingCapacity")279 or specs.get("Places Assises")) or 0) or None,280 vin=_clean(d.get("vehicleIdentificationNumber")),281 stock_number=_clean(d.get("stock")),282 dealer_name=branch or self.dealer_name,283 city=city,284 description=_clean(d.get("description"))[:4000],285 details=details,286 images=d.get("images") or [],287 )288