spb/auto-ka Public
Python 81.8%
TypeScript 12.4%
CSS 5.5%
1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/jlkauto.py : JLK Auto (Montréal) — plateforme EDealer5# (edealer.ca ; WordPress + inventaire EDealer, CDN media.edealer.ca).6#7# Stratégie : le sitemap Yoast `/inventory-listing-sitemap.xml` recense les8# pages véhicule (…-<token>vdp/) en FR et EN. Chaque page détail (VDP)9# expose un JSON-LD schema.org @graph avec un nœud @type [Product, Car]10# complet : prix, VIN, km, transmission, carrosserie, couleurs, moteur.11# Pages détail mises en cache BD (clé hebdomadaire), comme les sites D2C.12# -----------------------------------------------------------------------------13from __future__ import annotations1415import datetime16import json17import os18import re1920from ..schema import Vehicle21from .base import BaseConnector2223# …/inventaire/2005-chrysler-crossfire-limitee-jGKAbGVgRXus0SSkyI35kgvdp/24# le token (id EDealer) est le dernier segment alphanumérique SANS tiret :25# le même véhicule a un slug différent en FR et en EN, mais le même token26_VDP_RE = re.compile(r"-([A-Za-z0-9]{12,})vdp/?$")2728_LD_RE = re.compile(29 r'<script type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', re.S)3031_DRIVE_MAP = {32 "FrontWheelDriveConfiguration": "Traction",33 "RearWheelDriveConfiguration": "Propulsion",34 "AllWheelDriveConfiguration": "Intégrale / 4x4",35 "FourWheelDriveConfiguration": "Intégrale / 4x4",36}373839def _qty(node) -> float | None:40 """QuantitativeValue -> float ; accepte aussi un scalaire nu."""41 value = node.get("value") if isinstance(node, dict) else node42 try:43 v = float(value)44 return v if v > 0 else None45 except (TypeError, ValueError):46 return None474849class JLKAuto(BaseConnector):50 source_id = "jlkauto"51 base_url = "https://www.jlkauto.ca"52 dealer_name = "JLK Auto"53 city = "Montréal"54 request_delay: float = 1.055 max_details = 4005657 # -- sitemap ----------------------------------------------------------------58 def _vehicle_urls(self) -> list[str]:59 xml = self.get(f"{self.base_url}/inventory-listing-sitemap.xml").text60 by_token: dict[str, str] = {}61 for loc in re.findall(r"<loc>([^<]+)</loc>", xml):62 loc = loc.strip()63 m = _VDP_RE.search(loc)64 if not m:65 continue # pages listes (…vlp) et autres66 token = m.group(1)67 # préférer la page FR (/inventaire/) à la page EN (/inventory/)68 if token not in by_token or "/inventaire/" in loc:69 by_token[token] = loc70 return [by_token[t] for t in sorted(by_token)]7172 # -- page détail -> nœud JSON-LD [Product, Car] ------------------------------73 def _fetch_detail(self, url: str) -> dict:74 html = self.get(url).text75 for block in _LD_RE.findall(html):76 try:77 data = json.loads(block.strip())78 except ValueError:79 continue80 nodes = data.get("@graph") if isinstance(data, dict) else None81 if nodes is None:82 nodes = data if isinstance(data, list) else [data]83 for node in nodes:84 types = node.get("@type")85 types = types if isinstance(types, list) else [types]86 if "Car" in types or "Vehicle" in types:87 # galerie complète depuis le CDN EDealer (JSON-LD n'a88 # qu'une photo)89 imgs = re.findall(90 r'https://media\.edealer\.ca/[^"\'\\\s]+\.(?:webp|jpe?g|png)',91 html)92 seen: set[str] = set()93 node["_gallery"] = [u for u in imgs94 if not (u in seen or seen.add(u))][:20]95 return node96 return {}9798 # -- contrat -----------------------------------------------------------------99 def fetch(self) -> list[Vehicle]:100 urls = self._vehicle_urls()101 week = datetime.date.today().isocalendar()102 cache_key = f"v1:{week.year}w{week.week}"103 cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))104105 vehicles: list[Vehicle] = []106 real_fetches = 0107 for url in urls:108 ext_id = _VDP_RE.search(url).group(1)109110 def _fetch(u=url):111 return self._fetch_detail(u)112113 if real_fetches >= cap: # plafond : cache seulement114 from .. import db115 if self._detail_con is None:116 self._detail_con = db.connect()117 data = db.get_cached_detail(self._detail_con, self.source_id,118 ext_id, cache_key)119 if data is None:120 continue121 else:122 before = self._last_request123 data = self.detail(ext_id, cache_key, _fetch)124 if self._last_request != before:125 real_fetches += 1126 if not data:127 continue128 veh = self._to_vehicle(ext_id, url, data)129 if veh is not None:130 vehicles.append(veh)131 return vehicles132133 def _to_vehicle(self, ext_id: str, url: str, d: dict) -> Vehicle | None:134 # véhicules usagés uniquement135 cond = str(d.get("itemCondition") or "")136 if cond and "Used" not in cond:137 return None138139 title = " ".join(str(d.get("name") or "").split())140 if not title:141 return None142143 offers = d.get("offers") or {}144 price = None145 try:146 p = float(offers.get("price"))147 price = p if p > 0 else None148 except (TypeError, ValueError):149 pass150151 km = _qty(d.get("mileageFromOdometer"))152153 brand = d.get("brand") or {}154 make = brand.get("name") if isinstance(brand, dict) else str(brand)155156 drivetrain = ""157 drive_raw = str(d.get("driveWheelConfiguration") or "")158 for key, canon in _DRIVE_MAP.items():159 if key in drive_raw:160 drivetrain = canon161 break162163 year = None164 for k in ("vehicleModelDate", "modelDate", "productionDate"):165 try:166 year = int(str(d.get(k))[:4])167 break168 except (TypeError, ValueError):169 continue170171 engine = d.get("vehicleEngine") or {}172 engine_name = engine.get("name") if isinstance(engine, dict) else ""173174 images = d.get("_gallery") or []175 if not images and isinstance(d.get("image"), str):176 images = [d["image"]]177178 doors = d.get("numberOfDoors")179 doors = doors if isinstance(doors, int) else None180 try:181 seats = int(str(d.get("vehicleSeatingCapacity")))182 except (TypeError, ValueError):183 seats = None184185 details = {k: d[k] for k in ("numberOfForwardGears", "wheelbase",186 "fuelConsumption") if d.get(k)}187188 return Vehicle(189 source=self.source_id,190 external_id=ext_id,191 url=url,192 title=title,193 make=str(make or ""),194 model=str(d.get("model") or ""),195 trim=str(d.get("vehicleConfiguration") or ""),196 year=year,197 price=price,198 price_label=(f"{price:,.0f} $".replace(",", " ") if price else ""),199 mileage_km=km,200 mileage_label=(f"{km:,.0f} km".replace(",", " ") if km else ""),201 transmission=str(d.get("vehicleTransmission") or ""),202 fuel=str(d.get("fuelType") or ""),203 drivetrain=drivetrain,204 body_type=str(d.get("bodyType") or ""),205 exterior_color=str(d.get("color") or ""),206 interior_color=str(d.get("vehicleInteriorColor") or ""),207 engine=str(engine_name or ""),208 doors=doors,209 seats=seats,210 vin=str(d.get("vehicleIdentificationNumber") or ""),211 stock_number=str(d.get("sku") or ""),212 dealer_name=self.dealer_name,213 city=self.city,214 description=" ".join(str(d.get("description") or "").split())[:4000],215 details=details,216 images=images,217 )218