spb/auto-ka Public
Python 81.9%
TypeScript 12.4%
CSS 5.5%
1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/autodurocher.py : connecteur Entrepôt Auto Durocher5# (autodurocher.com), gros marchand indépendant de Mirabel (~200 véhicules,6# occasion seulement).7#8# Plateforme : WordPress + Elementor + JetEngine (CPT « vehicles »).9# Le sitemap /fr/sitemap_index.xml expose des vehicles-sitemapN.xml qui ne10# recensent QUE l'inventaire courant (/fr/vehicles/<slug>/). Chaque page11# détail embarque un JSON-LD (script class="fute-custom-schema-vehicles",12# Organization -> makesOffer -> itemOffered @type Car) : prix, km,13# transmission, couleur, carrosserie, rouage, année, marque, modèle — et la14# galerie photos (CDN AutoTrader 1s-photomanager). Le no de stock est le15# dernier segment du slug. Le JSON contient des sauts de ligne bruts ->16# json.loads(strict=False).17#18# Pages détail en cache BD (clé hebdomadaire) ; AUTOKA_MAX_DETAILS (env)19# plafonne les vraies requêtes détail par synchronisation.20# -----------------------------------------------------------------------------21from __future__ import annotations2223import datetime24import html as htmllib25import json26import os27import re2829from ..schema import Vehicle30from .base import BaseConnector3132_SITEMAP_RE = re.compile(r"vehicles-sitemap\d+\.xml$")33_VEH_URL_RE = re.compile(r"/fr/vehicles/([a-z0-9-]+)/?$")34_STOCK_RE = re.compile(r"-(?:19|20)\d{2}-([a-z0-9]+)$")3536_SCHEMA_RE = re.compile(37 r'<script type=["\']application/ld\+json["\'][^>]*'38 r'class="fute-custom-schema-vehicles"[^>]*>(.*?)</script>', re.S)39_LD_RE = re.compile(40 r'<script type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', re.S)41_IMG_RE = re.compile(r'https://1s-photomanager[^"\'\s\\<>]+', re.I)4243_FUEL_HINT_RE = re.compile(44 r"(hybride rechargeable|plug-?in|phev|hybride|hybrid|électrique|"45 r"electrique|electric|\bev\b|diesel|tdi)", re.I)4647_EXCLUDE_RE = re.compile(48 r"moto(?:cyclette|neige|marine)?\b|\bvtt\b|scooter|spyder|ryker|"49 r"can-?am|ski-?doo|sea-?doo|roulotte|remorque|\bvr\b|campeur", re.I)505152def _clean(text) -> str:53 if not text:54 return ""55 out = htmllib.unescape(str(text))56 out = re.sub(r"<[^>]+>", " ", out)57 return re.sub(r"\s+", " ", out).strip()585960def _num(value):61 try:62 v = float(str(value).replace(",", "").replace(" ", ""))63 except (TypeError, ValueError):64 return None65 return v if v else None666768class AutoDurocher(BaseConnector):69 """Entrepôt Auto Durocher — sitemaps vehicles + JSON-LD des pages détail."""7071 source_id = "autodurocher"72 base_url = "https://www.autodurocher.com"73 dealer_name = "Entrepôt Auto Durocher"74 city = "Mirabel"75 request_delay = 1.076 max_details = 300 # plafond de vraies requêtes par sync7778 # -- sitemaps ----------------------------------------------------------------79 def _vehicle_slugs(self) -> list[str]:80 index = self.get(f"{self.base_url}/fr/sitemap_index.xml").text81 maps = [loc.strip() for loc in re.findall(r"<loc>([^<]+)</loc>", index)82 if _SITEMAP_RE.search(loc.strip())]83 slugs: list[str] = []84 for sitemap_url in maps:85 try:86 xml = self.get(sitemap_url).text87 except Exception:88 continue # un sitemap HS ne bloque pas89 for loc in re.findall(r"<loc>([^<]+)</loc>", xml):90 m = _VEH_URL_RE.search(loc.strip())91 if m:92 slugs.append(m.group(1))93 return sorted(set(slugs))9495 # -- page détail -> payload ---------------------------------------------------96 def _fetch_detail(self, url: str) -> dict:97 html = self.get(url).text98 payload: dict = {}99100 blocks = _SCHEMA_RE.findall(html) or _LD_RE.findall(html)101 for block in blocks:102 try:103 data = json.loads(block.strip(), strict=False)104 except ValueError:105 continue106 offer = data.get("makesOffer") or {}107 car = offer.get("itemOffered") or {}108 if not isinstance(car, dict) or car.get("@type") != "Car":109 continue110 spec = offer.get("priceSpecification") or {}111 payload["price"] = spec.get("price")112 payload["car"] = {k: v for k, v in car.items()113 if not str(k).startswith("@")}114 break115116 images, seen = [], set()117 for u in _IMG_RE.findall(html):118 if u not in seen:119 seen.add(u)120 images.append(u)121 payload["images"] = images[:20]122 return payload123124 # -- contrat ---------------------------------------------------------------125 def fetch(self) -> list[Vehicle]:126 slugs = self._vehicle_slugs()127 week = datetime.date.today().isocalendar()128 cache_key = f"v1:{week.year}w{week.week}" # revalidation hebdomadaire129 cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))130131 vehicles: list[Vehicle] = []132 real_fetches = 0133 for slug in slugs:134 url = f"{self.base_url}/fr/vehicles/{slug}/"135136 def _fetch(u=url):137 return self._fetch_detail(u)138139 if real_fetches >= cap: # plafond : cache seulement140 from .. import db141 if self._detail_con is None:142 self._detail_con = db.connect()143 data = db.get_cached_detail(self._detail_con, self.source_id,144 slug, cache_key)145 if data is None:146 continue147 else:148 before = self._last_request149 try:150 data = self.detail(slug, cache_key, _fetch)151 except Exception:152 # page disparue (vendu) ou erreur ponctuelle : continuer153 if self._last_request != before:154 real_fetches += 1155 continue156 if self._last_request != before:157 real_fetches += 1158 if not data or not data.get("car"):159 continue160 veh = self._to_vehicle(slug, url, data)161 if veh is not None:162 vehicles.append(veh)163 return vehicles164165 # -- payload -> Vehicle ------------------------------------------------------166 def _to_vehicle(self, slug: str, url: str, d: dict) -> Vehicle | None:167 car = d.get("car") or {}168 make = _clean(car.get("manufacturer") or car.get("brand"))169 model = _clean(car.get("model"))170 name = _clean(str(car.get("name") or "").split("\n")[0])171 if not (make or model or name):172 return None173 body = _clean(car.get("bodyType"))174 if _EXCLUDE_RE.search(f"{make} {model} {body} {name}"):175 return None176177 year = None178 m = re.search(r"(19|20)\d{2}", str(car.get("vehicleModelDate")179 or car.get("modelDate") or ""))180 if m:181 year = int(m.group(0))182183 # « 2018 - Audi A3 » -> titre standard184 title = re.sub(r"^(?:19|20)\d{2}\s*-\s*", "", name)185 title = " ".join(x for x in (title or f"{make} {model}",186 str(year or "")) if x)187188 price = _num(d.get("price"))189 km = _num(car.get("mileageFromOdometer"))190191 stock = ""192 m = _STOCK_RE.search(slug)193 if m:194 stock = m.group(1).upper()195196 engine = ""197 eng = car.get("vehicleEngine")198 if isinstance(eng, dict):199 engine = _clean(eng.get("name"))200201 description = _clean(car.get("description"))[:4000]202 # indice carburant : le NOM seulement (la description parle de203 # « sièges électriques » etc. -> faux positifs)204 fuel = ""205 m = _FUEL_HINT_RE.search(name)206 if m:207 fuel = m.group(1)208209 images = d.get("images") or []210 ld_img = car.get("image")211 if not images and isinstance(ld_img, str):212 images = [ld_img]213214 return Vehicle(215 source=self.source_id,216 external_id=slug,217 url=url,218 title=title,219 make=make,220 model=model,221 year=year,222 price=price,223 price_label=f"{price:,.0f} $".replace(",", " ") if price else "",224 mileage_km=km,225 mileage_label=f"{km:,.0f} km".replace(",", " ") if km else "",226 transmission=_clean(car.get("vehicleTransmission")),227 fuel=fuel,228 drivetrain=_clean(car.get("driveWheelConfiguration")),229 body_type=body,230 exterior_color=_clean(car.get("color")),231 engine=engine,232 stock_number=stock,233 dealer_name=self.dealer_name,234 city=self.city,235 description=description,236 images=images,237 )238