# ----------------------------------------------------------------------------- # Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/yannicklaberge.py : Automobiles Yannick Laberge (Beauport) — # WordPress sur la plateforme AMVOQ (thème « theme_central », inventaire # autousagee.ca, images image.autousagee.ca). API REST verrouillée (401). # # Stratégie : le sitemap Yoast `/vehicules-sitemap.xml` recense les annonces # (/vehicule-usage///). Chaque page détail expose un JSON-LD # schema.org @type Vehicle : prix, VIN, km, transmission, carburant, # couleur intérieure, description. Galerie complétée par les URLs # image.autousagee.ca de la page. Pages détail en cache BD (clé hebdo). # ----------------------------------------------------------------------------- from __future__ import annotations import datetime import json import os import re from ..schema import Vehicle from .base import BaseConnector _VEH_URL_RE = re.compile(r"/vehicule-usage/[^/]+/(\d+)/?$") _LD_RE = re.compile( r'', re.S) _IMG_RE = re.compile(r'https://image\.autousagee\.ca/[^"\'\\\s)]+\.jpe?g') class YannickLaberge(BaseConnector): source_id = "automobilesyannicklaberge" base_url = "https://www.automobilesyannicklaberge.com" dealer_name = "Automobiles Yannick Laberge" city = "Beauport" request_delay: float = 1.0 max_details = 200 # -- sitemap ---------------------------------------------------------------- def _vehicle_urls(self) -> list[str]: xml = self.get(f"{self.base_url}/vehicules-sitemap.xml").text return [loc.strip() for loc in re.findall(r"([^<]+)", xml) if _VEH_URL_RE.search(loc.strip())] # -- page détail -> JSON-LD Vehicle + galerie --------------------------------- def _fetch_detail(self, url: str) -> dict: html = self.get(url).text for block in _LD_RE.findall(html): try: data = json.loads(block.strip()) except ValueError: continue nodes = data if isinstance(data, list) else [data] for node in nodes: types = node.get("@type") types = types if isinstance(types, list) else [types] if "Vehicle" in types or "Car" in types: seen: set[str] = set() node["_gallery"] = [ u for u in _IMG_RE.findall(html) if not (u in seen or seen.add(u))][:20] return node return {} # -- contrat ----------------------------------------------------------------- def fetch(self) -> list[Vehicle]: urls = self._vehicle_urls() week = datetime.date.today().isocalendar() cache_key = f"v1:{week.year}w{week.week}" cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details)) vehicles: list[Vehicle] = [] real_fetches = 0 for url in urls: ext_id = _VEH_URL_RE.search(url).group(1) def _fetch(u=url): return self._fetch_detail(u) if real_fetches >= cap: # plafond : cache seulement from .. import db if self._detail_con is None: self._detail_con = db.connect() data = db.get_cached_detail(self._detail_con, self.source_id, ext_id, cache_key) if data is None: continue else: before = self._last_request data = self.detail(ext_id, cache_key, _fetch) if self._last_request != before: real_fetches += 1 if not data: continue veh = self._to_vehicle(ext_id, url, data) if veh is not None: vehicles.append(veh) return vehicles def _to_vehicle(self, ext_id: str, url: str, d: dict) -> Vehicle | None: title = " ".join(str(d.get("name") or "").split()) if not title: return None offers = d.get("offers") or {} price = None try: p = float(offers.get("price")) price = p if p > 0 else None except (TypeError, ValueError): pass odo = d.get("mileageFromOdometer") or {} km = odo.get("value") if isinstance(odo, dict) else odo try: km = float(km) km = km if km > 0 else None except (TypeError, ValueError): km = None brand = d.get("brand") or {} make = brand.get("name") if isinstance(brand, dict) else str(brand) year = None try: year = int(str(d.get("productionDate"))[:4]) except (TypeError, ValueError): pass try: doors = int(str(d.get("numberOfDoors"))) except (TypeError, ValueError): doors = None images = d.get("_gallery") or [] if not images and isinstance(d.get("image"), str): images = [d["image"]] return Vehicle( source=self.source_id, external_id=ext_id, url=url, title=title, make=str(make or ""), year=year, price=price, price_label=(f"{price:,.0f} $".replace(",", " ") if price else ""), mileage_km=km, mileage_label=(f"{km:,.0f} km".replace(",", " ") if km else ""), transmission=str(d.get("vehicleTransmission") or ""), fuel=str(d.get("fuelType") or ""), interior_color=str(d.get("vehicleInteriorColor") or ""), doors=doors, vin=str(d.get("vehicleIdentificationNumber") or ""), dealer_name=self.dealer_name, city=self.city, description=" ".join(str(d.get("description") or "").split())[:4000], images=images, )