# ----------------------------------------------------------------------------- # Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/ototr.py : OTO Trois-Rivières — WordPress, thème « Motors » # (stm_vehicles_listing). Pas d'API REST exposée ni de JSON-LD. # # Stratégie : le sitemap natif WP `/wp-sitemap-posts-listings-1.xml` recense # les annonces (/listings//). Chaque page détail est parsée en HTML # (BeautifulSoup) : h1 = titre, .single-regular-price = prix, tableau # .single-car-data = fiche technique (Odomètre, Carburant, Transmission, # Propulsion, Carrosserie, Moteur, Couleurs…). Pages détail en cache BD # (clé hebdomadaire). # ----------------------------------------------------------------------------- from __future__ import annotations import datetime import os import re from bs4 import BeautifulSoup from ..schema import Vehicle from .base import BaseConnector _EXCLUDE_RE = re.compile( r"motoneige|vtt|bateau|ponton|motomarine|\bmoto\b|roulotte|remorque|" r"campeur|tracteur|souffleuse|scooter|spyder|atv", re.I) # vignettes WordPress (-350x205 etc.) à écarter de la galerie _THUMB_RE = re.compile(r"-\d{2,4}x\d{2,4}\.(?:jpe?g|png|webp)$", re.I) # libellés de la fiche technique -> champ Vehicle _SPEC_FIELDS = { "carrosserie": "body_type", "carburant": "fuel", "transmission": "transmission", "propulsion": "drivetrain", "moteur": "engine", "couleur exterieure": "exterior_color", "couleur interieure": "interior_color", } def _flat(text: str) -> str: from ..schema import strip_accents return strip_accents(" ".join(text.split())).strip().lower() class OtoTroisRivieres(BaseConnector): source_id = "ototroisrivieres" base_url = "https://ototroisrivieres.com" dealer_name = "OTO Trois-Rivières" city = "Trois-Rivières" request_delay: float = 1.0 max_details = 200 # -- sitemap ---------------------------------------------------------------- def _vehicle_urls(self) -> list[str]: xml = self.get(f"{self.base_url}/wp-sitemap-posts-listings-1.xml").text return [loc.strip() for loc in re.findall(r"([^<]+)", xml) if "/listings/" in loc] # -- page détail -> payload dict --------------------------------------------- def _fetch_detail(self, url: str) -> dict: soup = BeautifulSoup(self.get(url).text, "html.parser") payload: dict = {"specs": {}} h1 = soup.find("h1") if h1: payload["title"] = " ".join(h1.get_text(" ", strip=True).split()) price_el = soup.select_one(".single-regular-price") if price_el: payload["price_label"] = price_el.get_text(" ", strip=True) for row in soup.select(".single-car-data tr"): label = row.select_one(".t-label") value = row.select_one(".t-value") if label and value: payload["specs"][label.get_text(" ", strip=True)] = \ value.get_text(" ", strip=True) og_desc = soup.find("meta", attrs={"property": "og:description"}) if og_desc and og_desc.get("content"): payload["description"] = og_desc["content"] seen: set[str] = set() images = [] candidates = [img.get("src") or img.get("data-src") or "" for img in soup.find_all("img")] candidates += [a.get("href") or "" for a in soup.find_all("a")] for src in candidates: if "/wp-content/uploads/" not in src or _THUMB_RE.search(src) \ or not re.search(r"\.(?:jpe?g|png|webp)$", src, re.I): continue if src not in seen: seen.add(src) images.append(src) payload["images"] = images[:20] return payload # -- contrat ----------------------------------------------------------------- def fetch(self) -> list[Vehicle]: urls = self._vehicle_urls() week = datetime.date.today().isocalendar() cache_key = f"v1:{week.year}w{week.week}" cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details)) vehicles: list[Vehicle] = [] real_fetches = 0 for url in urls: ext_id = url.rstrip("/").rsplit("/", 1)[-1] def _fetch(u=url): return self._fetch_detail(u) if real_fetches >= cap: # plafond : cache seulement from .. import db if self._detail_con is None: self._detail_con = db.connect() data = db.get_cached_detail(self._detail_con, self.source_id, ext_id, cache_key) if data is None: continue else: before = self._last_request data = self.detail(ext_id, cache_key, _fetch) if self._last_request != before: real_fetches += 1 if not data or not data.get("title"): continue veh = self._to_vehicle(ext_id, url, data) if veh is not None: vehicles.append(veh) return vehicles def _to_vehicle(self, ext_id: str, url: str, d: dict) -> Vehicle | None: title = d.get("title", "") if not title or _EXCLUDE_RE.search(title): return None if re.search(r"vendu", title, re.I): # annonce conservée mais vendue return None title = re.sub(r"\*+\s*", " ", title).strip() fields = {"body_type": "", "fuel": "", "transmission": "", "drivetrain": "", "engine": "", "exterior_color": "", "interior_color": ""} mileage_label = "" details: dict = {} for label, value in (d.get("specs") or {}).items(): key = _flat(label) if key.startswith("odom"): # le thème affiche « 10500 KMkm » — garder tel quel, # parse_mileage s'en accommode mileage_label = value elif key in _SPEC_FIELDS: fields[_SPEC_FIELDS[key]] = value else: details[label] = value return Vehicle( source=self.source_id, external_id=ext_id, url=url, title=title.title() if title.isupper() else title, price_label=d.get("price_label", ""), mileage_label=mileage_label, dealer_name=self.dealer_name, city=self.city, description=" ".join(str(d.get("description") or "").split())[:4000], details=details, images=d.get("images") or [], **fields, )