spb/auto-ka Public
Python 81.8%
TypeScript 12.4%
CSS 5.5%
1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/ototr.py : OTO Trois-Rivières — WordPress, thème « Motors »5# (stm_vehicles_listing). Pas d'API REST exposée ni de JSON-LD.6#7# Stratégie : le sitemap natif WP `/wp-sitemap-posts-listings-1.xml` recense8# les annonces (/listings/<slug>/). Chaque page détail est parsée en HTML9# (BeautifulSoup) : h1 = titre, .single-regular-price = prix, tableau10# .single-car-data = fiche technique (Odomètre, Carburant, Transmission,11# Propulsion, Carrosserie, Moteur, Couleurs…). Pages détail en cache BD12# (clé hebdomadaire).13# -----------------------------------------------------------------------------14from __future__ import annotations1516import datetime17import os18import re1920from bs4 import BeautifulSoup2122from ..schema import Vehicle23from .base import BaseConnector2425_EXCLUDE_RE = re.compile(26 r"motoneige|vtt|bateau|ponton|motomarine|\bmoto\b|roulotte|remorque|"27 r"campeur|tracteur|souffleuse|scooter|spyder|atv", re.I)2829# vignettes WordPress (-350x205 etc.) à écarter de la galerie30_THUMB_RE = re.compile(r"-\d{2,4}x\d{2,4}\.(?:jpe?g|png|webp)$", re.I)3132# libellés de la fiche technique -> champ Vehicle33_SPEC_FIELDS = {34 "carrosserie": "body_type",35 "carburant": "fuel",36 "transmission": "transmission",37 "propulsion": "drivetrain",38 "moteur": "engine",39 "couleur exterieure": "exterior_color",40 "couleur interieure": "interior_color",41}424344def _flat(text: str) -> str:45 from ..schema import strip_accents46 return strip_accents(" ".join(text.split())).strip().lower()474849class OtoTroisRivieres(BaseConnector):50 source_id = "ototroisrivieres"51 base_url = "https://ototroisrivieres.com"52 dealer_name = "OTO Trois-Rivières"53 city = "Trois-Rivières"54 request_delay: float = 1.055 max_details = 2005657 # -- sitemap ----------------------------------------------------------------58 def _vehicle_urls(self) -> list[str]:59 xml = self.get(f"{self.base_url}/wp-sitemap-posts-listings-1.xml").text60 return [loc.strip() for loc in re.findall(r"<loc>([^<]+)</loc>", xml)61 if "/listings/" in loc]6263 # -- page détail -> payload dict ---------------------------------------------64 def _fetch_detail(self, url: str) -> dict:65 soup = BeautifulSoup(self.get(url).text, "html.parser")66 payload: dict = {"specs": {}}6768 h1 = soup.find("h1")69 if h1:70 payload["title"] = " ".join(h1.get_text(" ", strip=True).split())7172 price_el = soup.select_one(".single-regular-price")73 if price_el:74 payload["price_label"] = price_el.get_text(" ", strip=True)7576 for row in soup.select(".single-car-data tr"):77 label = row.select_one(".t-label")78 value = row.select_one(".t-value")79 if label and value:80 payload["specs"][label.get_text(" ", strip=True)] = \81 value.get_text(" ", strip=True)8283 og_desc = soup.find("meta", attrs={"property": "og:description"})84 if og_desc and og_desc.get("content"):85 payload["description"] = og_desc["content"]8687 seen: set[str] = set()88 images = []89 candidates = [img.get("src") or img.get("data-src") or ""90 for img in soup.find_all("img")]91 candidates += [a.get("href") or "" for a in soup.find_all("a")]92 for src in candidates:93 if "/wp-content/uploads/" not in src or _THUMB_RE.search(src) \94 or not re.search(r"\.(?:jpe?g|png|webp)$", src, re.I):95 continue96 if src not in seen:97 seen.add(src)98 images.append(src)99 payload["images"] = images[:20]100 return payload101102 # -- contrat -----------------------------------------------------------------103 def fetch(self) -> list[Vehicle]:104 urls = self._vehicle_urls()105 week = datetime.date.today().isocalendar()106 cache_key = f"v1:{week.year}w{week.week}"107 cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))108109 vehicles: list[Vehicle] = []110 real_fetches = 0111 for url in urls:112 ext_id = url.rstrip("/").rsplit("/", 1)[-1]113114 def _fetch(u=url):115 return self._fetch_detail(u)116117 if real_fetches >= cap: # plafond : cache seulement118 from .. import db119 if self._detail_con is None:120 self._detail_con = db.connect()121 data = db.get_cached_detail(self._detail_con, self.source_id,122 ext_id, cache_key)123 if data is None:124 continue125 else:126 before = self._last_request127 data = self.detail(ext_id, cache_key, _fetch)128 if self._last_request != before:129 real_fetches += 1130 if not data or not data.get("title"):131 continue132 veh = self._to_vehicle(ext_id, url, data)133 if veh is not None:134 vehicles.append(veh)135 return vehicles136137 def _to_vehicle(self, ext_id: str, url: str, d: dict) -> Vehicle | None:138 title = d.get("title", "")139 if not title or _EXCLUDE_RE.search(title):140 return None141 if re.search(r"vendu", title, re.I): # annonce conservée mais vendue142 return None143 title = re.sub(r"\*+\s*", " ", title).strip()144145 fields = {"body_type": "", "fuel": "", "transmission": "",146 "drivetrain": "", "engine": "", "exterior_color": "",147 "interior_color": ""}148 mileage_label = ""149 details: dict = {}150 for label, value in (d.get("specs") or {}).items():151 key = _flat(label)152 if key.startswith("odom"):153 # le thème affiche « 10500 KMkm » — garder tel quel,154 # parse_mileage s'en accommode155 mileage_label = value156 elif key in _SPEC_FIELDS:157 fields[_SPEC_FIELDS[key]] = value158 else:159 details[label] = value160161 return Vehicle(162 source=self.source_id,163 external_id=ext_id,164 url=url,165 title=title.title() if title.isupper() else title,166 price_label=d.get("price_label", ""),167 mileage_label=mileage_label,168 dealer_name=self.dealer_name,169 city=self.city,170 description=" ".join(str(d.get("description") or "").split())[:4000],171 details=details,172 images=d.get("images") or [],173 **fields,174 )175