spb/auto-ka Public
Python 81.9%
TypeScript 12.4%
CSS 5.5%
1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/magnetis_dealers.py : connecteurs des concessionnaires sur la5# plateforme WordPress « magnetis-dealer » (agence Magnetis, inventaire6# SyncAuto — photos syncauto-XX.s3.ca-central-1.amazonaws.com).7#8# Stratégie : le sitemap /inventory-sitemap.xml recense chaque page9# véhicule /inventaire/occasion/<année>/<marque>/<modèle>/<id> (versions FR10# et EN ; on ne garde que les FR, occasion seulement — le neuf vit sous11# /inventaire/neuf/). Chaque page détail embarque :12# - <script data-class="Vehicle"> : JSON complet (marque, modèle, année,13# version, km, prix, transmission, rouage, carburant, carrosserie,14# portes, passagers, couleur, no stock ET la concession réelle du15# véhicule avec sa ville — certains sites du réseau affichent16# l'inventaire de concessions sœurs) ;17# - des JSON-LD schema.org @type Vehicle (le bon = productID == id) :18# VIN, photos, description, vendeur.19# Pages détail en cache BD (clé hebdomadaire) ; AUTOKA_MAX_DETAILS (env)20# plafonne les vraies requêtes détail par synchronisation.21# -----------------------------------------------------------------------------22from __future__ import annotations2324import datetime25import html as htmllib26import json27import os28import re2930from ..schema import Vehicle31from .base import BaseConnector3233_VEH_PATH_RE = re.compile(34 r"/inventaire/occasion/(?:19|20)\d{2}/[^/]+/[^/]+/(\d+)/?$")3536_DATA_CLASS_RE = re.compile(37 r'<script[^>]*data-class=["\']?Vehicle["\']?[^>]*>(.*?)</script>', re.S)38_LD_RE = re.compile(39 r'<script type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', re.S)4041# garde-fou non-automobile (ces réseaux vendent parfois des VR/motoneiges)42_EXCLUDE_RE = re.compile(43 r"moto(?:cyclette|neige|marine)?\b|\bvtt\b|\batv\b|scooter|spyder|ryker|"44 r"can-?am|ski-?doo|sea-?doo|roulotte|remorque|\bvr\b|motoris|campeur|"45 r"trailer|snowmobile", re.I)464748def _clean(text) -> str:49 if not text:50 return ""51 out = htmllib.unescape(str(text))52 out = re.sub(r"<[^>]+>", " ", out)53 return re.sub(r"\s{2,}", " ", out).strip()545556def _num(value, cast=float):57 try:58 v = cast(value)59 except (TypeError, ValueError):60 return None61 return v if v else None626364class MagnetisConnector(BaseConnector):65 """Base commune aux sites WordPress « magnetis-dealer » (SyncAuto)."""6667 base_url: str = ""68 dealer_name: str = ""69 city: str = ""70 request_delay: float = 1.071 max_details = 400 # plafond de vraies requêtes par sync7273 # -- sitemap ---------------------------------------------------------------74 def _vehicle_urls(self) -> list[str]:75 xml = self.get(f"{self.base_url}/inventory-sitemap.xml").text76 urls: list[str] = []77 for loc in re.findall(r"<loc>([^<]+)</loc>", xml):78 loc = loc.strip()79 if "/en/" in loc or not loc.startswith(self.base_url):80 continue81 if _VEH_PATH_RE.search(loc):82 urls.append(loc)83 return sorted(set(urls))8485 # -- page détail -> payload ---------------------------------------------------86 def _fetch_detail(self, url: str, ext_id: str) -> dict:87 html = self.get(url).text88 payload: dict = {}8990 m = _DATA_CLASS_RE.search(html)91 if m:92 try:93 payload["vehicle"] = json.loads(m.group(1).strip())94 except ValueError:95 pass9697 # JSON-LD : la page inclut aussi des « véhicules similaires » — le98 # bon bloc est celui dont productID correspond à l'id de l'URL.99 for block in _LD_RE.findall(html):100 try:101 data = json.loads(block.strip(), strict=False)102 except ValueError:103 continue104 types = data.get("@type")105 types = types if isinstance(types, list) else [types]106 if "Vehicle" not in types and "Car" not in types:107 continue108 if str(data.get("productID") or "") == ext_id:109 payload["ld"] = data110 break111 return payload112113 # -- contrat ---------------------------------------------------------------114 def fetch(self) -> list[Vehicle]:115 urls = self._vehicle_urls()116 week = datetime.date.today().isocalendar()117 cache_key = f"v1:{week.year}w{week.week}" # revalidation hebdomadaire118 cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))119120 vehicles: list[Vehicle] = []121 real_fetches = 0122 for url in urls:123 ext_id = _VEH_PATH_RE.search(url).group(1)124125 def _fetch(u=url, e=ext_id):126 return self._fetch_detail(u, e)127128 # plafond atteint : ne servir que le cache (fetch réel interdit)129 if real_fetches >= cap:130 from .. import db131 if self._detail_con is None:132 self._detail_con = db.connect()133 data = db.get_cached_detail(self._detail_con, self.source_id,134 ext_id, cache_key)135 if data is None:136 continue137 else:138 before = self._last_request139 try:140 data = self.detail(ext_id, cache_key, _fetch)141 except Exception:142 # page disparue (vendu) ou erreur ponctuelle : ne jamais143 # bloquer la source entière144 if self._last_request != before:145 real_fetches += 1146 continue147 if self._last_request != before:148 real_fetches += 1149 if not data or not data.get("vehicle"):150 continue151 veh = self._to_vehicle(ext_id, url, data)152 if veh is not None:153 vehicles.append(veh)154 return vehicles155156 # -- payload -> Vehicle ------------------------------------------------------157 def _to_vehicle(self, ext_id: str, url: str, data: dict) -> Vehicle | None:158 v = data.get("vehicle") or {}159 ld = data.get("ld") or {}160161 if str(v.get("status") or "used").lower() != "used":162 return None # occasion seulement163164 make = _clean(v.get("make"))165 model = _clean(v.get("model"))166 body = _clean(v.get("body"))167 if not (make or model):168 return None169 if _EXCLUDE_RE.search(f"{make} {model} {body}"):170 return None171172 year = _num(v.get("year"), int)173 trim = _clean(v.get("trim"))174 title = " ".join(str(x) for x in (make, model, trim, year or "") if x)175176 price = _num(v.get("price"))177 km = _num(v.get("kms"))178179 images = ld.get("image") or []180 if isinstance(images, str):181 images = [images]182 images = [u for u in images if isinstance(u, str)183 and "img-bkg" not in u] # bannière de repli du site184185 # la concession réelle du véhicule (réseaux multi-concessions)186 dealer = _clean(v.get("dealer_name_fr") or v.get("dealer_name_en"))187 dealer = re.sub(r"\s+(?:used|occasion)$", "", dealer, flags=re.I)188 if dealer.lower() in ("", "sync"): # placeholder SyncAuto189 dealer = self.dealer_name190 city = _clean(v.get("dealer_city")).split(",")[0].strip() or self.city191192 details: dict = {}193 if v.get("msrp") and _num(v["msrp"]) != price:194 details["msrp"] = v["msrp"]195 if v.get("create_date"):196 details["listed_date"] = str(v["create_date"])197198 offers = ld.get("offers") or {}199 seller = offers.get("seller") or {}200 if isinstance(seller, dict) and seller.get("name"):201 details["branch"] = _clean(seller["name"])202203 return Vehicle(204 source=self.source_id,205 external_id=ext_id,206 url=url,207 title=title,208 make=make,209 model=model,210 trim=trim,211 year=year,212 price=price,213 price_label=f"{price:,.0f} $".replace(",", " ") if price else "",214 mileage_km=km,215 mileage_label=f"{km:,.0f} km".replace(",", " ") if km else "",216 transmission=_clean(v.get("transmission")),217 fuel=_clean(v.get("fuel_type")),218 drivetrain=_clean(v.get("drive")),219 body_type=body,220 exterior_color=_clean(v.get("exterior_color")),221 engine=_clean(v.get("engine_size")),222 doors=_num(v.get("doors"), int),223 seats=_num(v.get("passengers"), int),224 vin=_clean(ld.get("vehicleIdentificationNumber")),225 stock_number=_clean(v.get("stock_number") or ld.get("sku")),226 dealer_name=dealer,227 city=city,228 description=_clean(ld.get("description"))[:4000],229 details=details,230 images=images[:20],231 )232233234# ---------------------------------------------------------------------------235# Concessionnaires (1 sous-classe = 1 source)236# ---------------------------------------------------------------------------237238class BaieComeauToyota(MagnetisConnector):239 source_id = "baiecomeautoyota"240 base_url = "https://baiecomeautoyota.com"241 dealer_name = "Baie-Comeau Toyota"242 city = "Baie-Comeau"243244245class SherbrookeToyota(MagnetisConnector):246 source_id = "sherbrooketoyota"247 base_url = "https://sherbrooketoyota.ca"248 dealer_name = "Sherbrooke Toyota"249 city = "Sherbrooke"250251252class StGeorgesNissan(MagnetisConnector):253 source_id = "stgnissan"254 base_url = "https://www.stgnissan.com"255 dealer_name = "St-Georges Nissan"256 city = "Saint-Georges"257258259class LussierChevrolet(MagnetisConnector):260 """Lussier Chevrolet Buick GMC — remplace Auto Bas Prix St-Hyacinthe261 (autobasprixst-hyacinthe.com redirige vers lussierchevrolet.com depuis262 la fusion des deux commerces)."""263264 source_id = "lussierchevrolet"265 base_url = "https://www.lussierchevrolet.com"266 dealer_name = "Lussier Chevrolet Buick GMC"267 city = "Saint-Hyacinthe"268