spb/ora-ka Public

1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/occasioncharlevoix.py : connecteur Occasion Charlevoix5#   (occasioncharlevoix.com), regroupement de concessionnaires d'occasion de6#   Baie-Saint-Paul (Charlevoix, Capitale-Nationale) — ~30 véhicules.7#8#   Plateforme : OctoberCMS, thème « nerd2 » + plugins nerd/inventory9#   (plateforme québécoise NerdAuto). La page liste /fr/inventaire/occasion10#   est ENTIÈREMENT rendue côté serveur : chaque carte (div.vrp4-card)11#   contient marque, modèle, version, année, no de stock, kilométrage, photos12#   (carrousel) et un bloc « Autoverify » (div.av-srp-ecomm) avec VIN et prix13#   réel (data-av-price — le prix affiché en gros est un paiement périodique).14#15#   La page détail complète (cache BD, clé statique v1) : transmission,16#   cylindrée, motricité, couleurs, catégorie, lien Carfax.17# -----------------------------------------------------------------------------18from __future__ import annotations1920import os21import re2223from ..schema import Vehicle, parse_mileage24from .base import BaseConnector2526# garde-fou non-automobile27_EXCLUDE_RE = re.compile(28    r"motoneige|vtt|moto\b|motomarine|bateau|roulotte|remorque|spyder", re.I)2930_SPEC_RE = re.compile(31    r'class="specs-name[^"]*">\s*([^<]+?)\s*</div>\s*'32    r'<div class="specs-info[^"]*">\s*([^<]*?)\s*<', re.S)3334_CARFAX_RE = re.compile(r'data-lien-carfax="([^"]+)"')3536_AV_DETAIL_RE = re.compile(37    r'id="av_vehicle_information"[^>]*data-av-mileage="([^"]*)"')383940def _txt(node) -> str:41    return re.sub(r"\s+", " ", node.get_text(" ", strip=True)) if node else ""424344class OccasionCharlevoix(BaseConnector):45    """Occasion Charlevoix — cartes liste rendues serveur (OctoberCMS nerd)."""4647    source_id = "occasioncharlevoix"48    base_url = "https://www.occasioncharlevoix.com"49    dealer_name = "Occasion Charlevoix"50    city = "Baie-Saint-Paul"51    request_delay = 1.052    max_details = 2005354    def fetch(self) -> list[Vehicle]:55        from bs4 import BeautifulSoup56        cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))57        html = self.get(f"{self.base_url}/fr/inventaire/occasion").text58        soup = BeautifulSoup(html, "html.parser")5960        vehicles: list[Vehicle] = []61        seen: set[str] = set()62        real_fetches = 063        for card in soup.select("div.vrp4-card"):64            try:65                veh = self._card_to_vehicle(card)66            except Exception:67                continue                     # carte malformée isolée68            if veh is None or veh.external_id in seen:69                continue70            seen.add(veh.external_id)7172            # enrichissement page détail (transmission, couleurs, carfax…)73            if real_fetches < cap:74                url = veh.url7576                def _fetch(u=url):77                    return self._fetch_detail(u)7879                try:80                    extra = self.detail(veh.external_id, "v1", _fetch)81                    real_fetches += 182                    self._apply_detail(veh, extra)83                except Exception:84                    real_fetches += 1        # ne jamais bloquer la source85            vehicles.append(veh)86        return vehicles8788    # -- carte liste -> Vehicle -------------------------------------------------89    def _card_to_vehicle(self, card) -> Vehicle | None:90        link = card.select_one("a.vrp4-card-header[href]")91        if link is None:92            return None93        url = link["href"]94        if "/inventaire/vehicule/" not in url:95            return None96        if url.startswith("/"):97            url = self.base_url + url9899        # /fr/inventaire/vehicule/occasion/<stock>/<marque>/<modèle>/…100        m = re.search(r"/vehicule/[^/]+/([^/]+)/", url)101        ext_id = m.group(1) if m else url.rstrip("/").rsplit("/", 1)[-1]102103        make = _txt(card.select_one(".car-make"))104        model = _txt(card.select_one(".car-model"))105        trim = _txt(card.select_one(".car-trim"))106        if _EXCLUDE_RE.search(f"{make} {model} {trim}"):107            return None108109        year_txt = _txt(card.select_one(".car-year"))110        year = int(year_txt) if year_txt.isdigit() else None111112        stock = _txt(card.select_one(".car-stock")).lstrip("#")113        odo_label = _txt(card.select_one(".car-odometer"))114        km = parse_mileage(odo_label)115116        # bloc Autoverify : VIN + prix réel (le prix visible est un paiement)117        vin, price = "", None118        av = card.select_one(".av-srp-ecomm")119        if av is not None:120            vin = (av.get("data-av-vin") or "").strip().upper()121            try:122                price = float(av.get("data-av-price") or 0) or None123            except ValueError:124                price = None125            # garde-fou : < 1 500 $ = paiement périodique saisi comme prix126            if price is not None and price < 1500:127                price = None128129        images = []130        for img in card.select("img.car-img[src]"):131            src = img["src"]132            if src.startswith("/"):133                src = self.base_url + src134            if src not in images:135                images.append(src)136137        title = " ".join(x for x in (make, model, trim, year_txt) if x)138        if not title:139            return None140141        return Vehicle(142            source=self.source_id,143            external_id=ext_id,144            url=url,145            title=title,146            make=make,147            model=model,148            trim=trim,149            year=year,150            price=price,151            price_label=f"{price:,.0f} $".replace(",", " ") if price else "",152            mileage_km=km,153            mileage_label=odo_label,154            vin=vin,155            stock_number=stock or ext_id.upper(),156            dealer_name=self.dealer_name,157            city=self.city,158            images=images[:20],159        )160161    # -- page détail -------------------------------------------------------------162    def _fetch_detail(self, url: str) -> dict:163        html = self.get(url).text164        payload: dict = {"specs": {}}165        for name, value in _SPEC_RE.findall(html):166            payload["specs"][name.rstrip(": ").strip()] = value.strip()167        m = _CARFAX_RE.search(html)168        if m:169            payload["carfax"] = m.group(1)170        m = _AV_DETAIL_RE.search(html)171        if m and m.group(1):172            payload["mileage"] = m.group(1)173        return payload174175    def _apply_detail(self, veh: Vehicle, extra: dict) -> None:176        specs = (extra or {}).get("specs") or {}177        for name, value in specs.items():178            low = name.lower()179            if not value or value.lower() in ("autre", "n/d", "s/o"):180                continue181            if "transmission" in low and not veh.transmission:182                veh.transmission = value183            elif "motricit" in low and not veh.drivetrain:184                veh.drivetrain = value185            elif "cat" in low and not veh.body_type:186                veh.body_type = value187            elif "cylindr" in low and not veh.engine:188                veh.engine = value189            elif "ext" in low and "couleur" in low and not veh.exterior_color:190                veh.exterior_color = value191            elif "int" in low and "couleur" in low and not veh.interior_color:192                veh.interior_color = value193        if extra.get("carfax") and not veh.carfax_url:194            veh.carfax_url = extra["carfax"]195        if veh.mileage_km is None and extra.get("mileage"):196            try:197                veh.mileage_km = float(extra["mileage"])198            except ValueError:199                pass200