Vague 2 : dédup VIN inter-sources, Kijiji particuliers, rappels TC, coordonnées concessionnaires
- dedup.py : dédoublonnage par VIN exact (17 car. valides) recalculé après
chaque cycle ; autorité concessionnaire direct > portail > marketplace
(champ type de sources.json), canonique garde dup_sources, doublons
masqués via dup_of (colonnes additives + index vin/dup_of) ; filtre
dup_of IS NULL sur /api/vehicles, facets, stats et pages SEO
- connectors/kijiji.py : annonces de PARTICULIERS kijiji.ca c174/l9001
(kijijiautos.ca n existe plus — domaine SERVFAIL) : __NEXT_DATA__/Apollo,
attributs canoniques, GPS exact, VIN/Carfax quand présents, ~3 000 annonces
- recalls.py : rappels Transports Canada (dump CSV open data mensuel,
109 186 lignes / 16 969 rappels), croisement make/model/year +
GET /api/vehicles/{uid}/recalls
- dealers.py : coordonnées des 138 sources (JSON-LD AutoDealer + replis,
1 page/site, cache en base) + GET /api/dealers
- /api/stats : vin_duplicates_masked, recalls_total, dealers_total
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
9 changed files +864 −18
added
autoka/connectors/kijiji.py
+224 −0
@@ -0,0 +1,224 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec) | |
| 3 | +# Auteur : Simon-Pierre Boucher — contact@spboucher.ai | |
| 4 | +# connectors/kijiji.py : annonces de PARTICULIERS — Kijiji (kijiji.ca) | |
| 5 | +# | |
| 6 | +# Note : Kijiji Autos (kijijiautos.ca, plateforme MoVe/m.mobile.de) n'existe | |
| 7 | +# plus — le domaine ne résout plus (SERVFAIL, constaté 2026-08-18) ; les | |
| 8 | +# annonces ont été rapatriées sur kijiji.ca. On cible donc la catégorie | |
| 9 | +# « Autos et camions » (c174) du Québec (l9001), filtrée vendeur particulier | |
| 10 | +# (?for-sale-by=ownr) pour ne pas dupliquer l'inventaire des concessionnaires | |
| 11 | +# déjà couverts par les autres connecteurs. | |
| 12 | +# | |
| 13 | +# Stratégie : les pages liste (SRP) de kijiji.ca (Next.js) embarquent le | |
| 14 | +# cache Apollo complet dans <script id="__NEXT_DATA__"> — chaque annonce y | |
| 15 | +# est un objet AutosListing structuré : marque/modèle/année/km canoniques, | |
| 16 | +# prix (en cents), carburant/boîte/rouage/carrosserie, couleurs, portes, | |
| 17 | +# places, VIN (quand le vendeur l'a saisi), lien Carfax, photos CDN et | |
| 18 | +# surtout la géolocalisation exacte (lat/lng + adresse). Aucune page détail | |
| 19 | +# n'est nécessaire : tout est dans la liste — 40 annonces/requête. | |
| 20 | +# | |
| 21 | +# Pagination : /b-autos-camions/quebec/page-N/c174l9001?for-sale-by=ownr | |
| 22 | +# jusqu'à totalCount (bornée par MAX_PAGES par politesse). | |
| 23 | +# ----------------------------------------------------------------------------- | |
| 24 | +from __future__ import annotations | |
| 25 | + | |
| 26 | +import json | |
| 27 | +import re | |
| 28 | + | |
| 29 | +from ..schema import Vehicle | |
| 30 | +from .base import BaseConnector | |
| 31 | + | |
| 32 | +BASE = "https://www.kijiji.ca" | |
| 33 | +LIST_PATH = "/b-autos-camions/quebec/{page}c174l9001?for-sale-by=ownr" | |
| 34 | + | |
| 35 | +_NEXT_DATA_RE = re.compile( | |
| 36 | + r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', re.S) | |
| 37 | + | |
| 38 | +# suffixes d'adresse à écarter pour isoler la ville : « QC », code postal | |
| 39 | +# complet ou partiel (« J7V »), ou les deux (« QC H7X 2S6 ») | |
| 40 | +_ADDR_TAIL_RE = re.compile( | |
| 41 | + r"^(?:QC|Qc|Qu[ée]bec)?\s*(?:[A-Za-z]\d[A-Za-z](?:\s?\d[A-Za-z]\d)?)?$") | |
| 42 | + | |
| 43 | +# grandes photos plutôt que les vignettes 200 px de la liste | |
| 44 | +_IMG_RULE_RE = re.compile(r"rule=kijijica-\d+-") | |
| 45 | + | |
| 46 | +# valeurs canoniques Kijiji -> vocabulaire Auto-Ka (normalize.py gère le reste) | |
| 47 | +_TRANSMISSIONS = {"1": "Manuelle", "2": "Automatique", "3": "", | |
| 48 | + "auto": "Automatique", "man": "Manuelle"} | |
| 49 | +_BODIES = {"sedan": "Berline", "suvcrossover": "VUS", "htchbck": "Hayon", | |
| 50 | + "conv": "Cabriolet", "coup": "Coupé", "pickuptruck": "Camionnette", | |
| 51 | + "vanminicomma": "Fourgonnette", "wagon": "Familiale", | |
| 52 | + "othrbdytyp": ""} | |
| 53 | +_COLORS = {"white": "Blanc", "black": "Noir", "gray": "Gris", "grey": "Gris", | |
| 54 | + "silver": "Argent", "blue": "Bleu", "red": "Rouge", "brown": "Brun", | |
| 55 | + "green": "Vert", "burgundy": "Bourgogne", "gold": "Doré", | |
| 56 | + "orange": "Orange", "off_white": "Blanc cassé", "beige": "Beige", | |
| 57 | + "tan": "Beige", "yellow": "Jaune", "purple": "Violet", | |
| 58 | + "other": ""} | |
| 59 | + | |
| 60 | +MAX_PAGES = 90 # 90 × 40 = 3 600 annonces — couvre le volume QC actuel | |
| 61 | +PAGE_SIZE = 40 | |
| 62 | + | |
| 63 | + | |
| 64 | +def _attr_map(listing: dict) -> dict[str, str]: | |
| 65 | + out: dict[str, str] = {} | |
| 66 | + for a in ((listing.get("attributes") or {}).get("all") or []): | |
| 67 | + vals = a.get("canonicalValues") or [] | |
| 68 | + if vals and vals[0] is not None: | |
| 69 | + out[a.get("canonicalName") or ""] = str(vals[0]) | |
| 70 | + return out | |
| 71 | + | |
| 72 | + | |
| 73 | +def _city_from_location(loc: dict) -> str: | |
| 74 | + """Ville depuis l'adresse — formats observés : « Rue X, Laval, QC H7X 2S6 », | |
| 75 | + « Vaudreuil-Dorion, QC J7V », « Anjou, QC H1J 2W1 », « Laval, H7Y 2B7 ».""" | |
| 76 | + parts = [p.strip() for p in (loc.get("address") or "").split(",") if p.strip()] | |
| 77 | + # retirer depuis la fin : « QC », code postal (complet/partiel) ou les deux | |
| 78 | + while parts and _ADDR_TAIL_RE.match(parts[-1]): | |
| 79 | + parts.pop() | |
| 80 | + if parts: | |
| 81 | + return parts[-1] | |
| 82 | + # repli : nom de zone Kijiji (« Laval / North Shore » -> Laval) | |
| 83 | + return ((loc.get("name") or "").split("/")[0]).strip() | |
| 84 | + | |
| 85 | + | |
| 86 | +class KijijiParticuliers(BaseConnector): | |
| 87 | + """Annonces de particuliers — Kijiji, catégorie Autos et camions, Québec.""" | |
| 88 | + | |
| 89 | + source_id = "kijiji" | |
| 90 | + request_delay = 1.2 # politesse : gros site, gros volume | |
| 91 | + | |
| 92 | + def __init__(self) -> None: | |
| 93 | + super().__init__() | |
| 94 | + self.session.headers.update({ | |
| 95 | + "Accept": "text/html,application/xhtml+xml", | |
| 96 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.5", | |
| 97 | + }) | |
| 98 | + | |
| 99 | + # -- extraction ----------------------------------------------------------- | |
| 100 | + | |
| 101 | + def _fetch_page(self, page: int) -> tuple[list[dict], int]: | |
| 102 | + """Annonces AutosListing + totalCount d'une page SRP.""" | |
| 103 | + seg = "" if page <= 1 else f"page-{page}/" | |
| 104 | + html = self.get(BASE + LIST_PATH.format(page=seg)).text | |
| 105 | + m = _NEXT_DATA_RE.search(html) | |
| 106 | + if not m: | |
| 107 | + raise RuntimeError(f"kijiji : __NEXT_DATA__ introuvable (page {page})") | |
| 108 | + data = json.loads(m.group(1)) | |
| 109 | + apollo = (data.get("props", {}).get("pageProps", {}) | |
| 110 | + .get("__APOLLO_STATE__") or {}) | |
| 111 | + total = 0 | |
| 112 | + for key, val in (apollo.get("ROOT_QUERY") or {}).items(): | |
| 113 | + if key.startswith("searchResultsPageByUrl"): | |
| 114 | + total = int((val.get("pagination") or {}).get("totalCount") or 0) | |
| 115 | + break | |
| 116 | + listings = [v for k, v in apollo.items() | |
| 117 | + if k.startswith("AutosListing:") and isinstance(v, dict)] | |
| 118 | + return listings, total | |
| 119 | + | |
| 120 | + def _to_vehicle(self, l: dict) -> Vehicle | None: | |
| 121 | + ext_id = str(l.get("id") or "") | |
| 122 | + if not ext_id: | |
| 123 | + return None | |
| 124 | + attrs = _attr_map(l) | |
| 125 | + if attrs.get("forsaleby") not in ("", "ownr"): | |
| 126 | + return None # topListings = pubs de marchands | |
| 127 | + if attrs.get("vehicletype") == "new": | |
| 128 | + return None # occasion seulement | |
| 129 | + | |
| 130 | + price = None | |
| 131 | + p = l.get("price") or {} | |
| 132 | + if p.get("type") == "FIXED" and p.get("amount"): | |
| 133 | + price = round(p["amount"] / 100.0, 2) | |
| 134 | + if price < 500: # « 1 $ » = prix symbolique de petite annonce | |
| 135 | + price = None | |
| 136 | + | |
| 137 | + loc = l.get("location") or {} | |
| 138 | + coords = loc.get("coordinates") or {} | |
| 139 | + images = [_IMG_RULE_RE.sub("rule=kijijica-640-", u) | |
| 140 | + for u in (l.get("imageUrls") or [])] | |
| 141 | + | |
| 142 | + km = None | |
| 143 | + if attrs.get("carmileageinkms", "").replace(".", "", 1).isdigit(): | |
| 144 | + km = float(attrs["carmileageinkms"]) | |
| 145 | + | |
| 146 | + def _int(name: str) -> int | None: | |
| 147 | + v = attrs.get(name, "") | |
| 148 | + return int(v) if v.isdigit() else None | |
| 149 | + | |
| 150 | + details = {"forsaleby": "particulier"} | |
| 151 | + if l.get("activationDate"): | |
| 152 | + details["posted"] = l["activationDate"][:10] | |
| 153 | + if attrs.get("pricerating"): | |
| 154 | + details["kijiji_price_rating"] = attrs["pricerating"] | |
| 155 | + if attrs.get("electricrange", "").replace(".", "", 1).isdigit(): | |
| 156 | + details["electric_range_km"] = float(attrs["electricrange"]) | |
| 157 | + | |
| 158 | + vin = attrs.get("vin", "").strip().upper() | |
| 159 | + if not re.fullmatch(r"[A-HJ-NPR-Z0-9]{17}", vin): | |
| 160 | + vin = "" | |
| 161 | + | |
| 162 | + veh = Vehicle( | |
| 163 | + source=self.source_id, | |
| 164 | + external_id=ext_id, | |
| 165 | + url=l.get("url") or f"{BASE}/v-view-details.html?adId={ext_id}", | |
| 166 | + kind="auto", | |
| 167 | + title=l.get("title") or "", | |
| 168 | + make=attrs.get("carmake", ""), | |
| 169 | + model=attrs.get("carmodel", "").capitalize(), | |
| 170 | + trim=attrs.get("cartrim", ""), | |
| 171 | + year=_int("caryear"), | |
| 172 | + price=price, | |
| 173 | + price_label=(f"{price:,.0f} $".replace(",", " ") if price else ""), | |
| 174 | + mileage_km=km, | |
| 175 | + transmission=_TRANSMISSIONS.get(attrs.get("cartransmission", ""), | |
| 176 | + attrs.get("cartransmission", "")), | |
| 177 | + fuel=("" if attrs.get("carfueltype") == "other" | |
| 178 | + else attrs.get("carfueltype", "")), | |
| 179 | + drivetrain=("" if attrs.get("drivetrain") == "other" | |
| 180 | + else attrs.get("drivetrain", "")), | |
| 181 | + body_type=_BODIES.get(attrs.get("carbodytype", ""), | |
| 182 | + attrs.get("carbodytype", "")), | |
| 183 | + exterior_color=_COLORS.get(attrs.get("carcolor", ""), | |
| 184 | + attrs.get("carcolor", "").capitalize()), | |
| 185 | + interior_color=_COLORS.get(attrs.get("carinteriorcolor", ""), | |
| 186 | + attrs.get("carinteriorcolor", "").capitalize()), | |
| 187 | + doors=_int("noofdoors"), | |
| 188 | + seats=_int("noofseats"), | |
| 189 | + vin=vin, | |
| 190 | + dealer_name="Particulier (Kijiji)", | |
| 191 | + city=_city_from_location(loc), | |
| 192 | + lat=coords.get("latitude"), | |
| 193 | + lng=coords.get("longitude"), | |
| 194 | + description=l.get("description") or "", | |
| 195 | + details=details, | |
| 196 | + images=images, | |
| 197 | + carfax_url=attrs.get("carprooflink", ""), | |
| 198 | + ) | |
| 199 | + return veh | |
| 200 | + | |
| 201 | + # -- contrat --------------------------------------------------------------- | |
| 202 | + | |
| 203 | + def fetch(self) -> list[Vehicle]: | |
| 204 | + vehicles: dict[str, Vehicle] = {} | |
| 205 | + listings, total = self._fetch_page(1) | |
| 206 | + pages = min(MAX_PAGES, -(-max(total, 1) // PAGE_SIZE)) | |
| 207 | + for l in listings: | |
| 208 | + v = self._to_vehicle(l) | |
| 209 | + if v: | |
| 210 | + vehicles[v.external_id] = v | |
| 211 | + for page in range(2, pages + 1): | |
| 212 | + try: | |
| 213 | + listings, _ = self._fetch_page(page) | |
| 214 | + except Exception: | |
| 215 | + break # fin de pagination / page vide | |
| 216 | + new = 0 | |
| 217 | + for l in listings: | |
| 218 | + v = self._to_vehicle(l) | |
| 219 | + if v and v.external_id not in vehicles: | |
| 220 | + vehicles[v.external_id] = v | |
| 221 | + new += 1 | |
| 222 | + if new == 0: # au-delà de la dernière page | |
| 223 | + break | |
| 224 | + return list(vehicles.values()) | |
modified
autoka/db.py
+8 −0
@@ -115,6 +115,11 @@ _MIGRATIONS = { | ||
| 115 | 115 | # permet la recherche par rayon autour d'un point |
| 116 | 116 | "lat": "REAL", |
| 117 | 117 | "lng": "REAL", |
| 118 | + # dédoublonnage inter-sources par VIN (autoka/dedup.py) : | |
| 119 | + # dup_of = uid de l'annonce canonique (NULL = canonique ou unique) ; | |
| 120 | + # dup_sources = JSON des autres offres du même VIN (sur la canonique) | |
| 121 | + "dup_of": "TEXT", | |
| 122 | + "dup_sources": "TEXT", | |
| 118 | 123 | }, |
| 119 | 124 | } |
| 120 | 125 | |
@@ -136,6 +141,9 @@ def connect() -> sqlite3.Connection: | ||
| 136 | 141 | con.execute("CREATE INDEX IF NOT EXISTS idx_vehicles_kind ON vehicles(kind)") |
| 137 | 142 | con.execute( |
| 138 | 143 | "CREATE INDEX IF NOT EXISTS idx_vehicles_latlng ON vehicles(lat, lng)") |
| 144 | + con.execute("CREATE INDEX IF NOT EXISTS idx_vehicles_vin ON vehicles(vin)") | |
| 145 | + con.execute( | |
| 146 | + "CREATE INDEX IF NOT EXISTS idx_vehicles_dup_of ON vehicles(dup_of)") | |
| 139 | 147 | con.commit() |
| 140 | 148 | return con |
| 141 | 149 | |
added
autoka/dealers.py
+212 −0
@@ -0,0 +1,212 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec) | |
| 3 | +# Auteur : Simon-Pierre Boucher — contact@spboucher.ai | |
| 4 | +# dealers.py : coordonnées des concessionnaires (adresse, téléphone, GPS) | |
| 5 | +# | |
| 6 | +# 1 source = 1 site de concessionnaire (sauf portails/marketplaces) : on | |
| 7 | +# visite UNE page par site (l'accueil suffit — l'adresse et le téléphone | |
| 8 | +# sont dans l'en-tête/le pied de page, et la plupart des plateformes (D2C, | |
| 9 | +# SM360, Convertus…) émettent un JSON-LD AutoDealer/LocalBusiness complet). | |
| 10 | +# Extraction : JSON-LD d'abord (address/telephone/geo), replis regex | |
| 11 | +# (lien tel:, adresse avec code postal canadien). GPS par défaut : centre | |
| 12 | +# de la ville de la source (data/villes_gps.json). | |
| 13 | +# | |
| 14 | +# Budget : ~138 requêtes UNE fois (cache en base — on ne re-visite que les | |
| 15 | +# sources absentes de la table), throttle 1 s. | |
| 16 | +# | |
| 17 | +# One-shot : python3 -m autoka.dealers [--refresh] | |
| 18 | +# ----------------------------------------------------------------------------- | |
| 19 | +from __future__ import annotations | |
| 20 | + | |
| 21 | +import json | |
| 22 | +import re | |
| 23 | +import sqlite3 | |
| 24 | +import time | |
| 25 | +from pathlib import Path | |
| 26 | + | |
| 27 | +import requests | |
| 28 | + | |
| 29 | +from .normalize import city_gps | |
| 30 | + | |
| 31 | +SOURCES_PATH = Path(__file__).resolve().parent.parent / "data" / "sources.json" | |
| 32 | + | |
| 33 | +USER_AGENT = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " | |
| 34 | + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36 " | |
| 35 | + "AutoKaBot/1.0 (+contact@spboucher.ai)") | |
| 36 | + | |
| 37 | +_SCHEMA = """ | |
| 38 | +CREATE TABLE IF NOT EXISTS dealers ( | |
| 39 | + source TEXT PRIMARY KEY, -- id de la source (data/sources.json) | |
| 40 | + name TEXT, | |
| 41 | + address TEXT, | |
| 42 | + phone TEXT, | |
| 43 | + website TEXT, | |
| 44 | + city TEXT, | |
| 45 | + lat REAL, | |
| 46 | + lng REAL, | |
| 47 | + fetched_at REAL | |
| 48 | +); | |
| 49 | +""" | |
| 50 | + | |
| 51 | +_LD_RE = re.compile( | |
| 52 | + r'<script[^>]*type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', re.S) | |
| 53 | +_TEL_RE = re.compile(r'href=["\']tel:([+\d][\d\s().\-]{6,18}\d)["\']', re.I) | |
| 54 | +# « 123 rue Untel, Ville (QC) J0X 1A0 » — ancré sur le code postal canadien | |
| 55 | +# (MAJUSCULES obligatoires et ; : # { } exclus : évite les couleurs hex et | |
| 56 | +# variables CSS inline des thèmes D2C qui imitent le motif postal) | |
| 57 | +_ADDR_RE = re.compile( | |
| 58 | + r"(\d{1,5}[^<>{}|;:#\n]{5,90}?\b[A-Z]\d[A-Z]\s?\d[A-Z]\d\b)") | |
| 59 | +_STYLE_RE = re.compile(r"<(style|script)[^>]*>.*?</\1>", re.S | re.I) | |
| 60 | +_PHONE_TXT_RE = re.compile(r"(?<!\d)(\(?[2-9]\d{2}\)?[\s.\-]\d{3}[\s.\-]\d{4})(?!\d)") | |
| 61 | + | |
| 62 | +_DEALER_LD_TYPES = {"AutoDealer", "AutomotiveBusiness", "LocalBusiness", | |
| 63 | + "MotorcycleDealer", "Organization", "AutoRepair", "Store"} | |
| 64 | + | |
| 65 | + | |
| 66 | +def ensure_schema(con: sqlite3.Connection) -> None: | |
| 67 | + con.executescript(_SCHEMA) | |
| 68 | + con.commit() | |
| 69 | + | |
| 70 | + | |
| 71 | +def _iter_ld(html: str): | |
| 72 | + for blob in _LD_RE.findall(html): | |
| 73 | + try: | |
| 74 | + data = json.loads(blob.strip()) | |
| 75 | + except ValueError: | |
| 76 | + continue | |
| 77 | + stack = data if isinstance(data, list) else [data] | |
| 78 | + for item in stack: | |
| 79 | + if isinstance(item, dict): | |
| 80 | + yield from _flatten_ld(item) | |
| 81 | + | |
| 82 | + | |
| 83 | +def _flatten_ld(item: dict): | |
| 84 | + yield item | |
| 85 | + for v in item.get("@graph", []) if isinstance(item.get("@graph"), list) else []: | |
| 86 | + if isinstance(v, dict): | |
| 87 | + yield v | |
| 88 | + | |
| 89 | + | |
| 90 | +def _fmt_address(addr) -> str: | |
| 91 | + if isinstance(addr, str): | |
| 92 | + return " ".join(addr.split()) | |
| 93 | + if isinstance(addr, dict): | |
| 94 | + parts = [addr.get("streetAddress"), addr.get("addressLocality"), | |
| 95 | + addr.get("addressRegion"), addr.get("postalCode")] | |
| 96 | + return ", ".join(str(p).strip() for p in parts if p) | |
| 97 | + return "" | |
| 98 | + | |
| 99 | + | |
| 100 | +def _clean_phone(raw: str) -> str: | |
| 101 | + digits = re.sub(r"\D", "", raw or "") | |
| 102 | + if digits.startswith("1") and len(digits) == 11: | |
| 103 | + digits = digits[1:] | |
| 104 | + if len(digits) != 10: | |
| 105 | + return (raw or "").strip() | |
| 106 | + return f"({digits[0:3]}) {digits[3:6]}-{digits[6:]}" | |
| 107 | + | |
| 108 | + | |
| 109 | +def _extract(html: str) -> dict: | |
| 110 | + """address / phone / lat / lng depuis une page HTML de concessionnaire.""" | |
| 111 | + out: dict = {} | |
| 112 | + for item in _iter_ld(html): | |
| 113 | + types = item.get("@type") or "" | |
| 114 | + types = {types} if isinstance(types, str) else set(types) | |
| 115 | + if not (types & _DEALER_LD_TYPES): | |
| 116 | + continue | |
| 117 | + if not out.get("address") and item.get("address"): | |
| 118 | + out["address"] = _fmt_address(item["address"]) | |
| 119 | + if not out.get("phone") and item.get("telephone"): | |
| 120 | + out["phone"] = _clean_phone(str(item["telephone"])) | |
| 121 | + geo = item.get("geo") or {} | |
| 122 | + if isinstance(geo, dict) and not out.get("lat"): | |
| 123 | + try: | |
| 124 | + lat, lng = float(geo.get("latitude")), float(geo.get("longitude")) | |
| 125 | + if 44.0 <= lat <= 63.0 and -80.0 <= lng <= -57.0: # Québec | |
| 126 | + out["lat"], out["lng"] = lat, lng | |
| 127 | + except (TypeError, ValueError): | |
| 128 | + pass | |
| 129 | + if out.get("address") and out.get("phone"): | |
| 130 | + break | |
| 131 | + if not out.get("phone"): | |
| 132 | + m = _TEL_RE.search(html) or _PHONE_TXT_RE.search(html) | |
| 133 | + if m: | |
| 134 | + out["phone"] = _clean_phone(m.group(1)) | |
| 135 | + if not out.get("address"): | |
| 136 | + text = re.sub(r"<[^>]+>", " ", _STYLE_RE.sub(" ", html)) | |
| 137 | + m = _ADDR_RE.search(text) | |
| 138 | + # garde-fou : une vraie adresse contient un mot (rue, boul., ville…) | |
| 139 | + if m and re.search(r"[A-Za-zÀ-ÿ]{3,}", m.group(1)): | |
| 140 | + out["address"] = " ".join(m.group(1).split()) | |
| 141 | + return out | |
| 142 | + | |
| 143 | + | |
| 144 | +def populate(con: sqlite3.Connection, refresh: bool = False, | |
| 145 | + delay: float = 1.0) -> dict: | |
| 146 | + """Peuple la table dealers — 1 requête par source manquante, throttle 1 s.""" | |
| 147 | + ensure_schema(con) | |
| 148 | + registry = json.loads(SOURCES_PATH.read_text(encoding="utf-8"))["sources"] | |
| 149 | + have = {r["source"] for r in con.execute("SELECT source FROM dealers")} | |
| 150 | + session = requests.Session() | |
| 151 | + session.headers["User-Agent"] = USER_AGENT | |
| 152 | + fetched = filled_addr = filled_phone = errors = 0 | |
| 153 | + for s in registry: | |
| 154 | + sid = s["id"] | |
| 155 | + if s.get("type") == "marketplace": # Kijiji : pas un concessionnaire | |
| 156 | + continue | |
| 157 | + if sid in have and not refresh: | |
| 158 | + continue | |
| 159 | + info: dict = {} | |
| 160 | + try: | |
| 161 | + resp = session.get(s["url"], timeout=25) | |
| 162 | + resp.raise_for_status() | |
| 163 | + info = _extract(resp.text) | |
| 164 | + fetched += 1 | |
| 165 | + except Exception as exc: | |
| 166 | + errors += 1 | |
| 167 | + print(f"[auto-ka] dealers : {sid} — {exc}") | |
| 168 | + gps = (info.get("lat"), info.get("lng")) | |
| 169 | + if gps[0] is None or gps[1] is None: | |
| 170 | + hit = city_gps(s.get("city")) | |
| 171 | + gps = hit if hit else (None, None) | |
| 172 | + con.execute( | |
| 173 | + """INSERT INTO dealers (source, name, address, phone, website, | |
| 174 | + city, lat, lng, fetched_at) VALUES (?,?,?,?,?,?,?,?,?) | |
| 175 | + ON CONFLICT(source) DO UPDATE SET name=excluded.name, | |
| 176 | + address=excluded.address, phone=excluded.phone, | |
| 177 | + website=excluded.website, city=excluded.city, | |
| 178 | + lat=excluded.lat, lng=excluded.lng, | |
| 179 | + fetched_at=excluded.fetched_at""", | |
| 180 | + (sid, s.get("name", sid), info.get("address", ""), | |
| 181 | + info.get("phone", ""), s["url"], s.get("city", ""), | |
| 182 | + gps[0], gps[1], time.time())) | |
| 183 | + con.commit() | |
| 184 | + if info.get("address"): | |
| 185 | + filled_addr += 1 | |
| 186 | + if info.get("phone"): | |
| 187 | + filled_phone += 1 | |
| 188 | + time.sleep(delay) | |
| 189 | + total = con.execute("SELECT COUNT(*) c FROM dealers").fetchone()["c"] | |
| 190 | + out = {"fetched": fetched, "errors": errors, "with_address": filled_addr, | |
| 191 | + "with_phone": filled_phone, "dealers_total": total} | |
| 192 | + print(f"[auto-ka] dealers : {out}") | |
| 193 | + return out | |
| 194 | + | |
| 195 | + | |
| 196 | +def list_dealers(con: sqlite3.Connection) -> list[dict]: | |
| 197 | + ensure_schema(con) | |
| 198 | + counts = {r["source"]: r["n"] for r in con.execute( | |
| 199 | + "SELECT source, COUNT(*) n FROM vehicles WHERE active=1 GROUP BY source")} | |
| 200 | + rows = [dict(r) for r in con.execute( | |
| 201 | + "SELECT * FROM dealers ORDER BY name COLLATE NOCASE")] | |
| 202 | + for r in rows: | |
| 203 | + r["active_listings"] = counts.get(r["source"], 0) | |
| 204 | + return rows | |
| 205 | + | |
| 206 | + | |
| 207 | +if __name__ == "__main__": | |
| 208 | + import sys | |
| 209 | + from . import db | |
| 210 | + con = db.connect() | |
| 211 | + populate(con, refresh="--refresh" in sys.argv) | |
| 212 | + con.close() | |
added
autoka/dedup.py
+128 −0
@@ -0,0 +1,128 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec) | |
| 3 | +# Auteur : Simon-Pierre Boucher — contact@spboucher.ai | |
| 4 | +# dedup.py : dédoublonnage inter-sources par VIN | |
| 5 | +# | |
| 6 | +# Le même véhicule apparaît souvent chez plusieurs sources : le site du | |
| 7 | +# concessionnaire ET le portail de son groupe (Le Prix du Gros ↔ HGrégoire, | |
| 8 | +# Méga Centre ↔ concessions Laplante, Toutes les marques ↔ GM Côte-Nord…), | |
| 9 | +# voire une petite annonce Kijiji. Le VIN (17 caractères, unique au monde) | |
| 10 | +# est la clé de blocage parfaite : on ne regroupe QUE sur VIN identique et | |
| 11 | +# valide — dédoublonnage volontairement conservateur, zéro faux positif. | |
| 12 | +# | |
| 13 | +# Pour chaque groupe de doublons on élit une annonce CANONIQUE : | |
| 14 | +# autorité (concessionnaire direct > portail/regroupeur > petites | |
| 15 | +# annonces) puis complétude de la fiche, puis ancienneté (first_seen). | |
| 16 | +# La canonique garde la liste des autres offres dans `dup_sources` (JSON) ; | |
| 17 | +# les doublons pointent vers elle via `dup_of` (uid canonique). Les | |
| 18 | +# endpoints de liste filtrent `dup_of IS NULL` — les fiches détail restent | |
| 19 | +# toutes accessibles par uid. | |
| 20 | +# | |
| 21 | +# `recompute()` repart de zéro à chaque appel (idempotent) : appelé à la fin | |
| 22 | +# de chaque cycle d'ingestion (ingest.run) et disponible en one-shot : | |
| 23 | +# python3 -m autoka.dedup | |
| 24 | +# ----------------------------------------------------------------------------- | |
| 25 | +from __future__ import annotations | |
| 26 | + | |
| 27 | +import json | |
| 28 | +import re | |
| 29 | +import sqlite3 | |
| 30 | +from pathlib import Path | |
| 31 | + | |
| 32 | +# VIN valide : 17 caractères, alphanumériques sans I, O ni Q | |
| 33 | +_VIN_RE = re.compile(r"^[A-HJ-NPR-Z0-9]{17}$") | |
| 34 | + | |
| 35 | +_SOURCES_PATH = Path(__file__).resolve().parent.parent / "data" / "sources.json" | |
| 36 | + | |
| 37 | +# autorité par type de source (défaut : concessionnaire direct) | |
| 38 | +_AUTHORITY = {"direct": 2, "portail": 1, "marketplace": 0} | |
| 39 | + | |
| 40 | +# champs comptés pour la complétude de la fiche (départage à autorité égale) | |
| 41 | +_COMPLETENESS_FIELDS = ( | |
| 42 | + "price", "mileage_km", "transmission", "fuel", "drivetrain", "body_type", | |
| 43 | + "exterior_color", "engine", "description", "carfax_url", "trim", "year", | |
| 44 | +) | |
| 45 | + | |
| 46 | + | |
| 47 | +def _source_authority() -> dict[str, int]: | |
| 48 | + """source_id -> niveau d'autorité, depuis data/sources.json (champ `type`).""" | |
| 49 | + try: | |
| 50 | + registry = json.loads(_SOURCES_PATH.read_text(encoding="utf-8"))["sources"] | |
| 51 | + except (OSError, ValueError, KeyError): | |
| 52 | + return {} | |
| 53 | + return {s["id"]: _AUTHORITY.get(s.get("type", "direct"), 2) for s in registry} | |
| 54 | + | |
| 55 | + | |
| 56 | +def _score(row: sqlite3.Row, authority: dict[str, int]) -> tuple: | |
| 57 | + """Clé de tri décroissante : la meilleure annonce du groupe gagne.""" | |
| 58 | + completeness = sum(1 for f in _COMPLETENESS_FIELDS if row[f] not in (None, "")) | |
| 59 | + try: | |
| 60 | + n_images = len(json.loads(row["images"] or "[]")) | |
| 61 | + except ValueError: | |
| 62 | + n_images = 0 | |
| 63 | + return ( | |
| 64 | + authority.get(row["source"], 2), # concessionnaire direct d'abord | |
| 65 | + completeness + min(n_images, 10) / 10.0, | |
| 66 | + -(row["first_seen"] or 0), # à égalité : la plus ancienne | |
| 67 | + ) | |
| 68 | + | |
| 69 | + | |
| 70 | +def recompute(con: sqlite3.Connection, verbose: bool = False) -> dict: | |
| 71 | + """Recalcule dup_of/dup_sources sur toutes les annonces actives.""" | |
| 72 | + authority = _source_authority() | |
| 73 | + | |
| 74 | + # repartir de zéro : les groupes bougent à chaque cycle (ventes, retraits) | |
| 75 | + con.execute("UPDATE vehicles SET dup_of=NULL, dup_sources=NULL" | |
| 76 | + " WHERE dup_of IS NOT NULL OR dup_sources IS NOT NULL") | |
| 77 | + | |
| 78 | + rows = con.execute( | |
| 79 | + """SELECT uid, source, vin, url, price, dealer_name, city, first_seen, | |
| 80 | + images, mileage_km, transmission, fuel, drivetrain, | |
| 81 | + body_type, exterior_color, engine, description, carfax_url, | |
| 82 | + trim, year | |
| 83 | + FROM vehicles WHERE active=1 AND length(vin)=17""").fetchall() | |
| 84 | + | |
| 85 | + groups: dict[str, list[sqlite3.Row]] = {} | |
| 86 | + for r in rows: | |
| 87 | + vin = (r["vin"] or "").upper() | |
| 88 | + if _VIN_RE.match(vin): | |
| 89 | + groups.setdefault(vin, []).append(r) | |
| 90 | + | |
| 91 | + n_groups = n_dups = 0 | |
| 92 | + for vin, members in groups.items(): | |
| 93 | + if len(members) < 2: | |
| 94 | + continue | |
| 95 | + members.sort(key=lambda r: _score(r, authority), reverse=True) | |
| 96 | + canonical, dups = members[0], members[1:] | |
| 97 | + dup_sources = [ | |
| 98 | + {"uid": d["uid"], "source": d["source"], "url": d["url"], | |
| 99 | + "price": d["price"], "dealer_name": d["dealer_name"], | |
| 100 | + "city": d["city"]} | |
| 101 | + for d in dups | |
| 102 | + ] | |
| 103 | + con.execute("UPDATE vehicles SET dup_sources=? WHERE uid=?", | |
| 104 | + (json.dumps(dup_sources, ensure_ascii=False), | |
| 105 | + canonical["uid"])) | |
| 106 | + for d in dups: | |
| 107 | + con.execute("UPDATE vehicles SET dup_of=? WHERE uid=?", | |
| 108 | + (canonical["uid"], d["uid"])) | |
| 109 | + n_groups += 1 | |
| 110 | + n_dups += len(dups) | |
| 111 | + if verbose: | |
| 112 | + print(f" VIN {vin} : {canonical['uid']} <- " | |
| 113 | + + ", ".join(d["uid"] for d in dups)) | |
| 114 | + | |
| 115 | + con.commit() | |
| 116 | + out = {"vin_groups": n_groups, "duplicates_masked": n_dups, | |
| 117 | + "vins_actifs": len(groups)} | |
| 118 | + print(f"[auto-ka] dedup VIN : {n_groups} groupe(s), " | |
| 119 | + f"{n_dups} doublon(s) masqué(s)") | |
| 120 | + return out | |
| 121 | + | |
| 122 | + | |
| 123 | +if __name__ == "__main__": | |
| 124 | + import sys | |
| 125 | + from . import db | |
| 126 | + con = db.connect() | |
| 127 | + recompute(con, verbose="-v" in sys.argv) | |
| 128 | + con.close() | |
modified
autoka/ingest.py
+6 −0
@@ -46,6 +46,12 @@ def run(sources: list[str] | None = None) -> list[dict]: | ||
| 46 | 46 | db.log_failure(con, sid, f"{exc}") |
| 47 | 47 | traceback.print_exc() |
| 48 | 48 | results.append({"source": sid, "error": str(exc)}) |
| 49 | + # dédoublonnage inter-sources par VIN — recalculé après chaque cycle | |
| 50 | + try: | |
| 51 | + from . import dedup | |
| 52 | + dedup.recompute(con) | |
| 53 | + except Exception: | |
| 54 | + traceback.print_exc() | |
| 49 | 55 | con.close() |
| 50 | 56 | return results |
| 51 | 57 | |
added
autoka/recalls.py
+184 −0
@@ -0,0 +1,184 @@ | ||
| 1 | +# ----------------------------------------------------------------------------- | |
| 2 | +# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec) | |
| 3 | +# Auteur : Simon-Pierre Boucher — contact@spboucher.ai | |
| 4 | +# recalls.py : rappels de sécurité — Base de données des rappels de véhicules | |
| 5 | +# de Transports Canada (données ouvertes, gratuit) | |
| 6 | +# | |
| 7 | +# Source : dump mensuel complet CSV du jeu de données « Vehicle Recalls | |
| 8 | +# Database » (ouvert.canada.ca, jeu 1ec92326-47ef-4110-b7ca-959fab03f96d) : | |
| 9 | +# https://opendatatc.tc.canada.ca/vrdb_full_monthly.csv (~200 Mo) | |
| 10 | +# 1 ligne = 1 rappel × marque × modèle × année-modèle. L'API REST officielle | |
| 11 | +# (tc.api.canada.ca) exige une clé délivrée manuellement — le CSV ouvert est | |
| 12 | +# la voie sans clé, rafraîchi mensuellement par TC. | |
| 13 | +# | |
| 14 | +# Table `recalls` (make/model/year normalisés en MAJUSCULES sans accents) + | |
| 15 | +# croisement par (make, model, year) avec les annonces : le modèle TC est | |
| 16 | +# souvent plus court que le nôtre (« CIVIC » vs « Civic Sport Touring ») — | |
| 17 | +# on matche si l'un préfixe l'autre au premier mot près, année exacte. | |
| 18 | +# | |
| 19 | +# Peuplement one-shot / mensuel : python3 -m autoka.recalls [--limit-years N] | |
| 20 | +# ----------------------------------------------------------------------------- | |
| 21 | +from __future__ import annotations | |
| 22 | + | |
| 23 | +import csv | |
| 24 | +import io | |
| 25 | +import sqlite3 | |
| 26 | +import sys | |
| 27 | +import time | |
| 28 | +import unicodedata | |
| 29 | + | |
| 30 | +import requests | |
| 31 | + | |
| 32 | +CSV_URL = "https://opendatatc.tc.canada.ca/vrdb_full_monthly.csv" | |
| 33 | + | |
| 34 | +_SCHEMA = """ | |
| 35 | +CREATE TABLE IF NOT EXISTS recalls ( | |
| 36 | + recall_number TEXT NOT NULL, | |
| 37 | + make TEXT NOT NULL, -- MAJUSCULES sans accents (norme TC) | |
| 38 | + model TEXT NOT NULL, -- MAJUSCULES sans accents (norme TC) | |
| 39 | + year INTEGER, -- année-modèle visée | |
| 40 | + date TEXT, -- date du rappel (YYYY-MM-DD) | |
| 41 | + component TEXT, -- système visé (FR, repli EN) | |
| 42 | + description TEXT, -- description (FR, repli EN) | |
| 43 | + category TEXT, -- Car / Truck / Motorcycle... (norme TC) | |
| 44 | + units_affected INTEGER, | |
| 45 | + PRIMARY KEY (recall_number, make, model, year) | |
| 46 | +); | |
| 47 | +CREATE INDEX IF NOT EXISTS idx_recalls_mmy ON recalls(make, model, year); | |
| 48 | +""" | |
| 49 | + | |
| 50 | +# alias marques Auto-Ka -> marque TC quand l'orthographe diffère | |
| 51 | +_MAKE_ALIASES = { | |
| 52 | + "MERCEDES-BENZ": ("MERCEDES-BENZ", "MERCEDES"), | |
| 53 | + "LAND ROVER": ("LAND ROVER", "LANDROVER"), | |
| 54 | + "VOLKSWAGEN": ("VOLKSWAGEN", "VW"), | |
| 55 | + "CHEVROLET": ("CHEVROLET", "CHEV"), | |
| 56 | + "MINI": ("MINI", "BMW MINI"), | |
| 57 | +} | |
| 58 | + | |
| 59 | + | |
| 60 | +def _norm(text: str | None) -> str: | |
| 61 | + """MAJUSCULES sans accents, espaces réduits — clé de croisement.""" | |
| 62 | + if not text: | |
| 63 | + return "" | |
| 64 | + t = unicodedata.normalize("NFKD", text) | |
| 65 | + t = "".join(c for c in t if not unicodedata.combining(c)) | |
| 66 | + return " ".join(t.upper().split()) | |
| 67 | + | |
| 68 | + | |
| 69 | +def ensure_schema(con: sqlite3.Connection) -> None: | |
| 70 | + con.executescript(_SCHEMA) | |
| 71 | + con.commit() | |
| 72 | + | |
| 73 | + | |
| 74 | +# --------------------------------------------------------------------------- | |
| 75 | +# Peuplement depuis le dump CSV mensuel de Transports Canada | |
| 76 | +# --------------------------------------------------------------------------- | |
| 77 | + | |
| 78 | +def _fr_or_en(fr: str, en: str) -> str: | |
| 79 | + fr = (fr or "").strip() | |
| 80 | + if fr and fr.lower() not in ("translation not available", "non saisie"): | |
| 81 | + return fr | |
| 82 | + return (en or "").strip() | |
| 83 | + | |
| 84 | + | |
| 85 | +def populate(con: sqlite3.Connection, csv_path: str | None = None, | |
| 86 | + min_year: int = 1980) -> dict: | |
| 87 | + """(Re)charge la table depuis le dump complet (téléchargé si besoin). | |
| 88 | + | |
| 89 | + `min_year` écarte les années-modèles antérieures au parc plausible | |
| 90 | + d'Auto-Ka (schema.py borne les annonces à 1980+). | |
| 91 | + """ | |
| 92 | + ensure_schema(con) | |
| 93 | + t0 = time.time() | |
| 94 | + if csv_path: | |
| 95 | + fh = open(csv_path, encoding="utf-8", errors="replace", newline="") | |
| 96 | + else: | |
| 97 | + print(f"[auto-ka] rappels TC : téléchargement {CSV_URL} ...") | |
| 98 | + resp = requests.get(CSV_URL, timeout=600) | |
| 99 | + resp.raise_for_status() | |
| 100 | + fh = io.StringIO(resp.content.decode("utf-8", errors="replace")) | |
| 101 | + | |
| 102 | + con.execute("DELETE FROM recalls") | |
| 103 | + n = kept = 0 | |
| 104 | + with fh: | |
| 105 | + for row in csv.DictReader(fh): | |
| 106 | + n += 1 | |
| 107 | + try: | |
| 108 | + year = int(float(row.get("YEAR") or 0)) or None | |
| 109 | + except ValueError: | |
| 110 | + year = None | |
| 111 | + if year is not None and year < min_year: | |
| 112 | + continue | |
| 113 | + make = _norm(row.get("MAKE_NAME_NM")) | |
| 114 | + model = _norm(row.get("MODEL_NAME_NM")) | |
| 115 | + if not make or not row.get("RECALL_NUMBER_NUM"): | |
| 116 | + continue | |
| 117 | + try: | |
| 118 | + units = int(float((row.get("UNIT_AFFECTED_NBR") or "0") | |
| 119 | + .replace(",", ""))) | |
| 120 | + except ValueError: | |
| 121 | + units = 0 | |
| 122 | + con.execute( | |
| 123 | + """INSERT OR REPLACE INTO recalls (recall_number, make, model, | |
| 124 | + year, date, component, description, category, units_affected) | |
| 125 | + VALUES (?,?,?,?,?,?,?,?,?)""", | |
| 126 | + (row["RECALL_NUMBER_NUM"], make, model, year, | |
| 127 | + (row.get("RECALL_DATE_DTE") or "")[:10], | |
| 128 | + _fr_or_en(row.get("SYSTEM_TYPE_FTXT"), row.get("SYSTEM_TYPE_ETXT")), | |
| 129 | + _fr_or_en(row.get("COMMENT_FTXT"), row.get("COMMENT_ETXT")), | |
| 130 | + row.get("CATEGORY_ETXT") or "", | |
| 131 | + units)) | |
| 132 | + kept += 1 | |
| 133 | + con.commit() | |
| 134 | + total = con.execute("SELECT COUNT(*) c FROM recalls").fetchone()["c"] | |
| 135 | + out = {"csv_rows": n, "kept": kept, "recalls_rows": total, | |
| 136 | + "seconds": round(time.time() - t0, 1)} | |
| 137 | + print(f"[auto-ka] rappels TC : {out}") | |
| 138 | + return out | |
| 139 | + | |
| 140 | + | |
| 141 | +# --------------------------------------------------------------------------- | |
| 142 | +# Croisement avec une annonce (make/model/year normalisés) | |
| 143 | +# --------------------------------------------------------------------------- | |
| 144 | + | |
| 145 | +def for_vehicle(con: sqlite3.Connection, make: str, model: str, | |
| 146 | + year: int | None) -> list[dict]: | |
| 147 | + """Rappels TC visant ce (marque, modèle, année) — année exacte requise. | |
| 148 | + | |
| 149 | + Le modèle TC est générique (« CIVIC », « F-150 ») alors que le nôtre porte | |
| 150 | + parfois la version ; on matche par préfixe dans les deux sens, ancré sur | |
| 151 | + le premier mot pour éviter les faux positifs. | |
| 152 | + """ | |
| 153 | + if not make or not model or year is None: | |
| 154 | + return [] | |
| 155 | + ensure_schema(con) | |
| 156 | + n_make, n_model = _norm(make), _norm(model) | |
| 157 | + makes = _MAKE_ALIASES.get(n_make, (n_make,)) | |
| 158 | + first_word = n_model.split()[0] | |
| 159 | + | |
| 160 | + q = f"""SELECT recall_number, make, model, year, date, component, | |
| 161 | + description, category, units_affected | |
| 162 | + FROM recalls | |
| 163 | + WHERE make IN ({','.join('?' * len(makes))}) AND year=? | |
| 164 | + AND (model=? OR model LIKE ? OR ? LIKE model || ' %' | |
| 165 | + OR (length(model) >= 3 AND ? LIKE model || '%')) | |
| 166 | + ORDER BY date DESC""" | |
| 167 | + rows = con.execute(q, (*makes, year, n_model, f"{first_word} %", | |
| 168 | + n_model, first_word)).fetchall() | |
| 169 | + seen: set[str] = set() | |
| 170 | + out = [] | |
| 171 | + for r in rows: | |
| 172 | + if r["recall_number"] in seen: | |
| 173 | + continue | |
| 174 | + seen.add(r["recall_number"]) | |
| 175 | + out.append(dict(r)) | |
| 176 | + return out | |
| 177 | + | |
| 178 | + | |
| 179 | +if __name__ == "__main__": | |
| 180 | + from . import db | |
| 181 | + con = db.connect() | |
| 182 | + path = sys.argv[1] if len(sys.argv) > 1 else None | |
| 183 | + populate(con, csv_path=path) | |
| 184 | + con.close() | |
modified
autoka/seo.py
+4 −4
@@ -301,15 +301,15 @@ def _listing_body(*, h1: str, kicker: str, intro: str, st: dict, | ||
| 301 | 301 | def _stats(con, where: str, args: list) -> dict: |
| 302 | 302 | row = con.execute( |
| 303 | 303 | f"SELECT COUNT(*) n, ROUND(AVG(price)) avg_p, MIN(year) ymin, MAX(year) ymax" |
| 304 | − f" FROM vehicles WHERE active=1 AND {where}", args).fetchone() | |
| 304 | + f" FROM vehicles WHERE active=1 AND dup_of IS NULL AND {where}", args).fetchone() | |
| 305 | 305 | st = dict(row) |
| 306 | 306 | n_priced = con.execute( |
| 307 | − f"SELECT COUNT(*) c FROM vehicles WHERE active=1 AND price IS NOT NULL" | |
| 307 | + f"SELECT COUNT(*) c FROM vehicles WHERE active=1 AND dup_of IS NULL AND price IS NOT NULL" | |
| 308 | 308 | f" AND {where}", args).fetchone()["c"] |
| 309 | 309 | st["med_p"] = None |
| 310 | 310 | if n_priced: |
| 311 | 311 | st["med_p"] = con.execute( |
| 312 | − f"SELECT price FROM vehicles WHERE active=1 AND price IS NOT NULL AND {where}" | |
| 312 | + f"SELECT price FROM vehicles WHERE active=1 AND dup_of IS NULL AND price IS NOT NULL AND {where}" | |
| 313 | 313 | f" ORDER BY price LIMIT 1 OFFSET ?", args + [(n_priced - 1) // 2], |
| 314 | 314 | ).fetchone()["price"] |
| 315 | 315 | return st |
@@ -317,7 +317,7 @@ def _stats(con, where: str, args: list) -> dict: | ||
| 317 | 317 | |
| 318 | 318 | def _fetch(con, where: str, args: list, limit: int = LIST_SIZE) -> list[dict]: |
| 319 | 319 | rows = con.execute( |
| 320 | − f"SELECT * FROM vehicles WHERE active=1 AND {where}" | |
| 320 | + f"SELECT * FROM vehicles WHERE active=1 AND dup_of IS NULL AND {where}" | |
| 321 | 321 | f" ORDER BY price IS NULL, price ASC LIMIT ?", args + [limit]).fetchall() |
| 322 | 322 | return [dict(r) for r in rows] |
| 323 | 323 | |
modified
autoka/web.py
+64 −6
@@ -35,6 +35,12 @@ def _row_to_dict(row) -> dict: | ||
| 35 | 35 | d["features"] = json.loads(d.get("features") or "[]") |
| 36 | 36 | d["images"] = json.loads(d.get("images") or "[]") |
| 37 | 37 | d["details"] = json.loads(d.get("details") or "{}") |
| 38 | + # autres offres du même VIN (dédoublonnage inter-sources — autoka/dedup.py) | |
| 39 | + if d.get("dup_sources"): | |
| 40 | + try: | |
| 41 | + d["dup_sources"] = json.loads(d["dup_sources"]) | |
| 42 | + except ValueError: | |
| 43 | + d["dup_sources"] = [] | |
| 38 | 44 | return d |
| 39 | 45 | |
| 40 | 46 | |
@@ -109,6 +115,7 @@ def list_vehicles( | ||
| 109 | 115 | q: str | None = None, |
| 110 | 116 | sort: str = "price_asc", |
| 111 | 117 | active: int = 1, |
| 118 | + dedup: int = 1, | |
| 112 | 119 | limit: int = Query(60, le=500), |
| 113 | 120 | offset: int = 0, |
| 114 | 121 | ): |
@@ -117,6 +124,8 @@ def list_vehicles( | ||
| 117 | 124 | args: list = [] |
| 118 | 125 | if active in (0, 1): |
| 119 | 126 | sql += " AND active=?"; args.append(active) |
| 127 | + if dedup: # masquer les doublons inter-sources (même VIN) | |
| 128 | + sql += " AND dup_of IS NULL" | |
| 120 | 129 | if kind and kind != "tous": |
| 121 | 130 | sql += " AND kind=?"; args.append(kind) |
| 122 | 131 | sql = _apply_filters(sql, args, make=make, model=model, body_type=body_type, |
@@ -147,7 +156,8 @@ def get_vehicle(uid: str): | ||
| 147 | 156 | if d.get("make") and d.get("model"): |
| 148 | 157 | base_model = d["model"].split()[0] |
| 149 | 158 | d["similar"] = [_row_to_dict(r) for r in con.execute( |
| 150 | − "SELECT * FROM vehicles WHERE active=1 AND make=? AND model LIKE ?" | |
| 159 | + "SELECT * FROM vehicles WHERE active=1 AND dup_of IS NULL" | |
| 160 | + " AND make=? AND model LIKE ?" | |
| 151 | 161 | " AND uid<>? ORDER BY price IS NULL, price ASC LIMIT 6", |
| 152 | 162 | (d["make"], f"{base_model}%", uid)).fetchall()] |
| 153 | 163 | else: |
@@ -158,6 +168,37 @@ def get_vehicle(uid: str): | ||
| 158 | 168 | return d |
| 159 | 169 | |
| 160 | 170 | |
| 171 | +@app.get("/api/vehicles/{uid}/recalls") | |
| 172 | +def vehicle_recalls(uid: str): | |
| 173 | + """Rappels Transports Canada visant ce véhicule (marque/modèle/année).""" | |
| 174 | + from . import recalls as recalls_mod | |
| 175 | + con = db.connect() | |
| 176 | + row = con.execute("SELECT make, model, year FROM vehicles WHERE uid=?", | |
| 177 | + (uid,)).fetchone() | |
| 178 | + if row is None: | |
| 179 | + con.close() | |
| 180 | + raise HTTPException(404, "Véhicule introuvable") | |
| 181 | + try: | |
| 182 | + items = recalls_mod.for_vehicle(con, row["make"], row["model"], | |
| 183 | + row["year"]) | |
| 184 | + finally: | |
| 185 | + con.close() | |
| 186 | + return {"uid": uid, "make": row["make"], "model": row["model"], | |
| 187 | + "year": row["year"], "count": len(items), "recalls": items} | |
| 188 | + | |
| 189 | + | |
| 190 | +@app.get("/api/dealers") | |
| 191 | +def dealers(): | |
| 192 | + """Coordonnées des concessionnaires suivis (autoka/dealers.py).""" | |
| 193 | + from . import dealers as dealers_mod | |
| 194 | + con = db.connect() | |
| 195 | + try: | |
| 196 | + rows = dealers_mod.list_dealers(con) | |
| 197 | + finally: | |
| 198 | + con.close() | |
| 199 | + return {"count": len(rows), "dealers": rows} | |
| 200 | + | |
| 201 | + | |
| 161 | 202 | @app.get("/api/facets") |
| 162 | 203 | def facets(make: str | None = None, kind: str = "auto"): |
| 163 | 204 | """Valeurs distinctes pour construire les filtres du frontend. |
@@ -167,6 +208,7 @@ def facets(make: str | None = None, kind: str = "auto"): | ||
| 167 | 208 | """ |
| 168 | 209 | con = db.connect() |
| 169 | 210 | kf = "" if kind in ("", "tous") else f" AND kind='{'moto' if kind=='moto' else 'scooter' if kind=='scooter' else 'auto'}'" |
| 211 | + kf += " AND dup_of IS NULL" # compter chaque véhicule une seule fois | |
| 170 | 212 | model_sql = ("SELECT model, COUNT(*) n FROM vehicles" |
| 171 | 213 | " WHERE active=1 AND model<>''" + kf) |
| 172 | 214 | model_args: list = [] |
@@ -224,15 +266,19 @@ def stats(): | ||
| 224 | 266 | AVG(price) avg_price, |
| 225 | 267 | AVG(mileage_km) avg_km, |
| 226 | 268 | AVG(year) avg_year |
| 227 | − FROM vehicles WHERE active=1 AND kind='auto'""").fetchone() | |
| 269 | + FROM vehicles WHERE active=1 AND kind='auto' | |
| 270 | + AND dup_of IS NULL""").fetchone() | |
| 228 | 271 | by_region = [dict(r) for r in con.execute( |
| 229 | 272 | "SELECT region, COUNT(*) n, ROUND(AVG(price)) avg_price FROM vehicles" |
| 230 | − " WHERE active=1 AND kind='auto' AND region<>'' GROUP BY region ORDER BY n DESC")] | |
| 273 | + " WHERE active=1 AND kind='auto' AND dup_of IS NULL" | |
| 274 | + " AND region<>'' GROUP BY region ORDER BY n DESC")] | |
| 231 | 275 | by_make = [dict(r) for r in con.execute( |
| 232 | 276 | "SELECT make, COUNT(*) n, ROUND(AVG(price)) avg_price FROM vehicles" |
| 233 | − " WHERE active=1 AND kind='auto' AND make<>'' GROUP BY make ORDER BY n DESC LIMIT 20")] | |
| 277 | + " WHERE active=1 AND kind='auto' AND dup_of IS NULL" | |
| 278 | + " AND make<>'' GROUP BY make ORDER BY n DESC LIMIT 20")] | |
| 234 | 279 | by_body = [dict(r) for r in con.execute( |
| 235 | 280 | "SELECT body_type, COUNT(*) n FROM vehicles WHERE active=1 AND kind='auto'" |
| 281 | + " AND dup_of IS NULL" | |
| 236 | 282 | " AND body_type<>'' GROUP BY body_type ORDER BY n DESC")] |
| 237 | 283 | # baisses de prix récentes (signal d'aubaine) |
| 238 | 284 | drops = [dict(r) for r in con.execute( |
@@ -242,14 +288,26 @@ def stats(): | ||
| 242 | 288 | SELECT uid, price prev_price, |
| 243 | 289 | ROW_NUMBER() OVER (PARTITION BY uid ORDER BY ts DESC) rn |
| 244 | 290 | FROM price_log) p ON p.uid=v.uid AND p.rn=2 |
| 245 | − WHERE v.active=1 AND kind='auto' AND v.price IS NOT NULL AND p.prev_price > v.price | |
| 291 | + WHERE v.active=1 AND kind='auto' AND v.dup_of IS NULL | |
| 292 | + AND v.price IS NOT NULL AND p.prev_price > v.price | |
| 246 | 293 | ORDER BY (p.prev_price - v.price) DESC LIMIT 12""")] |
| 247 | 294 | for d in drops: |
| 248 | 295 | d["images"] = json.loads(d.get("images") or "[]")[:1] |
| 249 | 296 | log = [dict(r) for r in con.execute( |
| 250 | 297 | "SELECT * FROM sync_log ORDER BY ts DESC LIMIT 20")] |
| 298 | + # enrichissements vague 2 : doublons VIN masqués, rappels TC, dealers | |
| 299 | + extra = {"vin_duplicates_masked": con.execute( | |
| 300 | + "SELECT COUNT(*) c FROM vehicles WHERE active=1 AND dup_of IS NOT NULL" | |
| 301 | + ).fetchone()["c"]} | |
| 302 | + for key, query in ( | |
| 303 | + ("recalls_total", "SELECT COUNT(DISTINCT recall_number) c FROM recalls"), | |
| 304 | + ("dealers_total", "SELECT COUNT(*) c FROM dealers")): | |
| 305 | + try: | |
| 306 | + extra[key] = con.execute(query).fetchone()["c"] | |
| 307 | + except Exception: # table pas encore créée | |
| 308 | + extra[key] = 0 | |
| 251 | 309 | con.close() |
| 252 | − return {**dict(row), "by_region": by_region, "by_make": by_make, | |
| 310 | + return {**dict(row), **extra, "by_region": by_region, "by_make": by_make, | |
| 253 | 311 | "by_body": by_body, "price_drops": drops, "recent_syncs": log} |
| 254 | 312 | |
| 255 | 313 | |
modified
data/sources.json
+34 −8
@@ -120,7 +120,9 @@ | ||
| 120 | 120 | "region": "Abitibi-Témiscamingue", |
| 121 | 121 | "platform": "D2C Media", |
| 122 | 122 | "connector": "d2c_dealers", |
| 123 | − "status": "actif" | |
| 123 | + "status": "actif", | |
| 124 | + "type": "portail", | |
| 125 | + "type_note": "regroupeur crédit (cross-liste Lami Honda…)" | |
| 124 | 126 | }, |
| 125 | 127 | { |
| 126 | 128 | "id": "nicoloccasion", |
@@ -274,7 +276,9 @@ | ||
| 274 | 276 | "region": "Estrie", |
| 275 | 277 | "platform": "Next.js Autoroot / EvalAuto (multi-succursales : Sherbrooke, Magog, Granby, Drummondville, Victoriaville, Saint-Hyacinthe, Cowansville — ville par véhicule)", |
| 276 | 278 | "connector": "occasionbeaucage", |
| 277 | − "status": "actif" | |
| 279 | + "status": "actif", | |
| 280 | + "type": "portail", | |
| 281 | + "type_note": "regroupeur multi-succursales (cross-liste Mazda/Nissan/Kia…)" | |
| 278 | 282 | }, |
| 279 | 283 | { |
| 280 | 284 | "id": "megacentredeliquidation", |
@@ -285,7 +289,9 @@ | ||
| 285 | 289 | "region": "Bas-Saint-Laurent", |
| 286 | 290 | "platform": "Gatsby / EvalAuto (Groupe Laplante — Québec, Trois-Rivières, Rimouski, Matane, Mont-Joli, Amqui, Lanaudière — ville par véhicule)", |
| 287 | 291 | "connector": "megacentre", |
| 288 | − "status": "actif" | |
| 292 | + "status": "actif", | |
| 293 | + "type": "portail", | |
| 294 | + "type_note": "cross-liste l'inventaire du Groupe Laplante" | |
| 289 | 295 | }, |
| 290 | 296 | { |
| 291 | 297 | "id": "leprixdugros", |
@@ -296,7 +302,9 @@ | ||
| 296 | 302 | "region": "Mauricie", |
| 297 | 303 | "platform": "WordPress Stereodev (JSON paginé ?output=json — groupe Kia/Nissan/Hyundai/Mazda : Trois-Rivières, Shawinigan, Donnacona, Lévis, Québec, Laval, Sherbrooke, Joliette — ville par véhicule via page détail)", |
| 298 | 304 | "connector": "leprixdugros", |
| 299 | − "status": "actif" | |
| 305 | + "status": "actif", | |
| 306 | + "type": "portail", | |
| 307 | + "type_note": "portail de liquidation du groupe HGrégoire" | |
| 300 | 308 | }, |
| 301 | 309 | { |
| 302 | 310 | "id": "autoglobalmtl", |
@@ -670,7 +678,9 @@ | ||
| 670 | 678 | "region": "Saguenay–Lac-Saint-Jean", |
| 671 | 679 | "platform": "D2C Media", |
| 672 | 680 | "connector": "d2c_dealers", |
| 673 | − "status": "actif" | |
| 681 | + "status": "actif", | |
| 682 | + "type": "portail", | |
| 683 | + "type_note": "portail multiconcessionnaire (cross-liste Ford/Honda Baie-Comeau…)" | |
| 674 | 684 | }, |
| 675 | 685 | { |
| 676 | 686 | "id": "jeandumasfordbc", |
@@ -868,7 +878,9 @@ | ||
| 868 | 878 | "region": "Saguenay–Lac-Saint-Jean", |
| 869 | 879 | "platform": "sm360", |
| 870 | 880 | "connector": "sm360_dealers", |
| 871 | − "status": "actif" | |
| 881 | + "status": "actif", | |
| 882 | + "type": "portail", | |
| 883 | + "type_note": "portail regroupeur GM/multi-marques Côte-Nord/Saguenay" | |
| 872 | 884 | }, |
| 873 | 885 | { |
| 874 | 886 | "id": "toyotabdc", |
@@ -1374,7 +1386,9 @@ | ||
| 1374 | 1386 | "region": "Capitale-Nationale", |
| 1375 | 1387 | "platform": "OctoberCMS thème nerd2 / NerdAuto (cartes liste rendues serveur + bloc Autoverify data-av-* ; pages détail pour specs/carfax)", |
| 1376 | 1388 | "connector": "occasioncharlevoix", |
| 1377 | − "status": "actif" | |
| 1389 | + "status": "actif", | |
| 1390 | + "type": "portail", | |
| 1391 | + "type_note": "regroupeur des concessions de Charlevoix" | |
| 1378 | 1392 | }, |
| 1379 | 1393 | { |
| 1380 | 1394 | "id": "mathiassports", |
@@ -1531,6 +1545,18 @@ | ||
| 1531 | 1545 | "connector": "moto_dealers", |
| 1532 | 1546 | "vertical": "moto", |
| 1533 | 1547 | "status": "actif" |
| 1548 | + }, | |
| 1549 | + { | |
| 1550 | + "id": "kijiji", | |
| 1551 | + "name": "Kijiji (particuliers)", | |
| 1552 | + "url": "https://www.kijiji.ca", | |
| 1553 | + "listing_url": "https://www.kijiji.ca/b-autos-camions/quebec/c174l9001?for-sale-by=ownr", | |
| 1554 | + "city": "", | |
| 1555 | + "region": "", | |
| 1556 | + "type": "marketplace", | |
| 1557 | + "platform": "Next.js/Apollo — le SRP embarque les AutosListing complets dans __NEXT_DATA__ (attributs canoniques, GPS, VIN parfois) ; kijijiautos.ca (MoVe) n'existe plus (domaine SERVFAIL, 2026-08)", | |
| 1558 | + "connector": "kijiji", | |
| 1559 | + "status": "actif" | |
| 1534 | 1560 | } |
| 1535 | 1561 | ] |
| 1536 | −} | |
| 1562 | +} | |
| \ No newline at end of file | ||
| 1537 | 1563 | |