spb/auto-ka Public

1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# normalize.py : normalisation des champs véhicule — prix, kilométrage, année,5#                marque/modèle, transmission, carburant, motricité, carrosserie,6#                ville -> région administrative.7# -----------------------------------------------------------------------------8from __future__ import annotations910import re11import unicodedata1213__all__ = [14    "strip_accents", "parse_price", "price_is_from", "parse_mileage",15    "parse_year", "normalize_make", "split_title", "normalize_transmission",16    "normalize_fuel", "normalize_drivetrain", "normalize_body", "infer_region",17    "MAKES",18]192021def strip_accents(text: str) -> str:22    return "".join(c for c in unicodedata.normalize("NFD", text)23                   if unicodedata.category(c) != "Mn")242526# ---------------------------------------------------------------------------27# Prix et kilométrage28# ---------------------------------------------------------------------------2930_PRICE_RE = re.compile(r"(\d{1,3}(?:[  ,.]\d{3})+|\d{3,6})(?:[,.](\d{2}))?\s*\$")31_PRICE_BARE_RE = re.compile(r"\$\s*(\d{1,3}(?:[  ,.]\d{3})+|\d{3,6})")323334def parse_price(label: str | None) -> float | None:35    """« 21 495 $ », « $21,495 », « 21495$ » -> 21495.0 (None si absent/0)."""36    if not label:37        return None38    m = _PRICE_RE.search(label) or _PRICE_BARE_RE.search(label)39    if not m:40        return None41    raw = re.sub(r"[  ,.]", "", m.group(1))42    try:43        value = float(raw)44    except ValueError:45        return None46    # bornes de vraisemblance pour une auto usagée47    if value < 300 or value > 500_000:48        return None49    return value505152def price_is_from(label: str) -> bool:53    return bool(re.search(r"à partir|a partir|starting|from", label, re.I))545556_KM_RE = re.compile(r"(\d{1,3}(?:[  ,.]\d{3})+|\d+)\s*km", re.I)575859def parse_mileage(label: str | None) -> float | None:60    """« 45 678 km », « 45,678 KM » -> 45678.0."""61    if not label:62        return None63    m = _KM_RE.search(label)64    if not m:65        return None66    raw = re.sub(r"[  ,.]", "", m.group(1))67    try:68        value = float(raw)69    except ValueError:70        return None71    if value < 0 or value > 900_000:72        return None73    return value747576_YEAR_RE = re.compile(r"\b(19[89]\d|20[0-3]\d)\b")777879def parse_year(text: str | None) -> int | None:80    if not text:81        return None82    m = _YEAR_RE.search(text)83    return int(m.group(1)) if m else None848586# ---------------------------------------------------------------------------87# Marques (canoniques) — clé = version sans accents/majuscules/espaces88# ---------------------------------------------------------------------------8990MAKES = [91    "Acura", "Alfa Romeo", "Aston Martin", "Audi", "Bentley", "BMW", "Buick",92    "Cadillac", "Chevrolet", "Chrysler", "Dodge", "Ferrari", "Fiat", "Fisker",93    "Ford", "Genesis", "GMC", "Honda", "Hummer", "Hyundai", "Infiniti",94    "Jaguar", "Jeep", "Kia", "Lamborghini", "Land Rover", "Lexus", "Lincoln",95    "Lotus", "Maserati", "Mazda", "McLaren", "Mercedes-Benz", "Mercury",96    "MINI", "Mitsubishi", "Nissan", "Oldsmobile", "Plymouth", "Polestar",97    "Pontiac", "Porsche", "RAM", "Rivian", "Rolls-Royce", "Saab", "Saturn",98    "Scion", "Smart", "Subaru", "Suzuki", "Tesla", "Toyota", "VinFast",99    "Volkswagen", "Volvo",100]101102_MAKE_ALIASES = {103    "mercedes": "Mercedes-Benz", "mercedesbenz": "Mercedes-Benz",104    "mercedesamg": "Mercedes-Benz", "vw": "Volkswagen", "chevy": "Chevrolet",105    "landrover": "Land Rover", "rangerover": "Land Rover", "mini": "MINI",106    "ram": "RAM", "gmc": "GMC", "bmw": "BMW", "alfa": "Alfa Romeo",107    "alfaromeo": "Alfa Romeo", "rollsroyce": "Rolls-Royce", "vinfast": "VinFast",108}109110_MAKE_LOOKUP = {strip_accents(m).lower().replace(" ", "").replace("-", ""): m111                for m in MAKES}112_MAKE_LOOKUP.update(_MAKE_ALIASES)113114115def normalize_make(raw: str | None) -> str:116    if not raw:117        return ""118    key = strip_accents(raw.strip()).lower().replace(" ", "").replace("-", "")119    return _MAKE_LOOKUP.get(key, raw.strip().title() if raw.strip() else "")120121122# marques triées par longueur décroissante pour matcher « Land Rover » avant « Rover »123_MAKES_BY_LEN = sorted(MAKES + ["Mercedes"], key=len, reverse=True)124125126def split_title(title: str) -> tuple[int | None, str, str]:127    """« Toyota RAV4 2019 XLE AWD » ou « 2019 Toyota RAV4 » ->128    (année, marque, modèle+version). Champs vides si non détectés."""129    if not title:130        return None, "", ""131    year = parse_year(title)132    text = _YEAR_RE.sub(" ", title)133    flat = strip_accents(text).lower()134    for make in _MAKES_BY_LEN:135        mk = strip_accents(make).lower()136        idx = flat.find(mk)137        if idx >= 0:138            before = flat[idx - 1] if idx > 0 else " "139            after_i = idx + len(mk)140            after = flat[after_i] if after_i < len(flat) else " "141            if not before.isalnum() and not after.isalnum():142                model = text[after_i:].strip(" -–|,·")143                model = re.sub(r"\s{2,}", " ", model).strip()144                return year, normalize_make(make), model145    return year, "", text.strip()146147148# ---------------------------------------------------------------------------149# Transmission, carburant, motricité, carrosserie150# ---------------------------------------------------------------------------151152def normalize_transmission(raw: str | None) -> str:153    if not raw:154        return ""155    t = strip_accents(raw).lower()156    if "cvt" in t or "variation continue" in t:157        return "Automatique"          # CVT présentée comme automatique158    if "man" in t:159        return "Manuelle"160    if "auto" in t or "tiptronic" in t or "dsg" in t or "pdk" in t:161        return "Automatique"          # « Automatique 6 vitesses » = automatique162    if re.search(r"\b[56]\s?(vit|spd|speed)", t):163        return "Manuelle"             # « 6 vitesses » seul = boîte manuelle164    return ""165166167def normalize_fuel(raw: str | None) -> str:168    if not raw:169        return ""170    t = strip_accents(raw).lower()171    if "rechargeable" in t or "plug" in t or "phev" in t:172        return "Hybride rechargeable"173    if "hybrid" in t or "hybride" in t:174        return "Hybride"175    if "electr" in t or "ev" == t.strip() or "electric" in t:176        return "Électrique"177    if "diesel" in t or "tdi" in t:178        return "Diesel"179    if "essence" in t or "gas" in t or "petrol" in t or "sans plomb" in t:180        return "Essence"181    return ""182183184def normalize_drivetrain(raw: str | None) -> str:185    if not raw:186        return ""187    t = strip_accents(raw).lower()188    if "awd" in t or "4wd" in t or "4x4" in t or "integrale" in t or "all" in t \189            or "quattro" in t or "xdrive" in t or "4matic" in t or "4motion" in t:190        return "Intégrale / 4x4"191    if "rwd" in t or "propulsion" in t or "arriere" in t or "rear" in t:192        return "Propulsion"193    if "fwd" in t or "traction" in t or "avant" in t or "front" in t or "2wd" in t:194        return "Traction"195    return ""196197198# jetons STRICTS repérables dans un titre/une version (pas de faux positifs)199_DRIVE_TOKEN_RE = re.compile(200    r"\b(awd|4wd|4x4|4rm|quattro|xdrive|4matic|4motion|sh-awd|e-four|"201    r"traction integrale|integrale|awc)\b", re.I)202_RWD_TOKEN_RE = re.compile(r"\b(rwd|propulsion|sdrive)\b", re.I)203_FWD_TOKEN_RE = re.compile(r"\b(fwd|2rm|traction avant)\b", re.I)204205206def infer_drivetrain_from_text(text: str | None) -> str:207    """Motricité déduite des jetons explicites du titre/version (repli)."""208    if not text:209        return ""210    flat = strip_accents(text)211    if _DRIVE_TOKEN_RE.search(flat):212        return "Intégrale / 4x4"213    if _RWD_TOKEN_RE.search(flat):214        return "Propulsion"215    if _FWD_TOKEN_RE.search(flat):216        return "Traction"217    return ""218219220_FUEL_HR_RE = re.compile(r"\b(phev|plug-?in|hybride?\s+rechargeable)\b", re.I)221_FUEL_HYB_RE = re.compile(r"\b(hybride?|hybrid)\b", re.I)222_FUEL_EV_RE = re.compile(223    r"\b(electrique|electric|ev|bev|e-tron|ioniq \d|kona electric|"224    r"bolt|leaf|model [3sxy]|id\.?4)\b", re.I)225_FUEL_DIESEL_RE = re.compile(r"\b(diesel|tdi|duramax|cummins|ecodiesel)\b", re.I)226227228def infer_fuel_from_text(text: str | None) -> str:229    """Carburant déduit des jetons explicites du titre/version (repli).230231    Volontairement conservateur : ne regarde jamais la description232    (« sièges électriques » n'est pas un véhicule électrique).233    """234    if not text:235        return ""236    flat = strip_accents(text)237    if _FUEL_HR_RE.search(flat):238        return "Hybride rechargeable"239    if _FUEL_HYB_RE.search(flat):240        return "Hybride"241    if _FUEL_EV_RE.search(flat):242        return "Électrique"243    if _FUEL_DIESEL_RE.search(flat):244        return "Diesel"245    return ""246247248_BODY_MAP = [249    (("vus", "suv", "utilitaire", "crossover", "multisegment"), "VUS"),250    (("camionnette", "camion", "pickup", "pick-up", "truck", "cabine"), "Camionnette"),251    (("fourgonnette", "minivan", "mini-fourgonnette", "van", "fourgon"), "Fourgonnette"),252    (("cabriolet", "convertible", "decapotable"), "Cabriolet"),253    (("coupe",), "Coupé"),254    (("hayon", "hatchback", "a hayon", "5 portes"), "Hayon"),255    (("familiale", "wagon", "break"), "Familiale"),256    (("berline", "sedan", "4 portes"), "Berline"),257]258259260def normalize_body(raw: str | None) -> str:261    if not raw:262        return ""263    t = strip_accents(raw).lower()264    for keys, canon in _BODY_MAP:265        if any(k in t for k in keys):266            return canon267    return ""268269270# ---------------------------------------------------------------------------271# Ville -> région administrative du Québec272# ---------------------------------------------------------------------------273274_REGIONS: dict[str, tuple[str, ...]] = {275    "Capitale-Nationale": (276        "quebec", "ste-foy", "sainte-foy", "val-belair", "beauport", "charlesbourg",277        "loretteville", "l'ancienne-lorette", "ancienne-lorette", "donnacona",278        "cap-sante", "pont-rouge", "saint-augustin", "st-augustin", "baie-saint-paul",279        "la malbaie", "portneuf", "saint-raymond", "boischatel", "vanier",280    ),281    "Chaudière-Appalaches": (282        "levis", "saint-nicolas", "st-nicolas", "charny", "saint-romuald",283        "st-romuald", "saint-georges", "st-georges", "thetford", "montmagny",284        "sainte-marie", "ste-marie", "beauce", "saint-joseph", "lac-etchemin",285        "bernieres", "breakeyville", "laurier-station",286    ),287    "Montréal": (288        "montreal", "saint-leonard", "st-leonard", "anjou", "lasalle", "verdun",289        "dorval", "pointe-claire", "kirkland", "montreal-nord", "montreal-est",290        "ahuntsic", "villeray", "rosemont", "hochelaga", "lachine", "pierrefonds",291    ),292    "Laval": ("laval", "chomedey", "vimont", "duvernay", "sainte-rose", "ste-rose"),293    "Montérégie": (294        "longueuil", "brossard", "saint-hubert", "st-hubert", "boucherville",295        "chateauguay", "granby", "saint-hyacinthe", "st-hyacinthe", "sorel",296        "valleyfield", "salaberry", "vaudreuil", "saint-jean-sur-richelieu",297        "st-jean-sur-richelieu", "chambly", "beloeil", "la prairie", "candiac",298        "saint-constant", "st-constant", "varennes", "cowansville", "magog",299        "sainte-julie", "ste-julie", "mont-saint-hilaire", "acton vale",300        "carignan", "ile-perrot", "l'ile-perrot", "saint-basile", "st-basile",301        "marieville", "mont-saint-gregoire", "napierville", "saint-remi",302    ),303    "Laurentides": (304        "saint-jerome", "st-jerome", "blainville", "boisbriand", "mirabel",305        "saint-eustache", "st-eustache", "sainte-therese", "ste-therese",306        "lachute", "mont-tremblant", "sainte-agathe", "ste-agathe", "saint-sauveur",307        "st-sauveur", "rosemere", "deux-montagnes", "mont-laurier",308    ),309    "Lanaudière": (310        "joliette", "repentigny", "terrebonne", "mascouche", "l'assomption",311        "assomption", "rawdon", "berthierville", "lavaltrie", "saint-lin",312        "st-lin", "charlemagne",313    ),314    "Estrie": (315        "sherbrooke", "rock forest", "fleurimont", "coaticook", "windsor",316        "east angus", "lac-megantic", "asbestos", "val-des-sources", "bromont",317    ),318    "Mauricie": (319        "trois-rivieres", "shawinigan", "grand-mere", "louiseville", "nicolet",320        "la tuque", "cap-de-la-madeleine",321    ),322    "Centre-du-Québec": (323        "drummondville", "victoriaville", "plessisville", "becancour",324        "warwick", "princeville",325    ),326    "Saguenay–Lac-Saint-Jean": (327        "saguenay", "chicoutimi", "jonquiere", "alma", "roberval", "dolbeau",328        "saint-felicien", "st-felicien", "la baie",329    ),330    "Bas-Saint-Laurent": (331        "rimouski", "riviere-du-loup", "matane", "mont-joli", "amqui",332        "la pocatiere", "temiscouata", "trois-pistoles",333    ),334    "Gaspésie–Îles-de-la-Madeleine": (335        "gaspe", "caplan", "bonaventure", "carleton", "chandler", "new richmond",336        "sainte-anne-des-monts", "ste-anne-des-monts",337    ),338    "Côte-Nord": (339        "baie-comeau", "sept-iles", "port-cartier", "forestville", "havre-saint-pierre",340    ),341    "Abitibi-Témiscamingue": (342        "rouyn-noranda", "rouyn", "val-d'or", "val-dor", "amos", "la sarre",343        "ville-marie", "malartic", "senneterre",344    ),345    "Outaouais": (346        "gatineau", "hull", "aylmer", "buckingham", "maniwaki", "papineauville",347    ),348}349350_CITY_TO_REGION = {city: region for region, cities in _REGIONS.items()351                   for city in cities}352353354def infer_region(city: str | None) -> str:355    if not city:356        return ""357    key = strip_accents(city.strip()).lower()358    if key in _CITY_TO_REGION:359        return _CITY_TO_REGION[key]360    for c, region in _CITY_TO_REGION.items():361        if c in key:362            return region363    return ""364