# ----------------------------------------------------------------------------- # Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # normalize.py : normalisation des champs véhicule — prix, kilométrage, année, # marque/modèle, transmission, carburant, motricité, carrosserie, # ville -> région administrative. # ----------------------------------------------------------------------------- from __future__ import annotations import re import unicodedata __all__ = [ "strip_accents", "parse_price", "price_is_from", "parse_mileage", "parse_year", "normalize_make", "split_title", "normalize_transmission", "normalize_fuel", "normalize_drivetrain", "normalize_body", "infer_region", "MAKES", ] def strip_accents(text: str) -> str: return "".join(c for c in unicodedata.normalize("NFD", text) if unicodedata.category(c) != "Mn") # --------------------------------------------------------------------------- # Prix et kilométrage # --------------------------------------------------------------------------- _PRICE_RE = re.compile(r"(\d{1,3}(?:[ ,.]\d{3})+|\d{3,6})(?:[,.](\d{2}))?\s*\$") _PRICE_BARE_RE = re.compile(r"\$\s*(\d{1,3}(?:[ ,.]\d{3})+|\d{3,6})") def parse_price(label: str | None) -> float | None: """« 21 495 $ », « $21,495 », « 21495$ » -> 21495.0 (None si absent/0).""" if not label: return None m = _PRICE_RE.search(label) or _PRICE_BARE_RE.search(label) if not m: return None raw = re.sub(r"[ ,.]", "", m.group(1)) try: value = float(raw) except ValueError: return None # bornes de vraisemblance pour une auto usagée if value < 300 or value > 500_000: return None return value def price_is_from(label: str) -> bool: return bool(re.search(r"à partir|a partir|starting|from", label, re.I)) _KM_RE = re.compile(r"(\d{1,3}(?:[ ,.]\d{3})+|\d+)\s*km", re.I) def parse_mileage(label: str | None) -> float | None: """« 45 678 km », « 45,678 KM » -> 45678.0.""" if not label: return None m = _KM_RE.search(label) if not m: return None raw = re.sub(r"[ ,.]", "", m.group(1)) try: value = float(raw) except ValueError: return None if value < 0 or value > 900_000: return None return value _YEAR_RE = re.compile(r"\b(19[89]\d|20[0-3]\d)\b") def parse_year(text: str | None) -> int | None: if not text: return None m = _YEAR_RE.search(text) return int(m.group(1)) if m else None # --------------------------------------------------------------------------- # Marques (canoniques) — clé = version sans accents/majuscules/espaces # --------------------------------------------------------------------------- MAKES = [ "Acura", "Alfa Romeo", "Aston Martin", "Audi", "Bentley", "BMW", "Buick", "Cadillac", "Chevrolet", "Chrysler", "Dodge", "Ferrari", "Fiat", "Fisker", "Ford", "Genesis", "GMC", "Honda", "Hummer", "Hyundai", "Infiniti", "Jaguar", "Jeep", "Kia", "Lamborghini", "Land Rover", "Lexus", "Lincoln", "Lotus", "Maserati", "Mazda", "McLaren", "Mercedes-Benz", "Mercury", "MINI", "Mitsubishi", "Nissan", "Oldsmobile", "Plymouth", "Polestar", "Pontiac", "Porsche", "RAM", "Rivian", "Rolls-Royce", "Saab", "Saturn", "Scion", "Smart", "Subaru", "Suzuki", "Tesla", "Toyota", "VinFast", "Volkswagen", "Volvo", ] _MAKE_ALIASES = { "mercedes": "Mercedes-Benz", "mercedesbenz": "Mercedes-Benz", "mercedesamg": "Mercedes-Benz", "vw": "Volkswagen", "chevy": "Chevrolet", "landrover": "Land Rover", "rangerover": "Land Rover", "mini": "MINI", "ram": "RAM", "gmc": "GMC", "bmw": "BMW", "alfa": "Alfa Romeo", "alfaromeo": "Alfa Romeo", "rollsroyce": "Rolls-Royce", "vinfast": "VinFast", } _MAKE_LOOKUP = {strip_accents(m).lower().replace(" ", "").replace("-", ""): m for m in MAKES} _MAKE_LOOKUP.update(_MAKE_ALIASES) def normalize_make(raw: str | None) -> str: if not raw: return "" key = strip_accents(raw.strip()).lower().replace(" ", "").replace("-", "") return _MAKE_LOOKUP.get(key, raw.strip().title() if raw.strip() else "") # marques triées par longueur décroissante pour matcher « Land Rover » avant « Rover » _MAKES_BY_LEN = sorted(MAKES + ["Mercedes"], key=len, reverse=True) def split_title(title: str) -> tuple[int | None, str, str]: """« Toyota RAV4 2019 XLE AWD » ou « 2019 Toyota RAV4 » -> (année, marque, modèle+version). Champs vides si non détectés.""" if not title: return None, "", "" year = parse_year(title) text = _YEAR_RE.sub(" ", title) flat = strip_accents(text).lower() for make in _MAKES_BY_LEN: mk = strip_accents(make).lower() idx = flat.find(mk) if idx >= 0: before = flat[idx - 1] if idx > 0 else " " after_i = idx + len(mk) after = flat[after_i] if after_i < len(flat) else " " if not before.isalnum() and not after.isalnum(): model = text[after_i:].strip(" -–|,·") model = re.sub(r"\s{2,}", " ", model).strip() return year, normalize_make(make), model return year, "", text.strip() # --------------------------------------------------------------------------- # Transmission, carburant, motricité, carrosserie # --------------------------------------------------------------------------- def normalize_transmission(raw: str | None) -> str: if not raw: return "" t = strip_accents(raw).lower() if "cvt" in t or "variation continue" in t: return "Automatique" # CVT présentée comme automatique if "man" in t: return "Manuelle" if "auto" in t or "tiptronic" in t or "dsg" in t or "pdk" in t: return "Automatique" # « Automatique 6 vitesses » = automatique if re.search(r"\b[56]\s?(vit|spd|speed)", t): return "Manuelle" # « 6 vitesses » seul = boîte manuelle return "" def normalize_fuel(raw: str | None) -> str: if not raw: return "" t = strip_accents(raw).lower() if "rechargeable" in t or "plug" in t or "phev" in t: return "Hybride rechargeable" if "hybrid" in t or "hybride" in t: return "Hybride" if "electr" in t or "ev" == t.strip() or "electric" in t: return "Électrique" if "diesel" in t or "tdi" in t: return "Diesel" if "essence" in t or "gas" in t or "petrol" in t or "sans plomb" in t: return "Essence" return "" def normalize_drivetrain(raw: str | None) -> str: if not raw: return "" t = strip_accents(raw).lower() if "awd" in t or "4wd" in t or "4x4" in t or "integrale" in t or "all" in t \ or "quattro" in t or "xdrive" in t or "4matic" in t or "4motion" in t: return "Intégrale / 4x4" if "rwd" in t or "propulsion" in t or "arriere" in t or "rear" in t: return "Propulsion" if "fwd" in t or "traction" in t or "avant" in t or "front" in t or "2wd" in t: return "Traction" return "" # jetons STRICTS repérables dans un titre/une version (pas de faux positifs) _DRIVE_TOKEN_RE = re.compile( r"\b(awd|4wd|4x4|4rm|quattro|xdrive|4matic|4motion|sh-awd|e-four|" r"traction integrale|integrale|awc)\b", re.I) _RWD_TOKEN_RE = re.compile(r"\b(rwd|propulsion|sdrive)\b", re.I) _FWD_TOKEN_RE = re.compile(r"\b(fwd|2rm|traction avant)\b", re.I) def infer_drivetrain_from_text(text: str | None) -> str: """Motricité déduite des jetons explicites du titre/version (repli).""" if not text: return "" flat = strip_accents(text) if _DRIVE_TOKEN_RE.search(flat): return "Intégrale / 4x4" if _RWD_TOKEN_RE.search(flat): return "Propulsion" if _FWD_TOKEN_RE.search(flat): return "Traction" return "" _FUEL_HR_RE = re.compile(r"\b(phev|plug-?in|hybride?\s+rechargeable)\b", re.I) _FUEL_HYB_RE = re.compile(r"\b(hybride?|hybrid)\b", re.I) _FUEL_EV_RE = re.compile( r"\b(electrique|electric|ev|bev|e-tron|ioniq \d|kona electric|" r"bolt|leaf|model [3sxy]|id\.?4)\b", re.I) _FUEL_DIESEL_RE = re.compile(r"\b(diesel|tdi|duramax|cummins|ecodiesel)\b", re.I) def infer_fuel_from_text(text: str | None) -> str: """Carburant déduit des jetons explicites du titre/version (repli). Volontairement conservateur : ne regarde jamais la description (« sièges électriques » n'est pas un véhicule électrique). """ if not text: return "" flat = strip_accents(text) if _FUEL_HR_RE.search(flat): return "Hybride rechargeable" if _FUEL_HYB_RE.search(flat): return "Hybride" if _FUEL_EV_RE.search(flat): return "Électrique" if _FUEL_DIESEL_RE.search(flat): return "Diesel" return "" _BODY_MAP = [ (("vus", "suv", "utilitaire", "crossover", "multisegment"), "VUS"), (("camionnette", "camion", "pickup", "pick-up", "truck", "cabine"), "Camionnette"), (("fourgonnette", "minivan", "mini-fourgonnette", "van", "fourgon"), "Fourgonnette"), (("cabriolet", "convertible", "decapotable"), "Cabriolet"), (("coupe",), "Coupé"), (("hayon", "hatchback", "a hayon", "5 portes"), "Hayon"), (("familiale", "wagon", "break"), "Familiale"), (("berline", "sedan", "4 portes"), "Berline"), ] def normalize_body(raw: str | None) -> str: if not raw: return "" t = strip_accents(raw).lower() for keys, canon in _BODY_MAP: if any(k in t for k in keys): return canon return "" # --------------------------------------------------------------------------- # Ville -> région administrative du Québec # --------------------------------------------------------------------------- _REGIONS: dict[str, tuple[str, ...]] = { "Capitale-Nationale": ( "quebec", "ste-foy", "sainte-foy", "val-belair", "beauport", "charlesbourg", "loretteville", "l'ancienne-lorette", "ancienne-lorette", "donnacona", "cap-sante", "pont-rouge", "saint-augustin", "st-augustin", "baie-saint-paul", "la malbaie", "portneuf", "saint-raymond", "boischatel", "vanier", ), "Chaudière-Appalaches": ( "levis", "saint-nicolas", "st-nicolas", "charny", "saint-romuald", "st-romuald", "saint-georges", "st-georges", "thetford", "montmagny", "sainte-marie", "ste-marie", "beauce", "saint-joseph", "lac-etchemin", "bernieres", "breakeyville", "laurier-station", ), "Montréal": ( "montreal", "saint-leonard", "st-leonard", "anjou", "lasalle", "verdun", "dorval", "pointe-claire", "kirkland", "montreal-nord", "montreal-est", "ahuntsic", "villeray", "rosemont", "hochelaga", "lachine", "pierrefonds", ), "Laval": ("laval", "chomedey", "vimont", "duvernay", "sainte-rose", "ste-rose"), "Montérégie": ( "longueuil", "brossard", "saint-hubert", "st-hubert", "boucherville", "chateauguay", "granby", "saint-hyacinthe", "st-hyacinthe", "sorel", "valleyfield", "salaberry", "vaudreuil", "saint-jean-sur-richelieu", "st-jean-sur-richelieu", "chambly", "beloeil", "la prairie", "candiac", "saint-constant", "st-constant", "varennes", "cowansville", "magog", "sainte-julie", "ste-julie", "mont-saint-hilaire", "acton vale", "carignan", "ile-perrot", "l'ile-perrot", "saint-basile", "st-basile", "marieville", "mont-saint-gregoire", "napierville", "saint-remi", ), "Laurentides": ( "saint-jerome", "st-jerome", "blainville", "boisbriand", "mirabel", "saint-eustache", "st-eustache", "sainte-therese", "ste-therese", "lachute", "mont-tremblant", "sainte-agathe", "ste-agathe", "saint-sauveur", "st-sauveur", "rosemere", "deux-montagnes", "mont-laurier", ), "Lanaudière": ( "joliette", "repentigny", "terrebonne", "mascouche", "l'assomption", "assomption", "rawdon", "berthierville", "lavaltrie", "saint-lin", "st-lin", "charlemagne", ), "Estrie": ( "sherbrooke", "rock forest", "fleurimont", "coaticook", "windsor", "east angus", "lac-megantic", "asbestos", "val-des-sources", "bromont", ), "Mauricie": ( "trois-rivieres", "shawinigan", "grand-mere", "louiseville", "nicolet", "la tuque", "cap-de-la-madeleine", ), "Centre-du-Québec": ( "drummondville", "victoriaville", "plessisville", "becancour", "warwick", "princeville", ), "Saguenay–Lac-Saint-Jean": ( "saguenay", "chicoutimi", "jonquiere", "alma", "roberval", "dolbeau", "saint-felicien", "st-felicien", "la baie", ), "Bas-Saint-Laurent": ( "rimouski", "riviere-du-loup", "matane", "mont-joli", "amqui", "la pocatiere", "temiscouata", "trois-pistoles", ), "Gaspésie–Îles-de-la-Madeleine": ( "gaspe", "caplan", "bonaventure", "carleton", "chandler", "new richmond", "sainte-anne-des-monts", "ste-anne-des-monts", ), "Côte-Nord": ( "baie-comeau", "sept-iles", "port-cartier", "forestville", "havre-saint-pierre", ), "Abitibi-Témiscamingue": ( "rouyn-noranda", "rouyn", "val-d'or", "val-dor", "amos", "la sarre", "ville-marie", "malartic", "senneterre", ), "Outaouais": ( "gatineau", "hull", "aylmer", "buckingham", "maniwaki", "papineauville", ), } _CITY_TO_REGION = {city: region for region, cities in _REGIONS.items() for city in cities} def infer_region(city: str | None) -> str: if not city: return "" key = strip_accents(city.strip()).lower() if key in _CITY_TO_REGION: return _CITY_TO_REGION[key] for c, region in _CITY_TO_REGION.items(): if c in key: return region return ""