spb/ora-ka Public

1# -----------------------------------------------------------------------------2# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# normalize.py : normalisation des champs véhicule — prix, kilométrage, année,5#                marque/modèle, transmission, carburant, motricité, carrosserie,6#                ville -> région administrative.7# -----------------------------------------------------------------------------8from __future__ import annotations910import re11import unicodedata1213__all__ = [14    "strip_accents", "parse_price", "price_is_from", "parse_mileage",15    "parse_year", "normalize_make", "split_title", "normalize_transmission",16    "normalize_fuel", "normalize_drivetrain", "normalize_body", "infer_region",17    "MAKES",18]192021def strip_accents(text: str) -> str:22    return "".join(c for c in unicodedata.normalize("NFD", text)23                   if unicodedata.category(c) != "Mn")242526# ---------------------------------------------------------------------------27# Prix et kilométrage28# ---------------------------------------------------------------------------2930_PRICE_RE = re.compile(r"(\d{1,3}(?:[  ,.]\d{3})+|\d{3,6})(?:[,.](\d{2}))?\s*\$")31_PRICE_BARE_RE = re.compile(r"\$\s*(\d{1,3}(?:[  ,.]\d{3})+|\d{3,6})")323334def parse_price(label: str | None) -> float | None:35    """« 21 495 $ », « $21,495 », « 21495$ » -> 21495.0 (None si absent/0)."""36    if not label:37        return None38    m = _PRICE_RE.search(label) or _PRICE_BARE_RE.search(label)39    if not m:40        return None41    raw = re.sub(r"[  ,.]", "", m.group(1))42    try:43        value = float(raw)44    except ValueError:45        return None46    # bornes de vraisemblance pour une auto usagée47    if value < 300 or value > 500_000:48        return None49    return value505152def price_is_from(label: str) -> bool:53    return bool(re.search(r"à partir|a partir|starting|from", label, re.I))545556_KM_RE = re.compile(r"(\d{1,3}(?:[  ,.]\d{3})+|\d+)\s*km", re.I)575859def parse_mileage(label: str | None) -> float | None:60    """« 45 678 km », « 45,678 KM » -> 45678.0."""61    if not label:62        return None63    m = _KM_RE.search(label)64    if not m:65        return None66    raw = re.sub(r"[  ,.]", "", m.group(1))67    try:68        value = float(raw)69    except ValueError:70        return None71    if value < 0 or value > 900_000:72        return None73    return value747576_YEAR_RE = re.compile(r"\b(19[89]\d|20[0-3]\d)\b")777879def parse_year(text: str | None) -> int | None:80    if not text:81        return None82    m = _YEAR_RE.search(text)83    return int(m.group(1)) if m else None848586# ---------------------------------------------------------------------------87# Marques (canoniques) — clé = version sans accents/majuscules/espaces88# ---------------------------------------------------------------------------8990MAKES = [91    "Acura", "Alfa Romeo", "Aston Martin", "Audi", "Bentley", "BMW", "Buick",92    "Cadillac", "Chevrolet", "Chrysler", "Dodge", "Ferrari", "Fiat", "Fisker",93    "Ford", "Genesis", "GMC", "Honda", "Hummer", "Hyundai", "Infiniti",94    "Jaguar", "Jeep", "Kia", "Lamborghini", "Land Rover", "Lexus", "Lincoln",95    "Lotus", "Maserati", "Mazda", "McLaren", "Mercedes-Benz", "Mercury",96    "MINI", "Mitsubishi", "Nissan", "Oldsmobile", "Plymouth", "Polestar",97    "Pontiac", "Porsche", "RAM", "Rivian", "Rolls-Royce", "Saab", "Saturn",98    "Scion", "Smart", "Subaru", "Suzuki", "Tesla", "Toyota", "VinFast",99    "Volkswagen", "Volvo",100]101102_MAKE_ALIASES = {103    "mercedes": "Mercedes-Benz", "mercedesbenz": "Mercedes-Benz",104    "mercedesamg": "Mercedes-Benz", "vw": "Volkswagen", "chevy": "Chevrolet",105    "landrover": "Land Rover", "rangerover": "Land Rover", "mini": "MINI",106    "ram": "RAM", "gmc": "GMC", "bmw": "BMW", "alfa": "Alfa Romeo",107    "alfaromeo": "Alfa Romeo", "rollsroyce": "Rolls-Royce", "vinfast": "VinFast",108}109110_MAKE_LOOKUP = {strip_accents(m).lower().replace(" ", "").replace("-", ""): m111                for m in MAKES}112_MAKE_LOOKUP.update(_MAKE_ALIASES)113114# marques MOTO/SCOOTER (verticale moto — connectors/moto_dealers.py) :115# ajoutées au lookup de normalize_make SEULEMENT (pas à MAKES, pour ne pas116# influencer split_title côté autos). Sans elles, le repli .title() casserait117# les sigles : « KTM » → « Ktm », « CFMOTO » → « Cfmoto », etc.118_MOTO_MAKE_LOOKUP = {119    "harleydavidson": "Harley-Davidson", "harley": "Harley-Davidson",120    "indianmotorcycle": "Indian Motorcycle", "indian": "Indian Motorcycle",121    "ktm": "KTM", "brp": "BRP", "canam": "Can-Am",122    "cfmoto": "CFMOTO", "gasgas": "GASGAS", "kymco": "KYMCO",123    "mvagusta": "MV Agusta", "motoguzzi": "Moto Guzzi",124    "royalenfield": "Royal Enfield", "victorymotorcycles": "Victory",125    "victory": "Victory", "ssr": "SSR", "surron": "Sur-Ron",126}127_MAKE_LOOKUP.update(_MOTO_MAKE_LOOKUP)128129130def normalize_make(raw: str | None) -> str:131    if not raw:132        return ""133    key = strip_accents(raw.strip()).lower().replace(" ", "").replace("-", "")134    return _MAKE_LOOKUP.get(key, raw.strip().title() if raw.strip() else "")135136137# marques triées par longueur décroissante pour matcher « Land Rover » avant « Rover »138_MAKES_BY_LEN = sorted(MAKES + ["Mercedes"], key=len, reverse=True)139140141def split_title(title: str) -> tuple[int | None, str, str]:142    """« Toyota RAV4 2019 XLE AWD » ou « 2019 Toyota RAV4 » ->143    (année, marque, modèle+version). Champs vides si non détectés."""144    if not title:145        return None, "", ""146    year = parse_year(title)147    text = _YEAR_RE.sub(" ", title)148    flat = strip_accents(text).lower()149    for make in _MAKES_BY_LEN:150        mk = strip_accents(make).lower()151        idx = flat.find(mk)152        if idx >= 0:153            before = flat[idx - 1] if idx > 0 else " "154            after_i = idx + len(mk)155            after = flat[after_i] if after_i < len(flat) else " "156            if not before.isalnum() and not after.isalnum():157                model = text[after_i:].strip(" -–|,·")158                model = re.sub(r"\s{2,}", " ", model).strip()159                return year, normalize_make(make), model160    return year, "", text.strip()161162163# ---------------------------------------------------------------------------164# Transmission, carburant, motricité, carrosserie165# ---------------------------------------------------------------------------166167def normalize_transmission(raw: str | None) -> str:168    if not raw:169        return ""170    t = strip_accents(raw).lower()171    if "cvt" in t or "variation continue" in t:172        return "Automatique"          # CVT présentée comme automatique173    if "man" in t:174        return "Manuelle"175    if "auto" in t or "tiptronic" in t or "dsg" in t or "pdk" in t:176        return "Automatique"          # « Automatique 6 vitesses » = automatique177    if re.search(r"\b[56]\s?(vit|spd|speed)", t):178        return "Manuelle"             # « 6 vitesses » seul = boîte manuelle179    return ""180181182def normalize_fuel(raw: str | None) -> str:183    if not raw:184        return ""185    t = strip_accents(raw).lower()186    if "rechargeable" in t or "plug" in t or "phev" in t:187        return "Hybride rechargeable"188    if "hybrid" in t or "hybride" in t:189        return "Hybride"190    if "electr" in t or "ev" == t.strip() or "electric" in t:191        return "Électrique"192    if "diesel" in t or "tdi" in t:193        return "Diesel"194    if "essence" in t or "gas" in t or "petrol" in t or "sans plomb" in t:195        return "Essence"196    return ""197198199def normalize_drivetrain(raw: str | None) -> str:200    if not raw:201        return ""202    t = strip_accents(raw).lower()203    if "awd" in t or "4wd" in t or "4x4" in t or "integrale" in t or "all" in t \204            or "quattro" in t or "xdrive" in t or "4matic" in t or "4motion" in t:205        return "Intégrale / 4x4"206    if "rwd" in t or "propulsion" in t or "arriere" in t or "rear" in t:207        return "Propulsion"208    if "fwd" in t or "traction" in t or "avant" in t or "front" in t or "2wd" in t:209        return "Traction"210    return ""211212213# jetons STRICTS repérables dans un titre/une version (pas de faux positifs)214_DRIVE_TOKEN_RE = re.compile(215    r"\b(awd|4wd|4x4|4rm|quattro|xdrive|4matic|4motion|sh-awd|e-four|"216    r"traction integrale|integrale|awc)\b", re.I)217_RWD_TOKEN_RE = re.compile(r"\b(rwd|propulsion|sdrive)\b", re.I)218_FWD_TOKEN_RE = re.compile(r"\b(fwd|2rm|traction avant)\b", re.I)219220221def infer_drivetrain_from_text(text: str | None) -> str:222    """Motricité déduite des jetons explicites du titre/version (repli)."""223    if not text:224        return ""225    flat = strip_accents(text)226    if _DRIVE_TOKEN_RE.search(flat):227        return "Intégrale / 4x4"228    if _RWD_TOKEN_RE.search(flat):229        return "Propulsion"230    if _FWD_TOKEN_RE.search(flat):231        return "Traction"232    return ""233234235_FUEL_HR_RE = re.compile(r"\b(phev|plug-?in|hybride?\s+rechargeable)\b", re.I)236_FUEL_HYB_RE = re.compile(r"\b(hybride?|hybrid)\b", re.I)237_FUEL_EV_RE = re.compile(238    r"\b(electrique|electric|ev|bev|e-tron|ioniq \d|kona electric|"239    r"bolt|leaf|model [3sxy]|id\.?4)\b", re.I)240_FUEL_DIESEL_RE = re.compile(r"\b(diesel|tdi|duramax|cummins|ecodiesel)\b", re.I)241242243def infer_fuel_from_text(text: str | None) -> str:244    """Carburant déduit des jetons explicites du titre/version (repli).245246    Volontairement conservateur : ne regarde jamais la description247    (« sièges électriques » n'est pas un véhicule électrique).248    """249    if not text:250        return ""251    flat = strip_accents(text)252    if _FUEL_HR_RE.search(flat):253        return "Hybride rechargeable"254    if _FUEL_HYB_RE.search(flat):255        return "Hybride"256    if _FUEL_EV_RE.search(flat):257        return "Électrique"258    if _FUEL_DIESEL_RE.search(flat):259        return "Diesel"260    return ""261262263_BODY_MAP = [264    (("vus", "suv", "utilitaire", "crossover", "multisegment"), "VUS"),265    (("camionnette", "camion", "pickup", "pick-up", "truck", "cabine"), "Camionnette"),266    (("fourgonnette", "minivan", "mini-fourgonnette", "van", "fourgon"), "Fourgonnette"),267    (("cabriolet", "convertible", "decapotable"), "Cabriolet"),268    (("coupe",), "Coupé"),269    (("hayon", "hatchback", "a hayon", "5 portes"), "Hayon"),270    (("familiale", "wagon", "break"), "Familiale"),271    (("berline", "sedan", "4 portes"), "Berline"),272]273274275def normalize_body(raw: str | None) -> str:276    if not raw:277        return ""278    t = strip_accents(raw).lower()279    for keys, canon in _BODY_MAP:280        if any(k in t for k in keys):281            return canon282    return ""283284285# ---------------------------------------------------------------------------286# Ville -> région administrative du Québec287# ---------------------------------------------------------------------------288289_REGIONS: dict[str, tuple[str, ...]] = {290    "Capitale-Nationale": (291        "quebec", "ste-foy", "sainte-foy", "val-belair", "beauport", "charlesbourg",292        "loretteville", "l'ancienne-lorette", "ancienne-lorette", "donnacona",293        "cap-sante", "pont-rouge", "saint-augustin", "st-augustin", "baie-saint-paul",294        "la malbaie", "portneuf", "saint-raymond", "boischatel", "vanier",295    ),296    "Chaudière-Appalaches": (297        "levis", "saint-nicolas", "st-nicolas", "charny", "saint-romuald",298        "st-romuald", "saint-georges", "st-georges", "thetford", "montmagny",299        "sainte-marie", "ste-marie", "beauce", "saint-joseph", "lac-etchemin",300        "bernieres", "breakeyville", "laurier-station",301    ),302    "Montréal": (303        "montreal", "saint-leonard", "st-leonard", "anjou", "lasalle", "verdun",304        "dorval", "pointe-claire", "kirkland", "montreal-nord", "montreal-est",305        "ahuntsic", "villeray", "rosemont", "hochelaga", "lachine", "pierrefonds",306    ),307    "Laval": ("laval", "chomedey", "vimont", "duvernay", "sainte-rose", "ste-rose"),308    "Montérégie": (309        "longueuil", "brossard", "saint-hubert", "st-hubert", "boucherville",310        "chateauguay", "granby", "saint-hyacinthe", "st-hyacinthe", "sorel",311        "valleyfield", "salaberry", "vaudreuil", "saint-jean-sur-richelieu",312        "st-jean-sur-richelieu", "chambly", "beloeil", "la prairie", "candiac",313        "saint-constant", "st-constant", "varennes", "cowansville", "magog",314        "sainte-julie", "ste-julie", "mont-saint-hilaire", "acton vale",315        "carignan", "ile-perrot", "l'ile-perrot", "saint-basile", "st-basile",316        "marieville", "mont-saint-gregoire", "napierville", "saint-remi",317    ),318    "Laurentides": (319        "saint-jerome", "st-jerome", "blainville", "boisbriand", "mirabel",320        "saint-eustache", "st-eustache", "sainte-therese", "ste-therese",321        "lachute", "mont-tremblant", "sainte-agathe", "ste-agathe", "saint-sauveur",322        "st-sauveur", "rosemere", "deux-montagnes", "mont-laurier",323    ),324    "Lanaudière": (325        "joliette", "repentigny", "terrebonne", "mascouche", "l'assomption",326        "assomption", "rawdon", "berthierville", "lavaltrie", "saint-lin",327        "st-lin", "charlemagne",328    ),329    "Estrie": (330        "sherbrooke", "rock forest", "fleurimont", "coaticook", "windsor",331        "east angus", "lac-megantic", "asbestos", "val-des-sources", "bromont",332    ),333    "Mauricie": (334        "trois-rivieres", "shawinigan", "grand-mere", "louiseville", "nicolet",335        "la tuque", "cap-de-la-madeleine",336    ),337    "Centre-du-Québec": (338        "drummondville", "victoriaville", "plessisville", "becancour",339        "warwick", "princeville",340    ),341    "Saguenay–Lac-Saint-Jean": (342        "saguenay", "chicoutimi", "jonquiere", "alma", "roberval", "dolbeau",343        "saint-felicien", "st-felicien", "la baie",344    ),345    "Bas-Saint-Laurent": (346        "rimouski", "riviere-du-loup", "matane", "mont-joli", "amqui",347        "la pocatiere", "temiscouata", "trois-pistoles",348    ),349    "Gaspésie–Îles-de-la-Madeleine": (350        "gaspe", "caplan", "bonaventure", "carleton", "chandler", "new richmond",351        "sainte-anne-des-monts", "ste-anne-des-monts",352    ),353    "Côte-Nord": (354        "baie-comeau", "sept-iles", "port-cartier", "forestville", "havre-saint-pierre",355    ),356    "Abitibi-Témiscamingue": (357        "rouyn-noranda", "rouyn", "val-d'or", "val-dor", "amos", "la sarre",358        "ville-marie", "malartic", "senneterre",359    ),360    "Outaouais": (361        "gatineau", "hull", "aylmer", "buckingham", "maniwaki", "papineauville",362    ),363}364365_CITY_TO_REGION = {city: region for region, cities in _REGIONS.items()366                   for city in cities}367368369def infer_region(city: str | None) -> str:370    if not city:371        return ""372    key = strip_accents(city.strip()).lower()373    if key in _CITY_TO_REGION:374        return _CITY_TO_REGION[key]375    for c, region in _CITY_TO_REGION.items():376        if c in key:377            return region378    return ""379