Python 67%
TypeScript 18.2%
CSS 14.4%
1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# normalize.py : couche de normalisation commune (prix, types, adresses…)5# -----------------------------------------------------------------------------6"""Fonctions de normalisation partagées par tous les connecteurs.78Les connecteurs remplissent les champs bruts tels que vus sur le site source ;9`PropertyListing.finalize()` (schema.py) appelle ces fonctions pour produire10des valeurs canoniques comparables entre agences.11"""12from __future__ import annotations1314import re15import unicodedata1617__all__ = [18 "strip_accents", "clean_address", "parse_price", "price_is_from",19 "parse_int", "parse_float", "parse_area_sqft", "parse_lot_sqft",20 "parse_year", "normalize_property_type", "extract_bedrooms_bathrooms",21 "clean_title", "clean_description",22]232425def strip_accents(text: str) -> str:26 return "".join(c for c in unicodedata.normalize("NFD", text or "")27 if unicodedata.category(c) != "Mn")282930_SMALL_WORDS = {"a", "à", "au", "aux", "avec", "de", "des", "du", "en", "et",31 "la", "le", "les", "ou", "pour", "sur", "sous", "un", "une"}323334def clean_title(text: str) -> str:35 """Titre propre : espaces normalisés, et les titres CRIÉS EN MAJUSCULES36 (fréquents chez certaines sources) ramenés en casse naturelle — chaque mot37 capitalisé sauf les mots-outils (les noms de villes restent capitalisés)."""38 t = re.sub(r"\s+", " ", (text or "")).strip()39 letters = [c for c in t if c.isalpha()]40 if len(letters) >= 8 and sum(c.isupper() for c in letters) / len(letters) > 0.85:41 words = []42 for i, w in enumerate(t.lower().split(" ")):43 words.append(w if (i and w in _SMALL_WORDS) else w[:1].upper() + w[1:])44 t = " ".join(words)45 return t464748_TAG_RE = re.compile(r"<[^>]+>")49_BR_RE = re.compile(r"<br\s*/?>|</p>|</div>|</li>", re.I)505152def clean_description(text: str) -> str:53 """Description sans HTML brut visible : balises retirées (sauts de ligne54 préservés), entités décodées, espaces/blancs normalisés."""55 import html as _html56 t = text or ""57 if "<" in t and ">" in t:58 t = _BR_RE.sub("\n", t)59 t = _TAG_RE.sub(" ", t)60 t = _html.unescape(t)61 t = re.sub(r"[ \t]+", " ", t)62 t = re.sub(r" ?\n ?", "\n", t)63 t = re.sub(r"\n{3,}", "\n\n", t)64 return t.strip()656667def clean_address(text: str) -> str:68 """Nettoie une adresse civique (espaces, virgules doublées, apostrophes)."""69 t = re.sub(r"\s+", " ", (text or "").replace("’", "'")).strip()70 t = re.sub(r"\s*,\s*", ", ", t)71 t = re.sub(r"(, )+", ", ", t).strip(", ")72 return t737475# ---------------------------------------------------------------------------76# Prix77# ---------------------------------------------------------------------------7879_PRICE_RE = re.compile(r"(\d[\d\s .,]*)\s*(?:\$|CAD)?", re.UNICODE)808182def parse_price(label: str) -> float | None:83 """Extrait un prix de vente d'un libellé source.8485 Gère « 459 000 $ », « $459,000 », « 1 249 000$ +tx », « À partir de 399 900 $ ».86 Retourne None si aucun montant plausible (>= 10 000 $) n'est trouvé.87 """88 if not label:89 return None90 m = _PRICE_RE.search(label.replace(" ", " ").replace(" ", " "))91 if not m:92 return None93 raw = m.group(1).strip()94 # « 459 000 » / « 459,000 » / « 459000.00 » — retirer les séparateurs de milliers95 raw = raw.replace(" ", "")96 if "," in raw and "." in raw:97 raw = raw.replace(",", "") # 459,000.0098 elif raw.count(",") == 1 and len(raw.split(",")[1]) == 2:99 raw = raw.replace(",", ".") # 459000,00 (décimale FR)100 else:101 raw = raw.replace(",", "")102 try:103 value = float(raw)104 except ValueError:105 return None106 return value if value >= 10_000 else None107108109def price_is_from(label: str) -> bool:110 key = strip_accents((label or "").lower())111 return any(k in key for k in ("a partir", "starting", "from", "des "))112113114# ---------------------------------------------------------------------------115# Nombres génériques116# ---------------------------------------------------------------------------117118def parse_int(text) -> int | None:119 if text is None:120 return None121 if isinstance(text, (int, float)):122 return int(text)123 m = re.search(r"\d+", str(text))124 return int(m.group()) if m else None125126127def parse_float(text) -> float | None:128 if text is None:129 return None130 if isinstance(text, (int, float)):131 return float(text)132 m = re.search(r"\d[\d\s]*(?:[.,]\d+)?", str(text))133 if not m:134 return None135 try:136 return float(m.group().replace(" ", "").replace(",", "."))137 except ValueError:138 return None139140141_SQFT_RE = re.compile(r"([\d\s ,.]+)\s*(pi2|pi²|pc|sq\.?\s*?ft|ft2|ft²)",142 re.IGNORECASE)143_SQM_RE = re.compile(r"([\d\s ,.]+)\s*(m2|m²|mc)", re.IGNORECASE)144145146def parse_area_sqft(text: str) -> float | None:147 """Superficie habitable en pi² (convertit les m² au besoin)."""148 if not text:149 return None150 t = text.replace(" ", " ")151 m = _SQFT_RE.search(t)152 if m:153 v = parse_float(m.group(1))154 return round(v) if v and v > 50 else None155 m = _SQM_RE.search(t)156 if m:157 v = parse_float(m.group(1))158 return round(v * 10.7639) if v and v > 5 else None159 return None160161162def parse_lot_sqft(text: str) -> float | None:163 """Superficie de terrain en pi² — mêmes unités que parse_area_sqft."""164 return parse_area_sqft(text)165166167# « 1959 », « 2018 (Neuf) », « 1975, rénové »… mais JAMAIS « 20' X 34' irr. »168# ni « À construire » : la valeur doit COMMENCER par une année plausible.169_YEAR_RE = re.compile(r"^\s*(1[6-9]\d{2}|20[0-4]\d)\s*(?:$|[(,])")170171172def parse_year(text) -> int | None:173 """Année de construction plausible (1600-2049) depuis une valeur `details`.174175 Volontairement strict (année en tête de valeur, seule ou suivie d'une176 parenthèse/virgule) pour ne jamais promouvoir un libellé parasite vers la177 colonne year_built."""178 if text is None:179 return None180 if isinstance(text, (int, float)):181 y = int(text)182 return y if 1600 <= y <= 2049 else None183 m = _YEAR_RE.match(str(text))184 return int(m.group(1)) if m else None185186187# ---------------------------------------------------------------------------188# Type de propriété189# ---------------------------------------------------------------------------190191# House-Ka canonical vocabulary (frontend filters) — ENGLISH.192# Sources are CREA DDF sites (English labels), plus the odd French label.193_TYPE_MAP = [194 # (keywords in the normalized source text, canonical type)195 (("maison mobile", "unimodulaire", "mobile home", "manufactured home",196 "modular",), "Mobile home"),197 (("jumele", "semi-detache", "semi detache", "semi-detached", "semi detached",),198 "Semi-detached"),199 (("maison de ville", "townhouse", "town house", "en rangee", "row house",200 "row / town",), "Townhouse"),201 (("condo", "copropriete", "appartement", "apartment", "loft", "penthouse",202 "studio", "strata",), "Condo"),203 (("duplex",), "Duplex"),204 (("triplex",), "Triplex"),205 (("quadruplex", "quintuplex", "multiplex", "multilogement", "multi-logement",206 "immeuble a revenus", "revenus", "multi-family", "multi family",207 "multifamily", "fourplex",), "Multi-family"),208 (("chalet", "cottage 4 saisons", "acces au plan d'eau", "bord de l'eau",209 "recreational", "cabin",), "Cottage"),210 (("terre", "terrain", "lot ", "vacant land", "land",), "Land"),211 (("ferme", "fermette", "agricole", "agriculture", "hobby farm", "farm",212 "acreage",), "Farm"),213 (("plain-pied", "bungalow",), "House"),214 (("maison a etages", "a etage", "deux etages", "cottage",), "House"),215 (("unifamiliale", "maison", "house", "residence", "residential", "split",216 "detached", "single family", "single-family",), "House"),217 # enriched-sheet vocabulary: « 4 logements », « propriété à revenu »218 (("logements", "logement/", "unites et +", "revenu",), "Multi-family"),219 (("bi generation", "bi-generation", "bigeneration", "intergeneration",),220 "House"),221 (("domaine et villa", "villa", "domaine",), "House"),222 (("parking",), "Parking"),223 (("commercial", "commerce", "industriel", "industrie", "bureau", "local",224 "entreprise", "batisse", "restaurant", "depanneur", "hotel", "motel",225 "garage/", "concessionnaire", "coiffure", "esthetique", "camping",226 "retail", "office", "industrial", "warehouse", "business",227 "institutional",), "Commercial"),228]229230231def normalize_property_type(text: str) -> str:232 import html as _html233 key = strip_accents(_html.unescape(text or "").strip().lower())234 if not key:235 return ""236 for keywords, canon in _TYPE_MAP:237 if any(k in key for k in keywords):238 return canon239 return _html.unescape(text).strip().capitalize()240241242# ---------------------------------------------------------------------------243# Chambres / salles de bains depuis du texte libre244# ---------------------------------------------------------------------------245246_BED_RE = re.compile(r"(\d+)\s*(?:ch(?:ambre)?s?|cac|bed(?:room)?s?)\b",247 re.IGNORECASE)248_BATH_RE = re.compile(r"(\d+)\s*(?:sdb|salle?s?\s+de\s+bains?|bath(?:room)?s?)",249 re.IGNORECASE)250251252def extract_bedrooms_bathrooms(text: str) -> tuple[int | None, int | None]:253 if not text:254 return None, None255 beds = _BED_RE.search(text)256 baths = _BATH_RE.search(text)257 return (int(beds.group(1)) if beds else None,258 int(baths.group(1)) if baths else None)259