TeX 59%
Python 40.8%
1"""Normalise broker-entered fields (TEXT claims, French/English free strings) onto the ontology vocabularies so that2visual and text extractions can be compared with them. Mappings are explicit and conservative: a string that matches3no rule maps to None (excluded from agreement statistics, counted in `unmapped`)."""4from __future__ import annotations56import re7import unicodedata89import numpy as np10import pandas as pd111213def _n(s) -> str:14 if s is None or (isinstance(s, float) and np.isnan(s)):15 return ""16 s = unicodedata.normalize("NFKD", str(s)).encode("ascii", "ignore").decode().lower()17 return re.sub(r"\s+", " ", s)181920CLADDING_RULES = [21 (r"\bbrique|brick", "brick"), (r"\bpierre|stone|maconnerie", "stone"), (r"vinyle|vinyl", "vinyl"),22 (r"\bbois\b|wood|cedre|cedar|canexel|fibre de bois|maibec|declin de bois", "wood"),23 (r"aluminium|aluminum|acier|steel|tole", "aluminum_or_steel"), (r"acrylique|stucco|stuc|crepi|agregat", "stucco_or_acrylic"),24 (r"fibro|fibrociment|fibre-ciment|fiber cement|hardie|james hardie|ciment", "fiber_cement"),25]26ROOF_RULES = [27 (r"bardeau|shingle|asphalte(?!.*gravier)", "asphalt_shingle"), (r"tole|metal|acier|steel", "metal"),28 (r"membrane|elastomere|tpo|epdm|gravier|gravel|plat|flat|goudron|tar", "membrane_flat"),29]30BASEMENT_RULES = [31 (r"totalement|entierement|fully|finished(?! partially)|amenage(?! partiel)", "finished"),32 (r"partiel|partially", "partially_finished"), (r"non amenage|unfinished|brut|non fini", "unfinished"),33 (r"aucun|none|vide sanitaire|crawl|dalle", "no_basement"),34]35GARAGE_RULES = [(r"attach|integre|attached|built-in", "attached"), (r"detach|detached", "detached"),36 (r"abri|carport", "carport")]37COUNTER_RULES = [(r"quartz|granit|granite|marbre|marble|pierre|stone", "quartz_granite_or_stone"),38 (r"stratifie|laminate|melamine|formica|arborite", "laminate"), (r"bois|wood|boucher|butcher", "wood_or_butcher_block")]39CABINET_RULES = [(r"melamine|stratifie|laminate", "melamine_or_laminate"), (r"thermoplastique|thermofoil|polyester|peint|painted|mdf", "painted_or_thermofoil"),40 (r"\bbois\b|wood|merisier|chene|erable|oak|maple", "wood"), (r"lustre|gloss|laque|lacquer|acrylique", "modern_high_gloss")]414243def map_set(s, rules) -> set:44 t = _n(s)45 out = set()46 for pat, lab in rules:47 if re.search(pat, t):48 out.add(lab)49 return out505152def map_first(s, rules):53 """First matching rule in priority order (for mutually exclusive fields)."""54 t = _n(s)55 if not t:56 return None57 for pat, lab in rules:58 if re.search(pat, t):59 return lab60 return None616263def claim_cladding_set(s) -> set:64 return map_set(s, CLADDING_RULES)656667def claim_roof(s):68 t = _n(s)69 if "asphalte et gravier" in t or "gravier" in t:70 return "membrane_flat"71 return map_first(s, ROOF_RULES)727374def claim_basement(s):75 t = _n(s)76 if not t:77 return None78 if re.search(r"partiel|partially", t):79 return "partially_finished"80 return map_first(s, BASEMENT_RULES)818283def claim_garage(garage_field, parking_field=None):84 """Garage presence/type from 'Garage' or from 'Stationnement (total)' (e.g. 'Garage (1), Allée (2)')."""85 g = _n(garage_field)86 p = _n(parking_field)87 if g:88 lab = map_first(garage_field, GARAGE_RULES)89 return lab or "garage_unspecified"90 if p and "garage" in p:91 return "garage_unspecified"92 # NOTE: an exterior-parking count without a 'Garage' field does NOT imply the absence of a garage (verified against93 # visual and text channels, 2026-10-20) → no 'none' reference is derived from broker fields.94 return None959697def claim_counters(s):98 return map_first(s, COUNTER_RULES)99100101def claim_cabinets(s):102 return map_first(s, CABINET_RULES)103104105def roll_storeys_label(x):106 if x is None or (isinstance(x, float) and np.isnan(x)):107 return None108 return {1.0: "1", 1.5: "1.5", 2.0: "2", 2.5: "2.5", 3.0: "3+"}.get(float(x), "3+" if float(x) > 3 else None)109110111def listing_type_label(pt):112 return {"Maison": None, "Détachée": "detached", "Jumelé": "semi_detached", "Maison de ville": "row_or_townhouse"}.get(pt)113114115def add_reference_columns(frame: pd.DataFrame) -> pd.DataFrame:116 f = frame.copy()117 f["ref_cladding_set"] = f["claim_siding"].map(claim_cladding_set)118 f["ref_roof"] = f["claim_roof"].map(claim_roof)119 f["ref_basement"] = f["claim_basement"].map(claim_basement)120 park = f["claim_parking"] if "claim_parking" in f else pd.Series([None] * len(f), index=f.index)121 f["ref_garage"] = [claim_garage(g, p) for g, p in zip(f["claim_garage"], park)]122 f["ref_counters"] = f["claim_kitchen"].map(claim_counters)123 f["ref_cabinets"] = f["claim_kitchen"].map(claim_cabinets)124 f["ref_storeys"] = f["roll_storeys"].map(roll_storeys_label)125 f["ref_building_type"] = f["property_type"].map(listing_type_label)126 return f127