"""Normalise broker-entered fields (TEXT claims, French/English free strings) onto the ontology vocabularies so that visual and text extractions can be compared with them. Mappings are explicit and conservative: a string that matches no rule maps to None (excluded from agreement statistics, counted in `unmapped`).""" from __future__ import annotations import re import unicodedata import numpy as np import pandas as pd def _n(s) -> str: if s is None or (isinstance(s, float) and np.isnan(s)): return "" s = unicodedata.normalize("NFKD", str(s)).encode("ascii", "ignore").decode().lower() return re.sub(r"\s+", " ", s) CLADDING_RULES = [ (r"\bbrique|brick", "brick"), (r"\bpierre|stone|maconnerie", "stone"), (r"vinyle|vinyl", "vinyl"), (r"\bbois\b|wood|cedre|cedar|canexel|fibre de bois|maibec|declin de bois", "wood"), (r"aluminium|aluminum|acier|steel|tole", "aluminum_or_steel"), (r"acrylique|stucco|stuc|crepi|agregat", "stucco_or_acrylic"), (r"fibro|fibrociment|fibre-ciment|fiber cement|hardie|james hardie|ciment", "fiber_cement"), ] ROOF_RULES = [ (r"bardeau|shingle|asphalte(?!.*gravier)", "asphalt_shingle"), (r"tole|metal|acier|steel", "metal"), (r"membrane|elastomere|tpo|epdm|gravier|gravel|plat|flat|goudron|tar", "membrane_flat"), ] BASEMENT_RULES = [ (r"totalement|entierement|fully|finished(?! partially)|amenage(?! partiel)", "finished"), (r"partiel|partially", "partially_finished"), (r"non amenage|unfinished|brut|non fini", "unfinished"), (r"aucun|none|vide sanitaire|crawl|dalle", "no_basement"), ] GARAGE_RULES = [(r"attach|integre|attached|built-in", "attached"), (r"detach|detached", "detached"), (r"abri|carport", "carport")] COUNTER_RULES = [(r"quartz|granit|granite|marbre|marble|pierre|stone", "quartz_granite_or_stone"), (r"stratifie|laminate|melamine|formica|arborite", "laminate"), (r"bois|wood|boucher|butcher", "wood_or_butcher_block")] CABINET_RULES = [(r"melamine|stratifie|laminate", "melamine_or_laminate"), (r"thermoplastique|thermofoil|polyester|peint|painted|mdf", "painted_or_thermofoil"), (r"\bbois\b|wood|merisier|chene|erable|oak|maple", "wood"), (r"lustre|gloss|laque|lacquer|acrylique", "modern_high_gloss")] def map_set(s, rules) -> set: t = _n(s) out = set() for pat, lab in rules: if re.search(pat, t): out.add(lab) return out def map_first(s, rules): """First matching rule in priority order (for mutually exclusive fields).""" t = _n(s) if not t: return None for pat, lab in rules: if re.search(pat, t): return lab return None def claim_cladding_set(s) -> set: return map_set(s, CLADDING_RULES) def claim_roof(s): t = _n(s) if "asphalte et gravier" in t or "gravier" in t: return "membrane_flat" return map_first(s, ROOF_RULES) def claim_basement(s): t = _n(s) if not t: return None if re.search(r"partiel|partially", t): return "partially_finished" return map_first(s, BASEMENT_RULES) def claim_garage(garage_field, parking_field=None): """Garage presence/type from 'Garage' or from 'Stationnement (total)' (e.g. 'Garage (1), Allée (2)').""" g = _n(garage_field) p = _n(parking_field) if g: lab = map_first(garage_field, GARAGE_RULES) return lab or "garage_unspecified" if p and "garage" in p: return "garage_unspecified" # NOTE: an exterior-parking count without a 'Garage' field does NOT imply the absence of a garage (verified against # visual and text channels, 2026-10-20) → no 'none' reference is derived from broker fields. return None def claim_counters(s): return map_first(s, COUNTER_RULES) def claim_cabinets(s): return map_first(s, CABINET_RULES) def roll_storeys_label(x): if x is None or (isinstance(x, float) and np.isnan(x)): return None return {1.0: "1", 1.5: "1.5", 2.0: "2", 2.5: "2.5", 3.0: "3+"}.get(float(x), "3+" if float(x) > 3 else None) def listing_type_label(pt): return {"Maison": None, "Détachée": "detached", "Jumelé": "semi_detached", "Maison de ville": "row_or_townhouse"}.get(pt) def add_reference_columns(frame: pd.DataFrame) -> pd.DataFrame: f = frame.copy() f["ref_cladding_set"] = f["claim_siding"].map(claim_cladding_set) f["ref_roof"] = f["claim_roof"].map(claim_roof) f["ref_basement"] = f["claim_basement"].map(claim_basement) park = f["claim_parking"] if "claim_parking" in f else pd.Series([None] * len(f), index=f.index) f["ref_garage"] = [claim_garage(g, p) for g, p in zip(f["claim_garage"], park)] f["ref_counters"] = f["claim_kitchen"].map(claim_counters) f["ref_cabinets"] = f["claim_kitchen"].map(claim_cabinets) f["ref_storeys"] = f["roll_storeys"].map(roll_storeys_label) f["ref_building_type"] = f["property_type"].map(listing_type_label) return f