Python 49.6%
TypeScript 25.5%
CSS 24.1%
1# -----------------------------------------------------------------------------2# Home-Ka — US real-estate aggregator (Groupe KA)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# normalize.py : shared normalization layer (prices, types, addresses, areas)5#6# US counterpart of immo-ka/normalize.py. Every connector fills raw fields as7# seen at the source; Listing.finalize() (schema.py) calls these helpers to8# produce canonical values comparable across sources. Canonical vocabulary is9# aligned on the RESO Data Dictionary (PropertyType / PropertySubType,10# StandardStatus) while keeping the common Groupe KA internal model.11# -----------------------------------------------------------------------------12from __future__ import annotations1314import re15import unicodedata1617__all__ = [18 "strip_accents", "clean_address", "clean_title", "clean_description",19 "parse_price", "price_is_from", "parse_int", "parse_float",20 "parse_area_sqft", "parse_lot_sqft", "parse_lot_acres", "parse_year",21 "normalize_property_type", "normalize_status", "normalize_state",22 "normalize_zip", "extract_beds_baths", "STATE_NAMES",23]242526def strip_accents(text: str) -> str:27 return "".join(c for c in unicodedata.normalize("NFD", text or "")28 if unicodedata.category(c) != "Mn")293031# ---------------------------------------------------------------------------32# Text cleanup33# ---------------------------------------------------------------------------3435_SMALL_WORDS = {"a", "an", "and", "at", "by", "for", "in", "of", "on", "or",36 "the", "to", "with"}373839def clean_title(text: str) -> str:40 """Normalized title: whitespace collapsed, ALL-CAPS titles (frequent on41 MLS remarks) brought back to natural title case."""42 t = re.sub(r"\s+", " ", (text or "")).strip()43 letters = [c for c in t if c.isalpha()]44 if len(letters) >= 8 and sum(c.isupper() for c in letters) / len(letters) > 0.85:45 words = []46 for i, w in enumerate(t.lower().split(" ")):47 words.append(w if (i and w in _SMALL_WORDS) else w[:1].upper() + w[1:])48 t = " ".join(words)49 return t505152_TAG_RE = re.compile(r"<[^>]+>")53_BR_RE = re.compile(r"<br\s*/?>|</p>|</div>|</li>", re.I)545556def clean_description(text: str) -> str:57 """Description without raw HTML: tags stripped (line breaks preserved),58 entities decoded, whitespace normalized."""59 import html as _html60 t = text or ""61 if "<" in t and ">" in t:62 t = _BR_RE.sub("\n", t)63 t = _TAG_RE.sub(" ", t)64 t = _html.unescape(t)65 t = re.sub(r"[ \t]+", " ", t)66 t = re.sub(r" ?\n ?", "\n", t)67 t = re.sub(r"\n{3,}", "\n\n", t)68 return t.strip()697071_STREET_ABBR = {72 "street": "St", "avenue": "Ave", "boulevard": "Blvd", "drive": "Dr",73 "court": "Ct", "circle": "Cir", "lane": "Ln", "road": "Rd",74 "place": "Pl", "terrace": "Ter", "parkway": "Pkwy", "highway": "Hwy",75 "trail": "Trl", "square": "Sq",76}777879def clean_address(text: str) -> str:80 """Clean a street address (whitespace, doubled commas, smart quotes)."""81 t = re.sub(r"\s+", " ", (text or "").replace("’", "'")).strip()82 t = re.sub(r"\s*,\s*", ", ", t)83 t = re.sub(r"(, )+", ", ", t).strip(", ")84 return t858687# ---------------------------------------------------------------------------88# US states89# ---------------------------------------------------------------------------9091STATE_NAMES = {92 "AL": "Alabama", "AK": "Alaska", "AZ": "Arizona", "AR": "Arkansas",93 "CA": "California", "CO": "Colorado", "CT": "Connecticut", "DE": "Delaware",94 "FL": "Florida", "GA": "Georgia", "HI": "Hawaii", "ID": "Idaho",95 "IL": "Illinois", "IN": "Indiana", "IA": "Iowa", "KS": "Kansas",96 "KY": "Kentucky", "LA": "Louisiana", "ME": "Maine", "MD": "Maryland",97 "MA": "Massachusetts", "MI": "Michigan", "MN": "Minnesota",98 "MS": "Mississippi", "MO": "Missouri", "MT": "Montana", "NE": "Nebraska",99 "NV": "Nevada", "NH": "New Hampshire", "NJ": "New Jersey",100 "NM": "New Mexico", "NY": "New York", "NC": "North Carolina",101 "ND": "North Dakota", "OH": "Ohio", "OK": "Oklahoma", "OR": "Oregon",102 "PA": "Pennsylvania", "RI": "Rhode Island", "SC": "South Carolina",103 "SD": "South Dakota", "TN": "Tennessee", "TX": "Texas", "UT": "Utah",104 "VT": "Vermont", "VA": "Virginia", "WA": "Washington",105 "WV": "West Virginia", "WI": "Wisconsin", "WY": "Wyoming",106 "DC": "District of Columbia",107}108_NAME_TO_ABBR = {v.lower(): k for k, v in STATE_NAMES.items()}109110111def normalize_state(text: str) -> str:112 """State → 2-letter USPS code ('Texas'/'texas'/'TX'/'tx.' → 'TX')."""113 t = (text or "").strip().strip(".").strip()114 if not t:115 return ""116 if len(t) == 2 and t.upper() in STATE_NAMES:117 return t.upper()118 return _NAME_TO_ABBR.get(t.lower(), "")119120121_ZIP_RE = re.compile(r"\b(\d{5})(?:-\d{4})?\b")122123124def normalize_zip(text: str) -> str:125 """5-digit ZIP (ZIP+4 truncated)."""126 m = _ZIP_RE.search(str(text or ""))127 return m.group(1) if m else ""128129130# ---------------------------------------------------------------------------131# Price (USD)132# ---------------------------------------------------------------------------133134_PRICE_RE = re.compile(r"\$?\s*(\d[\d,\.\s]*)\s*([kKmM])?")135136137def parse_price(label) -> float | None:138 """Extract a sale price from a source label.139140 Handles '$459,000', '459000', '$1.25M', '750K', 'From $399,900'.141 Returns None when no plausible amount (>= $5,000) is found.142 """143 if label is None:144 return None145 if isinstance(label, (int, float)):146 return float(label) if label >= 5_000 else None147 m = _PRICE_RE.search(str(label))148 if not m:149 return None150 raw = m.group(1).strip().replace(" ", "").rstrip(".")151 mult = {"k": 1e3, "m": 1e6}.get((m.group(2) or "").lower(), 1)152 if "," in raw and "." in raw:153 raw = raw.replace(",", "")154 elif raw.count(",") >= 1:155 raw = raw.replace(",", "")156 try:157 value = float(raw) * mult158 except ValueError:159 return None160 return value if value >= 5_000 else None161162163def price_is_from(label: str) -> bool:164 key = (label or "").lower()165 return any(k in key for k in ("from ", "starting at", "starting from", "priced from"))166167168# ---------------------------------------------------------------------------169# Generic numbers170# ---------------------------------------------------------------------------171172def parse_int(text) -> int | None:173 if text is None:174 return None175 if isinstance(text, (int, float)):176 return int(text)177 m = re.search(r"\d+", str(text))178 return int(m.group()) if m else None179180181def parse_float(text) -> float | None:182 if text is None:183 return None184 if isinstance(text, (int, float)):185 return float(text)186 m = re.search(r"\d[\d,\s]*(?:\.\d+)?", str(text))187 if not m:188 return None189 try:190 return float(m.group().replace(",", "").replace(" ", ""))191 except ValueError:192 return None193194195_SQFT_RE = re.compile(r"([\d,\.\s]+)\s*(?:sq\.?\s*ft|sqft|ft2|ft²|square\s+feet)",196 re.IGNORECASE)197_ACRE_RE = re.compile(r"([\d,\.\s]+)\s*acres?\b", re.IGNORECASE)198199200def parse_area_sqft(text) -> float | None:201 """Living area in sq ft."""202 if text is None:203 return None204 if isinstance(text, (int, float)):205 return float(text) if text > 50 else None206 m = _SQFT_RE.search(str(text))207 if m:208 v = parse_float(m.group(1))209 return round(v) if v and v > 50 else None210 return None211212213def parse_lot_sqft(text) -> float | None:214 """Lot size in sq ft (acres converted: 1 acre = 43,560 sq ft)."""215 if text is None:216 return None217 if isinstance(text, (int, float)):218 return float(text) if text > 100 else None219 t = str(text)220 m = _ACRE_RE.search(t)221 if m:222 v = parse_float(m.group(1))223 return round(v * 43_560) if v and 0.005 <= v <= 50_000 else None224 m = _SQFT_RE.search(t)225 if m:226 v = parse_float(m.group(1))227 return round(v) if v and v > 100 else None228 return None229230231def parse_lot_acres(text) -> float | None:232 sqft = parse_lot_sqft(text)233 return round(sqft / 43_560, 3) if sqft else None234235236_YEAR_RE = re.compile(r"^\s*(1[6-9]\d{2}|20[0-4]\d)\s*(?:$|[(,])")237238239def parse_year(text) -> int | None:240 """Plausible year built (1600-2049); strict on purpose — the value must241 START with the year so stray labels never reach the year_built column."""242 if text is None:243 return None244 if isinstance(text, (int, float)):245 y = int(text)246 return y if 1600 <= y <= 2049 else None247 m = _YEAR_RE.match(str(text))248 return int(m.group(1)) if m else None249250251# ---------------------------------------------------------------------------252# Property type — canonical Home-Ka vocabulary (RESO-informed), shown in the253# frontend filters. Maps both RESO enumerations (PropertyType/PropertySubType)254# and free-text labels seen on brokerage sites.255# ---------------------------------------------------------------------------256257_TYPE_MAP = [258 # (keywords found in the normalized source text, canonical type)259 (("singlefamilyresidence", "single family", "single-family", "detached",260 "sfr", "residential - single", "house"), "Single Family"),261 (("townhouse", "townhome", "town house", "row house", "attached",262 "end unit"), "Townhouse"),263 (("condominium", "condo", "co-op", "coop", "stock cooperative",264 "apartment", "loft", "penthouse", "high rise", "flat"), "Condo"),265 (("duplex", "triplex", "quadruplex", "fourplex", "multi family",266 "multi-family", "multifamily", "income property", "residential income",267 "2 units", "3 units", "4 units"), "Multi-Family"),268 (("manufactured", "mobile home", "manufactured home", "modular",269 "manufacturedhome"), "Manufactured"),270 (("unimproved land", "vacant land", "land", "lot ", "lots and land",271 "acreage"), "Land"),272 (("farm", "ranch", "agricultural", "agriculture", "equestrian",273 "hobby farm"), "Farm/Ranch"),274 (("commercial", "office", "retail", "industrial", "warehouse", "business",275 "mixed use", "hotel", "motel", "commercialsale"), "Commercial"),276 (("cabin", "recreational", "waterfront", "lake house"), "Single Family"),277 (("residential",), "Single Family"), # RESO catch-all, after subtypes278]279280281def normalize_property_type(text: str) -> str:282 import html as _html283 key = strip_accents(_html.unescape(str(text or "")).strip().lower())284 if not key:285 return ""286 for keywords, canon in _TYPE_MAP:287 if any(k in key for k in keywords):288 return canon289 return _html.unescape(str(text)).strip().title()290291292# ---------------------------------------------------------------------------293# Listing status — RESO StandardStatus → canonical Home-Ka statuses294# ---------------------------------------------------------------------------295296_STATUS_MAP = [297 (("coming soon",), "coming-soon"),298 (("pending", "under contract", "active under contract", "contingent",299 "backup", "in contract"), "pending"),300 (("closed", "sold"), "sold"),301 (("withdrawn", "canceled", "cancelled", "delisted", "off market",302 "off-market", "expired", "hold", "temporarily off"), "withdrawn"),303 (("active", "for sale", "new", "price change", "back on market",304 "re-activated", "extended"), "active"),305]306307308def normalize_status(text: str) -> str:309 key = (str(text or "")).strip().lower()310 if not key:311 return "active"312 for keywords, canon in _STATUS_MAP:313 if any(k in key for k in keywords):314 return canon315 return "active"316317318# ---------------------------------------------------------------------------319# Beds / baths from free text320# ---------------------------------------------------------------------------321322_BED_RE = re.compile(r"(\d+)\s*(?:bed(?:room)?s?|br|bd)\b", re.IGNORECASE)323_BATH_RE = re.compile(r"(\d+(?:\.\d+)?)\s*(?:bath(?:room)?s?|ba)\b", re.IGNORECASE)324325326def extract_beds_baths(text: str) -> tuple[int | None, float | None]:327 if not text:328 return None, None329 beds = _BED_RE.search(text)330 baths = _BATH_RE.search(text)331 return (int(beds.group(1)) if beds else None,332 float(baths.group(1)) if baths else None)333