spb/wp9_uqo Public

1# Author: Simon-Pierre Boucher — contact@spboucher.ai2"""Parsers for the semi-structured raw MLS fields.34The raw ``listings`` table stores every attribute as free text. These5functions harmonise them to numeric analysis variables:67- bedrooms reported as ``"3 + 1"`` (main + lower level) are summed;8- living area is taken from the explicit floor-area measurement9  (``building.floor_area_measurements``), using the upper bound of banded10  entries such as ``"1100-1500 sqft"``, and falls back to11  ``building.size_interior``; square feet are converted at12  1 ft^2 = 0.0929 m^2;13- lot size is recovered from the free-text ``land.size_total`` where a14  numeric value with a recognisable unit (sqft, m2, acres, hectares, or15  frontage x depth in feet) is present.1617NOTE — reconstruction: the original cleaning code lived in the upstream18RE_DB_QC pipeline, which no longer exists. These rules were reverse-engineered19from the description in the paper's data section and validated against the20original sample counts, summary statistics and regression estimates21(see AUDIT.md, "Reproduction verification").22"""23import re2425import numpy as np2627from .config import SQFT_TO_M2, ACRE_TO_M2, HA_TO_M22829_RANGE = re.compile(r"([\d.,]+)\s*-\s*([\d.,]+)")30_NUMBER = re.compile(r"([\d.,]+)")31_UNFORMATTED = re.compile(r'"area_unformatted":\s*"([^"]+)"')32_SIZE_WITH_UNIT = re.compile(r"([\d.,]+)\s*(\w*)")33_LOT_UNIT = re.compile(r"([\d.,]+)\s*(sqft|sq ft|m2|ac|acre|acres|hectare|ha)\b")34_LOT_UNDER = re.compile(r"under\s+([\d/.]+)\s*acre")35_LOT_ACRE_RANGE = re.compile(r"([\d/.]+)\s*-\s*([\d.]+)\s*acres")36_LOT_DIMS = re.compile(r"(\d+(?:\.\d+)?)\s*x\s*(\d+(?:\.\d+)?)")373839def _to_float(text: str) -> float:40    return float(text.replace(",", ""))414243def parse_bedrooms(value) -> float:44    """Sum the integer components of a bedroom string ('3 + 1' -> 4)."""45    if not isinstance(value, str):46        return np.nan47    parts = re.findall(r"\d+", value)48    return float(sum(int(p) for p in parts)) if parts else np.nan495051def parse_count(value) -> float:52    """Parse a plain numeric count field (bathrooms, storeys, parking)."""53    try:54        return float(value)55    except (TypeError, ValueError):56        return np.nan575859def parse_floor_area(value) -> float:60    """Living area in m^2 from ``building.floor_area_measurements``.6162    Banded entries ("1100-1500 sqft") are mapped to the upper bound of the63    band; exact entries ("1698 sqft") are used as reported.64    """65    if not isinstance(value, str):66        return np.nan67    match = _UNFORMATTED.search(value)68    if not match:69        return np.nan70    text = match.group(1)71    band = _RANGE.match(text)72    if band:73        area = _to_float(band.group(2))74    else:75        number = _NUMBER.match(text)76        if not number:77            return np.nan78        area = _to_float(number.group(1))79    return area if "m2" in text else area * SQFT_TO_M2808182def parse_size_interior(value) -> float:83    """Living area in m^2 from ``building.size_interior`` ('102.19 m2', '1698 sqft')."""84    if not isinstance(value, str):85        return np.nan86    match = _SIZE_WITH_UNIT.match(value)87    if not match:88        return np.nan89    area = _to_float(match.group(1))90    unit = match.group(2).lower()91    return area if unit == "m2" else area * SQFT_TO_M2  # unitless values are sqft929394def parse_lot(value) -> float:95    """Lot area in m^2 from the free-text ``land.size_total`` field.9697    Recognises '<n> sqft|m2|ac|acres|hectare|ha', 'under <x> acre(s)',98    '<a> - <b> acres' (upper bound), and '<w> x <d>' frontage-by-depth in99    feet. Returns NaN when no numeric value can be recovered.100    """101    if not isinstance(value, str):102        return np.nan103    text = value.lower().strip()104    match = _LOT_UNIT.match(text)105    if match:106        size = _to_float(match.group(1))107        unit = match.group(2)108        if unit in ("sqft", "sq ft"):109            return size * SQFT_TO_M2110        if unit == "m2":111            return size112        if unit.startswith("ac"):113            return size * ACRE_TO_M2114        return size * HA_TO_M2115    match = _LOT_UNDER.match(text)116    if match:117        frac = match.group(1)118        if "/" in frac:119            num, den = frac.split("/", 1)120            acres = float(num) / float(den)121        else:122            acres = float(frac)123        return acres * ACRE_TO_M2124    match = _LOT_ACRE_RANGE.match(text)125    if match:126        return float(match.group(2)) * ACRE_TO_M2127    match = _LOT_DIMS.match(text)128    if match:129        return float(match.group(1)) * float(match.group(2)) * SQFT_TO_M2130    return np.nan131132133# Consolidation of raw dwelling types into the eight groups used in the paper134BUILDING_TYPE_MAP = {135    "House": "House",136    "Apartment": "Apartment",137    "Row / Townhouse": "Row/Townhouse",138    "Duplex": "Duplex",139    "Triplex": "Triplex",140    "Fourplex": "Fourplex",141    "Manufactured Home": "Manufactured",142    "Manufactured Home/Mobile": "Manufactured",143    "Mobile Home": "Manufactured",144    "Park Model Mobile Home": "Manufactured",145}146147148def consolidate_building_type(value) -> str:149    """Map the raw ``building.type`` to the paper's eight dwelling-type groups."""150    return BUILDING_TYPE_MAP.get(value, "Other")151152153def consolidate_ownership(value) -> str:154    """Map the raw ``ownership.type`` to the paper's ownership-form groups."""155    if not isinstance(value, str):156        return "Unknown"157    text = value.lower()158    if "lease" in text:159        return "Leasehold"160    if "condo" in text or "strata" in text:161        return "Condo/Strata"162    if "freehold" in text:163        return "Freehold"164    if "co-op" in text or "cooperative" in text or "co-ownership" in text:165        return "Co-op/Co-ownership"166    return "Other"167