TeX 59%
Python 40.8%
1"""Deterministic post-processing of the raw extraction results.23parse_results(name) → per-image (or per-listing for text) DataFrame from results/raw/batches/<name>.results.jsonl4aggregate_images(df, K) → one row per listing: cross-image aggregation rules (documented below), optionally truncated5 to the first K images in seller order6reconcile(visual, text, admin) → B-level property tables (B0…B4) with per-attribute evidence status78Aggregation rules (fixed before results were seen; docs/experiment_plan.md):9 - Exterior attributes (building type, storeys, cladding, roof, garage, exterior condition) use exterior scenes only.10 - Categorical: confidence-weighted plurality vote among informative values (not 'not_visible'/'not_applicable');11 if no informative vote → 'not_visible'. Garage: any attached/detached/carport vote beats 'none_visible'.12 - Ordinal grades (kitchen, bathroom): weighted median of ordinal codes among informative values.13 - Conditions and finish quality: confidence-weighted mean of non-zero scores.14 - Renovation evidence: maximum of (none_apparent < partial < extensive), 'unsure' ignored.15 - Defects: union excluding 'none_visible'. Photo quality: mean. Staging share: share of interior images not 'none_apparent'.16 - Composite condition = mean(exterior_condition, interior_condition) over those available.17 - Composite finish grade (for the cost schedule) = weighted ordinal 0.5 kitchen + 0.3 bathroom + 0.2 (interior finish18 quality rescaled to 1–4), rounded to {basic, standard, upgraded, luxury}; None when no interior grade is visible.19"""20from __future__ import annotations2122import json23from collections import Counter, defaultdict24from pathlib import Path2526import numpy as np27import pandas as pd2829ROOT = Path(__file__).resolve().parents[2]30RAW = ROOT / "results" / "raw" / "batches"3132EXTERIOR = {"exterior_front", "exterior_rear_or_side", "exterior_aerial_or_street"}33INTERIOR = {"kitchen", "bathroom", "living_or_dining", "bedroom", "basement"}34NONINFO = {"not_visible", "not_applicable", "not_stated", None}35GRADE_ORD = {"basic": 1, "standard": 2, "upgraded": 3, "luxury": 4}36ORD_GRADE = {v: k for k, v in GRADE_ORD.items()}37RENO_ORD = {"none_apparent": 0, "partial": 1, "extensive": 2}383940def parse_cid(cid: str) -> tuple[str, str, str]:41 tier, rest = cid.split("-", 1)42 uid, idx = rest.rsplit("-", 1)43 return tier, uid.replace("__", ":"), idx444546def parse_results(name: str) -> pd.DataFrame:47 f = RAW / f"{name}.results.jsonl"48 rows = []49 for line in open(f):50 r = json.loads(line)51 tier, uid, idx = parse_cid(r["custom_id"])52 base = {"tier": tier, "uid": uid, "image_index": None if idx == "T" else int(idx), "ok": r["type"] == "succeeded" and r.get("data") is not None,53 "error": r.get("error"), "error_message": r.get("error_message"), "stop_reason": r.get("stop_reason"),54 "input_tokens": (r.get("usage") or {}).get("input_tokens"), "output_tokens": (r.get("usage") or {}).get("output_tokens")}55 if r.get("data"):56 base.update(r["data"])57 rows.append(base)58 df = pd.DataFrame(rows)59 return df606162def _wvote(values, weights, exclude=NONINFO):63 c = defaultdict(float)64 for v, w in zip(values, weights):65 if v in exclude:66 continue67 c[v] += max(w, 0.05)68 if not c:69 return None, 0.070 v, w = max(c.items(), key=lambda kv: kv[1])71 return v, w / sum(c.values())727374def _wmedian_ord(values, weights, ordmap):75 pairs = [(ordmap[v], max(w, 0.05)) for v, w in zip(values, weights) if v in ordmap]76 if not pairs:77 return None78 pairs.sort()79 tot = sum(w for _, w in pairs)80 acc = 0.081 for v, w in pairs:82 acc += w83 if acc >= tot / 2:84 return v85 return pairs[-1][0]868788def _wmean(values, weights):89 pairs = [(float(v), max(w, 0.05)) for v, w in zip(values, weights) if v is not None and not pd.isna(v) and float(v) > 0]90 if not pairs:91 return np.nan92 return sum(v * w for v, w in pairs) / sum(w for _, w in pairs)939495def aggregate_images(img: pd.DataFrame, K: int | None = None) -> pd.DataFrame:96 df = img[img["ok"]].copy()97 if K is not None:98 df = df[df["image_index"] < K]99 out = []100 for uid, g in df.groupby("uid"):101 conf = g["confidence"].fillna(0.5).astype(float).tolist()102 ext = g[g["scene"].isin(EXTERIOR)]103 kit = g[g["scene"] == "kitchen"]104 bath = g[g["scene"] == "bathroom"]105 bsm = g[g["scene"] == "basement"]106 liv = g[g["scene"].isin({"living_or_dining", "bedroom", "kitchen"})]107 inte = g[g["scene"].isin(INTERIOR)]108 ec = ext["confidence"].fillna(0.5).astype(float).tolist()109 r = {"uid": uid, "n_images_ok": len(g), "n_exterior": len(ext), "n_kitchen": len(kit), "n_bathroom": len(bath),110 "n_basement": len(bsm), "n_living": len(liv), "n_unusable": int((g["scene"] == "unusable").sum()),111 "n_floorplan": int((g["scene"] == "floor_plan_or_document").sum()),112 "photo_quality_mean": g["photo_quality"].astype(float).mean(),113 "staging_share": float((inte["staging_or_rendering"] != "none_apparent").mean()) if len(inte) else np.nan,114 "confidence_mean": float(np.mean(conf)) if conf else np.nan}115 for col in ["building_type_visual", "storeys_visible", "roof_form", "roof_covering"]:116 v, s = _wvote(ext[col].tolist(), ec)117 r[col], r[col + "_share"] = (v or "not_visible"), s118 # cladding: votes over list elements119 cl_vals, cl_w = [], []120 for lst, w in zip(ext["cladding"].tolist(), ec):121 for m in (lst if isinstance(lst, (list, np.ndarray)) else []):122 cl_vals.append(m); cl_w.append(w)123 v, s = _wvote(cl_vals, cl_w)124 r["cladding_primary"], r["cladding_primary_share"] = (v or "not_visible"), s125 cnt = Counter(m for m in cl_vals if m not in NONINFO)126 r["cladding_set"] = sorted(cnt)127 # garage128 gv = ext["garage_visible"].tolist()129 pos = [x for x in gv if x in ("attached", "detached", "carport")]130 r["garage_visible"] = Counter(pos).most_common(1)[0][0] if pos else ("none_visible" if "none_visible" in gv else "not_visible")131 r["exterior_condition"] = _wmean(ext["exterior_condition"].tolist(), ec)132 r["interior_condition"] = _wmean(inte["interior_condition"].tolist(), inte["confidence"].fillna(0.5).tolist())133 r["interior_finish_quality"] = _wmean(inte["interior_finish_quality"].tolist(), inte["confidence"].fillna(0.5).tolist())134 kg = _wmedian_ord(kit["kitchen_grade"].tolist(), kit["confidence"].fillna(0.5).tolist(), GRADE_ORD)135 bg = _wmedian_ord(bath["bathroom_grade"].tolist(), bath["confidence"].fillna(0.5).tolist(), GRADE_ORD)136 r["kitchen_grade"] = ORD_GRADE.get(kg, "not_visible")137 r["bathroom_grade"] = ORD_GRADE.get(bg, "not_visible")138 for col, sub in [("kitchen_counters", kit), ("kitchen_cabinets", kit), ("basement_finish", bsm), ("flooring_main", liv)]:139 v, s = _wvote(sub[col].tolist(), sub["confidence"].fillna(0.5).tolist())140 r[col], r[col + "_share"] = (v or "not_visible"), s141 ren = [RENO_ORD[x] for x in g["renovation_evidence"].tolist() if x in RENO_ORD]142 r["renovation_evidence"] = {0: "none_apparent", 1: "partial", 2: "extensive"}[max(ren)] if ren else "unsure"143 defs = set()144 for lst in g["visible_defects"].tolist():145 for d in (lst if isinstance(lst, (list, np.ndarray)) else []):146 if d != "none_visible":147 defs.add(d)148 r["defects"] = sorted(defs)149 r["n_defects"] = len(defs)150 conds = [c for c in (r["exterior_condition"], r["interior_condition"]) if not pd.isna(c)]151 r["condition_composite"] = float(np.mean(conds)) if conds else np.nan152 # composite finish grade153 parts, wts = [], []154 if kg: parts.append(kg); wts.append(0.5)155 if bg: parts.append(bg); wts.append(0.3)156 if not pd.isna(r["interior_finish_quality"]):157 parts.append(1 + 3 * (r["interior_finish_quality"] - 1) / 4); wts.append(0.2)158 r["grade_score"] = float(np.average(parts, weights=wts)) if parts else np.nan159 r["grade_composite"] = ORD_GRADE[int(min(4, max(1, round(r["grade_score"]))))] if parts else None160 out.append(r)161 return pd.DataFrame(out)162163164# ------------------------------------------------------------------------------------------------ reconciliation165def _status(vis, txt):166 v_inf = vis not in NONINFO167 t_inf = txt not in NONINFO168 if v_inf and t_inf:169 return "agree" if vis == txt else "contradict"170 if v_inf:171 return "visual_only"172 if t_inf:173 return "text_only"174 return "unknown"175176177def build_levels(frame: pd.DataFrame, vis: pd.DataFrame | None, txt: pd.DataFrame | None) -> dict[str, pd.DataFrame]:178 """Return {'B0','B1','B2','B3','B4'} property tables with the columns required by src/cost_model/cost.py."""179 base = frame.set_index("uid")180 attached_from_type = base["property_type"].map({"Maison": "detached", "Détachée": "detached", "Jumelé": "semi_detached",181 "Maison de ville": "row_or_townhouse"}).fillna("detached")182 roll_storeys = base["roll_storeys"].map(lambda x: {1.0: "1", 1.5: "1.5", 2.0: "2", 2.5: "2.5", 3.0: "3"}.get(float(x) if x == x else 1.0, "1"))183 common = pd.DataFrame({"floor_area_m2": base["roll_floor_area_m2"], "age": base["age_2026"], "cma": base["cma"],184 "roll_land_value": base["roll_land_value"], "roll_market_date": base["roll_market_date"],185 "roll_building_value": base["roll_building_value"], "price": base["price"],186 "weight": base["weight"] if "weight" in base else 1.0, "source": base["source"],187 "age_band": base["age_band"].astype(str), "price_tier": base["price_tier"].astype(str),188 "roll_municipalite": base["roll_municipalite"]})189 levels = {}190 # B0 — administrative fields + typical assumptions191 b0 = common.copy()192 b0["storeys"] = roll_storeys; b0["attached"] = attached_from_type193 b0["grade"] = "standard"; b0["condition"] = 3.0; b0["renovation"] = "none_apparent"194 b0["basement"] = "unfinished"; b0["garage"] = "none"; b0["pool"] = "none"195 levels["B0"] = b0196 if txt is not None:197 t = txt.set_index("uid").reindex(base.index)198 b1 = b0.copy()199 tg = t["kitchen_grade"].where(~t["kitchen_grade"].isin(NONINFO))200 bgd = t["bathroom_grade"].where(~t["bathroom_grade"].isin(NONINFO))201 g = pd.concat([tg.map(GRADE_ORD), bgd.map(GRADE_ORD)], axis=1).mean(axis=1)202 b1["grade"] = g.round().map(ORD_GRADE).fillna("standard")203 b1["condition"] = t["condition_claim"].map({"needs_work": 2.0, "average": 3.0, "well_maintained": 3.5,204 "renovated_or_like_new": 4.5, "new_construction": 5.0}).fillna(3.0)205 b1["renovation"] = t["renovation_claim"].map({"none": "none_apparent", "partial": "partial", "extensive": "extensive"}).fillna("none_apparent")206 b1["basement"] = t["basement_finish"].map({"finished": "finished", "partially_finished": "partially_finished",207 "unfinished": "unfinished", "no_basement": "unfinished"}).fillna("unfinished")208 b1["garage"] = t["garage"].map({"attached": "attached", "detached": "detached", "carport": "carport", "none": "none"}).fillna("none")209 b1["pool"] = t["pool"].where(~t["pool"].isin(NONINFO), "none")210 b1["storeys"] = t["storeys"].where(~t["storeys"].isin(NONINFO), roll_storeys)211 b1["attached"] = t["building_type"].where(~t["building_type"].isin(NONINFO), attached_from_type)212 levels["B1"] = b1213 if vis is not None:214 v = vis.set_index("uid").reindex(base.index)215 b2 = b0.copy()216 b2["grade"] = v["grade_composite"].fillna("standard")217 b2["grade_visible"] = v["grade_composite"].notna()218 b2["condition"] = v["condition_composite"].fillna(3.0)219 b2["condition_visible"] = v["condition_composite"].notna()220 b2["renovation"] = v["renovation_evidence"].replace({"unsure": "none_apparent"}).fillna("none_apparent")221 b2["basement"] = v["basement_finish"].where(~v["basement_finish"].isin(NONINFO), "unfinished")222 b2["garage"] = v["garage_visible"].map({"attached": "attached", "detached": "detached", "carport": "carport"}).fillna("none")223 b2["storeys"] = v["storeys_visible"].map({"1": "1", "1.5": "1.5", "2": "2", "2.5": "2.5", "3+": "3"}).fillna(roll_storeys)224 b2["attached"] = v["building_type_visual"].where(v["building_type_visual"].isin(["detached", "semi_detached", "row_or_townhouse"]), attached_from_type)225 levels["B2"] = b2226 if vis is not None and txt is not None:227 b3 = b2.copy()228 # visual first, text fills gaps; contradictions recorded229 for col, vcol, tcol in [("basement", "basement_finish", "basement_finish"), ("garage", "garage_visible", "garage"),230 ("storeys", "storeys_visible", "storeys"), ("attached", "building_type_visual", "building_type")]:231 vv = v[vcol].replace({"3+": "3", "none_visible": "none"})232 tt = t[tcol].replace({"no_basement": "unfinished"})233 st = [_status(a, b) for a, b in zip(vv, tt)]234 b3[col + "_status"] = st235 b3[col] = np.where(~vv.isin(NONINFO), vv, np.where(~tt.isin(NONINFO), tt, b3[col]))236 # grade: visual composite, else text grade; condition: visual, else text claim237 b3["grade"] = v["grade_composite"].fillna(b1["grade"])238 b3["condition"] = v["condition_composite"].fillna(b1["condition"])239 b3["renovation"] = np.where(v["renovation_evidence"].isin(["partial", "extensive"]), v["renovation_evidence"], b1["renovation"])240 b3["pool"] = b1["pool"]241 b3["n_contradictions"] = sum((b3[c + "_status"] == "contradict").astype(int) for c in ["basement", "garage", "storeys", "attached"])242 levels["B3"] = b3243 # B4: administrative override (storeys from roll; attached from listing type when visual not visible) + flags for MC widening244 b4 = b3.copy()245 b4["storeys"] = roll_storeys246 b4["storeys_status"] = np.where(v["storeys_visible"].replace({"3+": "3"}) == roll_storeys, "agree",247 np.where(v["storeys_visible"].isin(NONINFO), "admin_only", "contradict"))248 b4["uncertain_grade"] = v["grade_composite"].isna()249 b4["uncertain_condition"] = v["condition_composite"].isna()250 levels["B4"] = b4251 return levels252