TeX 59%
Python 40.8%
1#!/usr/bin/env python32"""Experiments 1–4, 9 (empirical part), 11 (extraction part), 12, 16 — attribute extraction evaluation.34Reads data/processed/{images,text,props}_<tier>*.parquet and the frame; writes CSV tables to results/tables/ and a5machine-readable results/raw/exp_agreement.json. No API calls.6"""7import json8import sys9from pathlib import Path1011import numpy as np12import pandas as pd1314sys.path.insert(0, str(Path(__file__).resolve().parents[1]))15from src.evaluation.metrics import agreement_table, cohen_kappa, cluster_bootstrap, krippendorff_alpha_nominal # noqa: E40216from src.evaluation.reference import add_reference_columns # noqa: E40217from src.extraction.aggregate import GRADE_ORD, NONINFO # noqa: E4021819ROOT = Path(__file__).resolve().parents[1]20P = ROOT / "data" / "processed"21T = ROOT / "results" / "tables"22R = {}23TIERS = [t for t in ["haiku", "sonnet", "opus"] if (P / f"props_{t}_K10.parquet").exists()]24KCOMMON = 10252627def load_props(tier, k=KCOMMON):28 return pd.read_parquet(P / f"props_{tier}_K{k}.parquet").set_index("uid")293031def load_text(tier):32 f = P / f"text_{tier}.parquet"33 return pd.read_parquet(f).set_index("uid") if f.exists() else None343536smp = add_reference_columns(pd.read_parquet(P / "sample.parquet")).set_index("uid")373839# --------------------------------------------------------------------------- Experiment 1: agreement with reference fields40def exp1():41 rows = []42 for tier in TIERS:43 v = load_props(tier).reindex(smp.index)44 t = load_text(tier)45 pairs = [46 ("cladding_primary ∈ broker siding set", "visual", v["cladding_primary"],47 pd.Series([s if s else None for s in smp["ref_cladding_set"]], index=smp.index), "set"),48 ("roof_covering vs broker roofing", "visual", v["roof_covering"], smp["ref_roof"], "cat"),49 ("basement_finish vs broker basement", "visual", v["basement_finish"], smp["ref_basement"].replace({"no_basement": None}), "cat"),50 ("garage present (visual) where broker reports a garage", "visual",51 v["garage_visible"].map(lambda x: "present" if x in ("attached", "detached", "carport") else ("none" if x == "none_visible" else x)),52 smp["ref_garage"].map(lambda x: None if (x is None or (isinstance(x, float) and np.isnan(x))) else "present"), "cat"),53 ("garage type (attached/detached) vs broker", "visual", v["garage_visible"].where(v["garage_visible"].isin(["attached", "detached"]), "not_visible"),54 smp["ref_garage"].where(smp["ref_garage"].isin(["attached", "detached"]), None), "cat"),55 ("storeys_visible vs roll storeys (±0.5 tolerated)", "visual", v["storeys_visible"], smp["ref_storeys"], "storeys"),56 ("building_type_visual vs listing type", "visual", v["building_type_visual"], smp["ref_building_type"], "cat"),57 ("kitchen_counters vs broker kitchen field", "visual", v["kitchen_counters"], smp["ref_counters"], "cat"),58 ("kitchen_cabinets vs broker kitchen field", "visual", v["kitchen_cabinets"], smp["ref_cabinets"], "cat"),59 ]60 if t is not None:61 t = t.reindex(smp.index)62 pairs += [63 ("TEXT cladding ∈ broker siding set", "text", t["cladding"].map(lambda l: l[0] if isinstance(l, (list, np.ndarray)) and len(l) else None),64 pd.Series([s if s else None for s in smp["ref_cladding_set"]], index=smp.index), "set"),65 ("TEXT roof vs broker roofing", "text", t["roof_covering"], smp["ref_roof"], "cat"),66 ("TEXT basement vs broker basement", "text", t["basement_finish"].replace({"no_basement": None}), smp["ref_basement"].replace({"no_basement": None}), "cat"),67 ("TEXT storeys vs roll storeys (±0.5)", "text", t["storeys"], smp["ref_storeys"], "storeys"),68 ("TEXT year built vs roll year (±1)", "text", t["year_built_claim"].replace({0: None}), smp["roll_year_built"], "year"),69 ("TEXT building footprint × storeys vs roll floor area (±15 %)", "text",70 (t["building_footprint_claim_sqft"].replace({0: np.nan}) / 10.7639) * smp["roll_storeys"].fillna(1).clip(1, 3),71 smp["roll_floor_area_m2"], "area"),72 ]73 for name, channel, pred, ref, kind in pairs:74 if kind == "set":75 inf = ~pred.isin(NONINFO) & pred.notna()76 refok = ref.notna()77 both = inf & refok78 acc = float(np.mean([p in s for p, s in zip(pred[both], ref[both])])) if both.sum() else np.nan79 row = {"n_ref": int(refok.sum()), "coverage": float(inf[refok].mean()) if refok.sum() else np.nan,80 "n_compared": int(both.sum()), "accuracy": acc, "kappa": np.nan, "macro_f1": np.nan}81 elif kind == "storeys":82 num = pred.map({"1": 1, "1.5": 1.5, "2": 2, "2.5": 2.5, "3+": 3, "3": 3})83 refn = ref.map({"1": 1, "1.5": 1.5, "2": 2, "2.5": 2.5, "3+": 3})84 both = num.notna() & refn.notna()85 row = {"n_ref": int(refn.notna().sum()), "coverage": float(num.notna()[refn.notna()].mean()),86 "n_compared": int(both.sum()),87 "accuracy": float(((num - refn).abs() <= 0.5)[both].mean()) if both.sum() else np.nan,88 "accuracy_strict": float((num == refn)[both].mean()) if both.sum() else np.nan,89 "kappa": cohen_kappa(num[both].round().astype(int), refn[both].round().astype(int)) if both.sum() else np.nan,90 "macro_f1": np.nan}91 elif kind == "year":92 p = pd.to_numeric(pred, errors="coerce"); both = p.notna() & ref.notna()93 row = {"n_ref": int(ref.notna().sum()), "coverage": float(p.notna().mean()), "n_compared": int(both.sum()),94 "accuracy": float(((p - ref).abs() <= 1)[both].mean()) if both.sum() else np.nan,95 "mae_years": float((p - ref).abs()[both].median()) if both.sum() else np.nan, "kappa": np.nan, "macro_f1": np.nan}96 elif kind == "area":97 both = pred.notna() & ref.notna()98 ratio = (pred / ref)[both]99 row = {"n_ref": int(ref.notna().sum()), "coverage": float(pred.notna().mean()), "n_compared": int(both.sum()),100 "accuracy": float(((ratio - 1).abs() <= 0.15).mean()) if both.sum() else np.nan,101 "median_ratio": float(ratio.median()) if both.sum() else np.nan,102 "mdape": float((ratio - 1).abs().median() * 100) if both.sum() else np.nan, "kappa": np.nan, "macro_f1": np.nan}103 else:104 row = agreement_table(pred, ref)105 rows.append({"tier": tier, "channel": channel, "attribute": name, **row})106 df = pd.DataFrame(rows)107 df.to_csv(T / "exp1_agreement.csv", index=False)108 R["exp1"] = df.to_dict(orient="records")109 # confusion matrices for the key visual attributes (Sonnet, common K)110 tier = "sonnet" if "sonnet" in TIERS else TIERS[0]111 v = load_props(tier).reindex(smp.index)112 conf = {}113 for name, pred, ref in [("roof", v["roof_covering"], smp["ref_roof"]),114 ("basement", v["basement_finish"], smp["ref_basement"].replace({"no_basement": None})),115 ("storeys", v["storeys_visible"], smp["ref_storeys"]),116 ("garage", v["garage_visible"], smp["ref_garage"])]:117 m = pred.notna() & ref.notna() & ~pred.isin(["not_applicable"])118 ct = pd.crosstab(ref[m], pred[m])119 ct.to_csv(T / f"exp1_confusion_{name}_{tier}.csv")120 conf[name] = ct.to_dict()121 R["exp1_confusion_tier"] = tier122 return df123124125# --------------------------------------------------------------------------- Experiment 2: channel information content126def exp2():127 rows = []128 for tier in TIERS:129 v = load_props(tier).reindex(smp.index)130 t = load_text(tier)131 if t is None:132 continue133 t = t.reindex(smp.index)134 for attr, vcol, tcol in [("building_type", "building_type_visual", "building_type"), ("storeys", "storeys_visible", "storeys"),135 ("cladding", "cladding_primary", None), ("roof_covering", "roof_covering", "roof_covering"),136 ("garage", "garage_visible", "garage"), ("basement_finish", "basement_finish", "basement_finish"),137 ("kitchen_grade", "kitchen_grade", "kitchen_grade"), ("bathroom_grade", "bathroom_grade", "bathroom_grade"),138 ("kitchen_counters", "kitchen_counters", "kitchen_counters"), ("flooring_main", "flooring_main", "flooring_main")]:139 vi = ~v[vcol].isin(NONINFO) & v[vcol].notna()140 if tcol is None:141 ti = t["cladding"].map(lambda l: isinstance(l, (list, np.ndarray)) and len(l) > 0 and l[0] not in NONINFO)142 tv = t["cladding"].map(lambda l: l[0] if isinstance(l, (list, np.ndarray)) and len(l) else None)143 else:144 ti = ~t[tcol].isin(NONINFO) & t[tcol].notna(); tv = t[tcol]145 vv = v[vcol].replace({"3+": "3", "none_visible": "none"})146 both = vi & ti147 agree = float((vv[both] == tv[both]).mean()) if both.sum() else np.nan148 rows.append({"tier": tier, "attribute": attr, "visual_coverage": float(vi.mean()), "text_coverage": float(ti.mean()),149 "either_coverage": float((vi | ti).mean()), "both": int(both.sum()), "agreement_when_both": agree,150 "visual_only_share": float((vi & ~ti).mean()), "text_only_share": float((ti & ~vi).mean())})151 df = pd.DataFrame(rows)152 df.to_csv(T / "exp2_channels.csv", index=False)153 R["exp2"] = df.to_dict(orient="records")154 return df155156157# --------------------------------------------------------------------------- Experiment 3: image count sensitivity158def exp3():159 rows = []160 for tier in TIERS:161 ks = [1, 2, 4, 8, 10, 12] if tier == "haiku" else [1, 2, 4, 8, 10]162 ref = load_props(tier, ks[-1])163 for k in ks:164 p = load_props(tier, k).reindex(ref.index)165 row = {"tier": tier, "K": k, "n": int(p["n_images_ok"].notna().sum()),166 "share_with_kitchen": float((p["n_kitchen"] > 0).mean()), "share_with_bathroom": float((p["n_bathroom"] > 0).mean()),167 "share_with_basement": float((p["n_basement"] > 0).mean()), "share_with_exterior": float((p["n_exterior"] > 0).mean()),168 "grade_available": float(p["grade_composite"].notna().mean()), "condition_available": float(p["condition_composite"].notna().mean()),169 "grade_stable_vs_full": float((p["grade_composite"] == ref["grade_composite"])[ref["grade_composite"].notna()].mean()),170 "condition_mad_vs_full": float((p["condition_composite"] - ref["condition_composite"]).abs().median()),171 "cladding_stable_vs_full": float((p["cladding_primary"] == ref["cladding_primary"])[~ref["cladding_primary"].isin(NONINFO)].mean())}172 rows.append(row)173 df = pd.DataFrame(rows)174 df.to_csv(T / "exp3_image_count.csv", index=False)175 R["exp3"] = df.to_dict(orient="records")176 # photo quality moderator: agreement with broker roofing/siding by photo-quality tercile (Sonnet)177 tier = "sonnet" if "sonnet" in TIERS else TIERS[0]178 v = load_props(tier).reindex(smp.index)179 q = pd.qcut(v["photo_quality_mean"], 3, labels=["low", "mid", "high"])180 rows = []181 for lab in ["low", "mid", "high"]:182 m = q == lab183 rows.append({"tier": tier, "photo_quality_tercile": lab, "n": int(m.sum()),184 "roof_agreement": agreement_table(v.loc[m, "roof_covering"], smp.loc[m, "ref_roof"])["accuracy"],185 "cladding_in_set": float(np.mean([p in s for p, s in zip(v.loc[m, "cladding_primary"], smp.loc[m, "ref_cladding_set"]) if s and p not in NONINFO])) if m.sum() else np.nan,186 "confidence_mean": float(v.loc[m, "confidence_mean"].mean())})187 pd.DataFrame(rows).to_csv(T / "exp3_photo_quality.csv", index=False)188 R["exp3_quality"] = rows189 return df190191192# --------------------------------------------------------------------------- Experiment 4: cross-model agreement193def exp4():194 if len(TIERS) < 2:195 return None196 props = {t: load_props(t) for t in TIERS}197 common = set.intersection(*[set(p.index) for p in props.values()])198 rows = []199 attrs = ["building_type_visual", "storeys_visible", "cladding_primary", "roof_covering", "garage_visible", "basement_finish",200 "kitchen_grade", "bathroom_grade", "kitchen_counters", "kitchen_cabinets", "flooring_main", "renovation_evidence", "grade_composite"]201 for a in attrs:202 lab = np.array([[props[t].loc[u, a] if (props[t].loc[u, a] not in NONINFO) else None for u in sorted(common)] for t in TIERS], dtype=object)203 row = {"attribute": a, "n_listings": len(common), "alpha_nominal": krippendorff_alpha_nominal(lab)}204 for i in range(len(TIERS)):205 for j in range(i + 1, len(TIERS)):206 a1 = pd.Series(lab[i]); a2 = pd.Series(lab[j])207 m = a1.notna() & a2.notna()208 row[f"agree_{TIERS[i]}_{TIERS[j]}"] = float((a1[m] == a2[m]).mean()) if m.sum() else np.nan209 row[f"kappa_{TIERS[i]}_{TIERS[j]}"] = cohen_kappa(a1[m], a2[m]) if m.sum() else np.nan210 row[f"n_{TIERS[i]}_{TIERS[j]}"] = int(m.sum())211 rows.append(row)212 # continuous: condition composite213 for a in ["condition_composite", "exterior_condition", "interior_condition", "photo_quality_mean"]:214 row = {"attribute": a, "n_listings": len(common), "alpha_nominal": np.nan}215 for i in range(len(TIERS)):216 for j in range(i + 1, len(TIERS)):217 x = props[TIERS[i]].loc[sorted(common), a]; y = props[TIERS[j]].loc[sorted(common), a]218 m = x.notna() & y.notna()219 row[f"corr_{TIERS[i]}_{TIERS[j]}"] = float(np.corrcoef(x[m], y[m])[0, 1]) if m.sum() > 3 else np.nan220 row[f"mad_{TIERS[i]}_{TIERS[j]}"] = float((x - y).abs()[m].median()) if m.sum() else np.nan221 row[f"n_{TIERS[i]}_{TIERS[j]}"] = int(m.sum())222 rows.append(row)223 df = pd.DataFrame(rows)224 df.to_csv(T / "exp4_cross_model.csv", index=False)225 R["exp4"] = df.to_dict(orient="records")226 # per-image scene agreement227 imgs = {t: pd.read_parquet(P / f"images_{t}.parquet") for t in TIERS}228 key = ["uid", "image_index"]229 m = imgs[TIERS[0]][key + ["scene", "confidence"]].merge(imgs[TIERS[1]][key + ["scene", "confidence"]], on=key, suffixes=("_a", "_b"))230 R["exp4_scene_agreement"] = {"tiers": TIERS[:2], "n_images": int(len(m)), "agreement": float((m.scene_a == m.scene_b).mean()),231 "kappa": cohen_kappa(m.scene_a, m.scene_b)}232 pd.crosstab(m.scene_a, m.scene_b).to_csv(T / f"exp4_scene_confusion_{TIERS[0]}_{TIERS[1]}.csv")233 return df234235236# --------------------------------------------------------------------------- Experiment 9 (empirical): what sellers do not show237def exp9():238 rows = []239 for tier in TIERS:240 v = load_props(tier).reindex(smp.index)241 for scene, col in [("basement", "n_basement"), ("kitchen", "n_kitchen"), ("bathroom", "n_bathroom"), ("exterior", "n_exterior")]:242 shown = v[col] > 0243 row = {"tier": tier, "scene": scene, "share_shown": float(shown.mean())}244 # association of showing with broker basement claim / age / price / condition245 if scene == "basement":246 rb = smp["ref_basement"]247 for lab in ["finished", "partially_finished", "unfinished"]:248 m = rb == lab249 row[f"shown_given_{lab}"] = float(shown[m].mean()) if m.sum() else np.nan250 row[f"n_{lab}"] = int(m.sum())251 row["shown_median_price"] = float(smp.loc[shown, "price"].median()); row["notshown_median_price"] = float(smp.loc[~shown, "price"].median())252 row["shown_mean_age"] = float(smp.loc[shown, "age_2026"].mean()); row["notshown_mean_age"] = float(smp.loc[~shown, "age_2026"].mean())253 row["shown_cond"] = float(v.loc[shown, "condition_composite"].mean()); row["notshown_cond"] = float(v.loc[~shown, "condition_composite"].mean())254 rows.append(row)255 df = pd.DataFrame(rows)256 df.to_csv(T / "exp9_scene_presence.csv", index=False)257 R["exp9"] = df.to_dict(orient="records")258 # position of first basement / kitchen image in seller order (Haiku, 12 images)259 tier = "haiku" if "haiku" in TIERS else TIERS[0]260 img = pd.read_parquet(P / f"images_{tier}.parquet")261 pos = img[img.ok].groupby(["uid", "scene"])["image_index"].min().reset_index()262 R["exp9_first_position"] = pos.groupby("scene")["image_index"].describe()[["count", "mean", "50%"]].to_dict()263 pos.groupby("scene")["image_index"].describe().to_csv(T / "exp9_first_position.csv")264 return df265266267# --------------------------------------------------------------------------- Experiment 9 (simulation): strategic omission of poor rooms268def exp9_sim(n_draws=40, seed=20261020):269 """Counterfactual MNAR: sellers omit kitchen/bathroom/basement photographs with a probability that falls with the room's270 condition (p = 0.8 at condition 1 → 0.05 at condition 5). We delete such images from the observed set, re-aggregate, and271 measure how the property-level condition, grade and cost indication shift. This bounds the bias from selective272 presentation that is *not* already in the data (which already contains the sellers' actual omissions)."""273 from src.extraction.aggregate import aggregate_images274 import sys275 sys.path.insert(0, str(ROOT))276 from src.cost_model import cost277 rng = np.random.default_rng(seed)278 tier = "sonnet" if "sonnet" in TIERS else TIERS[0]279 img = pd.read_parquet(P / f"images_{tier}.parquet"); img = img[img.ok & img.image_index.lt(KCOMMON)]280 full = aggregate_images(img).set_index("uid")281 lv = pd.read_parquet(P / f"levels_{tier}.parquet"); b2 = lv[lv.level == "B2"].set_index("uid").reindex(full.index)282 base_cost = cost.cost_indication(b2)["V_cost"]283 rows = []284 interior = img.scene.isin(["kitchen", "bathroom", "basement"])285 cond = img["interior_condition"].astype(float).where(img["interior_condition"].astype(float) > 0, 3.0)286 p_drop = np.clip(0.8 - 0.1875 * (cond - 1), 0.05, 0.8) # 0.8 at 1 → 0.05 at 5287 for d in range(n_draws):288 keep = ~(interior & (rng.uniform(size=len(img)) < p_drop))289 agg = aggregate_images(img[keep]).set_index("uid").reindex(full.index)290 b = b2.copy()291 b["grade"] = agg["grade_composite"].fillna("standard"); b["condition"] = agg["condition_composite"].fillna(3.0)292 b["basement"] = agg["basement_finish"].where(~agg["basement_finish"].isin(NONINFO), "unfinished")293 v = cost.cost_indication(b)["V_cost"]294 rows.append({"draw": d, "share_images_dropped": float((~keep).mean()),295 "cond_shift": float((agg["condition_composite"] - full["condition_composite"]).mean()),296 "grade_available": float(agg["grade_composite"].notna().mean()), "grade_available_full": float(full["grade_composite"].notna().mean()),297 "share_grade_up": float((agg["grade_composite"].map(GRADE_ORD) > full["grade_composite"].map(GRADE_ORD)).mean()),298 "share_grade_down": float((agg["grade_composite"].map(GRADE_ORD) < full["grade_composite"].map(GRADE_ORD)).mean()),299 "median_V_ratio": float((v / base_cost).median()), "mean_V_ratio": float((v / base_cost).mean())})300 df = pd.DataFrame(rows); df.to_csv(T / "exp9_simulation_draws.csv", index=False)301 summ = df.drop(columns="draw").mean().to_dict(); summ["n_draws"] = n_draws; summ["tier"] = tier302 pd.DataFrame([summ]).to_csv(T / "exp9_simulation.csv", index=False)303 R["exp9_sim"] = summ304 return df305306307# --------------------------------------------------------------------------- Error taxonomy from evidence strings (roof and cladding disagreements)308def exp1_taxonomy():309 tier = "sonnet" if "sonnet" in TIERS else TIERS[0]310 img = pd.read_parquet(P / f"images_{tier}.parquet"); img = img[img.ok & img.image_index.lt(KCOMMON) & img.scene.str.startswith("exterior", na=False)]311 img = img.merge(smp[["ref_roof", "ref_cladding_set"]], left_on="uid", right_index=True)312 roof = img[img.ref_roof.notna() & ~img.roof_covering.isin(NONINFO)]313 dis = roof[roof.roof_covering != roof.ref_roof]314 cats = {"roof not fully visible / angle": r"not (fully )?visible|partial|angle|cannot see|obscur|hidden|only|edge",315 "snow or weather": r"snow|neige|winter|wet",316 "flat / low-slope confusion": r"flat|low[- ]slope|membrane|gravel",317 "metal vs shingle look-alike": r"metal|standing seam|tôle|tole|steel",318 "distance / resolution": r"distan|far|aerial|drone|small|resolution|blur",319 "other": r"."}320 rows = []321 for name, pat in cats.items():322 m = dis.evidence.fillna("").str.contains(pat, case=False, regex=True)323 rows.append({"attribute": "roof covering", "category": name, "n": int(m.sum()), "share_of_disagreements": float(m.mean()) if len(dis) else np.nan})324 dis = dis[~m] if name != "other" else dis325 ndis = int((roof.roof_covering != roof.ref_roof).sum())326 for r in rows:327 r["share_of_disagreements"] = r["n"] / ndis if ndis else np.nan328 clad = img[img.ref_cladding_set.map(lambda s: bool(s)) & img.cladding.map(lambda l: isinstance(l, (list, np.ndarray)) and len(l) > 0 and l[0] not in NONINFO)]329 cl_dis = clad[[not any(x in s for x in l) for s, l in zip(clad.ref_cladding_set, clad.cladding)]]330 pairs = pd.Series([f"{l[0]} (image) vs {'/'.join(sorted(s))} (broker)" for s, l in zip(cl_dis.ref_cladding_set, cl_dis.cladding)]).value_counts().head(8)331 for k, v in pairs.items():332 rows.append({"attribute": "cladding", "category": k, "n": int(v), "share_of_disagreements": float(v / len(cl_dis)) if len(cl_dis) else np.nan})333 df = pd.DataFrame(rows); df.to_csv(T / "exp1_error_taxonomy.csv", index=False)334 R["exp1_taxonomy"] = {"roof_disagreements": ndis, "roof_compared": int(len(roof)), "cladding_disagreements": int(len(cl_dis)), "cladding_compared": int(len(clad))}335 return df336337338# --------------------------------------------------------------------------- Experiment 12: confidence calibration339def exp12():340 rows = []341 for tier in TIERS:342 img = pd.read_parquet(P / f"images_{tier}.parquet")343 img = img[img.ok & img.image_index.lt(KCOMMON)]344 ext = img[img.scene.isin(["exterior_front", "exterior_rear_or_side", "exterior_aerial_or_street"])].copy()345 ext = ext.merge(smp[["ref_roof", "ref_cladding_set"]], left_on="uid", right_index=True)346 ext["roof_ok"] = np.where(ext.ref_roof.notna() & ~ext.roof_covering.isin(NONINFO), ext.roof_covering == ext.ref_roof, np.nan)347 ext["clad_ok"] = [np.nan if (not s or c in NONINFO or not isinstance(l, (list, np.ndarray)) or len(l) == 0) else float(any(x in s for x in l))348 for s, c, l in zip(ext.ref_cladding_set, ext.cladding.map(lambda l: l[0] if isinstance(l, (list, np.ndarray)) and len(l) else None), ext.cladding)]349 ext["conf_bin"] = pd.cut(ext.confidence.astype(float), [0, .6, .7, .8, .9, 1.0], include_lowest=True)350 g = ext.groupby("conf_bin", observed=True).agg(n=("uid", "size"), conf_mean=("confidence", "mean"),351 roof_acc=("roof_ok", "mean"), roof_n=("roof_ok", "count"),352 clad_acc=("clad_ok", "mean"), clad_n=("clad_ok", "count")).reset_index()353 g["tier"] = tier; g["conf_bin"] = g["conf_bin"].astype(str)354 rows.append(g)355 # abstention: share of not_visible by scene for scene-relevant fields356 df = pd.concat(rows)357 df.to_csv(T / "exp12_calibration.csv", index=False)358 R["exp12"] = df.to_dict(orient="records")359 # selective prediction: accuracy of roof when keeping only images above a confidence threshold360 sel = []361 for tier in TIERS:362 img = pd.read_parquet(P / f"images_{tier}.parquet")363 img = img[img.ok & img.image_index.lt(KCOMMON) & img.scene.str.startswith("exterior", na=False)].merge(smp[["ref_roof"]], left_on="uid", right_index=True)364 img = img[img.ref_roof.notna() & ~img.roof_covering.isin(NONINFO)]365 for thr in [0.0, 0.6, 0.7, 0.8, 0.9]:366 m = img.confidence.astype(float) >= thr367 sel.append({"tier": tier, "threshold": thr, "kept_share": float(m.mean()), "n": int(m.sum()),368 "roof_accuracy": float((img.roof_covering == img.ref_roof)[m].mean()) if m.sum() else np.nan})369 pd.DataFrame(sel).to_csv(T / "exp12_selective.csv", index=False)370 R["exp12_selective"] = sel371 return df372373374# --------------------------------------------------------------------------- Experiment 16: review-cost proxy375def exp16():376 rows = []377 for tier in TIERS:378 v = load_props(tier).reindex(smp.index)379 for thr in [0.7, 0.75, 0.8, 0.85]:380 flag = (v["confidence_mean"] < thr) | v["grade_composite"].isna() | (v["n_defects"] > 0)381 rows.append({"tier": tier, "rule": f"conf<{thr} or no grade or defects", "share_flagged": float(flag.mean()),382 "share_no_grade": float(v["grade_composite"].isna().mean()), "share_defects": float((v["n_defects"] > 0).mean())})383 df = pd.DataFrame(rows)384 df.to_csv(T / "exp16_review_load.csv", index=False)385 R["exp16"] = df.to_dict(orient="records")386 return df387388389def main():390 R["tiers"] = TIERS; R["K_common"] = KCOMMON391 R["n_sample"] = int(len(smp))392 R["reference_coverage"] = {c: float(smp[c].map(lambda x: bool(x) if isinstance(x, set) else x is not None and x == x).mean())393 for c in ["ref_cladding_set", "ref_roof", "ref_basement", "ref_garage", "ref_counters", "ref_cabinets", "ref_storeys", "ref_building_type"]}394 exp1(); exp1_taxonomy(); exp2(); exp3(); exp4(); exp9(); exp9_sim(); exp12(); exp16()395 json.dump(R, open(ROOT / "results" / "raw" / "exp_agreement.json", "w"), indent=1, default=str)396 print(json.dumps({k: v for k, v in R.items() if k in ("tiers", "reference_coverage", "exp4_scene_agreement", "exp9_first_position")}, indent=1, default=str))397 print(pd.read_csv(T / "exp1_agreement.csv").round(3).to_string())398399400if __name__ == "__main__":401 main()402