1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""FAKE DEMO — builds `results/runs/_fake_demo/` (predictions, transcripts, packets) from the SYNTHETIC data so that3scripts/20_evaluate.py and scripts/21_figures.py can be exercised end-to-end without any model run.45The run_id starts with `_` and is therefore SKIPPED by the collector of 20_evaluate.py unless `--include-fake` is6given. Nothing produced here is an empirical result.78 python3 scripts/29_fake_demo_predictions.py [--n 600] [--seed 20261027]9"""10from __future__ import annotations1112import argparse13import json14from datetime import date, timedelta15from pathlib import Path1617import numpy as np18import polars as pl19from scipy import stats2021from agentic_valuation.config import RESULTS, SEED, SYNTHETIC22from agentic_valuation.schemas import (AgentEstimate, AgentTurn, Adjustment, Comparable, DebateTranscript, EvidencePacket,23 MarketContext, SubjectProperty)2425ROLES = ["appraisal", "comparables", "hedonic", "economist", "condition", "geospatial", "underwriting", "buyer", "seller",26 "rental", "data_auditor", "skeptic"]27# arm -> (sigma of log error, bias, committee, aggregation, has_intervals, cost, latency, tokens, interval_scale)28ARMS = {29 "A_hedonic": (0.19, 0.00, "none", "model", False, 0.0, 0.001, 0, 1.0),30 "C_gbm": (0.16, 0.00, "none", "model", False, 0.0, 0.002, 0, 1.0),31 "D_avm": (0.145, 0.00, "none", "model", True, 0.0, 0.003, 0, 0.85),32 "E_single_llm": (0.175, 0.02, "mixed_random", "median", True, 0.02, 12.0, 6000, 0.7),33 "F_independent": (0.155, 0.01, "mixed_random", "median", True, 0.24, 15.0, 70000, 0.8),34 "G_debate_r2": (0.148, 0.01, "mixed_random", "median", True, 0.70, 95.0, 210000, 1.0),35 "H_debate_skeptic": (0.150, 0.00, "mixed_random", "median", True, 0.78, 105.0, 230000, 1.05),36 "I_placebo": (0.160, 0.01, "mixed_random", "median", True, 0.70, 95.0, 210000, 0.9),37 "J_cost_matched": (0.158, 0.02, "mixed_random", "median", True, 0.70, 60.0, 210000, 0.75),38}394041def make_predictions(truth: pl.DataFrame, rng: np.random.Generator, run_id: str) -> pl.DataFrame:42 frames = []43 base = truth.select("tx_id", "price").to_pandas()44 common = rng.normal(0, 0.08, len(base)) # shared component → correlated errors across arms (realistic for pairing)45 for arm, (sig, bias, committee, agg, has_q, cost, lat, tok, iscale) in ARMS.items():46 own = rng.normal(0, np.sqrt(max(sig**2 - 0.08**2, 1e-4)), len(base))47 e = common + own + bias48 point = base["price"].to_numpy() * np.exp(e)49 s = sig * iscale50 if has_q:51 z = stats.norm.ppf([0.05, 0.25, 0.5, 0.75, 0.95])52 q = np.exp(np.log(point)[:, None] + s * z[None, :])53 else:54 q = np.full((len(base), 5), np.nan)55 frames.append(pl.DataFrame({56 "tx_id": base["tx_id"].to_numpy(), "split": ["test"] * len(base), "arm": [arm] * len(base),57 "committee": [committee] * len(base), "aggregation": [agg] * len(base), "point": point,58 "q05": q[:, 0], "q25": q[:, 1], "q50": q[:, 2], "q75": q[:, 3], "q95": q[:, 4],59 "cost_usd": np.full(len(base), cost), "latency_s": np.full(len(base), lat) * rng.uniform(0.8, 1.2, len(base)),60 "tokens": np.full(len(base), tok, dtype=np.int64), "run_id": [run_id] * len(base),61 }))62 if arm in {"F_independent", "G_debate_r2"}: # second aggregation rule → exercises disambiguation63 alt = frames[-1].with_columns(pl.lit("mean").alias("aggregation"), (pl.col("point") * 1.01).alias("point"))64 frames.append(alt)65 return pl.concat(frames)666768def make_transcripts(truth: pl.DataFrame, rng: np.random.Generator, out: Path, n: int = 25) -> None:69 (out / "transcripts").mkdir(parents=True, exist_ok=True)70 (out / "packets").mkdir(parents=True, exist_ok=True)71 rows = truth.head(n).to_dicts()72 for r in rows:73 d = r["date"] if isinstance(r["date"], date) else date.fromisoformat(str(r["date"])[:10])74 comps = []75 for k in range(8):76 cp = float(r["price"] * np.exp(rng.normal(0, 0.15)))77 comps.append(Comparable(evidence_id=f"C{k + 1:02d}", sale_date=d - timedelta(days=int(rng.integers(30, 500))), price=cp,78 distance_km=float(rng.uniform(0.1, 3)), property_class=r["property_class"], municipality=r["mun_name"],79 floor_area_m2=r["floor_area_m2"], year_built=r["year_built"], months_before_valuation=float(rng.uniform(1, 17))))80 subj = SubjectProperty(tx_id=r["tx_id"], valuation_date=d, information_cutoff=d - timedelta(days=1), property_class=r["property_class"],81 region=r["region_name"], municipality=r["mun_name"], lat=round(r["lat"], 3), lng=round(r["lng"], 3),82 floor_area_m2=r["floor_area_m2"], year_built=r["year_built"], roll_total_value=r["roll_total_value"])83 packet = EvidencePacket(packet_id=f"pk_{r['tx_id']}", subject=subj, comparables=comps, market=MarketContext(as_of=d - timedelta(days=1)))84 (out / "packets" / f"{r['tx_id']}.json").write_text(packet.model_dump_json(indent=1), encoding="utf-8")85 turns, finals, finalq = [], {}, {}86 med0 = None87 r0 = {}88 for rnd in (0, 1, 2):89 for role in ROLES:90 if rnd == 0:91 est_log = np.log(r["price"]) + rng.normal(0.01, 0.12)92 r0[role] = est_log93 else:94 est_log = r0[role] + 0.4 * (med0 - r0[role]) + rng.normal(0, 0.02) # anchoring toward round-0 median95 p = float(np.exp(est_log))96 adj = [Adjustment(comparable_id=c.evidence_id, adjusted_price=float(c.price * np.exp(rng.normal(0, 0.1))), reason="area/age") for c in comps[:4]]97 cited = ["P", "M1"] + [c.evidence_id for c in comps[:4]]98 if rng.uniform() < 0.1:99 cited.append("C99") # hallucinated id100 if rng.uniform() < 0.05:101 adj[0] = Adjustment(comparable_id="C01", adjusted_price=float(comps[0].price * 1.6), reason="implausible")102 est = AgentEstimate(point_estimate=p, q05=p * 0.82, q25=p * 0.93, q50=p, q75=p * 1.07, q95=p * 1.2, evidence_ids_used=cited,103 adjustments=adj, key_assumptions=["roll ratio stable"], reasoning_summary=" ".join(["word"] * int(rng.integers(40, 140))))104 turns.append(AgentTurn(role=role, round=rnd, kind="estimate" if rnd == 0 else "revision", model="mock", provider="mock",105 estimate=est, input_tokens=5000, output_tokens=600, cost_usd=0.02, latency_s=6.0))106 finals[role], finalq[role] = p, [est.q05, est.q25, est.q50, est.q75, est.q95]107 if rnd == 0:108 med0 = float(np.median(list(r0.values())))109 tr = DebateTranscript(tx_id=r["tx_id"], arm="G_debate_r2", committee="mixed_random", protocol="structured_debate", rounds=2,110 packet_hash=packet.content_hash(), turns=turns, final_estimates=finals, final_quantiles=finalq,111 aggregated={"median": float(np.median(list(finals.values())))}, total_cost_usd=0.7, total_latency_s=95, total_tokens=210000)112 (out / "transcripts" / f"{r['tx_id']}.json").write_text(tr.to_json(), encoding="utf-8")113114115def main() -> None:116 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)117 ap.add_argument("--n", type=int, default=600)118 ap.add_argument("--seed", type=int, default=SEED)119 ap.add_argument("--run-id", default="_fake_demo")120 args = ap.parse_args()121 rng = np.random.default_rng(args.seed)122 syn = pl.read_parquet(SYNTHETIC / "synthetic_transactions.parquet")123 test = syn.filter(pl.col("date") >= date(2025, 4, 1)).filter(pl.col("date") <= date(2025, 12, 31))124 idx = rng.choice(len(test), size=min(args.n, len(test)), replace=False)125 sub = test[sorted(idx.tolist())]126 out = RESULTS / "runs" / args.run_id127 out.mkdir(parents=True, exist_ok=True)128 preds = make_predictions(sub, rng, args.run_id)129 preds.write_parquet(out / "predictions.parquet")130 make_transcripts(sub, rng, out)131 (out / "README.md").write_text("# FAKE DEMO RUN — synthetic data, simulated errors. Not a result. Skipped by scripts/20_evaluate.py.\n")132 (out / "manifest.json").write_text(json.dumps({"fake": True, "seed": args.seed, "n": len(sub), "arms": list(ARMS)}, indent=1))133 print(f"wrote {out} ({preds.shape[0]} prediction rows, {len(list((out / 'transcripts').glob('*.json')))} transcripts)")134135136if __name__ == "__main__":137 main()138