# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca """FAKE DEMO — builds `results/runs/_fake_demo/` (predictions, transcripts, packets) from the SYNTHETIC data so that scripts/20_evaluate.py and scripts/21_figures.py can be exercised end-to-end without any model run. The run_id starts with `_` and is therefore SKIPPED by the collector of 20_evaluate.py unless `--include-fake` is given. Nothing produced here is an empirical result. python3 scripts/29_fake_demo_predictions.py [--n 600] [--seed 20261027] """ from __future__ import annotations import argparse import json from datetime import date, timedelta from pathlib import Path import numpy as np import polars as pl from scipy import stats from agentic_valuation.config import RESULTS, SEED, SYNTHETIC from agentic_valuation.schemas import (AgentEstimate, AgentTurn, Adjustment, Comparable, DebateTranscript, EvidencePacket, MarketContext, SubjectProperty) ROLES = ["appraisal", "comparables", "hedonic", "economist", "condition", "geospatial", "underwriting", "buyer", "seller", "rental", "data_auditor", "skeptic"] # arm -> (sigma of log error, bias, committee, aggregation, has_intervals, cost, latency, tokens, interval_scale) ARMS = { "A_hedonic": (0.19, 0.00, "none", "model", False, 0.0, 0.001, 0, 1.0), "C_gbm": (0.16, 0.00, "none", "model", False, 0.0, 0.002, 0, 1.0), "D_avm": (0.145, 0.00, "none", "model", True, 0.0, 0.003, 0, 0.85), "E_single_llm": (0.175, 0.02, "mixed_random", "median", True, 0.02, 12.0, 6000, 0.7), "F_independent": (0.155, 0.01, "mixed_random", "median", True, 0.24, 15.0, 70000, 0.8), "G_debate_r2": (0.148, 0.01, "mixed_random", "median", True, 0.70, 95.0, 210000, 1.0), "H_debate_skeptic": (0.150, 0.00, "mixed_random", "median", True, 0.78, 105.0, 230000, 1.05), "I_placebo": (0.160, 0.01, "mixed_random", "median", True, 0.70, 95.0, 210000, 0.9), "J_cost_matched": (0.158, 0.02, "mixed_random", "median", True, 0.70, 60.0, 210000, 0.75), } def make_predictions(truth: pl.DataFrame, rng: np.random.Generator, run_id: str) -> pl.DataFrame: frames = [] base = truth.select("tx_id", "price").to_pandas() common = rng.normal(0, 0.08, len(base)) # shared component → correlated errors across arms (realistic for pairing) for arm, (sig, bias, committee, agg, has_q, cost, lat, tok, iscale) in ARMS.items(): own = rng.normal(0, np.sqrt(max(sig**2 - 0.08**2, 1e-4)), len(base)) e = common + own + bias point = base["price"].to_numpy() * np.exp(e) s = sig * iscale if has_q: z = stats.norm.ppf([0.05, 0.25, 0.5, 0.75, 0.95]) q = np.exp(np.log(point)[:, None] + s * z[None, :]) else: q = np.full((len(base), 5), np.nan) frames.append(pl.DataFrame({ "tx_id": base["tx_id"].to_numpy(), "split": ["test"] * len(base), "arm": [arm] * len(base), "committee": [committee] * len(base), "aggregation": [agg] * len(base), "point": point, "q05": q[:, 0], "q25": q[:, 1], "q50": q[:, 2], "q75": q[:, 3], "q95": q[:, 4], "cost_usd": np.full(len(base), cost), "latency_s": np.full(len(base), lat) * rng.uniform(0.8, 1.2, len(base)), "tokens": np.full(len(base), tok, dtype=np.int64), "run_id": [run_id] * len(base), })) if arm in {"F_independent", "G_debate_r2"}: # second aggregation rule → exercises disambiguation alt = frames[-1].with_columns(pl.lit("mean").alias("aggregation"), (pl.col("point") * 1.01).alias("point")) frames.append(alt) return pl.concat(frames) def make_transcripts(truth: pl.DataFrame, rng: np.random.Generator, out: Path, n: int = 25) -> None: (out / "transcripts").mkdir(parents=True, exist_ok=True) (out / "packets").mkdir(parents=True, exist_ok=True) rows = truth.head(n).to_dicts() for r in rows: d = r["date"] if isinstance(r["date"], date) else date.fromisoformat(str(r["date"])[:10]) comps = [] for k in range(8): cp = float(r["price"] * np.exp(rng.normal(0, 0.15))) comps.append(Comparable(evidence_id=f"C{k + 1:02d}", sale_date=d - timedelta(days=int(rng.integers(30, 500))), price=cp, distance_km=float(rng.uniform(0.1, 3)), property_class=r["property_class"], municipality=r["mun_name"], floor_area_m2=r["floor_area_m2"], year_built=r["year_built"], months_before_valuation=float(rng.uniform(1, 17)))) subj = SubjectProperty(tx_id=r["tx_id"], valuation_date=d, information_cutoff=d - timedelta(days=1), property_class=r["property_class"], region=r["region_name"], municipality=r["mun_name"], lat=round(r["lat"], 3), lng=round(r["lng"], 3), floor_area_m2=r["floor_area_m2"], year_built=r["year_built"], roll_total_value=r["roll_total_value"]) packet = EvidencePacket(packet_id=f"pk_{r['tx_id']}", subject=subj, comparables=comps, market=MarketContext(as_of=d - timedelta(days=1))) (out / "packets" / f"{r['tx_id']}.json").write_text(packet.model_dump_json(indent=1), encoding="utf-8") turns, finals, finalq = [], {}, {} med0 = None r0 = {} for rnd in (0, 1, 2): for role in ROLES: if rnd == 0: est_log = np.log(r["price"]) + rng.normal(0.01, 0.12) r0[role] = est_log else: est_log = r0[role] + 0.4 * (med0 - r0[role]) + rng.normal(0, 0.02) # anchoring toward round-0 median p = float(np.exp(est_log)) adj = [Adjustment(comparable_id=c.evidence_id, adjusted_price=float(c.price * np.exp(rng.normal(0, 0.1))), reason="area/age") for c in comps[:4]] cited = ["P", "M1"] + [c.evidence_id for c in comps[:4]] if rng.uniform() < 0.1: cited.append("C99") # hallucinated id if rng.uniform() < 0.05: adj[0] = Adjustment(comparable_id="C01", adjusted_price=float(comps[0].price * 1.6), reason="implausible") est = AgentEstimate(point_estimate=p, q05=p * 0.82, q25=p * 0.93, q50=p, q75=p * 1.07, q95=p * 1.2, evidence_ids_used=cited, adjustments=adj, key_assumptions=["roll ratio stable"], reasoning_summary=" ".join(["word"] * int(rng.integers(40, 140)))) turns.append(AgentTurn(role=role, round=rnd, kind="estimate" if rnd == 0 else "revision", model="mock", provider="mock", estimate=est, input_tokens=5000, output_tokens=600, cost_usd=0.02, latency_s=6.0)) finals[role], finalq[role] = p, [est.q05, est.q25, est.q50, est.q75, est.q95] if rnd == 0: med0 = float(np.median(list(r0.values()))) tr = DebateTranscript(tx_id=r["tx_id"], arm="G_debate_r2", committee="mixed_random", protocol="structured_debate", rounds=2, packet_hash=packet.content_hash(), turns=turns, final_estimates=finals, final_quantiles=finalq, aggregated={"median": float(np.median(list(finals.values())))}, total_cost_usd=0.7, total_latency_s=95, total_tokens=210000) (out / "transcripts" / f"{r['tx_id']}.json").write_text(tr.to_json(), encoding="utf-8") def main() -> None: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--n", type=int, default=600) ap.add_argument("--seed", type=int, default=SEED) ap.add_argument("--run-id", default="_fake_demo") args = ap.parse_args() rng = np.random.default_rng(args.seed) syn = pl.read_parquet(SYNTHETIC / "synthetic_transactions.parquet") test = syn.filter(pl.col("date") >= date(2025, 4, 1)).filter(pl.col("date") <= date(2025, 12, 31)) idx = rng.choice(len(test), size=min(args.n, len(test)), replace=False) sub = test[sorted(idx.tolist())] out = RESULTS / "runs" / args.run_id out.mkdir(parents=True, exist_ok=True) preds = make_predictions(sub, rng, args.run_id) preds.write_parquet(out / "predictions.parquet") make_transcripts(sub, rng, out) (out / "README.md").write_text("# FAKE DEMO RUN — synthetic data, simulated errors. Not a result. Skipped by scripts/20_evaluate.py.\n") (out / "manifest.json").write_text(json.dumps({"fake": True, "seed": args.seed, "n": len(sub), "arms": list(ARMS)}, indent=1)) print(f"wrote {out} ({preds.shape[0]} prediction rows, {len(list((out / 'transcripts').glob('*.json')))} transcripts)") if __name__ == "__main__": main()