1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Evidence packets: construction from the analytic row, least-privilege role views, user-prompt rendering3and the deterministic evidence-verification tool (arm H, faithfulness audit).45Invariants6----------7* A packet never contains the subject's sale price, street address or any identifier other than ``tx_id``.8* Coordinates are rounded to 3 decimals (~100 m).9* ``missing_fields`` lists the subject attributes that are null so that agents keep them missing.10* The user prompt embeds ``<evidence_packet>{json}</evidence_packet>`` (parsed by the mock provider) and,11 in deliberation rounds, ``<committee_state>{json}</committee_state>`` with ``committee_median``.12"""13from __future__ import annotations1415import hashlib16import json17import math18from datetime import date, timedelta19from typing import Any2021from ..schemas import AgentEstimate, Comparable, EvidencePacket, GeoContext, MarketContext, SubjectProperty22from .roles import SECTIONS, Role2324FORBIDDEN_KEYS = {"price", "log_price", "street", "address", "civic_number", "civic", "owner", "buyer", "seller", "remarks", "listing_remarks", "parcel_id"}25_SUBJECT_FIELDS = (26 "property_class", "subtype", "borough", "floor_area_m2", "land_area_m2", "year_built", "storeys", "n_units",27 "roll_land_value", "roll_building_value", "roll_total_value", "roll_reference_date", "prior_sale_price", "prior_sale_date", "new_construction",28)29_MISSING_CHECK = ("floor_area_m2", "land_area_m2", "year_built", "storeys", "n_units", "roll_land_value", "roll_building_value", "roll_total_value", "subtype")3031# ---------------------------------------------------------------------------------------------------32# task instructions (stable strings; part of the freeze through the code hash)33# ---------------------------------------------------------------------------------------------------34ESTIMATE_INSTRUCTION = "TASK: Return an AgentEstimate JSON object with your independent estimate of the market value in CAD as of the valuation date, using only the sections visible to you."35CRITIQUE_INSTRUCTION = "TASK: Return a CritiqueSet JSON object. For each committee member in committee_state.estimates (other than yourself), state whether you agree with its estimate, the evidence-based points that support or contradict it (cite ids), and a suggested value only when you can justify it. Do not critique yourself."36REVISION_INSTRUCTION = "TASK: Return a revised AgentEstimate JSON object. Consider the committee_state (estimates of the other members, the committee median, the critiques addressed to you and any verification flags). Change your estimate only for evidential reasons; otherwise keep it and explain why in confidence_rationale."37SKEPTIC_VERDICT_INSTRUCTION = "TASK: You are a blinded reviewer: you see the packet and the committee's final estimates only, not the discussion. Return an AgentEstimate JSON object with your own independent verdict on the market value, and use counterarguments to state the strongest reasons the committee could be wrong."38SELF_CRITIQUE_INSTRUCTION = "TASK: Return a CritiqueSet JSON object with exactly one Critique whose target_role is your own role: review your own previous estimate shown in committee_state for arithmetic errors, unsupported assumptions, missing evidence and over-confidence."39SELF_REFINE_INSTRUCTION = "TASK: Return a refined AgentEstimate JSON object that addresses your own critique shown in committee_state. Keep the estimate if the critique found nothing substantive."4041_INSTRUCTIONS = {42 "estimate": ESTIMATE_INSTRUCTION,43 "critique": CRITIQUE_INSTRUCTION,44 "revision": REVISION_INSTRUCTION,45 "skeptic_verdict": SKEPTIC_VERDICT_INSTRUCTION,46 "self_critique": SELF_CRITIQUE_INSTRUCTION,47 "self_refine": SELF_REFINE_INSTRUCTION,48}495051def _f(v: Any) -> float | None:52 if v is None:53 return None54 try:55 x = float(v)56 except (TypeError, ValueError):57 return None58 return None if math.isnan(x) else x596061def _i(v: Any) -> int | None:62 x = _f(v)63 return None if x is None else int(round(x))646566def _d(v: Any) -> date | None:67 if v is None:68 return None69 if isinstance(v, date):70 return v71 try:72 return date.fromisoformat(str(v)[:10])73 except ValueError:74 return None757677def build_packet(analytic_row: dict[str, Any], comps: list[Comparable], market: MarketContext, geo: GeoContext | None, cfg: dict[str, Any] | None = None) -> EvidencePacket:78 """Build the full (unfiltered) evidence packet for one subject. Price and address never enter."""79 row = analytic_row80 sale_date = _d(row.get("date") or row.get("valuation_date"))81 if sale_date is None:82 raise ValueError("analytic row needs a 'date' (valuation date)")83 cutoff = _d(row.get("information_cutoff")) or (sale_date - timedelta(days=1))84 sub = row.get("subtype") or "unknown"85 if sub not in ("detached", "semi", "row", "integrated", "unknown"):86 sub = "unknown"87 fields: dict[str, Any] = {88 "tx_id": str(row["tx_id"]),89 "valuation_date": sale_date,90 "information_cutoff": cutoff,91 "property_class": row["property_class"],92 "subtype": sub,93 "region": str(row.get("region_name") or row.get("region") or "unknown"),94 "municipality": str(row.get("mun_name") or row.get("municipality") or "unknown"),95 "borough": row.get("borough_name") or row.get("borough"),96 "lat": round(float(row["lat"]), 3),97 "lng": round(float(row["lng"]), 3),98 "floor_area_m2": _f(row.get("floor_area_m2")),99 "land_area_m2": _f(row.get("land_area_m2")),100 "year_built": _i(row.get("year_built")),101 "storeys": _f(row.get("storeys")),102 "n_units": _i(row.get("n_units")),103 "roll_land_value": _f(row.get("roll_land_value")),104 "roll_building_value": _f(row.get("roll_building_value")),105 "roll_total_value": _f(row.get("roll_total_value")),106 "roll_reference_date": (str(row["roll_reference_date"]) if row.get("roll_reference_date") else None),107 "prior_sale_price": _f(row.get("prior_sale_price")),108 "prior_sale_date": _d(row.get("prior_sale_date")),109 "new_construction": (None if row.get("new_construction") is None else bool(row.get("new_construction"))),110 }111 if fields["prior_sale_date"] is not None and fields["prior_sale_date"] >= cutoff: # schema: strictly before the cutoff112 fields["prior_sale_price"], fields["prior_sale_date"] = None, None # leakage guard113 missing = [k for k in _MISSING_CHECK if fields.get(k) in (None, "unknown")]114 if fields["roll_reference_date"] is None:115 missing.append("roll_reference_date")116 fields["missing_fields"] = missing117 subject = SubjectProperty(**fields)118119 max_comps = int((cfg or {}).get("comparables", {}).get("max_comps", 12)) if cfg else 12120 clean: list[Comparable] = []121 for c in comps[:max_comps]:122 if c.sale_date > cutoff:123 continue # strict as-of rule, defensive124 clean.append(c)125 clean = [c.model_copy(update={"evidence_id": f"C{i + 1:02d}"}) for i, c in enumerate(clean)]126 notes = [127 f"Valuation date {sale_date.isoformat()}; information cutoff {cutoff.isoformat()}: evidence dated after the cutoff is excluded.",128 "No inspection has been performed; condition, renovations and defects are unobserved for every role.",129 "The subject's sale price is not in the packet and is never disclosed.",130 ]131 if subject.roll_reference_date:132 notes.append(f"Roll values are those in force at the sale date; their market reference date is {subject.roll_reference_date}.")133 if not clean:134 notes.append("No comparable sale satisfied the retrieval rule before the cutoff.")135 if geo is None:136 notes.append("No geospatial context is available for this property.")137 pid = hashlib.sha256(f"{subject.tx_id}|{cutoff.isoformat()}|{len(clean)}".encode()).hexdigest()[:12]138 packet = EvidencePacket(packet_id=f"pkt_{subject.tx_id}_{pid}", subject=subject, comparables=clean, market=market, geo=geo, notes=notes)139 _assert_no_leak(packet)140 return packet141142143def _assert_no_leak(packet: EvidencePacket) -> None:144 """Defensive invariant: the subject carries no sale price and no address-like field."""145 subj = packet.subject.model_dump()146 for k in subj:147 if k in FORBIDDEN_KEYS:148 raise AssertionError(f"subject contains forbidden field {k!r}")149 txt = packet.model_dump_json().lower()150 for tok in ("\"street\"", "\"address\"", "\"civic"):151 if tok in txt:152 raise AssertionError(f"packet contains forbidden token {tok!r}")153154155def role_view(packet: EvidencePacket, role: Role) -> dict[str, Any]:156 """Least-privilege view: only the sections the role is entitled to, evidence ids preserved."""157 full = packet.model_dump(mode="json")158 view: dict[str, Any] = {159 "packet_id": packet.packet_id,160 "valuation_date": full["subject"]["valuation_date"],161 "information_cutoff": full["subject"]["information_cutoff"],162 "visible_sections": list(role.visible_sections),163 }164 ids: list[str] = []165 if role.sees("subject"):166 view["subject"] = full["subject"]167 ids.append(packet.subject.evidence_id)168 if role.sees("comparables"):169 view["comparables"] = full["comparables"]170 ids += [c.evidence_id for c in packet.comparables]171 if role.sees("market"):172 view["market"] = full["market"]173 ids.append(packet.market.evidence_id)174 if role.sees("geo"):175 view["geo"] = full["geo"] # may be None: absent is absent176 if packet.geo is not None:177 ids.append(packet.geo.evidence_id)178 view["evidence_ids_visible"] = ids179 view["notes"] = list(packet.notes)180 return view181182183def visible_ids(packet: EvidencePacket, role: Role) -> set[str]:184 return set(role_view(packet, role)["evidence_ids_visible"])185186187def render_user_prompt(view: dict[str, Any], extra: dict[str, Any] | None = None, *, kind: str = "estimate", verification_flags: list[str] | None = None) -> str:188 """User message = header + <evidence_packet> + optional <committee_state> + optional flags + task instruction."""189 if kind not in _INSTRUCTIONS:190 raise ValueError(f"unknown prompt kind {kind!r}")191 packet_json = json.dumps(view, ensure_ascii=False, separators=(",", ":"), default=str)192 parts = [193 f"Evidence packet {view.get('packet_id', '?')} — valuation date {view.get('valuation_date')}, information cutoff {view.get('information_cutoff')}.",194 f"Sections visible to your role: {', '.join(view.get('visible_sections', []))}.",195 f"<evidence_packet>{packet_json}</evidence_packet>",196 ]197 if extra is not None:198 parts.append(f"<committee_state>{json.dumps(extra, ensure_ascii=False, separators=(',', ':'), default=str)}</committee_state>")199 if verification_flags:200 parts.append(201 "<verification_flags>" + json.dumps(verification_flags, ensure_ascii=False) + "</verification_flags>\n"202 "A deterministic verification tool flagged the items above in your previous answer (unknown evidence id, adjustment outside ±40 % of the raw comparable price, or point estimate outside the adjusted-comparable hull ±25 % without justification). Correct them or justify them explicitly in key_assumptions."203 )204 parts.append(_INSTRUCTIONS[kind])205 return "\n".join(parts)206207208def committee_state(round_no: int, estimates: dict[str, AgentEstimate], *, for_role: str | None = None, critiques: list[dict[str, Any]] | None = None, visible: set[str] | None = None) -> dict[str, Any]:209 """Anonymised committee table (no model/provider identity) + committee median + critiques addressed to ``for_role``."""210 pts = sorted(e.point_estimate for e in estimates.values())211 med = None212 if pts:213 n = len(pts)214 med = pts[n // 2] if n % 2 else 0.5 * (pts[n // 2 - 1] + pts[n // 2])215 table = [216 {"role": rid, "point": e.point_estimate, "q05": e.q05, "q95": e.q95, "reasoning_summary": e.reasoning_summary[:600]}217 for rid, e in estimates.items()218 ]219 state: dict[str, Any] = {"round": round_no, "n_estimates": len(table), "committee_median": med, "estimates": table}220 if for_role is not None:221 state["your_role"] = for_role222 others = sorted({i for rid, e in estimates.items() if rid != for_role for i in e.evidence_ids_used})223 if visible is not None:224 state["evidence_ids_cited_by_others_outside_your_view"] = [i for i in others if i not in visible]225 state["critiques_addressed_to_you"] = critiques or []226 return state227228229# ---------------------------------------------------------------------------------------------------230# deterministic evidence verification (arm H and faithfulness audit)231# ---------------------------------------------------------------------------------------------------232ADJ_BAND = 0.40233HULL_BAND = 0.25234235236def verify_evidence(estimate: AgentEstimate, packet: EvidencePacket, allowed_ids: set[str] | None = None) -> list[str]:237 """Flags: ``unknown_evidence_id:<id>``, ``unknown_comparable:<id>``, ``adjustment_out_of_band:<id>``,238 ``point_outside_comparable_hull``. ``allowed_ids`` restricts citations to the role's view when given."""239 flags: list[str] = []240 ids = packet.evidence_ids() if allowed_ids is None else set(allowed_ids)241 for i in estimate.evidence_ids_used:242 if i not in ids:243 flags.append(f"unknown_evidence_id:{i}")244 raw = {c.evidence_id: c.price for c in packet.comparables}245 adjusted: list[float] = []246 for a in estimate.adjustments:247 if a.comparable_id not in raw:248 flags.append(f"unknown_comparable:{a.comparable_id}")249 continue250 p = raw[a.comparable_id]251 if p > 0 and abs(a.adjusted_price / p - 1.0) > ADJ_BAND:252 flags.append(f"adjustment_out_of_band:{a.comparable_id}")253 else:254 adjusted.append(a.adjusted_price)255 hull = adjusted or [c.price for c in packet.comparables]256 if hull:257 lo, hi = min(hull) * (1 - HULL_BAND), max(hull) * (1 + HULL_BAND)258 if not (lo <= estimate.point_estimate <= hi):259 flags.append("point_outside_comparable_hull")260 return flags261262263def faithfulness_summary(estimate: AgentEstimate, packet: EvidencePacket) -> dict[str, Any]:264 """Counts used by the evaluation layer (precision of cited ids, adjustment consistency, hull)."""265 ids = packet.evidence_ids()266 cited = estimate.evidence_ids_used267 ok = sum(1 for i in cited if i in ids)268 flags = verify_evidence(estimate, packet)269 return {270 "n_cited": len(cited),271 "n_cited_valid": ok,272 "citation_precision": (ok / len(cited)) if cited else None,273 "n_adjustments": len(estimate.adjustments),274 "n_adjustments_out_of_band": sum(1 for f in flags if f.startswith("adjustment_out_of_band")),275 "point_in_hull": "point_outside_comparable_hull" not in flags,276 "flags": flags,277 }278279280def sections_valid(sections: tuple[str, ...] | list[str]) -> bool:281 return all(s in SECTIONS for s in sections)282