1"""Phase 2 data-audit report: reports/phase2_data_audit.md from experiments/*.json (sample flow, siblings, store master, matching, audits)."""2from __future__ import annotations3import json4import pandas as pd5from src.common import ROOT, get_logger, BASE_PROC, BASE_EXP, P67log = get_logger("reports.phase2")8EXP = BASE_EXP9J = lambda n: json.load(open(EXP / n)) if (EXP / n).exists() else {}101112def main():13 sf, sib, smm, mt, ca, mv = J("sample_flow.json"), J("siblings.json"), J("store_master.json"), J("matching.json"), J("category_audit.json"), J("match_validation.json")14 sm = pd.read_parquet(ROOT / "data/interim/stores_master.parquet")15 L = ["# Phase 2 — Research database construction and data-quality audit (WP38)", "", "Generated by `src/reports/phase2_audit.py` from the pipeline outputs; snapshot 2026-11-01T17:38:30Z.", "",16 "## 2.1 Sample flow", "", "| step | listings |", "|---|---:|"] + [f"| {k} | {v:,} |".replace(",", " ") for k, v in sf.get("flow", [])] + ["",17 "## 2.2 Within-store duplicates, mirror domains, exclusions", "",18 f"- {sf['assortment_sample']['dup_in_store_listings_removed']:,} listings were non-representative duplicates of a (store, normalised title) pair — variants published as separate products; one representative (first seen) is kept.".replace(",", " "),19 f"- Mirror/sibling domains: {sib.get('sibling_groups')} groups, {sib.get('non_rep_stores')} non-representative stores ({sib.get('non_rep_listings')} listings) removed; rule = identical domain stem or title overlap ≥ 0.9 of the smaller catalogue (a 0.5 threshold wrongly captured seed resellers). Groups: " + "; ".join("/".join(g) for g in sib.get("largest_groups", [])) + ".",20 f"- Price sample exclusions: {json.dumps(sf.get('price_exclusions'))}; price quantiles (CAD): {json.dumps(sf.get('price_quantiles_cad'))}.", "",21 "## 2.3 Merchant geography", "",22 f"- Live stores {smm.get('live_stores')}; region source: {json.dumps(smm.get('region_source'))}; unknown-region listings {smm.get('unknown_region_listings'):,}.".replace(",", " "),23 f"- Agreement of Fabri-Ka labels with the FSA-centroid point-in-polygon (official MERN layer V2026-10): {smm.get('agreement_db_vs_pip')}; municipality lookup vs point-in-polygon: {smm.get('agreement_city_vs_pip')}; conflicts flagged: {smm.get('region_conflicts')} (69 % rural FSAs near borders). Rule and amendment A-001 in `research/decisions.md`.",24 f"- Phone numbers are shared by ≥ 3 stores for {smm.get('phone_shared_stores')} stores (directory contact copied by the discovery pipeline: e.g. 212 stores carry the Les Produits du Québec switchboard) — phones are never used for geography.",25 f"- Stores per region (final): {json.dumps(smm.get('region_final_counts'), ensure_ascii=False)}.", "",26 "## 2.4 Category labels (rule-based) — audit", ""]27 if ca:28 L += [f"- 500-listing stratified sample (25 per category), two LLM annotators: agreement rule–A {ca['agree_category_llm_a']}, rule–B {ca['agree_category_llm_b']}, A–B {ca['agree_llm_a_llm_b']}; consensus on {ca['consensus_share']:.0%}; precision of the rule label against consensus {ca['rule_precision_vs_consensus']}; {ca['n_disagreements']} listings with any disagreement listed in `data/processed/category_audit_disagreements.csv`.",29 "- Per-category precision (consensus n): " + ", ".join(f"{k} {v['precision']} ({v['n_consensus']})" for k, v in ca["per_category_precision_vs_consensus"].items() if v["precision"] is not None) + ".", ""]30 else:31 L += ["- audit pending", ""]32 L += ["## 2.5 Cross-merchant matching", "", f"- Universe {mt.get('universe_listings'):,} price-sample listings in {mt.get('universe_stores')} stores; SKU present for {mt.get('with_sku'):,}, brand for {mt.get('with_brand'):,}.".replace(",", " "),33 f"- Candidate pairs by tier {json.dumps(mt.get('pairs_by_tier'))}; comparable-variant pairs {json.dumps(mt.get('pairs_comparable_by_tier'))}; median |log gap| by tier {json.dumps(mt.get('median_abs_log_gap_by_tier'))}; share of zero gaps {json.dumps(mt.get('share_zero_gap_by_tier'))}.",34 f"- Groups over tiers A–C: {mt.get('groups_AC')} ({mt.get('listings_in_groups')} listings, {mt.get('stores_in_groups')} stores).", ""]35 if mv:36 L += [f"- Validation sample (60 pairs per tier): annotator agreement {mv['agree_llm']}, consensus {mv['consensus_share']:.0%}; share 'same product' by tier: " + ", ".join(f"{t} {v['yes']} (n={v['n']})" for t, v in mv["precision_by_tier_consensus"].items()) + f"; comparable-only: {json.dumps(mv['precision_by_tier_comparable_only'])}.", ""]37 L += ["## 2.6 Temporal coverage", "", "- Listings observed 2026-08-12 → 2026-11-01; store onboarding ended 2026-08-24 (ISO weeks 32–33); sync cadence median 6.3 h; inactive after two misses. No price history exists in the source (see `reports/00_source_audit.md`).", "",38 "## 2.7 Known limitations carried to the paper", "", "- Region unknown for " + str(int((sm.region_final == "Inconnue").sum())) + " live stores (" + f"{smm.get('unknown_region_listings'):,}".replace(",", " ") + " listings), concentrated in search-sweep and gift-directory discoveries and including the largest catalogues; reported separately, excluded from regional inference.",39 "- Category labels are rule-based; precision estimates from §2.4 propagate to a reassignment robustness run.", "- Origin class is a store-level prior; no product-level provenance (H6 untestable).", "- Posted prices, minimum variant price per listing; no shipping, taxes or transactions."]40 (ROOT / "reports" / "phase2_data_audit.md").write_text("\n".join(L)); log.info("phase2 audit written")414243if __name__ == "__main__":44 main()45