"""Reproducibility checks for the exhibit layer (protocol Part IX).

Run:  python3 -I -m pytest -q tests
1. every CSV export reconciles with stats.json (sums of the marginal tables equal the headline counts);
2. code/make_tables.py regenerated into a scratch directory reproduces the committed paper/tables/*.tex byte for byte
   and the committed results/inline_numbers.json;
3. make_tables.py never rewrites data/stats.json and contains no hand-typed data literal.
"""
import csv, hashlib, json, subprocess, sys, tokenize, io
from pathlib import Path
import pytest

ROOT = Path(__file__).resolve().parent.parent
D = ROOT / "data"
S = json.load(open(D / "stats.json"))


def rows(name):
    with open(D / f"{name}.csv", newline="") as f:
        return list(csv.DictReader(f))


def col_sum(name, col, where=None):
    return sum(int(float(r[col])) for r in rows(name) if where is None or where(r))


@pytest.mark.parametrize("name,col,key", [
    ("ledger_by_form", "sections", "ledger_total"),
    ("mentions_by_form", "n", "mentions"),
    ("mentions_by_year", "mentions", "mentions"),
    ("confidence_dist", "n", "mentions"),
    ("projects_by_type", "n", "projects"),
    ("projects_by_status", "n", "projects"),
    ("duration", "n", "projects"),
    ("mentions_per_project", "n", "projects"),
    ("projects_by_first_seen_year", "n", "projects"),
    ("projects_by_sector", "n", "projects"),
    ("projects_by_index", "projects", "projects"),
    ("filings_by_form", "n", "filings_total"),
    ("filings_by_year_form", "n", "filings_total"),
    ("issuers_by_year", "filings", "filings_total"),
    ("parse_status", "n", "filings_total"),
    ("sections_by_form", "sections", "sections_total"),
])
def test_csv_sums_reconcile_with_stats(name, col, key):
    assert col_sum(name, col) == S[key], f"{name}.{col} does not sum to stats.json[{key}]"


def test_universe_reconciles():
    U = S["universe"]
    assert U["both"] + U["sp_only"] + U["ru_only"] == U["n"]
    assert U["n_russell"] == U["russell_distinct_ciks"]
    assert S["issuers_with_filings"] + S["issuers_without_filings"] == U["n"]
    assert col_sum("companies_by_sector", "n") == U["n"]
    assert len(rows("issuers_without_filings")) == S["issuers_without_filings"]


def test_items_reconcile_with_ledger_by_form():
    lb = {r["form_type"]: r for r in rows("ledger_by_form")}
    for form, name in (("10-K", "ledger_10k_items"), ("10-Q", "ledger_10q_items")):
        assert col_sum(name, "n") == int(lb[form]["sections"])
        assert col_sum(name, "mentions") == int(lb[form]["mentions"])


def test_amounts_and_fields():
    assert int(rows("amount_dist")[0]["n"]) == S["projects_fields"][0]
    assert col_sum("projects_by_type", "with_amount") == S["projects_fields"][0]
    assert S["mentions_fields"][-1] == S["mentions"] and S["projects_fields"][-1] == S["projects"]


def test_ledger_mentions_exceed_stored_mentions_by_collisions():
    # the ledger counts mentions before id de-duplication; stored mentions can only be fewer
    assert col_sum("ledger_by_form", "mentions") >= S["mentions"]
    assert col_sum("ledger_by_form", "with_mentions") <= S["ledger_total"]


def _sha(p):
    return hashlib.sha256(p.read_bytes()).hexdigest()


def test_regeneration_reproduces_committed_tables(tmp_path):
    before = _sha(D / "stats.json")
    r = subprocess.run([sys.executable, "-I", str(ROOT / "code" / "make_tables.py"), "--out", str(tmp_path)],
                       capture_output=True, text=True, cwd=ROOT)
    assert r.returncode == 0, r.stderr
    assert _sha(D / "stats.json") == before, "make_tables.py must never rewrite stats.json"
    mism = []
    for p in sorted((tmp_path / "tables").glob("*.tex")):
        q = ROOT / "paper" / "tables" / p.name
        if not q.exists() or q.read_text() != p.read_text():
            mism.append(p.name)
    assert not mism, f"regenerated tables differ from the committed ones: {mism} (run python3 -I code/make_tables.py and commit)"
    gen = json.load(open(tmp_path / "results" / "inline_numbers.json"))
    com = json.load(open(ROOT / "results" / "inline_numbers.json"))
    assert gen == com, "results/inline_numbers.json is stale"


def test_make_tables_has_no_data_literals():
    """Integer literals with three or more digits are not allowed in the generator (the scales 100, 1e3, 1e6, 1e9 are)."""
    src = (ROOT / "code" / "make_tables.py").read_text()
    bad = []
    for tok in tokenize.generate_tokens(io.StringIO(src).readline):
        if tok.type == tokenize.NUMBER and "e" not in tok.string.lower() and "." not in tok.string and len(tok.string) >= 3 and tok.string != "100":
            bad.append((tok.start[0], tok.string))
    assert not bad, f"numeric literals in make_tables.py: {bad}"
    assert 'json.dump(S' not in src, "make_tables.py must not write stats.json"