"""F4-conf-03: harmonise five printed effect sizes. Reads only data/; writes out/results.json.
No randomness (random_seed = null). Prints every source line relied upon."""
import json, math, os, re

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
D = os.path.join(ROOT, "data")
SRC = os.path.join(D, "sources")


def read_lines(path):
    with open(path, encoding="utf-8") as f:
        return f.read().split("\n")


def wc_l(path):
    with open(path, "rb") as f:
        return f.read().count(b"\n")


def find(src, pattern, nth=0):
    """Return (line_no, line) of the nth line in source matching regex pattern; print it."""
    lines = read_lines(os.path.join(SRC, src + ".txt"))
    hits = [(i + 1, l) for i, l in enumerate(lines) if re.search(pattern, l)]
    if len(hits) <= nth:
        raise RuntimeError(f"pattern {pattern!r} not found in {src}")
    no, line = hits[nth]
    print(f"[{src} L{no}] {line.strip()}")
    return no, line


def cells(line):
    return [c.strip() for c in line.split("|")]


def num(s):
    m = re.search(r"-?\d[\d,]*\.?\d*", s.replace("−", "-"))
    return float(m.group(0).replace(",", ""))


effects = json.load(open(os.path.join(D, "effects.json"), encoding="utf-8"))
assert [e["id"] for e in effects] == ["E1", "E2", "E3", "E4", "E5"]

H = {}
checks = {}

# ---------- E1: DCR Table 6, ATT, 5 km (log points) -> % of counterfactual
s1 = "arxiv-2609.27732v1"
_, hdr = find(s1, r"^\| 1 km \| 3 km \| 5 km$")
ln, att = find(s1, r"^ATT \| -0\.399")
_, pct = find(s1, r"^% change \| -32\.9%")
col = cells(hdr).index("5 km")
b1_str = cells(att)[col]
b1 = num(b1_str)
v1 = 100 * (math.exp(b1) - 1)
printed_pct_5km = num(cells(pct)[col])
H["E1"] = dict(
    value=round(v1, 2), unit="percent of the counterfactual hospitalisation rate",
    transformation=f"100*(exp(b)-1) with b = {b1} (printed '{b1_str}') = 100*(exp({b1})-1) = {v1:.4f} %",
    harmonisable=True, missing_quantity=None,
    locator=f"{s1} | Table 6 | row 'ATT' (line {ln}) | column '5 km' (value {b1_str}); cross-check: row '% change', column '5 km' prints {cells(pct)[col]}")
checks["E1_matches_printed_pct_change_row"] = abs(round(v1, 1) - printed_pct_5km) < 1e-9

# ---------- E2: RESEA Table 2 Reemployment ATE (pp) / control mean (%)
s2 = "arxiv-2606.14621v1"
_, hdr2 = find(s2, r"^Outcome \| Treatment \| Control \| Treatment-control difference")
ln2, row2 = find(s2, r"^Reemployment \| ")
c2 = cells(hdr2)
r2 = cells(row2)
ctrl_str = r2[c2.index("Control")]
ate_str = r2[c2.index("Average treatment effect estimate")].split(";")[0]
p_ctrl, d_ate = num(ctrl_str), num(ate_str)
v2 = 100 * d_ate / p_ctrl
H["E2"] = dict(
    value=round(v2, 2), unit="percent of the control-group mean reemployment rate",
    transformation=f"100*Delta/p with Delta = {d_ate} pp (ATE) and p = {p_ctrl}% (control mean) = 100*{d_ate}/{p_ctrl} = {v2:.4f} %",
    harmonisable=True, missing_quantity=None,
    locator=f"{s2} | Table 2 | row 'Reemployment' (line {ln2}) | columns 'Average treatment effect estimate' ({ate_str}) and 'Control' ({ctrl_str})")

# ---------- E3: MW Table 2 elasticity x printed % MW change
s3 = "arxiv-2505.04555v3"
ln3a, row3a = find(s3, r"^Employment elasticity w\.r\.t MW \|")
ln3b, row3b = find(s3, r"^% MW changes \|")
find(s3, r"about 4% minimum wage increase")  # note line (documented inconsistency)
eta_str, g_str = cells(row3a)[1], cells(row3b)[1]
eta, g = num(eta_str), num(g_str)
v3 = 100 * eta * g
H["E3"] = dict(
    value=round(v3, 3), unit="percent change in employment",
    transformation=f"100*eta*g with eta = {eta} and g = {g} (proportional MW change as printed) = 100*({eta})*({g}) = {v3:.4f} %",
    harmonisable=True, missing_quantity=None,
    locator=f"{s3} | Table 2 | row 'Employment elasticity w.r.t MW' (line {ln3a}), column 'Estimate (SE)' ({eta_str}); row '% MW changes' (line {ln3b}), column 'Estimate (SE)' ({g_str})")

# ---------- E4: DCR Table 7 fatal, 5 km: absolute change per 100,000 needs a level
ln4, row4 = find(s1, r"^Drug-related \(fatal\) \|")
_, hdr7 = find(s1, r"^Outcome \| 1 km \| 3 km \| 5 km$")
b4_str = cells(row4)[cells(hdr7).index("5 km")]
b4 = num(b4_str)
rel4 = 100 * (math.exp(b4) - 1)
# Search the extract for any printed level of the outcome (mean / baseline rate) -- none expected.
lines1 = read_lines(os.path.join(SRC, s1 + ".txt"))
level_hits = [l for l in lines1 if re.search(r"(?i)mean|baseline rate|pre-treatment (mean|level)|dep\. var", l)]
checks["E4_no_outcome_level_printed"] = len(level_hits) == 0
H["E4"] = dict(
    value=None, unit="hospitalisations per 100,000 inhabitants per year",
    transformation=(f"Would require Delta = Y0*(exp(b)-1) with b = {b4} (printed '{b4_str}'); the relative change "
                    f"100*(exp({b4})-1) = {rel4:.2f} % is computable, but the counterfactual/baseline level Y0 is not printed"),
    harmonisable=False,
    missing_quantity=("baseline (pre-opening or counterfactual) level of the fatal drug-related hospitalisation rate per "
                      "100,000 inhabitants per year in the 5 km treated regions (e.g. treated-group pre-treatment mean); "
                      "the extract prints only log-point ATTs and no outcome means"),
    locator=f"{s1} | Table 7 | row 'Drug-related (fatal)' (line {ln4}) | column '5 km' (value {b4_str})")

# ---------- E5: Benin Table 3, Treatment, col (4): 0-1 scale -> pp
s5 = "arxiv-2510.03658v1"
_, hdr5 = find(s5, r"^\| \(1\) \| \(2\) \| \(3\) \| \(4\)$")
ln5, row5 = find(s5, r"^Treatment \| ")
_, pc5 = find(s5, r"^% Change \| ")
_, mean5 = find(s5, r"^Mean Dep\. Var\. \| ")
i5 = cells(hdr5).index("(4)")
b5_str = cells(row5)[i5]
b5 = num(b5_str)
v5 = 100 * b5
H["E5"] = dict(
    value=round(v5, 2), unit="percentage points",
    transformation=f"100*beta with beta = {b5} (printed '{b5_str}', 0-1 probability scale) = {v5:.2f} pp",
    harmonisable=True, missing_quantity=None,
    locator=f"{s5} | Table 3 | row 'Treatment' (line {ln5}) | column '(4)' (value {b5_str})")
checks["E5_pct_change_row_consistent"] = abs(round(b5 / num(cells(mean5)[i5]), 4) - num(cells(pc5)[i5])) <= 1e-4

# ---------- Source-consistency checks used to document data issues
_, a19h = find(s1, r"^\| CS-DR \| Sun & Abraham")
_, a19b = find(s1, r"^ATT \| -0\.431\*\* \| -0\.438")
_, a19se = find(s1, r"^\| \(0\.218\)")
_, a19p = find(s1, r"^% change \| -35\.0% \| -35\.5%")
_, t6se = find(s1, r"^\| \(0\.412\) \| \(0\.279\) \| \(0\.210\)")
find(s1, r"never-treated comparison regions \(640 regions\)")
find(s1, r"which fall by 35%")
find(s1, r"approximately 30% at 5 km")
find(s5, r"0\.0855/0\.455")
a19_mismatch = []
for name, bs, ps in zip(cells(a19h)[1:], cells(a19b)[1:], cells(a19p)[1:]):
    implied = round(100 * (math.exp(num(bs)) - 1), 1)
    if abs(implied - num(ps)) > 1e-9:
        a19_mismatch.append(f"{name}: b={num(bs)} implies {implied}% vs printed {ps}")

data_issues = [
    {"issue": "DCR extract: the same 5 km CS-DR ATT (-0.431) carries SE (0.210) in Table 6 but (0.218) in Table A19.",
     "handling": "Point estimate identical in both tables; E1 uses Table 6 as specified. SEs not used (no uncertainty propagation)."},
    {"issue": "DCR extract: Tables 6/7 and A19 body report 608 control regions, while the A19 note says 640 never-treated comparison regions.",
     "handling": "Recorded only; does not affect any harmonised value."},
    {"issue": f"DCR extract: abstract says fatal hospitalisations 'fall by 35%', but Table 7 5 km ATT {b4} implies {rel4:.1f}% (Section 6.1 says ~30%).",
     "handling": "Table 7 value used for E4; abstract figure not used. E4 is in any case not harmonisable to an absolute change."},
    {"issue": "DCR extract Table A19: printed '% change' differs by 0.1 from 100*(exp(b)-1) for: " + ("; ".join(a19_mismatch) if a19_mismatch else "none"),
     "handling": "Attributed to rounding of the printed coefficients; no effect on E1 (5 km CS-DR value matches -35.0%)."},
    {"issue": "DCR extract: no level (mean or baseline) of any hospitalisation rate is printed.",
     "handling": "E4 (absolute change per 100,000) declared not harmonisable; missing quantity named."},
    {"issue": f"Minimum-wage extract: Table 2 note says 'about 4% minimum wage increase' while the row '% MW changes' prints {g_str} (4.7%).",
     "handling": f"Used the table row value {g_str} as the target definition instructs; with 4% the result would be about {100*eta*0.04:.2f}% (not reported as the estimate)."},
    {"issue": "RESEA extract: the control-group mean (75.0%) is an unadjusted aggregate mean, whereas the ATE (1.47 pp) adjusts for randomisation week; the aggregate control mean is only an approximation of the counterfactual level for the treated.",
     "handling": "Used the printed control mean as the target definition specifies ('percent of the control-group mean'); flagged as approximate."},
    {"issue": "Benin extract: abstract and Section 4.1 headline (0.0855, 18.79%, mean 0.455) refer to Table 2, which is not in the extract; footnote markers are rendered inline ('percent^9^9 9').",
     "handling": "E5 uses Table 3 column (4) (-0.0962) as specified; Table 3 '% Change' (-0.3685) is consistent with -0.0962/0.261."},
]

n_input_rows = {"effects.json": len(effects), "README.md": wc_l(os.path.join(D, "README.md"))}
for f in sorted(os.listdir(SRC)):
    n_input_rows[f] = wc_l(os.path.join(SRC, f))

n_harm = sum(1 for v in H.values() if v["value"] is not None)
n_not = sum(1 for v in H.values() if v["harmonisable"] is False)
results = {
    "task_id": "F4-conf-03",
    "harmonised": H,
    "n_harmonised": n_harm,
    "n_not_harmonisable": n_not,
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(effects),
    "random_seed": None,
    "baseline_printed_conversions": {
        "E1_table6_pct_change_row_5km": printed_pct_5km,
        "E5_table3_pct_change_row_col4": num(cells(pc5)[i5]),
    },
    "internal_checks": checks,
}
assert n_harm + n_not == len(effects)
os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)
with open(os.path.join(ROOT, "out", "results.json"), "w", encoding="utf-8") as f:
    json.dump(results, f, indent=2, ensure_ascii=False)
for k, v in H.items():
    print(f"{k}: value={v['value']} [{v['unit']}] harmonisable={v['harmonisable']}")
print("checks:", checks)