"""F4-conf-03: harmonise five printed effect sizes. Reads only data/, writes out/results.json.
No randomness. Every printed input is located by regex in the extract and its line is printed."""
import json, math, os, re, sys

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
D = os.path.join(ROOT, "data")
S = os.path.join(D, "sources")

def lines_of(fn):
    with open(os.path.join(S, fn + ".txt"), encoding="utf-8") as f:
        return f.read().split("\n")

def find(src, pattern):
    """Return (line_no, line, match) for the unique line matching pattern."""
    L = lines_of(src)
    hits = [(i + 1, l, re.search(pattern, l)) for i, l in enumerate(L) if re.search(pattern, l)]
    if len(hits) != 1:
        sys.exit(f"pattern {pattern!r} matched {len(hits)} lines in {src}")
    n, l, m = hits[0]
    print(f"[{src} L{n}] {l.strip()}")
    return n, l, m

def cell(line, idx):
    return [c.strip() for c in line.split("|")][idx]

def num(s):
    return float(re.sub(r"[^\d.\-]", "", s))

effects = json.load(open(os.path.join(D, "effects.json"), encoding="utf-8"))
assert [e["id"] for e in effects] == ["E1", "E2", "E3", "E4", "E5"]
H = {}

# E1: Table 6, ATT row, 5 km column (4th cell)
src = "arxiv-2609.27732v1"
n, l, _ = find(src, r"^ATT \| -0\.399")
b1_txt = cell(l, 3)                      # '-0.431**'
b1 = num(b1_txt.replace("*", ""))
n_pc, l_pc, _ = find(src, r"^% change \| -32\.9%")
printed_pc = cell(l_pc, 3)
v1 = 100 * (math.exp(b1) - 1)
H["E1"] = dict(value=v1, unit="percent (relative change vs counterfactual rate)",
    transformation=f"100*(exp(b)-1) with b={b1} -> 100*(exp({b1})-1) = {v1:.4f}% (printed check: {printed_pc})",
    harmonisable=True, missing_quantity=None,
    locator=f"{src} | Table 6 | row 'ATT' (line {n}) | column '5 km' = {b1_txt}; cross-check row '% change' (line {n_pc}) | column '5 km' = {printed_pc}")

# E2: Table 2, Reemployment row
src = "arxiv-2606.14621v1"
n, l, _ = find(src, r"^Reemployment \| 75\.2%")
ctrl_txt = cell(l, 2); ate_txt = cell(l, 4).split(";")[0]
p = num(ctrl_txt); d = num(ate_txt)
v2 = 100 * d / p
H["E2"] = dict(value=v2, unit="percent (relative change vs control-group mean)",
    transformation=f"100*Delta/p with Delta={d} pp, p={p}% -> 100*{d}/{p} = {v2:.4f}%",
    harmonisable=True, missing_quantity=None,
    locator=f"{src} | Table 2 | row 'Reemployment' (line {n}) | column 'Average treatment effect estimate' = {ate_txt}; column 'Control' = {ctrl_txt}")

# E3: Table 2, elasticity row and '% MW changes' row
src = "arxiv-2505.04555v3"
n_e, l_e, _ = find(src, r"^Employment elasticity w\.r\.t MW \|")
eta = num(cell(l_e, 1))
n_g, l_g, _ = find(src, r"^% MW changes \|")
g = num(cell(l_g, 1))
find(src, r"about 4% minimum wage increase")
v3 = 100 * eta * g
H["E3"] = dict(value=v3, unit="percent (relative change in employment)",
    transformation=f"100*eta*g with eta={cell(l_e,1)}, g={cell(l_g,1)} (proportional MW change) -> 100*({eta})*({g}) = {v3:.4f}%",
    harmonisable=True, missing_quantity=None,
    locator=f"{src} | Table 2 | row 'Employment elasticity w.r.t MW' (line {n_e}) | column 'Estimate (SE)' = {cell(l_e,1)}; row '% MW changes' (line {n_g}) | column 'Estimate (SE)' = {cell(l_g,1)}")

# E4: Table 7, fatal row, 5 km column; absolute change needs the counterfactual level
src = "arxiv-2609.27732v1"
n, l, _ = find(src, r"^Drug-related \(fatal\) \|")
b4_txt = cell(l, 3); b4 = num(b4_txt.replace("*", ""))
rel4 = math.exp(b4) - 1
H["E4"] = dict(value=None, unit="hospitalisations per 100,000 inhabitants per year",
    transformation=(f"Absolute change = R0*(exp(b)-1) with b={b4} -> R0*({rel4:.4f}); "
                    "R0 (counterfactual/baseline fatal drug-related hospitalisation rate per 100,000 in treated 5 km regions) is not printed in the extract, so the value cannot be computed"),
    harmonisable=False,
    missing_quantity="baseline (counterfactual, e.g. pre-opening or control-group mean) level of the fatal drug-related hospitalisation rate per 100,000 inhabitants per year for the treated 5 km regions",
    locator=f"{src} | Table 7 | row 'Drug-related (fatal)' (line {n}) | column '5 km' = {b4_txt}")

# E5: Table 3, Treatment row, column (4)
src = "arxiv-2510.03658v1"
n, l, _ = find(src, r"^Treatment \| -0\.0911")
b5_txt = cell(l, 4); b5 = num(b5_txt)
v5 = 100 * b5
H["E5"] = dict(value=v5, unit="percentage points",
    transformation=f"100*beta with beta={b5_txt} (0-1 scale) -> 100*({b5}) = {v5:.2f} pp",
    harmonisable=True, missing_quantity=None,
    locator=f"{src} | Table 3 | row 'Treatment' (line {n}) | column '(4)' = {b5_txt}")

# ---- internal-consistency evidence for data_issues (all from the extracts) ----
src = "arxiv-2609.27732v1"
find(src, r"fatal outcomes, which fall by 35%")
find(src, r"by approximately 30% at 5 km")
n_a19, l_a19, _ = find(src, r"^\| \(0\.218\)")
n_se6, l_se6, _ = find(src, r"^\| \(0\.412\)")
find(src, r"never-treated comparison regions \(640 regions\)")
src = "arxiv-2510.03658v1"
find(src, r"by 0\.0855 \(a 18\.79 percentage change\)")
find(src, r"The result in table 2 suggests")
src = "arxiv-2606.14621v1"
find(src, r"increased reemployment by 1\.5 percentage points")

fatal_rel = 100 * rel4
data_issues = [
    {"issue": f"arxiv-2609.27732v1: abstract says fatal outcomes 'fall by 35%', whereas Section 6.1 says 'approximately 30% at 5 km'; Table 7 5 km fatal ATT {b4} implies 100*(exp({b4})-1) = {fatal_rel:.1f}%, consistent with ~30% not 35%.",
     "handling": "E4 is based on the Table 7 coefficient only; the abstract figure is not used. Noted as an internal inconsistency of the source."},
    {"issue": "arxiv-2609.27732v1: the CS-DR 5 km ATT -0.431 has SE (0.210) in Table 6 but (0.218) in Table A19; Table 6 note says 608 control regions while the Table A19 note says 640 never-treated comparison regions (Table A19 body says 608).",
     "handling": "Point estimate identical in both tables, so E1 is unaffected; no SE is propagated. Recorded only."},
    {"issue": f"arxiv-2505.04555v3: Table 2 prints '% MW changes' = {g} (4.7%) while the table note describes the 2023 change as 'about 4% minimum wage increase'.",
     "handling": f"E3 uses the printed table value g={g}, as the target definition asks for the size printed in the same table; with 0.04 the implied change would be {100*eta*0.04:.4f}% (reported for transparency only)."},
    {"issue": "arxiv-2510.03658v1: the abstract and Section 4.1 describe -0.0855 as 'a 18.79 percentage change' (a relative change computed against a mean of 0.455), and Section 4.1 attributes it to 'table 2', which is not in the extract; Table 3 relative change (-0.3685) is computed against the overall dependent-variable mean (0.261), not a counterfactual level.",
     "handling": "E5 only requires rescaling the Table 3 column (4) LPM coefficient to percentage points (exact); the relative-change figures are not used."},
    {"issue": "arxiv-2606.14621v1: the abstract rounds the reemployment ATE to 1.5 pp; Table 2 prints 1.47 pp. The ATE is block-adjusted while the 75.0% control mean is the raw (unadjusted) aggregate control-group mean.",
     "handling": "E2 uses the Table 2 values (1.47 pp, 75.0%) exactly as the target definition (percent of the control-group mean) specifies; noted as a caveat."},
]

def nlines(path):
    with open(path, "rb") as f:
        return f.read().count(b"\n")

n_input_rows = {"effects.json": len(effects), "README.md": nlines(os.path.join(D, "README.md"))}
for fn in sorted(os.listdir(S)):
    n_input_rows[fn] = nlines(os.path.join(S, fn))

n_h = sum(1 for v in H.values() if v["value"] is not None)
n_nh = sum(1 for v in H.values() if v["harmonisable"] is False)
res = {
    "task_id": "F4-conf-03",
    "harmonised": H,
    "n_harmonised": n_h,
    "n_not_harmonisable": n_nh,
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(effects),
    "random_seed": None,
}
assert n_h + n_nh == len(effects)
os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)
with open(os.path.join(ROOT, "out", "results.json"), "w", encoding="utf-8") as f:
    json.dump(res, f, indent=2, ensure_ascii=False)
for k, v in H.items():
    print(k, v["value"], v["unit"], "| harmonisable:", v["harmonisable"])
print("n_harmonised", n_h, "n_not_harmonisable", n_nh, "feasibility complete")