"""F4-conf-03: harmonise five printed effect sizes. Reads data/ only; writes out/results.json.
Deterministic; no randomness. Every printed input is parsed from the extract and the source line printed."""
import json, math, os, re

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
D = os.path.join(ROOT, "data")
SRC = os.path.join(D, "sources")


def lines_of(sid):
    with open(os.path.join(SRC, sid + ".txt"), encoding="utf-8") as f:
        return f.read().split("\n")


def find_line(sid, prefix, contains=None):
    """Return (line_no, text) of the first line starting with prefix (and containing `contains`)."""
    for i, l in enumerate(lines_of(sid), 1):
        if l.startswith(prefix) and (contains is None or contains in l):
            print(f"[{sid} L{i}] {l.strip()[:220]}")
            return i, l
    raise ValueError(f"line not found: {sid} {prefix}")


def cells(line):
    return [c.strip() for c in line.split("|")]


def num(s):
    m = re.search(r"-?\d+(?:\.\d+)?", s.replace(",", ""))
    return float(m.group(0)), m.group(0)


H = {}
issues = []

# ---- E1: DCR Table 6, ATT 5 km (log points) -> relative % change
sid = "arxiv-2609.27732v1"
i, l = find_line(sid, "ATT |")  # first 'ATT |' line is Table 6
b, b_s = num(cells(l)[3])
i2, l2 = find_line(sid, "% change |")
printed_pct = cells(l2)[3]
v1 = round(100 * (math.exp(b) - 1), 2)
H["E1"] = {
    "value": v1, "unit": "percent change relative to the counterfactual hospitalisation rate",
    "transformation": f"100*(exp(b)-1) with b = {b_s} log points -> 100*(exp({b_s})-1) = {100*(math.exp(b)-1):.4f}%, reported to 2 dp (consistent with printed '% change' {printed_pct})",
    "harmonisable": True, "missing_quantity": None,
    "locator": f"{sid} | Table 6 | row 'ATT' (line {i}) | column '5 km'; cross-check row '% change' (line {i2}) | column '5 km'",
}

# ---- E2: RESEA Table 2, Reemployment ATE pp / control mean
sid = "arxiv-2606.14621v1"
i, l = find_line(sid, "Reemployment |")
c = cells(l)
p, p_s = num(c[2]); d, d_s = num(c[4].split(";")[0])
v2 = round(100 * d / p, 2)
H["E2"] = {
    "value": v2, "unit": "percent change relative to the control-group mean reemployment rate",
    "transformation": f"100*Delta/p with Delta = {d_s} pp (ATE) and p = {p_s}% (control mean) -> 100*{d_s}/{p_s} = {100*d/p:.4f}%, reported to 2 dp",
    "harmonisable": True, "missing_quantity": None,
    "locator": f"{sid} | Table 2 | row 'Reemployment' (line {i}) | columns 'Average treatment effect estimate' (Delta) and 'Control' (p)",
}

# ---- E3: MW Table 2, elasticity x % MW change
sid = "arxiv-2505.04555v3"
i, l = find_line(sid, "Employment elasticity w.r.t MW")
eta, eta_s = num(cells(l)[1])
i2, l2 = find_line(sid, "% MW changes")
g, g_s = num(cells(l2)[1])
find_line(sid, "Note: The table reports the effects of a minimum wage increase")
v3 = round(100 * eta * g, 4)
H["E3"] = {
    "value": v3, "unit": "percent change in employment",
    "transformation": f"100*eta*g with eta = {eta_s} and g = {g_s} (proportional MW change) -> 100*({eta_s})*{g_s} = {100*eta*g:.4f}% (first-order/linear approximation)",
    "harmonisable": True, "missing_quantity": None,
    "locator": f"{sid} | Table 2 | row 'Employment elasticity w.r.t MW' (line {i}) | column 'Estimate (SE)'; row '% MW changes' (line {i2}) | column 'Estimate (SE)'",
}

# ---- E4: DCR Table 7 fatal 5 km -> absolute change per 100,000: needs baseline level
sid = "arxiv-2609.27732v1"
i, l = find_line(sid, "Drug-related (fatal) |")
bf, bf_s = num(cells(l)[3])
# search the extract for any printed level of the fatal hospitalisation rate (mean, baseline, pre-period)
level_hits = [k for k, t in enumerate(lines_of(sid), 1)
              if re.search(r"\b(mean|baseline|average rate|pre-treatment level|per 100,000 inhabitants was)\b", t, re.I)
              and re.search(r"fatal", t, re.I) and re.search(r"\d", t)]
print(f"[E4] lines with a printed level of the fatal rate: {level_hits}")
H["E4"] = {
    "value": None, "unit": "hospitalisations per 100,000 inhabitants per year",
    "transformation": f"absolute change = R0*(exp(b)-1) with b = {bf_s}; exp({bf_s})-1 = {math.exp(bf)-1:.4f} (relative), but R0 (counterfactual level of the fatal rate) is not printed, so the absolute change cannot be computed",
    "harmonisable": False,
    "missing_quantity": "counterfactual (or pre-treatment / control-group mean) level of the fatal drug-related hospitalisation rate per 100,000 inhabitants per year for the treated 5 km regions",
    "locator": f"{sid} | Table 7 | row 'Drug-related (fatal)' (line {i}) | column '5 km'",
}

# ---- E5: Benin Table 3, Treatment col (4), 0-1 scale -> pp
sid = "arxiv-2510.03658v1"
i, l = find_line(sid, "Treatment |")
bt, bt_s = num(cells(l)[4])
v5 = round(100 * bt, 2)
H["E5"] = {
    "value": v5, "unit": "percentage points",
    "transformation": f"100*beta with beta = {bt_s} (probability, 0-1 scale) -> 100*({bt_s}) = {100*bt:.2f} pp (exact rescaling)",
    "harmonisable": True, "missing_quantity": None,
    "locator": f"{sid} | Table 3 | row 'Treatment' (line {i}) | column '(4)'",
}

# ---- data issues (each backed by printed lines)
print("--- lines supporting data issues ---")
find_line("arxiv-2609.27732v1", "| (0.412)")
find_line("arxiv-2609.27732v1", "| (0.218)")
find_line("arxiv-2609.27732v1", "never-treated comparison regions (640")
issues.append({"issue": "arxiv-2609.27732v1: CS-DR 5 km ATT -0.431 printed with SE 0.210 in Table 6 but 0.218 in Table A19; Table A19 note says 640 never-treated regions while tables say 608.",
               "handling": "Internal inconsistency of the source; irrelevant to point estimates (no SE propagated); recorded only."})
fatal_rel = 100 * (math.exp(bf) - 1)
issues.append({"issue": f"arxiv-2609.27732v1: abstract says fatal hospitalisations 'fall by 35%', Section 6.1 says 'approximately 30%'; Table 7 5 km coefficient {bf_s} implies 100*(exp({bf_s})-1) = {fatal_rel:.1f}%.",
               "handling": "Abstract is inconsistent with Table 7; Table 7 coefficient used as the printed input for E4 (E4 not harmonisable anyway)."})
issues.append({"issue": "arxiv-2609.27732v1: no level (mean, baseline or control) of the fatal drug-related hospitalisation rate per 100,000 is printed in the extract.",
               "handling": "E4 declared not harmonisable; value null; missing quantity named. No external value substituted."})
issues.append({"issue": f"arxiv-2505.04555v3: Table 2 note describes 'about 4% minimum wage increase' while row '% MW changes' prints {g_s}.",
               "handling": f"Used the table row value g = {g_s} as instructed by the target definition; the note's '4%' is a rounded/loose description; with g = 0.04 the result would be {100*eta*0.04:.2f}%."})
issues.append({"issue": "arxiv-2606.14621v1: control mean 75.0% in Table 2 is the raw aggregate control-group mean (not adjusted for randomisation week), while the 1.47 pp ATE is block-adjusted; the paper states raw aggregate comparisons are biased.",
               "handling": "Used 75.0% as p because the target definition specifies 'percent of the control-group mean'; flagged as an approximation of the counterfactual level."})
issues.append({"issue": f"arxiv-2510.03658v1: text cites a 'table 2' estimate -0.0855 with mean 0.455 (table not in extract); printed % Change -0.3685 vs -0.0962/0.261 = {-0.0962/0.261:.4f} (rounding of inputs).",
               "handling": "Not relevant to E5 (pure x100 rescaling of Table 3 col (4)); recorded only."})

# ---- n_input_rows
n_rows = {}
for fn in sorted(os.listdir(SRC)):
    with open(os.path.join(SRC, fn), "rb") as f:
        n_rows[fn] = f.read().count(b"\n")
with open(os.path.join(D, "README.md"), "rb") as f:
    n_rows["README.md"] = f.read().count(b"\n")
with open(os.path.join(D, "effects.json"), encoding="utf-8") as f:
    effects = json.load(f)
n_rows["effects.json"] = len(effects)
assert [e["id"] for e in effects] == ["E1", "E2", "E3", "E4", "E5"]

n_h = sum(1 for v in H.values() if v["value"] is not None)
n_nh = sum(1 for v in H.values() if v["harmonisable"] is False)
res = {
    "task_id": "F4-conf-03",
    "harmonised": H,
    "n_harmonised": n_h,
    "n_not_harmonisable": n_nh,
    "feasibility": "complete" if n_h + n_nh == len(effects) else "partial",
    "feasibility_reason": None,
    "data_issues": issues,
    "n_input_rows": n_rows,
    "n_analysed": len(effects),
    "random_seed": None,
}
os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)
with open(os.path.join(ROOT, "out", "results.json"), "w", encoding="utf-8") as f:
    json.dump(res, f, indent=2, ensure_ascii=False)
print("--- harmonised ---")
for k, v in H.items():
    print(k, v["value"], v["unit"], "| harmonisable =", v["harmonisable"])
print("n_harmonised", n_h, "n_not_harmonisable", n_nh, "feasibility", res["feasibility"])