"""F4-conf-03: harmonise five printed effect sizes onto target definitions.
Reads only data/; writes out/results.json deterministically. No randomness."""
import json, math, os, re

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
D = os.path.join(ROOT, "data")
SRC = os.path.join(D, "sources")


def lines_of(path):
    with open(path, encoding="utf-8") as fh:
        return fh.read().split("\n")


def wc_l(path):
    with open(path, "rb") as fh:
        return fh.read().count(b"\n")


def find_row(src, label, after=None):
    """Return (line_no, cells) of the first table row whose first cell == label
    (searching after a line containing `after`, if given)."""
    L = lines_of(os.path.join(SRC, src + ".txt"))
    start = 0
    if after:
        start = next(i for i, l in enumerate(L) if after in l)
    for i in range(start, len(L)):
        cells = [c.strip() for c in L[i].split("|")]
        if cells[0] == label and len(cells) > 1:
            print(f"[{src} line {i+1}] {L[i]}")
            return i + 1, cells
    raise ValueError(f"row {label!r} not found in {src}")


def num(cell):
    m = re.search(r"-?\d[\d,]*\.?\d*", cell.replace("−", "-"))
    return float(m.group(0).replace(",", ""))


effects = json.load(open(os.path.join(D, "effects.json"), encoding="utf-8"))
ids = [e["id"] for e in effects]
assert ids == ["E1", "E2", "E3", "E4", "E5"], ids

H, baseline = {}, {}

# ---- E1: DCR Table 6, ATT, 5 km (log points) -> % change
s = "arxiv-2609.27732v1"
ln_hdr, hdr = find_row(s, "", after="Table 6: Overall Average")
col5 = hdr.index("5 km")
ln, row = find_row(s, "ATT", after="Table 6: Overall Average")
b1 = num(row[col5])
ln_pc, pc = find_row(s, "% change", after="Table 6: Overall Average")
v1 = 100 * (math.exp(b1) - 1)
baseline["E1"] = {"printed_value": b1, "printed_unit": "log points"}
H["E1"] = {"value": round(v1, 4), "unit": "percent of the counterfactual hospitalisation rate",
           "transformation": f"100*(exp(b)-1) with b={row[col5]} -> 100*(exp({b1})-1) = {v1:.4f}% (cross-check: printed '% change' row = {pc[col5]})",
           "harmonisable": True, "missing_quantity": None,
           "locator": f"{s} | Table 6 | row 'ATT' (line {ln}) | column '5 km'; cross-check row '% change' (line {ln_pc}) | column '5 km'"}

# ---- E2: RESEA Table 2, Reemployment, ATE (pp) / control mean (%)
s = "arxiv-2606.14621v1"
ln_h, hdr = find_row(s, "Outcome", after="[Table 2")
ln, row = find_row(s, "Reemployment", after="[Table 2")
ate = num(row[hdr.index("Average treatment effect estimate")])
ctrl = num(row[hdr.index("Control")])
v2 = 100 * ate / ctrl
baseline["E2"] = {"printed_value": ate, "printed_unit": "percentage points"}
H["E2"] = {"value": round(v2, 4), "unit": "percent of the control-group mean reemployment rate",
           "transformation": f"100*Delta/p with Delta={ate} pp, p={ctrl}% -> 100*{ate}/{ctrl} = {v2:.4f}%",
           "harmonisable": True, "missing_quantity": None,
           "locator": f"{s} | Table 2 | row 'Reemployment' (line {ln}) | column 'Average treatment effect estimate' (1.47 pp) and column 'Control' (75.0%)"}

# ---- E3: MW Table 2, elasticity x printed % MW change
s = "arxiv-2505.04555v3"
ln_e, row_e = find_row(s, "Employment elasticity w.r.t MW", after="Table 2:")
ln_g, row_g = find_row(s, "% MW changes", after="Table 2:")
eta, g = num(row_e[1]), num(row_g[1])
v3 = 100 * eta * g
baseline["E3"] = {"printed_value": eta, "printed_unit": "elasticity (dimensionless)"}
H["E3"] = {"value": round(v3, 4), "unit": "percent change in employment",
           "transformation": f"100*eta*g with eta={row_e[1]}, g={row_g[1]} (proportional MW change) -> 100*({eta})*({g}) = {v3:.4f}%",
           "harmonisable": True, "missing_quantity": None,
           "locator": f"{s} | Table 2 | row 'Employment elasticity w.r.t MW' (line {ln_e}) | column 'Estimate (SE)'; row '% MW changes' (line {ln_g}) | column 'Estimate (SE)'"}

# ---- E4: DCR Table 7, fatal, 5 km -> absolute change per 100,000: needs level
s = "arxiv-2609.27732v1"
ln_h7, hdr7 = find_row(s, "Outcome", after="Table 7:")
ln, row = find_row(s, "Drug-related (fatal)", after="Table 7:")
b4 = num(row[hdr7.index("5 km")])
rel4 = 100 * (math.exp(b4) - 1)
baseline["E4"] = {"printed_value": b4, "printed_unit": "log points",
                  "relative_percent_change_reachable": round(rel4, 4)}
H["E4"] = {"value": None, "unit": "hospitalisations per 100,000 inhabitants per year",
           "transformation": f"would be R0*(exp(b)-1) with b={row[hdr7.index('5 km')]}; the relative part 100*(exp({b4})-1) = {rel4:.4f}% is computable, but the level R0 is not printed",
           "harmonisable": False,
           "missing_quantity": "baseline/counterfactual level (mean) of the fatal drug-related hospitalisation rate per 100,000 inhabitants per year for the treated regions within 5 km (e.g. pre-opening or untreated-potential-outcome mean); not printed in the extract",
           "locator": f"{s} | Table 7 | row 'Drug-related (fatal)' (line {ln}) | column '5 km'"}

# ---- E5: Tossou Table 3, Treatment, col (4): 0-1 scale -> pp
s = "arxiv-2510.03658v1"
ln_h, hdr = find_row(s, "", after="Table 3: Probability")
ln_h2, hdr2 = find_row(s, "", after="| (1) | (2)")
ln, row = find_row(s, "Treatment", after="Table 3: Probability")
c4 = hdr2.index("(4)")
beta = num(row[c4])
v5 = 100 * beta
ln_m, rowm = find_row(s, "Mean Dep. Var.", after="Table 3: Probability")
baseline["E5"] = {"printed_value": beta, "printed_unit": "change in probability (0-1 scale)"}
H["E5"] = {"value": round(v5, 4), "unit": "percentage points",
           "transformation": f"100*beta with beta={row[c4]} -> {v5:.4f} pp (exact unit rescaling)",
           "harmonisable": True, "missing_quantity": None,
           "locator": f"{s} | Table 3 | row 'Treatment' (line {ln}) | column '(4)'"}

data_issues = [
    {"issue": "arxiv-2609.27732v1: abstract states fatal drug-related hospitalisations 'fall by 35%', but Table 7 (5 km) prints -0.354 log points, i.e. 100*(exp(-0.354)-1) = -29.8%, and Section 6.1 says 'approximately 30%'.",
     "handling": "Table 7 value used for E4; abstract figure not used."},
    {"issue": "arxiv-2609.27732v1: Table A19 prints SE (0.218) for the CS-DR ATT -0.431 while Table 6 prints (0.210); A19 note mentions 640 never-treated regions while the tables print 608 control regions.",
     "handling": "Point estimate identical in both tables; Table 6 used for E1. SEs not propagated."},
    {"issue": "arxiv-2609.27732v1: Table A19 '% change' entries for Synthetic DiD (-30.3%) and Stacked DiD (-13.9%) differ by about 0.1 from 100*(exp(b)-1) (-30.37%, -13.84%).",
     "handling": "Not used; noted only. The Table 6 '% change' for 5 km (-35.0%) agrees with the transform of -0.431 (-35.01%)."},
    {"issue": "arxiv-2505.04555v3: Table 2 prints '% MW changes | 0.047' while the table note says 'about 4% minimum wage increase'; the row label '%' with value 0.047 is ambiguous between a proportion (4.7%) and a percent (0.047%).",
     "handling": "Table value used as instructed, read as a proportion g = 0.047 (4.7%), consistent in magnitude with the note's 'about 4%'. With g = 0.04 the result would be -1.55%."},
    {"issue": "arxiv-2510.03658v1: Table 3 col (4) prints '% Change -0.3685' while -0.0962/0.261 = -0.3686 (printed mean rounded); text refers to a Table 2 (-0.0855/0.455) that is not in the extract; footnote markers are rendered as '^9^9 9' etc.",
     "handling": "Not needed for the pp conversion of E5; noted only. Rendering artefacts ignored."},
    {"issue": "arxiv-2606.14621v1: the control mean 75.0% is the unadjusted aggregate control-group mean (rounded to 0.1), whereas the 1.47 pp ATE is adjusted for randomisation week; the printed control mean is therefore only an approximation of the counterfactual level for the ATE.",
     "handling": "Used as the denominator because the target definition specifies the control-group mean."},
]

n_input_rows = {"effects.json": len(effects), "README.md": wc_l(os.path.join(D, "README.md"))}
for f in sorted(os.listdir(SRC)):
    n_input_rows[f] = wc_l(os.path.join(SRC, f))

n_h = sum(1 for v in H.values() if v["value"] is not None)
n_nh = sum(1 for v in H.values() if v["harmonisable"] is False)
results = {
    "task_id": "F4-conf-03",
    "harmonised": H,
    "n_harmonised": n_h,
    "n_not_harmonisable": n_nh,
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(effects),
    "random_seed": None,
    "baseline_printed": baseline,
}
os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)
with open(os.path.join(ROOT, "out", "results.json"), "w", encoding="utf-8") as fh:
    json.dump(results, fh, indent=2, ensure_ascii=False)
for k, v in H.items():
    print(k, v["value"], v["unit"], "| harmonisable:", v["harmonisable"])
print("n_harmonised", n_h, "n_not_harmonisable", n_nh)