"""F1-dev-01: build analytic firm dataset and revenue aggregates. Run from workspace root."""
import json, os
import numpy as np
import pandas as pd

SEED = 20261030

RAW = "data/firms.csv"
OUT = "out"
os.makedirs(OUT, exist_ok=True)

raw = pd.read_csv(RAW, dtype=str, keep_default_na=False)
n_rows = len(raw)
for c in raw.columns:
    raw[c] = raw[c].str.strip()

# Baseline: raw mean of revenue as in file (mixed units), non-empty rows, before cleaning
rev_raw = pd.to_numeric(raw["revenue"].replace("", None), errors="coerce")
mean_revenue_raw = float(rev_raw.mean())

issues, ledger = [], []
def step(rule, before, after, rationale):
    ledger.append(dict(step=len(ledger) + 1, rule=rule, n_before=before, n_after=after,
                       n_removed=before - after, rationale=rationale))

step("read data/firms.csv", n_rows, n_rows, "All data rows read (header excluded).")

# 1. duplicates
n_full_dup = int(raw.duplicated().sum())
n_id_dup = int(raw["firm_id"].duplicated().sum())
df = raw.drop_duplicates().copy()
if df["firm_id"].duplicated().any():
    raise RuntimeError("Conflicting duplicate firm_id rows found; resolve manually.")
step("drop exact duplicate rows (keep first occurrence)", n_rows, len(df),
     f"{n_full_dup} rows were exact copies of another row (same firm_id and all fields), an assembly error; "
     "one row per firm kept.")
if n_full_dup:
    issues.append({"issue": f"{n_full_dup} exact duplicate rows ({n_id_dup} repeated firm_id values; all duplicates identical in every field)",
                   "handling": "Dropped duplicates, keeping one row per firm_id; no conflicting duplicates existed."})

# 2. unit harmonisation
units = df["unit"].value_counts().to_dict()
factor = {"USD_millions": 1.0, "USD_thousands": 1e-3}
unknown = set(df["unit"]) - set(factor)
if unknown:
    raise RuntimeError(f"Unknown units: {unknown}")
df["revenue_num"] = pd.to_numeric(df["revenue"].replace("", None), errors="coerce")
df["revenue_musd"] = df["revenue_num"] * df["unit"].map(factor)
n_thou = int((df["unit"] == "USD_thousands").sum())
step("convert revenue to millions USD (USD_thousands / 1000)", len(df), len(df),
     f"Recoding only: {n_thou} firms reported revenue in USD_thousands; divided by 1000. No rows removed.")
issues.append({"issue": f"Mixed revenue units across rows (firm-level counts after de-duplication: {units})",
               "handling": "Converted all revenue to millions USD using the per-row unit column (thousands/1000)."})

# employees over all distinct firms (before dropping missing revenue)
df["employees"] = pd.to_numeric(df["employees"], errors="raise").astype(int)
mean_employees = float(df["employees"].mean())
n_unique = int(df["firm_id"].nunique())

# 3. missing revenue
miss = df["revenue_musd"].isna()
n_missing = int(df.loc[miss, "firm_id"].nunique())
before = len(df)
ana = df.loc[~miss].copy()
step("exclude firms with unknown (empty) revenue", before, len(ana),
     "Revenue unknown; cannot contribute to revenue aggregates. Firm IDs: " + ";".join(sorted(df.loc[miss, "firm_id"])))
issues.append({"issue": f"{n_missing} distinct firms have empty revenue",
               "handling": "Excluded from revenue aggregates (listed in exclusions.csv); retained in mean_employees, which is over all distinct firms."})

ana = ana.sort_values("firm_id")[["firm_id", "sector", "revenue_musd", "employees"]]
ana.to_csv(f"{OUT}/analytic.csv", index=False)
pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)

# Uncertainty (descriptive; firms treated as a sample from a superpopulation)
r = ana["revenue_musd"].to_numpy()
se_mean = float(r.std(ddof=1) / np.sqrt(len(r)))
rng = np.random.default_rng(SEED)
bs = rng.choice(r, size=(5000, len(r)), replace=True)
ci_mean = np.percentile(bs.mean(axis=1), [2.5, 97.5])
ci_med = np.percentile(np.median(bs, axis=1), [2.5, 97.5])

res = {
    "task_id": "F1-dev-01",
    "n_unique_firms": n_unique,
    "n_missing_revenue": n_missing,
    "n_analysed": int(len(ana)),
    "mean_revenue_musd": round(float(ana["revenue_musd"].mean()), 6),
    "median_revenue_musd": round(float(ana["revenue_musd"].median()), 6),
    "mean_employees": round(mean_employees, 6),
    "mean_revenue_raw": round(mean_revenue_raw, 6),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": issues,
    "n_input_rows": {"firms.csv": n_rows},
    "random_seed": SEED,
    "se_mean_revenue_musd": round(se_mean, 6),
    "bootstrap_ci95_mean_revenue_musd": [round(float(x), 6) for x in ci_mean],
    "bootstrap_ci95_median_revenue_musd": [round(float(x), 6) for x in ci_med],
}
with open(f"{OUT}/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps(res, indent=2))