"""F1-dev-01: analytic dataset and revenue aggregates. Run from workspace root."""
import json, os
import numpy as np
import pandas as pd

DATA = "data/firms.csv"
OUT = "out"
os.makedirs(OUT, exist_ok=True)

raw = pd.read_csv(DATA, dtype={"firm_id": str, "sector": str, "unit": str})
n_rows = len(raw)

# Baseline: raw revenue column mean, no cleaning
mean_revenue_raw = float(raw["revenue"].dropna().mean())

ledger, issues = [], []
def step(name, rule, before, after, why):
    ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,
                       n_removed=before - after, rationale=why))

# Step 1: exact duplicate rows
n_dup = int(raw.duplicated().sum())
d = raw.drop_duplicates().reset_index(drop=True)
step("1_drop_exact_duplicates", "drop rows identical to an earlier row on all columns",
     n_rows, len(d), "assembly error: same firm record appended more than once")
if n_dup:
    issues.append({"issue": f"{n_dup} rows are exact duplicates of another row (file assembled from several extracts)",
                   "handling": "kept first occurrence; one row per firm_id"})
# Check: no conflicting duplicates remain
n_conflict = int(d["firm_id"].duplicated().sum())
assert n_conflict == 0, "conflicting duplicate firm records"
n_unique_firms = int(raw["firm_id"].nunique())
assert len(d) == n_unique_firms

# Step 2: unit harmonisation
factor = {"USD_millions": 1.0, "USD_thousands": 1e-3}
unknown_units = set(d["unit"].unique()) - set(factor)
assert not unknown_units, f"unknown units {unknown_units}"
d["revenue_musd"] = d["revenue"] * d["unit"].map(factor)
n_thou = int((d["unit"] == "USD_thousands").sum())
step("2_convert_units", "revenue_musd = revenue/1000 if unit==USD_thousands else revenue",
     len(d), len(d), f"recoding only; {n_thou} firms reported in USD thousands")
issues.append({"issue": f"mixed revenue units: {n_thou} distinct firms in USD_thousands, rest in USD_millions",
               "handling": "converted all revenue to millions USD (thousands / 1000)"})

# mean employees over all distinct firms (before revenue exclusion)
mean_employees = float(d["employees"].mean())

# Step 3: missing revenue
miss = d["revenue_musd"].isna()
n_missing = int(miss.sum())
a = d.loc[~miss].copy()
step("3_drop_missing_revenue", "drop firms with empty revenue",
     len(d), len(a), "revenue unknown; cannot enter revenue aggregates (still counted in mean_employees)")
if n_missing:
    issues.append({"issue": f"{n_missing} distinct firms have empty (unknown) revenue: "
                   + ", ".join(sorted(d.loc[miss, 'firm_id'])),
                   "handling": "excluded from revenue aggregates and analytic.csv; included in n_unique_firms and mean_employees"})

a = a.sort_values("firm_id")[["firm_id", "sector", "revenue_musd", "employees"]]
a.to_csv(f"{OUT}/analytic.csv", index=False)
pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)

res = {
    "task_id": "F1-dev-01",
    "n_unique_firms": n_unique_firms,
    "n_missing_revenue": n_missing,
    "n_analysed": int(len(a)),
    "mean_revenue_musd": round(float(a["revenue_musd"].mean()), 6),
    "median_revenue_musd": round(float(a["revenue_musd"].median()), 6),
    "mean_employees": round(mean_employees, 6),
    "mean_revenue_raw": round(mean_revenue_raw, 6),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": issues,
    "n_input_rows": {"firms.csv": int(n_rows)},
    "random_seed": None,
    # supplementary (not required): descriptive uncertainty for the mean
    "sd_revenue_musd": round(float(a["revenue_musd"].std(ddof=1)), 6),
    "se_mean_revenue_musd": round(float(a["revenue_musd"].std(ddof=1) / np.sqrt(len(a))), 6),
}
with open(f"{OUT}/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps({k: res[k] for k in ["n_unique_firms", "n_missing_revenue", "n_analysed",
      "mean_revenue_musd", "median_revenue_musd", "mean_employees", "mean_revenue_raw"]}, indent=1))