"""F1-dev-01: build analytic firm dataset and revenue aggregates. Run from workspace root."""
import json, os
import numpy as np, pandas as pd

DATA = "data/firms.csv"
OUT = "out"
os.makedirs(OUT, exist_ok=True)
UNIT_FACTOR = {"USD_millions": 1.0, "USD_thousands": 1e-3}

raw = pd.read_csv(DATA, dtype={"firm_id": str, "sector": str, "unit": str})
n_rows = len(raw)
issues, ledger = [], []

def log(step, rule, nb, na, why):
    ledger.append(dict(step=step, rule=rule, n_before=nb, n_after=na, n_removed=nb - na, rationale=why))

# Baseline: raw revenue mean, no cleaning
mean_revenue_raw = float(raw["revenue"].dropna().mean())

# Step 1: exact duplicate rows
n_dup_id = int(raw["firm_id"].duplicated().sum())
d = raw.drop_duplicates().reset_index(drop=True)
log(1, "drop exact duplicate rows (all columns identical)", n_rows, len(d),
    "file assembled from several extracts; one intended row per firm")
if n_dup_id:
    issues.append({"issue": f"{n_dup_id} rows repeat an earlier firm_id; all are exact full-row duplicates",
                   "handling": "kept first occurrence of each firm (drop_duplicates); no conflicting values"})
if d["firm_id"].duplicated().any():
    raise SystemExit("conflicting duplicate firm records remain; no resolution rule")
n_unique = int(raw["firm_id"].nunique())
assert n_unique == len(d)

# Step 2: unit recoding
unknown_units = set(d["unit"].unique()) - set(UNIT_FACTOR)
if unknown_units:
    raise SystemExit(f"unknown units {unknown_units}")
d["revenue_musd"] = d["revenue"] * d["unit"].map(UNIT_FACTOR)
n_thou = int((d["unit"] == "USD_thousands").sum())
log(2, "convert revenue to millions USD (USD_thousands / 1000)", len(d), len(d),
    "mixed currency units across rows; recoding only, no rows removed")
issues.append({"issue": f"mixed revenue units: {n_thou} distinct firms in USD_thousands, rest in USD_millions",
               "handling": "revenue_musd = revenue/1000 for USD_thousands rows; unchanged for USD_millions"})

mean_employees = float(d["employees"].mean())

# Step 3: missing revenue
miss = d["revenue_musd"].isna()
n_missing = int(miss.sum())
a = d.loc[~miss, ["firm_id", "sector", "revenue_musd", "employees"]].sort_values("firm_id").reset_index(drop=True)
log(3, "exclude firms with empty revenue from revenue aggregates", len(d), len(a),
    "revenue unknown; firms: " + " ".join(sorted(d.loc[miss, "firm_id"])))
if n_missing:
    issues.append({"issue": f"{n_missing} distinct firms with empty (unknown) revenue",
                   "handling": "excluded from revenue aggregates (exclusions.csv step 3); kept in n_unique_firms and mean_employees"})

res = {
    "task_id": "F1-dev-01",
    "n_unique_firms": n_unique,
    "n_missing_revenue": n_missing,
    "n_analysed": int(len(a)),
    "mean_revenue_musd": float(a["revenue_musd"].mean()),
    "median_revenue_musd": float(a["revenue_musd"].median()),
    "mean_employees": mean_employees,
    "sd_revenue_musd": float(a["revenue_musd"].std(ddof=1)),
    "se_mean_revenue_musd": float(a["revenue_musd"].std(ddof=1) / np.sqrt(len(a))),
    "mean_revenue_raw": mean_revenue_raw,
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": issues,
    "n_input_rows": {"firms.csv": int(n_rows)},
    "random_seed": None,
}
a.to_csv(f"{OUT}/analytic.csv", index=False)
pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)
with open(f"{OUT}/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=2))