"""Record trimmed REAL EDGAR fixtures for the fundamentals tests (run manually, needs network).

    HFMD_DATA_ROOT=/tmp/hfmd-proto .venv/bin/python tests/fixtures/edgar/record.py

Prototype companies: Apple (CIK 320193, FYE late September, 52/53-week), Microsoft (CIK 789019, FYE June 30),
Shake Shack (CIK 1620533, Russell 2000, FYE last Wednesday of December, restaurant cost lines tagged with the
company extension `shak:OperatingMaterialsExpense` → cost_of_revenue legitimately unmapped).

Trimming keeps the fixtures small (< 1 MB gzipped total) while staying real:
* companyfacts: mapped tags + identity tags + a few unmapped us-gaap tags, facts filed since 2022-01-01
  with period end >= 2020-09-01;
* submissions: filings of the tracked forms since 2022-01-01 (older pages dropped);
* MetaLinks of the latest 10-K (SHAK): tag names only.
"""
from __future__ import annotations

import gzip
import json
import os
import sys
from datetime import date
from pathlib import Path

HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[2]
sys.path.insert(0, str(ROOT / "hfmarketdata" / "api"))
os.environ.setdefault("HFMD_DATA_ROOT", "/tmp/hfmd-proto")

from fundamentals import mapping as M  # noqa: E402
from fundamentals.edgar_client import TRACKED_FORMS, EdgarClient  # noqa: E402

COMPANIES = {"AAPL": 320193, "MSFT": 789019, "SHAK": 1620533}
EXTRA_TICKERS = {"GOOGL": 1652044, "GOOG": 1652044}     # one CIK, two share classes (mapping test only)
MIN_END = "2020-09-01"
MIN_FILED = "2022-01-01"
KEEP_UNMAPPED = ("OperatingExpenses", "LaborAndRelatedExpense", "OccupancyNet", "PreOpeningCosts",
                 "ComprehensiveIncomeNetOfTax", "OtherNonoperatingIncomeExpense", "IncreaseDecreaseInInventories")


def dump(path: Path, obj) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with gzip.open(path, "wt", encoding="utf-8", compresslevel=9) as fh:
        json.dump(obj, fh, separators=(",", ":"))
    print(f"  {path.name}: {path.stat().st_size / 1024:.0f} KB")


def trim_companyfacts(cf: dict) -> dict:
    keep = {(t.taxonomy, t.tag) for a in M.ACCOUNTS for t in a.tags} | set(M.IDENTITY_TAGS)
    keep |= {("us-gaap", t) for t in KEEP_UNMAPPED}
    out = {"cik": cf["cik"], "entityName": cf["entityName"], "facts": {}}
    for tax, tags in cf["facts"].items():
        for tag, body in tags.items():
            if (tax, tag) not in keep and tax != "dei":
                continue
            units = {}
            for unit, facts in body["units"].items():
                kept = [f for f in facts if f["end"] >= MIN_END and f["filed"] >= MIN_FILED]
                if kept:
                    units[unit] = kept
            if units:
                out["facts"].setdefault(tax, {})[tag] = {"label": body.get("label"), "description": "", "units": units}
    return out


def trim_submissions(sub: dict) -> dict:
    rec = sub["filings"]["recent"]
    idx = [i for i, f in enumerate(rec["form"]) if f in TRACKED_FORMS and rec["filingDate"][i] >= MIN_FILED]
    recent = {k: [v[i] for i in idx] for k, v in rec.items()}
    out = {k: v for k, v in sub.items() if k not in ("filings", "addresses", "formerNames", "description")}
    out["filings"] = {"recent": recent, "files": []}
    return out


def trim_metalinks(ml: dict) -> dict:
    key, inst = next(iter(ml["instance"].items()))
    keep_roles = {r["role"] for r in inst["report"].values() if r.get("menuCat") == "Statements"}
    tags = {}
    for name, info in inst["tag"].items():
        pres = [p for p in (info.get("presentation") or []) if p in keep_roles]
        tags[name] = {"xbrltype": info.get("xbrltype"), "nsuri": info.get("nsuri"), "presentation": pres}
    return {"instance": {key: {"nsprefix": inst["nsprefix"], "nsuri": inst["nsuri"], "baseTaxonomies": inst["baseTaxonomies"],
                               "report": {k: {"role": r.get("role"), "shortName": r.get("shortName"), "menuCat": r.get("menuCat")}
                                          for k, r in inst["report"].items() if r.get("menuCat") == "Statements"},
                               "tag": tags}}}


def main() -> None:
    c = EdgarClient()
    tickers = c.company_tickers()
    exch = c.company_tickers_exchange()
    wanted = set(COMPANIES.values()) | set(EXTRA_TICKERS.values())
    ct = {str(i): v for i, v in enumerate(v for v in tickers.values() if int(v["cik_str"]) in wanted)}
    dump(HERE / "company_tickers.json.gz", ct)
    dump(HERE / "company_tickers_exchange.json.gz", {"fields": exch["fields"],
                                                     "data": [r for r in exch["data"] if int(r[0]) in wanted]})
    for tk, cik in COMPANIES.items():
        print(tk, cik)
        cf = c.companyfacts(cik)
        dump(HERE / f"companyfacts_CIK{cik:010d}.json.gz", trim_companyfacts(cf))
        sub = c.submissions(cik, include_older=False)
        dump(HERE / f"submissions_CIK{cik:010d}.json.gz", trim_submissions(sub))
        if tk == "SHAK":
            rec = sub["filings"]["recent"]
            accn = next(rec["accessionNumber"][i] for i, f in enumerate(rec["form"]) if f == "10-K")
            dump(HERE / f"metalinks_CIK{cik:010d}_{accn.replace('-', '')}.json.gz", trim_metalinks(c.metalinks(cik, accn)))
    import re
    atom = c.atom_current("10-Q")
    head = atom.split("<entry>")[0]
    entries = re.findall(r"<entry>.*?</entry>", atom, flags=re.S)[:3]
    (HERE / "atom_10-Q.xml").write_text(head + "\n".join(entries) + "\n</feed>\n", encoding="utf-8")
    yesterday = date.fromordinal(date.today().toordinal() - 1)
    for back in range(0, 6):
        d = date.fromordinal(yesterday.toordinal() - back)
        q = (d.month - 1) // 3 + 1
        try:
            txt = c.get_text(f"https://www.sec.gov/Archives/edgar/daily-index/{d.year}/QTR{q}/master.{d:%Y%m%d}.idx")
        except Exception:
            continue
        lines = txt.splitlines()
        head = lines[:11]
        body = [ln for ln in lines[11:] if ln.split("|")[2:3] and ln.split("|")[2] in ("10-K", "10-Q", "8-K", "20-F")][:12]
        (HERE / "master.idx").write_text("\n".join(head + body) + "\n", encoding="utf-8")
        print("  master.idx from", d)
        break
    print(c.stats)


if __name__ == "__main__":
    main()