#!/usr/bin/env python3
"""Backfill SEC EDGAR fundamentals for the whole universe (resumable).

    # production (M3U96b)
    cd ~/hfmarketdata && HFMD_DATA_ROOT=~/firstratedata venv/bin/python scripts/edgar_backfill.py --workers 4
    # subset / dry run
    venv/bin/python scripts/edgar_backfill.py --tickers AAPL,MSFT,SHAK
    # from the SEC bulk archive (one 1+ GB download instead of ~7 600 API calls):
    #   curl -A "$HFMD_SEC_USER_AGENT" -o /tmp/companyfacts.zip https://www.sec.gov/Archives/edgar/daily-index/xbrl/companyfacts.zip
    venv/bin/python scripts/edgar_backfill.py --from-zip /tmp/companyfacts.zip

Steps: sync the CIK↔ticker universe (SEC lists ∩ price lake) → for every CIK fetch companyfacts + submissions
(token bucket ≤ 10 req/s, gzip cache under data_root/edgar/raw) → facts Parquet lake → standardized statements →
coverage → screener row. Progress is kept in data_root/edgar/backfill_manifest.json so a rerun only processes
CIKs that failed or were never done (use --force to redo). Ends with the bulk Parquet files and the health state.

Author: Simon-Pierre Boucher <contact@spboucher.ai>
"""
from __future__ import annotations

import argparse
import json
import logging
import sys
import time
import zipfile
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent / "hfmarketdata" / "api"))

from core.config import settings  # noqa: E402

log = logging.getLogger("edgar_backfill")


def manifest_path() -> Path:
    return settings.data_root / "edgar" / "backfill_manifest.json"


def load_manifest() -> dict:
    p = manifest_path()
    return json.loads(p.read_text()) if p.exists() else {"started_at": None, "companies": {}}


def save_manifest(m: dict) -> None:
    p = manifest_path()
    p.parent.mkdir(parents=True, exist_ok=True)
    tmp = p.with_suffix(".json.tmp")
    tmp.write_text(json.dumps(m, default=str))
    tmp.replace(p)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--tickers", help="comma-separated tickers (default: whole universe)")
    ap.add_argument("--ciks", help="comma-separated CIKs")
    ap.add_argument("--limit", type=int, help="process at most N companies")
    ap.add_argument("--workers", type=int, default=4, help="parallel companies (network is bounded by the 10 req/s bucket)")
    ap.add_argument("--rate", type=float, default=10.0, help="requests per second (SEC max 10)")
    ap.add_argument("--force", action="store_true", help="redo companies already done in the manifest")
    ap.add_argument("--refresh", action="store_true", help="ignore the raw JSON cache (refetch companyfacts/submissions)")
    ap.add_argument("--no-metalinks", action="store_true", help="skip MetaLinks.json (custom extension logging)")
    ap.add_argument("--from-zip", help="path to SEC companyfacts.zip (CIK##########.json inside) — no companyfacts API calls")
    ap.add_argument("--skip-universe", action="store_true", help="do not refresh edgar_companies from the SEC lists")
    ap.add_argument("--no-bulk", action="store_true", help="skip the bulk Parquet rebuild at the end")
    ap.add_argument("-v", "--verbose", action="store_true")
    args = ap.parse_args()
    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")

    from fundamentals import ingest, utcnow
    from fundamentals.edgar_client import EdgarClient
    from fundamentals.models import init_db

    init_db()
    client = EdgarClient(rate=args.rate)
    if not args.skip_universe:
        st = ingest.sync_universe(client)
        log.info("universe: lake=%s sec_tickers=%s companies=%s added=%s updated=%s delisted=%s unmatched(sample)=%s",
                 st.lake_tickers, st.sec_tickers, st.companies, st.added, st.updated, st.delisted, st.unmatched[:10])
    ciks = ingest.tracked_ciks()
    if args.ciks:
        want = {int(c) for c in args.ciks.split(",")}
        ciks = [c for c in ciks if c in want]
    if args.tickers:
        from fundamentals.service import resolve_company
        ciks = [resolve_company(t.strip()).cik for t in args.tickers.split(",") if t.strip()]
    manifest = load_manifest()
    manifest["started_at"] = manifest.get("started_at") or utcnow().isoformat()
    done = manifest["companies"]
    todo = [c for c in ciks if args.force or done.get(str(c), {}).get("status") not in ("ok", "no_facts")]
    if args.limit:
        todo = todo[:args.limit]
    log.info("%d companies tracked, %d to process", len(ciks), len(todo))
    ingest._set_state("backfill", last_run_at=utcnow(), companies_total=len(ciks),
                      companies_done=sum(1 for c in ciks if done.get(str(c), {}).get("status") == "ok"))

    zf = zipfile.ZipFile(args.from_zip) if args.from_zip else None

    def work(cik: int):
        cf = None
        if zf is not None:
            name = f"CIK{cik:010d}.json"
            try:
                cf = json.loads(zf.read(name))
            except KeyError:
                cf = {"cik": cik, "facts": {}}
        return ingest.ingest_company(client, cik, refresh=args.refresh, with_metalinks=not args.no_metalinks, companyfacts=cf)

    t0 = time.time()
    n_ok = n_err = 0
    with ThreadPoolExecutor(max_workers=max(1, args.workers)) as ex:
        futures = {ex.submit(work, c): c for c in todo}
        for i, fut in enumerate(as_completed(futures), start=1):
            cik = futures[fut]
            try:
                r = fut.result()
            except Exception as e:  # pragma: no cover
                r = None
                log.exception("cik %s crashed: %s", cik, e)
            if r is not None and r.error == ingest.NO_FACTS:
                # no XBRL facts on EDGAR (funds, trusts, paper filers): permanent — not retried on resume
                done[str(cik)] = {"status": "no_facts", "ticker": r.ticker, "at": utcnow().isoformat()}
            elif r is None or r.error:
                n_err += 1
                done[str(cik)] = {"status": "error", "error": r.error if r else "crash", "at": utcnow().isoformat()}
            else:
                n_ok += 1
                done[str(cik)] = {"status": "ok", "ticker": r.ticker, "rows": r.rows, "facts": r.facts, "completeness": r.completeness,
                                  "extensions": r.extensions, "at": utcnow().isoformat(), "seconds": round(r.seconds, 2)}
                log.info("[%d/%d] %s cik=%s facts=%s rows=%s derived=%s restated=%s completeness=%s ext=%s %.1fs",
                         i, len(todo), r.ticker, cik, r.facts, r.rows, r.derived_rows, r.restated_rows, r.completeness, r.extensions, r.seconds)
            if i % 25 == 0 or i == len(todo):
                save_manifest(manifest)
                ingest._set_state("backfill", companies_done=sum(1 for v in done.values() if v.get("status") == "ok"),
                                  failures_add=0, requests_made=client.stats.requests,
                                  failure_samples=[v.get("error") for v in done.values() if v.get("status") == "error"][:10])
                elapsed = time.time() - t0
                log.info("progress %d/%d — %.0f s elapsed, %.1f s/company, requests=%d cache_hits=%d retries=%d",
                         i, len(todo), elapsed, elapsed / i, client.stats.requests, client.stats.cache_hits, client.stats.retries)
    save_manifest(manifest)
    ingest._set_state("backfill", last_success_at=utcnow(), failures=n_err, requests_made=client.stats.requests)
    ingest._update_mapping_failure_rate()
    if not args.no_bulk:
        from bulk.build import build_all
        metas = build_all()
        log.info("bulk files rebuilt: %s", [(m["year"], m["rows"]) for m in metas])
    log.info("done: ok=%d errors=%d in %.0f s (%s)", n_ok, n_err, time.time() - t0, client.stats)
    return 0 if n_err == 0 else 1


if __name__ == "__main__":
    sys.exit(main())