#!/usr/bin/env python3
"""Incremental EDGAR poller: every 2 minutes, detect new filings of tracked CIKs and re-normalise them.

    # production (M3U96b) — run under PM2 next to the API:
    cd ~/hfmarketdata && HFMD_DATA_ROOT=~/firstratedata pm2 start venv/bin/python --name edgar-incremental -- scripts/edgar_incremental.py
    # one cycle (cron / debugging)
    venv/bin/python scripts/edgar_incremental.py --once

Sources (see fundamentals.ingest.poll_new_filings): the EDGAR Atom feed `browse-edgar?action=getcurrent` for
10-K / 10-Q / 8-K / 20-F (live, ~4 requests per cycle) + the daily master index of the last two business days
as a safety net (weekend indexes do not exist: 403 from the SEC, skipped silently). For every new
(cik, accession) pair of a tracked form: companyfacts + submissions are refetched (cache bypass), statements
re-normalised (new versions, restatements) and rewritten only when they changed, coverage + screener row
refreshed, and a `filing` event is published on Redis (`filings` channel + `filings:stream` buffer) for the
WebSocket. A Redis SET `edgar:seen_accn` (7 days) guarantees a pair is handled once even when EDGAR misbehaves.
Lag, failures (+ samples) and the mapping failure rate are written to `fund_ingest_state`
(GET /v1/fundamentals/_health).

Author: Simon-Pierre Boucher <contact@spboucher.ai>
"""
from __future__ import annotations

import argparse
import logging
import signal
import sys
import time
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent / "hfmarketdata" / "api"))

log = logging.getLogger("edgar_incremental")
_stop = False


def _sig(*_):
    global _stop
    _stop = True


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--interval", type=float, default=120.0, help="seconds between polls (default 120)")
    ap.add_argument("--once", action="store_true", help="run a single cycle and exit")
    ap.add_argument("--forms", default="10-K,10-Q,8-K,20-F", help="Atom feeds to poll")
    ap.add_argument("--no-daily-index", action="store_true")
    ap.add_argument("--rate", type=float, default=10.0)
    ap.add_argument("-v", "--verbose", action="store_true")
    args = ap.parse_args()
    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
    if not args.verbose:
        logging.getLogger("httpx").setLevel(logging.WARNING)   # one line per request otherwise (incl. expected 403/404)
    signal.signal(signal.SIGTERM, _sig)
    signal.signal(signal.SIGINT, _sig)

    from fundamentals import ingest, utcnow
    from fundamentals.edgar_client import EdgarClient
    from fundamentals.models import init_db

    init_db()
    client = EdgarClient(rate=args.rate)
    forms = tuple(f.strip() for f in args.forms.split(",") if f.strip())
    while not _stop:
        t0 = time.time()
        try:
            summary = ingest.poll_new_filings(client, forms=forms, include_daily_index=not args.no_daily_index)
            log.info("cycle: seen=%s new=%s affected=%s changed=%s events=%s errors=%s index=%s (%.1fs)", summary["seen"],
                     summary["new"], summary["affected_ciks"], summary["rows_changed"], summary["events"], summary["errors"],
                     summary["index_days"], time.time() - t0)
            if summary["rows_changed"]:                       # nothing new stored → the extracts are still fresh
                try:
                    from bulk.build import available_years, build_year
                    for y in available_years()[-2:]:          # keep the two most recent yearly extracts fresh
                        build_year(y)
                except Exception as e:  # pragma: no cover
                    log.warning("bulk rebuild failed: %s", e)
        except Exception as e:
            log.exception("cycle failed: %s", e)
            ingest._set_state("incremental", last_run_at=utcnow(), failures_add=1,
                              failure_samples_add=[f"cycle: {type(e).__name__}: {e}"])
        if args.once:
            break
        for _ in range(int(max(1.0, args.interval - (time.time() - t0)))):
            if _stop:
                break
            time.sleep(1)
    return 0


if __name__ == "__main__":
    sys.exit(main())