# ============================================================
# SPID — raw 10-K ingester
#
# The pre-parsed corpus only sectioned 10-Q & 8-K filings; the
# 10-K HTML was left raw.  This module cleans the raw 10-K HTML
# and extracts the project-dense Items (1 Business, 1A Risk,
# 7 MD&A, 2 Properties) into a SPID-owned `spid_sections` table
# that `v_sections` unions in — so 10-Ks flow through the same
# extraction pipeline.
# ============================================================
from __future__ import annotations

import glob
import re
import uuid
from pathlib import Path

from spid.config import get_logger, get_settings
from spid.store import Store

log = get_logger("ingest")

RAW_SEC = get_settings().source_data / "raw" / "sec"

# 10-K items we keep (where projects/capex/strategy concentrate)
_KEEP_ITEMS = {
    "1": "business", "1A": "risk_factors", "2": "properties",
    "7": "md_and_a", "7A": "quantitative_disclosures",
}
_ITEM_RE = re.compile(r"(?:^|\n)\s*ITEM\s+(\d+[AB]?)\s*[.\-:—–]", re.IGNORECASE)
_TAG_RE = re.compile(r"<[^>]+>")
_WS_RE = re.compile(r"[ \t]+")
_NL_RE = re.compile(r"\n{3,}")


def _clean_html(path: Path) -> str:
    raw = path.read_text(errors="ignore")
    try:
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(raw, "lxml")
        for t in soup(["script", "style", "table"]):
            t.decompose()
        text = soup.get_text("\n")
    except Exception:  # noqa: BLE001
        text = _TAG_RE.sub(" ", raw)
    import html as _html
    text = _html.unescape(text)
    text = _WS_RE.sub(" ", text)
    text = _NL_RE.sub("\n\n", text)
    return text


def _extract_items(text: str) -> list[tuple[str, str]]:
    matches = list(_ITEM_RE.finditer(text))
    if not matches:
        return [("full_body", text)]
    out: list[tuple[str, str]] = []
    for i, m in enumerate(matches):
        item = m.group(1).upper()
        if item not in _KEEP_ITEMS:
            continue
        start = m.start()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
        body = text[start:end].strip()
        if len(body) > 400:  # skip the table-of-contents stubs
            out.append((_KEEP_ITEMS[item], body[:600_000]))
    # keep the longest instance of each section (TOC vs real body)
    best: dict[str, str] = {}
    for name, body in out:
        if name not in best or len(body) > len(best[name]):
            best[name] = body
    return list(best.items())


def _find_primary(acc: str, cik: str) -> Path | None:
    d = RAW_SEC / cik / "10-K" / acc.replace("-", "")
    if not d.is_dir():
        return None
    cands = glob.glob(str(d / "*.htm")) + glob.glob(str(d / "*.html")) + glob.glob(str(d / "*.txt"))
    cands = [c for c in cands if "R" + "" not in Path(c).name or True]
    if not cands:
        return None
    return Path(max(cands, key=lambda p: Path(p).stat().st_size))


_SPID_SECTIONS_DDL = """
CREATE TABLE IF NOT EXISTS spid_sections (
    section_id VARCHAR PRIMARY KEY,
    accession_number VARCHAR,
    section_name VARCHAR,
    item_number VARCHAR,
    text VARCHAR,
    word_count INTEGER
);
"""


def ingest_10k(store: Store, *, ciks: list[str] | None = None, limit: int | None = None) -> dict:
    con = store.con
    con.execute(_SPID_SECTIONS_DDL)
    where = "form_type='10-K'"
    if ciks:
        q = ",".join("'" + c + "'" for c in ciks)
        where += f" AND cik IN ({q})"
    sql = f"SELECT accession_number, cik FROM src_filings WHERE {where} ORDER BY filing_date DESC"
    if limit:
        sql += f" LIMIT {int(limit)}"
    filings = con.execute(sql).fetchall()

    rows, done, ok = [], 0, 0
    for acc, cik in filings:
        done += 1
        # skip if already ingested
        if con.execute("SELECT 1 FROM spid_sections WHERE accession_number=? LIMIT 1", [acc]).fetchone():
            ok += 1
            continue
        path = _find_primary(acc, cik)
        if not path:
            continue
        try:
            text = _clean_html(path)
            for name, body in _extract_items(text):
                rows.append((str(uuid.uuid4()), acc, name, "", body, len(body.split())))
            ok += 1
        except Exception as e:  # noqa: BLE001
            log.warning("ingest failed %s: %s", acc, e)
        if len(rows) >= 400:
            con.executemany("INSERT OR REPLACE INTO spid_sections VALUES (?,?,?,?,?,?)", rows); rows = []
        if done % 50 == 0:
            log.info("  ... ingested %d/%d 10-Ks", done, len(filings))
    if rows:
        con.executemany("INSERT OR REPLACE INTO spid_sections VALUES (?,?,?,?,?,?)", rows)
    n = store.count("spid_sections")
    log.info("Ingested 10-K sections: %d filings processed, %d section rows total.", ok, n)
    return {"filings_processed": ok, "section_rows": n}