"""N1 - ledger reconciliation (clusters C, G, J-throughput, H-log numbers). Read-only.

(a) _PROJECT_GATE over every ledger section (DuckDB RE2, case-insensitive) -> gate pass and the
    true call count by form x item x run; productive rate / mentions / USD per call.
(b) backfilled block (rows at min(extracted_at)), word_count < 30 rows, outage-window rows,
    zero-mention share per flush timestamp and per 5-minute bin.
(c) sections where ledger n_mentions > stored mentions (id collisions).
(d) run-log parser (October data/logs/spid_extract.log, June project-db/logs/extract.log).
(e) wall-clock / outage / effective throughput from ledger timestamps.
(f) backend values, confidence = 0.600 count, amount_raw formatting rule, min confidence by run.
"""
from __future__ import annotations
import json, re, sys, datetime as dt
from pathlib import Path
from collections import Counter, defaultdict
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import *

GATE = (r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|"
        r"deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|"
        r"pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*")
OUTAGE_START = "2026-10-28 01:10:00"   # window named in the consolidated review (THEO-05 / DATA-02)
OUTAGE_END = "2026-10-28 06:22:00"
PAPER_USD_TOTAL = 775.0               # construct.tex:35 'roughly 775 USD'
PAPER_USD_PER_SECTION = 0.0008        # construct.tex:35
JUNE_LOG_SECTIONS = 153894            # project-db/logs/extract.log FULL EXTRACT COMPLETE
JUNE_LOG_MENTIONS = 39930             # TOTAL mentions in DB after the June run

r = Run("ledger")
con = connect(SPID_DB)

# ---- (a) section metadata + gate flag for every section visible to the pipeline --------------
print("building sec_meta (regex over the parquet text, streamed) ...", flush=True)
con.execute(f"""
CREATE TEMP TABLE sec_meta AS
SELECT section_id, accession_number, section_name, word_count, length(text) AS nchar,
       regexp_matches(text, '{GATE}', 'i') AS gate, 'parsed' AS segmenter
FROM src_sections
UNION ALL
SELECT section_id, accession_number, section_name, word_count, length(text) AS nchar,
       regexp_matches(text, '{GATE}', 'i') AS gate, 'legacy_ingest' AS segmenter
FROM spid_sections
""")
r.notes["sec_meta_rows"] = con.execute("select count(*) from sec_meta").fetchone()[0]

con.execute("""
CREATE TEMP TABLE stored AS
SELECT section_id, count(*) AS stored_mentions,
       list(project_type || ':' || left(project_name, 40)) AS stored_keys
FROM project_mentions GROUP BY 1
""")
tmin = con.execute("select min(extracted_at) from extracted_sections").fetchone()[0]
r.notes["min_extracted_at"] = str(tmin)
con.execute(f"""
CREATE TEMP TABLE ledger AS
SELECT e.section_id, e.accession_number, e.backend, e.n_mentions, e.extracted_at,
       CASE WHEN e.extracted_at = TIMESTAMP '{tmin}' THEN 'backfilled_june' ELSE 'october' END AS run,
       m.section_name, m.word_count, m.nchar, m.gate, m.segmenter,
       f.form_type, f.filing_date, f.cik, f.fiscal_year,
       co.in_sp500, co.in_russell3000,
       coalesce(s.stored_mentions, 0) AS stored_mentions, s.stored_keys
FROM extracted_sections e
LEFT JOIN sec_meta m USING (section_id)
LEFT JOIN src_filings f ON f.accession_number = e.accession_number
LEFT JOIN src_companies co ON f.cik = co.cik
LEFT JOIN stored s USING (section_id)
""")
n_ledger = con.execute("select count(*) from ledger").fetchone()[0]
n_unmatched = con.execute("select count(*) from ledger where section_name is null").fetchone()[0]
r.notes["ledger_rows"] = n_ledger
r.notes["ledger_rows_without_section_metadata"] = n_unmatched

agg = """
  count(*) AS ledger_rows,
  count(*) FILTER (WHERE word_count < 30) AS lt30_words,
  count(*) FILTER (WHERE gate) AS gate_pass,
  count(*) FILTER (WHERE gate AND word_count >= 30) AS calls,
  count(*) FILTER (WHERE NOT gate AND n_mentions = 0) AS gate_rejected_zero,
  count(*) FILTER (WHERE NOT gate AND n_mentions > 0) AS gate_fail_but_mentions,
  count(*) FILTER (WHERE n_mentions > 0) AS with_mentions,
  sum(n_mentions) AS ledger_mentions,
  sum(stored_mentions) AS stored_mentions,
  count(DISTINCT cik) AS issuers,
  round(100.0 * count(*) FILTER (WHERE n_mentions > 0) / count(*), 2) AS productive_pct_ledger,
  round(100.0 * count(*) FILTER (WHERE n_mentions > 0) / nullif(count(*) FILTER (WHERE gate AND word_count >= 30),0), 2) AS productive_pct_calls,
  round(sum(n_mentions) * 1.0 / count(*), 4) AS mentions_per_ledger_row,
  round(sum(n_mentions) * 1.0 / nullif(count(*) FILTER (WHERE gate AND word_count >= 30),0), 4) AS mentions_per_call
"""
r.dump("gate_pass_by_form_item.csv", con, f"SELECT form_type, section_name AS item, run, {agg} FROM ledger GROUP BY 1,2,3 ORDER BY 1,2,3")
r.dump("gate_pass_by_form.csv", con, f"SELECT form_type, {agg} FROM ledger GROUP BY 1 ORDER BY 1")
r.dump("gate_pass_by_run.csv", con, f"SELECT run, {agg} FROM ledger GROUP BY 1 ORDER BY 1")
r.dump("gate_pass_by_form_run.csv", con, f"SELECT form_type, run, {agg} FROM ledger GROUP BY 1,2 ORDER BY 1,2")
tot = con.execute(f"SELECT {agg} FROM ledger").fetchone()
cols = ["ledger_rows","lt30_words","gate_pass","calls","gate_rejected_zero","gate_fail_but_mentions","with_mentions","ledger_mentions","stored_mentions","issuers","productive_pct_ledger","productive_pct_calls","mentions_per_ledger_row","mentions_per_call"]
T = dict(zip(cols, tot))
oct_calls = con.execute("select count(*) filter (where gate and word_count>=30) from ledger where run='october'").fetchone()[0]
oct_rows = con.execute("select count(*) from ledger where run='october'").fetchone()[0]
T["usd_per_call_paper_total_775"] = round(PAPER_USD_TOTAL / oct_calls, 6)
T["usd_per_october_row_paper_total_775"] = round(PAPER_USD_TOTAL / oct_rows, 6)
T["usd_per_call_from_0_0008_x_ledger"] = round(PAPER_USD_PER_SECTION * n_ledger / T["calls"], 6)
T["october_rows"] = oct_rows; T["october_calls"] = oct_calls
T["headline_before"] = {"sections_submitted": 1123098, "productive_pct": 15.0, "mentions_per_section": 0.27, "usd_per_section": 0.0008}
r.write_json("gate_pass_totals.json", T)

# ---- (b) backfilled block, < 30 words, outage window, flush timestamps -----------------------
B = {}
B["min_extracted_at"] = str(tmin)
B["rows_at_min_timestamp"] = con.execute("select count(*) from ledger where run='backfilled_june'").fetchone()[0]
B["june_log_sections"] = JUNE_LOG_SECTIONS
B["gap_rows_minus_log"] = B["rows_at_min_timestamp"] - JUNE_LOG_SECTIONS
B["block_ledger_mentions"] = con.execute("select sum(n_mentions) from ledger where run='backfilled_june'").fetchone()[0]
B["block_stored_mentions"] = con.execute("select sum(stored_mentions) from ledger where run='backfilled_june'").fetchone()[0]
B["june_log_mentions"] = JUNE_LOG_MENTIONS
B["block_lt30_words"] = con.execute("select count(*) from ledger where run='backfilled_june' and word_count<30").fetchone()[0]
B["block_gate_fail"] = con.execute("select count(*) from ledger where run='backfilled_june' and not gate").fetchone()[0]
B["block_lt30_or_gate_fail"] = con.execute("select count(*) from ledger where run='backfilled_june' and (word_count<30 or not gate)").fetchone()[0]
B["block_ge30_and_gate"] = con.execute("select count(*) from ledger where run='backfilled_june' and word_count>=30 and gate").fetchone()[0]
B["october_lt30_words"] = con.execute("select count(*) from ledger where run='october' and word_count<30").fetchone()[0]
B["block_by_form"] = con.execute("select form_type, count(*), sum(n_mentions), count(*) filter (where word_count<30) from ledger where run='backfilled_june' group by 1 order by 1").fetchall()
B["block_by_segmenter"] = con.execute("select segmenter, count(*) from ledger where run='backfilled_june' group by 1").fetchall()
B["block_sp500_share"] = con.execute("select count(*) filter (where in_sp500), count(*) from ledger where run='backfilled_june'").fetchone()
B["block_ciks"] = con.execute("select count(distinct cik) from ledger where run='backfilled_june'").fetchone()[0]
B["outage_window"] = [OUTAGE_START, OUTAGE_END]
B["rows_in_outage_window"] = con.execute(f"select count(*), sum(n_mentions) from ledger where extracted_at between TIMESTAMP '{OUTAGE_START}' and TIMESTAMP '{OUTAGE_END}'").fetchone()
B["rows_0110_0640"] = con.execute("select count(*), sum(n_mentions) from ledger where extracted_at between TIMESTAMP '2026-10-28 01:10:00' and TIMESTAMP '2026-10-28 06:40:53'").fetchone()
B["rows_by_hour_1028"] = con.execute("select hour(extracted_at) h, count(*), count(*) filter (where n_mentions=0) zeros, sum(n_mentions) from ledger where run='october' and extracted_at::date = DATE '2026-10-28' group by 1 order by 1").fetchall()
B["rows_by_hour_1027"] = con.execute("select hour(extracted_at) h, count(*), count(*) filter (where n_mentions=0) zeros, sum(n_mentions) from ledger where run='october' and extracted_at::date = DATE '2026-10-27' group by 1 order by 1").fetchall()
B["paper_outage_rows_149352"] = 149352
r.write_json("backfill_block.json", B)

r.dump("ledger_by_5min.csv", con, """
SELECT time_bucket(INTERVAL '5 minutes', extracted_at) AS bin5, count(*) AS rows, count(*) FILTER (WHERE n_mentions>0) AS with_mentions,
       sum(n_mentions) AS mentions, round(100.0*count(*) FILTER (WHERE n_mentions=0)/count(*),2) AS zero_pct,
       count(*) FILTER (WHERE gate AND word_count>=30) AS calls, count(DISTINCT extracted_at) AS flushes
FROM ledger WHERE run='october' GROUP BY 1 ORDER BY 1""")
r.dump("ledger_by_minute.csv", con, """
SELECT time_bucket(INTERVAL '1 minute', extracted_at) AS bin1, count(*) AS rows, count(*) FILTER (WHERE n_mentions>0) AS with_mentions,
       sum(n_mentions) AS mentions, round(100.0*count(*) FILTER (WHERE n_mentions=0)/count(*),2) AS zero_pct,
       count(*) FILTER (WHERE gate AND word_count>=30) AS calls,
       round(100.0*count(*) FILTER (WHERE n_mentions=0 AND gate AND word_count>=30)/nullif(count(*) FILTER (WHERE gate AND word_count>=30),0),2) AS zero_pct_among_calls
FROM ledger WHERE run='october' GROUP BY 1 ORDER BY 1""")
B["october_distinct_timestamps"], B["october_rows_check"] = con.execute("select count(distinct extracted_at), count(*) from ledger where run='october'").fetchone()
B["timestamp_granularity_note"] = "extracted_at is stamped per row (executemany INSERT OR REPLACE, DEFAULT current_timestamp), not per flush: distinct timestamps = rows"
r.dump("flush_zero_groups_after_0622.csv", con, f"""
SELECT time_bucket(INTERVAL '1 minute', extracted_at) AS bin1, count(*) AS rows, count(*) FILTER (WHERE n_mentions=0) AS zeros,
       round(100.0*count(*) FILTER (WHERE n_mentions=0)/count(*),2) AS zero_pct,
       count(*) FILTER (WHERE gate AND word_count>=30) AS calls,
       round(100.0*count(*) FILTER (WHERE n_mentions=0 AND gate AND word_count>=30)/nullif(count(*) FILTER (WHERE gate AND word_count>=30),0),2) AS zero_pct_among_calls
FROM ledger WHERE run='october' AND extracted_at > TIMESTAMP '{OUTAGE_END}'
GROUP BY 1 HAVING count(*) >= 50 AND count(*) FILTER (WHERE n_mentions=0 AND gate AND word_count>=30) >= 0.99*count(*) FILTER (WHERE gate AND word_count>=30) ORDER BY 1""")
# the same test before the outage, for comparison
r.dump("flush_zero_groups_before_0110.csv", con, f"""
SELECT time_bucket(INTERVAL '1 minute', extracted_at) AS bin1, count(*) AS rows, count(*) FILTER (WHERE n_mentions=0) AS zeros,
       round(100.0*count(*) FILTER (WHERE n_mentions=0)/count(*),2) AS zero_pct,
       count(*) FILTER (WHERE gate AND word_count>=30) AS calls,
       round(100.0*count(*) FILTER (WHERE n_mentions=0 AND gate AND word_count>=30)/nullif(count(*) FILTER (WHERE gate AND word_count>=30),0),2) AS zero_pct_among_calls
FROM ledger WHERE run='october' AND extracted_at < TIMESTAMP '{OUTAGE_START}'
GROUP BY 1 HAVING count(*) >= 50 AND count(*) FILTER (WHERE n_mentions=0 AND gate AND word_count>=30) >= 0.99*count(*) FILTER (WHERE gate AND word_count>=30) ORDER BY 1""")

# ---- (c) id collisions: ledger n_mentions > stored ------------------------------------------
r.dump("ledger_collisions.csv", con, """
SELECT section_id, accession_number, form_type, section_name AS item, cik, run, n_mentions AS ledger_n, stored_mentions AS stored_n,
       n_mentions - stored_mentions AS deficit, stored_keys
FROM ledger WHERE n_mentions > stored_mentions ORDER BY deficit DESC, section_id""")
r.dump("ledger_collisions_by_form.csv", con, """
SELECT form_type, count(*) AS sections, sum(n_mentions - stored_mentions) AS excess, sum(n_mentions) AS ledger_mentions, sum(stored_mentions) AS stored_mentions
FROM ledger WHERE n_mentions > stored_mentions GROUP BY 1 ORDER BY 1""")
C = {}
C["sections_ledger_gt_stored"], C["excess_mentions"] = con.execute("select count(*), sum(n_mentions-stored_mentions) from ledger where n_mentions > stored_mentions").fetchone()
C["sections_ledger_lt_stored"], C["deficit_mentions"] = con.execute("select count(*), sum(stored_mentions-n_mentions) from ledger where n_mentions < stored_mentions").fetchone()
C["sum_ledger_n_mentions"] = con.execute("select sum(n_mentions) from ledger").fetchone()[0]
C["stored_project_mentions"] = con.execute("select count(*) from project_mentions").fetchone()[0]
C["mentions_whose_section_not_in_ledger"] = con.execute("select count(*) from project_mentions m where not exists (select 1 from extracted_sections e where e.section_id=m.section_id)").fetchone()[0]
C["ledger_vs_stored_by_form"] = con.execute("select form_type, sum(n_mentions), sum(stored_mentions), sum(n_mentions)-sum(stored_mentions) from ledger group by 1 order by 1").fetchall()
C["duplicate_key_sections_in_stored"] = con.execute("""
  select count(*) from (select section_id from project_mentions group by section_id, project_type, left(project_name,40) having count(*)>1)""").fetchone()[0]
r.write_json("collisions.json", C)

# ---- (f) misc code-vs-data checks ---------------------------------------------------------------
F = {}
F["distinct_backend_ledger"] = [x[0] for x in con.execute("select distinct backend from extracted_sections").fetchall()]
F["distinct_backend_mentions"] = [x[0] for x in con.execute("select distinct backend from project_mentions").fetchall()]
F["confidence_exact_0600_total"] = con.execute("select count(*) from project_mentions where round(confidence,3)=0.600").fetchone()[0]
F["confidence_exact_0600_by_run"] = con.execute("""
  select l.run, count(*) from project_mentions m join ledger l using(section_id) where round(m.confidence,3)=0.600 group by 1""").fetchall()
F["confidence_bins_by_run"] = con.execute("""
  select l.run, round(m.confidence,1) c, count(*) from project_mentions m join ledger l using(section_id) group by 1,2 order by 1,2""").fetchall()
F["min_confidence_by_run"] = con.execute("select l.run, min(m.confidence), max(m.confidence), count(*) from project_mentions m join ledger l using(section_id) group by 1").fetchall()
F["mentions_below_045"] = con.execute("select count(*) from project_mentions where confidence < 0.45").fetchone()[0]
F["amount_raw_rule_mismatch"] = con.execute("""
  select count(*) from project_mentions where amount_usd is not null and amount_raw is distinct from ('$' || format('{:,.0f}', amount_usd))""").fetchone()[0]
F["amount_raw_rule_mismatch_rows"] = con.execute("""
  select mention_id, amount_usd, amount_raw from project_mentions where amount_usd is not null and amount_raw is distinct from ('$' || format('{:,.0f}', amount_usd)) limit 20""").fetchall()
F["amount_raw_null_with_amount"] = con.execute("select count(*) from project_mentions where amount_usd is not null and amount_raw is null").fetchone()[0]
F["amount_raw_notnull_without_amount"] = con.execute("select count(*) from project_mentions where amount_usd is null and amount_raw is not null").fetchone()[0]
F["amount_le_0"] = con.execute("select count(*) from project_mentions where amount_usd <= 0").fetchone()[0]
F["mentions_with_run"] = con.execute("select l.run, count(*) from project_mentions m join ledger l using(section_id) group by 1").fetchall()
F["mentions_backfilled_block_vs_june_log"] = {"block_stored": B["block_stored_mentions"], "june_log": JUNE_LOG_MENTIONS}
F["fiscal_year_null_mentions_by_run"] = con.execute("select l.run, count(*) filter (where m.fiscal_year is null), count(*) from project_mentions m join ledger l using(section_id) group by 1").fetchall()
r.write_json("code_vs_data_checks.json", F)

# ---- (e) throughput from the ledger ----------------------------------------------------------
E = {}
ts = con.execute("select distinct extracted_at from ledger where run='october' order by 1").fetchall()
ts = [t[0] for t in ts]
E["october_first_flush"] = str(ts[0]); E["october_last_flush"] = str(ts[-1])
E["october_distinct_flush_timestamps"] = len(ts)
E["october_rows"] = oct_rows
E["october_calls"] = oct_calls
wall = (ts[-1] - ts[0]).total_seconds()
gaps = sorted(((ts[i+1]-ts[i]).total_seconds(), str(ts[i]), str(ts[i+1])) for i in range(len(ts)-1))
E["largest_gaps_seconds"] = gaps[-5:]
outage_gap = gaps[-1][0]
E["wall_clock_hours_first_to_last_flush"] = round(wall/3600, 3)
E["largest_gap_hours"] = round(outage_gap/3600, 3)
E["effective_hours_excluding_largest_gap"] = round((wall-outage_gap)/3600, 3)
E["rows_per_second_wall"] = round(oct_rows/wall, 2)
E["rows_per_second_effective"] = round(oct_rows/(wall-outage_gap), 2)
E["calls_per_second_effective"] = round(oct_calls/(wall-outage_gap), 2)
E["paper_claim"] = {"sections": 968134, "hours": "about twelve", "rate": "27-36 sections per second", "workers": 48}
E["paper_rate_implied_by_12h"] = round(968134/(12*3600), 2)
# segment rates: before the outage and after
pre = con.execute(f"select count(*), min(extracted_at), max(extracted_at) from ledger where run='october' and extracted_at < TIMESTAMP '{OUTAGE_START}'").fetchone()
post = con.execute(f"select count(*), min(extracted_at), max(extracted_at) from ledger where run='october' and extracted_at > TIMESTAMP '{OUTAGE_END}'").fetchone()
E["segment_before_outage"] = {"rows": pre[0], "from": str(pre[1]), "to": str(pre[2]), "rows_per_s": round(pre[0]/max(1,(pre[2]-pre[1]).total_seconds()),2)}
E["segment_after_outage"] = {"rows": post[0], "from": str(post[1]), "to": str(post[2]), "rows_per_s": round(post[0]/max(1,(post[2]-post[1]).total_seconds()),2)}
E["rows_by_hour_october"] = con.execute("select date_trunc('hour', extracted_at) h, count(*) from ledger where run='october' group by 1 order by 1").fetchall()
r.write_json("throughput.json", E)

# ---- (d) log parsers ---------------------------------------------------------------------------
LINE = re.compile(r"^(\d\d):(\d\d):(\d\d) (\w+)\s+(\S+) \| (.*)$")
def parse_log(path: Path, start_date: dt.date):
    day = start_date; prev = None; out = []
    for raw in open(path, errors="replace"):
        m = LINE.match(raw.rstrip("\n"))
        if not m:
            out.append((None, None, None, raw.rstrip("\n"))); continue
        h, mi, s = int(m.group(1)), int(m.group(2)), int(m.group(3))
        t = dt.time(h, mi, s)
        if prev is not None and (dt.datetime.combine(day, prev) - dt.datetime.combine(day, t)).total_seconds() > 3600:
            day = day + dt.timedelta(days=1)
        prev = t
        out.append((dt.datetime.combine(day, t), m.group(4), m.group(5), m.group(6)))
    return out

def classify(msg: str) -> str:
    if "retryable" in msg: return "retryable_http_429_5xx"
    if "Connection reset" in msg or "ConnectError" in msg or "ReadError" in msg or "Errno" in msg: return "connection_error"
    if "Unterminated string" in msg or "Expecting" in msg or "Extra data" in msg or "JSON" in msg: return "json_decode_error"
    if "KeyError" in msg: return "key_error"
    if "timed out" in msg.lower() or "Timeout" in msg: return "timeout"
    return "other"

def summarize(path: Path, start_date: dt.date, window=None):
    rows = parse_log(path, start_date)
    S = {"path": str(path), "lines": len(rows), "first": str(next((r_[0] for r_ in rows if r_[0]), None)), "last": str(next((r_[0] for r_ in reversed(rows) if r_[0]), None))}
    fails = [(t, classify(msg)) for t, lvl, name, msg in rows if msg and "LLM call failed" in msg]
    S["llm_call_failed_total"] = len(fails)
    S["llm_call_failed_by_reason"] = dict(Counter(c for _, c in fails))
    byhour = Counter((t.strftime("%Y-%m-%d %H") if t else "?") for t, _ in fails)
    S["llm_call_failed_by_hour"] = dict(sorted(byhour.items()))
    if window:
        w0, w1 = window
        inside = [(t, c) for t, c in fails if t and w0 <= t <= w1]
        outside = [(t, c) for t, c in fails if not (t and w0 <= t <= w1)]
        S["window"] = [str(w0), str(w1)]
        S["llm_call_failed_inside_window"] = len(inside)
        S["llm_call_failed_outside_window"] = len(outside)
        S["outside_window_by_reason"] = dict(Counter(c for _, c in outside))
        S["outside_window_list"] = [(str(t), c) for t, c in outside]
        if fails:
            S["first_failure"] = str(min(t for t, _ in fails if t)); S["last_failure"] = str(max(t for t, _ in fails if t))
            ins = [t for t, _ in inside]
            if ins: S["first_failure_in_window"] = str(min(ins)); S["last_failure_in_window"] = str(max(ins))
    S["section_failed"] = sum(1 for t, lvl, name, msg in rows if msg and "section failed" in msg)
    S["batch_failed_lines"] = sum(1 for t, lvl, name, msg in rows if msg and "BATCH" in msg and "FAILED" in msg)
    S["pending_lines"] = [(str(t), msg) for t, lvl, name, msg in rows if msg and msg.startswith("pending:")]
    S["ciks_total_lines"] = [(str(t), msg) for t, lvl, name, msg in rows if msg and msg.startswith("CIKs total")]
    S["run_complete_lines"] = [(str(t), msg) for t, lvl, name, msg in rows if msg and ("RUN COMPLETE" in msg or "EXTRACT COMPLETE" in msg)]
    ext = [int(re.search(r"Extracting from (\d+) sections", msg).group(1)) for t, lvl, name, msg in rows if msg and "Extracting from" in msg]
    S["extracting_from_lines"] = len(ext); S["extracting_from_sum_sections"] = sum(ext)
    done = [(int(a), int(b)) for t, lvl, name, msg in rows for a, b in re.findall(r"Extraction complete: (\d+) mentions from (\d+) sections", msg or "")]
    S["extraction_complete_lines"] = len(done); S["extraction_complete_sum_sections"] = sum(b for a, b in done); S["extraction_complete_sum_mentions"] = sum(a for a, b in done)
    S["workers_values"] = sorted(set(re.findall(r"workers=(\d+)", "\n".join(msg or "" for *_, msg in rows))))
    S["batch_values"] = sorted(set(re.findall(r"batch=(\d+)", "\n".join(msg or "" for *_, msg in rows))))
    S["model_id_strings"] = sorted(set(re.findall(r"gpt-[\w\-\.]+", open(path, errors="replace").read())))
    return S, fails

w0 = dt.datetime.fromisoformat(OUTAGE_START); w1 = dt.datetime.fromisoformat(OUTAGE_END)
oct_log, oct_fails = summarize(RDB / "data/logs/spid_extract.log", dt.date(2026, 10, 27), (w0, w1))
jun_log, jun_fails = summarize(RDB / "project-db/logs/extract.log", dt.date(2026, 6, 18), None)
# other October log files (small)
others = {}
for nm in ["spid.log", "spid_resume.log", "spid_post.log", "spid_interim.log"]:
    p = RDB / "data/logs" / nm
    if p.exists():
        txt = open(p, errors="replace").read()
        others[nm] = {"lines": txt.count("\n"), "head": txt[:1500]}
pend = RDB / "data/logs/spid_pending.json"
if pend.exists():
    try:
        pj = json.load(open(pend)); others["spid_pending.json"] = {k: (v if not isinstance(v, list) else v[:20]) for k, v in pj.items()} if isinstance(pj, dict) else str(pj)[:1500]
    except Exception as ex:
        others["spid_pending.json"] = str(ex)
# outage window measured from the log itself: the contiguous block of 'retryable' failures on 2026-10-28 between 01:00 and 07:00
retry_ts = [t for t, c in oct_fails if t and c == "retryable_http_429_5xx" and dt.datetime(2026,10,28,1,0) <= t <= dt.datetime(2026,10,28,7,0)]
oct_log["outage_window_from_log"] = [str(min(retry_ts)), str(max(retry_ts)), len(retry_ts)]
w0l, w1l = min(retry_ts), max(retry_ts)
oct_log["llm_call_failed_outside_log_window"] = [(str(t), c) for t, c in oct_fails if not (t and w0l <= t <= w1l)]
oct_log["silent_empty_results_outside_outage"] = len(oct_log["llm_call_failed_outside_log_window"])
oct_log["retryable_failures_outside_log_window"] = sum(1 for t, c in oct_fails if c == "retryable_http_429_5xx" and not (t and w0l <= t <= w1l))
oct_log["ledger_rows_in_log_window"] = con.execute(f"select count(*), sum(n_mentions) from ledger where extracted_at between TIMESTAMP '{w0l}' and TIMESTAMP '{w1l}'").fetchone()
oct_log["ledger_rows_in_largest_gap"] = con.execute(f"select count(*) from ledger where extracted_at > TIMESTAMP '{gaps[-1][1]}' and extracted_at < TIMESTAMP '{gaps[-1][2]}'").fetchone()[0]
pend2 = int(re.search(r"pending: (\d+)", oct_log["pending_lines"][1][1]).group(1)) if len(oct_log["pending_lines"]) > 1 else None
oct_log["outage_rows_reconciliation"] = {"extraction_complete_sum_sections": oct_log["extraction_complete_sum_sections"], "pending_at_second_start": pend2,
    "difference": (oct_log["extraction_complete_sum_sections"] - pend2) if pend2 else None, "paper_research_log_149352": 149352,
    "pending_at_resume": int(re.search(r"pending: (\d+)", oct_log["pending_lines"][-1][1]).group(1)),
    "rows_before_outage_plus_in_window": con.execute(f"select count(*) from ledger where run='october' and extracted_at < TIMESTAMP '{gaps[-1][2]}'").fetchone()[0]}
# batch-level productivity from the log (silent-failure test, THEO-05): mentions per section per batch
def batch_rows(rows_, run_):
    out = []
    for t, lvl, name, msg in rows_:
        m_ = re.search(r"BATCH (\d+)/(\d+) done \| batch_sec=(\d+) batch_men=(\d+)", msg or "")
        if m_:
            out.append((run_, int(m_.group(1)), int(m_.group(2)), str(t), int(m_.group(3)), int(m_.group(4)), round(int(m_.group(4))/max(1,int(m_.group(3))), 4)))
    return out
oct_rows_ = parse_log(RDB / "data/logs/spid_extract.log", dt.date(2026, 10, 27)); jun_rows_ = parse_log(RDB / "project-db/logs/extract.log", dt.date(2026, 6, 18))
br = batch_rows(oct_rows_, "october") + batch_rows(jun_rows_, "june")
import statistics
med_oct = statistics.median([b[6] for b in br if b[0] == "october"]); med_jun = statistics.median([b[6] for b in br if b[0] == "june"])
br2 = [b + (round(b[6]/(med_oct if b[0]=="october" else med_jun), 3), (b[6] < 0.25*(med_oct if b[0]=="october" else med_jun))) for b in br]
r.write_csv("batch_productivity_from_log.csv", ["run", "batch", "n_batches", "done_at", "sections", "mentions", "mentions_per_section", "ratio_to_median", "anomalous_lt_25pct_of_median"], br2)
oct_log["batch_productivity_median_mentions_per_section"] = med_oct
oct_log["anomalous_batches"] = [b for b in br2 if b[0] == "october" and b[8]]
jun_log["batch_productivity_median_mentions_per_section"] = med_jun
jun_log["anomalous_batches"] = [b for b in br2 if b[0] == "june" and b[8]]
# the first resumed batch: rows stamped between its start and end, with the expected productive count at the form x item rates of the rest of the October run
b1 = [b for b in br2 if b[0] == "october" and b[1] == 1 and b[2] == 81]
if b1:
    b1_end = b1[0][3]
    b1_start = str(dt.datetime.fromisoformat(oct_log["pending_lines"][-1][0]))
    con.execute(f"CREATE TEMP TABLE b1 AS SELECT * FROM ledger WHERE run='october' AND extracted_at BETWEEN TIMESTAMP '{b1_start}' AND TIMESTAMP '{b1_end}'")
    r.dump("resume_batch1_vs_expected.csv", con, """
    WITH rates AS (SELECT form_type, section_name, count(*) FILTER (WHERE n_mentions>0)*1.0/nullif(count(*) FILTER (WHERE gate AND word_count>=30),0) p_prod, sum(n_mentions)*1.0/nullif(count(*) FILTER (WHERE gate AND word_count>=30),0) m_per_call
                   FROM ledger WHERE run='october' AND section_id NOT IN (SELECT section_id FROM b1) GROUP BY 1,2)
    SELECT b.form_type, b.section_name AS item, count(*) rows, count(*) FILTER (WHERE gate AND word_count>=30) calls, count(*) FILTER (WHERE n_mentions>0) with_mentions, sum(n_mentions) mentions,
           round(sum(CASE WHEN gate AND word_count>=30 THEN r.p_prod ELSE 0 END),1) expected_with_mentions, round(sum(CASE WHEN gate AND word_count>=30 THEN r.m_per_call ELSE 0 END),1) expected_mentions,
           count(DISTINCT b.cik) ciks
    FROM b1 b LEFT JOIN rates r ON b.form_type=r.form_type AND b.section_name=r.section_name GROUP BY 1,2 ORDER BY 1,2""")
    oct_log["resume_batch1"] = {"window": [b1_start, b1_end], "log_sections": b1[0][4], "log_mentions": b1[0][5],
        "ledger_rows": con.execute("select count(*) from b1").fetchone()[0],
        "ledger_calls": con.execute("select count(*) from b1 where gate and word_count>=30").fetchone()[0],
        "ledger_with_mentions": con.execute("select count(*) from b1 where n_mentions>0").fetchone()[0],
        "ledger_mentions": con.execute("select sum(n_mentions) from b1").fetchone()[0],
        "ciks": con.execute("select list(distinct cik order by cik) from b1").fetchone()[0],
        "expected_with_mentions": con.execute("""WITH rates AS (SELECT form_type, section_name, count(*) FILTER (WHERE n_mentions>0)*1.0/nullif(count(*) FILTER (WHERE gate AND word_count>=30),0) p FROM ledger WHERE run='october' AND section_id NOT IN (SELECT section_id FROM b1) GROUP BY 1,2)
             SELECT round(sum(CASE WHEN gate AND word_count>=30 THEN r.p ELSE 0 END),1) FROM b1 b LEFT JOIN rates r ON b.form_type=r.form_type AND b.section_name=r.section_name""").fetchone()[0],
        "failed_call_lines_in_window": sum(1 for t, c in oct_fails if t and dt.datetime.fromisoformat(b1_start) <= t <= dt.datetime.fromisoformat(b1_end))}
r.write_json("log_summary.json", {"october": oct_log, "june": jun_log, "other_logs": others,
                                  "june_log_date_assumed": "2026-06-18 (file mtime; the log carries time-of-day only)"})
r.write_csv("log_failures_by_hour.csv", ["run", "hour", "reason", "n"],
            [("october", h, c, n) for (h, c), n in sorted(Counter((t.strftime("%Y-%m-%d %H") if t else "?", c) for t, c in oct_fails).items())] +
            [("june", h, c, n) for (h, c), n in sorted(Counter((t.strftime("%Y-%m-%d %H") if t else "?", c) for t, c in jun_fails).items())])

# ---- Table 3 replacement input ---------------------------------------------------------------
r.dump("table3_ledger_vs_stored.csv", con, """
SELECT form_type, count(*) AS ledger_rows, count(*) FILTER (WHERE gate AND word_count>=30) AS calls,
       count(*) FILTER (WHERE n_mentions>0) AS with_mentions, sum(n_mentions) AS returned_by_model_before_dedup,
       sum(stored_mentions) AS stored_mentions, count(DISTINCT cik) AS issuers,
       count(DISTINCT cik) FILTER (WHERE n_mentions>0) AS issuers_with_mentions
FROM ledger GROUP BY 1 ORDER BY 1""")
r.notes["issuers_total_distinct_in_ledger"] = con.execute("select count(distinct cik) from ledger").fetchone()[0]
r.notes["issuers_with_filings"] = con.execute("select count(distinct cik) from src_filings").fetchone()[0]
r.finish()