"""Phase C — PDF pre-processing: extract page text (PyMuPDF), detect candidate regression-table pages (density of numbers with
parentheses / asterisks and hedonic variable names), and write a compact dump per paper for LLM-assisted extraction:
data/extracted/dumps/<record_id>.txt (metadata header + abstract + candidate table pages + surrounding notes).
Also writes data/extracted/pdf_index.csv (pages, chars, n_candidate_pages, scanned flag)."""
import csv, re, sys
from pathlib import Path
import fitz, pandas as pd, yaml

ROOT = Path(__file__).resolve().parents[2]; PDFS = ROOT / "data" / "raw" / "pdfs"; DUMPS = ROOT / "data" / "extracted" / "dumps"; DUMPS.mkdir(parents=True, exist_ok=True)
tax = yaml.safe_load((ROOT / "scripts" / "harmonization" / "taxonomy.yaml").read_text())
VARWORDS = sorted({s for grp in tax.values() for v in grp.values() for s in v["synonyms"] if len(s) >= 4})
NUM_PAREN = re.compile(r"[-−–]?\d+\.\d+\s*\(\s*[-−–]?\d*\.\d+\s*\)|\d\.\d+\s*\*{1,3}|\(\s*\d+\.\d+\s*\)")
TABLE_WORDS = re.compile(r"\b(table|tableau|tabla|tabelle|coefficient|std\.? ?err|standard error|t-stat|t-value|p-value|r-squared|r2|adj|observations|log\(|ln\(|dependent variable)\b", re.I)


DECIMALS = re.compile(r"[-−–]?\d+\.\d{2,}")


def score_page(text):
    t = text.lower()
    nums = len(NUM_PAREN.findall(text)); words = len(TABLE_WORDS.findall(text)); vars_ = sum(1 for w in VARWORDS if w in t)
    dec = len(DECIMALS.findall(text)); has_table = bool(re.search(r"\btab(le|leau|la|elle)\s*[0-9ivx]", text, re.I))
    cand = nums >= 6 or (nums >= 3 and vars_ >= 3) or (has_table and dec >= 20 and vars_ >= 2 and words >= 2)
    return nums + 2 * words + vars_ + (dec // 10), nums, words, vars_, cand


def process(pdf: Path, meta: dict):
    doc = fitz.open(pdf)
    pages = [p.get_text("text") for p in doc]
    chars = sum(len(p) for p in pages)
    scanned = chars < 200 * max(len(pages), 1)
    scores = [score_page(p) for p in pages]
    cand = [i for i, s in enumerate(scores) if s[4]]
    # include the page after each candidate (table notes) and the first page (metadata)
    keep = sorted(set([0] + cand + [i + 1 for i in cand if i + 1 < len(pages)]))
    out = [f"# RECORD {pdf.stem}", f"# TITLE: {meta.get('title','')}", f"# YEAR: {meta.get('year','')}  VENUE: {meta.get('venue','')}  DOI: {meta.get('doi','')}",
           f"# AUTHORS: {meta.get('authors','')}", f"# PAGES: {len(pages)}  CANDIDATE TABLE PAGES (1-based): {[i+1 for i in cand]}", "# ABSTRACT: " + str(meta.get("abstract", ""))[:2500], ""]
    for i in keep:
        out.append(f"\n===== PAGE {i+1} (score {scores[i][0]}, numeric-paren {scores[i][1]}, table-words {scores[i][2]}, var-words {scores[i][3]}) =====\n")
        out.append(pages[i][:12000])
    (DUMPS / f"{pdf.stem}.txt").write_text("\n".join(out), encoding="utf-8")
    return {"record_id": pdf.stem, "pages": len(pages), "chars": chars, "scanned": scanned, "n_candidate_pages": len(cand), "candidate_pages": " ".join(str(i + 1) for i in cand), "dump_chars": sum(len(x) for x in out)}


def main():
    cand = pd.read_csv(ROOT / "data" / "metadata" / "stage1_candidates.csv")
    cand["record_id"] = cand.get("record_id", cand.sid).astype(str)
    meta = cand.set_index("record_id").to_dict("index")
    rows = []
    assigned = set()
    for b in (ROOT / "data" / "extracted" / "batches").glob("batch_*.txt"):
        assigned |= set(b.read_text().split())
    old_idx = ROOT / "data" / "extracted" / "pdf_index.csv"
    old = pd.read_csv(old_idx).set_index("record_id").to_dict("index") if old_idx.exists() else {}
    for pdf in sorted(PDFS.glob("*.pdf")):
        if pdf.stem in assigned and pdf.stem in old:
            rows.append({"record_id": pdf.stem, **old[pdf.stem]}); continue
        try:
            rows.append(process(pdf, meta.get(pdf.stem, {})))
        except Exception as e:
            rows.append({"record_id": pdf.stem, "pages": 0, "chars": 0, "scanned": True, "n_candidate_pages": 0, "candidate_pages": "", "dump_chars": 0, "error": repr(e)})
    pd.DataFrame(rows).to_csv(ROOT / "data" / "extracted" / "pdf_index.csv", index=False)
    df = pd.DataFrame(rows); print(len(df), "pdfs;", int(df.scanned.sum()), "scanned/empty;", int((df.n_candidate_pages > 0).sum()), "with candidate tables")


if __name__ == "__main__":
    main()