1"""Phase C — PDF pre-processing: extract page text (PyMuPDF), detect candidate regression-table pages (density of numbers with2parentheses / asterisks and hedonic variable names), and write a compact dump per paper for LLM-assisted extraction:3data/extracted/dumps/<record_id>.txt (metadata header + abstract + candidate table pages + surrounding notes).4Also writes data/extracted/pdf_index.csv (pages, chars, n_candidate_pages, scanned flag)."""5import csv, re, sys6from pathlib import Path7import fitz, pandas as pd, yaml89ROOT = Path(__file__).resolve().parents[2]; PDFS = ROOT / "data" / "raw" / "pdfs"; DUMPS = ROOT / "data" / "extracted" / "dumps"; DUMPS.mkdir(parents=True, exist_ok=True)10tax = yaml.safe_load((ROOT / "scripts" / "harmonization" / "taxonomy.yaml").read_text())11VARWORDS = sorted({s for grp in tax.values() for v in grp.values() for s in v["synonyms"] if len(s) >= 4})12NUM_PAREN = re.compile(r"[-−–]?\d+\.\d+\s*\(\s*[-−–]?\d*\.\d+\s*\)|\d\.\d+\s*\*{1,3}|\(\s*\d+\.\d+\s*\)")13TABLE_WORDS = re.compile(r"\b(table|tableau|tabla|tabelle|coefficient|std\.? ?err|standard error|t-stat|t-value|p-value|r-squared|r2|adj|observations|log\(|ln\(|dependent variable)\b", re.I)141516DECIMALS = re.compile(r"[-−–]?\d+\.\d{2,}")171819def score_page(text):20 t = text.lower()21 nums = len(NUM_PAREN.findall(text)); words = len(TABLE_WORDS.findall(text)); vars_ = sum(1 for w in VARWORDS if w in t)22 dec = len(DECIMALS.findall(text)); has_table = bool(re.search(r"\btab(le|leau|la|elle)\s*[0-9ivx]", text, re.I))23 cand = nums >= 6 or (nums >= 3 and vars_ >= 3) or (has_table and dec >= 20 and vars_ >= 2 and words >= 2)24 return nums + 2 * words + vars_ + (dec // 10), nums, words, vars_, cand252627def process(pdf: Path, meta: dict):28 doc = fitz.open(pdf)29 pages = [p.get_text("text") for p in doc]30 chars = sum(len(p) for p in pages)31 scanned = chars < 200 * max(len(pages), 1)32 scores = [score_page(p) for p in pages]33 cand = [i for i, s in enumerate(scores) if s[4]]34 # include the page after each candidate (table notes) and the first page (metadata)35 keep = sorted(set([0] + cand + [i + 1 for i in cand if i + 1 < len(pages)]))36 out = [f"# RECORD {pdf.stem}", f"# TITLE: {meta.get('title','')}", f"# YEAR: {meta.get('year','')} VENUE: {meta.get('venue','')} DOI: {meta.get('doi','')}",37 f"# AUTHORS: {meta.get('authors','')}", f"# PAGES: {len(pages)} CANDIDATE TABLE PAGES (1-based): {[i+1 for i in cand]}", "# ABSTRACT: " + str(meta.get("abstract", ""))[:2500], ""]38 for i in keep:39 out.append(f"\n===== PAGE {i+1} (score {scores[i][0]}, numeric-paren {scores[i][1]}, table-words {scores[i][2]}, var-words {scores[i][3]}) =====\n")40 out.append(pages[i][:12000])41 (DUMPS / f"{pdf.stem}.txt").write_text("\n".join(out), encoding="utf-8")42 return {"record_id": pdf.stem, "pages": len(pages), "chars": chars, "scanned": scanned, "n_candidate_pages": len(cand), "candidate_pages": " ".join(str(i + 1) for i in cand), "dump_chars": sum(len(x) for x in out)}434445def main():46 cand = pd.read_csv(ROOT / "data" / "metadata" / "stage1_candidates.csv")47 cand["record_id"] = cand.get("record_id", cand.sid).astype(str)48 meta = cand.set_index("record_id").to_dict("index")49 rows = []50 assigned = set()51 for b in (ROOT / "data" / "extracted" / "batches").glob("batch_*.txt"):52 assigned |= set(b.read_text().split())53 old_idx = ROOT / "data" / "extracted" / "pdf_index.csv"54 old = pd.read_csv(old_idx).set_index("record_id").to_dict("index") if old_idx.exists() else {}55 for pdf in sorted(PDFS.glob("*.pdf")):56 if pdf.stem in assigned and pdf.stem in old:57 rows.append({"record_id": pdf.stem, **old[pdf.stem]}); continue58 try:59 rows.append(process(pdf, meta.get(pdf.stem, {})))60 except Exception as e:61 rows.append({"record_id": pdf.stem, "pages": 0, "chars": 0, "scanned": True, "n_candidate_pages": 0, "candidate_pages": "", "dump_chars": 0, "error": repr(e)})62 pd.DataFrame(rows).to_csv(ROOT / "data" / "extracted" / "pdf_index.csv", index=False)63 df = pd.DataFrame(rows); print(len(df), "pdfs;", int(df.scanned.sum()), "scanned/empty;", int((df.n_candidate_pages > 0).sum()), "with candidate tables")646566if __name__ == "__main__":67 main()68