1"""Section-role mapping and annotation-unit construction (respects config/rights.yaml excerpt limits)."""2from __future__ import annotations3import re, json4from pathlib import Path5from . import config67ROLE_RULES = [8 ("introduction", r"^introduction$|^intro"), ("literature", r"literature|related (work|research)|prior (work|research)|background"),9 ("theory", r"model|theor|framework|hypothes|equilibrium|setup|setting|conceptual|propositions?|proof"),10 ("data", r"^data|data and|sample|measurement|variables?|descriptive|institutional (background|setting|detail)"),11 ("empirical_design", r"methodolog|empirical (strategy|design|approach|specification|framework|methodology)|identification|econometric|estimation|research design"),12 ("results", r"result|evidence|findings|estimates|main (analysis|tests)|baseline"),13 ("robustness", r"robust|additional (tests|analys|evidence)|alternative|placebo|sensitivity|extensions?|further (tests|evidence|analys)|heterogeneity|mechanism|channel"),14 ("conclusion", r"conclu|concluding|final remarks|summary|discussion"),15 ("appendix", r"^appendix|^internet appendix|^supporting information"),16]1718def section_role(heading: str, is_appendix: bool = False, is_intro: bool = False) -> str:19 if is_intro: return "introduction"20 if is_appendix: return "appendix"21 h = (heading or "").lower().strip()22 for role, pat in ROLE_RULES:23 if re.search(pat, h): return role24 return "other"2526def load_parsed(doc_id: str) -> dict:27 return json.loads((config.derived_dir() / "parsed" / f"{doc_id}.json").read_text())2829def paragraphs(sec: dict) -> list[dict]:30 return [p for p in sec["paragraphs"] if p.get("kind") != "subheading"]3132def clip(text: str, n: int) -> str:33 return text if len(text) <= n else text[:n].rsplit(" ", 1)[0] + " […]"3435def annotation_payload(d: dict) -> dict:36 """Build the (bounded) material sent to the annotator: abstract, intro, conclusion, first paragraph of every section,37 table/figure captions, headings with word counts. Total size is capped by rights.yaml (max_excerpt_chars_per_call)."""38 cap = config.load("rights")["external_api_excerpt_policy"]["max_excerpt_chars_per_call"]39 secs = []40 for i, s in enumerate(d["sections"]):41 ps = paragraphs(s); role = section_role(s["heading"], s.get("is_appendix", False), s.get("is_intro", False))42 secs.append({"idx": i, "number": s.get("number"), "heading": s["heading"], "role_rule": role, "n_words": s["n_words"], "n_paragraphs": len(ps), "page": s["page"],43 "subsections": [x["heading"] for x in s["subsections"]][:12], "first_paragraph": clip(ps[0]["text"], 700) if ps else ""})44 intro = next((s for s in d["sections"] if s.get("is_intro")), None)45 intro_paras = [clip(p["text"], 1500) for p in paragraphs(intro)] if intro else []46 concl = [s for s in d["sections"] if section_role(s["heading"], s.get("is_appendix", False)) == "conclusion"]47 concl_paras = [clip(p["text"], 1200) for p in paragraphs(concl[-1])] if concl else []48 payload = {"doc_id": d["doc_id"], "title": d["metadata"]["title"], "year": d["metadata"]["year"], "article_type": d["metadata"]["article_type"],49 "abstract": d["abstract"], "sections": secs, "introduction_paragraphs": intro_paras, "conclusion_paragraphs": concl_paras,50 "tables": [{"label": t["label"], "title": t["title"], "notes_excerpt": clip(t["notes_excerpt"] or "", 350)} for t in d["tables"]],51 "figures": [{"label": f["label"], "caption": clip(f["caption"], 300)} for f in d["figures"]],52 "stats": d["stats"], "n_equations_by_section": {}}53 for e in d["equations"]: payload["n_equations_by_section"][e["section"]] = payload["n_equations_by_section"].get(e["section"], 0) + 154 # enforce cap by trimming intro/conclusion paragraphs from the middle55 def size(): return len(json.dumps(payload, ensure_ascii=False))56 while size() > cap and len(payload["introduction_paragraphs"]) > 4:57 mid = len(payload["introduction_paragraphs"]) // 2; payload["introduction_paragraphs"].pop(mid)58 while size() > cap and len(payload["conclusion_paragraphs"]) > 2:59 payload["conclusion_paragraphs"].pop(len(payload["conclusion_paragraphs"]) // 2)60 while size() > cap and len(payload["sections"]) > 0:61 for s in payload["sections"]: s["first_paragraph"] = clip(s["first_paragraph"], 300)62 break63 payload["payload_chars"] = size()64 return payload65