1"""Phase 4 — measured writing profiles (train split only) → profiles/*.yaml, figures, data/derived/features.parquet."""2from __future__ import annotations3import json, re4from pathlib import Path5import numpy as np, pandas as pd, yaml6from . import config, segmentation7from .provenance import stamp, now_iso89PROFILE_DEFS = { # name -> (filter description, predicate on the merged feature row)10 "corpus_all": ("all included train documents", lambda r: True),11 "empirical_asset_pricing": ("annotation.subfield == asset_pricing and genre in {empirical, mixed_theory_empirical}", lambda r: r.get("subfield") == "asset_pricing" and r.get("genre") in ("empirical", "mixed_theory_empirical")),12 "corporate_finance": ("annotation.subfield == corporate_finance and genre in {empirical, mixed_theory_empirical}", lambda r: r.get("subfield") == "corporate_finance" and r.get("genre") in ("empirical", "mixed_theory_empirical")),13 "financial_intermediation_banking": ("annotation.subfield == financial_intermediation_banking (empirical/mixed)", lambda r: r.get("subfield") == "financial_intermediation_banking" and r.get("genre") in ("empirical", "mixed_theory_empirical")),14 "household_behavioral_finance": ("subfield in {household_finance, behavioral_finance} (empirical/mixed)", lambda r: r.get("subfield") in ("household_finance", "behavioral_finance") and r.get("genre") in ("empirical", "mixed_theory_empirical")),15 "financial_econometrics": ("subfield == financial_econometrics or genre == methodological_econometric", lambda r: r.get("subfield") == "financial_econometrics" or r.get("genre") == "methodological_econometric"),16 "theoretical_finance": ("genre == theoretical", lambda r: r.get("genre") == "theoretical"),17 "empirical_all": ("genre in {empirical, mixed_theory_empirical}", lambda r: r.get("genre") in ("empirical", "mixed_theory_empirical")),18}19NUMERIC = ["pages", "n_words_body", "n_words_appendix", "abstract_words", "n_sections", "n_subsections", "n_paragraphs", "n_equations", "n_tables", "n_figures", "n_footnotes", "n_references", "n_citations_in_text",20 "intro_words", "intro_paragraphs", "intro_share", "conclusion_words", "conclusion_share", "theory_share", "data_share", "results_share", "robustness_share", "words_per_table", "refs_per_1000_words", "equations_per_1000_words"]2122def doc_features(doc_id: str) -> dict:23 d = segmentation.load_parsed(doc_id); st = d["stats"]24 roles = {}25 for s in d["sections"]:26 role = segmentation.section_role(s["heading"], s.get("is_appendix", False), s.get("is_intro", False)); roles[role] = roles.get(role, 0) + s["n_words"]27 body = max(st["n_words_body"], 1); intro = next((s for s in d["sections"] if s.get("is_intro")), None)28 f = {"doc_id": doc_id, "year": d["metadata"]["year"], "article_type": d["metadata"]["article_type"], "pages": d["pages"], **{k: st[k] for k in ["n_words_body", "n_words_appendix", "abstract_words", "n_sections", "n_subsections", "n_paragraphs", "n_equations", "n_tables", "n_figures", "n_footnotes", "n_references", "n_citations_in_text"]},29 "intro_words": roles.get("introduction", 0), "intro_paragraphs": len(segmentation.paragraphs(intro)) if intro else 0, "intro_share": roles.get("introduction", 0) / body,30 "conclusion_words": roles.get("conclusion", 0), "conclusion_share": roles.get("conclusion", 0) / body, "theory_share": roles.get("theory", 0) / body, "data_share": roles.get("data", 0) / body,31 "results_share": roles.get("results", 0) / body, "robustness_share": roles.get("robustness", 0) / body, "has_robustness_section": roles.get("robustness", 0) > 0, "has_literature_section": roles.get("literature", 0) > 0,32 "has_appendix": st["n_words_appendix"] > 0, "has_internet_appendix": d["markers"]["has_internet_appendix"], "words_per_table": body / max(st["n_tables"], 1), "refs_per_1000_words": 1000 * st["n_references"] / body, "equations_per_1000_words": 1000 * st["n_equations"] / body,33 "section_headings": " | ".join(f"{s.get('number') or ''} {s['heading']}".strip() for s in d["sections"]), "parse_confidence": d["confidence"]["overall"]}34 return f3536def merged_features(split: str = "train") -> pd.DataFrame:37 sp = json.loads((config.ROOT / "data" / "splits" / "heldout_v1.json").read_text())38 ids = sp[split] if split in ("train", "heldout") else sp["train"] + sp["heldout"]39 df = pd.DataFrame([doc_features(i) for i in ids])40 ann_p = config.ROOT / "data" / "annotations" / "annotations_table.parquet"41 if ann_p.exists():42 ann = pd.read_parquet(ann_p); df = df.merge(ann.drop(columns=[c for c in ["model", "cost_usd"] if c in ann]), on="doc_id", how="left")43 df["split"] = split44 return df4546def q(s: pd.Series) -> dict:47 s = pd.to_numeric(s, errors="coerce").dropna()48 if len(s) == 0: return {"n": 0}49 rng = np.random.default_rng(45); boots = [float(np.median(rng.choice(s.values, len(s)))) for _ in range(500)] if len(s) >= 5 else [float(s.median())]50 return {"n": int(len(s)), "median": round(float(s.median()), 3), "q25": round(float(s.quantile(.25)), 3), "q75": round(float(s.quantile(.75)), 3), "mean": round(float(s.mean()), 3), "min": round(float(s.min()), 3), "max": round(float(s.max()), 3),51 "median_ci95_bootstrap": [round(float(np.percentile(boots, 2.5)), 3), round(float(np.percentile(boots, 97.5)), 3)]}5253def share(s: pd.Series) -> dict:54 s = s.dropna()55 return {"n": int(len(s)), "share": round(float(s.astype(bool).mean()), 3)} if len(s) else {"n": 0}5657def cat_dist(s: pd.Series, top=8) -> dict:58 s = s.dropna(); n = len(s)59 return {"n": int(n), **{str(k): round(float(v) / n, 3) for k, v in s.value_counts().head(top).items()}} if n else {"n": 0}6061def seq_dist(s: pd.Series, top=6) -> dict:62 """Most common introduction move orders (deduplicated consecutive moves)."""63 seqs = []64 for x in s.dropna():65 mv = [m for i, m in enumerate(x.split("|")) if i == 0 or m != x.split("|")[i - 1]]66 seqs.append(" > ".join(mv))67 s2 = pd.Series(seqs); return cat_dist(s2, top)6869def move_prevalence(s: pd.Series) -> dict:70 """Share of introductions containing each move at least once, and median relative position of its first occurrence."""71 s = s.dropna(); n = len(s); out = {"n": int(n)}72 if not n: return out73 from collections import Counter74 c = Counter(); pos = {}75 for x in s:76 mv = x.split("|")77 for m in set(mv): c[m] += 1; pos.setdefault(m, []).append(mv.index(m) / max(len(mv) - 1, 1))78 for m, k in c.most_common(): out[m] = {"share": round(k / n, 3), "median_relative_position": round(float(np.median(pos[m])), 2)}79 return out8081def first_move_dist(s: pd.Series) -> dict:82 return cat_dist(s.dropna().map(lambda x: x.split("|")[0] if x else None))8384def build_profile(name: str, df: pd.DataFrame) -> dict:85 desc, pred = PROFILE_DEFS[name]; sub = df[[pred(r) for r in df.to_dict("records")]]86 has_ann = "genre" in sub and sub["genre"].notna().any()87 prof = {"profile": name, "ontology_version": "1.0", "built": stamp(), "selection_rule": desc, "n_documents": int(len(sub)), "split": "train (heldout_v1 excluded)",88 "caveat": "Observed frequencies describe what published JoF articles did in 2017–2020; they are not evidence that these choices are editorially superior or required.",89 "structure": {k: q(sub[k]) for k in NUMERIC if k in sub},90 "presence": {k: share(sub[k]) for k in ["has_robustness_section", "has_literature_section", "has_appendix", "has_internet_appendix"] if k in sub},91 "by_year": {int(y): {"n": int(len(g)), "n_words_body_median": float(g["n_words_body"].median()), "n_tables_median": float(g["n_tables"].median())} for y, g in sub.groupby("year")}}92 if has_ann:93 a = sub[sub["schema_valid"] == True]94 prof["rhetoric"] = {"n_annotated": int(len(a)), "genre": cat_dist(a["genre"]), "subfield": cat_dist(a["subfield"]), "primary_method": cat_dist(a["primary_method"]),95 "abstract_moves_present": {k: share(a[f"abs_{k}"]) for k in ["question", "gap", "method", "results", "magnitude", "contribution"]},96 "abstract_move_order_median_sentence": {k: q(a[a[f"abs_{k}_idx"] >= 0][f"abs_{k}_idx"]) for k in ["question", "gap", "method", "results", "magnitude", "contribution"]},97 "abstract_sentences": q(a["abstract_n_sent"]), "magnitude_in_abstract_regex": share(a["magnitude_regex_abstract"]),98 "introduction": {"paragraphs": q(a["intro_n_paragraphs"]), "words": q(a["intro_n_words"]), "contribution_paragraph_index": q(a[a["contribution_para_idx"] >= 0]["contribution_para_idx"]),99 "contribution_relative_position": q((a[a["contribution_para_idx"] >= 0]["contribution_para_idx"] / a[a["contribution_para_idx"] >= 0]["intro_n_paragraphs"].clip(lower=1))),100 "first_move": first_move_dist(a["intro_moves_seq"]), "move_prevalence": move_prevalence(a["intro_moves_seq"]), "moves_per_intro": q(a["intro_n_moves"]), "common_move_sequences": seq_dist(a["intro_moves_seq"]), "roadmap_present_llm": share(a["roadmap_llm"]), "roadmap_present_regex": share(a["roadmap_regex"]),101 "literature_positioning_location": cat_dist(a["lit_positioning"]), "contribution_types": cat_dist(a["contribution_types"].dropna().str.split("|").explode())},102 "identification": {"causal_language_strength": q(a["causal_strength"]), "uses_causal_claim": share(a["uses_causal_claim"])},103 "results": {"economic_significance_discussed": share(a["econ_signif"]), "statistical_language": cat_dist(a["stat_lang"]), "magnitude_statements_in_intro": share(a["magnitude_in_intro"])},104 "robustness": {"location": cat_dist(a["robust_location"]), "types": cat_dist(a["robust_types"].dropna().str.split("|").explode())},105 "tables": {"standard_errors": cat_dist(a["se_desc"]), "units_in_notes": share(a["units_in_notes"]), "clustered_regex": share(a["clustered_se_regex"])},106 "figures": {"uncertainty_shown": cat_dist(a["fig_uncertainty"])},107 "conclusion": {k: share(a[c]) for k, c in [("bounded_claims", "concl_bounded"), ("policy_implications", "concl_policy"), ("open_questions", "concl_open")]},108 "hedging": cat_dist(a["hedging"]), "claims_with_stated_evidence_share": q(a["claims_with_evidence"] / a["n_claims"].clip(lower=1)),109 "llm_vs_rule_section_role_agreement": q(a["role_agreement"]), "llm_confidence": q(a["llm_confidence"])}110 return prof111112def build_profiles() -> dict:113 df = merged_features("train"); allf = pd.concat([df, merged_features("heldout")]); allf.to_parquet(config.derived_dir() / "features.parquet", index=False); allf.to_csv(config.derived_dir() / "features.csv", index=False)114 out = {}; pdir = config.ROOT / "profiles"; pdir.mkdir(exist_ok=True)115 for name in PROFILE_DEFS:116 prof = build_profile(name, df); out[name] = prof["n_documents"]117 (pdir / f"{name}.yaml").write_text("# Generated by `jof250 profiles build` — do not edit by hand (see docs/DATA_DICTIONARY.md)\n" + yaml.safe_dump(prof, sort_keys=False, allow_unicode=True, width=120))118 make_figures(df)119 return {"profiles": out, "n_train": int(len(df)), "annotated_in_train": int(df["schema_valid"].fillna(False).sum()) if "schema_valid" in df else 0}120121def make_figures(df: pd.DataFrame) -> None:122 from . import viz; viz.style(); import matplotlib.pyplot as plt123 fdir = config.ROOT / "figures"; fdir.mkdir(exist_ok=True); pf = config.ROOT / "paper" / "figures"; pf.mkdir(exist_ok=True)124 # 1. structural distributions125 fig, axes = plt.subplots(2, 3, figsize=(7.2, 4.2))126 for ax, (col, lab) in zip(axes.flat, [("n_words_body", "Body words"), ("n_tables", "Tables"), ("n_figures", "Figures"), ("n_equations", "Numbered equations"), ("n_references", "References"), ("intro_share", "Introduction share of body")]):127 ax.hist(df[col].dropna(), bins=20, color=viz.PALETTE[0], edgecolor=viz.SURFACE, linewidth=0.5); ax.set_title(lab, fontsize=9); ax.axvline(df[col].median(), color=viz.PALETTE[1], linewidth=1.2)128 fig.suptitle(f"Structural distributions, JoF train split (n = {len(df)}); orange line = median", fontsize=9); fig.tight_layout()129 for p in (fdir, pf): fig.savefig(p / "fig_structure_distributions.pdf"); fig.savefig(p / "fig_structure_distributions.png")130 plt.close(fig)131 if "genre" in df and df["genre"].notna().any():132 a = df[df["schema_valid"] == True]133 # 2. abstract moves prevalence by genre134 moves = ["question", "gap", "method", "results", "magnitude", "contribution"]; genres = [g for g in ["empirical", "mixed_theory_empirical", "theoretical"] if (a["genre"] == g).sum() >= 5]135 fig, ax = plt.subplots(figsize=(6.2, 2.8)); w = 0.8 / max(len(genres), 1)136 for i, g in enumerate(genres):137 sub = a[a["genre"] == g]; vals = [sub[f"abs_{m}"].mean() for m in moves]138 ax.bar(np.arange(len(moves)) + i * w, vals, w, label=f"{g} (n={len(sub)})", color=viz.PALETTE[i], edgecolor=viz.SURFACE, linewidth=0.8)139 ax.set_xticks(np.arange(len(moves)) + w * (len(genres) - 1) / 2); ax.set_xticklabels(moves); ax.set_ylabel("Share of abstracts"); ax.set_ylim(0, 1); ax.legend(fontsize=7); ax.set_title("Abstract moves present, by genre (LLM annotation, train split)", fontsize=9)140 fig.tight_layout(); [fig.savefig(p / "fig_abstract_moves.pdf") for p in (fdir, pf)]; plt.close(fig)141 # 3. contribution relative position142 rel = (a[a["contribution_para_idx"] >= 0]["contribution_para_idx"] / a[a["contribution_para_idx"] >= 0]["intro_n_paragraphs"].clip(lower=1))143 fig, ax = plt.subplots(figsize=(4.2, 2.6)); ax.hist(rel, bins=10, range=(0, 1), color=viz.PALETTE[0], edgecolor=viz.SURFACE); ax.set_xlabel("Relative position of the contribution paragraph in the introduction"); ax.set_ylabel("Documents")144 ax.set_title(f"Contribution placement (n = {len(rel)})", fontsize=9); fig.tight_layout(); [fig.savefig(p / "fig_contribution_position.pdf") for p in (fdir, pf)]; plt.close(fig)145 # 4. intro first moves + robustness location146 fig, axes = plt.subplots(1, 2, figsize=(7.2, 2.6))147 fm = a["intro_moves_seq"].dropna().map(lambda x: x.split("|")[0]).value_counts(normalize=True).head(6)148 axes[0].barh(fm.index[::-1], fm.values[::-1], color=viz.PALETTE[0]); axes[0].set_title("First rhetorical move of the introduction", fontsize=9); axes[0].set_xlabel("Share")149 rl = a["robust_location"].value_counts(normalize=True).head(6)150 axes[1].barh(rl.index[::-1], rl.values[::-1], color=viz.PALETTE[2]); axes[1].set_title("Where robustness checks are presented", fontsize=9); axes[1].set_xlabel("Share")151 fig.tight_layout(); [fig.savefig(p / "fig_moves_robustness.pdf") for p in (fdir, pf)]; plt.close(fig)152