# DATA DICTIONARY

# data/manifests/corpus_manifest.parquet (one row per file in the corpus folder)

relpath, filename, ext, size_bytes, sha256, pdf_valid, pages, pdf_title, pdf_author, doi_in_pdf (NFKC-normalized first 3 pages), first_page_chars, error; bib_* (from index.csv: title, authors, pages, published, doi, section, year, volume, issue, access), bib_joined; decision ∈ {include, exclude, review}, decision_reason, article_type ∈ {article, presidential_address, corrigendum_or_replication, retracted_article}; crossref_status/title/year/container/authors/refcount/cited_by, title_match_ratio, crossref_ok; doc_id (jof_<suffix of DOI>). corpus_v1_frozen.json: version, frozen_at, n_included, manifest_sha256, documents[]. inventory_ledger.jsonl: append-only build events.

# data/derived/parsed/<doc_id>.json (private)

doc_id, source_sha256, parser (version), parsed_at, pages, body_font_size, metadata{title, authors, doi, year, volume, issue, pages, article_type, bib_title, bib_authors}, abstract, sections[{number, heading, level, page, is_intro, is_appendix, paragraphs[{text, page, kind?}], subsections[{heading, page, para_index}], n_words}], equations[{tag, page, text_excerpt, section, confidence}], tables[{label, title, page, notes_excerpt, rotated, section}], figures[{label, caption, page, section}], footnotes[{page, text, n_lines}], references[{text, page, authors, year, rest, doi}], citations_in_text[], markers{references_page, appendix_page, supporting_information_page, has_internet_appendix}, stats{…}, confidence{title, authors, abstract, sections, references, equations, tables, figures, overall}. parse_summary.parquet: one row per doc with stats + confidences. parser_qa.json/.md: cross-parser agreement sample.

# data/annotations/documents/<doc_id>.json

doc_id, ontology_version, prompt_version, model, effort, annotated_at, source_sha256, parser, payload_chars, usage, cost_usd, request_id, validation_errors[], schema_valid, annotation{genre, subfield, subfield_secondary, primary_method, identification{estimand, strategy, key_assumptions[], threats_addressed[], causal_language_strength 1–5, uses_causal_claim}, abstract_moves{question|gap|method|results|magnitude|contribution: {present, sentence_index}}, introduction{moves[{paragraph_index, move}], contribution_paragraph_index, contribution_types[], roadmap_present, literature_positioning_location}, section_roles[{idx, role}], results_presentation{…}, robustness{location, types[]}, tables{convention_notes, units_stated_in_notes, standard_errors_description}, figures{uncertainty_shown}, conclusion{bounded_claims, policy_implications, open_questions, restates_contribution}, hedging{level, typical_devices[]}, claim_evidence[{claim, evidence_type, evidence_ref}], structural_summary, writing_profile_keywords[], confidence, notes}, deterministic_checks{roadmap_regex, contribution_regex_in_intro, magnitude_in_abstract, abstract_n_sentences, intro_n_paragraphs, intro_n_words, causal_words_in_abstract, hedge_words_in_abstract, stars_mentioned, clustered_se_mentioned, internet_appendix_mentioned, section_roles_rule}. annotations_table.parquet: flattened per-document table (see rhetoric.build_table). api_ledger.jsonl: one record per API call (purpose, prompt_id/version, model, effort, usage, cost_usd, latency, request_id, cache_key, code_version). engine_runs.jsonl: engine invocations.

# data/derived/features.parquet

Per document (train+heldout): structural features (doc_features) merged with the annotation table; split.

# profiles/.yaml

profile, selection_rule, n_documents, split, caveat, structure{metric: {n, median, q25, q75, mean, min, max, median_ci95_bootstrap}}, presence{…: {n, share}}, by_year, rhetoric{…} (only when annotations exist).

# data/derived/index/

units.parquet (unit_id, doc_id, sha256, year, title, article_type, doi, unit ∈ {document_summary, identification, claim_evidence, intro_move, section_opening, table_notes, figure_caption}, section, page, role, genre, subfield, method, text ≤ ~900 chars, moves, confidence, split), embeddings.npy (float32, normalized), index_meta.json, shingles.json (8-gram hashes per doc for overlap checks).

# experiments/

fixtures/heldout_v1/{manifest.json, /manuscript.md (intro withheld), manuscript_with_intro.md, published_introduction.txt, meta.json}; outputs//{runs.jsonl, judgments.jsonl, outputs/*.tex, summary.json}; analysis/results_.json + CSVs.