1#!/usr/bin/env python32"""3code_corpus.py -- automated (script-computed) content-coding variables for the4WP29 mandate corpus (research/corpus/*.md).56Standard library only. Run from anywhere:78 python3 -I research/code_corpus.py910Writes research/corpus_automated.csv (one row per file, including the11byte-identical WP15 duplicate, which is flagged in `duplicate_of`).1213Every variable here is AUTOMATED (computed from the bytes of the file). The14definitions below are the authoritative ones referenced by15research/corpus_coding_protocol.md. Keyword counts are *mentions*, not16judgements: a document that says "no budget is specified" would still count17one mention of the `budget` family.18"""1920import csv21import hashlib22import os23import re24import sys2526HERE = os.path.dirname(os.path.abspath(__file__))27CORPUS_DIR = os.path.join(HERE, "corpus")28MANIFEST = os.path.join(HERE, "corpus_manifest.csv")29OUT = os.path.join(HERE, "corpus_automated.csv")3031# --------------------------------------------------------------------------32# Regular expressions (documented for the codebook)33# --------------------------------------------------------------------------3435# Markdown ATX headings: 1-6 '#' at line start followed by a space.36RE_HEADING = re.compile(r"^#{1,6}\s+\S", re.MULTILINE)3738# Explicit phase labels anywhere in the text, e.g. "Phase 0", "PHASE 3 —",39# "phase_XX" is NOT matched (requires digits). Distinct numbers are counted.40RE_PHASE_LABEL = re.compile(r"\bphase\s*(\d+)", re.IGNORECASE)4142# Lettered stage labels ("Stage A", "Stage G") -- WP15 uses stages, not phases.43# Case-insensitive on the word "stage", but the label itself must be a capital44# letter. NB: WP28 also uses "Stage A..G" for its coordinator/worker message45# protocol; because WP28 has explicit "Phase N" labels those stages are not46# used as its phase count.47RE_STAGE_LABEL = re.compile(r"(?i:\bstage)\s+([A-Z])\b")4849# Gate labels ("Gate 1", "Gate 12") -- distinct numbers counted.50RE_GATE_LABEL = re.compile(r"\bgate\s*(\d+)", re.IGNORECASE)5152# Numbered list items at line start ("1. ", " 12. ").53RE_NUMBERED_ITEM = re.compile(r"^\s*\d+\.\s", re.MULTILINE)5455# Headings whose text indicates a phase/sequence/execution-order section. The56# numbered items *inside* such sections are counted as declared phases when the57# document has no explicit "Phase N" labels.58RE_PHASE_SECTION_HEADING = re.compile(59 r"^#{1,6}\s+.*\b(phase|phases|stage|stages|sequence|execution order|"60 r"work sequence|execution protocol|execution phases|milestones)\b",61 re.IGNORECASE | re.MULTILINE,62)6364# Markdown table separator rows: |---|---:| etc. One separator row == one table.65RE_TABLE_SEP = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$", re.MULTILINE)6667# Fenced code blocks: pairs of ``` fences.68RE_FENCE = re.compile(r"^\s*```", re.MULTILINE)6970# Keyword families (case-insensitive). Each is a mention count + boolean.71KEYWORD_FAMILIES = {72 "preregist": r"pre-?regist\w*",73 "estimand": r"\bestimands?\b",74 "hypothes": r"\bhypothes\w*",75 "falsif": r"\bfalsif\w*",76 "baseline": r"\bbaselines?\b",77 "leakage": r"\bleakage\b|\blook-?ahead\b|\blook ahead\b|\bpoint-in-time\b|\bpoint in time\b",78 "provenance": r"\bprovenance\b|\bhash(?:es|ed|ing)?\b|\bsha-?256\b|\bchecksums?\b",79 "reproduc": r"\breproduc\w*",80 "stopping": r"\bstopping\b|\bstop rules?\b|\bstop conditions?\b|\btermination\b|\bstop and\b|\bstop if\b|\bstop only\b",81 "recovery": r"\brecover\w*|\bresum\w*|\bcheckpoint\w*",82 "gate": r"\bgates?\b|\bacceptance\b|\bcriteria\b",83 "exhibit": r"\bscreenshots?\b|\bfigures?\b|\btables?\b",84 "disclos": r"\bdisclos\w*|\bAI-use\b|\bAI use\b|\bAI contribution\b",85 "budget": r"\bbudgets?\b|\bcosts?\b|\bUSD\b|\$",86 "license": r"\blicen[cs]e[sd]?\b|\blicensing\b",87 "seed": r"\bseeds?\b|\bseeded\b",88}89KEYWORD_RE = {k: re.compile(v, re.IGNORECASE) for k, v in KEYWORD_FAMILIES.items()}9091# Language heuristic: tokens that are common in French prose and rare in92# English technical prose ("et" is excluded because of "et al.").93# Tokens shared with English usage in this corpus ("conventions", "journal",94# "du" as in "Universite du Quebec", "et" as in "et al.") are deliberately excluded.95FR_TOKENS = {96 "le", "la", "les", "des", "est", "une", "ce", "cette", "dans", "pour",97 "sur", "avec", "pas", "aux", "qui", "que", "ci-dessous", "dossier",98 "appelle", "correspond", "auteur",99}100EN_TOKENS = {101 "the", "and", "of", "to", "in", "is", "for", "with", "that", "not", "must",102 "do", "never", "each", "every", "all", "or", "as", "from", "if", "use",103}104RE_WORD = re.compile(r"[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ'\-]*")105106107def sha256_of(path):108 h = hashlib.sha256()109 with open(path, "rb") as f:110 for chunk in iter(lambda: f.read(1 << 20), b""):111 h.update(chunk)112 return h.hexdigest()113114115def language_of(text):116 """Return (label, fr_share). fr_share = FR hits / (FR hits + EN hits).117118 en : fr_share < 0.015119 mixed : 0.015 <= fr_share < 0.50 (e.g. a French preamble on an English body)120 fr : fr_share >= 0.50121 (threshold chosen after inspection: WP18 = 0.043 and WP20 = 0.020 carry a122 French prefatory paragraph; all other files score 0.000)123 """124 toks = [t.lower() for t in RE_WORD.findall(text)]125 fr = sum(1 for t in toks if t in FR_TOKENS)126 en = sum(1 for t in toks if t in EN_TOKENS)127 total = fr + en128 share = (fr / total) if total else 0.0129 if share >= 0.50:130 label = "fr"131 elif share >= 0.015:132 label = "mixed"133 else:134 label = "en"135 return label, round(share, 4)136137138def numbered_items_in_phase_sections(text):139 """Count numbered list items inside sections whose heading mentions140 phase/stage/sequence/execution order. A section runs from its heading to141 the next heading of the same or higher level."""142 lines = text.splitlines()143 heading_idx = []144 for i, line in enumerate(lines):145 m = re.match(r"^(#{1,6})\s+(.*)$", line)146 if m:147 heading_idx.append((i, len(m.group(1)), m.group(2)))148 count = 0149 for n, (i, level, title) in enumerate(heading_idx):150 if not RE_PHASE_SECTION_HEADING.match("#" * level + " " + title):151 continue152 end = len(lines)153 for j, lvl, _ in heading_idx[n + 1:]:154 if lvl <= level:155 end = j156 break157 block = "\n".join(lines[i + 1:end])158 count += len(RE_NUMBERED_ITEM.findall(block))159 return count160161162def analyse(path):163 raw = open(path, "rb").read()164 text = raw.decode("utf-8", errors="replace")165 name = os.path.basename(path)166 wp = re.match(r"(WP\d+)", name).group(1)167168 phase_numbers = {int(x) for x in RE_PHASE_LABEL.findall(text)}169 stage_letters = set(RE_STAGE_LABEL.findall(text))170 gate_numbers = {int(x) for x in RE_GATE_LABEL.findall(text)}171 items_in_phase_sections = numbered_items_in_phase_sections(text)172173 # n_numbered_phases: distinct "Phase N" labels if any; else distinct174 # "Stage X" labels if any; else distinct "Gate N" labels if any; else175 # numbered items within phase/sequence sections. (Analyst-coded176 # n_phases_declared in corpus_coding.csv is the authoritative value; this177 # is the mechanical proxy.)178 if phase_numbers:179 n_numbered_phases = len(phase_numbers)180 phase_rule = "distinct 'Phase N' labels"181 elif stage_letters:182 n_numbered_phases = len(stage_letters)183 phase_rule = "distinct 'Stage X' labels"184 elif gate_numbers:185 n_numbered_phases = len(gate_numbers)186 phase_rule = "distinct 'Gate N' labels"187 else:188 n_numbered_phases = items_in_phase_sections189 phase_rule = "numbered items inside phase/sequence sections"190191 lang, fr_share = language_of(text)192 words = len(text.split())193 row = {194 "file": name,195 "wp": wp,196 "sha256": hashlib.sha256(raw).hexdigest(),197 "bytes": len(raw),198 "lines": text.count("\n") + (0 if text.endswith("\n") else 1),199 "words": words,200 "n_headings": len(RE_HEADING.findall(text)),201 "n_numbered_phases": n_numbered_phases,202 "n_numbered_phases_rule": phase_rule,203 "n_phase_labels_distinct": len(phase_numbers),204 "n_stage_labels_distinct": len(stage_letters),205 "n_gate_labels_distinct": len(gate_numbers),206 "n_numbered_items_in_phase_sections": items_in_phase_sections,207 "n_numbered_items_total": len(RE_NUMBERED_ITEM.findall(text)),208 "n_tables": len(RE_TABLE_SEP.findall(text)),209 "n_code_blocks": len(RE_FENCE.findall(text)) // 2,210 "language_of_document": lang,211 "fr_token_share": fr_share,212 }213 for fam, rx in KEYWORD_RE.items():214 n = len(rx.findall(text))215 row["mentions_%s_n" % fam] = n216 row["mentions_%s" % fam] = "yes" if n else "no"217 return row218219220def main():221 files = sorted(222 os.path.join(CORPUS_DIR, f) for f in os.listdir(CORPUS_DIR) if f.endswith(".md")223 )224 if not files:225 sys.exit("no .md files in %s" % CORPUS_DIR)226227 # manifest lookup (url, retrieved_utc, expected sha)228 manifest = {}229 if os.path.exists(MANIFEST):230 with open(MANIFEST, newline="", encoding="utf-8") as f:231 for r in csv.DictReader(f):232 if r.get("local_file"):233 manifest[os.path.basename(r["local_file"])] = r234235 rows = [analyse(p) for p in files]236237 # duplicates by sha256 (first file in sorted order is the canonical one)238 seen = {}239 for r in rows:240 r["duplicate_of"] = seen.get(r["sha256"], "")241 seen.setdefault(r["sha256"], r["file"])242 m = manifest.get(r["file"], {})243 r["manifest_url"] = m.get("url", "")244 r["manifest_retrieved_utc"] = m.get("retrieved_utc", "")245 r["sha256_matches_manifest"] = (246 "yes" if m and m.get("sha256") == r["sha256"] else ("no" if m else "not_in_manifest")247 )248 r["document_type"] = (249 "design_brief" if "BRIEF" in r["file"]250 else ("duplicate_claude_md" if r["duplicate_of"] else "claude_md")251 )252253 fieldnames = (254 ["file", "wp", "document_type", "duplicate_of", "sha256", "sha256_matches_manifest",255 "manifest_url", "manifest_retrieved_utc", "bytes", "lines", "words", "n_headings",256 "n_numbered_phases", "n_numbered_phases_rule", "n_phase_labels_distinct",257 "n_stage_labels_distinct", "n_gate_labels_distinct",258 "n_numbered_items_in_phase_sections", "n_numbered_items_total", "n_tables",259 "n_code_blocks", "language_of_document", "fr_token_share"]260 + ["mentions_%s" % k for k in KEYWORD_FAMILIES]261 + ["mentions_%s_n" % k for k in KEYWORD_FAMILIES]262 )263 with open(OUT, "w", newline="", encoding="utf-8") as f:264 w = csv.DictWriter(f, fieldnames=fieldnames, quoting=csv.QUOTE_MINIMAL)265 w.writeheader()266 for r in rows:267 w.writerow({k: r.get(k, "") for k in fieldnames})268269 # console summary270 print("wrote %s (%d rows)" % (OUT, len(rows)))271 for r in rows:272 print("%-38s %6d w %3d h phases=%-2s tables=%d code=%d lang=%s dup=%s" % (273 r["file"], r["words"], r["n_headings"], r["n_numbered_phases"], r["n_tables"],274 r["n_code_blocks"], r["language_of_document"], r["duplicate_of"] or "-"))275276277if __name__ == "__main__":278 main()279