Medium corpus v0 (13 372 docs / 371 419 eligible passages) summaries; tokenizer trainer on corpus passages; evidence summaries tracked

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 7 h ago (Nov 9, 2026) parent e4eec91 Browse files

22 changed files +1,270 −6

modified .gitignore +12 −0

+ .DS_Store
+ # demo
+ demo/node_modules/
++
++# Evidence summaries ARE committed (small JSON)
++!data/parsed/*/build_summary.json
++!data/parsed/*/relabel_log.json
++!data/splits/*/
++data/splits/*/splits.jsonl
++!data/pairs/*/
++data/pairs/*/rewrites.jsonl
++data/pairs/*/plan.jsonl
++data/pairs/*/validated/pairs.jsonl
++data/pairs/*/validated/quarantine.jsonl
++data/pairs/*/batches/

added data/manifests/discover_books_bookshelf_seed2.json +89 −0

++{
++  "created_at": "2026-11-09T18:28:01Z",
++  "git": {
++    "commit": "e4eec911d74f1ec613a2572cb35afde77ec2bdcc",
++    "branch": "main",
++    "dirty": false
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "script": "discover_books",
++  "args": {
++    "source": "bookshelf",
++    "max_docs": 300,
++    "max_attempts": 450,
++    "seed": 2,
++    "out": "data/raw/books",
++    "codes": "CC0,PUBLIC-DOMAIN,CC-BY,CC-BY-SA,CC-BY-NC,CC-BY-NC-SA",
++    "locc": "Q,R,T,H,G,B,L,Z",
++    "max_mb": 80.0,
++    "pool_only": false,
++    "report": null
++  },
++  "stats": {
++    "list_rows": 9733,
++    "pre2018_rows": 3941,
++    "attempted": 450,
++    "too_large": 9,
++    "books_kept": 163,
++    "chapters_written": 1351,
++    "license_excluded": 99,
++    "license_unknown": 142,
++    "not_pre2018_xml": 0,
++    "no_body_chapters": 37,
++    "errors": 0,
++    "by_license": {
++      "PUBLIC-DOMAIN": 169,
++      "CC-BY-NC-ND": 98,
++      "CC0": 25,
++      "UNKNOWN": 142,
++      "CC-BY-NC-SA": 1,
++      "CC-BY-ND": 1,
++      "CC-BY-NC": 3,
++      "CC-BY": 2
++    },
++    "by_publisher": {
++      "Agency for Healthcare Research and Quality (US)": 109,
++      "Centers for Disease Control and Prevention (US)": 19,
++      "Substance Abuse and Mental Health Services Administration (US)": 10,
++      "Office of the Surgeon General (US)": 2,
++      "Department of Veterans Affairs (US)": 13,
++      "American Epilepsy Society": 1,
++      "National Center for Biotechnology Information (US)": 3,
++      "Springer": 3,
++      "The International Bank for Reconstruction and Development / The World Bank": 1,
++      "National Institute of Environmental Health Sciences": 2
++    },
++    "by_era": {
++      "2010-2017": 92,
++      "2000-2009": 57,
++      "1980-1999": 14
++    },
++    "ftp": {
++      "requests": 895,
++      "retries": 4,
++      "bytes": 2310287545
++    },
++    "elapsed_s": 1822.4
++  }
++}

added data/manifests/discover_books_gutenberg_seed2.json +244 −0

++{
++  "created_at": "2026-11-09T18:38:44Z",
++  "git": {
++    "commit": "e4eec911d74f1ec613a2572cb35afde77ec2bdcc",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "script": "discover_books",
++  "args": {
++    "source": "gutenberg",
++    "max_docs": 500,
++    "max_attempts": 700,
++    "seed": 2,
++    "out": "data/raw/books",
++    "codes": "CC0,PUBLIC-DOMAIN,CC-BY,CC-BY-SA,CC-BY-NC,CC-BY-NC-SA",
++    "locc": "Q,R,T,H,G,B,L,Z",
++    "max_mb": 25.0,
++    "pool_only": false,
++    "report": null
++  },
++  "stats": {
++    "catalog_rows": 79769,
++    "candidates": 7799,
++    "attempted": 517,
++    "downloaded": 500,
++    "already_present": 0,
++    "not_found": 5,
++    "too_large": 3,
++    "copyrighted_header": 9,
++    "no_pd_header": 0,
++    "no_date": 60,
++    "date_from_frontmatter": 370,
++    "date_from_death_year": 70,
++    "errors": 0,
++    "by_locc": {
++      "RC": 23,
++      "LT": 2,
++      "QH": 11,
++      "GT": 17,
++      "HV": 13,
++      "TD": 3,
++      "BF": 26,
++      "RS": 1,
++      "LB": 28,
++      "QA": 1,
++      "GN": 11,
++      "HS": 1,
++      "TN": 3,
++      "Z": 62,
++      "QL": 13,
++      "G": 32,
++      "HX": 3,
++      "TA": 3,
++      "BJ": 14,
++      "LD": 4,
++      "TL": 6,
++      "B": 19,
++      "RZ": 2,
++      "LA": 12,
++      "QR": 1,
++      "HD": 4,
++      "TS": 3,
++      "GC": 1,
++      "HN": 3,
++      "T": 16,
++      "RM": 8,
++      "QK": 5,
++      "HE": 7,
++      "R": 12,
++      "QE": 6,
++      "H": 2,
++      "Q": 11,
++      "RA": 12,
++      "HQ": 13,
++      "TJ": 4,
++      "HF": 3,
++      "TT": 9,
++      "HM": 1,
++      "TC": 1,
++      "QP": 4,
++      "HT": 2,
++      "QC": 3,
++      "HG": 4,
++      "RG": 4,
++      "TP": 5,
++      "LF": 4,
++      "HB": 2,
++      "LC": 14,
++      "TH": 2,
++      "QB": 4,
++      "HC": 2,
++      "RJ": 2,
++      "TK": 4,
++      "BD": 3,
++      "HJ": 1,
++      "RD": 1,
++      "BC": 1,
++      "QD": 1,
++      "TF": 1,
++      "LH": 1,
++      "HA": 1,
++      "QM": 1,
++      "GB": 1
++    },
++    "by_era": {
++      "pre1900": 178,
++      "1950-1979": 20,
++      "1900-1949": 236,
++      "unknown": 60,
++      "1980-1999": 2,
++      "2000-2009": 4
++    },
++    "by_date_rule": {
++      "frontmatter:imprint": 164,
++      "frontmatter:copyright": 140,
++      "death_year": 70,
++      "frontmatter:published": 41,
++      "none": 60,
++      "frontmatter:roman": 25
++    },
++    "pool_by_class": {
++      "QH": 543,
++      "QL": 491,
++      "Z": 490,
++      "BF": 372,
++      "HV": 356,
++      "B": 344,
++      "HQ": 341,
++      "G": 256,
++      "BJ": 234,
++      "Q": 202,
++      "HD": 190,
++      "RC": 190,
++      "QC": 182,
++      "QK": 180,
++      "T": 167,
++      "TT": 164,
++      "QE": 161,
++      "HE": 156,
++      "QB": 139,
++      "LB": 132,
++      "TS": 130,
++      "QA": 120,
++      "RA": 112,
++      "HX": 99,
++      "GN": 99,
++      "TL": 96,
++      "TK": 89,
++      "HF": 87,
++      "RM": 84,
++      "HB": 81,
++      "GT": 81,
++      "TP": 80,
++      "QD": 78,
++      "R": 77,
++      "HG": 76,
++      "TJ": 63,
++      "HN": 62,
++      "QP": 61,
++      "HT": 55,
++      "LC": 55,
++      "TA": 50,
++      "LA": 48,
++      "HM": 47,
++      "HC": 43,
++      "TN": 42,
++      "TH": 38,
++      "TR": 37,
++      "RD": 34,
++      "LF": 33,
++      "BD": 33,
++      "RG": 32,
++      "TC": 32,
++      "TF": 27,
++      "HS": 26,
++      "TD": 25,
++      "BC": 24,
++      "GB": 19,
++      "LD": 16,
++      "RJ": 16,
++      "H": 16,
++      "RS": 16,
++      "QR": 15,
++      "HJ": 14,
++      "QM": 14,
++      "RT": 11,
++      "RE": 11,
++      "GA": 10,
++      "BH": 9,
++      "RZ": 8,
++      "RK": 8,
++      "RX": 7,
++      "RB": 7,
++      "LT": 7,
++      "GC": 6,
++      "TE": 6,
++      "LH": 6,
++      "GF": 5,
++      "GR": 5,
++      "TG": 5,
++      "HA": 4,
++      "L": 4,
++      "RL": 3,
++      "RF": 3,
++      "RV": 1,
++      "LE": 1
++    },
++    "catalog": {
++      "url": "https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv",
++      "path": "data/raw/books/_catalog/pg_catalog.csv",
++      "cached": true,
++      "sha256": "8c06ec1c46751dc184b640b8a8ed5842c873ae3b621d575f8e76837f12f33eb6",
++      "retrieved_or_cached_at": "2026-11-09T17:57:39Z"
++    },
++    "http": {
++      "requests": 527,
++      "retries": 0,
++      "bytes": 284820446
++    },
++    "elapsed_s": 2465.3
++  }
++}

added data/pairs/mock_smoke/ledger.json +11 −0

++{
++  "spent_usd": 0.0,
++  "n_ok": 360,
++  "n_error": 0,
++  "n_refused": 0,
++  "n_truncated": 0,
++  "n_empty": 0,
++  "batches": [],
++  "created_at": "2026-11-09T17:12:45Z",
++  "updated_at": "2026-11-09T17:12:47Z"
++}

added data/pairs/mock_smoke/manifest.json +57 −0

++{
++  "created_at": "2026-11-09T17:12:45Z",
++  "git": {
++    "commit": null,
++    "branch": null,
++    "dirty": false
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "job": "rewrite",
++  "provider": "mock",
++  "model": "mock-rewriter-1",
++  "prompt_version": "rw-1.0",
++  "prompt_fingerprint": "8af5d0bf61cfb0f6",
++  "budget_usd": 30.0,
++  "batch": false,
++  "corpus": "data/parsed/smoke",
++  "n_passages": 60,
++  "variants": [
++    "A",
++    "B",
++    "C",
++    "D",
++    "E"
++  ],
++  "selection_seed": 1,
++  "estimate": {
++    "pending_requests": 360,
++    "done": 0,
++    "est_input_tokens": 179232,
++    "est_output_tokens": 94260,
++    "est_cost_usd": null,
++    "budget_usd": 30.0,
++    "spent_usd": 0.0
++  },
++  "live": false
++}

added data/pairs/mock_smoke/validated/validation_summary.json +79 −0

++{
++  "created_at": "2026-11-09T17:12:55Z",
++  "git": {
++    "commit": null,
++    "branch": null,
++    "dirty": false
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "validator_version": "val-1.0",
++  "config": {
++    "min_len_ratio": 0.5,
++    "max_len_ratio": 1.8,
++    "concise_min_len_ratio": 0.4,
++    "max_norm_edit_similarity": 0.95,
++    "min_norm_edit_similarity": 0.05,
++    "min_semantic_sim": 0.7,
++    "max_negation_delta": 2,
++    "max_hedge_delta": 4,
++    "require_language": "en",
++    "entity_min_recall": 0.6,
++    "use_embeddings": true
++  },
++  "stats": {
++    "rewrites": 360,
++    "status": {
++      "ok": 360
++    },
++    "validated": 360,
++    "passed": 285,
++    "quarantined": 75,
++    "by_variant": {
++      "A": {
++        "n": 60,
++        "passed": 46
++      },
++      "E": {
++        "n": 120,
++        "passed": 102
++      },
++      "B": {
++        "n": 60,
++        "passed": 52
++      },
++      "C": {
++        "n": 60,
++        "passed": 43
++      },
++      "D": {
++        "n": 60,
++        "passed": 42
++      }
++    },
++    "fail_reasons": {
++      "near-verbatim": 75
++    },
++    "acceptance_rate": 0.7917
++  }
++}

added data/pairs/pilot_heldout_models/ledger.json +15 −0

++{
++  "spent_usd": 1.428675,
++  "n_ok": 600,
++  "n_error": 0,
++  "n_refused": 0,
++  "n_truncated": 0,
++  "n_empty": 0,
++  "batches": [
++    "msgbatch_01V4FduEDL26WRNLQEV3RHMP",
++    "msgbatch_01SWH3HaFBNZPCkKaQW3fwpQ"
++  ],
++  "created_at": "2026-11-09T17:43:49Z",
++  "submitted_est_usd": 0.6,
++  "updated_at": "2026-11-09T17:50:52Z"
++}

added data/pairs/pilot_heldout_models/manifest.json +54 −0

++{
++  "created_at": "2026-11-09T17:50:49Z",
++  "git": {
++    "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "job": "rewrite",
++  "provider": "anthropic",
++  "model": "claude-sonnet-5-5",
++  "prompt_version": "rw-1.0",
++  "prompt_fingerprint": "8af5d0bf61cfb0f6",
++  "budget_usd": 30.0,
++  "batch": true,
++  "corpus": "data/parsed/pilot_pmc",
++  "n_passages": 150,
++  "variants": [
++    "A",
++    "B"
++  ],
++  "selection_seed": 1,
++  "estimate": {
++    "pending_requests": 300,
++    "done": 0,
++    "est_input_tokens": 153430,
++    "est_output_tokens": 83198,
++    "est_cost_usd": 0.57,
++    "budget_usd": 30.0,
++    "spent_usd": 0.0
++  },
++  "live": true
++}

added data/pairs/pilot_heldout_models/validated/validation_summary.json +74 −0

++{
++  "created_at": "2026-11-09T17:51:06Z",
++  "git": {
++    "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "validator_version": "val-1.0",
++  "config": {
++    "min_len_ratio": 0.5,
++    "max_len_ratio": 1.8,
++    "concise_min_len_ratio": 0.4,
++    "max_norm_edit_similarity": 0.95,
++    "min_norm_edit_similarity": 0.05,
++    "min_semantic_sim": 0.7,
++    "max_negation_delta": 2,
++    "max_hedge_delta": 4,
++    "require_language": "en",
++    "entity_min_recall": 0.6,
++    "use_embeddings": true
++  },
++  "stats": {
++    "rewrites": 600,
++    "status": {
++      "ok": 600
++    },
++    "validated": 600,
++    "passed": 531,
++    "quarantined": 69,
++    "by_variant": {
++      "A": {
++        "n": 300,
++        "passed": 267
++      },
++      "B": {
++        "n": 300,
++        "passed": 264
++      }
++    },
++    "fail_reasons": {
++      "numbers missing=1": 37,
++      "entity recall": 7,
++      "numbers missing=2": 14,
++      "numbers missing=3": 7,
++      "formatting": 4,
++      "numbers missing=4": 1,
++      "near-verbatim": 1,
++      "numbers missing=7": 1
++    },
++    "acceptance_rate": 0.885
++  }
++}

added data/pairs/pilot_heldout_prompts/ledger.json +14 −0

++{
++  "spent_usd": 1.39825,
++  "n_ok": 297,
++  "n_error": 0,
++  "n_refused": 3,
++  "n_truncated": 0,
++  "n_empty": 0,
++  "batches": [
++    "msgbatch_01L5QkwBhyFtAydvR2X8g394"
++  ],
++  "created_at": "2026-11-09T17:43:55Z",
++  "submitted_est_usd": 1.14,
++  "updated_at": "2026-11-09T17:50:55Z"
++}

added data/pairs/pilot_heldout_prompts/manifest.json +54 −0

++{
++  "created_at": "2026-11-09T17:50:53Z",
++  "git": {
++    "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "job": "rewrite",
++  "provider": "anthropic",
++  "model": "claude-opus-5-5",
++  "prompt_version": "rw-1.0",
++  "prompt_fingerprint": "8af5d0bf61cfb0f6",
++  "budget_usd": 30.0,
++  "batch": true,
++  "corpus": "data/parsed/pilot_pmc",
++  "n_passages": 150,
++  "variants": [
++    "H1",
++    "H2"
++  ],
++  "selection_seed": 1,
++  "estimate": {
++    "pending_requests": 300,
++    "done": 0,
++    "est_input_tokens": 153430,
++    "est_output_tokens": 83198,
++    "est_cost_usd": 1.14,
++    "budget_usd": 30.0,
++    "spent_usd": 0.0
++  },
++  "live": true
++}

added data/pairs/pilot_heldout_prompts/validated/validation_summary.json +75 −0

++{
++  "created_at": "2026-11-09T17:51:14Z",
++  "git": {
++    "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "validator_version": "val-1.0",
++  "config": {
++    "min_len_ratio": 0.5,
++    "max_len_ratio": 1.8,
++    "concise_min_len_ratio": 0.4,
++    "max_norm_edit_similarity": 0.95,
++    "min_norm_edit_similarity": 0.05,
++    "min_semantic_sim": 0.7,
++    "max_negation_delta": 2,
++    "max_hedge_delta": 4,
++    "require_language": "en",
++    "entity_min_recall": 0.6,
++    "use_embeddings": true
++  },
++  "stats": {
++    "rewrites": 300,
++    "status": {
++      "ok": 297,
++      "refused": 3
++    },
++    "validated": 300,
++    "passed": 266,
++    "quarantined": 34,
++    "by_variant": {
++      "H2": {
++        "n": 150,
++        "passed": 138
++      },
++      "H1": {
++        "n": 150,
++        "passed": 128
++      }
++    },
++    "fail_reasons": {
++      "numbers missing=1": 20,
++      "numbers missing=2": 6,
++      "entity recall": 2,
++      "status:refused": 3,
++      "formatting": 1,
++      "placeholders missing=0": 1,
++      "numbers missing=0": 1,
++      "numbers missing=3": 2
++    },
++    "acceptance_rate": 0.8867
++  }
++}

added data/pairs/pilot_opus/ledger.json +19 −0

++{
++  "spent_usd": 26.85241,
++  "n_ok": 5801,
++  "n_error": 0,
++  "n_refused": 43,
++  "n_truncated": 0,
++  "n_empty": 0,
++  "batches": [
++    "msgbatch_01T8HKLMfnEWKK3JQUuWxunJ",
++    "msgbatch_01RMbxWJDx3NR88JcszbMADz",
++    "msgbatch_01Kvfx7j5b1VY1A5R1h9VGQj",
++    "msgbatch_014HKsCPuhepp7L2tprFi1tP",
++    "msgbatch_017Vob4KLjX8KgTVAc4zsYL8",
++    "msgbatch_01BwPVvZYm87hxjZFBH1iUws",
++    "msgbatch_013mYehGZpAYvaqfJTXaYkmc"
++  ],
++  "created_at": "2026-11-09T17:25:34Z",
++  "updated_at": "2026-11-09T17:41:05Z"
++}

added data/pairs/pilot_opus/manifest.json +57 −0

++{
++  "created_at": "2026-11-09T17:41:01Z",
++  "git": {
++    "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "job": "rewrite",
++  "provider": "anthropic",
++  "model": "claude-opus-5-5",
++  "prompt_version": "rw-1.0",
++  "prompt_fingerprint": "8af5d0bf61cfb0f6",
++  "budget_usd": 30.0,
++  "batch": true,
++  "corpus": "data/parsed/pilot_pmc",
++  "n_passages": 700,
++  "variants": [
++    "A",
++    "B",
++    "C",
++    "D",
++    "E"
++  ],
++  "selection_seed": 1,
++  "estimate": {
++    "pending_requests": 0,
++    "done": 4224,
++    "est_input_tokens": 0,
++    "est_output_tokens": 0,
++    "est_cost_usd": 0.0,
++    "budget_usd": 30.0,
++    "spent_usd": 18.89748
++  },
++  "live": true
++}

added data/pairs/pilot_opus/validated/validation_summary.json +92 −0

++{
++  "created_at": "2026-11-09T17:52:34Z",
++  "git": {
++    "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "validator_version": "val-1.0",
++  "config": {
++    "min_len_ratio": 0.5,
++    "max_len_ratio": 1.8,
++    "concise_min_len_ratio": 0.4,
++    "max_norm_edit_similarity": 0.95,
++    "min_norm_edit_similarity": 0.05,
++    "min_semantic_sim": 0.7,
++    "max_negation_delta": 2,
++    "max_hedge_delta": 4,
++    "require_language": "en",
++    "entity_min_recall": 0.6,
++    "use_embeddings": true
++  },
++  "stats": {
++    "rewrites": 5844,
++    "status": {
++      "ok": 5801,
++      "refused": 43
++    },
++    "validated": 5844,
++    "passed": 5383,
++    "quarantined": 461,
++    "by_variant": {
++      "B": {
++        "n": 974,
++        "passed": 884
++      },
++      "E": {
++        "n": 1948,
++        "passed": 1804
++      },
++      "C": {
++        "n": 974,
++        "passed": 894
++      },
++      "A": {
++        "n": 974,
++        "passed": 896
++      },
++      "D": {
++        "n": 974,
++        "passed": 905
++      }
++    },
++    "fail_reasons": {
++      "numbers missing=1": 265,
++      "entity recall": 39,
++      "numbers missing=2": 48,
++      "numbers missing=0": 31,
++      "negation": 11,
++      "status:refused": 43,
++      "sentence count": 3,
++      "formatting": 21,
++      "numbers missing=6": 1,
++      "numbers missing=3": 4,
++      "numbers missing=4": 1,
++      "hedge": 1,
++      "numbers missing=5": 1
++    },
++    "acceptance_rate": 0.9211
++  }
++}

added data/pairs/vs_smoke/ledger.json +11 −0

++{
++  "spent_usd": 0.0,
++  "n_ok": 720,
++  "n_error": 0,
++  "n_refused": 0,
++  "n_truncated": 0,
++  "n_empty": 0,
++  "batches": [],
++  "created_at": "2026-11-09T17:23:03Z",
++  "updated_at": "2026-11-09T17:23:06Z"
++}

added data/pairs/vs_smoke/manifest.json +57 −0

++{
++  "created_at": "2026-11-09T17:23:03Z",
++  "git": {
++    "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "job": "rewrite",
++  "provider": "mock",
++  "model": "mock-rewriter-1",
++  "prompt_version": "rw-1.0",
++  "prompt_fingerprint": "8af5d0bf61cfb0f6",
++  "budget_usd": 30.0,
++  "batch": false,
++  "corpus": "data/parsed/smoke",
++  "n_passages": 120,
++  "variants": [
++    "A",
++    "B",
++    "C",
++    "D",
++    "E"
++  ],
++  "selection_seed": 1,
++  "estimate": {
++    "pending_requests": 720,
++    "done": 0,
++    "est_input_tokens": 354066,
++    "est_output_tokens": 183744,
++    "est_cost_usd": null,
++    "budget_usd": 30.0,
++    "spent_usd": 0.0
++  },
++  "live": false
++}

added data/pairs/vs_smoke/validated/validation_summary.json +79 −0

++{
++  "created_at": "2026-11-09T17:23:07Z",
++  "git": {
++    "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "validator_version": "val-1.0",
++  "config": {
++    "min_len_ratio": 0.5,
++    "max_len_ratio": 1.8,
++    "concise_min_len_ratio": 0.4,
++    "max_norm_edit_similarity": 0.95,
++    "min_norm_edit_similarity": 0.05,
++    "min_semantic_sim": 0.7,
++    "max_negation_delta": 2,
++    "max_hedge_delta": 4,
++    "require_language": "en",
++    "entity_min_recall": 0.6,
++    "use_embeddings": false
++  },
++  "stats": {
++    "rewrites": 720,
++    "status": {
++      "ok": 720
++    },
++    "validated": 720,
++    "passed": 557,
++    "quarantined": 163,
++    "by_variant": {
++      "A": {
++        "n": 120,
++        "passed": 106
++      },
++      "E": {
++        "n": 240,
++        "passed": 196
++      },
++      "B": {
++        "n": 120,
++        "passed": 95
++      },
++      "C": {
++        "n": 120,
++        "passed": 81
++      },
++      "D": {
++        "n": 120,
++        "passed": 79
++      }
++    },
++    "fail_reasons": {
++      "near-verbatim": 163
++    },
++    "acceptance_rate": 0.7736
++  }
++}

added data/splits/medium_v0/splits_summary.json +54 −0

++{
++  "created_at": "2026-11-09T18:42:28Z",
++  "git": {
++    "commit": "e4eec911d74f1ec613a2572cb35afde77ec2bdcc",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "config": {
++    "seed": 20261109,
++    "train": 0.8,
++    "val": 0.1,
++    "test": 0.1,
++    "heldout_disciplines": [
++      "physics",
++      "earth_sciences"
++    ],
++    "heldout_before_year": 1900,
++    "author_safe": true,
++    "venue_safe": false
++  },
++  "summary": {
++    "heldout_domain": 257,
++    "heldout_period": 472,
++    "quarantine": 19,
++    "test": 1166,
++    "train": 10268,
++    "val": 1190
++  },
++  "n_docs": 13372,
++  "group_leaks": [],
++  "n_fixed": 1180
++}

added data/splits/pilot_pmc/splits_summary.json +52 −0

++{
++  "created_at": "2026-11-09T17:24:36Z",
++  "git": {
++    "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "config": {
++    "seed": 20261109,
++    "train": 0.8,
++    "val": 0.1,
++    "test": 0.1,
++    "heldout_disciplines": [
++      "chemistry"
++    ],
++    "heldout_before_year": 1950,
++    "author_safe": true,
++    "venue_safe": false
++  },
++  "summary": {
++    "heldout_domain": 47,
++    "heldout_period": 79,
++    "test": 100,
++    "train": 856,
++    "val": 98
++  },
++  "n_docs": 1180,
++  "group_leaks": [],
++  "n_fixed": 0
++}

added data/splits/smoke/splits_summary.json +51 −0

++{
++  "created_at": "2026-11-09T17:23:02Z",
++  "git": {
++    "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
++    "branch": "main",
++    "dirty": true
++  },
++  "libraries": {
++    "python": "3.14.4",
++    "torch": "2.12.0",
++    "numpy": "2.5.4",
++    "anthropic": "2.3.0",
++    "transformers": "5.8.1",
++    "sentencepiece": "0.2.2",
++    "lxml": "6.1.3",
++    "pyarrow": "24.0.0",
++    "sacrebleu": "2.6.0",
++    "datasketch": "2.0.0",
++    "rapidfuzz": "3.14.6"
++  },
++  "hardware": {
++    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
++    "machine": "arm64",
++    "cpu_count": 18,
++    "chip": "Apple M5 Max",
++    "ram_gb": 48.0,
++    "torch_mps": true,
++    "torch_cuda": false
++  },
++  "config": {
++    "seed": 20261109,
++    "train": 0.8,
++    "val": 0.1,
++    "test": 0.1,
++    "heldout_disciplines": [
++      "physics"
++    ],
++    "heldout_before_year": 1990,
++    "author_safe": true,
++    "venue_safe": false
++  },
++  "summary": {
++    "heldout_domain": 1,
++    "heldout_period": 12,
++    "test": 12,
++    "train": 69,
++    "val": 6
++  },
++  "n_docs": 100,
++  "group_leaks": []
++}

modified scripts/train_tokenizer.py +20 −6

				`@@ -15,23 +15,37 @@ from htr.utils.provenance import run_header # noqa: E402`
15		15
16	def main() -> None:	16	def main() -> None:
17	ap = argparse.ArgumentParser()	17	ap = argparse.ArgumentParser()
18	− ap.add_argument("--pairs", required=True)	18	+ ap.add_argument("--pairs", default=None, help="pairs.jsonl (originals + rewrites of the train split)")
		19	+ ap.add_argument("--corpus", default=None, help="parsed corpus dir: eligible train-split passages in placeholder form")
19	ap.add_argument("--splits", required=True)	20	ap.add_argument("--splits", required=True)
20	ap.add_argument("--out", required=True)	21	ap.add_argument("--out", required=True)
21	ap.add_argument("--vocab", type=int, default=32000)	22	ap.add_argument("--vocab", type=int, default=32000)
22	ap.add_argument("--model-type", default="unigram")	23	ap.add_argument("--model-type", default="unigram")
		24	+ ap.add_argument("--max-lines", type=int, default=600000)
23	a = ap.parse_args()	25	a = ap.parse_args()
		26	+ if not (a.pairs or a.corpus):
		27	+ ap.error("--pairs or --corpus required")
24	train_docs = {s["doc_id"] for s in read_jsonl(a.splits) if s["split"] == "train"}	28	train_docs = {s["doc_id"] for s in read_jsonl(a.splits) if s["split"] == "train"}
25	out = Path(a.out)	29	out = Path(a.out)
26	out.mkdir(parents=True, exist_ok=True)	30	out.mkdir(parents=True, exist_ok=True)
27	txt = out / "tokenizer_train_text.txt"	31	txt = out / "tokenizer_train_text.txt"
28	n = 0	32	n = 0
29	with open(txt, "w", encoding="utf-8") as f:	33	with open(txt, "w", encoding="utf-8") as f:
30	− for p in read_jsonl(a.pairs):	34	+ if a.pairs:
31	− if p["doc_id"] in train_docs:	35	+ for p in read_jsonl(a.pairs):
32	− f.write(p["target_text"].replace("\n", " ") + "\n")	36	+ if p["doc_id"] in train_docs:
33	− f.write(p["source_text"].replace("\n", " ") + "\n")	37	+ f.write(p["target_text"].replace("\n", " ") + "\n")
34	− n += 2	38	+ f.write(p["source_text"].replace("\n", " ") + "\n")
		39	+ n += 2
		40	+ if a.corpus:
		41	+ from htr.parse.protect import protect
		42	+
		43	+ for p in read_jsonl(Path(a.corpus) / "passages.jsonl"):
		44	+ if p["eligible"] and p["doc_id"] in train_docs:
		45	+ f.write(protect(p["text"]).text.replace("\n", " ") + "\n")
		46	+ n += 1
		47	+ if n >= a.max_lines:
		48	+ break
35	vocab = a.vocab	49	vocab = a.vocab
36	if n < 5000:	50	if n < 5000:
37	vocab = min(vocab, 8000)	51	vocab = min(vocab, 8000)
38		52

spb/wpai0002_uqo

Medium corpus v0 (13 372 docs / 371 419 eligible passages) summaries; tokenizer trainer on corpus passages; evidence summaries tracked

Theme