SPB Git
spboucher.ai

spb/wpai0002_uqo

Public

Medium corpus v0 (13 372 docs / 371 419 eligible passages) summaries; tokenizer trainer on corpus passages; evidence summaries tracked

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 9, 2026) parent e4eec91 Browse files

22 changed files +1,270 −6

22 changed files
Unified Split
modified .gitignore +12 −0
@@ -42,3 +42,15 @@ logs/
42 42 .DS_Store
43 43 # demo
44 44 demo/node_modules/
45 +
46 +# Evidence summaries ARE committed (small JSON)
47 +!data/parsed/*/build_summary.json
48 +!data/parsed/*/relabel_log.json
49 +!data/splits/*/
50 +data/splits/*/splits.jsonl
51 +!data/pairs/*/
52 +data/pairs/*/rewrites.jsonl
53 +data/pairs/*/plan.jsonl
54 +data/pairs/*/validated/pairs.jsonl
55 +data/pairs/*/validated/quarantine.jsonl
56 +data/pairs/*/batches/
added data/manifests/discover_books_bookshelf_seed2.json +89 −0
@@ -0,0 +1,89 @@
1 +{
2 + "created_at": "2026-11-09T18:28:01Z",
3 + "git": {
4 + "commit": "e4eec911d74f1ec613a2572cb35afde77ec2bdcc",
5 + "branch": "main",
6 + "dirty": false
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "script": "discover_books",
31 + "args": {
32 + "source": "bookshelf",
33 + "max_docs": 300,
34 + "max_attempts": 450,
35 + "seed": 2,
36 + "out": "data/raw/books",
37 + "codes": "CC0,PUBLIC-DOMAIN,CC-BY,CC-BY-SA,CC-BY-NC,CC-BY-NC-SA",
38 + "locc": "Q,R,T,H,G,B,L,Z",
39 + "max_mb": 80.0,
40 + "pool_only": false,
41 + "report": null
42 + },
43 + "stats": {
44 + "list_rows": 9733,
45 + "pre2018_rows": 3941,
46 + "attempted": 450,
47 + "too_large": 9,
48 + "books_kept": 163,
49 + "chapters_written": 1351,
50 + "license_excluded": 99,
51 + "license_unknown": 142,
52 + "not_pre2018_xml": 0,
53 + "no_body_chapters": 37,
54 + "errors": 0,
55 + "by_license": {
56 + "PUBLIC-DOMAIN": 169,
57 + "CC-BY-NC-ND": 98,
58 + "CC0": 25,
59 + "UNKNOWN": 142,
60 + "CC-BY-NC-SA": 1,
61 + "CC-BY-ND": 1,
62 + "CC-BY-NC": 3,
63 + "CC-BY": 2
64 + },
65 + "by_publisher": {
66 + "Agency for Healthcare Research and Quality (US)": 109,
67 + "Centers for Disease Control and Prevention (US)": 19,
68 + "Substance Abuse and Mental Health Services Administration (US)": 10,
69 + "Office of the Surgeon General (US)": 2,
70 + "Department of Veterans Affairs (US)": 13,
71 + "American Epilepsy Society": 1,
72 + "National Center for Biotechnology Information (US)": 3,
73 + "Springer": 3,
74 + "The International Bank for Reconstruction and Development / The World Bank": 1,
75 + "National Institute of Environmental Health Sciences": 2
76 + },
77 + "by_era": {
78 + "2010-2017": 92,
79 + "2000-2009": 57,
80 + "1980-1999": 14
81 + },
82 + "ftp": {
83 + "requests": 895,
84 + "retries": 4,
85 + "bytes": 2310287545
86 + },
87 + "elapsed_s": 1822.4
88 + }
89 +}
added data/manifests/discover_books_gutenberg_seed2.json +244 −0
@@ -0,0 +1,244 @@
1 +{
2 + "created_at": "2026-11-09T18:38:44Z",
3 + "git": {
4 + "commit": "e4eec911d74f1ec613a2572cb35afde77ec2bdcc",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "script": "discover_books",
31 + "args": {
32 + "source": "gutenberg",
33 + "max_docs": 500,
34 + "max_attempts": 700,
35 + "seed": 2,
36 + "out": "data/raw/books",
37 + "codes": "CC0,PUBLIC-DOMAIN,CC-BY,CC-BY-SA,CC-BY-NC,CC-BY-NC-SA",
38 + "locc": "Q,R,T,H,G,B,L,Z",
39 + "max_mb": 25.0,
40 + "pool_only": false,
41 + "report": null
42 + },
43 + "stats": {
44 + "catalog_rows": 79769,
45 + "candidates": 7799,
46 + "attempted": 517,
47 + "downloaded": 500,
48 + "already_present": 0,
49 + "not_found": 5,
50 + "too_large": 3,
51 + "copyrighted_header": 9,
52 + "no_pd_header": 0,
53 + "no_date": 60,
54 + "date_from_frontmatter": 370,
55 + "date_from_death_year": 70,
56 + "errors": 0,
57 + "by_locc": {
58 + "RC": 23,
59 + "LT": 2,
60 + "QH": 11,
61 + "GT": 17,
62 + "HV": 13,
63 + "TD": 3,
64 + "BF": 26,
65 + "RS": 1,
66 + "LB": 28,
67 + "QA": 1,
68 + "GN": 11,
69 + "HS": 1,
70 + "TN": 3,
71 + "Z": 62,
72 + "QL": 13,
73 + "G": 32,
74 + "HX": 3,
75 + "TA": 3,
76 + "BJ": 14,
77 + "LD": 4,
78 + "TL": 6,
79 + "B": 19,
80 + "RZ": 2,
81 + "LA": 12,
82 + "QR": 1,
83 + "HD": 4,
84 + "TS": 3,
85 + "GC": 1,
86 + "HN": 3,
87 + "T": 16,
88 + "RM": 8,
89 + "QK": 5,
90 + "HE": 7,
91 + "R": 12,
92 + "QE": 6,
93 + "H": 2,
94 + "Q": 11,
95 + "RA": 12,
96 + "HQ": 13,
97 + "TJ": 4,
98 + "HF": 3,
99 + "TT": 9,
100 + "HM": 1,
101 + "TC": 1,
102 + "QP": 4,
103 + "HT": 2,
104 + "QC": 3,
105 + "HG": 4,
106 + "RG": 4,
107 + "TP": 5,
108 + "LF": 4,
109 + "HB": 2,
110 + "LC": 14,
111 + "TH": 2,
112 + "QB": 4,
113 + "HC": 2,
114 + "RJ": 2,
115 + "TK": 4,
116 + "BD": 3,
117 + "HJ": 1,
118 + "RD": 1,
119 + "BC": 1,
120 + "QD": 1,
121 + "TF": 1,
122 + "LH": 1,
123 + "HA": 1,
124 + "QM": 1,
125 + "GB": 1
126 + },
127 + "by_era": {
128 + "pre1900": 178,
129 + "1950-1979": 20,
130 + "1900-1949": 236,
131 + "unknown": 60,
132 + "1980-1999": 2,
133 + "2000-2009": 4
134 + },
135 + "by_date_rule": {
136 + "frontmatter:imprint": 164,
137 + "frontmatter:copyright": 140,
138 + "death_year": 70,
139 + "frontmatter:published": 41,
140 + "none": 60,
141 + "frontmatter:roman": 25
142 + },
143 + "pool_by_class": {
144 + "QH": 543,
145 + "QL": 491,
146 + "Z": 490,
147 + "BF": 372,
148 + "HV": 356,
149 + "B": 344,
150 + "HQ": 341,
151 + "G": 256,
152 + "BJ": 234,
153 + "Q": 202,
154 + "HD": 190,
155 + "RC": 190,
156 + "QC": 182,
157 + "QK": 180,
158 + "T": 167,
159 + "TT": 164,
160 + "QE": 161,
161 + "HE": 156,
162 + "QB": 139,
163 + "LB": 132,
164 + "TS": 130,
165 + "QA": 120,
166 + "RA": 112,
167 + "HX": 99,
168 + "GN": 99,
169 + "TL": 96,
170 + "TK": 89,
171 + "HF": 87,
172 + "RM": 84,
173 + "HB": 81,
174 + "GT": 81,
175 + "TP": 80,
176 + "QD": 78,
177 + "R": 77,
178 + "HG": 76,
179 + "TJ": 63,
180 + "HN": 62,
181 + "QP": 61,
182 + "HT": 55,
183 + "LC": 55,
184 + "TA": 50,
185 + "LA": 48,
186 + "HM": 47,
187 + "HC": 43,
188 + "TN": 42,
189 + "TH": 38,
190 + "TR": 37,
191 + "RD": 34,
192 + "LF": 33,
193 + "BD": 33,
194 + "RG": 32,
195 + "TC": 32,
196 + "TF": 27,
197 + "HS": 26,
198 + "TD": 25,
199 + "BC": 24,
200 + "GB": 19,
201 + "LD": 16,
202 + "RJ": 16,
203 + "H": 16,
204 + "RS": 16,
205 + "QR": 15,
206 + "HJ": 14,
207 + "QM": 14,
208 + "RT": 11,
209 + "RE": 11,
210 + "GA": 10,
211 + "BH": 9,
212 + "RZ": 8,
213 + "RK": 8,
214 + "RX": 7,
215 + "RB": 7,
216 + "LT": 7,
217 + "GC": 6,
218 + "TE": 6,
219 + "LH": 6,
220 + "GF": 5,
221 + "GR": 5,
222 + "TG": 5,
223 + "HA": 4,
224 + "L": 4,
225 + "RL": 3,
226 + "RF": 3,
227 + "RV": 1,
228 + "LE": 1
229 + },
230 + "catalog": {
231 + "url": "https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv",
232 + "path": "data/raw/books/_catalog/pg_catalog.csv",
233 + "cached": true,
234 + "sha256": "8c06ec1c46751dc184b640b8a8ed5842c873ae3b621d575f8e76837f12f33eb6",
235 + "retrieved_or_cached_at": "2026-11-09T17:57:39Z"
236 + },
237 + "http": {
238 + "requests": 527,
239 + "retries": 0,
240 + "bytes": 284820446
241 + },
242 + "elapsed_s": 2465.3
243 + }
244 +}
added data/pairs/mock_smoke/ledger.json +11 −0
@@ -0,0 +1,11 @@
1 +{
2 + "spent_usd": 0.0,
3 + "n_ok": 360,
4 + "n_error": 0,
5 + "n_refused": 0,
6 + "n_truncated": 0,
7 + "n_empty": 0,
8 + "batches": [],
9 + "created_at": "2026-11-09T17:12:45Z",
10 + "updated_at": "2026-11-09T17:12:47Z"
11 +}
added data/pairs/mock_smoke/manifest.json +57 −0
@@ -0,0 +1,57 @@
1 +{
2 + "created_at": "2026-11-09T17:12:45Z",
3 + "git": {
4 + "commit": null,
5 + "branch": null,
6 + "dirty": false
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "job": "rewrite",
31 + "provider": "mock",
32 + "model": "mock-rewriter-1",
33 + "prompt_version": "rw-1.0",
34 + "prompt_fingerprint": "8af5d0bf61cfb0f6",
35 + "budget_usd": 30.0,
36 + "batch": false,
37 + "corpus": "data/parsed/smoke",
38 + "n_passages": 60,
39 + "variants": [
40 + "A",
41 + "B",
42 + "C",
43 + "D",
44 + "E"
45 + ],
46 + "selection_seed": 1,
47 + "estimate": {
48 + "pending_requests": 360,
49 + "done": 0,
50 + "est_input_tokens": 179232,
51 + "est_output_tokens": 94260,
52 + "est_cost_usd": null,
53 + "budget_usd": 30.0,
54 + "spent_usd": 0.0
55 + },
56 + "live": false
57 +}
added data/pairs/mock_smoke/validated/validation_summary.json +79 −0
@@ -0,0 +1,79 @@
1 +{
2 + "created_at": "2026-11-09T17:12:55Z",
3 + "git": {
4 + "commit": null,
5 + "branch": null,
6 + "dirty": false
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "validator_version": "val-1.0",
31 + "config": {
32 + "min_len_ratio": 0.5,
33 + "max_len_ratio": 1.8,
34 + "concise_min_len_ratio": 0.4,
35 + "max_norm_edit_similarity": 0.95,
36 + "min_norm_edit_similarity": 0.05,
37 + "min_semantic_sim": 0.7,
38 + "max_negation_delta": 2,
39 + "max_hedge_delta": 4,
40 + "require_language": "en",
41 + "entity_min_recall": 0.6,
42 + "use_embeddings": true
43 + },
44 + "stats": {
45 + "rewrites": 360,
46 + "status": {
47 + "ok": 360
48 + },
49 + "validated": 360,
50 + "passed": 285,
51 + "quarantined": 75,
52 + "by_variant": {
53 + "A": {
54 + "n": 60,
55 + "passed": 46
56 + },
57 + "E": {
58 + "n": 120,
59 + "passed": 102
60 + },
61 + "B": {
62 + "n": 60,
63 + "passed": 52
64 + },
65 + "C": {
66 + "n": 60,
67 + "passed": 43
68 + },
69 + "D": {
70 + "n": 60,
71 + "passed": 42
72 + }
73 + },
74 + "fail_reasons": {
75 + "near-verbatim": 75
76 + },
77 + "acceptance_rate": 0.7917
78 + }
79 +}
added data/pairs/pilot_heldout_models/ledger.json +15 −0
@@ -0,0 +1,15 @@
1 +{
2 + "spent_usd": 1.428675,
3 + "n_ok": 600,
4 + "n_error": 0,
5 + "n_refused": 0,
6 + "n_truncated": 0,
7 + "n_empty": 0,
8 + "batches": [
9 + "msgbatch_01V4FduEDL26WRNLQEV3RHMP",
10 + "msgbatch_01SWH3HaFBNZPCkKaQW3fwpQ"
11 + ],
12 + "created_at": "2026-11-09T17:43:49Z",
13 + "submitted_est_usd": 0.6,
14 + "updated_at": "2026-11-09T17:50:52Z"
15 +}
added data/pairs/pilot_heldout_models/manifest.json +54 −0
@@ -0,0 +1,54 @@
1 +{
2 + "created_at": "2026-11-09T17:50:49Z",
3 + "git": {
4 + "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "job": "rewrite",
31 + "provider": "anthropic",
32 + "model": "claude-sonnet-5-5",
33 + "prompt_version": "rw-1.0",
34 + "prompt_fingerprint": "8af5d0bf61cfb0f6",
35 + "budget_usd": 30.0,
36 + "batch": true,
37 + "corpus": "data/parsed/pilot_pmc",
38 + "n_passages": 150,
39 + "variants": [
40 + "A",
41 + "B"
42 + ],
43 + "selection_seed": 1,
44 + "estimate": {
45 + "pending_requests": 300,
46 + "done": 0,
47 + "est_input_tokens": 153430,
48 + "est_output_tokens": 83198,
49 + "est_cost_usd": 0.57,
50 + "budget_usd": 30.0,
51 + "spent_usd": 0.0
52 + },
53 + "live": true
54 +}
added data/pairs/pilot_heldout_models/validated/validation_summary.json +74 −0
@@ -0,0 +1,74 @@
1 +{
2 + "created_at": "2026-11-09T17:51:06Z",
3 + "git": {
4 + "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "validator_version": "val-1.0",
31 + "config": {
32 + "min_len_ratio": 0.5,
33 + "max_len_ratio": 1.8,
34 + "concise_min_len_ratio": 0.4,
35 + "max_norm_edit_similarity": 0.95,
36 + "min_norm_edit_similarity": 0.05,
37 + "min_semantic_sim": 0.7,
38 + "max_negation_delta": 2,
39 + "max_hedge_delta": 4,
40 + "require_language": "en",
41 + "entity_min_recall": 0.6,
42 + "use_embeddings": true
43 + },
44 + "stats": {
45 + "rewrites": 600,
46 + "status": {
47 + "ok": 600
48 + },
49 + "validated": 600,
50 + "passed": 531,
51 + "quarantined": 69,
52 + "by_variant": {
53 + "A": {
54 + "n": 300,
55 + "passed": 267
56 + },
57 + "B": {
58 + "n": 300,
59 + "passed": 264
60 + }
61 + },
62 + "fail_reasons": {
63 + "numbers missing=1": 37,
64 + "entity recall": 7,
65 + "numbers missing=2": 14,
66 + "numbers missing=3": 7,
67 + "formatting": 4,
68 + "numbers missing=4": 1,
69 + "near-verbatim": 1,
70 + "numbers missing=7": 1
71 + },
72 + "acceptance_rate": 0.885
73 + }
74 +}
added data/pairs/pilot_heldout_prompts/ledger.json +14 −0
@@ -0,0 +1,14 @@
1 +{
2 + "spent_usd": 1.39825,
3 + "n_ok": 297,
4 + "n_error": 0,
5 + "n_refused": 3,
6 + "n_truncated": 0,
7 + "n_empty": 0,
8 + "batches": [
9 + "msgbatch_01L5QkwBhyFtAydvR2X8g394"
10 + ],
11 + "created_at": "2026-11-09T17:43:55Z",
12 + "submitted_est_usd": 1.14,
13 + "updated_at": "2026-11-09T17:50:55Z"
14 +}
added data/pairs/pilot_heldout_prompts/manifest.json +54 −0
@@ -0,0 +1,54 @@
1 +{
2 + "created_at": "2026-11-09T17:50:53Z",
3 + "git": {
4 + "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "job": "rewrite",
31 + "provider": "anthropic",
32 + "model": "claude-opus-5-5",
33 + "prompt_version": "rw-1.0",
34 + "prompt_fingerprint": "8af5d0bf61cfb0f6",
35 + "budget_usd": 30.0,
36 + "batch": true,
37 + "corpus": "data/parsed/pilot_pmc",
38 + "n_passages": 150,
39 + "variants": [
40 + "H1",
41 + "H2"
42 + ],
43 + "selection_seed": 1,
44 + "estimate": {
45 + "pending_requests": 300,
46 + "done": 0,
47 + "est_input_tokens": 153430,
48 + "est_output_tokens": 83198,
49 + "est_cost_usd": 1.14,
50 + "budget_usd": 30.0,
51 + "spent_usd": 0.0
52 + },
53 + "live": true
54 +}
added data/pairs/pilot_heldout_prompts/validated/validation_summary.json +75 −0
@@ -0,0 +1,75 @@
1 +{
2 + "created_at": "2026-11-09T17:51:14Z",
3 + "git": {
4 + "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "validator_version": "val-1.0",
31 + "config": {
32 + "min_len_ratio": 0.5,
33 + "max_len_ratio": 1.8,
34 + "concise_min_len_ratio": 0.4,
35 + "max_norm_edit_similarity": 0.95,
36 + "min_norm_edit_similarity": 0.05,
37 + "min_semantic_sim": 0.7,
38 + "max_negation_delta": 2,
39 + "max_hedge_delta": 4,
40 + "require_language": "en",
41 + "entity_min_recall": 0.6,
42 + "use_embeddings": true
43 + },
44 + "stats": {
45 + "rewrites": 300,
46 + "status": {
47 + "ok": 297,
48 + "refused": 3
49 + },
50 + "validated": 300,
51 + "passed": 266,
52 + "quarantined": 34,
53 + "by_variant": {
54 + "H2": {
55 + "n": 150,
56 + "passed": 138
57 + },
58 + "H1": {
59 + "n": 150,
60 + "passed": 128
61 + }
62 + },
63 + "fail_reasons": {
64 + "numbers missing=1": 20,
65 + "numbers missing=2": 6,
66 + "entity recall": 2,
67 + "status:refused": 3,
68 + "formatting": 1,
69 + "placeholders missing=0": 1,
70 + "numbers missing=0": 1,
71 + "numbers missing=3": 2
72 + },
73 + "acceptance_rate": 0.8867
74 + }
75 +}
added data/pairs/pilot_opus/ledger.json +19 −0
@@ -0,0 +1,19 @@
1 +{
2 + "spent_usd": 26.85241,
3 + "n_ok": 5801,
4 + "n_error": 0,
5 + "n_refused": 43,
6 + "n_truncated": 0,
7 + "n_empty": 0,
8 + "batches": [
9 + "msgbatch_01T8HKLMfnEWKK3JQUuWxunJ",
10 + "msgbatch_01RMbxWJDx3NR88JcszbMADz",
11 + "msgbatch_01Kvfx7j5b1VY1A5R1h9VGQj",
12 + "msgbatch_014HKsCPuhepp7L2tprFi1tP",
13 + "msgbatch_017Vob4KLjX8KgTVAc4zsYL8",
14 + "msgbatch_01BwPVvZYm87hxjZFBH1iUws",
15 + "msgbatch_013mYehGZpAYvaqfJTXaYkmc"
16 + ],
17 + "created_at": "2026-11-09T17:25:34Z",
18 + "updated_at": "2026-11-09T17:41:05Z"
19 +}
added data/pairs/pilot_opus/manifest.json +57 −0
@@ -0,0 +1,57 @@
1 +{
2 + "created_at": "2026-11-09T17:41:01Z",
3 + "git": {
4 + "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "job": "rewrite",
31 + "provider": "anthropic",
32 + "model": "claude-opus-5-5",
33 + "prompt_version": "rw-1.0",
34 + "prompt_fingerprint": "8af5d0bf61cfb0f6",
35 + "budget_usd": 30.0,
36 + "batch": true,
37 + "corpus": "data/parsed/pilot_pmc",
38 + "n_passages": 700,
39 + "variants": [
40 + "A",
41 + "B",
42 + "C",
43 + "D",
44 + "E"
45 + ],
46 + "selection_seed": 1,
47 + "estimate": {
48 + "pending_requests": 0,
49 + "done": 4224,
50 + "est_input_tokens": 0,
51 + "est_output_tokens": 0,
52 + "est_cost_usd": 0.0,
53 + "budget_usd": 30.0,
54 + "spent_usd": 18.89748
55 + },
56 + "live": true
57 +}
added data/pairs/pilot_opus/validated/validation_summary.json +92 −0
@@ -0,0 +1,92 @@
1 +{
2 + "created_at": "2026-11-09T17:52:34Z",
3 + "git": {
4 + "commit": "b4b5fe4f40513d0672831db36c7d4a1a58527cae",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "validator_version": "val-1.0",
31 + "config": {
32 + "min_len_ratio": 0.5,
33 + "max_len_ratio": 1.8,
34 + "concise_min_len_ratio": 0.4,
35 + "max_norm_edit_similarity": 0.95,
36 + "min_norm_edit_similarity": 0.05,
37 + "min_semantic_sim": 0.7,
38 + "max_negation_delta": 2,
39 + "max_hedge_delta": 4,
40 + "require_language": "en",
41 + "entity_min_recall": 0.6,
42 + "use_embeddings": true
43 + },
44 + "stats": {
45 + "rewrites": 5844,
46 + "status": {
47 + "ok": 5801,
48 + "refused": 43
49 + },
50 + "validated": 5844,
51 + "passed": 5383,
52 + "quarantined": 461,
53 + "by_variant": {
54 + "B": {
55 + "n": 974,
56 + "passed": 884
57 + },
58 + "E": {
59 + "n": 1948,
60 + "passed": 1804
61 + },
62 + "C": {
63 + "n": 974,
64 + "passed": 894
65 + },
66 + "A": {
67 + "n": 974,
68 + "passed": 896
69 + },
70 + "D": {
71 + "n": 974,
72 + "passed": 905
73 + }
74 + },
75 + "fail_reasons": {
76 + "numbers missing=1": 265,
77 + "entity recall": 39,
78 + "numbers missing=2": 48,
79 + "numbers missing=0": 31,
80 + "negation": 11,
81 + "status:refused": 43,
82 + "sentence count": 3,
83 + "formatting": 21,
84 + "numbers missing=6": 1,
85 + "numbers missing=3": 4,
86 + "numbers missing=4": 1,
87 + "hedge": 1,
88 + "numbers missing=5": 1
89 + },
90 + "acceptance_rate": 0.9211
91 + }
92 +}
added data/pairs/vs_smoke/ledger.json +11 −0
@@ -0,0 +1,11 @@
1 +{
2 + "spent_usd": 0.0,
3 + "n_ok": 720,
4 + "n_error": 0,
5 + "n_refused": 0,
6 + "n_truncated": 0,
7 + "n_empty": 0,
8 + "batches": [],
9 + "created_at": "2026-11-09T17:23:03Z",
10 + "updated_at": "2026-11-09T17:23:06Z"
11 +}
added data/pairs/vs_smoke/manifest.json +57 −0
@@ -0,0 +1,57 @@
1 +{
2 + "created_at": "2026-11-09T17:23:03Z",
3 + "git": {
4 + "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "job": "rewrite",
31 + "provider": "mock",
32 + "model": "mock-rewriter-1",
33 + "prompt_version": "rw-1.0",
34 + "prompt_fingerprint": "8af5d0bf61cfb0f6",
35 + "budget_usd": 30.0,
36 + "batch": false,
37 + "corpus": "data/parsed/smoke",
38 + "n_passages": 120,
39 + "variants": [
40 + "A",
41 + "B",
42 + "C",
43 + "D",
44 + "E"
45 + ],
46 + "selection_seed": 1,
47 + "estimate": {
48 + "pending_requests": 720,
49 + "done": 0,
50 + "est_input_tokens": 354066,
51 + "est_output_tokens": 183744,
52 + "est_cost_usd": null,
53 + "budget_usd": 30.0,
54 + "spent_usd": 0.0
55 + },
56 + "live": false
57 +}
added data/pairs/vs_smoke/validated/validation_summary.json +79 −0
@@ -0,0 +1,79 @@
1 +{
2 + "created_at": "2026-11-09T17:23:07Z",
3 + "git": {
4 + "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "validator_version": "val-1.0",
31 + "config": {
32 + "min_len_ratio": 0.5,
33 + "max_len_ratio": 1.8,
34 + "concise_min_len_ratio": 0.4,
35 + "max_norm_edit_similarity": 0.95,
36 + "min_norm_edit_similarity": 0.05,
37 + "min_semantic_sim": 0.7,
38 + "max_negation_delta": 2,
39 + "max_hedge_delta": 4,
40 + "require_language": "en",
41 + "entity_min_recall": 0.6,
42 + "use_embeddings": false
43 + },
44 + "stats": {
45 + "rewrites": 720,
46 + "status": {
47 + "ok": 720
48 + },
49 + "validated": 720,
50 + "passed": 557,
51 + "quarantined": 163,
52 + "by_variant": {
53 + "A": {
54 + "n": 120,
55 + "passed": 106
56 + },
57 + "E": {
58 + "n": 240,
59 + "passed": 196
60 + },
61 + "B": {
62 + "n": 120,
63 + "passed": 95
64 + },
65 + "C": {
66 + "n": 120,
67 + "passed": 81
68 + },
69 + "D": {
70 + "n": 120,
71 + "passed": 79
72 + }
73 + },
74 + "fail_reasons": {
75 + "near-verbatim": 163
76 + },
77 + "acceptance_rate": 0.7736
78 + }
79 +}
added data/splits/medium_v0/splits_summary.json +54 −0
@@ -0,0 +1,54 @@
1 +{
2 + "created_at": "2026-11-09T18:42:28Z",
3 + "git": {
4 + "commit": "e4eec911d74f1ec613a2572cb35afde77ec2bdcc",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "config": {
31 + "seed": 20261109,
32 + "train": 0.8,
33 + "val": 0.1,
34 + "test": 0.1,
35 + "heldout_disciplines": [
36 + "physics",
37 + "earth_sciences"
38 + ],
39 + "heldout_before_year": 1900,
40 + "author_safe": true,
41 + "venue_safe": false
42 + },
43 + "summary": {
44 + "heldout_domain": 257,
45 + "heldout_period": 472,
46 + "quarantine": 19,
47 + "test": 1166,
48 + "train": 10268,
49 + "val": 1190
50 + },
51 + "n_docs": 13372,
52 + "group_leaks": [],
53 + "n_fixed": 1180
54 +}
added data/splits/pilot_pmc/splits_summary.json +52 −0
@@ -0,0 +1,52 @@
1 +{
2 + "created_at": "2026-11-09T17:24:36Z",
3 + "git": {
4 + "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "config": {
31 + "seed": 20261109,
32 + "train": 0.8,
33 + "val": 0.1,
34 + "test": 0.1,
35 + "heldout_disciplines": [
36 + "chemistry"
37 + ],
38 + "heldout_before_year": 1950,
39 + "author_safe": true,
40 + "venue_safe": false
41 + },
42 + "summary": {
43 + "heldout_domain": 47,
44 + "heldout_period": 79,
45 + "test": 100,
46 + "train": 856,
47 + "val": 98
48 + },
49 + "n_docs": 1180,
50 + "group_leaks": [],
51 + "n_fixed": 0
52 +}
added data/splits/smoke/splits_summary.json +51 −0
@@ -0,0 +1,51 @@
1 +{
2 + "created_at": "2026-11-09T17:23:02Z",
3 + "git": {
4 + "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "config": {
31 + "seed": 20261109,
32 + "train": 0.8,
33 + "val": 0.1,
34 + "test": 0.1,
35 + "heldout_disciplines": [
36 + "physics"
37 + ],
38 + "heldout_before_year": 1990,
39 + "author_safe": true,
40 + "venue_safe": false
41 + },
42 + "summary": {
43 + "heldout_domain": 1,
44 + "heldout_period": 12,
45 + "test": 12,
46 + "train": 69,
47 + "val": 6
48 + },
49 + "n_docs": 100,
50 + "group_leaks": []
51 +}
modified scripts/train_tokenizer.py +20 −6
@@ -15,23 +15,37 @@ from htr.utils.provenance import run_header # noqa: E402
15 15
16 16 def main() -> None:
17 17 ap = argparse.ArgumentParser()
18 − ap.add_argument("--pairs", required=True)
18 + ap.add_argument("--pairs", default=None, help="pairs.jsonl (originals + rewrites of the train split)")
19 + ap.add_argument("--corpus", default=None, help="parsed corpus dir: eligible train-split passages in placeholder form")
19 20 ap.add_argument("--splits", required=True)
20 21 ap.add_argument("--out", required=True)
21 22 ap.add_argument("--vocab", type=int, default=32000)
22 23 ap.add_argument("--model-type", default="unigram")
24 + ap.add_argument("--max-lines", type=int, default=600000)
23 25 a = ap.parse_args()
26 + if not (a.pairs or a.corpus):
27 + ap.error("--pairs or --corpus required")
24 28 train_docs = {s["doc_id"] for s in read_jsonl(a.splits) if s["split"] == "train"}
25 29 out = Path(a.out)
26 30 out.mkdir(parents=True, exist_ok=True)
27 31 txt = out / "tokenizer_train_text.txt"
28 32 n = 0
29 33 with open(txt, "w", encoding="utf-8") as f:
30 − for p in read_jsonl(a.pairs):
31 − if p["doc_id"] in train_docs:
32 − f.write(p["target_text"].replace("\n", " ") + "\n")
33 − f.write(p["source_text"].replace("\n", " ") + "\n")
34 − n += 2
34 + if a.pairs:
35 + for p in read_jsonl(a.pairs):
36 + if p["doc_id"] in train_docs:
37 + f.write(p["target_text"].replace("\n", " ") + "\n")
38 + f.write(p["source_text"].replace("\n", " ") + "\n")
39 + n += 2
40 + if a.corpus:
41 + from htr.parse.protect import protect
42 +
43 + for p in read_jsonl(Path(a.corpus) / "passages.jsonl"):
44 + if p["eligible"] and p["doc_id"] in train_docs:
45 + f.write(protect(p["text"]).text.replace("\n", " ") + "\n")
46 + n += 1
47 + if n >= a.max_lines:
48 + break
35 49 vocab = a.vocab
36 50 if n < 5000:
37 51 vocab = min(vocab, 8000)
38 52
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere