SPB Git

spb/wp28_uqo

Public

WP28: literature audits C-D, PRIOR_ART_MATRIX (56 rows), PRIORITY_AND_GAP, ASSUMPTIONS, THEOREM_AUDIT, MC driver + cluster scatter

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Oct 28, 2026) parent 7a86c23 Browse files

32 changed files +19,995 −0

32 changed files
added experiments/cluster_assignment.json +876 −0
@@ -0,0 +1,876 @@
1+{
2+ "nodes": {
3+ "M3U96a": 6,
4+ "M3U96b": 6,
5+ "M2U64": 4,
6+ "M4M64a": 4,
7+ "M4M64b": 4,
8+ "M4M64c": 4,
9+ "M5M36": 4,
10+ "M4BP48": 3,
11+ "M4BP36": 3,
12+ "M2M32c": 3,
13+ "m4mm": 3,
14+ "m4mn": 3,
15+ "m6ma": 3,
16+ "m6mb": 3,
17+ "m5ma": 3,
18+ "m4md": 2,
19+ "m4ml": 2,
20+ "M3BA24": 2
21+ },
22+ "assignment": {
23+ "M3U96a": [
24+ {
25+ "config": "experiments/configs/main_logistic.yaml",
26+ "cell": 49,
27+ "cost": 187574977245.9856,
28+ "name": "logit_p20_p20_N1048576_K8192"
29+ },
30+ {
31+ "config": "experiments/configs/main_logistic.yaml",
32+ "cell": 48,
33+ "cost": 46893744311.4964,
34+ "name": "logit_p20_p20_N262144_K128"
35+ },
36+ {
37+ "config": "experiments/configs/main_logistic.yaml",
38+ "cell": 36,
39+ "cost": 21794242891.16812,
40+ "name": "logit_p3_N1048576_K8192"
41+ },
42+ {
43+ "config": "experiments/configs/robustness.yaml",
44+ "cell": 12,
45+ "cost": 21794242891.16812,
46+ "name": "logit_misspec_p3_N1048576_K8192"
47+ },
48+ {
49+ "config": "experiments/configs/robustness.yaml",
50+ "cell": 14,
51+ "cost": 21794242891.16812,
52+ "name": "logit_misspec_p3_N1048576_K2048"
53+ },
54+ {
55+ "config": "experiments/configs/robustness.yaml",
56+ "cell": 33,
57+ "cost": 4194304000.0,
58+ "name": "exp_contam_p1_N1048576_K65536"
59+ },
60+ {
61+ "config": "experiments/configs/main_logistic.yaml",
62+ "cell": 6,
63+ "cost": 1482910400.3789308,
64+ "name": "logit_p8_N16384_K64"
65+ },
66+ {
67+ "config": "experiments/configs/main_poisson.yaml",
68+ "cell": 10,
69+ "cost": 1482910400.3789308,
70+ "name": "pois_p8_N16384_K16"
71+ },
72+ {
73+ "config": "experiments/configs/main_poisson.yaml",
74+ "cell": 12,
75+ "cost": 1362140180.6980076,
76+ "name": "pois_p3_N65536_K1024"
77+ },
78+ {
79+ "config": "experiments/configs/robustness.yaml",
80+ "cell": 2,
81+ "cost": 1362140180.6980076,
82+ "name": "logit_misspec_p3_N65536_K512"
83+ },
84+ {
85+ "config": "experiments/configs/main_logistic.yaml",
86+ "cell": 9,
87+ "cost": 340535045.1745019,
88+ "name": "logit_p3_N16384_K16"
89+ },
90+ {
91+ "config": "experiments/configs/main_poisson.yaml",
92+ "cell": 5,
93+ "cost": 340535045.1745019,
94+ "name": "pois_p3_N16384_K64"
95+ }
96+ ],
97+ "M3U96b": [
98+ {
99+ "config": "experiments/configs/main_logistic.yaml",
100+ "cell": 50,
101+ "cost": 187574977245.9856,
102+ "name": "logit_p20_p20_N1048576_K4096"
103+ },
104+ {
105+ "config": "experiments/configs/main_logistic.yaml",
106+ "cell": 24,
107+ "cost": 23726566406.062893,
108+ "name": "logit_p8_N262144_K4096"
109+ },
110+ {
111+ "config": "experiments/configs/main_poisson.yaml",
112+ "cell": 32,
113+ "cost": 23726566406.062893,
114+ "name": "pois_p8_N262144_K256"
115+ },
116+ {
117+ "config": "experiments/configs/main_logistic.yaml",
118+ "cell": 38,
119+ "cost": 21794242891.16812,
120+ "name": "logit_p3_N1048576_K4096"
121+ },
122+ {
123+ "config": "experiments/configs/robustness.yaml",
124+ "cell": 13,
125+ "cost": 21794242891.16812,
126+ "name": "logit_misspec_p3_N1048576_K4096"
127+ },
128+ {
129+ "config": "experiments/configs/robustness.yaml",
130+ "cell": 20,
131+ "cost": 11863283203.031446,
132+ "name": "logit_illcond_p8_N262144_K2048"
133+ },
134+ {
135+ "config": "experiments/configs/main_logistic.yaml",
136+ "cell": 29,
137+ "cost": 5448560722.79203,
138+ "name": "logit_p3_N262144_K512"
139+ },
140+ {
141+ "config": "experiments/configs/main_poisson.yaml",
142+ "cell": 31,
143+ "cost": 5448560722.79203,
144+ "name": "pois_p3_N262144_K256"
145+ },
146+ {
147+ "config": "experiments/configs/robustness.yaml",
148+ "cell": 15,
149+ "cost": 2965820800.7578616,
150+ "name": "logit_illcond_p8_N65536_K1024"
151+ },
152+ {
153+ "config": "experiments/configs/main_logistic.yaml",
154+ "cell": 10,
155+ "cost": 1482910400.3789308,
156+ "name": "logit_p8_N16384_K16"
157+ },
158+ {
159+ "config": "experiments/configs/main_logistic.yaml",
160+ "cell": 11,
161+ "cost": 1362140180.6980076,
162+ "name": "logit_p3_N65536_K2048"
163+ },
164+ {
165+ "config": "experiments/configs/main_poisson.yaml",
166+ "cell": 16,
167+ "cost": 1362140180.6980076,
168+ "name": "pois_p3_N65536_K256"
169+ },
170+ {
171+ "config": "experiments/configs/robustness.yaml",
172+ "cell": 3,
173+ "cost": 1362140180.6980076,
174+ "name": "logit_misspec_p3_N65536_K256"
175+ },
176+ {
177+ "config": "experiments/configs/main_poisson.yaml",
178+ "cell": 0,
179+ "cost": 340535045.1745019,
180+ "name": "pois_p3_N16384_K512"
181+ }
182+ ],
183+ "M2U64": [
184+ {
185+ "config": "experiments/configs/main_logistic.yaml",
186+ "cell": 51,
187+ "cost": 187574977245.9856,
188+ "name": "logit_p20_p20_N1048576_K2048"
189+ },
190+ {
191+ "config": "experiments/configs/main_poisson.yaml",
192+ "cell": 13,
193+ "cost": 5931641601.515723,
194+ "name": "pois_p8_N65536_K1024"
195+ },
196+ {
197+ "config": "experiments/configs/main_logistic.yaml",
198+ "cell": 23,
199+ "cost": 5448560722.79203,
200+ "name": "logit_p3_N262144_K4096"
201+ },
202+ {
203+ "config": "experiments/configs/robustness.yaml",
204+ "cell": 6,
205+ "cost": 5448560722.79203,
206+ "name": "logit_misspec_p3_N262144_K4096"
207+ },
208+ {
209+ "config": "experiments/configs/main_logistic.yaml",
210+ "cell": 18,
211+ "cost": 1362140180.6980076,
212+ "name": "logit_p3_N65536_K128"
213+ },
214+ {
215+ "config": "experiments/configs/robustness.yaml",
216+ "cell": 4,
217+ "cost": 1362140180.6980076,
218+ "name": "logit_misspec_p3_N65536_K128"
219+ }
220+ ],
221+ "M4M64a": [
222+ {
223+ "config": "experiments/configs/main_logistic.yaml",
224+ "cell": 52,
225+ "cost": 187574977245.9856,
226+ "name": "logit_p20_p20_N1048576_K1024"
227+ },
228+ {
229+ "config": "experiments/configs/main_poisson.yaml",
230+ "cell": 15,
231+ "cost": 5931641601.515723,
232+ "name": "pois_p8_N65536_K512"
233+ },
234+ {
235+ "config": "experiments/configs/main_logistic.yaml",
236+ "cell": 25,
237+ "cost": 5448560722.79203,
238+ "name": "logit_p3_N262144_K2048"
239+ },
240+ {
241+ "config": "experiments/configs/robustness.yaml",
242+ "cell": 7,
243+ "cost": 5448560722.79203,
244+ "name": "logit_misspec_p3_N262144_K2048"
245+ },
246+ {
247+ "config": "experiments/configs/main_logistic.yaml",
248+ "cell": 20,
249+ "cost": 1362140180.6980076,
250+ "name": "logit_p3_N65536_K64"
251+ },
252+ {
253+ "config": "experiments/configs/robustness.yaml",
254+ "cell": 30,
255+ "cost": 1048576000.0,
256+ "name": "exp_contam_p1_N262144_K16384"
257+ },
258+ {
259+ "config": "experiments/configs/main_poisson.yaml",
260+ "cell": 7,
261+ "cost": 340535045.1745019,
262+ "name": "pois_p3_N16384_K32"
263+ }
264+ ],
265+ "M4M64b": [
266+ {
267+ "config": "experiments/configs/main_logistic.yaml",
268+ "cell": 53,
269+ "cost": 187574977245.9856,
270+ "name": "logit_p20_p20_N1048576_K512"
271+ },
272+ {
273+ "config": "experiments/configs/main_poisson.yaml",
274+ "cell": 17,
275+ "cost": 5931641601.515723,
276+ "name": "pois_p8_N65536_K256"
277+ },
278+ {
279+ "config": "experiments/configs/main_logistic.yaml",
280+ "cell": 27,
281+ "cost": 5448560722.79203,
282+ "name": "logit_p3_N262144_K1024"
283+ },
284+ {
285+ "config": "experiments/configs/robustness.yaml",
286+ "cell": 8,
287+ "cost": 5448560722.79203,
288+ "name": "logit_misspec_p3_N262144_K1024"
289+ },
290+ {
291+ "config": "experiments/configs/main_poisson.yaml",
292+ "cell": 11,
293+ "cost": 1362140180.6980076,
294+ "name": "pois_p3_N65536_K2048"
295+ },
296+ {
297+ "config": "experiments/configs/robustness.yaml",
298+ "cell": 31,
299+ "cost": 1048576000.0,
300+ "name": "exp_contam_p1_N262144_K4096"
301+ },
302+ {
303+ "config": "experiments/configs/main_poisson.yaml",
304+ "cell": 9,
305+ "cost": 340535045.1745019,
306+ "name": "pois_p3_N16384_K16"
307+ }
308+ ],
309+ "M4M64c": [
310+ {
311+ "config": "experiments/configs/main_logistic.yaml",
312+ "cell": 35,
313+ "cost": 94906265624.25157,
314+ "name": "logit_p8_N1048576_K16384"
315+ },
316+ {
317+ "config": "experiments/configs/robustness.yaml",
318+ "cell": 26,
319+ "cost": 47453132812.125786,
320+ "name": "logit_illcond_p8_N1048576_K2048"
321+ },
322+ {
323+ "config": "experiments/configs/main_logistic.yaml",
324+ "cell": 34,
325+ "cost": 21794242891.16812,
326+ "name": "logit_p3_N1048576_K16384"
327+ },
328+ {
329+ "config": "experiments/configs/robustness.yaml",
330+ "cell": 11,
331+ "cost": 21794242891.16812,
332+ "name": "logit_misspec_p3_N1048576_K16384"
333+ },
334+ {
335+ "config": "experiments/configs/robustness.yaml",
336+ "cell": 21,
337+ "cost": 11863283203.031446,
338+ "name": "logit_illcond_p8_N262144_K1024"
339+ },
340+ {
341+ "config": "experiments/configs/robustness.yaml",
342+ "cell": 5,
343+ "cost": 5448560722.79203,
344+ "name": "logit_misspec_p3_N262144_K8192"
345+ },
346+ {
347+ "config": "experiments/configs/main_poisson.yaml",
348+ "cell": 8,
349+ "cost": 1482910400.3789308,
350+ "name": "pois_p8_N16384_K32"
351+ },
352+ {
353+ "config": "experiments/configs/main_poisson.yaml",
354+ "cell": 14,
355+ "cost": 1362140180.6980076,
356+ "name": "pois_p3_N65536_K512"
357+ },
358+ {
359+ "config": "experiments/configs/robustness.yaml",
360+ "cell": 32,
361+ "cost": 1048576000.0,
362+ "name": "exp_contam_p1_N262144_K1024"
363+ }
364+ ],
365+ "M5M36": [
366+ {
367+ "config": "experiments/configs/main_logistic.yaml",
368+ "cell": 37,
369+ "cost": 94906265624.25157,
370+ "name": "logit_p8_N1048576_K8192"
371+ },
372+ {
373+ "config": "experiments/configs/main_logistic.yaml",
374+ "cell": 44,
375+ "cost": 46893744311.4964,
376+ "name": "logit_p20_p20_N262144_K2048"
377+ },
378+ {
379+ "config": "experiments/configs/main_logistic.yaml",
380+ "cell": 33,
381+ "cost": 21794242891.16812,
382+ "name": "logit_p3_N1048576_K32768"
383+ },
384+ {
385+ "config": "experiments/configs/robustness.yaml",
386+ "cell": 10,
387+ "cost": 21794242891.16812,
388+ "name": "logit_misspec_p3_N1048576_K32768"
389+ },
390+ {
391+ "config": "experiments/configs/robustness.yaml",
392+ "cell": 19,
393+ "cost": 11863283203.031446,
394+ "name": "logit_illcond_p8_N262144_K4096"
395+ },
396+ {
397+ "config": "experiments/configs/main_poisson.yaml",
398+ "cell": 29,
399+ "cost": 5448560722.79203,
400+ "name": "pois_p3_N262144_K512"
401+ },
402+ {
403+ "config": "experiments/configs/main_logistic.yaml",
404+ "cell": 8,
405+ "cost": 1482910400.3789308,
406+ "name": "logit_p8_N16384_K32"
407+ },
408+ {
409+ "config": "experiments/configs/main_logistic.yaml",
410+ "cell": 12,
411+ "cost": 1362140180.6980076,
412+ "name": "logit_p3_N65536_K1024"
413+ },
414+ {
415+ "config": "experiments/configs/robustness.yaml",
416+ "cell": 1,
417+ "cost": 1362140180.6980076,
418+ "name": "logit_misspec_p3_N65536_K1024"
419+ }
420+ ],
421+ "M4BP48": [
422+ {
423+ "config": "experiments/configs/main_logistic.yaml",
424+ "cell": 39,
425+ "cost": 94906265624.25157,
426+ "name": "logit_p8_N1048576_K4096"
427+ },
428+ {
429+ "config": "experiments/configs/main_logistic.yaml",
430+ "cell": 26,
431+ "cost": 23726566406.062893,
432+ "name": "logit_p8_N262144_K2048"
433+ },
434+ {
435+ "config": "experiments/configs/main_logistic.yaml",
436+ "cell": 40,
437+ "cost": 21794242891.16812,
438+ "name": "logit_p3_N1048576_K2048"
439+ },
440+ {
441+ "config": "experiments/configs/robustness.yaml",
442+ "cell": 22,
443+ "cost": 11863283203.031446,
444+ "name": "logit_illcond_p8_N262144_K512"
445+ },
446+ {
447+ "config": "experiments/configs/main_poisson.yaml",
448+ "cell": 2,
449+ "cost": 1482910400.3789308,
450+ "name": "pois_p8_N16384_K256"
451+ },
452+ {
453+ "config": "experiments/configs/main_poisson.yaml",
454+ "cell": 18,
455+ "cost": 1362140180.6980076,
456+ "name": "pois_p3_N65536_K128"
457+ }
458+ ],
459+ "M4BP36": [
460+ {
461+ "config": "experiments/configs/main_logistic.yaml",
462+ "cell": 41,
463+ "cost": 94906265624.25157,
464+ "name": "logit_p8_N1048576_K2048"
465+ },
466+ {
467+ "config": "experiments/configs/main_logistic.yaml",
468+ "cell": 28,
469+ "cost": 23726566406.062893,
470+ "name": "logit_p8_N262144_K1024"
471+ },
472+ {
473+ "config": "experiments/configs/main_logistic.yaml",
474+ "cell": 42,
475+ "cost": 21794242891.16812,
476+ "name": "logit_p3_N1048576_K1024"
477+ },
478+ {
479+ "config": "experiments/configs/robustness.yaml",
480+ "cell": 36,
481+ "cost": 11863283203.031446,
482+ "name": "logit_small_m_p8_N262144_K16384"
483+ },
484+ {
485+ "config": "experiments/configs/main_poisson.yaml",
486+ "cell": 4,
487+ "cost": 1482910400.3789308,
488+ "name": "pois_p8_N16384_K128"
489+ },
490+ {
491+ "config": "experiments/configs/main_poisson.yaml",
492+ "cell": 20,
493+ "cost": 1362140180.6980076,
494+ "name": "pois_p3_N65536_K64"
495+ }
496+ ],
497+ "M2M32c": [
498+ {
499+ "config": "experiments/configs/main_logistic.yaml",
500+ "cell": 43,
501+ "cost": 94906265624.25157,
502+ "name": "logit_p8_N1048576_K1024"
503+ },
504+ {
505+ "config": "experiments/configs/main_logistic.yaml",
506+ "cell": 30,
507+ "cost": 23726566406.062893,
508+ "name": "logit_p8_N262144_K512"
509+ },
510+ {
511+ "config": "experiments/configs/main_poisson.yaml",
512+ "cell": 33,
513+ "cost": 21794242891.16812,
514+ "name": "pois_p3_N1048576_K32768"
515+ },
516+ {
517+ "config": "experiments/configs/robustness.yaml",
518+ "cell": 37,
519+ "cost": 11863283203.031446,
520+ "name": "logit_small_m_p8_N262144_K8192"
521+ },
522+ {
523+ "config": "experiments/configs/main_poisson.yaml",
524+ "cell": 6,
525+ "cost": 1482910400.3789308,
526+ "name": "pois_p8_N16384_K64"
527+ },
528+ {
529+ "config": "experiments/configs/robustness.yaml",
530+ "cell": 0,
531+ "cost": 1362140180.6980076,
532+ "name": "logit_misspec_p3_N65536_K2048"
533+ }
534+ ],
535+ "m4mm": [
536+ {
537+ "config": "experiments/configs/main_poisson.yaml",
538+ "cell": 35,
539+ "cost": 94906265624.25157,
540+ "name": "pois_p8_N1048576_K16384"
541+ },
542+ {
543+ "config": "experiments/configs/main_logistic.yaml",
544+ "cell": 32,
545+ "cost": 23726566406.062893,
546+ "name": "logit_p8_N262144_K256"
547+ },
548+ {
549+ "config": "experiments/configs/main_poisson.yaml",
550+ "cell": 34,
551+ "cost": 21794242891.16812,
552+ "name": "pois_p3_N1048576_K16384"
553+ },
554+ {
555+ "config": "experiments/configs/main_logistic.yaml",
556+ "cell": 13,
557+ "cost": 5931641601.515723,
558+ "name": "logit_p8_N65536_K1024"
559+ },
560+ {
561+ "config": "experiments/configs/main_logistic.yaml",
562+ "cell": 31,
563+ "cost": 5448560722.79203,
564+ "name": "logit_p3_N262144_K256"
565+ },
566+ {
567+ "config": "experiments/configs/robustness.yaml",
568+ "cell": 16,
569+ "cost": 2965820800.7578616,
570+ "name": "logit_illcond_p8_N65536_K512"
571+ },
572+ {
573+ "config": "experiments/configs/main_logistic.yaml",
574+ "cell": 3,
575+ "cost": 340535045.1745019,
576+ "name": "logit_p3_N16384_K128"
577+ },
578+ {
579+ "config": "experiments/configs/robustness.yaml",
580+ "cell": 27,
581+ "cost": 262144000.0,
582+ "name": "exp_contam_p1_N65536_K4096"
583+ }
584+ ],
585+ "m4mn": [
586+ {
587+ "config": "experiments/configs/main_poisson.yaml",
588+ "cell": 37,
589+ "cost": 94906265624.25157,
590+ "name": "pois_p8_N1048576_K8192"
591+ },
592+ {
593+ "config": "experiments/configs/main_poisson.yaml",
594+ "cell": 24,
595+ "cost": 23726566406.062893,
596+ "name": "pois_p8_N262144_K4096"
597+ },
598+ {
599+ "config": "experiments/configs/main_poisson.yaml",
600+ "cell": 36,
601+ "cost": 21794242891.16812,
602+ "name": "pois_p3_N1048576_K8192"
603+ },
604+ {
605+ "config": "experiments/configs/main_logistic.yaml",
606+ "cell": 15,
607+ "cost": 5931641601.515723,
608+ "name": "logit_p8_N65536_K512"
609+ },
610+ {
611+ "config": "experiments/configs/main_poisson.yaml",
612+ "cell": 22,
613+ "cost": 5448560722.79203,
614+ "name": "pois_p3_N262144_K8192"
615+ },
616+ {
617+ "config": "experiments/configs/robustness.yaml",
618+ "cell": 17,
619+ "cost": 2965820800.7578616,
620+ "name": "logit_illcond_p8_N65536_K256"
621+ },
622+ {
623+ "config": "experiments/configs/main_logistic.yaml",
624+ "cell": 5,
625+ "cost": 340535045.1745019,
626+ "name": "logit_p3_N16384_K64"
627+ },
628+ {
629+ "config": "experiments/configs/robustness.yaml",
630+ "cell": 28,
631+ "cost": 262144000.0,
632+ "name": "exp_contam_p1_N65536_K1024"
633+ }
634+ ],
635+ "m6ma": [
636+ {
637+ "config": "experiments/configs/main_poisson.yaml",
638+ "cell": 39,
639+ "cost": 94906265624.25157,
640+ "name": "pois_p8_N1048576_K4096"
641+ },
642+ {
643+ "config": "experiments/configs/main_poisson.yaml",
644+ "cell": 26,
645+ "cost": 23726566406.062893,
646+ "name": "pois_p8_N262144_K2048"
647+ },
648+ {
649+ "config": "experiments/configs/main_poisson.yaml",
650+ "cell": 38,
651+ "cost": 21794242891.16812,
652+ "name": "pois_p3_N1048576_K4096"
653+ },
654+ {
655+ "config": "experiments/configs/main_logistic.yaml",
656+ "cell": 17,
657+ "cost": 5931641601.515723,
658+ "name": "logit_p8_N65536_K256"
659+ },
660+ {
661+ "config": "experiments/configs/main_poisson.yaml",
662+ "cell": 23,
663+ "cost": 5448560722.79203,
664+ "name": "pois_p3_N262144_K4096"
665+ },
666+ {
667+ "config": "experiments/configs/robustness.yaml",
668+ "cell": 18,
669+ "cost": 2965820800.7578616,
670+ "name": "logit_illcond_p8_N65536_K128"
671+ },
672+ {
673+ "config": "experiments/configs/main_logistic.yaml",
674+ "cell": 7,
675+ "cost": 340535045.1745019,
676+ "name": "logit_p3_N16384_K32"
677+ },
678+ {
679+ "config": "experiments/configs/robustness.yaml",
680+ "cell": 29,
681+ "cost": 262144000.0,
682+ "name": "exp_contam_p1_N65536_K256"
683+ }
684+ ],
685+ "m6mb": [
686+ {
687+ "config": "experiments/configs/main_poisson.yaml",
688+ "cell": 41,
689+ "cost": 94906265624.25157,
690+ "name": "pois_p8_N1048576_K2048"
691+ },
692+ {
693+ "config": "experiments/configs/main_poisson.yaml",
694+ "cell": 28,
695+ "cost": 23726566406.062893,
696+ "name": "pois_p8_N262144_K1024"
697+ },
698+ {
699+ "config": "experiments/configs/main_poisson.yaml",
700+ "cell": 40,
701+ "cost": 21794242891.16812,
702+ "name": "pois_p3_N1048576_K2048"
703+ },
704+ {
705+ "config": "experiments/configs/main_logistic.yaml",
706+ "cell": 19,
707+ "cost": 5931641601.515723,
708+ "name": "logit_p8_N65536_K128"
709+ },
710+ {
711+ "config": "experiments/configs/main_poisson.yaml",
712+ "cell": 25,
713+ "cost": 5448560722.79203,
714+ "name": "pois_p3_N262144_K2048"
715+ },
716+ {
717+ "config": "experiments/configs/main_logistic.yaml",
718+ "cell": 2,
719+ "cost": 1482910400.3789308,
720+ "name": "logit_p8_N16384_K256"
721+ },
722+ {
723+ "config": "experiments/configs/main_logistic.yaml",
724+ "cell": 14,
725+ "cost": 1362140180.6980076,
726+ "name": "logit_p3_N65536_K512"
727+ },
728+ {
729+ "config": "experiments/configs/main_logistic.yaml",
730+ "cell": 0,
731+ "cost": 340535045.1745019,
732+ "name": "logit_p3_N16384_K512"
733+ },
734+ {
735+ "config": "experiments/configs/main_poisson.yaml",
736+ "cell": 1,
737+ "cost": 340535045.1745019,
738+ "name": "pois_p3_N16384_K256"
739+ }
740+ ],
741+ "m5ma": [
742+ {
743+ "config": "experiments/configs/main_poisson.yaml",
744+ "cell": 43,
745+ "cost": 94906265624.25157,
746+ "name": "pois_p8_N1048576_K1024"
747+ },
748+ {
749+ "config": "experiments/configs/main_poisson.yaml",
750+ "cell": 30,
751+ "cost": 23726566406.062893,
752+ "name": "pois_p8_N262144_K512"
753+ },
754+ {
755+ "config": "experiments/configs/main_poisson.yaml",
756+ "cell": 42,
757+ "cost": 21794242891.16812,
758+ "name": "pois_p3_N1048576_K1024"
759+ },
760+ {
761+ "config": "experiments/configs/main_logistic.yaml",
762+ "cell": 21,
763+ "cost": 5931641601.515723,
764+ "name": "logit_p8_N65536_K64"
765+ },
766+ {
767+ "config": "experiments/configs/main_poisson.yaml",
768+ "cell": 27,
769+ "cost": 5448560722.79203,
770+ "name": "pois_p3_N262144_K1024"
771+ },
772+ {
773+ "config": "experiments/configs/main_logistic.yaml",
774+ "cell": 4,
775+ "cost": 1482910400.3789308,
776+ "name": "logit_p8_N16384_K128"
777+ },
778+ {
779+ "config": "experiments/configs/main_logistic.yaml",
780+ "cell": 16,
781+ "cost": 1362140180.6980076,
782+ "name": "logit_p3_N65536_K256"
783+ },
784+ {
785+ "config": "experiments/configs/main_logistic.yaml",
786+ "cell": 1,
787+ "cost": 340535045.1745019,
788+ "name": "logit_p3_N16384_K256"
789+ },
790+ {
791+ "config": "experiments/configs/main_poisson.yaml",
792+ "cell": 3,
793+ "cost": 340535045.1745019,
794+ "name": "pois_p3_N16384_K128"
795+ }
796+ ],
797+ "m4md": [
798+ {
799+ "config": "experiments/configs/robustness.yaml",
800+ "cell": 23,
801+ "cost": 47453132812.125786,
802+ "name": "logit_illcond_p8_N1048576_K16384"
803+ },
804+ {
805+ "config": "experiments/configs/main_logistic.yaml",
806+ "cell": 45,
807+ "cost": 46893744311.4964,
808+ "name": "logit_p20_p20_N262144_K1024"
809+ },
810+ {
811+ "config": "experiments/configs/main_poisson.yaml",
812+ "cell": 19,
813+ "cost": 5931641601.515723,
814+ "name": "pois_p8_N65536_K128"
815+ },
816+ {
817+ "config": "experiments/configs/robustness.yaml",
818+ "cell": 34,
819+ "cost": 4194304000.0,
820+ "name": "exp_contam_p1_N1048576_K16384"
821+ }
822+ ],
823+ "m4ml": [
824+ {
825+ "config": "experiments/configs/robustness.yaml",
826+ "cell": 24,
827+ "cost": 47453132812.125786,
828+ "name": "logit_illcond_p8_N1048576_K8192"
829+ },
830+ {
831+ "config": "experiments/configs/main_logistic.yaml",
832+ "cell": 46,
833+ "cost": 46893744311.4964,
834+ "name": "logit_p20_p20_N262144_K512"
835+ },
836+ {
837+ "config": "experiments/configs/main_poisson.yaml",
838+ "cell": 21,
839+ "cost": 5931641601.515723,
840+ "name": "pois_p8_N65536_K64"
841+ },
842+ {
843+ "config": "experiments/configs/robustness.yaml",
844+ "cell": 35,
845+ "cost": 4194304000.0,
846+ "name": "exp_contam_p1_N1048576_K4096"
847+ }
848+ ],
849+ "M3BA24": [
850+ {
851+ "config": "experiments/configs/robustness.yaml",
852+ "cell": 25,
853+ "cost": 47453132812.125786,
854+ "name": "logit_illcond_p8_N1048576_K4096"
855+ },
856+ {
857+ "config": "experiments/configs/main_logistic.yaml",
858+ "cell": 47,
859+ "cost": 46893744311.4964,
860+ "name": "logit_p20_p20_N262144_K256"
861+ },
862+ {
863+ "config": "experiments/configs/main_logistic.yaml",
864+ "cell": 22,
865+ "cost": 5448560722.79203,
866+ "name": "logit_p3_N262144_K8192"
867+ },
868+ {
869+ "config": "experiments/configs/robustness.yaml",
870+ "cell": 9,
871+ "cost": 5448560722.79203,
872+ "name": "logit_misspec_p3_N262144_K512"
873+ }
874+ ]
875+ }
876+}
\ No newline at end of file
added experiments/collect_cluster.sh +8 −0
@@ -0,0 +1,8 @@
1+#!/bin/bash
2+# Pull finished raw results and logs from all nodes used by scatter_cluster.py
3+cd "$(dirname "$0")/.."
4+for n in $(python3 -c "import json;print(' '.join(json.load(open('experiments/cluster_assignment.json'))['nodes']))"); do
5+ rsync -az --ignore-existing "$n:~/wp28/repo/results/raw/" results/raw/ 2>/dev/null &
6+ rsync -az "$n:~/wp28/repo/logs/" logs/cluster/$n/ 2>/dev/null &
7+done; wait
8+echo "raw csv files: $(ls results/raw/*.csv 2>/dev/null | wc -l)"
added experiments/configs/main_exponential.yaml +11 −0
@@ -0,0 +1,11 @@
1+# Main rate study, exponential(rate) model: exact MLE, all (r,q) cells, large N affordable.
2+seed_base: 20261028
3+grids:
4+ - model: exponential
5+ label: exp
6+ N: [16384, 65536, 262144, 1048576, 4194304]
7+ m: [8, 16, 32, 64, 128, 256, 512, 1024]
8+ reps: 2000
9+ splits_max: 3
10+ q_max: 2
11+ dgp: {theta0: 1.0}
added experiments/configs/main_logistic.yaml +20 −0
@@ -0,0 +1,20 @@
1+seed_base: 20261028
2+grids:
3+ - model: logistic
4+ label: logit
5+ p: [3, 8]
6+ N: [16384, 65536, 262144, 1048576]
7+ m: [32, 64, 128, 256, 512, 1024]
8+ reps: 1000
9+ splits_max: 3
10+ q_max: 2
11+ min_m_over_p: 8
12+ - model: logistic
13+ label: logit_p20
14+ p: [20]
15+ N: [262144, 1048576]
16+ m: [128, 256, 512, 1024, 2048]
17+ reps: 500
18+ splits_max: 3
19+ q_max: 2
20+ min_m_over_p: 6
added experiments/configs/main_poisson.yaml +11 −0
@@ -0,0 +1,11 @@
1+seed_base: 20261028
2+grids:
3+ - model: poisson
4+ label: pois
5+ p: [3, 8]
6+ N: [16384, 65536, 262144, 1048576]
7+ m: [32, 64, 128, 256, 512, 1024]
8+ reps: 1000
9+ splits_max: 3
10+ q_max: 2
11+ min_m_over_p: 8
added experiments/configs/robustness.yaml +29 −0
@@ -0,0 +1,29 @@
1+seed_base: 20261028
2+grids:
3+ - model: logistic
4+ label: logit_misspec
5+ p: [3]
6+ N: [65536, 262144, 1048576]
7+ m: [32, 64, 128, 256, 512]
8+ reps: 1000
9+ dgp: {kind: misspecified, omitted: 0.5}
10+ - model: logistic
11+ label: logit_illcond
12+ p: [8]
13+ N: [65536, 262144, 1048576]
14+ m: [64, 128, 256, 512]
15+ reps: 500
16+ dgp: {cond: 100.0}
17+ - model: exponential
18+ label: exp_contam
19+ N: [65536, 262144, 1048576]
20+ m: [16, 64, 256]
21+ reps: 1000
22+ dgp: {kind: contaminated, eps: 0.01, scale: 10.0}
23+ - model: logistic
24+ label: logit_small_m
25+ p: [8]
26+ N: [262144]
27+ m: [16, 24, 32, 48]
28+ reps: 500
29+ min_m_over_p: 2
added experiments/run_monte_carlo.py +168 −0
@@ -0,0 +1,168 @@
1+#!/usr/bin/env python3
2+"""Monte Carlo driver for WP28. One *cell* = (model, p, N, K, DGP options, reps, seed). Every estimator is computed on the
3+same simulated sample and the same partition (paired design). Output: one CSV per cell in results/raw/ with one row per
4+replication, columns <estimator>_<j> for parameter j, plus diagnostics and timings, and a JSON manifest.
5+
6+Usage
7+ python3 experiments/run_monte_carlo.py --config experiments/configs/main_exponential.yaml [--cells 0,3,5] [--reps 200]
8+ python3 experiments/run_monte_carlo.py --list --config ...
9+"""
10+from __future__ import annotations
11+import argparse, hashlib, json, os, pathlib, platform, socket, sys, time
12+import numpy as np, yaml
13+
14+ROOT = pathlib.Path(__file__).resolve().parents[1]
15+sys.path.insert(0, str(ROOT / "src"))
16+from pamle.likelihoods import make_model, gather_blocks, STATUS_OK # noqa: E402
17+from pamle.algorithms import estimate_all, aggregated_score_hessian, local_split_fits # noqa: E402
18+from pamle.partitioning import skewed_sizes # noqa: E402
19+from pamle import __version__ # noqa: E402
20+
21+SE_ESTIMATORS = ("central", "r1_q0", "r1_q1", "r1_q2", "r2_q0", "r2_q1", "r3_q1", "cs_q1")
22+
23+
24+def cell_seed(base: int, cell: dict) -> int:
25+ key = json.dumps({k: cell[k] for k in sorted(cell) if k not in ("reps",)}, sort_keys=True, default=str)
26+ return (base + int(hashlib.sha256(key.encode()).hexdigest(), 16)) % (2 ** 63)
27+
28+
29+def expand_cells(cfg: dict):
30+ cells = []
31+ for g in cfg["grids"]:
32+ for N in g["N"]:
33+ for m in g["m"]:
34+ if N % m:
35+ continue
36+ for p in g.get("p", [1]):
37+ if m < g.get("min_m_over_p", 4) * p:
38+ continue
39+ c = {"model": g["model"], "p": p, "N": int(N), "K": int(N // m), "m": int(m), "reps": int(g["reps"]),
40+ "splits_max": int(g.get("splits_max", 3)), "q_max": int(g.get("q_max", 2)),
41+ "dgp": g.get("dgp", {}), "label": g.get("label", g["model"]), "extras": bool(g.get("extras", True))}
42+ cells.append(c)
43+ return cells
44+
45+
46+def true_theta(cell):
47+ p = cell["p"]; dgp = cell["dgp"]
48+ if cell["model"] == "exponential":
49+ return np.array([float(dgp.get("theta0", 1.0))])
50+ if "theta0" in dgp:
51+ return np.asarray(dgp["theta0"], float)
52+ # default: intercept 0.3, slopes alternating +-0.5 decaying
53+ th = np.array([0.3] + [0.5 * (-1) ** j / (1 + 0.15 * j) for j in range(p - 1)])
54+ if cell["model"] == "poisson":
55+ th = th * 0.5
56+ return th
57+
58+
59+def simulate(model, cell, theta0, rng):
60+ dgp = cell["dgp"]; N = cell["N"]; p = cell["p"]
61+ kind = dgp.get("kind", "well_specified")
62+ if cell["model"] == "exponential":
63+ x = rng.exponential(1.0 / theta0[0], size=N)
64+ if kind == "contaminated":
65+ eps = float(dgp.get("eps", 0.01)); scale = float(dgp.get("scale", 10.0))
66+ mask = rng.random(N) < eps
67+ x[mask] = rng.exponential(scale / theta0[0], size=mask.sum())
68+ return {"x": x}
69+ cov = None
70+ if dgp.get("cond", None) is not None and p > 1: # covariate covariance with prescribed condition number
71+ q = p - 1; ev = np.geomspace(1.0, 1.0 / float(dgp["cond"]), q)
72+ cov = np.diag(ev)
73+ data = model.simulate(rng, N, theta0, cov=cov)
74+ if kind == "misspecified": # omitted quadratic term in the index
75+ X = data["X"]; eta = X @ theta0 + float(dgp.get("omitted", 0.5)) * (X[:, 1] ** 2 - 1.0)
76+ if cell["model"] == "logistic":
77+ data["y"] = (rng.random(N) < 1 / (1 + np.exp(-eta))).astype(float)
78+ else:
79+ data["y"] = rng.poisson(np.exp(eta)).astype(float)
80+ return data
81+
82+
83+def run_cell(cell, out_dir: pathlib.Path, base_seed: int, reps_override=None, verbose=True):
84+ model = make_model(cell["model"], cell["p"]); p = model.p
85+ reps = reps_override or cell["reps"]
86+ seed = cell_seed(base_seed, cell); rng = np.random.default_rng(seed)
87+ theta0 = true_theta(cell)
88+ N, K, m = cell["N"], cell["K"], cell["m"]
89+ skew = float(cell["dgp"].get("skew", 0.0))
90+ name = f'{cell["label"]}_p{p}_N{N}_K{K}'
91+ rows = []; t_start = time.time()
92+ for b in range(reps):
93+ data = simulate(model, cell, theta0, rng)
94+ t0 = time.perf_counter()
95+ if skew > 1.0:
96+ # unequal contiguous blocks; estimate_all needs equal shapes -> use K blocks of sizes from skewed_sizes,
97+ # handled by padding is not possible: we instead run a dedicated weighted/naive comparison (see run_skewed)
98+ raise NotImplementedError("use run_skewed.py for unequal partitions")
99+ db = gather_blocks(data, np.arange(N).reshape(K, m))
100+ out, diag = estimate_all(model, db, splits_max=cell["splits_max"], q_max=cell["q_max"], extras=cell["extras"])
101+ t_est = time.perf_counter() - t0
102+ row = {"rep": b, "seed": seed, "N": N, "K": K, "m": m, "p": p, "model": cell["model"], "label": cell["label"],
103+ "t_estimate_s": t_est, "central_status": diag["central_status"]}
104+ for s, cnt in diag["local_status_counts"].items():
105+ row[f"local_fail_s{s}"] = int(sum(cnt[1:]))
106+ for r in range(1, cell["splits_max"] + 1):
107+ row[f"valid_r{r}"] = int(diag[f"valid_r{r}"].sum())
108+ nd = diag[f"newton_r{r}"]
109+ for q, d in enumerate(nd, start=1):
110+ row[f"cond_r{r}_q{q}"] = d["cond"]; row[f"step_r{r}_q{q}"] = d["step_norm"]; row[f"grad_r{r}_q{q}"] = d["grad_norm"]
111+ for k, v in out.items():
112+ for j in range(p):
113+ row[f"{k}_{j}"] = float(v[j])
114+ # standard errors (Fisher at the estimate; sandwich at the centralized estimate)
115+ for k in SE_ESTIMATORS:
116+ if k in out and np.all(np.isfinite(out[k])):
117+ g, H, gk, Hk = aggregated_score_hessian(model, db, out[k])
118+ try:
119+ cov = np.linalg.inv(-H)
120+ for j in range(p):
121+ row[f"se_{k}_{j}"] = float(np.sqrt(max(cov[j, j], 0) / N))
122+ except np.linalg.LinAlgError:
123+ pass
124+ th = np.broadcast_to(out["central"], (K, p))
125+ J = model.outer_scores(th, db).mean(axis=0); g, H, _, _ = aggregated_score_hessian(model, db, out["central"])
126+ Hi = np.linalg.inv(H); S = Hi @ J @ Hi
127+ for j in range(p):
128+ row[f"se_sand_central_{j}"] = float(np.sqrt(max(S[j, j], 0) / N))
129+ for j in range(p):
130+ row[f"theta0_{j}"] = float(theta0[j])
131+ rows.append(row)
132+ if verbose and (b + 1) % max(1, reps // 10) == 0:
133+ print(f"[{name}] {b+1}/{reps} reps, {time.time()-t_start:.1f}s", flush=True)
134+ import pandas as pd
135+ df = pd.DataFrame(rows)
136+ out_dir.mkdir(parents=True, exist_ok=True)
137+ f = out_dir / f"{name}.csv"; df.to_csv(f, index=False)
138+ sha = hashlib.sha256(f.read_bytes()).hexdigest()
139+ manifest = {"run_id": name, "cell": cell, "reps": reps, "seed": seed, "pamle_version": __version__,
140+ "python": platform.python_version(), "numpy": np.__version__, "host": socket.gethostname(),
141+ "machine": platform.machine(), "wall_seconds": time.time() - t_start, "raw_output_sha256": sha,
142+ "git_commit": os.popen(f"git -C {ROOT} rev-parse --short HEAD").read().strip(), "status": "completed"}
143+ (out_dir / f"{name}.manifest.json").write_text(json.dumps(manifest, indent=1))
144+ return f, manifest
145+
146+
147+def main():
148+ ap = argparse.ArgumentParser()
149+ ap.add_argument("--config", required=True)
150+ ap.add_argument("--cells", default=None, help="comma-separated cell indices (default all)")
151+ ap.add_argument("--reps", type=int, default=None)
152+ ap.add_argument("--out", default=str(ROOT / "results" / "raw"))
153+ ap.add_argument("--list", action="store_true")
154+ a = ap.parse_args()
155+ cfg = yaml.safe_load(open(a.config))
156+ cells = expand_cells(cfg)
157+ if a.list:
158+ for i, c in enumerate(cells):
159+ print(i, c["label"], "p", c["p"], "N", c["N"], "K", c["K"], "m", c["m"], "reps", c["reps"])
160+ return
161+ idx = range(len(cells)) if a.cells is None else [int(x) for x in a.cells.split(",")]
162+ for i in idx:
163+ f, man = run_cell(cells[i], pathlib.Path(a.out), int(cfg.get("seed_base", 20261028)), a.reps)
164+ print("wrote", f, f"{man['wall_seconds']:.1f}s")
165+
166+
167+if __name__ == "__main__":
168+ main()
added experiments/scatter_cluster.py +58 −0
@@ -0,0 +1,58 @@
1+#!/usr/bin/env python3
2+"""Distribute Monte Carlo cells over MacLustr nodes (greedy by estimated cost, weighted by worker slots).
3+
4+Writes experiments/cluster_assignment.json, syncs the repo to each node, and launches `xargs -P` worker pools.
5+Usage: python3 experiments/scatter_cluster.py --configs a.yaml b.yaml --launch
6+Collect later with experiments/collect_cluster.sh.
7+"""
8+import argparse, json, os, pathlib, subprocess, sys
9+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "experiments"))
10+import yaml
11+from run_monte_carlo import expand_cells
12+
13+ROOT = pathlib.Path(__file__).resolve().parents[1]
14+NODES = {"M3U96a": 6, "M3U96b": 6, "M2U64": 4, "M4M64a": 4, "M4M64b": 4, "M4M64c": 4, "M5M36": 4, "M4BP48": 3, "M4BP36": 3,
15+ "M2M32c": 3, "m4mm": 3, "m4mn": 3, "m6ma": 3, "m6mb": 3, "m5ma": 3, "m4md": 2, "m4ml": 2, "M3BA24": 2}
16+
17+
18+def cost(c):
19+ extras = 1.6 if c["extras"] else 1.0
20+ return c["N"] * (c["p"] ** 1.5) * c["reps"] * (1 + 0.5 * c["splits_max"]) * extras
21+
22+
23+def main():
24+ ap = argparse.ArgumentParser(); ap.add_argument("--configs", nargs="+", required=True); ap.add_argument("--launch", action="store_true")
25+ a = ap.parse_args()
26+ jobs = []
27+ for cfg in a.configs:
28+ cells = expand_cells(yaml.safe_load(open(cfg)))
29+ for i, c in enumerate(cells):
30+ jobs.append({"config": os.path.relpath(cfg, ROOT), "cell": i, "cost": cost(c), "name": f'{c["label"]}_p{c["p"]}_N{c["N"]}_K{c["K"]}'})
31+ jobs.sort(key=lambda j: -j["cost"])
32+ load = {n: 0.0 for n in NODES}; assign = {n: [] for n in NODES}
33+ for j in jobs:
34+ n = min(NODES, key=lambda k: load[k] / NODES[k])
35+ assign[n].append(j); load[n] += j["cost"]
36+ total = sum(j["cost"] for j in jobs)
37+ print(f"{len(jobs)} jobs; est. share per node (%):", {n: round(100 * load[n] / total, 1) for n in NODES})
38+ json.dump({"nodes": NODES, "assignment": assign}, open(ROOT / "experiments" / "cluster_assignment.json", "w"), indent=1)
39+ if not a.launch:
40+ return
41+ for n, js in assign.items():
42+ if not js:
43+ continue
44+ script = "\n".join(f'{j["config"]} {j["cell"]}' for j in js) + "\n"
45+ (ROOT / "logs" / f"jobs_{n}.txt").write_text(script)
46+ subprocess.run(["rsync", "-az", "--exclude", "results", "--exclude", "logs", "--exclude", "references", "--exclude", "paper",
47+ "--exclude", ".git", f"{ROOT}/", f"{n}:~/wp28/repo/"], check=True)
48+ subprocess.run(["scp", "-q", str(ROOT / "logs" / f"jobs_{n}.txt"), f"{n}:~/wp28/jobs.txt"], check=True)
49+ W = NODES[n]
50+ cmd = (f"cd ~/wp28/repo && mkdir -p logs results/raw && nohup sh -c 'cat ~/wp28/jobs.txt | xargs -P {W} -L 1 sh -c "
51+ f"\"OMP_NUM_THREADS=2 ~/wp28/venv/bin/python experiments/run_monte_carlo.py --config \\$0 --cells \\$1 "
52+ f">> logs/cell_\\$(basename \\$0 .yaml)_\\$1.log 2>&1\"' > logs/pool.log 2>&1 &")
53+ subprocess.run(["ssh", n, cmd], check=True)
54+ print(f"launched {len(js)} jobs on {n} with {W} workers")
55+
56+
57+if __name__ == "__main__":
58+ main()
added literature/PRIORITY_AND_GAP.md +56 −0
@@ -0,0 +1,56 @@
1+# PRIORITY_AND_GAP — WP28 (literature cutoff 2026-10-28)
2+
3+Sources: four theorem-level verification tasks (audits A–D in this directory; 56 rows in `PRIOR_ART_MATRIX.csv`, 24 of them read in
4+full text, the rest at abstract/metadata level and marked so). Page/theorem numbers refer to the versions read (arXiv or accepted
5+manuscripts when publisher pages were bot-blocked).
6+
7+## 1. Verified anchors (theorem level)
8+| Paper | What it proves (our notation: K blocks of size m = N/K) | Rounds / uplink | Read |
9+|---|---|---|---|
10+| Zhang–Duchi–Wainwright 2013 (JMLR) | AVGM MSE O(1/N + 1/m²) ⇒ K ≲ √N (eq. 10); SAVGM = (θ̄₁ − rθ̄₂)/(1−r) with one random r-subsample, MSE O(1/N + 1/(r(1−r)²m³)), leading constant inflated by (2+3r)/(1−r)², K ≲ N^{2/3} (Thm 4, eq. 11). No jackknife, no Newton, multiple subsamples "open". | 1 / p (2p) | full |
11+| Huang–Huo 2019 (Math. Prog.) | Plain average + ONE aggregated-Hessian Newton step: √N-normal and equivalent to the full M-estimator when k = O(√N) (Thm 3.4); MSE ≤ 2TrΣ/N + O(N^{-2}) + O(k⁴N^{-4}) (Thm 3.5) — the k⁴N^{-4} term implicitly allows k = o(N^{3/4}) (not stated). No bias correction. | 2 / p + p² | arXiv v2 |
12+| Jordan–Lee–Yang 2019 (JASA) | CSL: machine-1 Hessian, gradients only; ‖θ̃ − θ̂‖ ≤ C(‖θ̄−θ̂‖² + (N^{-1/2} + m^{-1/2})‖θ̄−θ̂‖) (Thm 3.1/3.3); contraction C/√m per round; Cor 3.4 sandwich expansion. One round from the average ⇒ K = o(N^{2/3}) (translation). | t / p | arXiv v3 |
13+| Fan–Guo–Wang 2023 (JASA, CEASE) | Rounds frontier (p. 16, verbatim): one single-machine-Hessian step from the average matches θ̂ iff n³ ≫ N·polylog ⇒ K = o(N^{2/3}); t steps ⇒ K = o(N^{(t+1)/(t+2)}); averaged local-Hessian step in the linear model ⇒ K = o(N^{3/4}). | t / p | arXiv v2 + PMC |
14+| Zhu–Li–Wang 2021 (JCGS) | TWLSE: average → one LOCAL Newton step per machine → local-Hessian-weighted average: efficient iff m/N^{1/4} → ∞, "allows K = O(N^{3/4})" (Thm 2); iterated version with m/log N → ∞ rounds (Prop 2); unequal n_k. | 2 / p + p² | arXiv v4 |
15+| Gu–Chen 2023 (JMLR) | Analytic (Rilstone-type) local bias correction with sample splitting, then Hessian-weighted average: χ² limit if K = o(m²) ⇔ K = o(N^{2/3}) (Thms 4–6); beats the SAVGM constant; quasi-likelihood/sandwich; heterogeneity. | 1 / p + p² | arXiv v1 |
16+| Wu–Zhu–Wang 2023 (Statistica Sinica) | Delete-one jackknife within each subsample, then average: bias O(m^{-2}) + O(N^{-1}), normal under N = o(m⁴) (⇔ K = o(N^{3/4})) for smooth functions of means (Thm 2, 4). | 1 / p | arXiv v1 |
17+| Yin 2022 (arXiv 2207.06253) | Bias-adjusted one-step estimator from the plain average (analytic removal of quadratic terms): efficient iff K = o(m²) ⇔ K = o(N^{2/3}) (Thm 3.2); Fisher information from gradients with error O_p(m^{-1} + K^{-1}). | 2 / p + summaries | arXiv v1 |
18+| Dhaene–Jochmans 2015 (RES) | Half-panel jackknife 2θ̂ − θ̄_{1/2}; general h-th order split-panel jackknife with Vandermonde-type weights (eqs. 5.1–5.3); bias-optimal partitions G = {2, 3, …, h+1}; first-order variance unchanged; higher-order variance left open. Panel, not distributed. | — | manuscript |
19+| Hahn–Hughes–Kuersteiner–Newey 2024 (QE) | Split-sample bias estimates are inconsistent (O_p(1)); split-sample corrections have twice the higher-order variance of delete-one (Props 5–7). | — | full |
20+| Volgushev–Chao–Cheng 2019 (AoS) | Plain averaging of quantile regressions: sufficient S = o(√N/log N), NECESSARY S = o(√N) via an explicit worst-case model (Thm 3.2, Cor 3.3). Template for sharpness statements. | 1 / p | arXiv + NSF |
21+| Lin–Xi 2011; Rosenblatt–Nadler 2016; Battey et al. 2018; Shamir et al. 2014 (DANE); Wang et al. 2017 (EDSL); Chen–Liu–Zhang 2022 (FONE); Shang–Cheng 2017; Gao et al. 2022 (review) | One-shot weighting (K ≪ N^{1/4}); bias of the average = m × centralized bias; √N barrier for plain averaging; local-Hessian contraction (β/λ)² ≍ 1/m per round; gradient-only multi-round; sharp machine counts in nonparametrics; review confirming "one-step C₁/N + O(m^{-4} + N^{-2})" and no (order, rounds) frontier as of 2022. | | full/abstract |
22+
23+## 2. Claim-by-claim map
24+| # | Claim in the prior draft / mandate | Status after audit |
25+|---|---|---|
26+| (i) | Half-block (and quarter-block) jackknife of local MLEs cancels the O(1/m) (and O(1/m²)) bias while preserving the first-order influence term | **Known in components**: the extrapolation algebra is Quenouille (1956) / Schucany–Gray–Owen (1971) / Dhaene–Jochmans (2015, panel); "debias locally then average" exists with analytic corrections (Gu–Chen 2023; Yin 2022), delete-one jackknife (Wu–Zhu–Wang 2023) and one-level subsampling (ZDW 2013 SAVGM). **Possibly new**: the general r-th order *nested split* construction of local MLEs with the influence-preservation argument and explicit remainder constants A_r (Lemma 3) in the distributed setting; r ≥ 3 not found anywhere. |
27+| (ii) | Half-block jackknife + one aggregated-Hessian Newton step ⇒ first-order equivalence for K = o(N^{7/8}) | **Not found**; closest: Huang–Huo (plain average + one step, stated √N, implicit 3/4), Zhu–Li–Wang (3/4 with local steps), Yin (2/3 with analytic adjustment), CEASE (2/3 single-machine). The exponent 7/8 appears nowhere. **Demonstrably new with specified proof** (Theorem 1, cell (2,1)). |
28+| (iii) | General (r, q) frontier K = o(N^{1 − 1/(2^{q+1} r)}) | **Not found**; rounds-only frontiers exist (CEASE (t+1)/(t+2) with a single-machine Hessian; JLY ⌈log K/log m⌉ rounds). **Demonstrably new with specified proof** (Theorem 1). The CEASE rounds frontier is recovered exactly as the single-machine-Hessian case of our Theorem 3 (α = 1 − 1/(2r + q), r = 1). |
29+| (iv) | Exact exponential boundary limit −4θ0c⁴ at K ~ cN^{7/8} | **New** (sharpness for a corrected construction). Generalised to all (r, q): −θ0(β_r c^r)^{2^q} (Theorem 2). Necessity results in the literature concern plain averaging only (Volgushev et al.; Shang–Cheng). |
30+| (v) | Inexact / stale / single-machine / quantized Hessians change the admissible exponent | **Partially known** (JLY factor m^{-1/2}; DANE/CEASE factor 1/m; Zhu–Li–Wang 3/4) but **never expressed as an exponent frontier nor with byte/bit accounting**; the quantized-Hessian statistical statement was not found. Theorem 3 unifies and extends. |
31+| (vi) | Unequal block sizes; misspecification | **Known** as extensions (Zhu–Li–Wang, Gu–Chen, Lin–Xi, JLY Cor 3.4). Present as Proposition 4 / Remark, not as a contribution. |
32+| (vii) | Communication–computation decision rule (one more local level vs one more round) | **No explicit rule found**; only qualitative remarks. Proposition 5 is a *conditional cost-model* result calibrated on measured hardware, labelled as such. |
33+| (viii) | Communication accounting in rounds, messages, numbers and bytes | Only O(p)/O(p²) remarks in the literature; our accounting is bookkeeping, not a theorem. |
34+
35+## 3. Candidate contributions
36+- **A (strongest, GO):** the joint local-order / global-round frontier (Theorem 1) with its sharpness in an exact model (Theorem 2) and the inexact-derivative extension that recovers the CSL/CEASE and averaged-local-Hessian cases and adds quantization/bit requirements (Theorem 3). Novelty: no joint frontier, no sharpness for corrected constructions, no bit-precision statement in the audited literature.
37+- **B (feasible backup, GO as supporting material):** the general nested-split extrapolation family with explicit remainder/amplification constants A_r, exact finite-sample bias and variance identities in the exponential model, the Newton basin of attraction, and the finite-sample comparison with analytic (Gu–Chen-type) and delete-one corrections. Novelty: incremental; positioned as the local building block.
38+- **C (negative results, GO):** (a) Proposition: higher-order split corrections lose in RMSE to lower orders for moderate m and have positive probability of leaving the Newton basin (exponential model, exact); (b) in exponential families with aggregable sufficient statistics (and in the exponential model for the full CSL surrogate) the approximate protocols are dominated by exact pooling — no computational lower bound is claimed; (c) unweighted averaging with unequal sizes is not first-order equivalent without a Newton step.
39+
40+## 4. Decisions
41+| Proposed claim | Decision |
42+|---|---|
43+| "Jackknife + Newton is a novel estimator" | **DROP** (components known: HH19, ZDW13, DJ15, GC23, WZW23). Rephrase as "a known local correction and a known global correction, analysed jointly for the first time". |
44+| K = o(N^{7/8}) for (2,1) and the general α(r, q) | **GO** (Theorem 1) with explicit comparison table against HH19 (√N stated / 3/4 implicit), ZLW21 (3/4), GC23 (2/3), Yin22 (2/3), CEASE (rounds frontier). |
45+| Sharpness (−4θ0c⁴ and general) | **GO** (Theorem 2), scoped as "for this construction, in this model"; cite Volgushev et al. as the template and state that it is not a universal lower bound. |
46+| Inexact Hessian frontier + quantization bits | **GO** (Theorem 3); cite JLY Thm 3.1/3.3, DANE Thm 2, CEASE Thm 4.1, ZLW Thm 2, Alimisis et al. (optimization only). |
47+| Heterogeneous sizes / misspecification | **REVISE** → extension Proposition 4 + remark, with citations; no novelty claim. |
48+| Cost-model decision rule | **REVISE** → conditional Proposition 5 ("under the measured cost model"), calibrated on the cluster benchmark; empirical, not general. |
49+| Adaptive (r, q) selection with proof | **DROP as theorem** → heuristic + numerical evaluation only, labelled open. |
50+| Wall-clock / energy / dollar claims | **DROP** unless measured in Phase 8; report measured quantities only. |
51+| Minimax / communication lower bounds | **DROP** (explicitly disclaimed; exact sufficient-statistic pooling dominates in exponential families). |
52+
53+## 5. Items not verifiable in this audit (to check against library copies before journal submission)
54+Quenouille 1949/1956 texts; Tukey 1958 abstract; Schucany–Gray–Owen 1971 determinant system; Sharot 1976; Rilstone et al. 1996 and
55+Bao–Ullah 2007 texts; Le Cam 1956; Bickel 1975 technical statements; van der Vaart §5.7 content; Cordeiro–McCullagh original notation;
56+published JASA theorem numbering for JLY, CEASE, FONE; Springer text of Huang–Huo; Yin's 2022 Temple thesis.
modified literature/PRIOR_ART_MATRIX.csv +44 −0
@@ -11,3 +11,47 @@ wang2017edsl,"Wang, Jialei; Kolar, Mladen; Srebro, Nathan; Zhang, Tong",2017,Eff
11 battey2018distributed,"Battey, Heather; Fan, Jianqing; Liu, Han; Lu, Junwei; Zhu, Ziwei",2018,Distributed testing and estimation under sparse high dimensional models,"Annals of Statistics 46(3):1352-1382",10.1214/17-AOS1587,"full text arXiv 1509.05457v1 + PMC6051757 (low-dim theorems moved to supplement); journal page mapping NOT VERIFIED","lasso/SCAD/MCP error rates; bounded eigenvalues, sub-Gaussian rows; GLM analogues; finite fourth moments","1 round (one-shot)","d numbers per subsample","local lasso + debiasing (nodewise lassos); low-dim local OLS/GLM","Wald normality k=o((s log d)^-1 sqrt n) (Thm 3.3); lasso rate retained iff k=O(sqrt(n/(s^2 log d))) (Thm 4.3); low-dim GLM k=o(sqrt n/(d v log n)) (arXiv Thm 4.13)","high-probability decomposition + CLT","sparse high-dim linear & GLM; low-dim linear & GLM","no bias correction beyond lasso debiasing; establishes the sqrt(N) baseline (r=1,q=0); Remark 4.5 = same leading term, remainders differ",low,"n = total sample size in their notation","arXiv pp. 1-2, 8, 13-17; PMC lines 92-99, 368-442, 531-537" 11 battey2018distributed,"Battey, Heather; Fan, Jianqing; Liu, Han; Lu, Junwei; Zhu, Ziwei",2018,Distributed testing and estimation under sparse high dimensional models,"Annals of Statistics 46(3):1352-1382",10.1214/17-AOS1587,"full text arXiv 1509.05457v1 + PMC6051757 (low-dim theorems moved to supplement); journal page mapping NOT VERIFIED","lasso/SCAD/MCP error rates; bounded eigenvalues, sub-Gaussian rows; GLM analogues; finite fourth moments","1 round (one-shot)","d numbers per subsample","local lasso + debiasing (nodewise lassos); low-dim local OLS/GLM","Wald normality k=o((s log d)^-1 sqrt n) (Thm 3.3); lasso rate retained iff k=O(sqrt(n/(s^2 log d))) (Thm 4.3); low-dim GLM k=o(sqrt n/(d v log n)) (arXiv Thm 4.13)","high-probability decomposition + CLT","sparse high-dim linear & GLM; low-dim linear & GLM","no bias correction beyond lasso debiasing; establishes the sqrt(N) baseline (r=1,q=0); Remark 4.5 = same leading term, remainders differ",low,"n = total sample size in their notation","arXiv pp. 1-2, 8, 13-17; PMC lines 92-99, 368-442, 531-537"
12 volgushev2019quantile,"Volgushev, Stanislav; Chao, Shih-Kang; Cheng, Guang",2019,Distributed inference for quantile regression processes,"Annals of Statistics 47(3):1634-1662",10.1214/18-AOS1730,"full text arXiv 1701.06088v3 + NSF-PAR accepted manuscript (numbering identical)","bounded design with eigenvalue bounds; conditional density bounded with bounded derivative; Holder smoothness of tau -> beta(tau)","1 round (one-shot)","m numbers per machine per quantile level","local quantile regression fits","sup_tau ||beta_bar - beta_or|| = O_P(S log N/N + S^1/4 (log N)^7/4/N^3/4) + o_P(N^-1/2) (Thm 3.1); lower bound >= C S/N for some model (Thm 3.2); sufficient S=o(N^1/2/log N), NECESSARY S=o(N^1/2) (Cor 3.3)","new Bahadur representation; explicit worst-case construction","linear quantile regression; series models; conditional distribution functions","template for sharp (necessary + sufficient) partition-count conditions; no correction; no Newton",low,"necessity is worst-case over a model class","arXiv pp. 1, 6, 9-11" 12 volgushev2019quantile,"Volgushev, Stanislav; Chao, Shih-Kang; Cheng, Guang",2019,Distributed inference for quantile regression processes,"Annals of Statistics 47(3):1634-1662",10.1214/18-AOS1730,"full text arXiv 1701.06088v3 + NSF-PAR accepted manuscript (numbering identical)","bounded design with eigenvalue bounds; conditional density bounded with bounded derivative; Holder smoothness of tau -> beta(tau)","1 round (one-shot)","m numbers per machine per quantile level","local quantile regression fits","sup_tau ||beta_bar - beta_or|| = O_P(S log N/N + S^1/4 (log N)^7/4/N^3/4) + o_P(N^-1/2) (Thm 3.1); lower bound >= C S/N for some model (Thm 3.2); sufficient S=o(N^1/2/log N), NECESSARY S=o(N^1/2) (Cor 3.3)","new Bahadur representation; explicit worst-case construction","linear quantile regression; series models; conditional distribution functions","template for sharp (necessary + sufficient) partition-count conditions; no correction; no Newton",low,"necessity is worst-case over a model class","arXiv pp. 1, 6, 9-11"
13 shang2017limits,"Shang, Zuofeng; Cheng, Guang",2017,Computational Limits of A Distributed Algorithm for Smoothing Spline,"JMLR 18(108):1-37",https://jmlr.org/papers/v18/16-289.html,"full text JMLR PDF","periodic m-th order smoothing spline; evenly spaced design; iid errors; Sobolev ball","1 round (one-shot averaging)","one function estimate per machine","local smoothing spline with common lambda","minimax optimal iff s=O(N^2m/(2m+1)) (Thm 3.1); worst-case squared bias >= C(a_m n^-2m - 8N^-1) (Thm 3.2) => sharp; testing s=O(N^(4m-1)/(4m+1)) sharp","non-asymptotic bounds in eigen-basis; phase transition","nonparametric regression; testing","sharp maximal number of machines set by local bias vs 1/N variance; same mechanism, nonparametric",low,"improves ZDW bound","JMLR pp. 1, 5-7, 9-10" 13 shang2017limits,"Shang, Zuofeng; Cheng, Guang",2017,Computational Limits of A Distributed Algorithm for Smoothing Spline,"JMLR 18(108):1-37",https://jmlr.org/papers/v18/16-289.html,"full text JMLR PDF","periodic m-th order smoothing spline; evenly spaced design; iid errors; Sobolev ball","1 round (one-shot averaging)","one function estimate per machine","local smoothing spline with common lambda","minimax optimal iff s=O(N^2m/(2m+1)) (Thm 3.1); worst-case squared bias >= C(a_m n^-2m - 8N^-1) (Thm 3.2) => sharp; testing s=O(N^(4m-1)/(4m+1)) sharp","non-asymptotic bounds in eigen-basis; phase transition","nonparametric regression; testing","sharp maximal number of machines set by local bias vs 1/N variance; same mechanism, nonparametric",low,"improves ZDW bound","JMLR pp. 1, 5-7, 9-10"
14+dhaene2015split,"Dhaene, Geert; Jochmans, Koen",2015,"Split-panel Jackknife Estimation of Fixed-effect Models","Review of Economic Studies 82(3):991-1030",10.1093/restud/rdv007,"yes (accepted manuscript)","alpha-mixing stationary panel; influence-function representation; bias expansion B1/T + ... + Bk/T^k; N/T -> rho",NA,NA,NA,"bias o(T^-h) with h-th order jackknife; sqrt(NT) normal with variance Sigma^-1 unchanged (first order)","asymptotic rectangular-array","nonlinear fixed-effect panel MLE","same extrapolation algebra (weights from a Vandermonde-type system in 1/T); bias-optimal partitions {2,3,...,h+1}, not {2,4,...}; higher-order variance left open","high for the extrapolation step; no distributed/Newton component","Supplement Thms S.2.1, S.2.2, S.3.1 cited from the manuscript","manuscript pp. 7-10, 24-26, 33"
15+hahn2024efficient,"Hahn, Jinyong; Hughes, David W.; Kuersteiner, Guido; Newey, Whitney K.",2024,"Efficient bias correction for cross-section and panel data","Quantitative Economics 15(3):783-816",10.3982/QE2350,"yes","iid; third-order stochastic expansion; regularity App. B",NA,NA,NA,"split-sample bias estimate inconsistent (O_p(1)); higher-order variance of split-sample corrections = 2x leave-one-out (Props 5-7)","asymptotic expansions","MLE cross-section; FE panel","variance cost of split-sample corrections",high,"analytical, bootstrap and leave-one-out share the same higher-order variance (Thms 1-2)","pp. 783-787, 797-802, 806-807"
16+quenouille1956notes,"Quenouille, M. H.",1956,"Notes on bias in estimation","Biometrika 43(3/4):353-360",10.1093/biomet/43.3-4.353,no,"bias expansion in powers of 1/n (per Kosmidis 2014)",NA,NA,NA,"bias O(n^-2) after jackknife",NOT VERIFIED,"general estimators","origin of the jackknife bias cancellation",low,"text not read",none
17+quenouille1949approximate,"Quenouille, M. H.",1949,"Approximate tests of correlation in time-series","JRSS B 11(1):68-84",10.1111/j.2517-6161.1949.tb00023.x,no,NOT VERIFIED,NA,NA,NA,NOT VERIFIED,NOT VERIFIED,"time series","split-half bias cancellation (attribution per secondary sources)",low,"text behind paywall",none
18+tukey1958bias,"Tukey, J. W.",1958,"Bias and confidence in not-quite large samples (abstract)","Annals of Mathematical Statistics 29(2):614",10.1214/aoms/1177706647,no,NA,NA,NA,NA,NA,"abstract","general","names the jackknife",low,"DOI covers Abstracts of Papers pp. 614-623",none
19+schucany1971bias,"Schucany, W. R.; Gray, H. L.; Owen, D. B.",1971,"On bias reduction in estimation","JASA 66(335):524-533",10.1080/01621459.1971.10482296,no,"known ratio of bias functions (per secondary)",NA,NA,NA,"removes k bias terms with k+1 estimators (per secondary)",NOT VERIFIED,"general estimators","generalized-jackknife weights = same linear-system idea",medium,"brief's DOI was wrong",none
20+sharot1976generalized,"Sharot, Trevor",1976,"The generalized jackknife: finite samples and subsample sizes","JASA 71(354):451-454",10.1080/01621459.1976.10480367,no,NOT VERIFIED,NA,NA,NA,NOT VERIFIED,NOT VERIFIED,"general","subsample-size choice",medium,"no text available",none
21+efron1982jackknife,"Efron, Bradley",1982,"The Jackknife, the Bootstrap and Other Resampling Plans","SIAM CBMS-NSF 38",10.1137/1.9781611970319,no,NA,NA,NA,NA,NA,"monograph","general","jackknife as linear extrapolation in 1/n (sec 2.3 per Kosmidis)",low,"",none
22+cox1968general,"Cox, D. R.; Snell, E. J.",1968,"A general definition of residuals","JRSS B 30(2):248-265",10.1111/j.2517-6161.1968.tb00724.x,no,"regular likelihood; cumulants O(n)",NA,NA,NA,"O(1/n) bias formula","formal expansion","parametric MLE","analytic first-order bias targeted by the extrapolation",low,"formula per Giles 2011",none
23+cordeiro1991bias,"Cordeiro, Gauss M.; McCullagh, Peter",1991,"Bias correction in generalized linear models","JRSS B 53(3):629-643",10.1111/j.2517-6161.1991.tb01852.x,"abstract","GLM with known/estimated dispersion",NA,NA,NA,"O(1/n) bias removed","formal expansion","GLMs","closed-form analytic bias (benchmark for the jackknife)",low,"formula per Kosmidis et al. 2018",none
24+firth1993bias,"Firth, David",1993,"Bias reduction of maximum likelihood estimates","Biometrika 80(1):27-38",10.1093/biomet/80.1.27,"abstract","regular parametric problems",NA,NA,NA,"first-order bias removed via score modification","asymptotic","parametric MLE, GLMs","implicit alternative to the jackknife",low,"",none
25+rilstone1996second,"Rilstone, Paul; Srivastava, V. K.; Ullah, Aman",1996,"The second-order bias and mean squared error of nonlinear estimators","Journal of Econometrics 75(2):369-395",10.1016/0304-4076(96)89457-7,no,NOT VERIFIED,NA,NA,NA,"O(1/n) bias and MSE expansions",NOT VERIFIED,"M-estimators","analytic second-order bias of M-estimators",low,"corrigendum 2005 124(1):203-204",none
26+hahn2004jackknife,"Hahn, Jinyong; Newey, Whitney",2004,"Jackknife and analytical bias reduction for nonlinear panel models","Econometrica 72(4):1295-1319",10.1111/j.1468-0262.2004.00533.x,"abstract + 2002 WP","static nonlinear FE panel; expansion theta[T] = theta + beta/T + gamma/T^2 + O(T^-3)",NA,NA,NA,"bias O(T^-2)","asymptotic","static nonlinear panel MLE","delete-one jackknife vs analytic correction",low-medium,"",none
27+lecam1956asymptotic,"Le Cam, Lucien",1956,"On the asymptotic theory of estimation and testing hypotheses","Proc. Third Berkeley Symp. 1:129-156",10.1525/9780520313880-014,no,NOT VERIFIED,NA,NA,NA,NOT VERIFIED,NOT VERIFIED,"parametric","one-step efficiency origin",low,"",none
28+bickel1975one,"Bickel, P. J.",1975,"One-step Huber estimates in the linear model","JASA 70(350):428-434",10.1080/01621459.1975.10479884,"abstract","mild regularity conditions",NA,NA,NA,"one-step asymptotically equivalent to the full M-estimate",NOT VERIFIED,"linear model M-estimators","one-step Newton from a preliminary estimator",low,"",none
29+vandervaart1998asymptotic,"van der Vaart, A. W.",1998,"Asymptotic Statistics","Cambridge University Press",10.1017/CBO9780511802256,"TOC",NA,NA,NA,NA,NA,"textbook","general","Sec. 5.7 One-Step Estimators p. 71",low,"",TOC
30+bhattacharya1978validity,"Bhattacharya, R. N.; Ghosh, J. K.",1978,"On the validity of the formal Edgeworth expansion","Annals of Statistics 6(2):434-451",10.1214/aos/1176344134,"abstract","iid, smooth H, moment + Cramer conditions",NA,NA,NA,"valid Edgeworth expansion to arbitrary order","rigorous","MLE, minimum contrast","justifies the bias expansion in 1/n",low,"",none
31+bach2021richardson,"Bach, Francis",2021,"On the effectiveness of Richardson extrapolation in data science","SIAM J. Math. Data Sci. 3(4):1251-1277",10.1137/21M1397349,yes,"power-series dependence in the hyperparameter",NA,NA,NA,"O(1/k) -> O(1/k^2) in iterations","asymptotic","optimisation, ridge, smoothing","Richardson in iterations/regularisation, not sample size",low,"",arXiv
32+lin2024jackknife,"Lin, Licong; Su, Fangzhou; Mou, Wenlong; Ding, Peng; Wainwright, Martin J.",2024,"When is it worthwhile to jackknife? Breaking the quadratic barrier for Z-estimators","arXiv 2411.02909",https://arxiv.org/abs/2411.02909,"abstract","high-dim Z-estimators",NA,NA,NA,"sqrt(n)-consistent when n >> d^{3/2}","non-asymptotic","Z-estimators, GLMs","leave-one-out jackknife only",low-medium,"",abstract
33+wu2023subsampling,"Wu, Shuyuan; Zhu, Xuening; Wang, Hansheng",2023,"Subsampling and jackknifing: a practically convenient solution for large data analysis with limited computational resources","Statistica Sinica 33:2041-2064",10.5705/ss.202021.0257,"abstract","mild conditions",NA,NA,NA,"consistent, asymptotically normal","asymptotic","general estimators","jackknife debiasing within subsamples; no nested sizes",low,"",abstract
34+chudik2018half,"Chudik, Alexander; Pesaran, M. Hashem; Yang, Jui-Chung",2018,"Half-panel jackknife fixed-effects estimation of linear panels with weakly exogenous regressors","Journal of Applied Econometrics 33(6):816-836",10.1002/jae.2623,"abstract","weakly exogenous regressors",NA,NA,NA,"bias O(T^-2); needs N/T^3 -> 0","asymptotic","linear dynamic panels","half-panel jackknife application",low,"",none
35+zhu2021lsa,"Zhu, Xuening; Li, Feng; Wang, Hansheng",2021,"Least-Square Approximation for a Distributed System","Journal of Computational and Graphical Statistics 30(4):1004-1018",10.1080/10618600.2021.1923517 (arXiv:1908.04904v4),"yes (arXiv v4)","iid C1-C5; c1 <= n_k/n <= c2","1 (WLSE) / 2 (TWLSE) / m (iterated)","K(p + p^2) per round","local M-estimation; TWLSE adds one LOCAL Newton step per machine before Hessian-weighted averaging","Prop 1 bias O_p(K/sqrt N); Thm 1 efficient if n/N^1/2 -> inf; Thm 2 TWLSE bias O_p(n^-2 sqrt N), efficient if n/N^1/4 -> inf (K = O(N^3/4)); Prop 2 m-WLSE with m/log N -> inf rounds allows n = O(1)","asymptotic normality","M-estimators","closest to 'raw average + one step' with local Hessians at local points; unequal sizes","STRONG for (r=1,q=1) exponent 3/4 and for the stale/local-Hessian variant; no bias correction; no joint frontier","trade-off remark p.12","pp. 6, 10-12"
36+gu2023heterogeneity,"Gu, Jia; Chen, Songxi",2023,"Distributed statistical inference under heterogeneity","Journal of Machine Learning Research 24(387):1-56",https://jmlr.org/papers/v24/22-1274.html (arXiv:2209.06482v1),"yes (arXiv v1)","heterogeneous blocks; Assumptions 1-7; moments v >= 4; unequal n_k","1","K(p + p^2)","local M-estimation + analytic leading-bias estimate B_k with sample splitting","WD: K = o(N^1/2) (Thms 2-3); debiased dWD/dSaC: MSE <= C1/(nK) + C2/(n^2 K) + C3/n^3, chi^2 limit if K = o(n^2) = o(N^2/3) (Thms 4-6); Cor 1 beats the SAVGM constant","MSE bounds + chi^2 limits","likelihood and quasi-likelihood (sandwich weights)","analytic local debiasing before weighted averaging (our analytic Cox-Snell baseline is this idea); unequal sizes; heterogeneity","STRONG for 'debias then average' (analytic, not split-sample); none for Newton rounds / frontier","SM 1.10 compares with Huang-Huo","pp. 3, 17, 20-24"
37+wu2023subsampling_jds,"Wu, Shuyuan; Zhu, Xuening; Wang, Hansheng",2023,"Subsampling and jackknifing: a practically convenient solution for large data analysis with limited computational resources","Statistica Sinica 33:2041-2064",10.5705/ss.202021.0257 (arXiv:2304.06231v1),"yes (arXiv v1)","sub-Gaussian; g eight times differentiable; N = o(n^4); log K = o(sqrt n); with-replacement subsamples","1","K estimates","delete-one jackknife (n refits) per subsample, then average","Thm 2: bias O(n^-2) + O(N^-1) vs tau_2/n for plain averaging; Thm 4 normal under N = o(n^4)","expansion + CLT","smooth functions of means","jackknife-debiased local estimates then averaged (delete-one, not split-sample; n refits vs 2^r - 1 nested fits)","STRONG for 'jackknife then average' (different jackknife); none for Newton / frontier","Li et al. 2024 review extends to distributed M-estimators","pp. 10-16"
38+yin2022newton,"Yin, Zhuoran",2022,"Communication-efficient distributed Newton-like optimization with gradients and M-estimators","arXiv 2207.06253 (Temple PhD thesis 2022 NOT VERIFIED)",https://arxiv.org/abs/2207.06253,"yes (arXiv v1)","iid; third-order smooth loss; sqrt(n)-consistent local M-estimators","2 / T","M-estimators + gradients + local second/third-derivative summaries","local M-estimation + derivative summaries","Fisher information from gradients and M-estimators with error O_p(n^-1 + m^-1); Thm 3.2 bias-adjusted one-step error O_p(n^-3/2 + m^-1 n^-1/2), efficient iff m = o(n^2) (K = o(N^2/3))","expansion bounds","M-estimators","analytic bias adjustment of the one-step estimator from the plain average","PARTIAL/STRONG for 'debiased averaging + Newton' (analytic, 2/3); none for split-sample / frontier","thesis may contain a non-normal limit for many centres (NOT VERIFIED)","pp. 4, 10-11"
39+gao2022review,"Gao, Yuan; Liu, Weidong; Wang, Hansheng; Wang, Xiaozhou; Yan, Yibo; Zhang, Riquan",2022,"A review of distributed statistical inference","Statistical Theory and Related Fields 6(2):89-99",10.1080/24754269.2021.1974158 (arXiv:2304.06245),"yes","review","-","-","-","averaging MSE C1/N + C2/n^2; one-step C1/N + O(n^-4 + N^-2); K = o(sqrt N) necessary (citing Volgushev)","review","-","confirms no (order, rounds) frontier as of 2022; open problem: unbalanced sizes",NONE,"p. 20 open problem","pp. 4-7, 11, 20"
40+chen2024binary,"Chen, Xi; Jing, Wenbo; Liu, Weidong; Zhang, Yichen",2024,"Distributed estimation and inference for semiparametric binary response models","Annals of Statistics 52(3):922-947",10.1214/24-AOS2376 (arXiv:2210.08393v4),"yes","smoothed max-score kernel order alpha; heterogeneous sizes (Sec 4)","1 / T","L(p + p^2)","smoothed local max score","Thm 3.1 one-shot L condition; Thms 3.3-3.4 multi-round removes it; Thm 4.1 weighted average with unequal m_l","nonsmooth asymptotics","binary choice","multi-round removes the machine-count condition; unequal sizes","PARTIAL","","pp. 5-6, 18-24, 30-32"
41+wang2021osup,"Wang, Fei; Zhu, Yuan; Huang, Danyang; Qi, Haobo; Wang, Hansheng",2021,"Distributed one-step upgraded estimation for non-uniformly and non-randomly distributed data","Computational Statistics & Data Analysis 162:107265",10.1016/j.csda.2021.107265,"abstract","non-random unequal partitions; pilot sample","2","pilot + derivatives","one Newton step from a pilot","as efficient as the full MLE","asymptotic normality","MLE / GLM","pilot + one step; unequal non-random partitions","PARTIAL (C2); STRONG (C5)","","abstract"
42+zhou2020glm,"Zhou, Ping; Yu, Zhen; Ma, Jingyi; Tian, Maozai; Fan, Ye",2020,"Communication-efficient distributed estimator for generalized linear models with a diverging number of covariates","arXiv 2001.06194v2 (journal NOT VERIFIED)",https://arxiv.org/abs/2001.06194,"yes","GLM, p_n = o(n^alpha); max n_k / min n_k = O(1)","2","K(p + p^2)","weighted average + one Fisher-scoring step","AEE needs K = o(sqrt n/p_n) (Thm 3); weighted average + one step needs K = O(sqrt(n/p_n)) (Thm 4)","efficiency","GLM","one-step from a weighted average; unequal sizes","PARTIAL","","pp. 4, 11, 13"
43+duan2022heterogeneity,"Duan, Rui; Ning, Yang; Chen, Yong",2022,"Heterogeneity-aware and communication-efficient distributed statistical inference","Biometrika 109(1):67-83",10.1093/biomet/asab007,"abstract","heterogeneous sites; density-ratio tilting","1-2","O(p + p^2)","local MLE + score","Cramer-Rao bound attained if the number of sites is smaller in rate than n","two-index asymptotics","parametric likelihood","heterogeneity-aware surrogate","PARTIAL (C5/C4)","","abstract"
44+chen2019quantile,"Chen, Xi; Liu, Weidong; Zhang, Yichen",2019,"Quantile regression under memory constraint","Annals of Statistics 47(6):3244-3273",10.1214/18-AOS1777,"abstract","QR; batch size m","few","p^2 per round","smoothed scores","naive DC needs n = o(m^2); multi-round removes","Bahadur","quantile regression","multi-round removes the constraint","PARTIAL (C2)","","abstract"
45+chen2014split,"Chen, Xueying; Xie, Min-ge",2014,"A split-and-conquer approach for analysis of extraordinarily large data","Statistica Sinica 24:1655-1684",10.5705/ss.2013.088,"abstract","penalized GLM; K subsets","1","K(p + p^2)","penalized GLM","equivalence when K is controlled (condition NOT VERIFIED)","asymptotic normality","penalized GLM","weighted combination with local information","PARTIAL (C5)","","abstract"
46+dobriban2021averaging,"Dobriban, Edgar; Sheng, Yue",2021,"Distributed linear regression by averaging","Annals of Statistics 49(2):918-943",10.1214/20-AOS1984,"abstract","linear regression; random-matrix asymptotics","1","Kp","OLS","exact efficiency loss of weighted averaging","random matrix theory","linear regression","efficiency loss of averaging","low","","abstract"
47+szabo2020adaptive,"Szabo, Botond; van Zanten, Harry",2020,"Adaptive distributed methods under communication constraints","Annals of Statistics 48(4):2347-2380",10.1214/19-AOS1890,"abstract","nonparametric; bit budget","1-2","bits","-","minimax bounds under communication constraints","information-theoretic","nonparametric","bit accounting precedent","low","","abstract"
48+cai2022gaussian,"Cai, T. Tony; Wei, Hongji",2022,"Distributed Gaussian mean estimation under communication constraints: optimal rates and communication-efficient algorithms","Annals of Statistics 50(4):1992-2020",10.1214/21-AOS2167,"abstract","Gaussian mean; bit budget","1","bits","-","optimal rates (localization + refinement)","information-theoretic","Gaussian mean","bit accounting precedent","low","","abstract"
49+duchi2014optimality,"Duchi, John C.; Jordan, Michael I.; Wainwright, Martin J.; Zhang, Yuchen",2014,"Optimality guarantees for distributed statistical estimation","arXiv 1405.0782 (NeurIPS 2013 version)",https://arxiv.org/abs/1405.0782,"abstract","bit budgets","-","bits","-","communication lower bounds","information-theoretic","location / regression","lower bounds (universal, not algorithm-family)","low","Garg-Ma-Nguyen 2014 and Braverman et al. 2016 NOT VERIFIED","abstract"
50+kal2025adaptive,"Kal, Dennis; Szabo, Botond; Guhaniyogi, Rajarshi; Pillai, Natesh; Pati, Debdeep",2025,"Adaptive divide and conquer with two rounds of communication","arXiv 2508.17073",https://arxiv.org/abs/2508.17073,"abstract/intro","white noise model","2","(log2 n)^2 bits + optimal bits","local summaries","rate-optimal adaptive","minimax","nonparametric","bit accounting with two rounds","low","","pp. 1-2"
51+banerjee2019divide,"Banerjee, Moulinath; Durot, Cecile; Sen, Bodhisattva",2019,"Divide and conquer in nonstandard problems and the super-efficiency phenomenon","Annals of Statistics 47(2):720-757",10.1214/17-AOS1633,"abstract","cube-root problems","1","Kp","isotonic","super-efficiency","nonstandard","isotonic regression","none","NONE","","abstract"
52+minsker2019distributed,"Minsker, Stanislav",2019,"Distributed statistical estimation and rates of convergence in normal approximation","Electronic Journal of Statistics 13:5213-5252",10.1214/19-EJS1647,"abstract","robust aggregation","1","Kp","local","same rate if K <= O(sqrt N)","Berry-Esseen","M-estimators / MLE","none","NONE","","abstract"
53+luo2020renewable,"Luo, Lan; Song, Peter X.-K.",2020,"Renewable estimation and incremental inference in generalized linear models with streaming data sets","Journal of the Royal Statistical Society Series B 82(1):69-97",10.1111/rssb.12352,"abstract","streaming data","sequential","summaries","incremental Newton","consistency + normality","asymptotic","GLM","none","NONE","","abstract"
54+tan2022quantile,"Tan, Kean Ming; Battey, Heather; Zhou, Wen-Xin",2022,"Communication-constrained distributed quantile regression with optimal statistical guarantees","Journal of Machine Learning Research 23(272):1-61",https://jmlr.org/papers/v23/21-1269.html,"abstract","double-smoothed QR","near-constant","gradients / Hessians","smoothed objectives","communication-vs-error trade-off","finite-sample","quantile regression","trade-off (no Hessian compression)","low","","abstract"
55+alimisis2021quantized,"Alimisis, Foivos; Davies, Peter; Alistarh, Dan",2021,"Communication-efficient distributed optimization with quantized preconditioners","Proc. 38th ICML (arXiv 2102.07214)",https://arxiv.org/abs/2102.07214,"abstract","strongly convex; lattice quantization","iterative","quantized Hessians","local Hessians","communication complexity (optimization only)","optimization","convex ERM","quantized-Hessian algorithm without statistical K-exponent","low (no statistics)","","abstract"
56+lin2023proforma,"Lin, Lu; Li, Feng",2023,"Global debiased DC estimations for biased estimators via pro forma regression","TEST 32(2):726-758",10.1007/s11749-023-00850-5 (arXiv:1904.07477v2),"yes","non-identically distributed batches","1","batches x (estimate + covariate)","local estimator + structural covariate","Thms 3.3-3.4 sqrt(n)-consistent / normal for any N, m","regression-based","biased estimators","not block-size extrapolation","NONE / weak","","pp. 10-15"
57+li2024selective,"Li, Xuetong; Gao, Yuan; Chang, Hong; Huang, Danyang; Ma, Yingying; Pan, Rui; Qi, Haobo; Wang, Feifei; Wu, Shuyuan; Xu, Ke; Zhou, Jing; Zhu, Xuening; Zhu, Yingqiu; Wang, Hansheng",2024,"A selective review on statistical methods for massive data computation","arXiv 2403.11163 (journal NOT VERIFIED)",https://arxiv.org/abs/2403.11163,"yes","review","-","-","-","JDS bias O(1/n^2) + O(1/N); per-machine delete-one M-estimators","review","M-estimators","JDS 'readily applicable to distributed computing'","PARTIAL (C1)","","pp. 15-16"
added literature/audit_C_jackknife_bias_classics.md +31 −0
@@ -0,0 +1,31 @@
1+# Literature audit C — jackknife / bias-correction classics and 2010–2026 extrapolation search
2+
3+Method (verification task, 2026-10-28 18:47–19:03): Crossref-verified metadata for every item; full texts read for Dhaene–Jochmans (accepted manuscript), Hahn–Hughes–Kuersteiner–Newey 2024 (QE PDF), ZDW 2013, Hahn–Newey 2002 WP, Lin et al. 2024; secondary sources for formulas (Kosmidis 2014; Kosmidis–Kenne Pagui–Sartori 2018; Giles 2011). JSTOR / T&F / ScienceDirect / SIAM / Euclid blocked: several classics are "metadata verified, text NOT VERIFIED".
4+
5+## Corrections to the brief
6+- Schucany, Gray & Owen (1971) JASA 66(335):524–533: DOI **10.1080/01621459.1971.10482296** (the brief's …297 is Light & Margolin).
7+- Sharot (1976) JASA 71(354):451–454: DOI 10.1080/01621459.1976.10480367. (Also Sharot 1976 Biometrika 63(2):315–321 "Sharpening the jackknife".)
8+- Cox & Snell (1968) JRSS B 30(2): 248–265 on OUP/Crossref (248–275 with discussion per JSTOR).
9+
10+## Dhaene & Jochmans (2015), RES 82(3):991–1030, DOI 10.1093/restud/rdv007 (accepted manuscript read)
11+- Assumptions 2.1–2.3: α-mixing stationary panel; influence-function representation; bias expansion θ_T − θ0 = B₁/T + o(1/T). Eq. (2.4) half-panel jackknife θ̃_{1/2} = 2θ̂ − ½(θ̄_{S1} + θ̄_{S2}). Theorem 2.1: √(NT)(θ̃ − θ0) ⇒ N(0, Σ^{-1}) — "bias reduction can be achieved without variance inflation" (first-order).
12+- Higher order (§5.1, eqs. 5.1–5.3): with expansion (2.5) to order k, the h-th order jackknife θ̃_{1/G} = (1 + Σ_g a_{1/g})θ̂ − Σ_g a_{1/g} θ̄_{1/g}, weights a = (1 − ι'A^{-1}ι)^{-1}A^{-1}ι, [A]_{r,s} = Σ_{S∈S_{g_s}} (T/|S|)^{r−1}; for G = {2,3}: 3θ̂ − 3θ̄_{1/2} + θ̄_{1/3}, inconsistency o(T^{-2}). Supplement Theorem S.2.2: **G = {2, 3, …, h+1} minimises the inflation of the non-eliminated bias terms** (thirds, not quarters). First-order variance unchanged; "How to choose h optimally in practice is a difficult issue because the choice should also be guided by variance considerations" — higher-order variance left open. Overlapping subpanels inflate variance by d_g = ½g/(g−1) (Thm S.3.1).
13+
14+## Hahn, Hughes, Kuersteiner & Newey (2024), Quantitative Economics 15(3):783–816, DOI 10.3982/QE2350 (full text read)
15+- Split-sample bias estimate b̂_SS = ½(θ̂(1) + θ̂(2)) − θ̂ is unbiased but inconsistent (O_p(1), U-statistic structure; Prop. 5, App. A.1). Panel Prop. 6: Var(θ̂_SS) ≈ Var(A₁) + 2ῦ/T vs Var(θ̂_J) ≈ Var(A₁) + ῦ/(T−1): split-sample corrections have twice the higher-order variance of leave-one-out. Prop. 7: √(nT)(b̂_SS/T − B) = O_p(T^{-1/2}) vs O_p(T^{-1}) for leave-one-out. Analytical, bootstrap and leave-one-out corrections share the same higher-order variance (Thms 1–2). Direct relevance: quantifies the finite-sample variance cost of split-sample (and quarter-sample) extrapolation.
16+
17+## Classics (metadata verified; texts behind paywalls unless noted)
18+- Quenouille (1949) JRSS B 11(1):68–84, DOI 10.1111/j.2517-6161.1949.tb00023.x; Quenouille (1956) Biometrika 43(3/4):353–360, DOI 10.1093/biomet/43.3-4.353 (per Kosmidis 2014: nθ̂ − (n−1)θ̄ has bias −b₂/n² + O(n^{-3})).
19+- Tukey (1958) Ann. Math. Statist. 29(2):614 (abstract within "Abstracts of Papers" pp. 614–623, DOI 10.1214/aoms/1177706647).
20+- Schucany, Gray & Owen (1971): generalized jackknife (θ̂₁ − Rθ̂₂)/(1 − R); multi-term system NOT VERIFIED from the original.
21+- Efron (1982) SIAM CBMS-NSF 38, DOI 10.1137/1.9781611970319 (§2.3 jackknife as linear extrapolation in 1/n, per Kosmidis).
22+- Cox & Snell (1968): O(1/n) bias Bias(θ̂_s) = Σ k^{si}k^{jl}[½k_{ijl} + k_{ij,l}] + O(n^{-2}) (form per Giles 2011 eq. 10).
23+- Cordeiro & McCullagh (1991) JRSS B 53(3):629–643, DOI 10.1111/j.2517-6161.1991.tb01852.x (abstract read): first-order GLM biases via a supplementary weighted regression; form b_β = −i^{-1}X'Wξ, ξ_i = h_i d'_i/(2d_i w_i) per Kosmidis et al. 2018 eq. (4).
24+- Firth (1993) Biometrika 80(1):27–38, DOI 10.1093/biomet/80.1.27 (abstract read).
25+- Rilstone, Srivastava & Ullah (1996) J. Econometrics 75(2):369–395, DOI 10.1016/0304-4076(96)89457-7 (corrigendum 2005, 124(1):203–204); Bao & Ullah (2007) J. Econometrics 140(2):650–669, DOI 10.1016/j.jeconom.2006.07.007.
26+- Hahn & Newey (2004) Econometrica 72(4):1295–1319, DOI 10.1111/j.1468-0262.2004.00533.x (abstract + 2002 WP read): delete-one-period jackknife Tθ̂ − (T−1)Σθ̂_(t)/T, bias O(T^{-2}).
27+- Le Cam (1956) Proc. Third Berkeley Symp. 1:129–156, DOI 10.1525/9780520313880-014; Bickel (1975) JASA 70(350):428–434, DOI 10.1080/01621459.1975.10479884 (abstract); van der Vaart (1998) CUP, DOI 10.1017/CBO9780511802256, §5.7 "One-Step Estimators" p. 71 (TOC verified).
28+- Bhattacharya & Ghosh (1978) AoS 6(2):434–451, DOI 10.1214/aos/1176344134 (abstract); Ghosh (1994) NSF-CBMS 4, DOI 10.1214/cbms/1462297300; Pfanzagl (1985) LNS 31, DOI 10.1007/978-1-4615-6479-9.
29+
30+## 2010–2026 search: nested-subsample Richardson extrapolation of MLEs + Newton in divide-and-conquer
31+No paper found that extrapolates over nested sub-sample sizes m, m/2, m/4 for MLEs/M-estimators and combines it with a Newton correction in a distributed setting. Nearest: ZDW (2013) SAVGM (one-level subsampled extrapolation, Theorem 4, eq. 8); Hahn et al. (2024) (variance of split-sample corrections); Bach (2021) SIMODS 3(4):1251–1277, DOI 10.1137/21M1397349 (Richardson in iterations/regularisation, not sample size); Lin, Su, Mou, Ding & Wainwright (2024) arXiv 2411.02909 (leave-one-out jackknife, high-dim Z-estimators); Wu, Zhu & Wang (2023) Statistica Sinica 33:2041–2064, DOI 10.5705/ss.202021.0257; Chudik, Pesaran & Yang (2018) JAE 33(6):816–836, DOI 10.1002/jae.2623; Fernández-Val & Weidner (2016) J. Econometrics 192(1):291–312, DOI 10.1016/j.jeconom.2015.12.014.
added literature/audit_D_priority_sweep_2013_2026.md +27 −0
@@ -0,0 +1,27 @@
1+# Literature audit D — priority / novelty-threat sweep (2013 → 2026-10-28)
2+
3+Method (verification task, 2026-10-28 18:47–19:05): 40+ web searches (arXiv, JMLR, AoS, JASA, Biometrika, Statistica Sinica, JCGS, QE, ReStud, TEST, CSDA; Crossref for metadata); full texts read for the 14 most relevant papers. Local dumps /tmp/litsearch/.
4+
5+## Threat per candidate claim (auditor's assessment)
6+- **C1 r-th order split-sample extrapolation of local MLEs, then average: PARTIAL→STRONG.** "Debias each local estimator then average" exists: ZDW13 SAVGM (subsampling); **Gu & Chen (2023) JMLR 24(387)** (analytic Rilstone-type local bias correction with sample splitting, weighted by n_k Ĥ_k^{-1}; χ² limit if K = o(n²) ⇔ K = o(N^{2/3}); beats the SAVGM constant); **Wu, Zhu & Wang (2023) Statistica Sinica 33:2041–2064** (delete-one jackknife within each subsample, then average; bias O(n^{-2}) + O(N^{-1}); normality under N = o(n⁴)); Li et al. (2024) review says JDS "is readily applicable to distributed computing"; Yin (2022) (analytic). The nested-subsample Vandermonde construction is Dhaene & Jochmans (2015) / Schucany–Gray–Owen (1971) / Quenouille (1956). **Not found:** a general r-th order split-sample extrapolation of local MLEs in divide-and-conquer with r ≥ 3 and the influence-function-preservation argument. Caveat: Hahn–Hughes–Kuersteiner–Newey (QE 2024) split-sample jackknife has twice the higher-order variance of delete-one.
7+- **C2 debiased average + q exact-Hessian Newton steps, frontier K = o(N^{1−1/(2^{q+1} r)}): PARTIAL.** Average + 1 exact step: Huang & Huo (2019) (stated k = O(√N); MSE term k⁴N^{-4} ⇒ implicitly 3/4); **Zhu, Li & Wang (2021) JCGS 30(4):1004–1018, DOI 10.1080/10618600.2021.1923517** Theorem 2: TWLSE (average → one local Newton step per machine → local-Hessian-weighted average) efficient iff n/N^{1/4} → ∞, "allows K = O(N^{3/4})"; Prop. 2 iterated version with m/log N → ∞ rounds allows n = O(1); CEASE rounds frontier n^{t+2} ≫ N polylog ⇒ K = o(N^{(t+1)/(t+2)}); JLY contraction n^{-1/2}; **Yin (2022) arXiv 2207.06253** Theorem 3.2: bias-adjusted one-step estimator (analytic removal of quadratic terms, pilot = average) efficient iff m = o(n²) ⇔ K = o(N^{2/3}). **Nobody combines an extrapolated initial estimator with exact-Hessian Newton rounds; no joint (order, rounds) frontier; no 7/8 or 11/12 anywhere.**
8+- **C3 sharpness: PARTIAL.** Necessity results exist only for plain averaging: Volgushev–Chao–Cheng (2019) Thm 3.2 + Cor 3.3; Shang–Cheng (2017); ZDW Example 1. Huang–Huo claim necessity of k = O(√N) from simulation only. Yin's 2022 Temple PhD thesis may derive a non-normal limit for one-step estimators with many centres — NOT VERIFIED (repository 403). No sharpness result for a debiased/Newton construction.
9+- **C4 inexact Newton: PARTIAL.** Single-machine Hessian: JLY Cor 3.4, Wang–Kolar–Srebro–Zhang 2017; local Hessians at local points: Zhu–Li–Wang (K = O(N^{3/4})); gradient-built Fisher information with O_p(n^{-1} + m^{-1}) error: Yin 2022; CEASE contraction p/n; Lin–Xi AEE. **Quantized/compressed Hessian with a statistical K-exponent and byte count: nothing found** (Alimisis–Davies–Alistarh ICML 2021 is optimization-only).
10+- **C5 unequal sizes / misspecification: STRONG (sizes) / PARTIAL (misspecification).** Weighted aggregation with unequal n_k is routine (Zhu–Li–Wang; Gu–Chen with quasi-likelihood ⇒ sandwich; Wang et al. CSDA 2021; Zhou et al. 2020; Lin–Xi; Chen–Xie 2014; Shi–Lu–Song 2018; Chen–Jing–Liu–Zhang AoS 2024 Thm 4.1). Misspecification: JLY Cor 3.4 is the sandwich; Gu–Chen quasi-likelihood; Volgushev p. 10. → present as an extension, not a contribution.
11+- **C6 cost model (extra extrapolation level vs extra synchronization round): PARTIAL-low.** Only qualitative trade-off statements (JLY p. 15; Zhu–Li–Wang p. 12; CEASE §4.3; Kal–Szabó–Guhaniyogi–Pillai–Pati 2025 bit accounting; Duchi et al. 2014, Szabó–van Zanten 2020, Cai–Wei 2024 lower bounds). No explicit decision rule found.
12+
13+## Additional verified records (beyond audits A–C)
14+- Zhu, Xuening; Li, Feng; Wang, Hansheng (2021) "Least-Square Approximation for a Distributed System", JCGS 30(4):1004–1018, DOI 10.1080/10618600.2021.1923517 (arXiv 1908.04904v4 read): WLSE weights α_k Σ̂_k^{-1}; c₁ ≤ n_k/n ≤ c₂; Prop. 1 bias O_p(K/√N); Thm 1 efficient if n/N^{1/2} → ∞; Thm 2 TWLSE bias O_p(n^{-2}√N), efficient if n/N^{1/4} → ∞; Prop. 2 m-WLSE.
15+- Gu, Jia; Chen, Songxi (2023) "Distributed statistical inference under heterogeneity", JMLR 24(387):1–56 (arXiv 2209.06482v1 read): WD weights n_k Ĥ_k^{-1}; Thms 2–3 K = o(N^{1/2}); §5 analytic leading bias n_k^{-1}B_k with sample splitting; Thms 4–6 dWD/dSaC MSE ≤ C₁/(nK) + C₂/(n²K) + C₃/n³, χ² limit if K = o(n²); Cor. 1 constant beats SAVGM's (2+3r)/(2(1−r)²); likelihood and quasi-likelihood (sandwich); one round; SM §1.10 vs Huang–Huo.
16+- Wu, Shuyuan; Zhu, Xuening; Wang, Hansheng (2023) Statistica Sinica 33:2041–2064, DOI 10.5705/ss.202021.0257 (arXiv 2304.06231v1 read): JDS = delete-one jackknife bias estimate per subsample then average (§2.3); Thm 2 bias O(n^{-2}) + O(N^{-1}); Thm 4 normal under N = o(n⁴). Smooth functions of means.
17+- Yin, Z. (2022) "Communication-efficient distributed Newton-like optimization with gradients and M-estimators", arXiv 2207.06253 (read): Fisher information from gradients + M-estimators with error O_p(n^{-1} + m^{-1}); Thm 3.2 bias-adjusted one-step error O_p(n^{-3/2} + m^{-1}n^{-1/2}); efficient iff m = o(n²).
18+- Gao, Yuan; Liu, Weidong; Wang, Hansheng; Wang, Xiaozhou; Yan, Yibo; Zhang, Riquan (2022) "A review of distributed statistical inference", Statistical Theory and Related Fields 6(2):89–99, DOI 10.1080/24754269.2021.1974158 (read): averaging MSE C₁/N + C₂/n²; one-step C₁/N + O(n^{-4} + N^{-2}); "K = o(√N) necessary (Volgushev)"; open problem: unbalanced sizes (p. 20). Confirms no (order, rounds) frontier as of 2022.
19+- Li et al. (2024) "A selective review on statistical methods for massive data computation", arXiv 2403.11163 (journal NOT VERIFIED).
20+- Chen, Xi; Jing, Wenbo; Liu, Weidong; Zhang, Yichen (2024) AoS 52(3):922–947, DOI 10.1214/24-AOS2376 (arXiv 2210.08393v4 read): Thm 3.1 one-shot L condition; Thms 3.3–3.4 multi-round; Thm 4.1 weighted average with unequal m_ℓ.
21+- Wang, Zhu, Huang, Qi, Wang (2021) CSDA 162:107265, DOI 10.1016/j.csda.2021.107265 (abstract): OSUP pilot + one step; non-random unequal partitions.
22+- Zhou, Yu, Ma, Tian, Fan (2020) arXiv 2001.06194v2 (read): AEE K = o(√n/p_n); weighted average + Fisher scoring K = O(√(n/p_n)).
23+- Duan, Ning, Chen (2022) Biometrika 109(1):67–83, DOI 10.1093/biomet/asab007 (abstract): heterogeneity-aware surrogate.
24+- Chen, Liu, Zhang (2019) AoS 47(6):3244–3273, DOI 10.1214/18-AOS1777 (abstract): naive DC needs n = o(m²); multi-round removes.
25+- Chen & Xie (2014) Statistica Sinica 24:1655–1684, DOI 10.5705/ss.2013.088 (abstract). Lee, Liu, Sun & Taylor (2017) JMLR 18(5):1–30 (abstract). Dobriban & Sheng (2021) AoS 49(2):918–943, DOI 10.1214/20-AOS1984. Szabó & van Zanten (2020) AoS 48(4):2347–2380, DOI 10.1214/19-AOS1890. Cai & Wei (2022) AoS 50(4):1992–2020, DOI 10.1214/21-AOS2167. Duchi, Jordan, Wainwright & Zhang (2014) arXiv 1405.0782. Kal, Szabó, Guhaniyogi, Pillai & Pati (2025) arXiv 2508.17073. Banerjee, Durot & Sen (2019) AoS 47(2):720–757, DOI 10.1214/17-AOS1633. Minsker (2019) EJS 13:5213–5252, DOI 10.1214/19-EJS1647. Luo & Song (2020) JRSS-B 82(1):69–97, DOI 10.1111/rssb.12352. Tan, Battey & Zhou (2022) JMLR 23(272):1–61. Alimisis, Davies & Alistarh (2021) ICML, arXiv 2102.07214. Lin & Li (2023) TEST 32(2):726–758, DOI 10.1007/s11749-023-00850-5. Tu, Liu, Mao & Chen (2021) JMLR 22(84):1–67.
26+
27+Searches with no relevant hit: "Richardson extrapolation" + distributed; "split-sample jackknife" + distributed; "K = o(N^{7/8})"; distributed pseudo-MLE/sandwich as a topic; quantized-Hessian one-step with asymptotic normality.
modified research_log.md +2 −0
@@ -11,3 +11,5 @@
11 - Decision: identifier WP28 kept (see SOURCE_PROVENANCE.md collision note). 11 - Decision: identifier WP28 kept (see SOURCE_PROVENANCE.md collision note).
12 - 18:55 Audit A received and archived (literature/audit_A_*.md, PRIOR_ART_MATRIX.csv rows 1-4). Key facts: HH19 proves one-step equivalence only under k=O(√N) from an uncorrected average; ZDW SAVGM = single-ratio subsampling correction with inflated variance constant, no jackknife/higher order; LX11 K≪N^{1/4}; RN16 m fixed. 12 - 18:55 Audit A received and archived (literature/audit_A_*.md, PRIOR_ART_MATRIX.csv rows 1-4). Key facts: HH19 proves one-step equivalence only under k=O(√N) from an uncorrected average; ZDW SAVGM = single-ratio subsampling correction with inflated variance constant, no jackknife/higher order; LX11 K≪N^{1/4}; RN16 m fixed.
13 - 19:00 Audit B archived (8 papers). Fixed sign of the analytic Cox–Snell correction (b_1 = +½ I^{-1} T; exponential check b_1 = θ0). 13 - 19:00 Audit B archived (8 papers). Fixed sign of the analytic Cox–Snell correction (b_1 = +½ I^{-1} T; exponential check b_1 = θ0).
14+- 19:08 Audit C archived (jackknife classics; two DOI corrections; Dhaene–Jochmans partitions {2,3,..} bias-optimal; Hahn et al. 2024 variance of split-sample corrections). Patterns (1,2,3) added to the experimental plan.
15+- 19:12 Audit D archived (priority sweep; 23 more rows). Closest prior art for the combination: Zhu–Li–Wang 2021 (K=O(N^{3/4}) TWLSE), Gu–Chen 2023 (analytic debias+average, 2/3), Wu–Zhu–Wang 2023 (delete-one jackknife per block), Yin 2022 (bias-adjusted one-step, 2/3). No joint (r,q) frontier, no sharpness for corrected constructions, no quantized-Hessian statistical result found.
added results/raw/exp_p1_N16384_K1024.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K1024.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K128.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K128.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K16.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K16.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K2048.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K2048.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K256.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K256.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K32.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K32.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K512.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K512.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K64.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N16384_K64.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N65536_K1024.csv +2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N65536_K1024.manifest.json +29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added results/raw/exp_p1_N65536_K128.csv +384 −0
Large diff (384 lines) is not loaded by default to keep the page fast.Load diff

Diff truncated — file too large.

Menu

Theme