WP28: literature audits C-D, PRIOR_ART_MATRIX (56 rows), PRIORITY_AND_GAP, ASSUMPTIONS, THEOREM_AUDIT, MC driver + cluster scatter
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
32 changed files +19,995 −0
added
experiments/cluster_assignment.json
+876 −0
@@ -0,0 +1,876 @@ | |||||
| 1 | +{ | ||||
| 2 | + "nodes": { | ||||
| 3 | + "M3U96a": 6, | ||||
| 4 | + "M3U96b": 6, | ||||
| 5 | + "M2U64": 4, | ||||
| 6 | + "M4M64a": 4, | ||||
| 7 | + "M4M64b": 4, | ||||
| 8 | + "M4M64c": 4, | ||||
| 9 | + "M5M36": 4, | ||||
| 10 | + "M4BP48": 3, | ||||
| 11 | + "M4BP36": 3, | ||||
| 12 | + "M2M32c": 3, | ||||
| 13 | + "m4mm": 3, | ||||
| 14 | + "m4mn": 3, | ||||
| 15 | + "m6ma": 3, | ||||
| 16 | + "m6mb": 3, | ||||
| 17 | + "m5ma": 3, | ||||
| 18 | + "m4md": 2, | ||||
| 19 | + "m4ml": 2, | ||||
| 20 | + "M3BA24": 2 | ||||
| 21 | + }, | ||||
| 22 | + "assignment": { | ||||
| 23 | + "M3U96a": [ | ||||
| 24 | + { | ||||
| 25 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 26 | + "cell": 49, | ||||
| 27 | + "cost": 187574977245.9856, | ||||
| 28 | + "name": "logit_p20_p20_N1048576_K8192" | ||||
| 29 | + }, | ||||
| 30 | + { | ||||
| 31 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 32 | + "cell": 48, | ||||
| 33 | + "cost": 46893744311.4964, | ||||
| 34 | + "name": "logit_p20_p20_N262144_K128" | ||||
| 35 | + }, | ||||
| 36 | + { | ||||
| 37 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 38 | + "cell": 36, | ||||
| 39 | + "cost": 21794242891.16812, | ||||
| 40 | + "name": "logit_p3_N1048576_K8192" | ||||
| 41 | + }, | ||||
| 42 | + { | ||||
| 43 | + "config": "experiments/configs/robustness.yaml", | ||||
| 44 | + "cell": 12, | ||||
| 45 | + "cost": 21794242891.16812, | ||||
| 46 | + "name": "logit_misspec_p3_N1048576_K8192" | ||||
| 47 | + }, | ||||
| 48 | + { | ||||
| 49 | + "config": "experiments/configs/robustness.yaml", | ||||
| 50 | + "cell": 14, | ||||
| 51 | + "cost": 21794242891.16812, | ||||
| 52 | + "name": "logit_misspec_p3_N1048576_K2048" | ||||
| 53 | + }, | ||||
| 54 | + { | ||||
| 55 | + "config": "experiments/configs/robustness.yaml", | ||||
| 56 | + "cell": 33, | ||||
| 57 | + "cost": 4194304000.0, | ||||
| 58 | + "name": "exp_contam_p1_N1048576_K65536" | ||||
| 59 | + }, | ||||
| 60 | + { | ||||
| 61 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 62 | + "cell": 6, | ||||
| 63 | + "cost": 1482910400.3789308, | ||||
| 64 | + "name": "logit_p8_N16384_K64" | ||||
| 65 | + }, | ||||
| 66 | + { | ||||
| 67 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 68 | + "cell": 10, | ||||
| 69 | + "cost": 1482910400.3789308, | ||||
| 70 | + "name": "pois_p8_N16384_K16" | ||||
| 71 | + }, | ||||
| 72 | + { | ||||
| 73 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 74 | + "cell": 12, | ||||
| 75 | + "cost": 1362140180.6980076, | ||||
| 76 | + "name": "pois_p3_N65536_K1024" | ||||
| 77 | + }, | ||||
| 78 | + { | ||||
| 79 | + "config": "experiments/configs/robustness.yaml", | ||||
| 80 | + "cell": 2, | ||||
| 81 | + "cost": 1362140180.6980076, | ||||
| 82 | + "name": "logit_misspec_p3_N65536_K512" | ||||
| 83 | + }, | ||||
| 84 | + { | ||||
| 85 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 86 | + "cell": 9, | ||||
| 87 | + "cost": 340535045.1745019, | ||||
| 88 | + "name": "logit_p3_N16384_K16" | ||||
| 89 | + }, | ||||
| 90 | + { | ||||
| 91 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 92 | + "cell": 5, | ||||
| 93 | + "cost": 340535045.1745019, | ||||
| 94 | + "name": "pois_p3_N16384_K64" | ||||
| 95 | + } | ||||
| 96 | + ], | ||||
| 97 | + "M3U96b": [ | ||||
| 98 | + { | ||||
| 99 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 100 | + "cell": 50, | ||||
| 101 | + "cost": 187574977245.9856, | ||||
| 102 | + "name": "logit_p20_p20_N1048576_K4096" | ||||
| 103 | + }, | ||||
| 104 | + { | ||||
| 105 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 106 | + "cell": 24, | ||||
| 107 | + "cost": 23726566406.062893, | ||||
| 108 | + "name": "logit_p8_N262144_K4096" | ||||
| 109 | + }, | ||||
| 110 | + { | ||||
| 111 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 112 | + "cell": 32, | ||||
| 113 | + "cost": 23726566406.062893, | ||||
| 114 | + "name": "pois_p8_N262144_K256" | ||||
| 115 | + }, | ||||
| 116 | + { | ||||
| 117 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 118 | + "cell": 38, | ||||
| 119 | + "cost": 21794242891.16812, | ||||
| 120 | + "name": "logit_p3_N1048576_K4096" | ||||
| 121 | + }, | ||||
| 122 | + { | ||||
| 123 | + "config": "experiments/configs/robustness.yaml", | ||||
| 124 | + "cell": 13, | ||||
| 125 | + "cost": 21794242891.16812, | ||||
| 126 | + "name": "logit_misspec_p3_N1048576_K4096" | ||||
| 127 | + }, | ||||
| 128 | + { | ||||
| 129 | + "config": "experiments/configs/robustness.yaml", | ||||
| 130 | + "cell": 20, | ||||
| 131 | + "cost": 11863283203.031446, | ||||
| 132 | + "name": "logit_illcond_p8_N262144_K2048" | ||||
| 133 | + }, | ||||
| 134 | + { | ||||
| 135 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 136 | + "cell": 29, | ||||
| 137 | + "cost": 5448560722.79203, | ||||
| 138 | + "name": "logit_p3_N262144_K512" | ||||
| 139 | + }, | ||||
| 140 | + { | ||||
| 141 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 142 | + "cell": 31, | ||||
| 143 | + "cost": 5448560722.79203, | ||||
| 144 | + "name": "pois_p3_N262144_K256" | ||||
| 145 | + }, | ||||
| 146 | + { | ||||
| 147 | + "config": "experiments/configs/robustness.yaml", | ||||
| 148 | + "cell": 15, | ||||
| 149 | + "cost": 2965820800.7578616, | ||||
| 150 | + "name": "logit_illcond_p8_N65536_K1024" | ||||
| 151 | + }, | ||||
| 152 | + { | ||||
| 153 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 154 | + "cell": 10, | ||||
| 155 | + "cost": 1482910400.3789308, | ||||
| 156 | + "name": "logit_p8_N16384_K16" | ||||
| 157 | + }, | ||||
| 158 | + { | ||||
| 159 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 160 | + "cell": 11, | ||||
| 161 | + "cost": 1362140180.6980076, | ||||
| 162 | + "name": "logit_p3_N65536_K2048" | ||||
| 163 | + }, | ||||
| 164 | + { | ||||
| 165 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 166 | + "cell": 16, | ||||
| 167 | + "cost": 1362140180.6980076, | ||||
| 168 | + "name": "pois_p3_N65536_K256" | ||||
| 169 | + }, | ||||
| 170 | + { | ||||
| 171 | + "config": "experiments/configs/robustness.yaml", | ||||
| 172 | + "cell": 3, | ||||
| 173 | + "cost": 1362140180.6980076, | ||||
| 174 | + "name": "logit_misspec_p3_N65536_K256" | ||||
| 175 | + }, | ||||
| 176 | + { | ||||
| 177 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 178 | + "cell": 0, | ||||
| 179 | + "cost": 340535045.1745019, | ||||
| 180 | + "name": "pois_p3_N16384_K512" | ||||
| 181 | + } | ||||
| 182 | + ], | ||||
| 183 | + "M2U64": [ | ||||
| 184 | + { | ||||
| 185 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 186 | + "cell": 51, | ||||
| 187 | + "cost": 187574977245.9856, | ||||
| 188 | + "name": "logit_p20_p20_N1048576_K2048" | ||||
| 189 | + }, | ||||
| 190 | + { | ||||
| 191 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 192 | + "cell": 13, | ||||
| 193 | + "cost": 5931641601.515723, | ||||
| 194 | + "name": "pois_p8_N65536_K1024" | ||||
| 195 | + }, | ||||
| 196 | + { | ||||
| 197 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 198 | + "cell": 23, | ||||
| 199 | + "cost": 5448560722.79203, | ||||
| 200 | + "name": "logit_p3_N262144_K4096" | ||||
| 201 | + }, | ||||
| 202 | + { | ||||
| 203 | + "config": "experiments/configs/robustness.yaml", | ||||
| 204 | + "cell": 6, | ||||
| 205 | + "cost": 5448560722.79203, | ||||
| 206 | + "name": "logit_misspec_p3_N262144_K4096" | ||||
| 207 | + }, | ||||
| 208 | + { | ||||
| 209 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 210 | + "cell": 18, | ||||
| 211 | + "cost": 1362140180.6980076, | ||||
| 212 | + "name": "logit_p3_N65536_K128" | ||||
| 213 | + }, | ||||
| 214 | + { | ||||
| 215 | + "config": "experiments/configs/robustness.yaml", | ||||
| 216 | + "cell": 4, | ||||
| 217 | + "cost": 1362140180.6980076, | ||||
| 218 | + "name": "logit_misspec_p3_N65536_K128" | ||||
| 219 | + } | ||||
| 220 | + ], | ||||
| 221 | + "M4M64a": [ | ||||
| 222 | + { | ||||
| 223 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 224 | + "cell": 52, | ||||
| 225 | + "cost": 187574977245.9856, | ||||
| 226 | + "name": "logit_p20_p20_N1048576_K1024" | ||||
| 227 | + }, | ||||
| 228 | + { | ||||
| 229 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 230 | + "cell": 15, | ||||
| 231 | + "cost": 5931641601.515723, | ||||
| 232 | + "name": "pois_p8_N65536_K512" | ||||
| 233 | + }, | ||||
| 234 | + { | ||||
| 235 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 236 | + "cell": 25, | ||||
| 237 | + "cost": 5448560722.79203, | ||||
| 238 | + "name": "logit_p3_N262144_K2048" | ||||
| 239 | + }, | ||||
| 240 | + { | ||||
| 241 | + "config": "experiments/configs/robustness.yaml", | ||||
| 242 | + "cell": 7, | ||||
| 243 | + "cost": 5448560722.79203, | ||||
| 244 | + "name": "logit_misspec_p3_N262144_K2048" | ||||
| 245 | + }, | ||||
| 246 | + { | ||||
| 247 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 248 | + "cell": 20, | ||||
| 249 | + "cost": 1362140180.6980076, | ||||
| 250 | + "name": "logit_p3_N65536_K64" | ||||
| 251 | + }, | ||||
| 252 | + { | ||||
| 253 | + "config": "experiments/configs/robustness.yaml", | ||||
| 254 | + "cell": 30, | ||||
| 255 | + "cost": 1048576000.0, | ||||
| 256 | + "name": "exp_contam_p1_N262144_K16384" | ||||
| 257 | + }, | ||||
| 258 | + { | ||||
| 259 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 260 | + "cell": 7, | ||||
| 261 | + "cost": 340535045.1745019, | ||||
| 262 | + "name": "pois_p3_N16384_K32" | ||||
| 263 | + } | ||||
| 264 | + ], | ||||
| 265 | + "M4M64b": [ | ||||
| 266 | + { | ||||
| 267 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 268 | + "cell": 53, | ||||
| 269 | + "cost": 187574977245.9856, | ||||
| 270 | + "name": "logit_p20_p20_N1048576_K512" | ||||
| 271 | + }, | ||||
| 272 | + { | ||||
| 273 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 274 | + "cell": 17, | ||||
| 275 | + "cost": 5931641601.515723, | ||||
| 276 | + "name": "pois_p8_N65536_K256" | ||||
| 277 | + }, | ||||
| 278 | + { | ||||
| 279 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 280 | + "cell": 27, | ||||
| 281 | + "cost": 5448560722.79203, | ||||
| 282 | + "name": "logit_p3_N262144_K1024" | ||||
| 283 | + }, | ||||
| 284 | + { | ||||
| 285 | + "config": "experiments/configs/robustness.yaml", | ||||
| 286 | + "cell": 8, | ||||
| 287 | + "cost": 5448560722.79203, | ||||
| 288 | + "name": "logit_misspec_p3_N262144_K1024" | ||||
| 289 | + }, | ||||
| 290 | + { | ||||
| 291 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 292 | + "cell": 11, | ||||
| 293 | + "cost": 1362140180.6980076, | ||||
| 294 | + "name": "pois_p3_N65536_K2048" | ||||
| 295 | + }, | ||||
| 296 | + { | ||||
| 297 | + "config": "experiments/configs/robustness.yaml", | ||||
| 298 | + "cell": 31, | ||||
| 299 | + "cost": 1048576000.0, | ||||
| 300 | + "name": "exp_contam_p1_N262144_K4096" | ||||
| 301 | + }, | ||||
| 302 | + { | ||||
| 303 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 304 | + "cell": 9, | ||||
| 305 | + "cost": 340535045.1745019, | ||||
| 306 | + "name": "pois_p3_N16384_K16" | ||||
| 307 | + } | ||||
| 308 | + ], | ||||
| 309 | + "M4M64c": [ | ||||
| 310 | + { | ||||
| 311 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 312 | + "cell": 35, | ||||
| 313 | + "cost": 94906265624.25157, | ||||
| 314 | + "name": "logit_p8_N1048576_K16384" | ||||
| 315 | + }, | ||||
| 316 | + { | ||||
| 317 | + "config": "experiments/configs/robustness.yaml", | ||||
| 318 | + "cell": 26, | ||||
| 319 | + "cost": 47453132812.125786, | ||||
| 320 | + "name": "logit_illcond_p8_N1048576_K2048" | ||||
| 321 | + }, | ||||
| 322 | + { | ||||
| 323 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 324 | + "cell": 34, | ||||
| 325 | + "cost": 21794242891.16812, | ||||
| 326 | + "name": "logit_p3_N1048576_K16384" | ||||
| 327 | + }, | ||||
| 328 | + { | ||||
| 329 | + "config": "experiments/configs/robustness.yaml", | ||||
| 330 | + "cell": 11, | ||||
| 331 | + "cost": 21794242891.16812, | ||||
| 332 | + "name": "logit_misspec_p3_N1048576_K16384" | ||||
| 333 | + }, | ||||
| 334 | + { | ||||
| 335 | + "config": "experiments/configs/robustness.yaml", | ||||
| 336 | + "cell": 21, | ||||
| 337 | + "cost": 11863283203.031446, | ||||
| 338 | + "name": "logit_illcond_p8_N262144_K1024" | ||||
| 339 | + }, | ||||
| 340 | + { | ||||
| 341 | + "config": "experiments/configs/robustness.yaml", | ||||
| 342 | + "cell": 5, | ||||
| 343 | + "cost": 5448560722.79203, | ||||
| 344 | + "name": "logit_misspec_p3_N262144_K8192" | ||||
| 345 | + }, | ||||
| 346 | + { | ||||
| 347 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 348 | + "cell": 8, | ||||
| 349 | + "cost": 1482910400.3789308, | ||||
| 350 | + "name": "pois_p8_N16384_K32" | ||||
| 351 | + }, | ||||
| 352 | + { | ||||
| 353 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 354 | + "cell": 14, | ||||
| 355 | + "cost": 1362140180.6980076, | ||||
| 356 | + "name": "pois_p3_N65536_K512" | ||||
| 357 | + }, | ||||
| 358 | + { | ||||
| 359 | + "config": "experiments/configs/robustness.yaml", | ||||
| 360 | + "cell": 32, | ||||
| 361 | + "cost": 1048576000.0, | ||||
| 362 | + "name": "exp_contam_p1_N262144_K1024" | ||||
| 363 | + } | ||||
| 364 | + ], | ||||
| 365 | + "M5M36": [ | ||||
| 366 | + { | ||||
| 367 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 368 | + "cell": 37, | ||||
| 369 | + "cost": 94906265624.25157, | ||||
| 370 | + "name": "logit_p8_N1048576_K8192" | ||||
| 371 | + }, | ||||
| 372 | + { | ||||
| 373 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 374 | + "cell": 44, | ||||
| 375 | + "cost": 46893744311.4964, | ||||
| 376 | + "name": "logit_p20_p20_N262144_K2048" | ||||
| 377 | + }, | ||||
| 378 | + { | ||||
| 379 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 380 | + "cell": 33, | ||||
| 381 | + "cost": 21794242891.16812, | ||||
| 382 | + "name": "logit_p3_N1048576_K32768" | ||||
| 383 | + }, | ||||
| 384 | + { | ||||
| 385 | + "config": "experiments/configs/robustness.yaml", | ||||
| 386 | + "cell": 10, | ||||
| 387 | + "cost": 21794242891.16812, | ||||
| 388 | + "name": "logit_misspec_p3_N1048576_K32768" | ||||
| 389 | + }, | ||||
| 390 | + { | ||||
| 391 | + "config": "experiments/configs/robustness.yaml", | ||||
| 392 | + "cell": 19, | ||||
| 393 | + "cost": 11863283203.031446, | ||||
| 394 | + "name": "logit_illcond_p8_N262144_K4096" | ||||
| 395 | + }, | ||||
| 396 | + { | ||||
| 397 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 398 | + "cell": 29, | ||||
| 399 | + "cost": 5448560722.79203, | ||||
| 400 | + "name": "pois_p3_N262144_K512" | ||||
| 401 | + }, | ||||
| 402 | + { | ||||
| 403 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 404 | + "cell": 8, | ||||
| 405 | + "cost": 1482910400.3789308, | ||||
| 406 | + "name": "logit_p8_N16384_K32" | ||||
| 407 | + }, | ||||
| 408 | + { | ||||
| 409 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 410 | + "cell": 12, | ||||
| 411 | + "cost": 1362140180.6980076, | ||||
| 412 | + "name": "logit_p3_N65536_K1024" | ||||
| 413 | + }, | ||||
| 414 | + { | ||||
| 415 | + "config": "experiments/configs/robustness.yaml", | ||||
| 416 | + "cell": 1, | ||||
| 417 | + "cost": 1362140180.6980076, | ||||
| 418 | + "name": "logit_misspec_p3_N65536_K1024" | ||||
| 419 | + } | ||||
| 420 | + ], | ||||
| 421 | + "M4BP48": [ | ||||
| 422 | + { | ||||
| 423 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 424 | + "cell": 39, | ||||
| 425 | + "cost": 94906265624.25157, | ||||
| 426 | + "name": "logit_p8_N1048576_K4096" | ||||
| 427 | + }, | ||||
| 428 | + { | ||||
| 429 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 430 | + "cell": 26, | ||||
| 431 | + "cost": 23726566406.062893, | ||||
| 432 | + "name": "logit_p8_N262144_K2048" | ||||
| 433 | + }, | ||||
| 434 | + { | ||||
| 435 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 436 | + "cell": 40, | ||||
| 437 | + "cost": 21794242891.16812, | ||||
| 438 | + "name": "logit_p3_N1048576_K2048" | ||||
| 439 | + }, | ||||
| 440 | + { | ||||
| 441 | + "config": "experiments/configs/robustness.yaml", | ||||
| 442 | + "cell": 22, | ||||
| 443 | + "cost": 11863283203.031446, | ||||
| 444 | + "name": "logit_illcond_p8_N262144_K512" | ||||
| 445 | + }, | ||||
| 446 | + { | ||||
| 447 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 448 | + "cell": 2, | ||||
| 449 | + "cost": 1482910400.3789308, | ||||
| 450 | + "name": "pois_p8_N16384_K256" | ||||
| 451 | + }, | ||||
| 452 | + { | ||||
| 453 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 454 | + "cell": 18, | ||||
| 455 | + "cost": 1362140180.6980076, | ||||
| 456 | + "name": "pois_p3_N65536_K128" | ||||
| 457 | + } | ||||
| 458 | + ], | ||||
| 459 | + "M4BP36": [ | ||||
| 460 | + { | ||||
| 461 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 462 | + "cell": 41, | ||||
| 463 | + "cost": 94906265624.25157, | ||||
| 464 | + "name": "logit_p8_N1048576_K2048" | ||||
| 465 | + }, | ||||
| 466 | + { | ||||
| 467 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 468 | + "cell": 28, | ||||
| 469 | + "cost": 23726566406.062893, | ||||
| 470 | + "name": "logit_p8_N262144_K1024" | ||||
| 471 | + }, | ||||
| 472 | + { | ||||
| 473 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 474 | + "cell": 42, | ||||
| 475 | + "cost": 21794242891.16812, | ||||
| 476 | + "name": "logit_p3_N1048576_K1024" | ||||
| 477 | + }, | ||||
| 478 | + { | ||||
| 479 | + "config": "experiments/configs/robustness.yaml", | ||||
| 480 | + "cell": 36, | ||||
| 481 | + "cost": 11863283203.031446, | ||||
| 482 | + "name": "logit_small_m_p8_N262144_K16384" | ||||
| 483 | + }, | ||||
| 484 | + { | ||||
| 485 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 486 | + "cell": 4, | ||||
| 487 | + "cost": 1482910400.3789308, | ||||
| 488 | + "name": "pois_p8_N16384_K128" | ||||
| 489 | + }, | ||||
| 490 | + { | ||||
| 491 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 492 | + "cell": 20, | ||||
| 493 | + "cost": 1362140180.6980076, | ||||
| 494 | + "name": "pois_p3_N65536_K64" | ||||
| 495 | + } | ||||
| 496 | + ], | ||||
| 497 | + "M2M32c": [ | ||||
| 498 | + { | ||||
| 499 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 500 | + "cell": 43, | ||||
| 501 | + "cost": 94906265624.25157, | ||||
| 502 | + "name": "logit_p8_N1048576_K1024" | ||||
| 503 | + }, | ||||
| 504 | + { | ||||
| 505 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 506 | + "cell": 30, | ||||
| 507 | + "cost": 23726566406.062893, | ||||
| 508 | + "name": "logit_p8_N262144_K512" | ||||
| 509 | + }, | ||||
| 510 | + { | ||||
| 511 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 512 | + "cell": 33, | ||||
| 513 | + "cost": 21794242891.16812, | ||||
| 514 | + "name": "pois_p3_N1048576_K32768" | ||||
| 515 | + }, | ||||
| 516 | + { | ||||
| 517 | + "config": "experiments/configs/robustness.yaml", | ||||
| 518 | + "cell": 37, | ||||
| 519 | + "cost": 11863283203.031446, | ||||
| 520 | + "name": "logit_small_m_p8_N262144_K8192" | ||||
| 521 | + }, | ||||
| 522 | + { | ||||
| 523 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 524 | + "cell": 6, | ||||
| 525 | + "cost": 1482910400.3789308, | ||||
| 526 | + "name": "pois_p8_N16384_K64" | ||||
| 527 | + }, | ||||
| 528 | + { | ||||
| 529 | + "config": "experiments/configs/robustness.yaml", | ||||
| 530 | + "cell": 0, | ||||
| 531 | + "cost": 1362140180.6980076, | ||||
| 532 | + "name": "logit_misspec_p3_N65536_K2048" | ||||
| 533 | + } | ||||
| 534 | + ], | ||||
| 535 | + "m4mm": [ | ||||
| 536 | + { | ||||
| 537 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 538 | + "cell": 35, | ||||
| 539 | + "cost": 94906265624.25157, | ||||
| 540 | + "name": "pois_p8_N1048576_K16384" | ||||
| 541 | + }, | ||||
| 542 | + { | ||||
| 543 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 544 | + "cell": 32, | ||||
| 545 | + "cost": 23726566406.062893, | ||||
| 546 | + "name": "logit_p8_N262144_K256" | ||||
| 547 | + }, | ||||
| 548 | + { | ||||
| 549 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 550 | + "cell": 34, | ||||
| 551 | + "cost": 21794242891.16812, | ||||
| 552 | + "name": "pois_p3_N1048576_K16384" | ||||
| 553 | + }, | ||||
| 554 | + { | ||||
| 555 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 556 | + "cell": 13, | ||||
| 557 | + "cost": 5931641601.515723, | ||||
| 558 | + "name": "logit_p8_N65536_K1024" | ||||
| 559 | + }, | ||||
| 560 | + { | ||||
| 561 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 562 | + "cell": 31, | ||||
| 563 | + "cost": 5448560722.79203, | ||||
| 564 | + "name": "logit_p3_N262144_K256" | ||||
| 565 | + }, | ||||
| 566 | + { | ||||
| 567 | + "config": "experiments/configs/robustness.yaml", | ||||
| 568 | + "cell": 16, | ||||
| 569 | + "cost": 2965820800.7578616, | ||||
| 570 | + "name": "logit_illcond_p8_N65536_K512" | ||||
| 571 | + }, | ||||
| 572 | + { | ||||
| 573 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 574 | + "cell": 3, | ||||
| 575 | + "cost": 340535045.1745019, | ||||
| 576 | + "name": "logit_p3_N16384_K128" | ||||
| 577 | + }, | ||||
| 578 | + { | ||||
| 579 | + "config": "experiments/configs/robustness.yaml", | ||||
| 580 | + "cell": 27, | ||||
| 581 | + "cost": 262144000.0, | ||||
| 582 | + "name": "exp_contam_p1_N65536_K4096" | ||||
| 583 | + } | ||||
| 584 | + ], | ||||
| 585 | + "m4mn": [ | ||||
| 586 | + { | ||||
| 587 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 588 | + "cell": 37, | ||||
| 589 | + "cost": 94906265624.25157, | ||||
| 590 | + "name": "pois_p8_N1048576_K8192" | ||||
| 591 | + }, | ||||
| 592 | + { | ||||
| 593 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 594 | + "cell": 24, | ||||
| 595 | + "cost": 23726566406.062893, | ||||
| 596 | + "name": "pois_p8_N262144_K4096" | ||||
| 597 | + }, | ||||
| 598 | + { | ||||
| 599 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 600 | + "cell": 36, | ||||
| 601 | + "cost": 21794242891.16812, | ||||
| 602 | + "name": "pois_p3_N1048576_K8192" | ||||
| 603 | + }, | ||||
| 604 | + { | ||||
| 605 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 606 | + "cell": 15, | ||||
| 607 | + "cost": 5931641601.515723, | ||||
| 608 | + "name": "logit_p8_N65536_K512" | ||||
| 609 | + }, | ||||
| 610 | + { | ||||
| 611 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 612 | + "cell": 22, | ||||
| 613 | + "cost": 5448560722.79203, | ||||
| 614 | + "name": "pois_p3_N262144_K8192" | ||||
| 615 | + }, | ||||
| 616 | + { | ||||
| 617 | + "config": "experiments/configs/robustness.yaml", | ||||
| 618 | + "cell": 17, | ||||
| 619 | + "cost": 2965820800.7578616, | ||||
| 620 | + "name": "logit_illcond_p8_N65536_K256" | ||||
| 621 | + }, | ||||
| 622 | + { | ||||
| 623 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 624 | + "cell": 5, | ||||
| 625 | + "cost": 340535045.1745019, | ||||
| 626 | + "name": "logit_p3_N16384_K64" | ||||
| 627 | + }, | ||||
| 628 | + { | ||||
| 629 | + "config": "experiments/configs/robustness.yaml", | ||||
| 630 | + "cell": 28, | ||||
| 631 | + "cost": 262144000.0, | ||||
| 632 | + "name": "exp_contam_p1_N65536_K1024" | ||||
| 633 | + } | ||||
| 634 | + ], | ||||
| 635 | + "m6ma": [ | ||||
| 636 | + { | ||||
| 637 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 638 | + "cell": 39, | ||||
| 639 | + "cost": 94906265624.25157, | ||||
| 640 | + "name": "pois_p8_N1048576_K4096" | ||||
| 641 | + }, | ||||
| 642 | + { | ||||
| 643 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 644 | + "cell": 26, | ||||
| 645 | + "cost": 23726566406.062893, | ||||
| 646 | + "name": "pois_p8_N262144_K2048" | ||||
| 647 | + }, | ||||
| 648 | + { | ||||
| 649 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 650 | + "cell": 38, | ||||
| 651 | + "cost": 21794242891.16812, | ||||
| 652 | + "name": "pois_p3_N1048576_K4096" | ||||
| 653 | + }, | ||||
| 654 | + { | ||||
| 655 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 656 | + "cell": 17, | ||||
| 657 | + "cost": 5931641601.515723, | ||||
| 658 | + "name": "logit_p8_N65536_K256" | ||||
| 659 | + }, | ||||
| 660 | + { | ||||
| 661 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 662 | + "cell": 23, | ||||
| 663 | + "cost": 5448560722.79203, | ||||
| 664 | + "name": "pois_p3_N262144_K4096" | ||||
| 665 | + }, | ||||
| 666 | + { | ||||
| 667 | + "config": "experiments/configs/robustness.yaml", | ||||
| 668 | + "cell": 18, | ||||
| 669 | + "cost": 2965820800.7578616, | ||||
| 670 | + "name": "logit_illcond_p8_N65536_K128" | ||||
| 671 | + }, | ||||
| 672 | + { | ||||
| 673 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 674 | + "cell": 7, | ||||
| 675 | + "cost": 340535045.1745019, | ||||
| 676 | + "name": "logit_p3_N16384_K32" | ||||
| 677 | + }, | ||||
| 678 | + { | ||||
| 679 | + "config": "experiments/configs/robustness.yaml", | ||||
| 680 | + "cell": 29, | ||||
| 681 | + "cost": 262144000.0, | ||||
| 682 | + "name": "exp_contam_p1_N65536_K256" | ||||
| 683 | + } | ||||
| 684 | + ], | ||||
| 685 | + "m6mb": [ | ||||
| 686 | + { | ||||
| 687 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 688 | + "cell": 41, | ||||
| 689 | + "cost": 94906265624.25157, | ||||
| 690 | + "name": "pois_p8_N1048576_K2048" | ||||
| 691 | + }, | ||||
| 692 | + { | ||||
| 693 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 694 | + "cell": 28, | ||||
| 695 | + "cost": 23726566406.062893, | ||||
| 696 | + "name": "pois_p8_N262144_K1024" | ||||
| 697 | + }, | ||||
| 698 | + { | ||||
| 699 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 700 | + "cell": 40, | ||||
| 701 | + "cost": 21794242891.16812, | ||||
| 702 | + "name": "pois_p3_N1048576_K2048" | ||||
| 703 | + }, | ||||
| 704 | + { | ||||
| 705 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 706 | + "cell": 19, | ||||
| 707 | + "cost": 5931641601.515723, | ||||
| 708 | + "name": "logit_p8_N65536_K128" | ||||
| 709 | + }, | ||||
| 710 | + { | ||||
| 711 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 712 | + "cell": 25, | ||||
| 713 | + "cost": 5448560722.79203, | ||||
| 714 | + "name": "pois_p3_N262144_K2048" | ||||
| 715 | + }, | ||||
| 716 | + { | ||||
| 717 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 718 | + "cell": 2, | ||||
| 719 | + "cost": 1482910400.3789308, | ||||
| 720 | + "name": "logit_p8_N16384_K256" | ||||
| 721 | + }, | ||||
| 722 | + { | ||||
| 723 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 724 | + "cell": 14, | ||||
| 725 | + "cost": 1362140180.6980076, | ||||
| 726 | + "name": "logit_p3_N65536_K512" | ||||
| 727 | + }, | ||||
| 728 | + { | ||||
| 729 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 730 | + "cell": 0, | ||||
| 731 | + "cost": 340535045.1745019, | ||||
| 732 | + "name": "logit_p3_N16384_K512" | ||||
| 733 | + }, | ||||
| 734 | + { | ||||
| 735 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 736 | + "cell": 1, | ||||
| 737 | + "cost": 340535045.1745019, | ||||
| 738 | + "name": "pois_p3_N16384_K256" | ||||
| 739 | + } | ||||
| 740 | + ], | ||||
| 741 | + "m5ma": [ | ||||
| 742 | + { | ||||
| 743 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 744 | + "cell": 43, | ||||
| 745 | + "cost": 94906265624.25157, | ||||
| 746 | + "name": "pois_p8_N1048576_K1024" | ||||
| 747 | + }, | ||||
| 748 | + { | ||||
| 749 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 750 | + "cell": 30, | ||||
| 751 | + "cost": 23726566406.062893, | ||||
| 752 | + "name": "pois_p8_N262144_K512" | ||||
| 753 | + }, | ||||
| 754 | + { | ||||
| 755 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 756 | + "cell": 42, | ||||
| 757 | + "cost": 21794242891.16812, | ||||
| 758 | + "name": "pois_p3_N1048576_K1024" | ||||
| 759 | + }, | ||||
| 760 | + { | ||||
| 761 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 762 | + "cell": 21, | ||||
| 763 | + "cost": 5931641601.515723, | ||||
| 764 | + "name": "logit_p8_N65536_K64" | ||||
| 765 | + }, | ||||
| 766 | + { | ||||
| 767 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 768 | + "cell": 27, | ||||
| 769 | + "cost": 5448560722.79203, | ||||
| 770 | + "name": "pois_p3_N262144_K1024" | ||||
| 771 | + }, | ||||
| 772 | + { | ||||
| 773 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 774 | + "cell": 4, | ||||
| 775 | + "cost": 1482910400.3789308, | ||||
| 776 | + "name": "logit_p8_N16384_K128" | ||||
| 777 | + }, | ||||
| 778 | + { | ||||
| 779 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 780 | + "cell": 16, | ||||
| 781 | + "cost": 1362140180.6980076, | ||||
| 782 | + "name": "logit_p3_N65536_K256" | ||||
| 783 | + }, | ||||
| 784 | + { | ||||
| 785 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 786 | + "cell": 1, | ||||
| 787 | + "cost": 340535045.1745019, | ||||
| 788 | + "name": "logit_p3_N16384_K256" | ||||
| 789 | + }, | ||||
| 790 | + { | ||||
| 791 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 792 | + "cell": 3, | ||||
| 793 | + "cost": 340535045.1745019, | ||||
| 794 | + "name": "pois_p3_N16384_K128" | ||||
| 795 | + } | ||||
| 796 | + ], | ||||
| 797 | + "m4md": [ | ||||
| 798 | + { | ||||
| 799 | + "config": "experiments/configs/robustness.yaml", | ||||
| 800 | + "cell": 23, | ||||
| 801 | + "cost": 47453132812.125786, | ||||
| 802 | + "name": "logit_illcond_p8_N1048576_K16384" | ||||
| 803 | + }, | ||||
| 804 | + { | ||||
| 805 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 806 | + "cell": 45, | ||||
| 807 | + "cost": 46893744311.4964, | ||||
| 808 | + "name": "logit_p20_p20_N262144_K1024" | ||||
| 809 | + }, | ||||
| 810 | + { | ||||
| 811 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 812 | + "cell": 19, | ||||
| 813 | + "cost": 5931641601.515723, | ||||
| 814 | + "name": "pois_p8_N65536_K128" | ||||
| 815 | + }, | ||||
| 816 | + { | ||||
| 817 | + "config": "experiments/configs/robustness.yaml", | ||||
| 818 | + "cell": 34, | ||||
| 819 | + "cost": 4194304000.0, | ||||
| 820 | + "name": "exp_contam_p1_N1048576_K16384" | ||||
| 821 | + } | ||||
| 822 | + ], | ||||
| 823 | + "m4ml": [ | ||||
| 824 | + { | ||||
| 825 | + "config": "experiments/configs/robustness.yaml", | ||||
| 826 | + "cell": 24, | ||||
| 827 | + "cost": 47453132812.125786, | ||||
| 828 | + "name": "logit_illcond_p8_N1048576_K8192" | ||||
| 829 | + }, | ||||
| 830 | + { | ||||
| 831 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 832 | + "cell": 46, | ||||
| 833 | + "cost": 46893744311.4964, | ||||
| 834 | + "name": "logit_p20_p20_N262144_K512" | ||||
| 835 | + }, | ||||
| 836 | + { | ||||
| 837 | + "config": "experiments/configs/main_poisson.yaml", | ||||
| 838 | + "cell": 21, | ||||
| 839 | + "cost": 5931641601.515723, | ||||
| 840 | + "name": "pois_p8_N65536_K64" | ||||
| 841 | + }, | ||||
| 842 | + { | ||||
| 843 | + "config": "experiments/configs/robustness.yaml", | ||||
| 844 | + "cell": 35, | ||||
| 845 | + "cost": 4194304000.0, | ||||
| 846 | + "name": "exp_contam_p1_N1048576_K4096" | ||||
| 847 | + } | ||||
| 848 | + ], | ||||
| 849 | + "M3BA24": [ | ||||
| 850 | + { | ||||
| 851 | + "config": "experiments/configs/robustness.yaml", | ||||
| 852 | + "cell": 25, | ||||
| 853 | + "cost": 47453132812.125786, | ||||
| 854 | + "name": "logit_illcond_p8_N1048576_K4096" | ||||
| 855 | + }, | ||||
| 856 | + { | ||||
| 857 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 858 | + "cell": 47, | ||||
| 859 | + "cost": 46893744311.4964, | ||||
| 860 | + "name": "logit_p20_p20_N262144_K256" | ||||
| 861 | + }, | ||||
| 862 | + { | ||||
| 863 | + "config": "experiments/configs/main_logistic.yaml", | ||||
| 864 | + "cell": 22, | ||||
| 865 | + "cost": 5448560722.79203, | ||||
| 866 | + "name": "logit_p3_N262144_K8192" | ||||
| 867 | + }, | ||||
| 868 | + { | ||||
| 869 | + "config": "experiments/configs/robustness.yaml", | ||||
| 870 | + "cell": 9, | ||||
| 871 | + "cost": 5448560722.79203, | ||||
| 872 | + "name": "logit_misspec_p3_N262144_K512" | ||||
| 873 | + } | ||||
| 874 | + ] | ||||
| 875 | + } | ||||
| 876 | +} | ||||
| \ No newline at end of file | |||||
added
experiments/collect_cluster.sh
+8 −0
@@ -0,0 +1,8 @@ | |||||
| 1 | +#!/bin/bash | ||||
| 2 | +# Pull finished raw results and logs from all nodes used by scatter_cluster.py | ||||
| 3 | +cd "$(dirname "$0")/.." | ||||
| 4 | +for n in $(python3 -c "import json;print(' '.join(json.load(open('experiments/cluster_assignment.json'))['nodes']))"); do | ||||
| 5 | + rsync -az --ignore-existing "$n:~/wp28/repo/results/raw/" results/raw/ 2>/dev/null & | ||||
| 6 | + rsync -az "$n:~/wp28/repo/logs/" logs/cluster/$n/ 2>/dev/null & | ||||
| 7 | +done; wait | ||||
| 8 | +echo "raw csv files: $(ls results/raw/*.csv 2>/dev/null | wc -l)" | ||||
added
experiments/configs/main_exponential.yaml
+11 −0
@@ -0,0 +1,11 @@ | |||||
| 1 | +# Main rate study, exponential(rate) model: exact MLE, all (r,q) cells, large N affordable. | ||||
| 2 | +seed_base: 20261028 | ||||
| 3 | +grids: | ||||
| 4 | + - model: exponential | ||||
| 5 | + label: exp | ||||
| 6 | + N: [16384, 65536, 262144, 1048576, 4194304] | ||||
| 7 | + m: [8, 16, 32, 64, 128, 256, 512, 1024] | ||||
| 8 | + reps: 2000 | ||||
| 9 | + splits_max: 3 | ||||
| 10 | + q_max: 2 | ||||
| 11 | + dgp: {theta0: 1.0} | ||||
added
experiments/configs/main_logistic.yaml
+20 −0
@@ -0,0 +1,20 @@ | |||||
| 1 | +seed_base: 20261028 | ||||
| 2 | +grids: | ||||
| 3 | + - model: logistic | ||||
| 4 | + label: logit | ||||
| 5 | + p: [3, 8] | ||||
| 6 | + N: [16384, 65536, 262144, 1048576] | ||||
| 7 | + m: [32, 64, 128, 256, 512, 1024] | ||||
| 8 | + reps: 1000 | ||||
| 9 | + splits_max: 3 | ||||
| 10 | + q_max: 2 | ||||
| 11 | + min_m_over_p: 8 | ||||
| 12 | + - model: logistic | ||||
| 13 | + label: logit_p20 | ||||
| 14 | + p: [20] | ||||
| 15 | + N: [262144, 1048576] | ||||
| 16 | + m: [128, 256, 512, 1024, 2048] | ||||
| 17 | + reps: 500 | ||||
| 18 | + splits_max: 3 | ||||
| 19 | + q_max: 2 | ||||
| 20 | + min_m_over_p: 6 | ||||
added
experiments/configs/main_poisson.yaml
+11 −0
@@ -0,0 +1,11 @@ | |||||
| 1 | +seed_base: 20261028 | ||||
| 2 | +grids: | ||||
| 3 | + - model: poisson | ||||
| 4 | + label: pois | ||||
| 5 | + p: [3, 8] | ||||
| 6 | + N: [16384, 65536, 262144, 1048576] | ||||
| 7 | + m: [32, 64, 128, 256, 512, 1024] | ||||
| 8 | + reps: 1000 | ||||
| 9 | + splits_max: 3 | ||||
| 10 | + q_max: 2 | ||||
| 11 | + min_m_over_p: 8 | ||||
added
experiments/configs/robustness.yaml
+29 −0
@@ -0,0 +1,29 @@ | |||||
| 1 | +seed_base: 20261028 | ||||
| 2 | +grids: | ||||
| 3 | + - model: logistic | ||||
| 4 | + label: logit_misspec | ||||
| 5 | + p: [3] | ||||
| 6 | + N: [65536, 262144, 1048576] | ||||
| 7 | + m: [32, 64, 128, 256, 512] | ||||
| 8 | + reps: 1000 | ||||
| 9 | + dgp: {kind: misspecified, omitted: 0.5} | ||||
| 10 | + - model: logistic | ||||
| 11 | + label: logit_illcond | ||||
| 12 | + p: [8] | ||||
| 13 | + N: [65536, 262144, 1048576] | ||||
| 14 | + m: [64, 128, 256, 512] | ||||
| 15 | + reps: 500 | ||||
| 16 | + dgp: {cond: 100.0} | ||||
| 17 | + - model: exponential | ||||
| 18 | + label: exp_contam | ||||
| 19 | + N: [65536, 262144, 1048576] | ||||
| 20 | + m: [16, 64, 256] | ||||
| 21 | + reps: 1000 | ||||
| 22 | + dgp: {kind: contaminated, eps: 0.01, scale: 10.0} | ||||
| 23 | + - model: logistic | ||||
| 24 | + label: logit_small_m | ||||
| 25 | + p: [8] | ||||
| 26 | + N: [262144] | ||||
| 27 | + m: [16, 24, 32, 48] | ||||
| 28 | + reps: 500 | ||||
| 29 | + min_m_over_p: 2 | ||||
added
experiments/run_monte_carlo.py
+168 −0
@@ -0,0 +1,168 @@ | |||||
| 1 | +#!/usr/bin/env python3 | ||||
| 2 | +"""Monte Carlo driver for WP28. One *cell* = (model, p, N, K, DGP options, reps, seed). Every estimator is computed on the | ||||
| 3 | +same simulated sample and the same partition (paired design). Output: one CSV per cell in results/raw/ with one row per | ||||
| 4 | +replication, columns <estimator>_<j> for parameter j, plus diagnostics and timings, and a JSON manifest. | ||||
| 5 | + | ||||
| 6 | +Usage | ||||
| 7 | + python3 experiments/run_monte_carlo.py --config experiments/configs/main_exponential.yaml [--cells 0,3,5] [--reps 200] | ||||
| 8 | + python3 experiments/run_monte_carlo.py --list --config ... | ||||
| 9 | +""" | ||||
| 10 | +from __future__ import annotations | ||||
| 11 | +import argparse, hashlib, json, os, pathlib, platform, socket, sys, time | ||||
| 12 | +import numpy as np, yaml | ||||
| 13 | + | ||||
| 14 | +ROOT = pathlib.Path(__file__).resolve().parents[1] | ||||
| 15 | +sys.path.insert(0, str(ROOT / "src")) | ||||
| 16 | +from pamle.likelihoods import make_model, gather_blocks, STATUS_OK # noqa: E402 | ||||
| 17 | +from pamle.algorithms import estimate_all, aggregated_score_hessian, local_split_fits # noqa: E402 | ||||
| 18 | +from pamle.partitioning import skewed_sizes # noqa: E402 | ||||
| 19 | +from pamle import __version__ # noqa: E402 | ||||
| 20 | + | ||||
| 21 | +SE_ESTIMATORS = ("central", "r1_q0", "r1_q1", "r1_q2", "r2_q0", "r2_q1", "r3_q1", "cs_q1") | ||||
| 22 | + | ||||
| 23 | + | ||||
| 24 | +def cell_seed(base: int, cell: dict) -> int: | ||||
| 25 | + key = json.dumps({k: cell[k] for k in sorted(cell) if k not in ("reps",)}, sort_keys=True, default=str) | ||||
| 26 | + return (base + int(hashlib.sha256(key.encode()).hexdigest(), 16)) % (2 ** 63) | ||||
| 27 | + | ||||
| 28 | + | ||||
| 29 | +def expand_cells(cfg: dict): | ||||
| 30 | + cells = [] | ||||
| 31 | + for g in cfg["grids"]: | ||||
| 32 | + for N in g["N"]: | ||||
| 33 | + for m in g["m"]: | ||||
| 34 | + if N % m: | ||||
| 35 | + continue | ||||
| 36 | + for p in g.get("p", [1]): | ||||
| 37 | + if m < g.get("min_m_over_p", 4) * p: | ||||
| 38 | + continue | ||||
| 39 | + c = {"model": g["model"], "p": p, "N": int(N), "K": int(N // m), "m": int(m), "reps": int(g["reps"]), | ||||
| 40 | + "splits_max": int(g.get("splits_max", 3)), "q_max": int(g.get("q_max", 2)), | ||||
| 41 | + "dgp": g.get("dgp", {}), "label": g.get("label", g["model"]), "extras": bool(g.get("extras", True))} | ||||
| 42 | + cells.append(c) | ||||
| 43 | + return cells | ||||
| 44 | + | ||||
| 45 | + | ||||
| 46 | +def true_theta(cell): | ||||
| 47 | + p = cell["p"]; dgp = cell["dgp"] | ||||
| 48 | + if cell["model"] == "exponential": | ||||
| 49 | + return np.array([float(dgp.get("theta0", 1.0))]) | ||||
| 50 | + if "theta0" in dgp: | ||||
| 51 | + return np.asarray(dgp["theta0"], float) | ||||
| 52 | + # default: intercept 0.3, slopes alternating +-0.5 decaying | ||||
| 53 | + th = np.array([0.3] + [0.5 * (-1) ** j / (1 + 0.15 * j) for j in range(p - 1)]) | ||||
| 54 | + if cell["model"] == "poisson": | ||||
| 55 | + th = th * 0.5 | ||||
| 56 | + return th | ||||
| 57 | + | ||||
| 58 | + | ||||
| 59 | +def simulate(model, cell, theta0, rng): | ||||
| 60 | + dgp = cell["dgp"]; N = cell["N"]; p = cell["p"] | ||||
| 61 | + kind = dgp.get("kind", "well_specified") | ||||
| 62 | + if cell["model"] == "exponential": | ||||
| 63 | + x = rng.exponential(1.0 / theta0[0], size=N) | ||||
| 64 | + if kind == "contaminated": | ||||
| 65 | + eps = float(dgp.get("eps", 0.01)); scale = float(dgp.get("scale", 10.0)) | ||||
| 66 | + mask = rng.random(N) < eps | ||||
| 67 | + x[mask] = rng.exponential(scale / theta0[0], size=mask.sum()) | ||||
| 68 | + return {"x": x} | ||||
| 69 | + cov = None | ||||
| 70 | + if dgp.get("cond", None) is not None and p > 1: # covariate covariance with prescribed condition number | ||||
| 71 | + q = p - 1; ev = np.geomspace(1.0, 1.0 / float(dgp["cond"]), q) | ||||
| 72 | + cov = np.diag(ev) | ||||
| 73 | + data = model.simulate(rng, N, theta0, cov=cov) | ||||
| 74 | + if kind == "misspecified": # omitted quadratic term in the index | ||||
| 75 | + X = data["X"]; eta = X @ theta0 + float(dgp.get("omitted", 0.5)) * (X[:, 1] ** 2 - 1.0) | ||||
| 76 | + if cell["model"] == "logistic": | ||||
| 77 | + data["y"] = (rng.random(N) < 1 / (1 + np.exp(-eta))).astype(float) | ||||
| 78 | + else: | ||||
| 79 | + data["y"] = rng.poisson(np.exp(eta)).astype(float) | ||||
| 80 | + return data | ||||
| 81 | + | ||||
| 82 | + | ||||
| 83 | +def run_cell(cell, out_dir: pathlib.Path, base_seed: int, reps_override=None, verbose=True): | ||||
| 84 | + model = make_model(cell["model"], cell["p"]); p = model.p | ||||
| 85 | + reps = reps_override or cell["reps"] | ||||
| 86 | + seed = cell_seed(base_seed, cell); rng = np.random.default_rng(seed) | ||||
| 87 | + theta0 = true_theta(cell) | ||||
| 88 | + N, K, m = cell["N"], cell["K"], cell["m"] | ||||
| 89 | + skew = float(cell["dgp"].get("skew", 0.0)) | ||||
| 90 | + name = f'{cell["label"]}_p{p}_N{N}_K{K}' | ||||
| 91 | + rows = []; t_start = time.time() | ||||
| 92 | + for b in range(reps): | ||||
| 93 | + data = simulate(model, cell, theta0, rng) | ||||
| 94 | + t0 = time.perf_counter() | ||||
| 95 | + if skew > 1.0: | ||||
| 96 | + # unequal contiguous blocks; estimate_all needs equal shapes -> use K blocks of sizes from skewed_sizes, | ||||
| 97 | + # handled by padding is not possible: we instead run a dedicated weighted/naive comparison (see run_skewed) | ||||
| 98 | + raise NotImplementedError("use run_skewed.py for unequal partitions") | ||||
| 99 | + db = gather_blocks(data, np.arange(N).reshape(K, m)) | ||||
| 100 | + out, diag = estimate_all(model, db, splits_max=cell["splits_max"], q_max=cell["q_max"], extras=cell["extras"]) | ||||
| 101 | + t_est = time.perf_counter() - t0 | ||||
| 102 | + row = {"rep": b, "seed": seed, "N": N, "K": K, "m": m, "p": p, "model": cell["model"], "label": cell["label"], | ||||
| 103 | + "t_estimate_s": t_est, "central_status": diag["central_status"]} | ||||
| 104 | + for s, cnt in diag["local_status_counts"].items(): | ||||
| 105 | + row[f"local_fail_s{s}"] = int(sum(cnt[1:])) | ||||
| 106 | + for r in range(1, cell["splits_max"] + 1): | ||||
| 107 | + row[f"valid_r{r}"] = int(diag[f"valid_r{r}"].sum()) | ||||
| 108 | + nd = diag[f"newton_r{r}"] | ||||
| 109 | + for q, d in enumerate(nd, start=1): | ||||
| 110 | + row[f"cond_r{r}_q{q}"] = d["cond"]; row[f"step_r{r}_q{q}"] = d["step_norm"]; row[f"grad_r{r}_q{q}"] = d["grad_norm"] | ||||
| 111 | + for k, v in out.items(): | ||||
| 112 | + for j in range(p): | ||||
| 113 | + row[f"{k}_{j}"] = float(v[j]) | ||||
| 114 | + # standard errors (Fisher at the estimate; sandwich at the centralized estimate) | ||||
| 115 | + for k in SE_ESTIMATORS: | ||||
| 116 | + if k in out and np.all(np.isfinite(out[k])): | ||||
| 117 | + g, H, gk, Hk = aggregated_score_hessian(model, db, out[k]) | ||||
| 118 | + try: | ||||
| 119 | + cov = np.linalg.inv(-H) | ||||
| 120 | + for j in range(p): | ||||
| 121 | + row[f"se_{k}_{j}"] = float(np.sqrt(max(cov[j, j], 0) / N)) | ||||
| 122 | + except np.linalg.LinAlgError: | ||||
| 123 | + pass | ||||
| 124 | + th = np.broadcast_to(out["central"], (K, p)) | ||||
| 125 | + J = model.outer_scores(th, db).mean(axis=0); g, H, _, _ = aggregated_score_hessian(model, db, out["central"]) | ||||
| 126 | + Hi = np.linalg.inv(H); S = Hi @ J @ Hi | ||||
| 127 | + for j in range(p): | ||||
| 128 | + row[f"se_sand_central_{j}"] = float(np.sqrt(max(S[j, j], 0) / N)) | ||||
| 129 | + for j in range(p): | ||||
| 130 | + row[f"theta0_{j}"] = float(theta0[j]) | ||||
| 131 | + rows.append(row) | ||||
| 132 | + if verbose and (b + 1) % max(1, reps // 10) == 0: | ||||
| 133 | + print(f"[{name}] {b+1}/{reps} reps, {time.time()-t_start:.1f}s", flush=True) | ||||
| 134 | + import pandas as pd | ||||
| 135 | + df = pd.DataFrame(rows) | ||||
| 136 | + out_dir.mkdir(parents=True, exist_ok=True) | ||||
| 137 | + f = out_dir / f"{name}.csv"; df.to_csv(f, index=False) | ||||
| 138 | + sha = hashlib.sha256(f.read_bytes()).hexdigest() | ||||
| 139 | + manifest = {"run_id": name, "cell": cell, "reps": reps, "seed": seed, "pamle_version": __version__, | ||||
| 140 | + "python": platform.python_version(), "numpy": np.__version__, "host": socket.gethostname(), | ||||
| 141 | + "machine": platform.machine(), "wall_seconds": time.time() - t_start, "raw_output_sha256": sha, | ||||
| 142 | + "git_commit": os.popen(f"git -C {ROOT} rev-parse --short HEAD").read().strip(), "status": "completed"} | ||||
| 143 | + (out_dir / f"{name}.manifest.json").write_text(json.dumps(manifest, indent=1)) | ||||
| 144 | + return f, manifest | ||||
| 145 | + | ||||
| 146 | + | ||||
| 147 | +def main(): | ||||
| 148 | + ap = argparse.ArgumentParser() | ||||
| 149 | + ap.add_argument("--config", required=True) | ||||
| 150 | + ap.add_argument("--cells", default=None, help="comma-separated cell indices (default all)") | ||||
| 151 | + ap.add_argument("--reps", type=int, default=None) | ||||
| 152 | + ap.add_argument("--out", default=str(ROOT / "results" / "raw")) | ||||
| 153 | + ap.add_argument("--list", action="store_true") | ||||
| 154 | + a = ap.parse_args() | ||||
| 155 | + cfg = yaml.safe_load(open(a.config)) | ||||
| 156 | + cells = expand_cells(cfg) | ||||
| 157 | + if a.list: | ||||
| 158 | + for i, c in enumerate(cells): | ||||
| 159 | + print(i, c["label"], "p", c["p"], "N", c["N"], "K", c["K"], "m", c["m"], "reps", c["reps"]) | ||||
| 160 | + return | ||||
| 161 | + idx = range(len(cells)) if a.cells is None else [int(x) for x in a.cells.split(",")] | ||||
| 162 | + for i in idx: | ||||
| 163 | + f, man = run_cell(cells[i], pathlib.Path(a.out), int(cfg.get("seed_base", 20261028)), a.reps) | ||||
| 164 | + print("wrote", f, f"{man['wall_seconds']:.1f}s") | ||||
| 165 | + | ||||
| 166 | + | ||||
| 167 | +if __name__ == "__main__": | ||||
| 168 | + main() | ||||
added
experiments/scatter_cluster.py
+58 −0
@@ -0,0 +1,58 @@ | |||||
| 1 | +#!/usr/bin/env python3 | ||||
| 2 | +"""Distribute Monte Carlo cells over MacLustr nodes (greedy by estimated cost, weighted by worker slots). | ||||
| 3 | + | ||||
| 4 | +Writes experiments/cluster_assignment.json, syncs the repo to each node, and launches `xargs -P` worker pools. | ||||
| 5 | +Usage: python3 experiments/scatter_cluster.py --configs a.yaml b.yaml --launch | ||||
| 6 | +Collect later with experiments/collect_cluster.sh. | ||||
| 7 | +""" | ||||
| 8 | +import argparse, json, os, pathlib, subprocess, sys | ||||
| 9 | +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "experiments")) | ||||
| 10 | +import yaml | ||||
| 11 | +from run_monte_carlo import expand_cells | ||||
| 12 | + | ||||
| 13 | +ROOT = pathlib.Path(__file__).resolve().parents[1] | ||||
| 14 | +NODES = {"M3U96a": 6, "M3U96b": 6, "M2U64": 4, "M4M64a": 4, "M4M64b": 4, "M4M64c": 4, "M5M36": 4, "M4BP48": 3, "M4BP36": 3, | ||||
| 15 | + "M2M32c": 3, "m4mm": 3, "m4mn": 3, "m6ma": 3, "m6mb": 3, "m5ma": 3, "m4md": 2, "m4ml": 2, "M3BA24": 2} | ||||
| 16 | + | ||||
| 17 | + | ||||
| 18 | +def cost(c): | ||||
| 19 | + extras = 1.6 if c["extras"] else 1.0 | ||||
| 20 | + return c["N"] * (c["p"] ** 1.5) * c["reps"] * (1 + 0.5 * c["splits_max"]) * extras | ||||
| 21 | + | ||||
| 22 | + | ||||
| 23 | +def main(): | ||||
| 24 | + ap = argparse.ArgumentParser(); ap.add_argument("--configs", nargs="+", required=True); ap.add_argument("--launch", action="store_true") | ||||
| 25 | + a = ap.parse_args() | ||||
| 26 | + jobs = [] | ||||
| 27 | + for cfg in a.configs: | ||||
| 28 | + cells = expand_cells(yaml.safe_load(open(cfg))) | ||||
| 29 | + for i, c in enumerate(cells): | ||||
| 30 | + jobs.append({"config": os.path.relpath(cfg, ROOT), "cell": i, "cost": cost(c), "name": f'{c["label"]}_p{c["p"]}_N{c["N"]}_K{c["K"]}'}) | ||||
| 31 | + jobs.sort(key=lambda j: -j["cost"]) | ||||
| 32 | + load = {n: 0.0 for n in NODES}; assign = {n: [] for n in NODES} | ||||
| 33 | + for j in jobs: | ||||
| 34 | + n = min(NODES, key=lambda k: load[k] / NODES[k]) | ||||
| 35 | + assign[n].append(j); load[n] += j["cost"] | ||||
| 36 | + total = sum(j["cost"] for j in jobs) | ||||
| 37 | + print(f"{len(jobs)} jobs; est. share per node (%):", {n: round(100 * load[n] / total, 1) for n in NODES}) | ||||
| 38 | + json.dump({"nodes": NODES, "assignment": assign}, open(ROOT / "experiments" / "cluster_assignment.json", "w"), indent=1) | ||||
| 39 | + if not a.launch: | ||||
| 40 | + return | ||||
| 41 | + for n, js in assign.items(): | ||||
| 42 | + if not js: | ||||
| 43 | + continue | ||||
| 44 | + script = "\n".join(f'{j["config"]} {j["cell"]}' for j in js) + "\n" | ||||
| 45 | + (ROOT / "logs" / f"jobs_{n}.txt").write_text(script) | ||||
| 46 | + subprocess.run(["rsync", "-az", "--exclude", "results", "--exclude", "logs", "--exclude", "references", "--exclude", "paper", | ||||
| 47 | + "--exclude", ".git", f"{ROOT}/", f"{n}:~/wp28/repo/"], check=True) | ||||
| 48 | + subprocess.run(["scp", "-q", str(ROOT / "logs" / f"jobs_{n}.txt"), f"{n}:~/wp28/jobs.txt"], check=True) | ||||
| 49 | + W = NODES[n] | ||||
| 50 | + cmd = (f"cd ~/wp28/repo && mkdir -p logs results/raw && nohup sh -c 'cat ~/wp28/jobs.txt | xargs -P {W} -L 1 sh -c " | ||||
| 51 | + f"\"OMP_NUM_THREADS=2 ~/wp28/venv/bin/python experiments/run_monte_carlo.py --config \\$0 --cells \\$1 " | ||||
| 52 | + f">> logs/cell_\\$(basename \\$0 .yaml)_\\$1.log 2>&1\"' > logs/pool.log 2>&1 &") | ||||
| 53 | + subprocess.run(["ssh", n, cmd], check=True) | ||||
| 54 | + print(f"launched {len(js)} jobs on {n} with {W} workers") | ||||
| 55 | + | ||||
| 56 | + | ||||
| 57 | +if __name__ == "__main__": | ||||
| 58 | + main() | ||||
added
literature/PRIORITY_AND_GAP.md
+56 −0
@@ -0,0 +1,56 @@ | |||||
| 1 | +# PRIORITY_AND_GAP — WP28 (literature cutoff 2026-10-28) | ||||
| 2 | + | ||||
| 3 | +Sources: four theorem-level verification tasks (audits A–D in this directory; 56 rows in `PRIOR_ART_MATRIX.csv`, 24 of them read in | ||||
| 4 | +full text, the rest at abstract/metadata level and marked so). Page/theorem numbers refer to the versions read (arXiv or accepted | ||||
| 5 | +manuscripts when publisher pages were bot-blocked). | ||||
| 6 | + | ||||
| 7 | +## 1. Verified anchors (theorem level) | ||||
| 8 | +| Paper | What it proves (our notation: K blocks of size m = N/K) | Rounds / uplink | Read | | ||||
| 9 | +|---|---|---|---| | ||||
| 10 | +| Zhang–Duchi–Wainwright 2013 (JMLR) | AVGM MSE O(1/N + 1/m²) ⇒ K ≲ √N (eq. 10); SAVGM = (θ̄₁ − rθ̄₂)/(1−r) with one random r-subsample, MSE O(1/N + 1/(r(1−r)²m³)), leading constant inflated by (2+3r)/(1−r)², K ≲ N^{2/3} (Thm 4, eq. 11). No jackknife, no Newton, multiple subsamples "open". | 1 / p (2p) | full | | ||||
| 11 | +| Huang–Huo 2019 (Math. Prog.) | Plain average + ONE aggregated-Hessian Newton step: √N-normal and equivalent to the full M-estimator when k = O(√N) (Thm 3.4); MSE ≤ 2TrΣ/N + O(N^{-2}) + O(k⁴N^{-4}) (Thm 3.5) — the k⁴N^{-4} term implicitly allows k = o(N^{3/4}) (not stated). No bias correction. | 2 / p + p² | arXiv v2 | | ||||
| 12 | +| Jordan–Lee–Yang 2019 (JASA) | CSL: machine-1 Hessian, gradients only; ‖θ̃ − θ̂‖ ≤ C(‖θ̄−θ̂‖² + (N^{-1/2} + m^{-1/2})‖θ̄−θ̂‖) (Thm 3.1/3.3); contraction C/√m per round; Cor 3.4 sandwich expansion. One round from the average ⇒ K = o(N^{2/3}) (translation). | t / p | arXiv v3 | | ||||
| 13 | +| Fan–Guo–Wang 2023 (JASA, CEASE) | Rounds frontier (p. 16, verbatim): one single-machine-Hessian step from the average matches θ̂ iff n³ ≫ N·polylog ⇒ K = o(N^{2/3}); t steps ⇒ K = o(N^{(t+1)/(t+2)}); averaged local-Hessian step in the linear model ⇒ K = o(N^{3/4}). | t / p | arXiv v2 + PMC | | ||||
| 14 | +| Zhu–Li–Wang 2021 (JCGS) | TWLSE: average → one LOCAL Newton step per machine → local-Hessian-weighted average: efficient iff m/N^{1/4} → ∞, "allows K = O(N^{3/4})" (Thm 2); iterated version with m/log N → ∞ rounds (Prop 2); unequal n_k. | 2 / p + p² | arXiv v4 | | ||||
| 15 | +| Gu–Chen 2023 (JMLR) | Analytic (Rilstone-type) local bias correction with sample splitting, then Hessian-weighted average: χ² limit if K = o(m²) ⇔ K = o(N^{2/3}) (Thms 4–6); beats the SAVGM constant; quasi-likelihood/sandwich; heterogeneity. | 1 / p + p² | arXiv v1 | | ||||
| 16 | +| Wu–Zhu–Wang 2023 (Statistica Sinica) | Delete-one jackknife within each subsample, then average: bias O(m^{-2}) + O(N^{-1}), normal under N = o(m⁴) (⇔ K = o(N^{3/4})) for smooth functions of means (Thm 2, 4). | 1 / p | arXiv v1 | | ||||
| 17 | +| Yin 2022 (arXiv 2207.06253) | Bias-adjusted one-step estimator from the plain average (analytic removal of quadratic terms): efficient iff K = o(m²) ⇔ K = o(N^{2/3}) (Thm 3.2); Fisher information from gradients with error O_p(m^{-1} + K^{-1}). | 2 / p + summaries | arXiv v1 | | ||||
| 18 | +| Dhaene–Jochmans 2015 (RES) | Half-panel jackknife 2θ̂ − θ̄_{1/2}; general h-th order split-panel jackknife with Vandermonde-type weights (eqs. 5.1–5.3); bias-optimal partitions G = {2, 3, …, h+1}; first-order variance unchanged; higher-order variance left open. Panel, not distributed. | — | manuscript | | ||||
| 19 | +| Hahn–Hughes–Kuersteiner–Newey 2024 (QE) | Split-sample bias estimates are inconsistent (O_p(1)); split-sample corrections have twice the higher-order variance of delete-one (Props 5–7). | — | full | | ||||
| 20 | +| Volgushev–Chao–Cheng 2019 (AoS) | Plain averaging of quantile regressions: sufficient S = o(√N/log N), NECESSARY S = o(√N) via an explicit worst-case model (Thm 3.2, Cor 3.3). Template for sharpness statements. | 1 / p | arXiv + NSF | | ||||
| 21 | +| Lin–Xi 2011; Rosenblatt–Nadler 2016; Battey et al. 2018; Shamir et al. 2014 (DANE); Wang et al. 2017 (EDSL); Chen–Liu–Zhang 2022 (FONE); Shang–Cheng 2017; Gao et al. 2022 (review) | One-shot weighting (K ≪ N^{1/4}); bias of the average = m × centralized bias; √N barrier for plain averaging; local-Hessian contraction (β/λ)² ≍ 1/m per round; gradient-only multi-round; sharp machine counts in nonparametrics; review confirming "one-step C₁/N + O(m^{-4} + N^{-2})" and no (order, rounds) frontier as of 2022. | | full/abstract | | ||||
| 22 | + | ||||
| 23 | +## 2. Claim-by-claim map | ||||
| 24 | +| # | Claim in the prior draft / mandate | Status after audit | | ||||
| 25 | +|---|---|---| | ||||
| 26 | +| (i) | Half-block (and quarter-block) jackknife of local MLEs cancels the O(1/m) (and O(1/m²)) bias while preserving the first-order influence term | **Known in components**: the extrapolation algebra is Quenouille (1956) / Schucany–Gray–Owen (1971) / Dhaene–Jochmans (2015, panel); "debias locally then average" exists with analytic corrections (Gu–Chen 2023; Yin 2022), delete-one jackknife (Wu–Zhu–Wang 2023) and one-level subsampling (ZDW 2013 SAVGM). **Possibly new**: the general r-th order *nested split* construction of local MLEs with the influence-preservation argument and explicit remainder constants A_r (Lemma 3) in the distributed setting; r ≥ 3 not found anywhere. | | ||||
| 27 | +| (ii) | Half-block jackknife + one aggregated-Hessian Newton step ⇒ first-order equivalence for K = o(N^{7/8}) | **Not found**; closest: Huang–Huo (plain average + one step, stated √N, implicit 3/4), Zhu–Li–Wang (3/4 with local steps), Yin (2/3 with analytic adjustment), CEASE (2/3 single-machine). The exponent 7/8 appears nowhere. **Demonstrably new with specified proof** (Theorem 1, cell (2,1)). | | ||||
| 28 | +| (iii) | General (r, q) frontier K = o(N^{1 − 1/(2^{q+1} r)}) | **Not found**; rounds-only frontiers exist (CEASE (t+1)/(t+2) with a single-machine Hessian; JLY ⌈log K/log m⌉ rounds). **Demonstrably new with specified proof** (Theorem 1). The CEASE rounds frontier is recovered exactly as the single-machine-Hessian case of our Theorem 3 (α = 1 − 1/(2r + q), r = 1). | | ||||
| 29 | +| (iv) | Exact exponential boundary limit −4θ0c⁴ at K ~ cN^{7/8} | **New** (sharpness for a corrected construction). Generalised to all (r, q): −θ0(β_r c^r)^{2^q} (Theorem 2). Necessity results in the literature concern plain averaging only (Volgushev et al.; Shang–Cheng). | | ||||
| 30 | +| (v) | Inexact / stale / single-machine / quantized Hessians change the admissible exponent | **Partially known** (JLY factor m^{-1/2}; DANE/CEASE factor 1/m; Zhu–Li–Wang 3/4) but **never expressed as an exponent frontier nor with byte/bit accounting**; the quantized-Hessian statistical statement was not found. Theorem 3 unifies and extends. | | ||||
| 31 | +| (vi) | Unequal block sizes; misspecification | **Known** as extensions (Zhu–Li–Wang, Gu–Chen, Lin–Xi, JLY Cor 3.4). Present as Proposition 4 / Remark, not as a contribution. | | ||||
| 32 | +| (vii) | Communication–computation decision rule (one more local level vs one more round) | **No explicit rule found**; only qualitative remarks. Proposition 5 is a *conditional cost-model* result calibrated on measured hardware, labelled as such. | | ||||
| 33 | +| (viii) | Communication accounting in rounds, messages, numbers and bytes | Only O(p)/O(p²) remarks in the literature; our accounting is bookkeeping, not a theorem. | | ||||
| 34 | + | ||||
| 35 | +## 3. Candidate contributions | ||||
| 36 | +- **A (strongest, GO):** the joint local-order / global-round frontier (Theorem 1) with its sharpness in an exact model (Theorem 2) and the inexact-derivative extension that recovers the CSL/CEASE and averaged-local-Hessian cases and adds quantization/bit requirements (Theorem 3). Novelty: no joint frontier, no sharpness for corrected constructions, no bit-precision statement in the audited literature. | ||||
| 37 | +- **B (feasible backup, GO as supporting material):** the general nested-split extrapolation family with explicit remainder/amplification constants A_r, exact finite-sample bias and variance identities in the exponential model, the Newton basin of attraction, and the finite-sample comparison with analytic (Gu–Chen-type) and delete-one corrections. Novelty: incremental; positioned as the local building block. | ||||
| 38 | +- **C (negative results, GO):** (a) Proposition: higher-order split corrections lose in RMSE to lower orders for moderate m and have positive probability of leaving the Newton basin (exponential model, exact); (b) in exponential families with aggregable sufficient statistics (and in the exponential model for the full CSL surrogate) the approximate protocols are dominated by exact pooling — no computational lower bound is claimed; (c) unweighted averaging with unequal sizes is not first-order equivalent without a Newton step. | ||||
| 39 | + | ||||
| 40 | +## 4. Decisions | ||||
| 41 | +| Proposed claim | Decision | | ||||
| 42 | +|---|---| | ||||
| 43 | +| "Jackknife + Newton is a novel estimator" | **DROP** (components known: HH19, ZDW13, DJ15, GC23, WZW23). Rephrase as "a known local correction and a known global correction, analysed jointly for the first time". | | ||||
| 44 | +| K = o(N^{7/8}) for (2,1) and the general α(r, q) | **GO** (Theorem 1) with explicit comparison table against HH19 (√N stated / 3/4 implicit), ZLW21 (3/4), GC23 (2/3), Yin22 (2/3), CEASE (rounds frontier). | | ||||
| 45 | +| Sharpness (−4θ0c⁴ and general) | **GO** (Theorem 2), scoped as "for this construction, in this model"; cite Volgushev et al. as the template and state that it is not a universal lower bound. | | ||||
| 46 | +| Inexact Hessian frontier + quantization bits | **GO** (Theorem 3); cite JLY Thm 3.1/3.3, DANE Thm 2, CEASE Thm 4.1, ZLW Thm 2, Alimisis et al. (optimization only). | | ||||
| 47 | +| Heterogeneous sizes / misspecification | **REVISE** → extension Proposition 4 + remark, with citations; no novelty claim. | | ||||
| 48 | +| Cost-model decision rule | **REVISE** → conditional Proposition 5 ("under the measured cost model"), calibrated on the cluster benchmark; empirical, not general. | | ||||
| 49 | +| Adaptive (r, q) selection with proof | **DROP as theorem** → heuristic + numerical evaluation only, labelled open. | | ||||
| 50 | +| Wall-clock / energy / dollar claims | **DROP** unless measured in Phase 8; report measured quantities only. | | ||||
| 51 | +| Minimax / communication lower bounds | **DROP** (explicitly disclaimed; exact sufficient-statistic pooling dominates in exponential families). | | ||||
| 52 | + | ||||
| 53 | +## 5. Items not verifiable in this audit (to check against library copies before journal submission) | ||||
| 54 | +Quenouille 1949/1956 texts; Tukey 1958 abstract; Schucany–Gray–Owen 1971 determinant system; Sharot 1976; Rilstone et al. 1996 and | ||||
| 55 | +Bao–Ullah 2007 texts; Le Cam 1956; Bickel 1975 technical statements; van der Vaart §5.7 content; Cordeiro–McCullagh original notation; | ||||
| 56 | +published JASA theorem numbering for JLY, CEASE, FONE; Springer text of Huang–Huo; Yin's 2022 Temple thesis. | ||||
modified
literature/PRIOR_ART_MATRIX.csv
+44 −0
@@ -11,3 +11,47 @@ wang2017edsl,"Wang, Jialei; Kolar, Mladen; Srebro, Nathan; Zhang, Tong",2017,Eff | |||||
| 11 | battey2018distributed,"Battey, Heather; Fan, Jianqing; Liu, Han; Lu, Junwei; Zhu, Ziwei",2018,Distributed testing and estimation under sparse high dimensional models,"Annals of Statistics 46(3):1352-1382",10.1214/17-AOS1587,"full text arXiv 1509.05457v1 + PMC6051757 (low-dim theorems moved to supplement); journal page mapping NOT VERIFIED","lasso/SCAD/MCP error rates; bounded eigenvalues, sub-Gaussian rows; GLM analogues; finite fourth moments","1 round (one-shot)","d numbers per subsample","local lasso + debiasing (nodewise lassos); low-dim local OLS/GLM","Wald normality k=o((s log d)^-1 sqrt n) (Thm 3.3); lasso rate retained iff k=O(sqrt(n/(s^2 log d))) (Thm 4.3); low-dim GLM k=o(sqrt n/(d v log n)) (arXiv Thm 4.13)","high-probability decomposition + CLT","sparse high-dim linear & GLM; low-dim linear & GLM","no bias correction beyond lasso debiasing; establishes the sqrt(N) baseline (r=1,q=0); Remark 4.5 = same leading term, remainders differ",low,"n = total sample size in their notation","arXiv pp. 1-2, 8, 13-17; PMC lines 92-99, 368-442, 531-537" | 11 | battey2018distributed,"Battey, Heather; Fan, Jianqing; Liu, Han; Lu, Junwei; Zhu, Ziwei",2018,Distributed testing and estimation under sparse high dimensional models,"Annals of Statistics 46(3):1352-1382",10.1214/17-AOS1587,"full text arXiv 1509.05457v1 + PMC6051757 (low-dim theorems moved to supplement); journal page mapping NOT VERIFIED","lasso/SCAD/MCP error rates; bounded eigenvalues, sub-Gaussian rows; GLM analogues; finite fourth moments","1 round (one-shot)","d numbers per subsample","local lasso + debiasing (nodewise lassos); low-dim local OLS/GLM","Wald normality k=o((s log d)^-1 sqrt n) (Thm 3.3); lasso rate retained iff k=O(sqrt(n/(s^2 log d))) (Thm 4.3); low-dim GLM k=o(sqrt n/(d v log n)) (arXiv Thm 4.13)","high-probability decomposition + CLT","sparse high-dim linear & GLM; low-dim linear & GLM","no bias correction beyond lasso debiasing; establishes the sqrt(N) baseline (r=1,q=0); Remark 4.5 = same leading term, remainders differ",low,"n = total sample size in their notation","arXiv pp. 1-2, 8, 13-17; PMC lines 92-99, 368-442, 531-537" | ||
| 12 | volgushev2019quantile,"Volgushev, Stanislav; Chao, Shih-Kang; Cheng, Guang",2019,Distributed inference for quantile regression processes,"Annals of Statistics 47(3):1634-1662",10.1214/18-AOS1730,"full text arXiv 1701.06088v3 + NSF-PAR accepted manuscript (numbering identical)","bounded design with eigenvalue bounds; conditional density bounded with bounded derivative; Holder smoothness of tau -> beta(tau)","1 round (one-shot)","m numbers per machine per quantile level","local quantile regression fits","sup_tau ||beta_bar - beta_or|| = O_P(S log N/N + S^1/4 (log N)^7/4/N^3/4) + o_P(N^-1/2) (Thm 3.1); lower bound >= C S/N for some model (Thm 3.2); sufficient S=o(N^1/2/log N), NECESSARY S=o(N^1/2) (Cor 3.3)","new Bahadur representation; explicit worst-case construction","linear quantile regression; series models; conditional distribution functions","template for sharp (necessary + sufficient) partition-count conditions; no correction; no Newton",low,"necessity is worst-case over a model class","arXiv pp. 1, 6, 9-11" | 12 | volgushev2019quantile,"Volgushev, Stanislav; Chao, Shih-Kang; Cheng, Guang",2019,Distributed inference for quantile regression processes,"Annals of Statistics 47(3):1634-1662",10.1214/18-AOS1730,"full text arXiv 1701.06088v3 + NSF-PAR accepted manuscript (numbering identical)","bounded design with eigenvalue bounds; conditional density bounded with bounded derivative; Holder smoothness of tau -> beta(tau)","1 round (one-shot)","m numbers per machine per quantile level","local quantile regression fits","sup_tau ||beta_bar - beta_or|| = O_P(S log N/N + S^1/4 (log N)^7/4/N^3/4) + o_P(N^-1/2) (Thm 3.1); lower bound >= C S/N for some model (Thm 3.2); sufficient S=o(N^1/2/log N), NECESSARY S=o(N^1/2) (Cor 3.3)","new Bahadur representation; explicit worst-case construction","linear quantile regression; series models; conditional distribution functions","template for sharp (necessary + sufficient) partition-count conditions; no correction; no Newton",low,"necessity is worst-case over a model class","arXiv pp. 1, 6, 9-11" | ||
| 13 | shang2017limits,"Shang, Zuofeng; Cheng, Guang",2017,Computational Limits of A Distributed Algorithm for Smoothing Spline,"JMLR 18(108):1-37",https://jmlr.org/papers/v18/16-289.html,"full text JMLR PDF","periodic m-th order smoothing spline; evenly spaced design; iid errors; Sobolev ball","1 round (one-shot averaging)","one function estimate per machine","local smoothing spline with common lambda","minimax optimal iff s=O(N^2m/(2m+1)) (Thm 3.1); worst-case squared bias >= C(a_m n^-2m - 8N^-1) (Thm 3.2) => sharp; testing s=O(N^(4m-1)/(4m+1)) sharp","non-asymptotic bounds in eigen-basis; phase transition","nonparametric regression; testing","sharp maximal number of machines set by local bias vs 1/N variance; same mechanism, nonparametric",low,"improves ZDW bound","JMLR pp. 1, 5-7, 9-10" | 13 | shang2017limits,"Shang, Zuofeng; Cheng, Guang",2017,Computational Limits of A Distributed Algorithm for Smoothing Spline,"JMLR 18(108):1-37",https://jmlr.org/papers/v18/16-289.html,"full text JMLR PDF","periodic m-th order smoothing spline; evenly spaced design; iid errors; Sobolev ball","1 round (one-shot averaging)","one function estimate per machine","local smoothing spline with common lambda","minimax optimal iff s=O(N^2m/(2m+1)) (Thm 3.1); worst-case squared bias >= C(a_m n^-2m - 8N^-1) (Thm 3.2) => sharp; testing s=O(N^(4m-1)/(4m+1)) sharp","non-asymptotic bounds in eigen-basis; phase transition","nonparametric regression; testing","sharp maximal number of machines set by local bias vs 1/N variance; same mechanism, nonparametric",low,"improves ZDW bound","JMLR pp. 1, 5-7, 9-10" | ||
| 14 | +dhaene2015split,"Dhaene, Geert; Jochmans, Koen",2015,"Split-panel Jackknife Estimation of Fixed-effect Models","Review of Economic Studies 82(3):991-1030",10.1093/restud/rdv007,"yes (accepted manuscript)","alpha-mixing stationary panel; influence-function representation; bias expansion B1/T + ... + Bk/T^k; N/T -> rho",NA,NA,NA,"bias o(T^-h) with h-th order jackknife; sqrt(NT) normal with variance Sigma^-1 unchanged (first order)","asymptotic rectangular-array","nonlinear fixed-effect panel MLE","same extrapolation algebra (weights from a Vandermonde-type system in 1/T); bias-optimal partitions {2,3,...,h+1}, not {2,4,...}; higher-order variance left open","high for the extrapolation step; no distributed/Newton component","Supplement Thms S.2.1, S.2.2, S.3.1 cited from the manuscript","manuscript pp. 7-10, 24-26, 33" | ||||
| 15 | +hahn2024efficient,"Hahn, Jinyong; Hughes, David W.; Kuersteiner, Guido; Newey, Whitney K.",2024,"Efficient bias correction for cross-section and panel data","Quantitative Economics 15(3):783-816",10.3982/QE2350,"yes","iid; third-order stochastic expansion; regularity App. B",NA,NA,NA,"split-sample bias estimate inconsistent (O_p(1)); higher-order variance of split-sample corrections = 2x leave-one-out (Props 5-7)","asymptotic expansions","MLE cross-section; FE panel","variance cost of split-sample corrections",high,"analytical, bootstrap and leave-one-out share the same higher-order variance (Thms 1-2)","pp. 783-787, 797-802, 806-807" | ||||
| 16 | +quenouille1956notes,"Quenouille, M. H.",1956,"Notes on bias in estimation","Biometrika 43(3/4):353-360",10.1093/biomet/43.3-4.353,no,"bias expansion in powers of 1/n (per Kosmidis 2014)",NA,NA,NA,"bias O(n^-2) after jackknife",NOT VERIFIED,"general estimators","origin of the jackknife bias cancellation",low,"text not read",none | ||||
| 17 | +quenouille1949approximate,"Quenouille, M. H.",1949,"Approximate tests of correlation in time-series","JRSS B 11(1):68-84",10.1111/j.2517-6161.1949.tb00023.x,no,NOT VERIFIED,NA,NA,NA,NOT VERIFIED,NOT VERIFIED,"time series","split-half bias cancellation (attribution per secondary sources)",low,"text behind paywall",none | ||||
| 18 | +tukey1958bias,"Tukey, J. W.",1958,"Bias and confidence in not-quite large samples (abstract)","Annals of Mathematical Statistics 29(2):614",10.1214/aoms/1177706647,no,NA,NA,NA,NA,NA,"abstract","general","names the jackknife",low,"DOI covers Abstracts of Papers pp. 614-623",none | ||||
| 19 | +schucany1971bias,"Schucany, W. R.; Gray, H. L.; Owen, D. B.",1971,"On bias reduction in estimation","JASA 66(335):524-533",10.1080/01621459.1971.10482296,no,"known ratio of bias functions (per secondary)",NA,NA,NA,"removes k bias terms with k+1 estimators (per secondary)",NOT VERIFIED,"general estimators","generalized-jackknife weights = same linear-system idea",medium,"brief's DOI was wrong",none | ||||
| 20 | +sharot1976generalized,"Sharot, Trevor",1976,"The generalized jackknife: finite samples and subsample sizes","JASA 71(354):451-454",10.1080/01621459.1976.10480367,no,NOT VERIFIED,NA,NA,NA,NOT VERIFIED,NOT VERIFIED,"general","subsample-size choice",medium,"no text available",none | ||||
| 21 | +efron1982jackknife,"Efron, Bradley",1982,"The Jackknife, the Bootstrap and Other Resampling Plans","SIAM CBMS-NSF 38",10.1137/1.9781611970319,no,NA,NA,NA,NA,NA,"monograph","general","jackknife as linear extrapolation in 1/n (sec 2.3 per Kosmidis)",low,"",none | ||||
| 22 | +cox1968general,"Cox, D. R.; Snell, E. J.",1968,"A general definition of residuals","JRSS B 30(2):248-265",10.1111/j.2517-6161.1968.tb00724.x,no,"regular likelihood; cumulants O(n)",NA,NA,NA,"O(1/n) bias formula","formal expansion","parametric MLE","analytic first-order bias targeted by the extrapolation",low,"formula per Giles 2011",none | ||||
| 23 | +cordeiro1991bias,"Cordeiro, Gauss M.; McCullagh, Peter",1991,"Bias correction in generalized linear models","JRSS B 53(3):629-643",10.1111/j.2517-6161.1991.tb01852.x,"abstract","GLM with known/estimated dispersion",NA,NA,NA,"O(1/n) bias removed","formal expansion","GLMs","closed-form analytic bias (benchmark for the jackknife)",low,"formula per Kosmidis et al. 2018",none | ||||
| 24 | +firth1993bias,"Firth, David",1993,"Bias reduction of maximum likelihood estimates","Biometrika 80(1):27-38",10.1093/biomet/80.1.27,"abstract","regular parametric problems",NA,NA,NA,"first-order bias removed via score modification","asymptotic","parametric MLE, GLMs","implicit alternative to the jackknife",low,"",none | ||||
| 25 | +rilstone1996second,"Rilstone, Paul; Srivastava, V. K.; Ullah, Aman",1996,"The second-order bias and mean squared error of nonlinear estimators","Journal of Econometrics 75(2):369-395",10.1016/0304-4076(96)89457-7,no,NOT VERIFIED,NA,NA,NA,"O(1/n) bias and MSE expansions",NOT VERIFIED,"M-estimators","analytic second-order bias of M-estimators",low,"corrigendum 2005 124(1):203-204",none | ||||
| 26 | +hahn2004jackknife,"Hahn, Jinyong; Newey, Whitney",2004,"Jackknife and analytical bias reduction for nonlinear panel models","Econometrica 72(4):1295-1319",10.1111/j.1468-0262.2004.00533.x,"abstract + 2002 WP","static nonlinear FE panel; expansion theta[T] = theta + beta/T + gamma/T^2 + O(T^-3)",NA,NA,NA,"bias O(T^-2)","asymptotic","static nonlinear panel MLE","delete-one jackknife vs analytic correction",low-medium,"",none | ||||
| 27 | +lecam1956asymptotic,"Le Cam, Lucien",1956,"On the asymptotic theory of estimation and testing hypotheses","Proc. Third Berkeley Symp. 1:129-156",10.1525/9780520313880-014,no,NOT VERIFIED,NA,NA,NA,NOT VERIFIED,NOT VERIFIED,"parametric","one-step efficiency origin",low,"",none | ||||
| 28 | +bickel1975one,"Bickel, P. J.",1975,"One-step Huber estimates in the linear model","JASA 70(350):428-434",10.1080/01621459.1975.10479884,"abstract","mild regularity conditions",NA,NA,NA,"one-step asymptotically equivalent to the full M-estimate",NOT VERIFIED,"linear model M-estimators","one-step Newton from a preliminary estimator",low,"",none | ||||
| 29 | +vandervaart1998asymptotic,"van der Vaart, A. W.",1998,"Asymptotic Statistics","Cambridge University Press",10.1017/CBO9780511802256,"TOC",NA,NA,NA,NA,NA,"textbook","general","Sec. 5.7 One-Step Estimators p. 71",low,"",TOC | ||||
| 30 | +bhattacharya1978validity,"Bhattacharya, R. N.; Ghosh, J. K.",1978,"On the validity of the formal Edgeworth expansion","Annals of Statistics 6(2):434-451",10.1214/aos/1176344134,"abstract","iid, smooth H, moment + Cramer conditions",NA,NA,NA,"valid Edgeworth expansion to arbitrary order","rigorous","MLE, minimum contrast","justifies the bias expansion in 1/n",low,"",none | ||||
| 31 | +bach2021richardson,"Bach, Francis",2021,"On the effectiveness of Richardson extrapolation in data science","SIAM J. Math. Data Sci. 3(4):1251-1277",10.1137/21M1397349,yes,"power-series dependence in the hyperparameter",NA,NA,NA,"O(1/k) -> O(1/k^2) in iterations","asymptotic","optimisation, ridge, smoothing","Richardson in iterations/regularisation, not sample size",low,"",arXiv | ||||
| 32 | +lin2024jackknife,"Lin, Licong; Su, Fangzhou; Mou, Wenlong; Ding, Peng; Wainwright, Martin J.",2024,"When is it worthwhile to jackknife? Breaking the quadratic barrier for Z-estimators","arXiv 2411.02909",https://arxiv.org/abs/2411.02909,"abstract","high-dim Z-estimators",NA,NA,NA,"sqrt(n)-consistent when n >> d^{3/2}","non-asymptotic","Z-estimators, GLMs","leave-one-out jackknife only",low-medium,"",abstract | ||||
| 33 | +wu2023subsampling,"Wu, Shuyuan; Zhu, Xuening; Wang, Hansheng",2023,"Subsampling and jackknifing: a practically convenient solution for large data analysis with limited computational resources","Statistica Sinica 33:2041-2064",10.5705/ss.202021.0257,"abstract","mild conditions",NA,NA,NA,"consistent, asymptotically normal","asymptotic","general estimators","jackknife debiasing within subsamples; no nested sizes",low,"",abstract | ||||
| 34 | +chudik2018half,"Chudik, Alexander; Pesaran, M. Hashem; Yang, Jui-Chung",2018,"Half-panel jackknife fixed-effects estimation of linear panels with weakly exogenous regressors","Journal of Applied Econometrics 33(6):816-836",10.1002/jae.2623,"abstract","weakly exogenous regressors",NA,NA,NA,"bias O(T^-2); needs N/T^3 -> 0","asymptotic","linear dynamic panels","half-panel jackknife application",low,"",none | ||||
| 35 | +zhu2021lsa,"Zhu, Xuening; Li, Feng; Wang, Hansheng",2021,"Least-Square Approximation for a Distributed System","Journal of Computational and Graphical Statistics 30(4):1004-1018",10.1080/10618600.2021.1923517 (arXiv:1908.04904v4),"yes (arXiv v4)","iid C1-C5; c1 <= n_k/n <= c2","1 (WLSE) / 2 (TWLSE) / m (iterated)","K(p + p^2) per round","local M-estimation; TWLSE adds one LOCAL Newton step per machine before Hessian-weighted averaging","Prop 1 bias O_p(K/sqrt N); Thm 1 efficient if n/N^1/2 -> inf; Thm 2 TWLSE bias O_p(n^-2 sqrt N), efficient if n/N^1/4 -> inf (K = O(N^3/4)); Prop 2 m-WLSE with m/log N -> inf rounds allows n = O(1)","asymptotic normality","M-estimators","closest to 'raw average + one step' with local Hessians at local points; unequal sizes","STRONG for (r=1,q=1) exponent 3/4 and for the stale/local-Hessian variant; no bias correction; no joint frontier","trade-off remark p.12","pp. 6, 10-12" | ||||
| 36 | +gu2023heterogeneity,"Gu, Jia; Chen, Songxi",2023,"Distributed statistical inference under heterogeneity","Journal of Machine Learning Research 24(387):1-56",https://jmlr.org/papers/v24/22-1274.html (arXiv:2209.06482v1),"yes (arXiv v1)","heterogeneous blocks; Assumptions 1-7; moments v >= 4; unequal n_k","1","K(p + p^2)","local M-estimation + analytic leading-bias estimate B_k with sample splitting","WD: K = o(N^1/2) (Thms 2-3); debiased dWD/dSaC: MSE <= C1/(nK) + C2/(n^2 K) + C3/n^3, chi^2 limit if K = o(n^2) = o(N^2/3) (Thms 4-6); Cor 1 beats the SAVGM constant","MSE bounds + chi^2 limits","likelihood and quasi-likelihood (sandwich weights)","analytic local debiasing before weighted averaging (our analytic Cox-Snell baseline is this idea); unequal sizes; heterogeneity","STRONG for 'debias then average' (analytic, not split-sample); none for Newton rounds / frontier","SM 1.10 compares with Huang-Huo","pp. 3, 17, 20-24" | ||||
| 37 | +wu2023subsampling_jds,"Wu, Shuyuan; Zhu, Xuening; Wang, Hansheng",2023,"Subsampling and jackknifing: a practically convenient solution for large data analysis with limited computational resources","Statistica Sinica 33:2041-2064",10.5705/ss.202021.0257 (arXiv:2304.06231v1),"yes (arXiv v1)","sub-Gaussian; g eight times differentiable; N = o(n^4); log K = o(sqrt n); with-replacement subsamples","1","K estimates","delete-one jackknife (n refits) per subsample, then average","Thm 2: bias O(n^-2) + O(N^-1) vs tau_2/n for plain averaging; Thm 4 normal under N = o(n^4)","expansion + CLT","smooth functions of means","jackknife-debiased local estimates then averaged (delete-one, not split-sample; n refits vs 2^r - 1 nested fits)","STRONG for 'jackknife then average' (different jackknife); none for Newton / frontier","Li et al. 2024 review extends to distributed M-estimators","pp. 10-16" | ||||
| 38 | +yin2022newton,"Yin, Zhuoran",2022,"Communication-efficient distributed Newton-like optimization with gradients and M-estimators","arXiv 2207.06253 (Temple PhD thesis 2022 NOT VERIFIED)",https://arxiv.org/abs/2207.06253,"yes (arXiv v1)","iid; third-order smooth loss; sqrt(n)-consistent local M-estimators","2 / T","M-estimators + gradients + local second/third-derivative summaries","local M-estimation + derivative summaries","Fisher information from gradients and M-estimators with error O_p(n^-1 + m^-1); Thm 3.2 bias-adjusted one-step error O_p(n^-3/2 + m^-1 n^-1/2), efficient iff m = o(n^2) (K = o(N^2/3))","expansion bounds","M-estimators","analytic bias adjustment of the one-step estimator from the plain average","PARTIAL/STRONG for 'debiased averaging + Newton' (analytic, 2/3); none for split-sample / frontier","thesis may contain a non-normal limit for many centres (NOT VERIFIED)","pp. 4, 10-11" | ||||
| 39 | +gao2022review,"Gao, Yuan; Liu, Weidong; Wang, Hansheng; Wang, Xiaozhou; Yan, Yibo; Zhang, Riquan",2022,"A review of distributed statistical inference","Statistical Theory and Related Fields 6(2):89-99",10.1080/24754269.2021.1974158 (arXiv:2304.06245),"yes","review","-","-","-","averaging MSE C1/N + C2/n^2; one-step C1/N + O(n^-4 + N^-2); K = o(sqrt N) necessary (citing Volgushev)","review","-","confirms no (order, rounds) frontier as of 2022; open problem: unbalanced sizes",NONE,"p. 20 open problem","pp. 4-7, 11, 20" | ||||
| 40 | +chen2024binary,"Chen, Xi; Jing, Wenbo; Liu, Weidong; Zhang, Yichen",2024,"Distributed estimation and inference for semiparametric binary response models","Annals of Statistics 52(3):922-947",10.1214/24-AOS2376 (arXiv:2210.08393v4),"yes","smoothed max-score kernel order alpha; heterogeneous sizes (Sec 4)","1 / T","L(p + p^2)","smoothed local max score","Thm 3.1 one-shot L condition; Thms 3.3-3.4 multi-round removes it; Thm 4.1 weighted average with unequal m_l","nonsmooth asymptotics","binary choice","multi-round removes the machine-count condition; unequal sizes","PARTIAL","","pp. 5-6, 18-24, 30-32" | ||||
| 41 | +wang2021osup,"Wang, Fei; Zhu, Yuan; Huang, Danyang; Qi, Haobo; Wang, Hansheng",2021,"Distributed one-step upgraded estimation for non-uniformly and non-randomly distributed data","Computational Statistics & Data Analysis 162:107265",10.1016/j.csda.2021.107265,"abstract","non-random unequal partitions; pilot sample","2","pilot + derivatives","one Newton step from a pilot","as efficient as the full MLE","asymptotic normality","MLE / GLM","pilot + one step; unequal non-random partitions","PARTIAL (C2); STRONG (C5)","","abstract" | ||||
| 42 | +zhou2020glm,"Zhou, Ping; Yu, Zhen; Ma, Jingyi; Tian, Maozai; Fan, Ye",2020,"Communication-efficient distributed estimator for generalized linear models with a diverging number of covariates","arXiv 2001.06194v2 (journal NOT VERIFIED)",https://arxiv.org/abs/2001.06194,"yes","GLM, p_n = o(n^alpha); max n_k / min n_k = O(1)","2","K(p + p^2)","weighted average + one Fisher-scoring step","AEE needs K = o(sqrt n/p_n) (Thm 3); weighted average + one step needs K = O(sqrt(n/p_n)) (Thm 4)","efficiency","GLM","one-step from a weighted average; unequal sizes","PARTIAL","","pp. 4, 11, 13" | ||||
| 43 | +duan2022heterogeneity,"Duan, Rui; Ning, Yang; Chen, Yong",2022,"Heterogeneity-aware and communication-efficient distributed statistical inference","Biometrika 109(1):67-83",10.1093/biomet/asab007,"abstract","heterogeneous sites; density-ratio tilting","1-2","O(p + p^2)","local MLE + score","Cramer-Rao bound attained if the number of sites is smaller in rate than n","two-index asymptotics","parametric likelihood","heterogeneity-aware surrogate","PARTIAL (C5/C4)","","abstract" | ||||
| 44 | +chen2019quantile,"Chen, Xi; Liu, Weidong; Zhang, Yichen",2019,"Quantile regression under memory constraint","Annals of Statistics 47(6):3244-3273",10.1214/18-AOS1777,"abstract","QR; batch size m","few","p^2 per round","smoothed scores","naive DC needs n = o(m^2); multi-round removes","Bahadur","quantile regression","multi-round removes the constraint","PARTIAL (C2)","","abstract" | ||||
| 45 | +chen2014split,"Chen, Xueying; Xie, Min-ge",2014,"A split-and-conquer approach for analysis of extraordinarily large data","Statistica Sinica 24:1655-1684",10.5705/ss.2013.088,"abstract","penalized GLM; K subsets","1","K(p + p^2)","penalized GLM","equivalence when K is controlled (condition NOT VERIFIED)","asymptotic normality","penalized GLM","weighted combination with local information","PARTIAL (C5)","","abstract" | ||||
| 46 | +dobriban2021averaging,"Dobriban, Edgar; Sheng, Yue",2021,"Distributed linear regression by averaging","Annals of Statistics 49(2):918-943",10.1214/20-AOS1984,"abstract","linear regression; random-matrix asymptotics","1","Kp","OLS","exact efficiency loss of weighted averaging","random matrix theory","linear regression","efficiency loss of averaging","low","","abstract" | ||||
| 47 | +szabo2020adaptive,"Szabo, Botond; van Zanten, Harry",2020,"Adaptive distributed methods under communication constraints","Annals of Statistics 48(4):2347-2380",10.1214/19-AOS1890,"abstract","nonparametric; bit budget","1-2","bits","-","minimax bounds under communication constraints","information-theoretic","nonparametric","bit accounting precedent","low","","abstract" | ||||
| 48 | +cai2022gaussian,"Cai, T. Tony; Wei, Hongji",2022,"Distributed Gaussian mean estimation under communication constraints: optimal rates and communication-efficient algorithms","Annals of Statistics 50(4):1992-2020",10.1214/21-AOS2167,"abstract","Gaussian mean; bit budget","1","bits","-","optimal rates (localization + refinement)","information-theoretic","Gaussian mean","bit accounting precedent","low","","abstract" | ||||
| 49 | +duchi2014optimality,"Duchi, John C.; Jordan, Michael I.; Wainwright, Martin J.; Zhang, Yuchen",2014,"Optimality guarantees for distributed statistical estimation","arXiv 1405.0782 (NeurIPS 2013 version)",https://arxiv.org/abs/1405.0782,"abstract","bit budgets","-","bits","-","communication lower bounds","information-theoretic","location / regression","lower bounds (universal, not algorithm-family)","low","Garg-Ma-Nguyen 2014 and Braverman et al. 2016 NOT VERIFIED","abstract" | ||||
| 50 | +kal2025adaptive,"Kal, Dennis; Szabo, Botond; Guhaniyogi, Rajarshi; Pillai, Natesh; Pati, Debdeep",2025,"Adaptive divide and conquer with two rounds of communication","arXiv 2508.17073",https://arxiv.org/abs/2508.17073,"abstract/intro","white noise model","2","(log2 n)^2 bits + optimal bits","local summaries","rate-optimal adaptive","minimax","nonparametric","bit accounting with two rounds","low","","pp. 1-2" | ||||
| 51 | +banerjee2019divide,"Banerjee, Moulinath; Durot, Cecile; Sen, Bodhisattva",2019,"Divide and conquer in nonstandard problems and the super-efficiency phenomenon","Annals of Statistics 47(2):720-757",10.1214/17-AOS1633,"abstract","cube-root problems","1","Kp","isotonic","super-efficiency","nonstandard","isotonic regression","none","NONE","","abstract" | ||||
| 52 | +minsker2019distributed,"Minsker, Stanislav",2019,"Distributed statistical estimation and rates of convergence in normal approximation","Electronic Journal of Statistics 13:5213-5252",10.1214/19-EJS1647,"abstract","robust aggregation","1","Kp","local","same rate if K <= O(sqrt N)","Berry-Esseen","M-estimators / MLE","none","NONE","","abstract" | ||||
| 53 | +luo2020renewable,"Luo, Lan; Song, Peter X.-K.",2020,"Renewable estimation and incremental inference in generalized linear models with streaming data sets","Journal of the Royal Statistical Society Series B 82(1):69-97",10.1111/rssb.12352,"abstract","streaming data","sequential","summaries","incremental Newton","consistency + normality","asymptotic","GLM","none","NONE","","abstract" | ||||
| 54 | +tan2022quantile,"Tan, Kean Ming; Battey, Heather; Zhou, Wen-Xin",2022,"Communication-constrained distributed quantile regression with optimal statistical guarantees","Journal of Machine Learning Research 23(272):1-61",https://jmlr.org/papers/v23/21-1269.html,"abstract","double-smoothed QR","near-constant","gradients / Hessians","smoothed objectives","communication-vs-error trade-off","finite-sample","quantile regression","trade-off (no Hessian compression)","low","","abstract" | ||||
| 55 | +alimisis2021quantized,"Alimisis, Foivos; Davies, Peter; Alistarh, Dan",2021,"Communication-efficient distributed optimization with quantized preconditioners","Proc. 38th ICML (arXiv 2102.07214)",https://arxiv.org/abs/2102.07214,"abstract","strongly convex; lattice quantization","iterative","quantized Hessians","local Hessians","communication complexity (optimization only)","optimization","convex ERM","quantized-Hessian algorithm without statistical K-exponent","low (no statistics)","","abstract" | ||||
| 56 | +lin2023proforma,"Lin, Lu; Li, Feng",2023,"Global debiased DC estimations for biased estimators via pro forma regression","TEST 32(2):726-758",10.1007/s11749-023-00850-5 (arXiv:1904.07477v2),"yes","non-identically distributed batches","1","batches x (estimate + covariate)","local estimator + structural covariate","Thms 3.3-3.4 sqrt(n)-consistent / normal for any N, m","regression-based","biased estimators","not block-size extrapolation","NONE / weak","","pp. 10-15" | ||||
| 57 | +li2024selective,"Li, Xuetong; Gao, Yuan; Chang, Hong; Huang, Danyang; Ma, Yingying; Pan, Rui; Qi, Haobo; Wang, Feifei; Wu, Shuyuan; Xu, Ke; Zhou, Jing; Zhu, Xuening; Zhu, Yingqiu; Wang, Hansheng",2024,"A selective review on statistical methods for massive data computation","arXiv 2403.11163 (journal NOT VERIFIED)",https://arxiv.org/abs/2403.11163,"yes","review","-","-","-","JDS bias O(1/n^2) + O(1/N); per-machine delete-one M-estimators","review","M-estimators","JDS 'readily applicable to distributed computing'","PARTIAL (C1)","","pp. 15-16" | ||||
added
literature/audit_C_jackknife_bias_classics.md
+31 −0
@@ -0,0 +1,31 @@ | |||||
| 1 | +# Literature audit C — jackknife / bias-correction classics and 2010–2026 extrapolation search | ||||
| 2 | + | ||||
| 3 | +Method (verification task, 2026-10-28 18:47–19:03): Crossref-verified metadata for every item; full texts read for Dhaene–Jochmans (accepted manuscript), Hahn–Hughes–Kuersteiner–Newey 2024 (QE PDF), ZDW 2013, Hahn–Newey 2002 WP, Lin et al. 2024; secondary sources for formulas (Kosmidis 2014; Kosmidis–Kenne Pagui–Sartori 2018; Giles 2011). JSTOR / T&F / ScienceDirect / SIAM / Euclid blocked: several classics are "metadata verified, text NOT VERIFIED". | ||||
| 4 | + | ||||
| 5 | +## Corrections to the brief | ||||
| 6 | +- Schucany, Gray & Owen (1971) JASA 66(335):524–533: DOI **10.1080/01621459.1971.10482296** (the brief's …297 is Light & Margolin). | ||||
| 7 | +- Sharot (1976) JASA 71(354):451–454: DOI 10.1080/01621459.1976.10480367. (Also Sharot 1976 Biometrika 63(2):315–321 "Sharpening the jackknife".) | ||||
| 8 | +- Cox & Snell (1968) JRSS B 30(2): 248–265 on OUP/Crossref (248–275 with discussion per JSTOR). | ||||
| 9 | + | ||||
| 10 | +## Dhaene & Jochmans (2015), RES 82(3):991–1030, DOI 10.1093/restud/rdv007 (accepted manuscript read) | ||||
| 11 | +- Assumptions 2.1–2.3: α-mixing stationary panel; influence-function representation; bias expansion θ_T − θ0 = B₁/T + o(1/T). Eq. (2.4) half-panel jackknife θ̃_{1/2} = 2θ̂ − ½(θ̄_{S1} + θ̄_{S2}). Theorem 2.1: √(NT)(θ̃ − θ0) ⇒ N(0, Σ^{-1}) — "bias reduction can be achieved without variance inflation" (first-order). | ||||
| 12 | +- Higher order (§5.1, eqs. 5.1–5.3): with expansion (2.5) to order k, the h-th order jackknife θ̃_{1/G} = (1 + Σ_g a_{1/g})θ̂ − Σ_g a_{1/g} θ̄_{1/g}, weights a = (1 − ι'A^{-1}ι)^{-1}A^{-1}ι, [A]_{r,s} = Σ_{S∈S_{g_s}} (T/|S|)^{r−1}; for G = {2,3}: 3θ̂ − 3θ̄_{1/2} + θ̄_{1/3}, inconsistency o(T^{-2}). Supplement Theorem S.2.2: **G = {2, 3, …, h+1} minimises the inflation of the non-eliminated bias terms** (thirds, not quarters). First-order variance unchanged; "How to choose h optimally in practice is a difficult issue because the choice should also be guided by variance considerations" — higher-order variance left open. Overlapping subpanels inflate variance by d_g = ½g/(g−1) (Thm S.3.1). | ||||
| 13 | + | ||||
| 14 | +## Hahn, Hughes, Kuersteiner & Newey (2024), Quantitative Economics 15(3):783–816, DOI 10.3982/QE2350 (full text read) | ||||
| 15 | +- Split-sample bias estimate b̂_SS = ½(θ̂(1) + θ̂(2)) − θ̂ is unbiased but inconsistent (O_p(1), U-statistic structure; Prop. 5, App. A.1). Panel Prop. 6: Var(θ̂_SS) ≈ Var(A₁) + 2ῦ/T vs Var(θ̂_J) ≈ Var(A₁) + ῦ/(T−1): split-sample corrections have twice the higher-order variance of leave-one-out. Prop. 7: √(nT)(b̂_SS/T − B) = O_p(T^{-1/2}) vs O_p(T^{-1}) for leave-one-out. Analytical, bootstrap and leave-one-out corrections share the same higher-order variance (Thms 1–2). Direct relevance: quantifies the finite-sample variance cost of split-sample (and quarter-sample) extrapolation. | ||||
| 16 | + | ||||
| 17 | +## Classics (metadata verified; texts behind paywalls unless noted) | ||||
| 18 | +- Quenouille (1949) JRSS B 11(1):68–84, DOI 10.1111/j.2517-6161.1949.tb00023.x; Quenouille (1956) Biometrika 43(3/4):353–360, DOI 10.1093/biomet/43.3-4.353 (per Kosmidis 2014: nθ̂ − (n−1)θ̄ has bias −b₂/n² + O(n^{-3})). | ||||
| 19 | +- Tukey (1958) Ann. Math. Statist. 29(2):614 (abstract within "Abstracts of Papers" pp. 614–623, DOI 10.1214/aoms/1177706647). | ||||
| 20 | +- Schucany, Gray & Owen (1971): generalized jackknife (θ̂₁ − Rθ̂₂)/(1 − R); multi-term system NOT VERIFIED from the original. | ||||
| 21 | +- Efron (1982) SIAM CBMS-NSF 38, DOI 10.1137/1.9781611970319 (§2.3 jackknife as linear extrapolation in 1/n, per Kosmidis). | ||||
| 22 | +- Cox & Snell (1968): O(1/n) bias Bias(θ̂_s) = Σ k^{si}k^{jl}[½k_{ijl} + k_{ij,l}] + O(n^{-2}) (form per Giles 2011 eq. 10). | ||||
| 23 | +- Cordeiro & McCullagh (1991) JRSS B 53(3):629–643, DOI 10.1111/j.2517-6161.1991.tb01852.x (abstract read): first-order GLM biases via a supplementary weighted regression; form b_β = −i^{-1}X'Wξ, ξ_i = h_i d'_i/(2d_i w_i) per Kosmidis et al. 2018 eq. (4). | ||||
| 24 | +- Firth (1993) Biometrika 80(1):27–38, DOI 10.1093/biomet/80.1.27 (abstract read). | ||||
| 25 | +- Rilstone, Srivastava & Ullah (1996) J. Econometrics 75(2):369–395, DOI 10.1016/0304-4076(96)89457-7 (corrigendum 2005, 124(1):203–204); Bao & Ullah (2007) J. Econometrics 140(2):650–669, DOI 10.1016/j.jeconom.2006.07.007. | ||||
| 26 | +- Hahn & Newey (2004) Econometrica 72(4):1295–1319, DOI 10.1111/j.1468-0262.2004.00533.x (abstract + 2002 WP read): delete-one-period jackknife Tθ̂ − (T−1)Σθ̂_(t)/T, bias O(T^{-2}). | ||||
| 27 | +- Le Cam (1956) Proc. Third Berkeley Symp. 1:129–156, DOI 10.1525/9780520313880-014; Bickel (1975) JASA 70(350):428–434, DOI 10.1080/01621459.1975.10479884 (abstract); van der Vaart (1998) CUP, DOI 10.1017/CBO9780511802256, §5.7 "One-Step Estimators" p. 71 (TOC verified). | ||||
| 28 | +- Bhattacharya & Ghosh (1978) AoS 6(2):434–451, DOI 10.1214/aos/1176344134 (abstract); Ghosh (1994) NSF-CBMS 4, DOI 10.1214/cbms/1462297300; Pfanzagl (1985) LNS 31, DOI 10.1007/978-1-4615-6479-9. | ||||
| 29 | + | ||||
| 30 | +## 2010–2026 search: nested-subsample Richardson extrapolation of MLEs + Newton in divide-and-conquer | ||||
| 31 | +No paper found that extrapolates over nested sub-sample sizes m, m/2, m/4 for MLEs/M-estimators and combines it with a Newton correction in a distributed setting. Nearest: ZDW (2013) SAVGM (one-level subsampled extrapolation, Theorem 4, eq. 8); Hahn et al. (2024) (variance of split-sample corrections); Bach (2021) SIMODS 3(4):1251–1277, DOI 10.1137/21M1397349 (Richardson in iterations/regularisation, not sample size); Lin, Su, Mou, Ding & Wainwright (2024) arXiv 2411.02909 (leave-one-out jackknife, high-dim Z-estimators); Wu, Zhu & Wang (2023) Statistica Sinica 33:2041–2064, DOI 10.5705/ss.202021.0257; Chudik, Pesaran & Yang (2018) JAE 33(6):816–836, DOI 10.1002/jae.2623; Fernández-Val & Weidner (2016) J. Econometrics 192(1):291–312, DOI 10.1016/j.jeconom.2015.12.014. | ||||
added
literature/audit_D_priority_sweep_2013_2026.md
+27 −0
@@ -0,0 +1,27 @@ | |||||
| 1 | +# Literature audit D — priority / novelty-threat sweep (2013 → 2026-10-28) | ||||
| 2 | + | ||||
| 3 | +Method (verification task, 2026-10-28 18:47–19:05): 40+ web searches (arXiv, JMLR, AoS, JASA, Biometrika, Statistica Sinica, JCGS, QE, ReStud, TEST, CSDA; Crossref for metadata); full texts read for the 14 most relevant papers. Local dumps /tmp/litsearch/. | ||||
| 4 | + | ||||
| 5 | +## Threat per candidate claim (auditor's assessment) | ||||
| 6 | +- **C1 r-th order split-sample extrapolation of local MLEs, then average: PARTIAL→STRONG.** "Debias each local estimator then average" exists: ZDW13 SAVGM (subsampling); **Gu & Chen (2023) JMLR 24(387)** (analytic Rilstone-type local bias correction with sample splitting, weighted by n_k Ĥ_k^{-1}; χ² limit if K = o(n²) ⇔ K = o(N^{2/3}); beats the SAVGM constant); **Wu, Zhu & Wang (2023) Statistica Sinica 33:2041–2064** (delete-one jackknife within each subsample, then average; bias O(n^{-2}) + O(N^{-1}); normality under N = o(n⁴)); Li et al. (2024) review says JDS "is readily applicable to distributed computing"; Yin (2022) (analytic). The nested-subsample Vandermonde construction is Dhaene & Jochmans (2015) / Schucany–Gray–Owen (1971) / Quenouille (1956). **Not found:** a general r-th order split-sample extrapolation of local MLEs in divide-and-conquer with r ≥ 3 and the influence-function-preservation argument. Caveat: Hahn–Hughes–Kuersteiner–Newey (QE 2024) split-sample jackknife has twice the higher-order variance of delete-one. | ||||
| 7 | +- **C2 debiased average + q exact-Hessian Newton steps, frontier K = o(N^{1−1/(2^{q+1} r)}): PARTIAL.** Average + 1 exact step: Huang & Huo (2019) (stated k = O(√N); MSE term k⁴N^{-4} ⇒ implicitly 3/4); **Zhu, Li & Wang (2021) JCGS 30(4):1004–1018, DOI 10.1080/10618600.2021.1923517** Theorem 2: TWLSE (average → one local Newton step per machine → local-Hessian-weighted average) efficient iff n/N^{1/4} → ∞, "allows K = O(N^{3/4})"; Prop. 2 iterated version with m/log N → ∞ rounds allows n = O(1); CEASE rounds frontier n^{t+2} ≫ N polylog ⇒ K = o(N^{(t+1)/(t+2)}); JLY contraction n^{-1/2}; **Yin (2022) arXiv 2207.06253** Theorem 3.2: bias-adjusted one-step estimator (analytic removal of quadratic terms, pilot = average) efficient iff m = o(n²) ⇔ K = o(N^{2/3}). **Nobody combines an extrapolated initial estimator with exact-Hessian Newton rounds; no joint (order, rounds) frontier; no 7/8 or 11/12 anywhere.** | ||||
| 8 | +- **C3 sharpness: PARTIAL.** Necessity results exist only for plain averaging: Volgushev–Chao–Cheng (2019) Thm 3.2 + Cor 3.3; Shang–Cheng (2017); ZDW Example 1. Huang–Huo claim necessity of k = O(√N) from simulation only. Yin's 2022 Temple PhD thesis may derive a non-normal limit for one-step estimators with many centres — NOT VERIFIED (repository 403). No sharpness result for a debiased/Newton construction. | ||||
| 9 | +- **C4 inexact Newton: PARTIAL.** Single-machine Hessian: JLY Cor 3.4, Wang–Kolar–Srebro–Zhang 2017; local Hessians at local points: Zhu–Li–Wang (K = O(N^{3/4})); gradient-built Fisher information with O_p(n^{-1} + m^{-1}) error: Yin 2022; CEASE contraction p/n; Lin–Xi AEE. **Quantized/compressed Hessian with a statistical K-exponent and byte count: nothing found** (Alimisis–Davies–Alistarh ICML 2021 is optimization-only). | ||||
| 10 | +- **C5 unequal sizes / misspecification: STRONG (sizes) / PARTIAL (misspecification).** Weighted aggregation with unequal n_k is routine (Zhu–Li–Wang; Gu–Chen with quasi-likelihood ⇒ sandwich; Wang et al. CSDA 2021; Zhou et al. 2020; Lin–Xi; Chen–Xie 2014; Shi–Lu–Song 2018; Chen–Jing–Liu–Zhang AoS 2024 Thm 4.1). Misspecification: JLY Cor 3.4 is the sandwich; Gu–Chen quasi-likelihood; Volgushev p. 10. → present as an extension, not a contribution. | ||||
| 11 | +- **C6 cost model (extra extrapolation level vs extra synchronization round): PARTIAL-low.** Only qualitative trade-off statements (JLY p. 15; Zhu–Li–Wang p. 12; CEASE §4.3; Kal–Szabó–Guhaniyogi–Pillai–Pati 2025 bit accounting; Duchi et al. 2014, Szabó–van Zanten 2020, Cai–Wei 2024 lower bounds). No explicit decision rule found. | ||||
| 12 | + | ||||
| 13 | +## Additional verified records (beyond audits A–C) | ||||
| 14 | +- Zhu, Xuening; Li, Feng; Wang, Hansheng (2021) "Least-Square Approximation for a Distributed System", JCGS 30(4):1004–1018, DOI 10.1080/10618600.2021.1923517 (arXiv 1908.04904v4 read): WLSE weights α_k Σ̂_k^{-1}; c₁ ≤ n_k/n ≤ c₂; Prop. 1 bias O_p(K/√N); Thm 1 efficient if n/N^{1/2} → ∞; Thm 2 TWLSE bias O_p(n^{-2}√N), efficient if n/N^{1/4} → ∞; Prop. 2 m-WLSE. | ||||
| 15 | +- Gu, Jia; Chen, Songxi (2023) "Distributed statistical inference under heterogeneity", JMLR 24(387):1–56 (arXiv 2209.06482v1 read): WD weights n_k Ĥ_k^{-1}; Thms 2–3 K = o(N^{1/2}); §5 analytic leading bias n_k^{-1}B_k with sample splitting; Thms 4–6 dWD/dSaC MSE ≤ C₁/(nK) + C₂/(n²K) + C₃/n³, χ² limit if K = o(n²); Cor. 1 constant beats SAVGM's (2+3r)/(2(1−r)²); likelihood and quasi-likelihood (sandwich); one round; SM §1.10 vs Huang–Huo. | ||||
| 16 | +- Wu, Shuyuan; Zhu, Xuening; Wang, Hansheng (2023) Statistica Sinica 33:2041–2064, DOI 10.5705/ss.202021.0257 (arXiv 2304.06231v1 read): JDS = delete-one jackknife bias estimate per subsample then average (§2.3); Thm 2 bias O(n^{-2}) + O(N^{-1}); Thm 4 normal under N = o(n⁴). Smooth functions of means. | ||||
| 17 | +- Yin, Z. (2022) "Communication-efficient distributed Newton-like optimization with gradients and M-estimators", arXiv 2207.06253 (read): Fisher information from gradients + M-estimators with error O_p(n^{-1} + m^{-1}); Thm 3.2 bias-adjusted one-step error O_p(n^{-3/2} + m^{-1}n^{-1/2}); efficient iff m = o(n²). | ||||
| 18 | +- Gao, Yuan; Liu, Weidong; Wang, Hansheng; Wang, Xiaozhou; Yan, Yibo; Zhang, Riquan (2022) "A review of distributed statistical inference", Statistical Theory and Related Fields 6(2):89–99, DOI 10.1080/24754269.2021.1974158 (read): averaging MSE C₁/N + C₂/n²; one-step C₁/N + O(n^{-4} + N^{-2}); "K = o(√N) necessary (Volgushev)"; open problem: unbalanced sizes (p. 20). Confirms no (order, rounds) frontier as of 2022. | ||||
| 19 | +- Li et al. (2024) "A selective review on statistical methods for massive data computation", arXiv 2403.11163 (journal NOT VERIFIED). | ||||
| 20 | +- Chen, Xi; Jing, Wenbo; Liu, Weidong; Zhang, Yichen (2024) AoS 52(3):922–947, DOI 10.1214/24-AOS2376 (arXiv 2210.08393v4 read): Thm 3.1 one-shot L condition; Thms 3.3–3.4 multi-round; Thm 4.1 weighted average with unequal m_ℓ. | ||||
| 21 | +- Wang, Zhu, Huang, Qi, Wang (2021) CSDA 162:107265, DOI 10.1016/j.csda.2021.107265 (abstract): OSUP pilot + one step; non-random unequal partitions. | ||||
| 22 | +- Zhou, Yu, Ma, Tian, Fan (2020) arXiv 2001.06194v2 (read): AEE K = o(√n/p_n); weighted average + Fisher scoring K = O(√(n/p_n)). | ||||
| 23 | +- Duan, Ning, Chen (2022) Biometrika 109(1):67–83, DOI 10.1093/biomet/asab007 (abstract): heterogeneity-aware surrogate. | ||||
| 24 | +- Chen, Liu, Zhang (2019) AoS 47(6):3244–3273, DOI 10.1214/18-AOS1777 (abstract): naive DC needs n = o(m²); multi-round removes. | ||||
| 25 | +- Chen & Xie (2014) Statistica Sinica 24:1655–1684, DOI 10.5705/ss.2013.088 (abstract). Lee, Liu, Sun & Taylor (2017) JMLR 18(5):1–30 (abstract). Dobriban & Sheng (2021) AoS 49(2):918–943, DOI 10.1214/20-AOS1984. Szabó & van Zanten (2020) AoS 48(4):2347–2380, DOI 10.1214/19-AOS1890. Cai & Wei (2022) AoS 50(4):1992–2020, DOI 10.1214/21-AOS2167. Duchi, Jordan, Wainwright & Zhang (2014) arXiv 1405.0782. Kal, Szabó, Guhaniyogi, Pillai & Pati (2025) arXiv 2508.17073. Banerjee, Durot & Sen (2019) AoS 47(2):720–757, DOI 10.1214/17-AOS1633. Minsker (2019) EJS 13:5213–5252, DOI 10.1214/19-EJS1647. Luo & Song (2020) JRSS-B 82(1):69–97, DOI 10.1111/rssb.12352. Tan, Battey & Zhou (2022) JMLR 23(272):1–61. Alimisis, Davies & Alistarh (2021) ICML, arXiv 2102.07214. Lin & Li (2023) TEST 32(2):726–758, DOI 10.1007/s11749-023-00850-5. Tu, Liu, Mao & Chen (2021) JMLR 22(84):1–67. | ||||
| 26 | + | ||||
| 27 | +Searches with no relevant hit: "Richardson extrapolation" + distributed; "split-sample jackknife" + distributed; "K = o(N^{7/8})"; distributed pseudo-MLE/sandwich as a topic; quantized-Hessian one-step with asymptotic normality. | ||||
modified
research_log.md
+2 −0
@@ -11,3 +11,5 @@ | |||||
| 11 | - Decision: identifier WP28 kept (see SOURCE_PROVENANCE.md collision note). | 11 | - Decision: identifier WP28 kept (see SOURCE_PROVENANCE.md collision note). | ||
| 12 | - 18:55 Audit A received and archived (literature/audit_A_*.md, PRIOR_ART_MATRIX.csv rows 1-4). Key facts: HH19 proves one-step equivalence only under k=O(√N) from an uncorrected average; ZDW SAVGM = single-ratio subsampling correction with inflated variance constant, no jackknife/higher order; LX11 K≪N^{1/4}; RN16 m fixed. | 12 | - 18:55 Audit A received and archived (literature/audit_A_*.md, PRIOR_ART_MATRIX.csv rows 1-4). Key facts: HH19 proves one-step equivalence only under k=O(√N) from an uncorrected average; ZDW SAVGM = single-ratio subsampling correction with inflated variance constant, no jackknife/higher order; LX11 K≪N^{1/4}; RN16 m fixed. | ||
| 13 | - 19:00 Audit B archived (8 papers). Fixed sign of the analytic Cox–Snell correction (b_1 = +½ I^{-1} T; exponential check b_1 = θ0). | 13 | - 19:00 Audit B archived (8 papers). Fixed sign of the analytic Cox–Snell correction (b_1 = +½ I^{-1} T; exponential check b_1 = θ0). | ||
| 14 | +- 19:08 Audit C archived (jackknife classics; two DOI corrections; Dhaene–Jochmans partitions {2,3,..} bias-optimal; Hahn et al. 2024 variance of split-sample corrections). Patterns (1,2,3) added to the experimental plan. | ||||
| 15 | +- 19:12 Audit D archived (priority sweep; 23 more rows). Closest prior art for the combination: Zhu–Li–Wang 2021 (K=O(N^{3/4}) TWLSE), Gu–Chen 2023 (analytic debias+average, 2/3), Wu–Zhu–Wang 2023 (delete-one jackknife per block), Yin 2022 (bias-adjusted one-step, 2/3). No joint (r,q) frontier, no sharpness for corrected constructions, no quantized-Hessian statistical result found. | ||||
added
results/raw/exp_p1_N16384_K1024.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K1024.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K128.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K128.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K16.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K16.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K2048.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K2048.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K256.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K256.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K32.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K32.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K512.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K512.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K64.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N16384_K64.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N65536_K1024.csv
+2001 −0
Large diff (2,001 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N65536_K1024.manifest.json
+29 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added
results/raw/exp_p1_N65536_K128.csv
+384 −0
Large diff (384 lines) is not loaded by default to keep the page fast.Load diff
Diff truncated — file too large.