| 370 |
370 |
out.push(fail("results.consistency", "hard", `metrics.json invalid: ${metrics.error}`)); |
| 371 |
371 |
else { |
| 372 |
372 |
const rv = metrics.value.reportedValues; |
| 373 |
|
− const missing = rv.filter((v) => !texContainsNumber(tex, v.value)); |
|
373 |
+ const reportTexts: string[] = []; |
|
374 |
+ for (const a of ctx.artifacts) |
|
375 |
+ if (/^results\/.*\.(md|tex)$/.test(a.path) && a.bytes < 4 * 1024 * 1024) |
|
376 |
+ reportTexts.push((await a.read()).toString("utf8")); |
|
377 |
+ const corpus = `${tex}\n${reportTexts.join("\n")}`; |
|
378 |
+ const missing = rv.filter((v) => !texContainsNumber(corpus, v.value)); |
| 374 |
379 |
out.push( |
| 375 |
380 |
rv.length >= 3 && missing.length === 0 |
| 376 |
381 |
? pass( |
| 390 |
395 |
if (metrics.value.synthetic && ctx.mode === "empirical") |
| 391 |
396 |
out.push(fail("results.real", "hard", "empirical results flagged synthetic")); |
| 392 |
397 |
} |
|
398 |
+ // execution evidence: platform reproduction (strongest), then the experiment log (tolerant of the agent's log shape) |
|
399 |
+ const repro = await readJson( |
|
400 |
+ find(ctx, "results/reproduction.json"), |
|
401 |
+ (x) => |
|
402 |
+ x as { |
|
403 |
+ reproduceScript?: { ran?: boolean; exitCode?: number | null }; |
|
404 |
+ metricsCompared?: number; |
|
405 |
+ metricsMismatched?: unknown[]; |
|
406 |
+ metricsMissingInRerun?: number; |
|
407 |
+ }, |
|
408 |
+ ); |
|
409 |
+ const reproOk = repro.value?.reproduceScript?.ran === true && repro.value.reproduceScript.exitCode === 0; |
| 393 |
410 |
const logA = find(ctx, P.experimentLog); |
| 394 |
|
− const entries = logA |
|
411 |
+ const rawEntries = logA |
| 395 |
412 |
? (await logA.read()) |
| 396 |
413 |
.toString("utf8") |
| 397 |
414 |
.split(/\r?\n/) |
| 398 |
415 |
.filter(Boolean) |
| 399 |
416 |
.map((l) => { |
| 400 |
417 |
try { |
| 401 |
|
− return ExperimentLogEntry.parse(JSON.parse(l)); |
|
418 |
+ return JSON.parse(l) as Record<string, unknown>; |
| 402 |
419 |
} catch { |
| 403 |
420 |
return null; |
| 404 |
421 |
} |
| 405 |
422 |
}) |
|
423 |
+ .filter((e): e is Record<string, unknown> => !!e) |
| 406 |
424 |
: []; |
| 407 |
|
− const ok = entries.filter((e) => e && e.exitCode === 0).length; |
|
425 |
+ const okEntries = rawEntries.filter((e) => { |
|
426 |
+ const strict = ExperimentLogEntry.safeParse(e); |
|
427 |
+ if (strict.success) return strict.data.exitCode === 0; |
|
428 |
+ const code = e.exitCode ?? e.exit_code ?? e.returncode ?? e.status; |
|
429 |
+ if (code !== undefined) return code === 0 || code === "0" || code === "ok" || code === "success"; |
|
430 |
+ return !!(e.script || e.command || e.name) && !(e.error || e.failed === true); |
|
431 |
+ }).length; |
| 408 |
432 |
const codeFiles = ctx.artifacts.filter( |
| 409 |
|
− (a) => a.path.startsWith("code/") && a.path !== P.codeReadme, |
|
433 |
+ (a) => a.path.startsWith("code/") && a.path !== P.codeReadme && !/\.(md|txt)$/.test(a.path), |
| 410 |
434 |
).length; |
| 411 |
|
− out.push( |
| 412 |
|
− ok >= 1 && codeFiles >= 1 |
| 413 |
|
− ? pass("execution.evidence", "hard", `${ok} successful executions logged, ${codeFiles} code files`) |
| 414 |
|
− : fail( |
| 415 |
|
− "execution.evidence", |
| 416 |
|
− "hard", |
| 417 |
|
− `insufficient execution evidence (successful runs: ${ok}, code files: ${codeFiles})`, |
| 418 |
|
− ), |
| 419 |
|
− ); |
|
435 |
+ const mismatched = repro.value?.metricsMismatched?.length ?? 0; |
|
436 |
+ if ((reproOk || okEntries >= 1) && codeFiles >= 1) { |
|
437 |
+ out.push( |
|
438 |
+ pass( |
|
439 |
+ "execution.evidence", |
|
440 |
+ "hard", |
|
441 |
+ `${reproOk ? "platform reproduction succeeded; " : ""}${okEntries} logged execution(s), ${codeFiles} code files`, |
|
442 |
+ { reproOk, okEntries, codeFiles, metricsCompared: repro.value?.metricsCompared, mismatched }, |
|
443 |
+ ), |
|
444 |
+ ); |
|
445 |
+ } else { |
|
446 |
+ out.push( |
|
447 |
+ fail( |
|
448 |
+ "execution.evidence", |
|
449 |
+ "hard", |
|
450 |
+ `insufficient execution evidence (platform reproduction ok: ${reproOk}, logged successful executions: ${okEntries}, code files: ${codeFiles})`, |
|
451 |
+ { reproOk, okEntries, codeFiles }, |
|
452 |
+ ), |
|
453 |
+ ); |
|
454 |
+ } |
|
455 |
+ if (repro.value?.reproduceScript?.ran) { |
|
456 |
+ out.push( |
|
457 |
+ reproOk && mismatched === 0 |
|
458 |
+ ? pass( |
|
459 |
+ "reproduction.clean", |
|
460 |
+ "soft", |
|
461 |
+ `reproduce.sh re-ran cleanly; ${repro.value.metricsCompared ?? 0} metrics compared, 0 mismatches`, |
|
462 |
+ ) |
|
463 |
+ : fail( |
|
464 |
+ "reproduction.clean", |
|
465 |
+ "soft", |
|
466 |
+ reproOk |
|
467 |
+ ? `${mismatched} metric(s) differed on re-execution` |
|
468 |
+ : `reproduce.sh exited with ${repro.value.reproduceScript.exitCode}`, |
|
469 |
+ { mismatched: repro.value.metricsMismatched?.slice(0, 10) }, |
|
470 |
+ ), |
|
471 |
+ ); |
|
472 |
+ } |
| 420 |
473 |
} else out.push(skip("results.consistency", "hard", "not required for this mode")); |
| 421 |
474 |
|
| 422 |
475 |
// G11 data provenance |
| 423 |
476 |
|