| 370 | out.push(fail("results.consistency", "hard", `metrics.json invalid: ${metrics.error}`)); |
370 | out.push(fail("results.consistency", "hard", `metrics.json invalid: ${metrics.error}`)); |
| 371 | else { |
371 | else { |
| 372 | const rv = metrics.value.reportedValues; |
372 | const rv = metrics.value.reportedValues; |
| 373 | − const missing = rv.filter((v) => !texContainsNumber(tex, v.value)); |
373 | + const reportTexts: string[] = []; |
| |
374 | + for (const a of ctx.artifacts) |
| |
375 | + if (/^results\/.*\.(md|tex)$/.test(a.path) && a.bytes < 4 * 1024 * 1024) |
| |
376 | + reportTexts.push((await a.read()).toString("utf8")); |
| |
377 | + const corpus = `${tex}\n${reportTexts.join("\n")}`; |
| |
378 | + const missing = rv.filter((v) => !texContainsNumber(corpus, v.value)); |
| 374 | out.push( |
379 | out.push( |
| 375 | rv.length >= 3 && missing.length === 0 |
380 | rv.length >= 3 && missing.length === 0 |
| 376 | ? pass( |
381 | ? pass( |
| 390 | if (metrics.value.synthetic && ctx.mode === "empirical") |
395 | if (metrics.value.synthetic && ctx.mode === "empirical") |
| 391 | out.push(fail("results.real", "hard", "empirical results flagged synthetic")); |
396 | out.push(fail("results.real", "hard", "empirical results flagged synthetic")); |
| 392 | } |
397 | } |
| |
398 | + // execution evidence: platform reproduction (strongest), then the experiment log (tolerant of the agent's log shape) |
| |
399 | + const repro = await readJson( |
| |
400 | + find(ctx, "results/reproduction.json"), |
| |
401 | + (x) => |
| |
402 | + x as { |
| |
403 | + reproduceScript?: { ran?: boolean; exitCode?: number | null }; |
| |
404 | + metricsCompared?: number; |
| |
405 | + metricsMismatched?: unknown[]; |
| |
406 | + metricsMissingInRerun?: number; |
| |
407 | + }, |
| |
408 | + ); |
| |
409 | + const reproOk = repro.value?.reproduceScript?.ran === true && repro.value.reproduceScript.exitCode === 0; |
| 393 | const logA = find(ctx, P.experimentLog); |
410 | const logA = find(ctx, P.experimentLog); |
| 394 | − const entries = logA |
411 | + const rawEntries = logA |
| 395 | ? (await logA.read()) |
412 | ? (await logA.read()) |
| 396 | .toString("utf8") |
413 | .toString("utf8") |
| 397 | .split(/\r?\n/) |
414 | .split(/\r?\n/) |
| 398 | .filter(Boolean) |
415 | .filter(Boolean) |
| 399 | .map((l) => { |
416 | .map((l) => { |
| 400 | try { |
417 | try { |
| 401 | − return ExperimentLogEntry.parse(JSON.parse(l)); |
418 | + return JSON.parse(l) as Record<string, unknown>; |
| 402 | } catch { |
419 | } catch { |
| 403 | return null; |
420 | return null; |
| 404 | } |
421 | } |
| 405 | }) |
422 | }) |
| |
423 | + .filter((e): e is Record<string, unknown> => !!e) |
| 406 | : []; |
424 | : []; |
| 407 | − const ok = entries.filter((e) => e && e.exitCode === 0).length; |
425 | + const okEntries = rawEntries.filter((e) => { |
| |
426 | + const strict = ExperimentLogEntry.safeParse(e); |
| |
427 | + if (strict.success) return strict.data.exitCode === 0; |
| |
428 | + const code = e.exitCode ?? e.exit_code ?? e.returncode ?? e.status; |
| |
429 | + if (code !== undefined) return code === 0 || code === "0" || code === "ok" || code === "success"; |
| |
430 | + return !!(e.script || e.command || e.name) && !(e.error || e.failed === true); |
| |
431 | + }).length; |
| 408 | const codeFiles = ctx.artifacts.filter( |
432 | const codeFiles = ctx.artifacts.filter( |
| 409 | − (a) => a.path.startsWith("code/") && a.path !== P.codeReadme, |
433 | + (a) => a.path.startsWith("code/") && a.path !== P.codeReadme && !/\.(md|txt)$/.test(a.path), |
| 410 | ).length; |
434 | ).length; |
| 411 | − out.push( |
435 | + const mismatched = repro.value?.metricsMismatched?.length ?? 0; |
| 412 | − ok >= 1 && codeFiles >= 1 |
436 | + if ((reproOk || okEntries >= 1) && codeFiles >= 1) { |
| 413 | − ? pass("execution.evidence", "hard", `${ok} successful executions logged, ${codeFiles} code files`) |
437 | + out.push( |
| 414 | − : fail( |
438 | + pass( |
| 415 | − "execution.evidence", |
439 | + "execution.evidence", |
| 416 | − "hard", |
440 | + "hard", |
| 417 | − `insufficient execution evidence (successful runs: ${ok}, code files: ${codeFiles})`, |
441 | + `${reproOk ? "platform reproduction succeeded; " : ""}${okEntries} logged execution(s), ${codeFiles} code files`, |
| 418 | − ), |
442 | + { reproOk, okEntries, codeFiles, metricsCompared: repro.value?.metricsCompared, mismatched }, |
| 419 | − ); |
443 | + ), |
| |
444 | + ); |
| |
445 | + } else { |
| |
446 | + out.push( |
| |
447 | + fail( |
| |
448 | + "execution.evidence", |
| |
449 | + "hard", |
| |
450 | + `insufficient execution evidence (platform reproduction ok: ${reproOk}, logged successful executions: ${okEntries}, code files: ${codeFiles})`, |
| |
451 | + { reproOk, okEntries, codeFiles }, |
| |
452 | + ), |
| |
453 | + ); |
| |
454 | + } |
| |
455 | + if (repro.value?.reproduceScript?.ran) { |
| |
456 | + out.push( |
| |
457 | + reproOk && mismatched === 0 |
| |
458 | + ? pass( |
| |
459 | + "reproduction.clean", |
| |
460 | + "soft", |
| |
461 | + `reproduce.sh re-ran cleanly; ${repro.value.metricsCompared ?? 0} metrics compared, 0 mismatches`, |
| |
462 | + ) |
| |
463 | + : fail( |
| |
464 | + "reproduction.clean", |
| |
465 | + "soft", |
| |
466 | + reproOk |
| |
467 | + ? `${mismatched} metric(s) differed on re-execution` |
| |
468 | + : `reproduce.sh exited with ${repro.value.reproduceScript.exitCode}`, |
| |
469 | + { mismatched: repro.value.metricsMismatched?.slice(0, 10) }, |
| |
470 | + ), |
| |
471 | + ); |
| |
472 | + } |
| 420 | } else out.push(skip("results.consistency", "hard", "not required for this mode")); |
473 | } else out.push(skip("results.consistency", "hard", "not required for this mode")); |
| 421 | |
474 | |
| 422 | // G11 data provenance |
475 | // G11 data provenance |
| 423 | |
476 | |