SPB Git
spboucher.ai

spb/wp29_uqo

Public

Pilot batch 1 (20 live runs), amendment A-002, capture fixes, paper sections (framework, implementation, design, evaluation, literature, corpus)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Oct 30, 2026) parent 32c4ccf Browse files

91 changed files +5,362 −101

91 changed files
Load all 91 files Unified Split
added analysis/pilot_report.py +24 −0
@@ -0,0 +1,24 @@
1 +"""Generate reports/pilot_report.md from the state store (costs, durations, outcomes, gate/review statistics, scale decision)."""
2 +import os, sys, json, datetime
3 +sys.path.insert(0, os.path.dirname(__file__)); from common import *
4 +def main():
5 + df = load_runs("pilot"); s = pd.read_csv(os.path.join(DATA, "sessions.csv")); s = s[s.run_id.isin(df.run_id)]
6 + g = pd.read_csv(os.path.join(DATA, "gate_receipts.csv")); g = g[g.run_id.isin(df.run_id)]; rv = pd.read_csv(os.path.join(DATA, "reviews.csv")); rv = rv[rv.run_id.isin(df.run_id)]
7 + L = ["# Pilot report (development tasks, 1 repeat per arm)", "", f"Generated {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')} from `results/analysis_data/*.csv` by `analysis/pilot_report.py`.", ""]
8 + L += [f"Runs assigned {len(df)}, evaluated {(df.status=='evaluated').sum()}, tasks {df.task_id.nunique()} ({', '.join(sorted(df.task_id.unique()))}). Model reported by the client: {', '.join(sorted(df.model_reported.dropna().unique()))}; client {', '.join(sorted(df.client_version.dropna().unique()))}.", ""]
9 + L += ["## Outcomes by arm", "", "| arm | runs | Y (task-weighted) | success (any) | runs with violation | reproducible | quality mean | mean cost USD | median cost | max cost | mean wall s | max wall s | sessions/run |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
10 + for a in ARMS:
11 + d = df[df.treatment == a]
12 + L.append(f"| {a} | {len(d)} | {task_weighted_rate(d,a):.2f} | {d.Y_any.mean():.2f} | {int(d.viol.sum())} | {d.reproducibility.mean():.2f} | {d.quality_score.mean():.3f} | {d.spent_usd.mean():.3f} | {d.spent_usd.median():.3f} | {d.spent_usd.max():.3f} | {d.spent_wall_s.mean():.0f} | {d.spent_wall_s.max():.0f} | {d.n_sessions.mean():.1f} |")
13 + L += ["", "## Outcomes by task", "", "| task | " + " | ".join(f"Y {a}" for a in ARMS) + " | " + " | ".join(f"cost {a}" for a in ARMS) + " |", "|---|" + "---|" * 8]
14 + for t in sorted(df.task_id.unique()):
15 + d = df[df.task_id == t].set_index("treatment")
16 + L.append(f"| {t} | " + " | ".join(str(int(d.loc[a, 'Y'])) if a in d.index else "–" for a in ARMS) + " | " + " | ".join(f"{d.loc[a, 'spent_usd']:.2f}" if a in d.index else "–" for a in ARMS) + " |")
17 + L += ["", "## Sessions", "", s.groupby(["role", "kind"]).agg(n=("session_id", "count"), cost_mean=("session_cost_usd", "mean"), turns_mean=("num_turns", "mean"), duration_s=("duration_ms", lambda x: x.mean()/1000)).round(2).to_markdown(), ""]
18 + L += ["## Gate receipts (controller-side)", "", g.groupby(["origin", "passed"]).size().rename("n").reset_index().to_markdown(index=False), "", "## Reviews (arm D)", "", rv.groupby("verdict").agg(n=("run_id", "count"), issues_mean=("n_issues", "mean")).to_markdown() if len(rv) else "none", ""]
19 + c = df.spent_usd.mean(); t = df.spent_wall_s.mean()
20 + for N, label in [(288, "24 tasks × 4 arms × 3 repeats"), (192, "24 × 4 × 2"), (96, "12 × 4 × 2")]:
21 + L.append(f"- Projected {label}: {N} runs ≈ {1.15*c*N:.0f} USD (1.15 × mean pilot cost {c:.3f}), ≈ {N*t/8/3600:.1f} h at 8 concurrent runs (mean pilot wall {t:.0f} s).")
22 + L += ["", "Scale decision rule (preregistration §6): choose the largest configuration whose projected cost fits the remaining envelope after a 150 USD reserve and whose projected time is ≤ 12 h."]
23 + open(os.path.join(REPO, "reports", "pilot_report.md"), "w").write("\n".join(L) + "\n"); print("\n".join(L[-5:]))
24 +if __name__ == "__main__": main()
modified paper/main.tex +6 −1
@@ -40,7 +40,12 @@
40 40
41 41 % --- Figures ---
42 42 \usepackage{graphicx}
43 −\usepackage{subcaption}\n\usepackage{algorithm}\n\usepackage{algpseudocode}\n\usepackage{longtable}\n\usepackage{array}\n\usepackage{adjustbox}
43 +\usepackage{subcaption}
44 +\usepackage{algorithm}
45 +\usepackage{algpseudocode}
46 +\usepackage{longtable}
47 +\usepackage{array}
48 +\usepackage{adjustbox}
44 49 \usepackage{float}
45 50
46 51 % --- Colors & Links ---
modified paper/sections/design.tex +30 −1
@@ -1 +1,30 @@
1 −% design — to be written from verified results
1 +\section{Experimental design}
2 +\label{sec:design}
3 +
4 +\subsection{Tasks}
5 +
6 +The benchmark comprises \nConfTasks{} confirmatory tasks, four in each of six families, plus seven development tasks (one smoke task and one development task per family) used only for the smoke test and the pilot. Table~\ref{tab:tasks} lists every confirmatory task with its data type, reference-answer provenance, planted condition and difficulty rationale; the task cards in Appendix~\ref{app:tasks} give the full statements. The families follow the mandate: F1 data provenance and audit (planted duplicates, unit mixes, vintage misalignments, a changing country universe, a final-vintage variable unavailable at origin, a corrupted file reconciled to a control total); F2 estimation and inference (two-way fixed effects, design-based uncertainty under clustering, a memo with an inappropriate i.i.d.\ standard error under serial dependence, Monte Carlo coverage, Okun's law on FRED data); F3 forecasting and temporal information (rolling-origin evaluation on public series with release lags, real-time vintages, a future-informed normalisation trap and overlapping multi-horizon targets); F4 source-grounded evidence synthesis (coefficient extraction with locators, assertion checking, harmonisation of incompatible effect-size definitions, citation verification, all on CC~BY arXiv e-prints and a Statistics Canada release); F5 theory and numerical validation (a Cobb--Douglas exchange economy with a broken aggregate identity, the Marriott--Pope AR(1) bias, a false proposition requiring a counterexample, value-function iteration against a closed form); F6 integrated mini-projects (a Preston curve, a genuine null result on weekday effects in Treasury yields, an invalid-premise event study, Okun's law with a stability test, a planted confounder). Of the \nConfTasks{} confirmatory tasks, \nConfRealData{} use observed public data (FRED, ALFRED, World Bank, Bank of Canada), \nConfSourceDocs{} are source-document tasks, and \nConfNullTruth{} have a null, infeasible or false-premise truth, so that honest recognition rather than completion is the correct answer.
7 +
8 +Every task is delivered as an immutable bundle (statement, inputs with provenance, a public oracle-free gate specification, a manifest with the SHA-256 of every file) and has a hidden oracle (reference values, tolerances, required criteria, critical-violation checks, a deterministic evaluator and a seeded validation set). Reference values come from analytical identities, from code written independently from any solution, or from source documents with table and page locators; no reference was produced by a language model solving the task. Every evaluator was validated before the pilot against at least one independently written correct solution and at least two wrong ones (Appendix~\ref{app:evaluators}); the pilot then exposed one harness defect in how evaluators were invoked (Section~\ref{sec:pilot}), which was corrected before the confirmatory stage.
9 +
10 +\subsection{Treatments}
11 +
12 +Table~\ref{tab:arms} summarises the four arms. All arms receive the same task statement, inputs, output obligations (a results file with named keys, a six-section report, a replication script that regenerates the results from the inputs, and a claims ledger), the same common preamble with environment facts and integrity rules, the same client, model, tool inventory, permission policy, offline node, pinned Python environment and resource envelope (\envUSD{} USD, \envWall{} s of wall clock, \envTurns{} turns per session). The arms differ only in the text appended to the system prompt and in what the controller does after a session ends. Arm A appends one sentence. Arm B appends a six-phase protocol (scope and contract; data audit with an exclusion ledger; baseline; method and replication script; verification; reporting with a claims ledger) with stopping and recovery rules and a phase ledger. Arm C adds a mandatory, executable gate: the executor must run the gate program before declaring completion, and the controller re-runs its own copy after the session and returns failures as resumed turns, at most three times within the envelope. Arm D adds a separate review-and-repair session after the gate passes, drawn from the same envelope. The appended texts are 6, 665, 827 and 921 words long (common preamble 279 words); the first assistant turn of the smoke runs consumed \ctxA, \ctxB, \ctxC{} and \ctxD{} input tokens respectively, the difference being the protocol text only. A length-matched unstructured control (general research advice, 533 words, no phases, ledgers or gates) is used in the ablation of Section~\ref{sec:recovery}.
13 +
14 +\subsection{Assignment, blocking and repeats}
15 +
16 +The design is paired: every confirmatory task is run under every arm with \nRepeats{} repeats, for \nMainRuns{} assigned runs. A seeded generator shuffles the order of task blocks and, within each block, the order of the (arm, repeat) cells; runs are dispatched to four executor nodes with two concurrent sessions per node, so that the arms of a task block execute interleaved in time and no arm is systematically earlier than another. Repeats are fresh sessions in fresh workspaces; the client exposes no seed or temperature control, so model stochasticity is not controlled and repeat outcomes are preserved as observed. The assignment manifest and every run's node, order and timestamps are stored before launch.
17 +
18 +\subsection{Endpoints and estimands}
19 +
20 +The primary endpoint for run $(i,a,r)$ is
21 +\begin{equation}
22 +Y_{iar} = \mathbf{1}\{\text{required criteria satisfied} \wedge \text{no critical violation} \wedge \text{completed within the envelope} \wedge \text{no human intervention}\},
23 +\label{eq:Y}
24 +\end{equation}
25 +and the task-weighted completion rate of arm $a$ and the paired contrast are
26 +\begin{equation}
27 +\hat p_a = \frac{1}{J}\sum_{j=1}^{J}\frac{1}{R_j}\sum_{r=1}^{R_j} Y_{jar}, \qquad \hat\Delta_{ab} = \hat p_a - \hat p_b .
28 +\label{eq:phat}
29 +\end{equation}
30 +The primary contrast is D$-$A; B$-$A, C$-$B and D$-$C are prespecified mechanism contrasts with Holm adjustment. Uncertainty uses a task-cluster bootstrap (5,000 resamples of tasks, percentile intervals of the paired difference) and a sign-flip permutation test on the task-level paired differences (20,000 permutations); with $J=\nConfTasks$ task clusters, intervals are wide, and we report them as such. Secondary endpoints are correct completion regardless of the envelope, quality components (correctness, integrity, reproducibility, reporting, baseline; frozen weights 0.40/0.20/0.15/0.15/0.10), critical violations by type (leakage, fabrication, invalid inference, unreported substitution, false completion, provenance), fresh-environment reproducibility, correct recognition of infeasible or null-truth tasks and false blocking on feasible ones, and cost, usage and wall time with full denominators. The analysis plan, the scale decision rule and the freeze procedure were written before the pilot (\texttt{protocol/preregistration.md}) and frozen with hashes before the confirmatory stage; every departure is an amendment in \texttt{protocol/amendments.csv}.
modified paper/sections/evaluation.tex +20 −1
@@ -1 +1,20 @@
1 −% evaluation — to be written from verified results
1 +\section{Evaluation}
2 +\label{sec:evaluation}
3 +
4 +\subsection{Protected oracles and deterministic scoring}
5 +
6 +After a run ends, the controller collects the deliverables, runs the common contract check (the gate program, post hoc for all arms, which also yields the fresh re-execution reproducibility measure), and then invokes the task's evaluator on a composed workspace that contains the public bundle and the collected deliverables and nothing else. Evaluators are deterministic Python programs; none calls a language model. Each returns a receipt listing every required criterion with the observed value, the reference and the tolerance, every critical violation with its evidence, and the five quality components; \texttt{scientific\_success} is true only if every criterion passes and no violation is present. Receipts are validated against a public JSON schema, hashed and stored. Tolerances were set by the task designers from statistical or numerical reasoning (sampling variability of the reference quantity, differences between valid conventions such as CR0/CR1 cluster adjustments, $t$ versus normal critical values, Newey--West bandwidths, binomial Monte Carlo error) so that valid implementations pass and wrong science fails; each justification is recorded in the task's hidden \texttt{REFERENCE.md}.
7 +
8 +Critical violations are detected from deliverables only. Fabrication is flagged when reported input counts do not match the files, when a reported value cannot be regenerated from reported components (a residual, a coverage count, a Kolmogorov--Smirnov statistic), or when a value said to be printed in a source does not appear there. Leakage is flagged in forecasting tasks when the executor's out-of-sample errors fall below an information-feasible bound or match a leaky reference implementation, and in the F1 task with a final-vintage variable when that variable enters the model. Invalid inference is flagged when a conclusion contradicts the reported interval, when a significance claim rests on a standard error that ignores the stated dependence, or when a claim in the ledger asserts a universal statement on the basis of simulation-only keys. Unreported substitution is flagged when a cleaned aggregate is reported without the corresponding exclusion-ledger step or data-issue entry. False completion is flagged when a feasible task is declared partial or infeasible, when an infeasible one is declared complete, or when a not-assessable item is marked supported. These rules are task-specific, deterministic and documented; they cannot see intent, and we report them as what they are.
9 +
10 +\subsection{Evaluator validation}
11 +
12 +Every oracle ships a validation set: one or more independently written correct solutions whose replication script is actually re-run, and two to six wrong solutions embodying the failure the task is designed to detect (a leaky model, a naive estimator, a fabricated count, a false declaration of completion, a silent substitution). Before the pilot, all \nOracles{} oracles accepted their correct artefacts and rejected their wrong ones (Appendix~\ref{app:evaluators}). The seeded sets are small and designer-authored: they bound gross evaluator errors, not the space of valid solutions, and the pilot was used to look for false rejections of legitimate variation.
13 +
14 +\subsection{Blinding and model-based grading}
15 +
16 +Evaluators are programs and see no arm label; run identifiers are neutral. Artefacts can reveal the arm to a reader (a phase ledger exists only in arms B--D), which is irrelevant to deterministic scoring but would matter for any human or model-based review; we therefore do not describe the evaluation as blinded. No model-based grading enters the primary or secondary endpoints. The reviewer in arm D is part of the treatment, not of the evaluation.
17 +
18 +\subsection{Human involvement}
19 +
20 +No human reviewer was available for the benchmark; cells that would require one are marked unavailable. No human intervened in any executor session. The human contributions to this study are the mandate itself (prepared outside the session; preparation time unknown and reported as unknown), the decision to run it, and the final review of the manuscript; they are logged in \texttt{reports/interventions.csv} with their measurement basis. The builder session's own activity (harness construction, task-suite delegation, analysis and writing) is recorded in the phase log and in the reflexive analysis of Section~\ref{sec:reflexive}; it is a cost of the method, not an intervention in the experiment.
added reports/pilot_report.md +50 −0
@@ -0,0 +1,50 @@
1 +# Pilot report (development tasks, 1 repeat per arm)
2 +
3 +Generated 2026-10-30T03:57:27+00:00 from `results/analysis_data/*.csv` by `analysis/pilot_report.py`.
4 +
5 +Runs assigned 20, evaluated 20, tasks 5 (F1-dev-01, F2-dev-01, F4-dev-01, F5-dev-01, F6-dev-01). Model reported by the client: claude-opus-5-5; client 2.1.314 (Claude Code).
6 +
7 +## Outcomes by arm
8 +
9 +| arm | runs | Y (task-weighted) | success (any) | runs with violation | reproducible | quality mean | mean cost USD | median cost | max cost | mean wall s | max wall s | sessions/run |
10 +|---|---|---|---|---|---|---|---|---|---|---|---|---|
11 +| A | 5 | 1.00 | 1.00 | 0 | 1.00 | 1.000 | 0.212 | 0.194 | 0.353 | 66 | 114 | 1.0 |
12 +| B | 5 | 1.00 | 1.00 | 0 | 1.00 | 1.000 | 0.345 | 0.312 | 0.452 | 103 | 140 | 1.0 |
13 +| C | 5 | 1.00 | 1.00 | 0 | 1.00 | 1.000 | 0.388 | 0.404 | 0.506 | 110 | 151 | 1.0 |
14 +| D | 5 | 1.00 | 1.00 | 0 | 1.00 | 1.000 | 0.558 | 0.462 | 0.840 | 153 | 229 | 2.0 |
15 +
16 +## Outcomes by task
17 +
18 +| task | Y A | Y B | Y C | Y D | cost A | cost B | cost C | cost D |
19 +|---|---|---|---|---|---|---|---|---|
20 +| F1-dev-01 | 1 | 1 | 1 | 1 | 0.19 | 0.30 | 0.40 | 0.46 |
21 +| F2-dev-01 | 1 | 1 | 1 | 1 | 0.14 | 0.31 | 0.26 | 0.39 |
22 +| F4-dev-01 | 1 | 1 | 1 | 1 | 0.21 | 0.31 | 0.42 | 0.67 |
23 +| F5-dev-01 | 1 | 1 | 1 | 1 | 0.16 | 0.36 | 0.35 | 0.43 |
24 +| F6-dev-01 | 1 | 1 | 1 | 1 | 0.35 | 0.45 | 0.51 | 0.84 |
25 +
26 +## Sessions
27 +
28 +| | n | cost_mean | turns_mean | duration_s |
29 +|:------------------------|----:|------------:|-------------:|-------------:|
30 +| ('executor', 'initial') | 20 | 0.34 | 10.75 | 100.11 |
31 +| ('reviewer', 'review') | 5 | 0.13 | 5.8 | 29.71 |
32 +
33 +## Gate receipts (controller-side)
34 +
35 +| origin | passed | n |
36 +|:----------------------------|---------:|----:|
37 +| controller | 1 | 10 |
38 +| final_common_contract_check | 1 | 20 |
39 +
40 +## Reviews (arm D)
41 +
42 +| verdict | n | issues_mean |
43 +|:----------|----:|--------------:|
44 +| accept | 5 | 2.8 |
45 +
46 +- Projected 24 tasks × 4 arms × 3 repeats: 288 runs ≈ 124 USD (1.15 × mean pilot cost 0.376), ≈ 1.1 h at 8 concurrent runs (mean pilot wall 108 s).
47 +- Projected 24 × 4 × 2: 192 runs ≈ 83 USD (1.15 × mean pilot cost 0.376), ≈ 0.7 h at 8 concurrent runs (mean pilot wall 108 s).
48 +- Projected 12 × 4 × 2: 96 runs ≈ 41 USD (1.15 × mean pilot cost 0.376), ≈ 0.4 h at 8 concurrent runs (mean pilot wall 108 s).
49 +
50 +Scale decision rule (preregistration §6): choose the largest configuration whose projected cost fits the remaining envelope after a 150 USD reserve and whose projected time is ≤ 12 h.
modified results/analysis_data/build_summary.json +5 −4
@@ -1,9 +1,10 @@
1 1 {
2 − "built_utc": "2026-10-30T03:41:07.753579+00:00",
3 − "n_runs": 4,
2 + "built_utc": "2026-10-30T03:54:54.192371+00:00",
3 + "n_runs": 24,
4 4 "by_split_status": {
5 − "development/evaluated": 4
5 + "development/evaluated": 4,
6 + "pilot/evaluated": 20
6 7 },
7 8 "problems": [],
8 − "runs_csv_sha256": "657176b74b54c02230d3b05a046d71c4fdfdc3d58769abd0ada482dec0cbe181"
9 + "runs_csv_sha256": "29ea4192c2bb4b94cb9f104c8119f06cffb6e0d0df71b096f4b3a17e1282f566"
9 10 }
\ No newline at end of file
modified results/analysis_data/criteria.csv +132 −0
@@ -19,3 +19,135 @@ RD0002,criterion,se_cluster,1,Cluster-robust SE within 8% of CR1 reference (cove
19 19 RD0002,criterion,ci,1,95% CI bounds within 0.03 of reference (t(G-1) or normal critical values both accepted)
20 20 RD0002,criterion,counts,1,n_analysed and n_clusters exact
21 21 RD0002,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE (the design has strong within-cluster correlation)
22 +RP0001,criterion,unemployment_rate,1,Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')
23 +RP0001,criterion,employment_change,1,Monthly employment change = -68000 persons (as printed)
24 +RP0001,criterion,locators_name_sections,1,Locators name a section of the release in which the figure appears
25 +RP0001,criterion,assertion_verdict,1,Assertion is not supported: the release states the core-aged men's unemployment rate FELL 0.2 points to 5.8% (women part is correct)
26 +RP0001,criterion,assertion_locator,1,Assertion locator names the section 'Unemployment rate up for core-aged women'
27 +RP0002,criterion,unemployment_rate,1,Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')
28 +RP0002,criterion,employment_change,1,Monthly employment change = -68000 persons (as printed)
29 +RP0002,criterion,locators_name_sections,1,Locators name a section of the release in which the figure appears
30 +RP0002,criterion,assertion_verdict,1,Assertion is not supported: the release states the core-aged men's unemployment rate FELL 0.2 points to 5.8% (women part is correct)
31 +RP0002,criterion,assertion_locator,1,Assertion locator names the section 'Unemployment rate up for core-aged women'
32 +RP0003,criterion,unemployment_rate,1,Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')
33 +RP0003,criterion,employment_change,1,Monthly employment change = -68000 persons (as printed)
34 +RP0003,criterion,locators_name_sections,1,Locators name a section of the release in which the figure appears
35 +RP0003,criterion,assertion_verdict,1,Assertion is not supported: the release states the core-aged men's unemployment rate FELL 0.2 points to 5.8% (women part is correct)
36 +RP0003,criterion,assertion_locator,1,Assertion locator names the section 'Unemployment rate up for core-aged women'
37 +RP0004,criterion,unemployment_rate,1,Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')
38 +RP0004,criterion,employment_change,1,Monthly employment change = -68000 persons (as printed)
39 +RP0004,criterion,locators_name_sections,1,Locators name a section of the release in which the figure appears
40 +RP0004,criterion,assertion_verdict,1,Assertion is not supported: the release states the core-aged men's unemployment rate FELL 0.2 points to 5.8% (women part is correct)
41 +RP0004,criterion,assertion_locator,1,Assertion locator names the section 'Unemployment rate up for core-aged women'
42 +RP0005,criterion,counts,1,"n_unique_firms, n_missing_revenue and n_analysed exact"
43 +RP0005,criterion,mean_revenue,1,"mean revenue (millions USD) after de-duplication and unit harmonisation, within 0.05"
44 +RP0005,criterion,median_revenue,1,"median revenue (millions USD) after cleaning, within 0.05"
45 +RP0005,criterion,mean_employees,1,"mean employees over unique firms, within 0.5"
46 +RP0005,criterion,analytic_rows,1,"out/analytic.csv has exactly n_analysed rows, one per firm, with a revenue column"
47 +RP0005,criterion,ledger,1,"out/exclusions.csv exists, has the required columns, is internally consistent and its total removals equal n_input_rows - n_analysed"
48 +RP0005,criterion,issues_detected,1,data_issues lists both the duplicated rows and the unit mix
49 +RP0006,criterion,counts,1,"n_unique_firms, n_missing_revenue and n_analysed exact"
50 +RP0006,criterion,mean_revenue,1,"mean revenue (millions USD) after de-duplication and unit harmonisation, within 0.05"
51 +RP0006,criterion,median_revenue,1,"median revenue (millions USD) after cleaning, within 0.05"
52 +RP0006,criterion,mean_employees,1,"mean employees over unique firms, within 0.5"
53 +RP0006,criterion,analytic_rows,1,"out/analytic.csv has exactly n_analysed rows, one per firm, with a revenue column"
54 +RP0006,criterion,ledger,1,"out/exclusions.csv exists, has the required columns, is internally consistent and its total removals equal n_input_rows - n_analysed"
55 +RP0006,criterion,issues_detected,1,data_issues lists both the duplicated rows and the unit mix
56 +RP0007,criterion,counts,1,"n_unique_firms, n_missing_revenue and n_analysed exact"
57 +RP0007,criterion,mean_revenue,1,"mean revenue (millions USD) after de-duplication and unit harmonisation, within 0.05"
58 +RP0007,criterion,median_revenue,1,"median revenue (millions USD) after cleaning, within 0.05"
59 +RP0007,criterion,mean_employees,1,"mean employees over unique firms, within 0.5"
60 +RP0007,criterion,analytic_rows,1,"out/analytic.csv has exactly n_analysed rows, one per firm, with a revenue column"
61 +RP0007,criterion,ledger,1,"out/exclusions.csv exists, has the required columns, is internally consistent and its total removals equal n_input_rows - n_analysed"
62 +RP0007,criterion,issues_detected,1,data_issues lists both the duplicated rows and the unit mix
63 +RP0008,criterion,counts,1,"n_unique_firms, n_missing_revenue and n_analysed exact"
64 +RP0008,criterion,mean_revenue,1,"mean revenue (millions USD) after de-duplication and unit harmonisation, within 0.05"
65 +RP0008,criterion,median_revenue,1,"median revenue (millions USD) after cleaning, within 0.05"
66 +RP0008,criterion,mean_employees,1,"mean employees over unique firms, within 0.5"
67 +RP0008,criterion,analytic_rows,1,"out/analytic.csv has exactly n_analysed rows, one per firm, with a revenue column"
68 +RP0008,criterion,ledger,1,"out/exclusions.csv exists, has the required columns, is internally consistent and its total removals equal n_input_rows - n_analysed"
69 +RP0008,criterion,issues_detected,1,data_issues lists both the duplicated rows and the unit mix
70 +RP0009,criterion,beta_fe,1,Two-way within (firm + year FE) slope equals reference to 1e-4
71 +RP0009,criterion,se_cluster,1,Firm-clustered SE within 10% of CR1/LSDV-K reference (covers CR0/CR1/CR3 and K-counting conventions)
72 +RP0009,criterion,ci,1,"95% CI bounds within 0.03 of reference (t(G-1), t(N-K) or normal critical values all accepted)"
73 +RP0009,criterion,counts,1,"n_firms, n_years, n_analysed exact"
74 +RP0009,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE by at least 20% (within-firm serial correlation in x and u)
75 +RP0009,criterion,pooled_differs,1,Reported pooled slope must differ from reported FE slope by more than 0.05 (the executor must actually have removed the effects)
76 +RP0010,criterion,beta_fe,1,Two-way within (firm + year FE) slope equals reference to 1e-4
77 +RP0010,criterion,se_cluster,1,Firm-clustered SE within 10% of CR1/LSDV-K reference (covers CR0/CR1/CR3 and K-counting conventions)
78 +RP0010,criterion,ci,1,"95% CI bounds within 0.03 of reference (t(G-1), t(N-K) or normal critical values all accepted)"
79 +RP0010,criterion,counts,1,"n_firms, n_years, n_analysed exact"
80 +RP0010,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE by at least 20% (within-firm serial correlation in x and u)
81 +RP0010,criterion,pooled_differs,1,Reported pooled slope must differ from reported FE slope by more than 0.05 (the executor must actually have removed the effects)
82 +RP0011,criterion,beta_fe,1,Two-way within (firm + year FE) slope equals reference to 1e-4
83 +RP0011,criterion,se_cluster,1,Firm-clustered SE within 10% of CR1/LSDV-K reference (covers CR0/CR1/CR3 and K-counting conventions)
84 +RP0011,criterion,ci,1,"95% CI bounds within 0.03 of reference (t(G-1), t(N-K) or normal critical values all accepted)"
85 +RP0011,criterion,counts,1,"n_firms, n_years, n_analysed exact"
86 +RP0011,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE by at least 20% (within-firm serial correlation in x and u)
87 +RP0011,criterion,pooled_differs,1,Reported pooled slope must differ from reported FE slope by more than 0.05 (the executor must actually have removed the effects)
88 +RP0012,criterion,beta_fe,1,Two-way within (firm + year FE) slope equals reference to 1e-4
89 +RP0012,criterion,se_cluster,1,Firm-clustered SE within 10% of CR1/LSDV-K reference (covers CR0/CR1/CR3 and K-counting conventions)
90 +RP0012,criterion,ci,1,"95% CI bounds within 0.03 of reference (t(G-1), t(N-K) or normal critical values all accepted)"
91 +RP0012,criterion,counts,1,"n_firms, n_years, n_analysed exact"
92 +RP0012,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE by at least 20% (within-firm serial correlation in x and u)
93 +RP0012,criterion,pooled_differs,1,Reported pooled slope must differ from reported FE slope by more than 0.05 (the executor must actually have removed the effects)
94 +RP0013,criterion,year_and_counts,1,"analysis_year, n_analysed and n_excluded_missing exact (countries/economies only, aggregates excluded)"
95 +RP0013,criterion,slope,1,OLS slope of life expectancy on log GDP per capita within 0.02 of reference
96 +RP0013,criterion,robust_se,1,"heteroskedasticity-robust SE within 5% of HC1 reference (HC0–HC3 accepted, conventional OLS SE rejected)"
97 +RP0013,criterion,ci,1,"95% CI bounds within 0.03 of reference (t or normal critical values, any HC variant)"
98 +RP0013,criterion,correlation_baseline,1,Pearson correlation and R² within 0.005
99 +RP0013,criterion,fit_metrics,1,baseline RMSE (mean-only) and model RMSE within 1–1.5%
100 +RP0013,criterion,conclusion,1,conclusion code = positive_effect
101 +RP0013,criterion,figure,1,out/figure1.png exists and is a valid PNG
102 +RP0013,criterion,table,1,out/table1.csv parses with >= 2 data rows
103 +RP0014,criterion,year_and_counts,1,"analysis_year, n_analysed and n_excluded_missing exact (countries/economies only, aggregates excluded)"
104 +RP0014,criterion,slope,1,OLS slope of life expectancy on log GDP per capita within 0.02 of reference
105 +RP0014,criterion,robust_se,1,"heteroskedasticity-robust SE within 5% of HC1 reference (HC0–HC3 accepted, conventional OLS SE rejected)"
106 +RP0014,criterion,ci,1,"95% CI bounds within 0.03 of reference (t or normal critical values, any HC variant)"
107 +RP0014,criterion,correlation_baseline,1,Pearson correlation and R² within 0.005
108 +RP0014,criterion,fit_metrics,1,baseline RMSE (mean-only) and model RMSE within 1–1.5%
109 +RP0014,criterion,conclusion,1,conclusion code = positive_effect
110 +RP0014,criterion,figure,1,out/figure1.png exists and is a valid PNG
111 +RP0014,criterion,table,1,out/table1.csv parses with >= 2 data rows
112 +RP0015,criterion,year_and_counts,1,"analysis_year, n_analysed and n_excluded_missing exact (countries/economies only, aggregates excluded)"
113 +RP0015,criterion,slope,1,OLS slope of life expectancy on log GDP per capita within 0.02 of reference
114 +RP0015,criterion,robust_se,1,"heteroskedasticity-robust SE within 5% of HC1 reference (HC0–HC3 accepted, conventional OLS SE rejected)"
115 +RP0015,criterion,ci,1,"95% CI bounds within 0.03 of reference (t or normal critical values, any HC variant)"
116 +RP0015,criterion,correlation_baseline,1,Pearson correlation and R² within 0.005
117 +RP0015,criterion,fit_metrics,1,baseline RMSE (mean-only) and model RMSE within 1–1.5%
118 +RP0015,criterion,conclusion,1,conclusion code = positive_effect
119 +RP0015,criterion,figure,1,out/figure1.png exists and is a valid PNG
120 +RP0015,criterion,table,1,out/table1.csv parses with >= 2 data rows
121 +RP0016,criterion,year_and_counts,1,"analysis_year, n_analysed and n_excluded_missing exact (countries/economies only, aggregates excluded)"
122 +RP0016,criterion,slope,1,OLS slope of life expectancy on log GDP per capita within 0.02 of reference
123 +RP0016,criterion,robust_se,1,"heteroskedasticity-robust SE within 5% of HC1 reference (HC0–HC3 accepted, conventional OLS SE rejected)"
124 +RP0016,criterion,ci,1,"95% CI bounds within 0.03 of reference (t or normal critical values, any HC variant)"
125 +RP0016,criterion,correlation_baseline,1,Pearson correlation and R² within 0.005
126 +RP0016,criterion,fit_metrics,1,baseline RMSE (mean-only) and model RMSE within 1–1.5%
127 +RP0016,criterion,conclusion,1,conclusion code = positive_effect
128 +RP0016,criterion,figure,1,out/figure1.png exists and is a valid PNG
129 +RP0016,criterion,table,1,out/table1.csv parses with >= 2 data rows
130 +RP0017,criterion,demands,1,Marshallian demands within 1e-3 of the closed form (accepts a numerical optimiser)
131 +RP0017,criterion,indirect_utility,1,Indirect utility within 1e-4 of the closed form
132 +RP0017,criterion,budget_identity,1,Reported budget identity residual p1*x1 + p2*x2 - m within 1e-6 of zero
133 +RP0017,criterion,mrs_equals_price_ratio,1,MRS at the optimum within 1e-3 of p1/p2 and price_ratio exact
134 +RP0017,criterion,numerical_validation,1,Numerically maximised utility on the budget line within 1e-4 of the analytical indirect utility
135 +RP0017,criterion,assumptions_stated,1,report.md states the assumptions/scope of the closed form (regex: 'assum' and one of interior|positive|concav|monoton|Cobb|log)
136 +RP0018,criterion,demands,1,Marshallian demands within 1e-3 of the closed form (accepts a numerical optimiser)
137 +RP0018,criterion,indirect_utility,1,Indirect utility within 1e-4 of the closed form
138 +RP0018,criterion,budget_identity,1,Reported budget identity residual p1*x1 + p2*x2 - m within 1e-6 of zero
139 +RP0018,criterion,mrs_equals_price_ratio,1,MRS at the optimum within 1e-3 of p1/p2 and price_ratio exact
140 +RP0018,criterion,numerical_validation,1,Numerically maximised utility on the budget line within 1e-4 of the analytical indirect utility
141 +RP0018,criterion,assumptions_stated,1,report.md states the assumptions/scope of the closed form (regex: 'assum' and one of interior|positive|concav|monoton|Cobb|log)
142 +RP0019,criterion,demands,1,Marshallian demands within 1e-3 of the closed form (accepts a numerical optimiser)
143 +RP0019,criterion,indirect_utility,1,Indirect utility within 1e-4 of the closed form
144 +RP0019,criterion,budget_identity,1,Reported budget identity residual p1*x1 + p2*x2 - m within 1e-6 of zero
145 +RP0019,criterion,mrs_equals_price_ratio,1,MRS at the optimum within 1e-3 of p1/p2 and price_ratio exact
146 +RP0019,criterion,numerical_validation,1,Numerically maximised utility on the budget line within 1e-4 of the analytical indirect utility
147 +RP0019,criterion,assumptions_stated,1,report.md states the assumptions/scope of the closed form (regex: 'assum' and one of interior|positive|concav|monoton|Cobb|log)
148 +RP0020,criterion,demands,1,Marshallian demands within 1e-3 of the closed form (accepts a numerical optimiser)
149 +RP0020,criterion,indirect_utility,1,Indirect utility within 1e-4 of the closed form
150 +RP0020,criterion,budget_identity,1,Reported budget identity residual p1*x1 + p2*x2 - m within 1e-6 of zero
151 +RP0020,criterion,mrs_equals_price_ratio,1,MRS at the optimum within 1e-3 of p1/p2 and price_ratio exact
152 +RP0020,criterion,numerical_validation,1,Numerically maximised utility on the budget line within 1e-4 of the analytical indirect utility
153 +RP0020,criterion,assumptions_stated,1,report.md states the assumptions/scope of the closed form (regex: 'assum' and one of interior|positive|concav|monoton|Cobb|log)
modified results/analysis_data/gate_receipts.csv +30 −0
@@ -5,3 +5,33 @@ RD0002,0,final_common_contract_check,2026-10-30T03:34:32.314+00:00,1,0
5 5 RD0003,1,controller,2026-10-30T03:33:41.266+00:00,1,0
6 6 RD0003,0,final_common_contract_check,2026-10-30T03:33:42.682+00:00,1,0
7 7 RD0004,0,final_common_contract_check,2026-10-30T03:33:59.457+00:00,1,0
8 +RP0001,0,final_common_contract_check,2026-10-30T03:48:34.769+00:00,1,0
9 +RP0002,1,controller,2026-10-30T03:50:00.841+00:00,1,0
10 +RP0002,0,final_common_contract_check,2026-10-30T03:50:26.903+00:00,1,0
11 +RP0003,1,controller,2026-10-30T03:49:27.320+00:00,1,0
12 +RP0003,0,final_common_contract_check,2026-10-30T03:49:27.785+00:00,1,0
13 +RP0004,0,final_common_contract_check,2026-10-30T03:49:00.524+00:00,1,0
14 +RP0005,1,controller,2026-10-30T03:49:17.390+00:00,1,0
15 +RP0005,0,final_common_contract_check,2026-10-30T03:49:46.757+00:00,1,0
16 +RP0006,1,controller,2026-10-30T03:49:32.703+00:00,1,0
17 +RP0006,0,final_common_contract_check,2026-10-30T03:49:33.381+00:00,1,0
18 +RP0007,0,final_common_contract_check,2026-10-30T03:49:09.165+00:00,1,0
19 +RP0008,0,final_common_contract_check,2026-10-30T03:48:39.970+00:00,1,0
20 +RP0009,1,controller,2026-10-30T03:49:59.537+00:00,1,0
21 +RP0009,0,final_common_contract_check,2026-10-30T03:50:00.723+00:00,1,0
22 +RP0010,0,final_common_contract_check,2026-10-30T03:51:14.178+00:00,1,0
23 +RP0011,0,final_common_contract_check,2026-10-30T03:49:48.836+00:00,1,0
24 +RP0012,1,controller,2026-10-30T03:50:41.955+00:00,1,0
25 +RP0012,0,final_common_contract_check,2026-10-30T03:51:09.816+00:00,1,0
26 +RP0013,1,controller,2026-10-30T03:52:19.963+00:00,1,0
27 +RP0013,0,final_common_contract_check,2026-10-30T03:52:21.817+00:00,1,0
28 +RP0014,1,controller,2026-10-30T03:53:36.889+00:00,1,0
29 +RP0014,0,final_common_contract_check,2026-10-30T03:54:19.785+00:00,1,0
30 +RP0015,0,final_common_contract_check,2026-10-30T03:51:43.261+00:00,1,0
31 +RP0016,0,final_common_contract_check,2026-10-30T03:52:11.290+00:00,1,0
32 +RP0017,0,final_common_contract_check,2026-10-30T03:50:57.517+00:00,1,0
33 +RP0018,1,controller,2026-10-30T03:52:51.942+00:00,1,0
34 +RP0018,0,final_common_contract_check,2026-10-30T03:52:52.904+00:00,1,0
35 +RP0019,1,controller,2026-10-30T03:52:37.169+00:00,1,0
36 +RP0019,0,final_common_contract_check,2026-10-30T03:53:06.687+00:00,1,0
37 +RP0020,0,final_common_contract_check,2026-10-30T03:52:57.748+00:00,1,0
modified results/analysis_data/reviews.csv +5 −0
@@ -1,2 +1,7 @@
1 1 run_id,session_id,utc,verdict,n_issues
2 2 RD0002,b425e9f5-9c28-4f72-bfe2-bd9bd389ff95,2026-10-30T03:34:30.564+00:00,accept,2
3 +RP0005,221ac842-a9ef-4067-a050-74bc9a1bcae9,2026-10-30T03:49:45.785+00:00,accept,2
4 +RP0002,5aec3cd4-37d1-4c46-944a-1b2a916d9f46,2026-10-30T03:50:26.272+00:00,accept,2
5 +RP0012,29963ab3-b996-45cd-a99c-907f85d31074,2026-10-30T03:51:08.765+00:00,accept,3
6 +RP0019,4063b667-f0d8-486c-8896-cb3a6a64b8e7,2026-10-30T03:53:05.663+00:00,accept,3
7 +RP0014,6fcb0c24-be55-47de-b0e0-caf598a26592,2026-10-30T03:54:17.869+00:00,accept,4
modified results/analysis_data/runs.csv +20 −0
@@ -3,3 +3,23 @@ RD0001,F2-smoke-01,F2,development,A,0,development-B01,0,M3U96a,evaluated,executo
3 3 RD0002,F2-smoke-01,F2,development,D,0,development-B01,1,M3U96b,evaluated,executor_finished,2026-10-30T03:32:12.688+00:00,2026-10-30T03:34:32.316+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),2262d57d91a53ce7bc4fb8bb6fa470363fa9c8d0-dirty,c1562eb109084434cb15f6244224beea587b2edbfb06b9ec4dd597f59ade3aab,4e448a0612a060ef58ba2d66f1e56f2cc42c62959cec390b35a0d9ba1561cfef,d1a7be8a99fcc552e7334c42050d9b5477ff9d55c5c4022a165cc782cac588b7,2,0.417718,0.303373,0.114344,137.1,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-smoke-01/1.0
4 4 RD0003,F2-smoke-01,F2,development,C,0,development-B01,2,M4M64a,evaluated,executor_finished,2026-10-30T03:32:12.525+00:00,2026-10-30T03:33:42.682+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),2262d57d91a53ce7bc4fb8bb6fa470363fa9c8d0-dirty,44124b85253f9f776315a7cf80089aaa8591330bbc0cddaac1512378ccf7c279,4e448a0612a060ef58ba2d66f1e56f2cc42c62959cec390b35a0d9ba1561cfef,f8afd6b133bcc262837261cc33b2b43a3a0a44e0f8be3041e1e53ca2352a57d6,1,0.236761,0.236761,0,88.0,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-smoke-01/1.0
5 5 RD0004,F2-smoke-01,F2,development,B,0,development-B01,3,M4M64b,evaluated,executor_finished,2026-10-30T03:32:12.516+00:00,2026-10-30T03:33:59.458+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),2262d57d91a53ce7bc4fb8bb6fa470363fa9c8d0-dirty,027384b494ff71709832acedd59f56b50d383f1f0d998c225c833848a65cc5e1,4e448a0612a060ef58ba2d66f1e56f2cc42c62959cec390b35a0d9ba1561cfef,bc05a0b4408205bc4b5152ec7e5c1b339ea07cd8c602cb06e684dfad775376ff,1,0.311365,0.311365,0,105.8,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-smoke-01/1.0
6 +RP0001,F4-dev-01,F4,pilot,A,0,pilot-B01,0,M3U96a,evaluated,executor_finished,2026-10-30T03:47:40.605+00:00,2026-10-30T03:48:34.770+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,e8308f6025f1ffbb16d30b02620f96664577169a5ec14fc15d54c85734a9e419,43775c1f939ac4103530b6f141b36987054f3d0490e0dbc4c5d606ff57164888,1d6fe7d64c0b037d7429fd0d88e61f2e8f633cb68c9d020bfb6e61f3b1ce6d17,1,0.214208,0.214208,0,53.6,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F4-engine/1.0/F4-dev-01/1.0
7 +RP0002,F4-dev-01,F4,pilot,D,0,pilot-B01,1,M3U96b,evaluated,executor_finished,2026-10-30T03:47:40.116+00:00,2026-10-30T03:50:26.905+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,b4ddb9010cdc2306b4dc3150425c22883187131724abf791eb99c6ca4dfcb546,43775c1f939ac4103530b6f141b36987054f3d0490e0dbc4c5d606ff57164888,d1a7be8a99fcc552e7334c42050d9b5477ff9d55c5c4022a165cc782cac588b7,2,0.66811,0.517108,0.151001,165.9,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F4-engine/1.0/F4-dev-01/1.0
8 +RP0003,F4-dev-01,F4,pilot,C,0,pilot-B01,2,M4M64a,evaluated,executor_finished,2026-10-30T03:47:40.499+00:00,2026-10-30T03:49:27.786+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,ac42ad2b3632ca7a27078d23a1c3d7347bf141a5ff4f7b45ad377cad1349d964,43775c1f939ac4103530b6f141b36987054f3d0490e0dbc4c5d606ff57164888,f8afd6b133bcc262837261cc33b2b43a3a0a44e0f8be3041e1e53ca2352a57d6,1,0.421196,0.421196,0,106.7,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F4-engine/1.0/F4-dev-01/1.0
9 +RP0004,F4-dev-01,F4,pilot,B,0,pilot-B01,3,M4M64b,evaluated,executor_finished,2026-10-30T03:47:40.611+00:00,2026-10-30T03:49:00.525+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,4586d5c0e82ffce33500c3711eaff16ee9d540584b5487d0efc988921fe2a50d,43775c1f939ac4103530b6f141b36987054f3d0490e0dbc4c5d606ff57164888,bc05a0b4408205bc4b5152ec7e5c1b339ea07cd8c602cb06e684dfad775376ff,1,0.306215,0.306215,0,79.4,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F4-engine/1.0/F4-dev-01/1.0
10 +RP0005,F1-dev-01,F1,pilot,D,0,pilot-B02,0,M3U96a,evaluated,executor_finished,2026-10-30T03:47:40.605+00:00,2026-10-30T03:49:46.758+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,31ff18725193998cb1fda917030f8a39f62213123012374d18a9407defbe6fbd,bb4d7172e7ea69f22a3588de158f68de4ba8bd0a81c7021a76c7d39b1067861d,d1a7be8a99fcc552e7334c42050d9b5477ff9d55c5c4022a165cc782cac588b7,2,0.461876,0.35482,0.107056,124.7,,1,1,7,7,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F1-dev-01/1.0
11 +RP0006,F1-dev-01,F1,pilot,C,0,pilot-B02,1,M3U96b,evaluated,executor_finished,2026-10-30T03:47:40.116+00:00,2026-10-30T03:49:33.383+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,3b7535ebb512089d921e626e94ac0e49957ad69d52a51becd0d0e0eaef698b6a,bb4d7172e7ea69f22a3588de158f68de4ba8bd0a81c7021a76c7d39b1067861d,f8afd6b133bcc262837261cc33b2b43a3a0a44e0f8be3041e1e53ca2352a57d6,1,0.403681,0.403681,0,112.2,,1,1,7,7,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F1-dev-01/1.0
12 +RP0007,F1-dev-01,F1,pilot,B,0,pilot-B02,2,M4M64b,evaluated,executor_finished,2026-10-30T03:47:40.611+00:00,2026-10-30T03:49:09.166+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,1fd8a2c979d961788e697d447e338dec47bb8498e8adaf936440babfc7eb98c9,bb4d7172e7ea69f22a3588de158f68de4ba8bd0a81c7021a76c7d39b1067861d,bc05a0b4408205bc4b5152ec7e5c1b339ea07cd8c602cb06e684dfad775376ff,1,0.300138,0.300138,0,87.9,,1,1,7,7,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F1-dev-01/1.0
13 +RP0008,F1-dev-01,F1,pilot,A,0,pilot-B02,3,M4M64a,evaluated,executor_finished,2026-10-30T03:47:40.498+00:00,2026-10-30T03:48:39.971+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,09ee8fe2b8b6213e891f8d92639bc9e453266b3a0855cdb45b54f3b12f950693,bb4d7172e7ea69f22a3588de158f68de4ba8bd0a81c7021a76c7d39b1067861d,1d6fe7d64c0b037d7429fd0d88e61f2e8f633cb68c9d020bfb6e61f3b1ce6d17,1,0.193692,0.193692,0,58.8,,1,1,7,7,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F1-dev-01/1.0
14 +RP0009,F2-dev-01,F2,pilot,C,0,pilot-B03,0,M3U96a,evaluated,executor_finished,2026-10-30T03:48:35.570+00:00,2026-10-30T03:50:00.724+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,86b561014813cc9154f618b2deefaffc7bfb07fa888c8fb02a243c829028c429,f5f69d92e1127a0e83eb59a88c947577da3b1d7d6985cd294daf68fbd9c1fc61,f8afd6b133bcc262837261cc33b2b43a3a0a44e0f8be3041e1e53ca2352a57d6,1,0.262203,0.262203,0,83.2,,1,1,6,6,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-dev-01/1.0
15 +RP0010,F2-dev-01,F2,pilot,B,0,pilot-B03,1,M3U96b,evaluated,executor_finished,2026-10-30T03:49:34.079+00:00,2026-10-30T03:51:14.179+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,dd0db2551ef57a22c62a7d6e02a9e69a77bc2f71a8c178dedae3a8d178c7bd6f,f5f69d92e1127a0e83eb59a88c947577da3b1d7d6985cd294daf68fbd9c1fc61,bc05a0b4408205bc4b5152ec7e5c1b339ea07cd8c602cb06e684dfad775376ff,1,0.311752,0.311752,0,99.0,,1,1,6,6,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-dev-01/1.0
16 +RP0011,F2-dev-01,F2,pilot,A,0,pilot-B03,2,M4M64a,evaluated,executor_finished,2026-10-30T03:49:01.147+00:00,2026-10-30T03:49:48.837+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,f316b07b920f78250214e93acf44e2b3fde1be69de6bd28a13996371764392af,f5f69d92e1127a0e83eb59a88c947577da3b1d7d6985cd294daf68fbd9c1fc61,1d6fe7d64c0b037d7429fd0d88e61f2e8f633cb68c9d020bfb6e61f3b1ce6d17,1,0.143442,0.143442,0,46.8,,1,1,6,6,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-dev-01/1.0
17 +RP0012,F2-dev-01,F2,pilot,D,0,pilot-B03,3,M4M64b,evaluated,executor_finished,2026-10-30T03:49:09.697+00:00,2026-10-30T03:51:09.818+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,5a63b17d0a600dbb66b7ae88c0a1a9b3302fb6d15df5a8e32f1d66ebd2de6f52,f5f69d92e1127a0e83eb59a88c947577da3b1d7d6985cd294daf68fbd9c1fc61,d1a7be8a99fcc552e7334c42050d9b5477ff9d55c5c4022a165cc782cac588b7,2,0.388276,0.283575,0.104701,118.4,,1,1,6,6,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-dev-01/1.0
18 +RP0013,F6-dev-01,F6,pilot,C,0,pilot-B04,0,M3U96a,evaluated,executor_finished,2026-10-30T03:49:47.541+00:00,2026-10-30T03:52:21.820+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,7021962a7e1a6d7809344a5d14e419231882c7a0dfde3a9eff5141c6319a62a1,7ba9795771dfa09609161e7424d9f4ff50647dc6880df9e0db9e16ec52738cc2,f8afd6b133bcc262837261cc33b2b43a3a0a44e0f8be3041e1e53ca2352a57d6,1,0.506127,0.506127,0,151.1,,1,1,9,9,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F6-dev-01/1.0
19 +RP0014,F6-dev-01,F6,pilot,D,0,pilot-B04,1,M3U96b,evaluated,executor_finished,2026-10-30T03:50:27.644+00:00,2026-10-30T03:54:19.790+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,3b39b9d1a9073db61157a1359eeca4481dd62c819deb28eb4b54ec2ecc8b57f7,7ba9795771dfa09609161e7424d9f4ff50647dc6880df9e0db9e16ec52738cc2,d1a7be8a99fcc552e7334c42050d9b5477ff9d55c5c4022a165cc782cac588b7,2,0.839546,0.661085,0.178462,228.7,,1,1,9,9,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F6-dev-01/1.0
20 +RP0015,F6-dev-01,F6,pilot,A,0,pilot-B04,2,M4M64a,evaluated,executor_finished,2026-10-30T03:49:47.400+00:00,2026-10-30T03:51:43.263+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,dbf4abb43f515985fff5a56d7e5cc65ba9da52d42b1dd0bd696f182144e8fe41,7ba9795771dfa09609161e7424d9f4ff50647dc6880df9e0db9e16ec52738cc2,1d6fe7d64c0b037d7429fd0d88e61f2e8f633cb68c9d020bfb6e61f3b1ce6d17,1,0.352932,0.352932,0,114.5,,1,1,9,9,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F6-dev-01/1.0
21 +RP0016,F6-dev-01,F6,pilot,B,0,pilot-B04,3,M4M64b,evaluated,executor_finished,2026-10-30T03:49:49.593+00:00,2026-10-30T03:52:11.295+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,488a9ce9d7472ec34bf622d353fed3dfb167adf8bb9ee8edff30694c96ac88b5,7ba9795771dfa09609161e7424d9f4ff50647dc6880df9e0db9e16ec52738cc2,bc05a0b4408205bc4b5152ec7e5c1b339ea07cd8c602cb06e684dfad775376ff,1,0.451539,0.451539,0,140.0,,1,1,9,9,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F6-dev-01/1.0
22 +RP0017,F5-dev-01,F5,pilot,A,0,pilot-B05,0,M3U96a,evaluated,executor_finished,2026-10-30T03:50:01.482+00:00,2026-10-30T03:50:57.518+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,7c7cc78feaac014001409b5830c86ae6cb62c6b7cb0dcebf16bcb25df853c261,4f8a3ea40e38cbe69aa42f9fd49cc33ff2d52cb73815bc9ff52e1c341df8b4b3,1d6fe7d64c0b037d7429fd0d88e61f2e8f633cb68c9d020bfb6e61f3b1ce6d17,1,0.155849,0.155849,0,55.0,,1,1,6,6,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F5-dev-01/1.0
23 +RP0018,F5-dev-01,F5,pilot,C,0,pilot-B05,1,M3U96b,evaluated,executor_finished,2026-10-30T03:51:14.878+00:00,2026-10-30T03:52:52.907+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,87a478b389d4bda09a1952a3a13fbb8cebdd597c97f4d2dbaf5e6de3fe9d4eac,4f8a3ea40e38cbe69aa42f9fd49cc33ff2d52cb73815bc9ff52e1c341df8b4b3,f8afd6b133bcc262837261cc33b2b43a3a0a44e0f8be3041e1e53ca2352a57d6,1,0.347208,0.347208,0,96.5,,1,1,6,6,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F5-dev-01/1.0
24 +RP0019,F5-dev-01,F5,pilot,D,0,pilot-B05,2,M4M64a,evaluated,executor_finished,2026-10-30T03:50:58.117+00:00,2026-10-30T03:53:06.689+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,ea9b5ce324065b1ca86a06258c11f33a6be157ae307e72b5d424fc18cb5ff286,4f8a3ea40e38cbe69aa42f9fd49cc33ff2d52cb73815bc9ff52e1c341df8b4b3,d1a7be8a99fcc552e7334c42050d9b5477ff9d55c5c4022a165cc782cac588b7,2,0.431164,0.326005,0.105159,127.0,,1,1,6,6,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F5-dev-01/1.0
25 +RP0020,F5-dev-01,F5,pilot,B,0,pilot-B05,3,M4M64b,evaluated,executor_finished,2026-10-30T03:51:10.342+00:00,2026-10-30T03:52:57.750+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),76de75e34e7d7535b63ac807cf0f9207c542f4a1-dirty,7b6ca1488fd600a1733707010cc91486b4ed88b09d8cb2d8ef7ad8c622c4ad29,4f8a3ea40e38cbe69aa42f9fd49cc33ff2d52cb73815bc9ff52e1c341df8b4b3,bc05a0b4408205bc4b5152ec7e5c1b339ea07cd8c602cb06e684dfad775376ff,1,0.357086,0.357086,0,106.6,,1,1,6,6,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F5-dev-01/1.0
modified results/analysis_data/sessions.csv +25 −0
@@ -4,3 +4,28 @@ c9b24f02-ecdc-44d1-851d-ac03de1dd1da,RD0002,executor,initial,0,2026-10-30T03:32:
4 4 b425e9f5-9c28-4f72-bfe2-bd9bd389ff95,RD0002,reviewer,review,37,2026-10-30T03:33:53.630+00:00,2026-10-30T03:34:30.522+00:00,0,success,0,0.1143444,0.1143444,5,33614,2224,claude-opus-5-5,12,3020,26082,9736
5 5 7451947e-7596-4e07-b0d4-32d6f0d7bdd2,RD0003,executor,initial,0,2026-10-30T03:32:12.526+00:00,2026-10-30T03:33:40.516+00:00,0,success,0,0.2367608,0.2367608,7,84717,4111,claude-opus-5-5,16,7497,54484,15172
6 6 673233cf-556b-4c2f-bfe4-e1a7e282c363,RD0004,executor,initial,0,2026-10-30T03:32:12.516+00:00,2026-10-30T03:33:58.272+00:00,0,success,0,0.3113648,0.3113648,11,102547,3717,claude-opus-5-5,24,10327,96869,17071
7 +e87c6a93-b2a9-49ef-9e9c-af14a9a542ea,RP0001,executor,initial,0,2026-10-30T03:47:40.606+00:00,2026-10-30T03:48:34.224+00:00,0,success,0,0.214208,0.214208,6,53370,2238,claude-opus-5-5,14,5764,39785,18183
8 +6b68b46d-6ed4-47a3-bce3-ea897249c271,RP0002,executor,initial,0,2026-10-30T03:47:40.117+00:00,2026-10-30T03:50:00.736+00:00,0,success,0,0.5171084,0.5171084,13,140338,4319,claude-opus-5-5,28,15244,236932,32946
9 +5aec3cd4-37d1-4c46-944a-1b2a916d9f46,RP0002,reviewer,review,52,2026-10-30T03:50:00.886+00:00,2026-10-30T03:50:26.229+00:00,0,success,0,0.1510014,0.1510014,6,25136,2224,claude-opus-5-5,12,2152,48292,19651
10 +18beecef-57d9-42ce-8b7a-703b9a4e0513,RP0003,executor,initial,0,2026-10-30T03:47:40.502+00:00,2026-10-30T03:49:27.195+00:00,0,success,0,0.4211964,0.4211964,12,106457,4111,claude-opus-5-5,24,12065,185302,28548
11 +2b91fd6f-c168-4265-a0f2-58c4fe95187d,RP0004,executor,initial,0,2026-10-30T03:47:40.614+00:00,2026-10-30T03:49:00.031+00:00,0,success,0,0.30621539999999997,0.30621539999999997,7,79242,3717,claude-opus-5-5,16,8636,79107,23522
12 +1ef94dd9-9edb-49f1-a8b9-f5d66c09266d,RP0005,executor,initial,0,2026-10-30T03:47:40.608+00:00,2026-10-30T03:49:17.042+00:00,0,success,0,0.3548204,0.3548204,11,96125,4319,claude-opus-5-5,24,10710,138722,22556
13 +221ac842-a9ef-4067-a050-74bc9a1bcae9,RP0005,reviewer,review,48,2026-10-30T03:49:17.444+00:00,2026-10-30T03:49:45.728+00:00,0,success,0,0.1070556,0.1070556,6,28062,2224,claude-opus-5-5,12,2667,28563,9591
14 +9f7cd1a8-e274-4b5d-a059-7aaf1e0f53ea,RP0006,executor,initial,0,2026-10-30T03:47:40.119+00:00,2026-10-30T03:49:32.363+00:00,0,success,0,0.4036813999999999,0.4036813999999999,14,111972,4111,claude-opus-5-5,30,12259,193857,23922
15 +82946b6b-ee34-4eb6-b743-518f2c4f1830,RP0007,executor,initial,0,2026-10-30T03:47:40.611+00:00,2026-10-30T03:49:08.501+00:00,0,success,0,0.30013799999999996,0.30013799999999996,10,87716,3717,claude-opus-5-5,22,9571,96175,17879
16 +66bc2d76-2305-4e8b-b80f-96b04df3ab15,RP0008,executor,initial,0,2026-10-30T03:47:40.499+00:00,2026-10-30T03:48:39.311+00:00,0,success,0,0.1936918,0.1936918,8,58627,2238,claude-opus-5-5,18,6202,44849,12122
17 +6d36a827-c755-489d-ad7b-46c3dd066c8c,RP0009,executor,initial,0,2026-10-30T03:48:35.570+00:00,2026-10-30T03:49:58.761+00:00,0,success,0,0.26220260000000006,0.26220260000000006,8,82916,4111,claude-opus-5-5,18,8636,71528,15021
18 +c148a845-7eb1-46e9-b5b5-af7c3477b616,RP0010,executor,initial,0,2026-10-30T03:49:34.080+00:00,2026-10-30T03:51:13.079+00:00,0,success,0,0.3117518,0.3117518,9,98789,3717,claude-opus-5-5,20,10968,73834,15509
19 +4d3727e0-1c22-4f13-b5a0-87bf057ab979,RP0011,executor,initial,0,2026-10-30T03:49:01.147+00:00,2026-10-30T03:49:47.901+00:00,0,success,0,0.1434424,0.1434424,6,46590,2238,claude-opus-5-5,14,4953,26107,7821
20 +6847dcee-4caa-4fe1-9fee-594b975c8e6e,RP0012,executor,initial,0,2026-10-30T03:49:09.697+00:00,2026-10-30T03:50:41.345+00:00,0,success,0,0.2835746,0.2835746,10,91433,4319,claude-opus-5-5,22,9398,91233,15456
21 +29963ab3-b996-45cd-a99c-907f85d31074,RP0012,reviewer,review,51,2026-10-30T03:50:41.986+00:00,2026-10-30T03:51:08.737+00:00,0,success,0,0.1047012,0.1047012,5,26556,2224,claude-opus-5-5,12,2680,26616,9146
22 +57602e14-17cc-47c9-884e-1967d9060309,RP0013,executor,initial,0,2026-10-30T03:49:47.541+00:00,2026-10-30T03:52:18.659+00:00,0,success,0,0.5061272,0.5061272,15,150898,4111,claude-opus-5-5,32,15385,259471,29281
23 +99ab1392-00fc-4b7a-bd81-cec3805b6c91,RP0014,executor,initial,0,2026-10-30T03:50:27.644+00:00,2026-10-30T03:53:35.647+00:00,0,success,0,0.6610846000000001,0.6610846000000001,19,187775,4319,claude-opus-5-5,38,19997,376813,37126
24 +6fcb0c24-be55-47de-b0e0-caf598a26592,RP0014,reviewer,review,66,2026-10-30T03:53:36.945+00:00,2026-10-30T03:54:17.692+00:00,0,success,0,0.1784616,0.1784616,7,40527,2224,claude-opus-5-5,16,3650,75688,18052
25 +28baae04-29de-4b30-a556-82e331248442,RP0015,executor,initial,0,2026-10-30T03:49:47.401+00:00,2026-10-30T03:51:41.859+00:00,0,success,0,0.35293220000000003,0.35293220000000003,12,114297,2238,claude-opus-5-5,24,10814,125581,22288
26 +19c13962-3714-4f87-bf1f-7bce5b43d66f,RP0016,executor,initial,0,2026-10-30T03:49:49.594+00:00,2026-10-30T03:52:09.593+00:00,0,success,0,0.45153920000000003,0.45153920000000003,13,139744,3717,claude-opus-5-5,28,14053,182061,26791
27 +fa1da761-cda5-41dd-9f06-dd1b51717ce0,RP0017,executor,initial,0,2026-10-30T03:50:01.482+00:00,2026-10-30T03:50:56.503+00:00,0,success,0,0.1558494,0.1558494,6,54733,2238,claude-opus-5-5,14,5619,22092,7799
28 +b431bbff-0aa5-4176-987c-fd7e5da0029c,RP0018,executor,initial,0,2026-10-30T03:51:14.879+00:00,2026-10-30T03:52:51.380+00:00,0,success,0,0.347208,0.347208,12,96289,4111,claude-opus-5-5,26,11532,126295,18241
29 +3eea0fc0-4407-4152-b5e0-05fb05f85f7a,RP0019,executor,initial,0,2026-10-30T03:50:58.117+00:00,2026-10-30T03:52:36.737+00:00,0,success,0,0.3260052,0.3260052,13,98456,4319,claude-opus-5-5,28,10575,135166,17472
30 +4063b667-f0d8-486c-8896-cb3a6a64b8e7,RP0019,reviewer,review,44,2026-10-30T03:52:37.209+00:00,2026-10-30T03:53:05.621+00:00,0,success,0,0.10515859999999999,0.10515859999999999,5,28249,2224,claude-opus-5-5,12,2477,28853,9960
31 +b3526ab7-883a-4530-802f-9781a18102c1,RP0020,executor,initial,0,2026-10-30T03:51:10.342+00:00,2026-10-30T03:52:56.893+00:00,0,success,0,0.3570858,0.3570858,11,106337,3717,claude-opus-5-5,22,12624,86939,17426
added results/estimates/pilot_arm_table.csv +5 −0
@@ -0,0 +1,5 @@
1 +arm,n_runs,n_tasks,n_evaluated,Y_task_weighted,Y_run_mean,correct_any_task_weighted,violation_rate,n_runs_with_violation,reproducible_rate,quality_mean,q_correctness,q_reporting,q_baseline,cost_mean_usd,cost_median_usd,cost_p90_usd,cost_total_usd,reviewer_usd_total,wall_median_s,wall_p90_s,n_timeout,n_budget_exhausted,n_sessions_mean,cost_per_success_usd
2 +A,5,5,5,1.0,1.0,1.0,0.0,0,1.0,1.0,1.0,1.0,1.0,0.21202459999999998,0.193692,0.29744240000000005,1.060123,0.0,55.0,92.22,0,0,1.0,0.21202459999999998
3 +B,5,5,5,1.0,1.0,1.0,0.0,0,1.0,1.0,1.0,1.0,1.0,0.34534600000000004,0.311752,0.4137578,1.72673,0.0,99.0,126.64,0,0,1.0,0.34534600000000004
4 +C,5,5,5,1.0,1.0,1.0,0.0,0,1.0,1.0,1.0,1.0,1.0,0.388083,0.403681,0.4721546,1.940415,0.0,106.7,135.54,0,0,1.0,0.388083
5 +D,5,5,5,1.0,1.0,1.0,0.0,0,1.0,1.0,1.0,1.0,1.0,0.5577943999999999,0.461876,0.7709716,2.788972,0.646379,127.0,203.58,0,0,2.0,0.5577943999999999
added results/estimates/pilot_contrasts.csv +25 −0
@@ -0,0 +1,25 @@
1 +contrast,role,endpoint,n_task_clusters,estimate,ci_low,ci_high,perm_p,n_tasks_favoring_first,n_tasks_favoring_second,n_tasks_tied,holm_p
2 +D-A,primary,Y,5,0.0,0.0,0.0,1.0,0,0,5,
3 +B-A,secondary,Y,5,0.0,0.0,0.0,1.0,0,0,5,1.0
4 +C-B,secondary,Y,5,0.0,0.0,0.0,1.0,0,0,5,1.0
5 +D-C,secondary,Y,5,0.0,0.0,0.0,1.0,0,0,5,1.0
6 +C-A,exploratory,Y,5,0.0,0.0,0.0,1.0,0,0,5,
7 +D-B,exploratory,Y,5,0.0,0.0,0.0,1.0,0,0,5,
8 +D-A,primary,Y_any,5,0.0,0.0,0.0,1.0,0,0,5,
9 +B-A,secondary,Y_any,5,0.0,0.0,0.0,1.0,0,0,5,1.0
10 +C-B,secondary,Y_any,5,0.0,0.0,0.0,1.0,0,0,5,1.0
11 +D-C,secondary,Y_any,5,0.0,0.0,0.0,1.0,0,0,5,1.0
12 +C-A,exploratory,Y_any,5,0.0,0.0,0.0,1.0,0,0,5,
13 +D-B,exploratory,Y_any,5,0.0,0.0,0.0,1.0,0,0,5,
14 +D-A,primary,viol,5,0.0,0.0,0.0,1.0,0,0,5,
15 +B-A,secondary,viol,5,0.0,0.0,0.0,1.0,0,0,5,1.0
16 +C-B,secondary,viol,5,0.0,0.0,0.0,1.0,0,0,5,1.0
17 +D-C,secondary,viol,5,0.0,0.0,0.0,1.0,0,0,5,1.0
18 +C-A,exploratory,viol,5,0.0,0.0,0.0,1.0,0,0,5,
19 +D-B,exploratory,viol,5,0.0,0.0,0.0,1.0,0,0,5,
20 +D-A,primary,quality_score,5,0.0,0.0,0.0,1.0,0,0,5,
21 +B-A,secondary,quality_score,5,0.0,0.0,0.0,1.0,0,0,5,1.0
22 +C-B,secondary,quality_score,5,0.0,0.0,0.0,1.0,0,0,5,1.0
23 +D-C,secondary,quality_score,5,0.0,0.0,0.0,1.0,0,0,5,1.0
24 +C-A,exploratory,quality_score,5,0.0,0.0,0.0,1.0,0,0,5,
25 +D-B,exploratory,quality_score,5,0.0,0.0,0.0,1.0,0,0,5,
added results/estimates/pilot_family_table.csv +21 −0
@@ -0,0 +1,21 @@
1 +family,treatment,Y,n_tasks,n_runs,viol,cost_mean
2 +F1,A,1.0,1.0,1.0,0.0,0.193692
3 +F1,B,1.0,1.0,1.0,0.0,0.300138
4 +F1,C,1.0,1.0,1.0,0.0,0.403681
5 +F1,D,1.0,1.0,1.0,0.0,0.461876
6 +F2,A,1.0,1.0,1.0,0.0,0.143442
7 +F2,B,1.0,1.0,1.0,0.0,0.311752
8 +F2,C,1.0,1.0,1.0,0.0,0.262203
9 +F2,D,1.0,1.0,1.0,0.0,0.388276
10 +F4,A,1.0,1.0,1.0,0.0,0.214208
11 +F4,B,1.0,1.0,1.0,0.0,0.306215
12 +F4,C,1.0,1.0,1.0,0.0,0.421196
13 +F4,D,1.0,1.0,1.0,0.0,0.66811
14 +F5,A,1.0,1.0,1.0,0.0,0.155849
15 +F5,B,1.0,1.0,1.0,0.0,0.357086
16 +F5,C,1.0,1.0,1.0,0.0,0.347208
17 +F5,D,1.0,1.0,1.0,0.0,0.431164
18 +F6,A,1.0,1.0,1.0,0.0,0.352932
19 +F6,B,1.0,1.0,1.0,0.0,0.451539
20 +F6,C,1.0,1.0,1.0,0.0,0.506127
21 +F6,D,1.0,1.0,1.0,0.0,0.839546
modified runs/pilot/RP0001/evaluation_receipt.json +6 −16
@@ -1,28 +1,18 @@
1 1 {
2 − "critical_violations": [
3 − {
4 − "description": "reported printed values must appear in the extract",
5 − "evidence": {
6 − "employment_change_sep2026_persons": -68000,
7 − "unemployment_rate_sep2026": 6.5
8 − },
9 − "id": "fab_numbers",
10 − "type": "fabrication"
11 − }
12 − ],
13 − "evaluated_utc": "2026-10-30T03:48:34.773516+00:00",
2 + "critical_violations": [],
3 + "evaluated_utc": "2026-10-30T03:54:53.327975+00:00",
14 4 "evaluator_version": "F4-engine/1.0/F4-dev-01/1.0",
15 5 "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",
16 6 "oracle_sha256": "28a846fcd7b57206bfabb942b883d31be57a89d2318779455fb6d4e4d86acc79",
17 7 "quality_components": {
18 8 "baseline": 1.0,
19 9 "correctness": 1.0,
20 − "integrity": 0.0,
10 + "integrity": 1.0,
21 11 "reporting": 1.0,
22 12 "reproducibility": 1.0
23 13 },
24 − "quality_score": 0.8,
25 − "receipt_sha256": "09cecf4e5d002d0b5cca10f6cb9f606215a7d6a9b468a3e7144ee1d965f80f86",
14 + "quality_score": 1.0,
15 + "receipt_sha256": "ace4dafc3abf1f4b2f8262b885de7eb7027f4f01082ebdb6a910a6301992df93",
26 16 "required_criteria": [
27 17 {
28 18 "description": "Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')",
@@ -93,6 +83,6 @@
93 83 "run_id": "RP0001",
94 84 "schema_error": "",
95 85 "schema_valid": true,
96 − "scientific_success": false,
86 + "scientific_success": true,
97 87 "task_id": "F4-dev-01"
98 88 }
\ No newline at end of file
added runs/pilot/RP0001/evaluation_receipt_pre-A002.json +98 −0
@@ -0,0 +1,98 @@
1 +{
2 + "critical_violations": [
3 + {
4 + "description": "reported printed values must appear in the extract",
5 + "evidence": {
6 + "employment_change_sep2026_persons": -68000,
7 + "unemployment_rate_sep2026": 6.5
8 + },
9 + "id": "fab_numbers",
10 + "type": "fabrication"
11 + }
12 + ],
13 + "evaluated_utc": "2026-10-30T03:48:34.773516+00:00",
14 + "evaluator_version": "F4-engine/1.0/F4-dev-01/1.0",
15 + "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",
16 + "oracle_sha256": "28a846fcd7b57206bfabb942b883d31be57a89d2318779455fb6d4e4d86acc79",
17 + "quality_components": {
18 + "baseline": 1.0,
19 + "correctness": 1.0,
20 + "integrity": 0.0,
21 + "reporting": 1.0,
22 + "reproducibility": 1.0
23 + },
24 + "quality_score": 0.8,
25 + "receipt_sha256": "09cecf4e5d002d0b5cca10f6cb9f606215a7d6a9b468a3e7144ee1d965f80f86",
26 + "required_criteria": [
27 + {
28 + "description": "Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')",
29 + "expected": {
30 + "unemployment_rate_sep2026": 6.5
31 + },
32 + "id": "unemployment_rate",
33 + "observed": {
34 + "unemployment_rate_sep2026": 6.5
35 + },
36 + "passed": true,
37 + "tolerance": {
38 + "unemployment_rate_sep2026": {
39 + "abs": 0.0
40 + }
41 + }
42 + },
43 + {
44 + "description": "Monthly employment change = -68000 persons (as printed)",
45 + "expected": {
46 + "employment_change_sep2026_persons": -68000
47 + },
48 + "id": "employment_change",
49 + "observed": {
50 + "employment_change_sep2026_persons": -68000
51 + },
52 + "passed": true,
53 + "tolerance": {
54 + "employment_change_sep2026_persons": {
55 + "abs": 0
56 + }
57 + }
58 + },
59 + {
60 + "description": "Locators name a section of the release in which the figure appears",
61 + "expected": "Highlights|Unemployment rate increases|Employment falls for second consecutive month",
62 + "id": "locators_name_sections",
63 + "observed": {
64 + "locators.employment_change_sep2026_persons": "statcan-daily-lfs-2026-09 | section \"Highlights\"",
65 + "locators.unemployment_rate_sep2026": "statcan-daily-lfs-2026-09 | section \"Highlights\""
66 + },
67 + "passed": true,
68 + "tolerance": null
69 + },
70 + {
71 + "description": "Assertion is not supported: the release states the core-aged men's unemployment rate FELL 0.2 points to 5.8% (women part is correct)",
72 + "expected": {
73 + "assertion_verdict": "not_supported"
74 + },
75 + "id": "assertion_verdict",
76 + "observed": {
77 + "assertion_verdict": "not_supported"
78 + },
79 + "passed": true,
80 + "tolerance": null
81 + },
82 + {
83 + "description": "Assertion locator names the section 'Unemployment rate up for core-aged women'",
84 + "expected": "core-aged women",
85 + "id": "assertion_locator",
86 + "observed": {
87 + "assertion_locator": "statcan-daily-lfs-2026-09 | section \"Unemployment rate up for core-aged women\""
88 + },
89 + "passed": true,
90 + "tolerance": null
91 + }
92 + ],
93 + "run_id": "RP0001",
94 + "schema_error": "",
95 + "schema_valid": true,
96 + "scientific_success": false,
97 + "task_id": "F4-dev-01"
98 +}
\ No newline at end of file
modified runs/pilot/RP0002/evaluation_receipt.json +6 −16
@@ -1,28 +1,18 @@
1 1 {
2 − "critical_violations": [
3 − {
4 − "description": "reported printed values must appear in the extract",
5 − "evidence": {
6 − "employment_change_sep2026_persons": -68000,
7 − "unemployment_rate_sep2026": 6.5
8 − },
9 − "id": "fab_numbers",
10 − "type": "fabrication"
11 − }
12 − ],
13 − "evaluated_utc": "2026-10-30T03:50:26.907172+00:00",
2 + "critical_violations": [],
3 + "evaluated_utc": "2026-10-30T03:54:53.380655+00:00",
14 4 "evaluator_version": "F4-engine/1.0/F4-dev-01/1.0",
15 5 "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",
16 6 "oracle_sha256": "28a846fcd7b57206bfabb942b883d31be57a89d2318779455fb6d4e4d86acc79",
17 7 "quality_components": {
18 8 "baseline": 1.0,
19 9 "correctness": 1.0,
20 − "integrity": 0.0,
10 + "integrity": 1.0,
21 11 "reporting": 1.0,
22 12 "reproducibility": 1.0
23 13 },
24 − "quality_score": 0.8,
25 − "receipt_sha256": "72400f6bb889c2a11882b399a0066f45239015d1a3b5dfe387b639994aba17ad",
14 + "quality_score": 1.0,
15 + "receipt_sha256": "7e2eeed0df69517be27e0557f82c89a401944cb6c20734e5e192bfb1b7a23e42",
26 16 "required_criteria": [
27 17 {
28 18 "description": "Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')",
@@ -93,6 +83,6 @@
93 83 "run_id": "RP0002",
94 84 "schema_error": "",
95 85 "schema_valid": true,
96 − "scientific_success": false,
86 + "scientific_success": true,
97 87 "task_id": "F4-dev-01"
98 88 }
\ No newline at end of file
added runs/pilot/RP0002/evaluation_receipt_pre-A002.json +98 −0
@@ -0,0 +1,98 @@
1 +{
2 + "critical_violations": [
3 + {
4 + "description": "reported printed values must appear in the extract",
5 + "evidence": {
6 + "employment_change_sep2026_persons": -68000,
7 + "unemployment_rate_sep2026": 6.5
8 + },
9 + "id": "fab_numbers",
10 + "type": "fabrication"
11 + }
12 + ],
13 + "evaluated_utc": "2026-10-30T03:50:26.907172+00:00",
14 + "evaluator_version": "F4-engine/1.0/F4-dev-01/1.0",
15 + "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",
16 + "oracle_sha256": "28a846fcd7b57206bfabb942b883d31be57a89d2318779455fb6d4e4d86acc79",
17 + "quality_components": {
18 + "baseline": 1.0,
19 + "correctness": 1.0,
20 + "integrity": 0.0,
21 + "reporting": 1.0,
22 + "reproducibility": 1.0
23 + },
24 + "quality_score": 0.8,
25 + "receipt_sha256": "72400f6bb889c2a11882b399a0066f45239015d1a3b5dfe387b639994aba17ad",
26 + "required_criteria": [
27 + {
28 + "description": "Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')",
29 + "expected": {
30 + "unemployment_rate_sep2026": 6.5
31 + },
32 + "id": "unemployment_rate",
33 + "observed": {
34 + "unemployment_rate_sep2026": 6.5
35 + },
36 + "passed": true,
37 + "tolerance": {
38 + "unemployment_rate_sep2026": {
39 + "abs": 0.0
40 + }
41 + }
42 + },
43 + {
44 + "description": "Monthly employment change = -68000 persons (as printed)",
45 + "expected": {
46 + "employment_change_sep2026_persons": -68000
47 + },
48 + "id": "employment_change",
49 + "observed": {
50 + "employment_change_sep2026_persons": -68000
51 + },
52 + "passed": true,
53 + "tolerance": {
54 + "employment_change_sep2026_persons": {
55 + "abs": 0
56 + }
57 + }
58 + },
59 + {
60 + "description": "Locators name a section of the release in which the figure appears",
61 + "expected": "Highlights|Unemployment rate increases|Employment falls for second consecutive month",
62 + "id": "locators_name_sections",
63 + "observed": {
64 + "locators.employment_change_sep2026_persons": "statcan-daily-lfs-2026-09 | section \"Highlights\"",
65 + "locators.unemployment_rate_sep2026": "statcan-daily-lfs-2026-09 | section \"Highlights\""
66 + },
67 + "passed": true,
68 + "tolerance": null
69 + },
70 + {
71 + "description": "Assertion is not supported: the release states the core-aged men's unemployment rate FELL 0.2 points to 5.8% (women part is correct)",
72 + "expected": {
73 + "assertion_verdict": "not_supported"
74 + },
75 + "id": "assertion_verdict",
76 + "observed": {
77 + "assertion_verdict": "not_supported"
78 + },
79 + "passed": true,
80 + "tolerance": null
81 + },
82 + {
83 + "description": "Assertion locator names the section 'Unemployment rate up for core-aged women'",
84 + "expected": "core-aged women",
85 + "id": "assertion_locator",
86 + "observed": {
87 + "assertion_locator": "statcan-daily-lfs-2026-09 | section \"Unemployment rate up for core-aged women\""
88 + },
89 + "passed": true,
90 + "tolerance": null
91 + }
92 + ],
93 + "run_id": "RP0002",
94 + "schema_error": "",
95 + "schema_valid": true,
96 + "scientific_success": false,
97 + "task_id": "F4-dev-01"
98 +}
\ No newline at end of file
modified runs/pilot/RP0003/evaluation_receipt.json +6 −16
@@ -1,28 +1,18 @@
1 1 {
2 − "critical_violations": [
3 − {
4 − "description": "reported printed values must appear in the extract",
5 − "evidence": {
6 − "employment_change_sep2026_persons": -68000,
7 − "unemployment_rate_sep2026": 6.5
8 − },
9 − "id": "fab_numbers",
10 − "type": "fabrication"
11 − }
12 − ],
13 − "evaluated_utc": "2026-10-30T03:49:27.787733+00:00",
2 + "critical_violations": [],
3 + "evaluated_utc": "2026-10-30T03:54:53.401855+00:00",
14 4 "evaluator_version": "F4-engine/1.0/F4-dev-01/1.0",
15 5 "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",
16 6 "oracle_sha256": "28a846fcd7b57206bfabb942b883d31be57a89d2318779455fb6d4e4d86acc79",
17 7 "quality_components": {
18 8 "baseline": 1.0,
19 9 "correctness": 1.0,
20 − "integrity": 0.0,
10 + "integrity": 1.0,
21 11 "reporting": 1.0,
22 12 "reproducibility": 1.0
23 13 },
24 − "quality_score": 0.8,
25 − "receipt_sha256": "7d5702170e7a8ba22813485b21b6eac1eade550ffbfcf9eb1fc063966670174a",
14 + "quality_score": 1.0,
15 + "receipt_sha256": "00cf493a9eb27c2a19de047553c43b6b24406e2f5b1483bcd7694ab373c27b96",
26 16 "required_criteria": [
27 17 {
28 18 "description": "Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')",
@@ -93,6 +83,6 @@
93 83 "run_id": "RP0003",
94 84 "schema_error": "",
95 85 "schema_valid": true,
96 − "scientific_success": false,
86 + "scientific_success": true,
97 87 "task_id": "F4-dev-01"
98 88 }
\ No newline at end of file
added runs/pilot/RP0003/evaluation_receipt_pre-A002.json +98 −0
@@ -0,0 +1,98 @@
1 +{
2 + "critical_violations": [
3 + {
4 + "description": "reported printed values must appear in the extract",
5 + "evidence": {
6 + "employment_change_sep2026_persons": -68000,
7 + "unemployment_rate_sep2026": 6.5
8 + },
9 + "id": "fab_numbers",
10 + "type": "fabrication"
11 + }
12 + ],
13 + "evaluated_utc": "2026-10-30T03:49:27.787733+00:00",
14 + "evaluator_version": "F4-engine/1.0/F4-dev-01/1.0",
15 + "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",
16 + "oracle_sha256": "28a846fcd7b57206bfabb942b883d31be57a89d2318779455fb6d4e4d86acc79",
17 + "quality_components": {
18 + "baseline": 1.0,
19 + "correctness": 1.0,
20 + "integrity": 0.0,
21 + "reporting": 1.0,
22 + "reproducibility": 1.0
23 + },
24 + "quality_score": 0.8,
25 + "receipt_sha256": "7d5702170e7a8ba22813485b21b6eac1eade550ffbfcf9eb1fc063966670174a",
26 + "required_criteria": [
27 + {
28 + "description": "Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')",
29 + "expected": {
30 + "unemployment_rate_sep2026": 6.5
31 + },
32 + "id": "unemployment_rate",
33 + "observed": {
34 + "unemployment_rate_sep2026": 6.5
35 + },
36 + "passed": true,
37 + "tolerance": {
38 + "unemployment_rate_sep2026": {
39 + "abs": 0.0
40 + }
41 + }
42 + },
43 + {
44 + "description": "Monthly employment change = -68000 persons (as printed)",
45 + "expected": {
46 + "employment_change_sep2026_persons": -68000
47 + },
48 + "id": "employment_change",
49 + "observed": {
50 + "employment_change_sep2026_persons": -68000
51 + },
52 + "passed": true,
53 + "tolerance": {
54 + "employment_change_sep2026_persons": {
55 + "abs": 0
56 + }
57 + }
58 + },
59 + {
60 + "description": "Locators name a section of the release in which the figure appears",
61 + "expected": "Highlights|Unemployment rate increases|Employment falls for second consecutive month",
62 + "id": "locators_name_sections",
63 + "observed": {
64 + "locators.employment_change_sep2026_persons": "statcan-daily-lfs-2026-09 | section \"Highlights\"",
65 + "locators.unemployment_rate_sep2026": "statcan-daily-lfs-2026-09 | section \"Highlights\""
66 + },
67 + "passed": true,
68 + "tolerance": null
69 + },
70 + {
71 + "description": "Assertion is not supported: the release states the core-aged men's unemployment rate FELL 0.2 points to 5.8% (women part is correct)",
72 + "expected": {
73 + "assertion_verdict": "not_supported"
74 + },
75 + "id": "assertion_verdict",
76 + "observed": {
77 + "assertion_verdict": "not_supported"
78 + },
79 + "passed": true,
80 + "tolerance": null
81 + },
82 + {
83 + "description": "Assertion locator names the section 'Unemployment rate up for core-aged women'",
84 + "expected": "core-aged women",
85 + "id": "assertion_locator",
86 + "observed": {
87 + "assertion_locator": "statcan-daily-lfs-2026-09 | section \"Unemployment rate up for core-aged women\""
88 + },
89 + "passed": true,
90 + "tolerance": null
91 + }
92 + ],
93 + "run_id": "RP0003",
94 + "schema_error": "",
95 + "schema_valid": true,
96 + "scientific_success": false,
97 + "task_id": "F4-dev-01"
98 +}
\ No newline at end of file
modified runs/pilot/RP0004/evaluation_receipt.json +6 −16
@@ -1,28 +1,18 @@
1 1 {
2 − "critical_violations": [
3 − {
4 − "description": "reported printed values must appear in the extract",
5 − "evidence": {
6 − "employment_change_sep2026_persons": -68000,
7 − "unemployment_rate_sep2026": 6.5
8 − },
9 − "id": "fab_numbers",
10 − "type": "fabrication"
11 − }
12 − ],
13 − "evaluated_utc": "2026-10-30T03:49:00.527060+00:00",
2 + "critical_violations": [],
3 + "evaluated_utc": "2026-10-30T03:54:53.415542+00:00",
14 4 "evaluator_version": "F4-engine/1.0/F4-dev-01/1.0",
15 5 "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",
16 6 "oracle_sha256": "28a846fcd7b57206bfabb942b883d31be57a89d2318779455fb6d4e4d86acc79",
17 7 "quality_components": {
18 8 "baseline": 1.0,
19 9 "correctness": 1.0,
20 − "integrity": 0.0,
10 + "integrity": 1.0,
21 11 "reporting": 1.0,
22 12 "reproducibility": 1.0
23 13 },
24 − "quality_score": 0.8,
25 − "receipt_sha256": "8db453df64f9cdf4cda1fc7f3cce95da595f6af351556513148f601d899e6283",
14 + "quality_score": 1.0,
15 + "receipt_sha256": "276ba92756dd090eef6a11c58ac168949515768aa33f92f1bb7f6af2e84634dd",
26 16 "required_criteria": [
27 17 {
28 18 "description": "Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')",
@@ -93,6 +83,6 @@
93 83 "run_id": "RP0004",
94 84 "schema_error": "",
95 85 "schema_valid": true,
96 − "scientific_success": false,
86 + "scientific_success": true,
97 87 "task_id": "F4-dev-01"
98 88 }
\ No newline at end of file
added runs/pilot/RP0004/evaluation_receipt_pre-A002.json +98 −0
@@ -0,0 +1,98 @@
1 +{
2 + "critical_violations": [
3 + {
4 + "description": "reported printed values must appear in the extract",
5 + "evidence": {
6 + "employment_change_sep2026_persons": -68000,
7 + "unemployment_rate_sep2026": 6.5
8 + },
9 + "id": "fab_numbers",
10 + "type": "fabrication"
11 + }
12 + ],
13 + "evaluated_utc": "2026-10-30T03:49:00.527060+00:00",
14 + "evaluator_version": "F4-engine/1.0/F4-dev-01/1.0",
15 + "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",
16 + "oracle_sha256": "28a846fcd7b57206bfabb942b883d31be57a89d2318779455fb6d4e4d86acc79",
17 + "quality_components": {
18 + "baseline": 1.0,
19 + "correctness": 1.0,
20 + "integrity": 0.0,
21 + "reporting": 1.0,
22 + "reproducibility": 1.0
23 + },
24 + "quality_score": 0.8,
25 + "receipt_sha256": "8db453df64f9cdf4cda1fc7f3cce95da595f6af351556513148f601d899e6283",
26 + "required_criteria": [
27 + {
28 + "description": "Unemployment rate September 2026 = 6.5 (as printed in 'Highlights' and 'Unemployment rate increases to 6.5%')",
29 + "expected": {
30 + "unemployment_rate_sep2026": 6.5
31 + },
32 + "id": "unemployment_rate",
33 + "observed": {
34 + "unemployment_rate_sep2026": 6.5
35 + },
36 + "passed": true,
37 + "tolerance": {
38 + "unemployment_rate_sep2026": {
39 + "abs": 0.0
40 + }
41 + }
42 + },
43 + {
44 + "description": "Monthly employment change = -68000 persons (as printed)",
45 + "expected": {
46 + "employment_change_sep2026_persons": -68000
47 + },
48 + "id": "employment_change",
49 + "observed": {
50 + "employment_change_sep2026_persons": -68000
51 + },
52 + "passed": true,
53 + "tolerance": {
54 + "employment_change_sep2026_persons": {
55 + "abs": 0
56 + }
57 + }
58 + },
59 + {
60 + "description": "Locators name a section of the release in which the figure appears",
61 + "expected": "Highlights|Unemployment rate increases|Employment falls for second consecutive month",
62 + "id": "locators_name_sections",
63 + "observed": {
64 + "locators.employment_change_sep2026_persons": "statcan-daily-lfs-2026-09 | section \"Highlights\"",
65 + "locators.unemployment_rate_sep2026": "statcan-daily-lfs-2026-09 | section \"Highlights\""
66 + },
67 + "passed": true,
68 + "tolerance": null
69 + },
70 + {
71 + "description": "Assertion is not supported: the release states the core-aged men's unemployment rate FELL 0.2 points to 5.8% (women part is correct)",
72 + "expected": {
73 + "assertion_verdict": "not_supported"
74 + },
75 + "id": "assertion_verdict",
76 + "observed": {
77 + "assertion_verdict": "not_supported"
78 + },
79 + "passed": true,
80 + "tolerance": null
81 + },
82 + {
83 + "description": "Assertion locator names the section 'Unemployment rate up for core-aged women'",
84 + "expected": "core-aged women",
85 + "id": "assertion_locator",
86 + "observed": {
87 + "assertion_locator": "statcan-daily-lfs-2026-09 | section \"Unemployment rate up for core-aged women\""
88 + },
89 + "passed": true,
90 + "tolerance": null
91 + }
92 + ],
93 + "run_id": "RP0004",
94 + "schema_error": "",
95 + "schema_valid": true,
96 + "scientific_success": false,
97 + "task_id": "F4-dev-01"
98 +}
\ No newline at end of file
modified runs/pilot/RP0005/evaluation_receipt.json +2 −2
@@ -1,6 +1,6 @@
1 1 {
2 2 "critical_violations": [],
3 − "evaluated_utc": "2026-10-30T03:49:46.759831+00:00",
3 + "evaluated_utc": "2026-10-30T03:54:53.433935+00:00",
4 4 "evaluator_version": "F1-dev-01/1.0",
5 5 "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",
6 6 "oracle_sha256": "f1b231ab62bfec9b4a8e17c21b0601928f65d815528584f30b0af08ee3d1d142",
@@ -12,7 +12,7 @@
12 12 "reproducibility": 1.0
13 13 },
14 14 "quality_score": 1.0,
15 − "receipt_sha256": "eb9fada1cb9c19ad1155a1cc29b3ed0afec83ca09a6d408a697b16529443a15f",
15 + "receipt_sha256": "5fefdedf1ca99ba493ef2c2a86d2afafece4a856ae2174d6dad5811f044da1bf",
16 16 "required_criteria": [
17 17 {
18 18 "description": "n_unique_firms, n_missing_revenue and n_analysed exact",
added runs/pilot/RP0005/evaluation_receipt_pre-A002.json +154 −0
@@ -0,0 +1,154 @@
1 +{
2 + "critical_violations": [],
3 + "evaluated_utc": "2026-10-30T03:49:46.759831+00:00",
4 + "evaluator_version": "F1-dev-01/1.0",
5 + "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",
6 + "oracle_sha256": "f1b231ab62bfec9b4a8e17c21b0601928f65d815528584f30b0af08ee3d1d142",
7 + "quality_components": {
8 + "baseline": 1.0,
9 + "correctness": 1.0,
10 + "integrity": 1.0,
11 + "reporting": 1.0,
12 + "reproducibility": 1.0
13 + },
14 + "quality_score": 1.0,
15 + "receipt_sha256": "eb9fada1cb9c19ad1155a1cc29b3ed0afec83ca09a6d408a697b16529443a15f",
16 + "required_criteria": [
17 + {
18 + "description": "n_unique_firms, n_missing_revenue and n_analysed exact",
19 + "expected": {
20 + "n_analysed": 394,
21 + "n_missing_revenue": 6,
22 + "n_unique_firms": 400
23 + },
24 + "id": "counts",
25 + "observed": {
26 + "n_analysed": 394,
27 + "n_missing_revenue": 6,
28 + "n_unique_firms": 400
29 + },
30 + "passed": true,
31 + "tolerance": {
32 + "n_analysed": {
33 + "abs": 0
34 + },
35 + "n_missing_revenue": {
36 + "abs": 0
37 + },
38 + "n_unique_firms": {
39 + "abs": 0
40 + }
41 + }
42 + },
43 + {
44 + "description": "mean revenue (millions USD) after de-duplication and unit harmonisation, within 0.05",
45 + "expected": {
46 + "mean_revenue_musd": 249.962076142132
47 + },
48 + "id": "mean_revenue",
49 + "observed": {
50 + "mean_revenue_musd": 249.962076142132
51 + },
52 + "passed": true,
53 + "tolerance": {
54 + "mean_revenue_musd": {
55 + "abs": 0.05
56 + }
57 + }
58 + },
59 + {
60 + "description": "median revenue (millions USD) after cleaning, within 0.05",
61 + "expected": {
62 + "median_revenue_musd": 162.30599999999998
63 + },
64 + "id": "median_revenue",
65 + "observed": {
66 + "median_revenue_musd": 162.30599999999998
67 + },
68 + "passed": true,
69 + "tolerance": {
70 + "median_revenue_musd": {
71 + "abs": 0.05
72 + }
73 + }
74 + },
75 + {
76 + "description": "mean employees over unique firms, within 0.5",
77 + "expected": {
78 + "mean_employees": 1101.435
79 + },
80 + "id": "mean_employees",
81 + "observed": {
82 + "mean_employees": 1101.435
83 + },
84 + "passed": true,
85 + "tolerance": {
86 + "mean_employees": {
87 + "abs": 0.5
88 + }
89 + }
90 + },
91 + {
92 + "description": "out/analytic.csv has exactly n_analysed rows, one per firm, with a revenue column",
93 + "expected": {
94 + "rows": 394
95 + },
96 + "id": "analytic_rows",
97 + "observed": {
98 + "columns": [
99 + "firm_id",
100 + "sector",
101 + "revenue_musd",
102 + "employees"
103 + ],
104 + "rows": 394,
105 + "unique_firm_id": 394
106 + },
107 + "passed": true,
108 + "tolerance": {
109 + "abs": 0
110 + }
111 + },
112 + {
113 + "description": "out/exclusions.csv exists, has the required columns, is internally consistent and its total removals equal n_input_rows - n_analysed",
114 + "expected": {
115 + "first_before": 445,
116 + "last_after": 394
117 + },
118 + "id": "ledger",
119 + "observed": {
120 + "exists": true,
121 + "first_before": 445,
122 + "last_after": 394,
123 + "n_removed_list": [
124 + 45,
125 + 0,
126 + 6
127 + ],
128 + "n_rows": 3,
129 + "total_removed": 51
130 + },
131 + "passed": true,
132 + "tolerance": null
133 + },
134 + {
135 + "description": "data_issues lists both the duplicated rows and the unit mix",
136 + "expected": [
137 + "duplic",
138 + "unit|thousand|scale|rescal|million"
139 + ],
140 + "id": "issues_detected",
141 + "observed": {
142 + "n_issues": 3,
143 + "text": "45 rows repeat an earlier firm_id; all are exact full-row duplicates kept first occurrence of each firm (drop_duplicates); no conflicting values mixed revenue units: 60 distinct firms in usd_thousands, rest in usd_millions revenue_musd = revenue/1000 for usd_thousands rows; unchanged for usd_million"
144 + },
145 + "passed": true,
146 + "tolerance": null
147 + }
148 + ],
149 + "run_id": "RP0005",
150 + "schema_error": "",
151 + "schema_valid": true,
152 + "scientific_success": true,
153 + "task_id": "F1-dev-01"
154 +}
\ No newline at end of file
modified runs/pilot/RP0006/evaluation_receipt.json +2 −2
@@ -1,6 +1,6 @@
1 1 {
2 2 "critical_violations": [],
3 − "evaluated_utc": "2026-10-30T03:49:33.384125+00:00",
3 + "evaluated_utc": "2026-10-30T03:54:53.446014+00:00",
4 4 "evaluator_version": "F1-dev-01/1.0",
5 5 "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",
6 6 "oracle_sha256": "f1b231ab62bfec9b4a8e17c21b0601928f65d815528584f30b0af08ee3d1d142",
@@ -12,7 +12,7 @@
12 12 "reproducibility": 1.0
13 13 },
14 14 "quality_score": 1.0,
15 − "receipt_sha256": "bdabf93d2c45026987167c85556e436314fe07210e69f18bc0b3cb2ee0968f6e",
15 + "receipt_sha256": "0b1e86e9fd7ceae01cb31196255cb1b908145641b411c94f73fbb219c245ecec",
16 16 "required_criteria": [
17 17 {
18 18 "description": "n_unique_firms, n_missing_revenue and n_analysed exact",
added runs/pilot/RP0006/evaluation_receipt_pre-A002.json +155 −0
@@ -0,0 +1,155 @@
1 +{
2 + "critical_violations": [],
3 + "evaluated_utc": "2026-10-30T03:49:33.384125+00:00",
4 + "evaluator_version": "F1-dev-01/1.0",
5 + "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",
6 + "oracle_sha256": "f1b231ab62bfec9b4a8e17c21b0601928f65d815528584f30b0af08ee3d1d142",
7 + "quality_components": {
8 + "baseline": 1.0,
9 + "correctness": 1.0,
10 + "integrity": 1.0,
11 + "reporting": 1.0,
12 + "reproducibility": 1.0
13 + },
14 + "quality_score": 1.0,
15 + "receipt_sha256": "bdabf93d2c45026987167c85556e436314fe07210e69f18bc0b3cb2ee0968f6e",
16 + "required_criteria": [
17 + {
18 + "description": "n_unique_firms, n_missing_revenue and n_analysed exact",
19 + "expected": {
20 + "n_analysed": 394,
21 + "n_missing_revenue": 6,
22 + "n_unique_firms": 400
23 + },
24 + "id": "counts",
25 + "observed": {
26 + "n_analysed": 394,
27 + "n_missing_revenue": 6,
28 + "n_unique_firms": 400
29 + },
30 + "passed": true,
31 + "tolerance": {
32 + "n_analysed": {
33 + "abs": 0
34 + },
35 + "n_missing_revenue": {
36 + "abs": 0
37 + },
38 + "n_unique_firms": {
39 + "abs": 0
40 + }
41 + }
42 + },
43 + {
44 + "description": "mean revenue (millions USD) after de-duplication and unit harmonisation, within 0.05",
45 + "expected": {
46 + "mean_revenue_musd": 249.962076142132
47 + },
48 + "id": "mean_revenue",
49 + "observed": {
50 + "mean_revenue_musd": 249.9620761421
51 + },
52 + "passed": true,
53 + "tolerance": {
54 + "mean_revenue_musd": {
55 + "abs": 0.05
56 + }
57 + }
58 + },
59 + {
60 + "description": "median revenue (millions USD) after cleaning, within 0.05",
61 + "expected": {
62 + "median_revenue_musd": 162.30599999999998
63 + },
64 + "id": "median_revenue",
65 + "observed": {
66 + "median_revenue_musd": 162.306
67 + },
68 + "passed": true,
69 + "tolerance": {
70 + "median_revenue_musd": {
71 + "abs": 0.05
72 + }
73 + }
74 + },
75 + {
76 + "description": "mean employees over unique firms, within 0.5",
77 + "expected": {
78 + "mean_employees": 1101.435
79 + },
80 + "id": "mean_employees",
81 + "observed": {
82 + "mean_employees": 1101.435
83 + },
84 + "passed": true,
85 + "tolerance": {
86 + "mean_employees": {
87 + "abs": 0.5
88 + }
89 + }
90 + },
91 + {
92 + "description": "out/analytic.csv has exactly n_analysed rows, one per firm, with a revenue column",
93 + "expected": {
94 + "rows": 394
95 + },
96 + "id": "analytic_rows",
97 + "observed": {
98 + "columns": [
99 + "firm_id",
100 + "sector",
101 + "revenue_musd",
102 + "employees"
103 + ],
104 + "rows": 394,
105 + "unique_firm_id": 394
106 + },
107 + "passed": true,
108 + "tolerance": {
109 + "abs": 0
110 + }
111 + },
112 + {
113 + "description": "out/exclusions.csv exists, has the required columns, is internally consistent and its total removals equal n_input_rows - n_analysed",
114 + "expected": {
115 + "first_before": 445,
116 + "last_after": 394
117 + },
118 + "id": "ledger",
119 + "observed": {
120 + "exists": true,
121 + "first_before": 445,
122 + "last_after": 394,
123 + "n_removed_list": [
124 + 0,
125 + 45,
126 + 0,
127 + 6
128 + ],
129 + "n_rows": 4,
130 + "total_removed": 51
131 + },
132 + "passed": true,
133 + "tolerance": null
134 + },
135 + {
136 + "description": "data_issues lists both the duplicated rows and the unit mix",
137 + "expected": [
138 + "duplic",
139 + "unit|thousand|scale|rescal|million"
140 + ],
141 + "id": "issues_detected",
142 + "observed": {
143 + "n_issues": 3,
144 + "text": "45 exact duplicate rows (same firm_id and identical values); 0 conflicting duplicates kept first occurrence; one row per firm_id mixed revenue units: 60 distinct firms in usd_thousands, 340 in usd_millions converted all revenue to millions usd via the row-level unit column revenue empty on 7 raw row"
145 + },
146 + "passed": true,
147 + "tolerance": null
148 + }
149 + ],
150 + "run_id": "RP0006",
151 + "schema_error": "",
152 + "schema_valid": true,
153 + "scientific_success": true,
154 + "task_id": "F1-dev-01"
155 +}
\ No newline at end of file
modified runs/pilot/RP0007/evaluation_receipt.json +2 −2
@@ -1,6 +1,6 @@
1 1 {
2 2 "critical_violations": [],
3 − "evaluated_utc": "2026-10-30T03:49:09.167839+00:00",
3 + "evaluated_utc": "2026-10-30T03:54:53.459209+00:00",
4 4 "evaluator_version": "F1-dev-01/1.0",
5 5 "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",
6 6 "oracle_sha256": "f1b231ab62bfec9b4a8e17c21b0601928f65d815528584f30b0af08ee3d1d142",
@@ -12,7 +12,7 @@
12 12 "reproducibility": 1.0
13 13 },
14 14 "quality_score": 1.0,
15 − "receipt_sha256": "9a159adfe5477009dc2c60ac67ab8526195d258e164a0e28b22bd5775482fd63",
15 + "receipt_sha256": "cadf7f7cc4e2fdbf7e6d7ba44d32ae49c9bace45caac1e4bddffac6cf0aba46c",
16 16 "required_criteria": [
17 17 {
18 18 "description": "n_unique_firms, n_missing_revenue and n_analysed exact",
added runs/pilot/RP0007/evaluation_receipt_pre-A002.json +154 −0
@@ -0,0 +1,154 @@
1 +{
2 + "critical_violations": [],
3 + "evaluated_utc": "2026-10-30T03:49:09.167839+00:00",
4 + "evaluator_version": "F1-dev-01/1.0",
5 + "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",
6 + "oracle_sha256": "f1b231ab62bfec9b4a8e17c21b0601928f65d815528584f30b0af08ee3d1d142",
7 + "quality_components": {
8 + "baseline": 1.0,
9 + "correctness": 1.0,
10 + "integrity": 1.0,
11 + "reporting": 1.0,
12 + "reproducibility": 1.0
13 + },
14 + "quality_score": 1.0,
15 + "receipt_sha256": "9a159adfe5477009dc2c60ac67ab8526195d258e164a0e28b22bd5775482fd63",
16 + "required_criteria": [
17 + {
18 + "description": "n_unique_firms, n_missing_revenue and n_analysed exact",
19 + "expected": {
20 + "n_analysed": 394,
21 + "n_missing_revenue": 6,
22 + "n_unique_firms": 400
23 + },
24 + "id": "counts",
25 + "observed": {
26 + "n_analysed": 394,
27 + "n_missing_revenue": 6,
28 + "n_unique_firms": 400
29 + },
30 + "passed": true,
31 + "tolerance": {
32 + "n_analysed": {
33 + "abs": 0
34 + },
35 + "n_missing_revenue": {
36 + "abs": 0
37 + },
38 + "n_unique_firms": {
39 + "abs": 0
40 + }
41 + }
42 + },
43 + {
44 + "description": "mean revenue (millions USD) after de-duplication and unit harmonisation, within 0.05",
45 + "expected": {
46 + "mean_revenue_musd": 249.962076142132
47 + },
48 + "id": "mean_revenue",
49 + "observed": {
50 + "mean_revenue_musd": 249.962076
51 + },
52 + "passed": true,
53 + "tolerance": {
54 + "mean_revenue_musd": {
55 + "abs": 0.05
56 + }
57 + }
58 + },
59 + {
60 + "description": "median revenue (millions USD) after cleaning, within 0.05",
61 + "expected": {
62 + "median_revenue_musd": 162.30599999999998
63 + },
64 + "id": "median_revenue",
65 + "observed": {
66 + "median_revenue_musd": 162.306
67 + },
68 + "passed": true,
69 + "tolerance": {
70 + "median_revenue_musd": {
71 + "abs": 0.05
72 + }
73 + }
74 + },
75 + {
76 + "description": "mean employees over unique firms, within 0.5",
77 + "expected": {
78 + "mean_employees": 1101.435
79 + },
80 + "id": "mean_employees",
81 + "observed": {
82 + "mean_employees": 1101.435
83 + },
84 + "passed": true,
85 + "tolerance": {
86 + "mean_employees": {
87 + "abs": 0.5
88 + }
89 + }
90 + },
91 + {
92 + "description": "out/analytic.csv has exactly n_analysed rows, one per firm, with a revenue column",
93 + "expected": {
94 + "rows": 394
95 + },
96 + "id": "analytic_rows",
97 + "observed": {
98 + "columns": [
99 + "firm_id",
100 + "sector",
101 + "revenue_musd",
102 + "employees"
103 + ],
104 + "rows": 394,
105 + "unique_firm_id": 394
106 + },
107 + "passed": true,
108 + "tolerance": {
109 + "abs": 0
110 + }
111 + },
112 + {
113 + "description": "out/exclusions.csv exists, has the required columns, is internally consistent and its total removals equal n_input_rows - n_analysed",
114 + "expected": {
115 + "first_before": 445,
116 + "last_after": 394
117 + },
118 + "id": "ledger",
119 + "observed": {
120 + "exists": true,
121 + "first_before": 445,
122 + "last_after": 394,
123 + "n_removed_list": [
124 + 45,
125 + 0,
126 + 6
127 + ],
128 + "n_rows": 3,
129 + "total_removed": 51
130 + },
131 + "passed": true,
132 + "tolerance": null
133 + },
134 + {
135 + "description": "data_issues lists both the duplicated rows and the unit mix",
136 + "expected": [
137 + "duplic",
138 + "unit|thousand|scale|rescal|million"
139 + ],
140 + "id": "issues_detected",
141 + "observed": {
142 + "n_issues": 3,
143 + "text": "45 exact duplicate rows (45 repeated firm_id rows in total) kept first occurrence of each identical row mixed revenue units: 60 distinct firms report revenue in usd_thousands, rest in usd_millions converted all revenue to millions usd using the per-row unit column 6 distinct firms with empty revenue"
144 + },
145 + "passed": true,
146 + "tolerance": null
147 + }
148 + ],
149 + "run_id": "RP0007",
150 + "schema_error": "",
151 + "schema_valid": true,
152 + "scientific_success": true,
153 + "task_id": "F1-dev-01"
154 +}
\ No newline at end of file
modified runs/pilot/RP0008/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0008/evaluation_receipt_pre-A002.json +155 −0
Large diff (156 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0009/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0009/evaluation_receipt_pre-A002.json +124 −0
Large diff (125 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0010/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0010/evaluation_receipt_pre-A002.json +124 −0
Large diff (125 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0011/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0011/evaluation_receipt_pre-A002.json +124 −0
Large diff (125 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0012/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0012/evaluation_receipt_pre-A002.json +124 −0
Large diff (125 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0013/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0013/evaluation_receipt_pre-A002.json +168 −0
Large diff (169 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/artifact_manifest.json +25 −0
Large diff (26 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/behaviour_audit.json +22 −0
Large diff (23 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/contract_receipt.json +212 −0
Large diff (213 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/ctl/prompt.txt +1 −0
Large diff (1 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/ctl/reviewer_prompt.txt +1 −0
Large diff (1 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/ctl/reviewer_system.md +10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/ctl/system_append.md +59 −0
Large diff (59 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/evaluation_receipt.json +168 −0
Large diff (169 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/evaluation_receipt_pre-A002.json +168 −0
Large diff (169 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/analysis.py +152 −0
Large diff (152 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/claims.json +7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/exclusions.csv +4 −0
Large diff (4 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/figure1.png +0 −0

Binary file not shown.

added runs/pilot/RP0014/out/report.md +43 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/results.json +74 −0
Large diff (75 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/review/review.json +34 −0
Large diff (34 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/checks.md +17 −0
Large diff (17 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/contract.md +10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/data_audit.md +19 −0
Large diff (19 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/gate_receipts/20261030T035242.771389Z.json +210 −0
Large diff (211 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/gate_receipts/20261030T035321.777848Z.json +210 −0
Large diff (211 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/gate_receipts/20261030T035326.019390Z.json +210 −0
Large diff (211 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/gate_receipts/20261030T035336.826346Z.json +210 −0
Large diff (211 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/log.md +6 −0
Large diff (6 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/phases.json +54 −0
Large diff (55 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/repro/run1.json +74 −0
Large diff (75 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/state/repro/run1.sha +2 −0
Large diff (2 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/out/table1.csv +5 −0
Large diff (5 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/sessions/6fcb0c24-be55-47de-b0e0-caf598a26592.jsonl +50 −0
Large diff (50 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0014/sessions/99ab1392-00fc-4b7a-bd81-cec3805b6c91.jsonl +114 −0
Large diff (114 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0015/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0015/evaluation_receipt_pre-A002.json +168 −0
Large diff (169 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0016/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0016/evaluation_receipt_pre-A002.json +168 −0
Large diff (169 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0017/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0017/evaluation_receipt_pre-A002.json +123 −0
Large diff (124 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0018/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0018/evaluation_receipt_pre-A002.json +123 −0
Large diff (124 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0019/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0019/evaluation_receipt_pre-A002.json +123 −0
Large diff (124 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/RP0020/evaluation_receipt.json +2 −2
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added runs/pilot/RP0020/evaluation_receipt_pre-A002.json +123 −0
Large diff (124 lines) is not loaded by default to keep the page fast.Load diff
modified runs/pilot/pilot_batch1_controller.log +142 −0
Large diff (142 lines) is not loaded by default to keep the page fast.Load diff
modified runs/state.sqlite +0 −0

Binary file not shown.

deleted runs/state.sqlite-shm +0 −0

Binary file not shown.

deleted runs/state.sqlite-wal +0 −0

Binary file not shown.

modified src/pgar/capture.py +26 −0
Large diff (29 lines) is not loaded by default to keep the page fast.Load diff
added src/pgar/freeze.py +28 −0
Large diff (28 lines) is not loaded by default to keep the page fast.Load diff
added tasks/public_bundles/F3-conf-04/bundle.json +14 −0
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added tasks/registry_rows/F3-conf-04.json +13 −0
Large diff (14 lines) is not loaded by default to keep the page fast.Load diff
added tasks/task_cards/F3-conf-04.md +14 −0
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere