{
 "critical_violations": [],
 "evaluated_utc": "2026-10-30T05:34:18.275614+00:00",
 "evaluator_version": "T3-synthesis/1.0",
 "notes": "Deterministic evaluation; reproducibility filled by the harness. Reference = designer extraction with locators + analytical transforms + independent inverse-variance pooling code.",
 "oracle_sha256": "1900db9319ac39a3fb4c4da47dbe8d7be16a596361bf233e529209d362371da0",
 "quality_components": {
  "baseline": 1.0,
  "correctness": 1.0,
  "integrity": 1.0,
  "reporting": 1.0,
  "reproducibility": 1.0
 },
 "quality_score": 1.0,
 "receipt_sha256": "5a791474888b83b07f2b2c4ee54e28876f741199cdf1983d29c7e08be03c71c9",
 "required_criteria": [
  {
   "description": "printed values of the five items present in estimate/uncertainty_as_printed",
   "expected": {
    "S1": "Table III Model 2 ExpCon -0.24 (0.14); Section VI-C 95% CI [-0.51, 0.03], p=0.086; n=96",
    "S2": "Section 3: 19% more time (regression S = exp(beta)-1); no numeric CI in the extract (Figure 1 only)",
    "S3": "Table 3 All Content: overall 53.84, treatment 44.29, control 63.10 minutes, -29.79%***; stars only; 181 subjects",
    "S4": "Table 3 Full Corpus: Aware 50.5 s, Control 56.9 s, % time savings 11.1, p-value 0.471; Aware N=56, Control N=52",
    "S5": "Table 1 panel A col (3) Output: Treat x Post -0.278 (0.295), N 18332 (col (2) Release -0.012** (0.005) also accepted as the printed value)"
   },
   "id": "extraction_printed",
   "observed": {
    "S1": {
     "ok": true,
     "text": "-0.24 (ExpCon, Model 2, LogToT; marker '.'); text: beta_1=-0.24, effect of AI = 21% SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086"
    },
    "S2": {
     "ok": true,
     "text": "19% more time on average (AI allowed increases completion time by 19%) none printed (confidence interval shown only in Figure 1 / Figure 15, not reproduced in t"
    },
    "S3": {
     "ok": true,
     "text": "Treatment 44.29 min, Control 63.10 min (Overall 53.84); Improvement -29.79%*** significance stars only (***; per the paper's scheme 0<p<=0.01); no SE or CI prin"
    },
    "S4": {
     "ok": true,
     "text": "Aware 50.5, Control 56.9 (row labels: Time per Email (s)); % Time Savings 11.1 p-value 0.471"
    },
    "S5": {
     "ok": true,
     "text": "-0.278 (Treat x Post, Output, panel A, column (3)) SE (0.295) (clustered at user level)"
    }
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "harmonised theta S1-S4 within tolerance; S5 null",
   "expected": null,
   "id": "harmonised_values",
   "observed": {
    "S1": {
     "expected": -0.24,
     "reported": -0.24
    },
    "S2": {
     "expected": 0.17395330712343798,
     "reported": 0.17395330712343798
    },
    "S3": {
     "expected": -0.35382064328549734,
     "reported": -0.353679434958934
    },
    "S4": {
     "expected": -0.11849002415960175,
     "reported": -0.11765804346823246
    },
    "S5": {
     "expected": null,
     "reported": null
    }
   },
   "passed": true,
   "tolerance": {
    "S1_est": {
     "abs": 0.001
    },
    "S2_est": {
     "abs": 0.002
    },
    "S3_est": {
     "abs": 0.0015
    },
    "S4_est": {
     "abs": 0.0015
    },
    "pooled_est": {
     "abs": 0.004
    }
   }
  },
  {
   "description": "harmonised SE for S1 and S4 within tolerance; null for S2, S3, S5",
   "expected": null,
   "id": "harmonised_ses",
   "observed": {
    "S1": {
     "expected": 0.14,
     "reported": 0.14
    },
    "S2": {
     "expected": null,
     "reported": null
    },
    "S3": {
     "expected": null,
     "reported": null
    },
    "S4": {
     "expected": 0.16437475344672545,
     "reported": 0.1632205919720678
    },
    "S5": {
     "expected": null,
     "reported": null
    }
   },
   "passed": true,
   "tolerance": {
    "S1_se": {
     "abs": 0.0035
    },
    "S4_se": {
     "abs": 0.0025
    },
    "pooled_se": {
     "abs": 0.003
    }
   }
  },
  {
   "description": "harmonisable/poolable flags and k counts equal the reference",
   "expected": {
    "S1": {
     "harmonisable": true,
     "poolable": true
    },
    "S2": {
     "harmonisable": true,
     "poolable": false
    },
    "S3": {
     "harmonisable": true,
     "poolable": false
    },
    "S4": {
     "harmonisable": true,
     "poolable": true
    },
    "S5": {
     "harmonisable": false,
     "poolable": false
    },
    "counts": {
     "k_extracted": 5,
     "k_harmonised": 4,
     "k_pooled": 2
    }
   },
   "id": "flags_counts",
   "observed": {
    "S1": {
     "harmonisable": true,
     "poolable": true
    },
    "S2": {
     "harmonisable": true,
     "poolable": false
    },
    "S3": {
     "harmonisable": true,
     "poolable": false
    },
    "S4": {
     "harmonisable": true,
     "poolable": true
    },
    "S5": {
     "harmonisable": false,
     "poolable": false
    },
    "counts": {
     "k_extracted": 5,
     "k_harmonised": 4,
     "k_pooled": 2
    }
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "FE/RE estimates, SEs, tau2, I2, Q within tolerance; CIs and pct change consistent",
   "expected": {
    "fe_estimate": -0.18938,
    "fe_se": 0.10608,
    "i2": 0.0,
    "pooled_pct_change_re": -17.252799241919426,
    "q_stat": 0.319,
    "re_estimate": -0.18938,
    "re_se": 0.10608,
    "tau2": 0.0
   },
   "id": "pooled",
   "observed": {
    "fe_estimate": -0.18814332871148873,
    "fe_se": 0.10626483095451474,
    "i2": 0.0,
    "pooled_pct_change_re": -17.15040485348688,
    "q_stat": 0.3236860522786328,
    "re_estimate": -0.18814332871148873,
    "re_se": 0.10626483095451474,
    "tau2": 0.0
   },
   "passed": true,
   "tolerance": {
    "i2": {
     "abs": 5.0
    },
    "pooled_est": {
     "abs": 0.004
    },
    "pooled_se": {
     "abs": 0.003
    },
    "q_stat": {
     "abs": 0.05
    },
    "tau2": {
     "abs": 0.002
    }
   }
  },
  {
   "description": "heterogeneity_interpretation = low; pooled_significant_5pct = false",
   "expected": {
    "heterogeneity_interpretation": "low",
    "pooled_significant_5pct": false
   },
   "id": "interpretation",
   "observed": {
    "heterogeneity_interpretation": "low",
    "pooled_significant_5pct": false
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "table1.csv (5 rows, required columns) and figure1.png present",
   "expected": null,
   "id": "artifacts",
   "observed": {
    "figure": true,
    "table1_rows": 5
   },
   "passed": true,
   "tolerance": null
  }
 ],
 "run_id": "RT0007",
 "schema_error": "",
 "schema_valid": true,
 "scientific_success": true,
 "task_id": "T3-synthesis"
}