{
 "critical_violations": [],
 "evaluated_utc": "2026-10-30T05:33:44.696086+00:00",
 "evaluator_version": "T3-synthesis/1.0",
 "notes": "Deterministic evaluation; reproducibility filled by the harness. Reference = designer extraction with locators + analytical transforms + independent inverse-variance pooling code.",
 "oracle_sha256": "1900db9319ac39a3fb4c4da47dbe8d7be16a596361bf233e529209d362371da0",
 "quality_components": {
  "baseline": 1.0,
  "correctness": 1.0,
  "integrity": 1.0,
  "reporting": 1.0,
  "reproducibility": 1.0
 },
 "quality_score": 1.0,
 "receipt_sha256": "eba5b9998a95b85bbb2e96294b4a9a13a3f6bc919c6a3d18f1f3938f9655cdcd",
 "required_criteria": [
  {
   "description": "printed values of the five items present in estimate/uncertainty_as_printed",
   "expected": {
    "S1": "Table III Model 2 ExpCon -0.24 (0.14); Section VI-C 95% CI [-0.51, 0.03], p=0.086; n=96",
    "S2": "Section 3: 19% more time (regression S = exp(beta)-1); no numeric CI in the extract (Figure 1 only)",
    "S3": "Table 3 All Content: overall 53.84, treatment 44.29, control 63.10 minutes, -29.79%***; stars only; 181 subjects",
    "S4": "Table 3 Full Corpus: Aware 50.5 s, Control 56.9 s, % time savings 11.1, p-value 0.471; Aware N=56, Control N=52",
    "S5": "Table 1 panel A col (3) Output: Treat x Post -0.278 (0.295), N 18332 (col (2) Release -0.012** (0.005) also accepted as the printed value)"
   },
   "id": "extraction_printed",
   "observed": {
    "S1": {
     "ok": true,
     "text": "-0.24. (ExpCon coefficient, Model 2, LogToT; marker '.' = p<.1); text: beta_1=-0.24, 'about 21%' faster SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086"
    },
    "S2": {
     "ok": true,
     "text": "+19% ('allowing AI actually increases completion time by 19%'; 'implement issues in 19% more time on average') none printed in the extract (95% CI shown only in"
    },
    "S3": {
     "ok": true,
     "text": "Treatment 44.29 min, Control 63.10 min, Overall 53.84 min, Improvement -29.79%*** significance stars only: *** (defined as 0<p<=0.01); no SE or CI printed"
    },
    "S4": {
     "ok": true,
     "text": "Aware 50.5 s, Control 56.9 s per email; % Time Savings 11.1 p-value 0.471"
    },
    "S5": {
     "ok": true,
     "text": "-0.278 (Treat x Post, Output, Panel A column (3)) SE (0.295) (clustered at user level); not marked significant"
    }
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "harmonised theta S1-S4 within tolerance; S5 null",
   "expected": null,
   "id": "harmonised_values",
   "observed": {
    "S1": {
     "expected": -0.24,
     "reported": -0.24
    },
    "S2": {
     "expected": 0.17395330712343798,
     "reported": 0.1739533071
    },
    "S3": {
     "expected": -0.35382064328549734,
     "reported": -0.353679435
    },
    "S4": {
     "expected": -0.11849002415960175,
     "reported": -0.1176580435
    },
    "S5": {
     "expected": null,
     "reported": null
    }
   },
   "passed": true,
   "tolerance": {
    "S1_est": {
     "abs": 0.001
    },
    "S2_est": {
     "abs": 0.002
    },
    "S3_est": {
     "abs": 0.0015
    },
    "S4_est": {
     "abs": 0.0015
    },
    "pooled_est": {
     "abs": 0.004
    }
   }
  },
  {
   "description": "harmonised SE for S1 and S4 within tolerance; null for S2, S3, S5",
   "expected": null,
   "id": "harmonised_ses",
   "observed": {
    "S1": {
     "expected": 0.14,
     "reported": 0.14
    },
    "S2": {
     "expected": null,
     "reported": null
    },
    "S3": {
     "expected": null,
     "reported": null
    },
    "S4": {
     "expected": 0.16437475344672545,
     "reported": 0.163220592
    },
    "S5": {
     "expected": null,
     "reported": null
    }
   },
   "passed": true,
   "tolerance": {
    "S1_se": {
     "abs": 0.0035
    },
    "S4_se": {
     "abs": 0.0025
    },
    "pooled_se": {
     "abs": 0.003
    }
   }
  },
  {
   "description": "harmonisable/poolable flags and k counts equal the reference",
   "expected": {
    "S1": {
     "harmonisable": true,
     "poolable": true
    },
    "S2": {
     "harmonisable": true,
     "poolable": false
    },
    "S3": {
     "harmonisable": true,
     "poolable": false
    },
    "S4": {
     "harmonisable": true,
     "poolable": true
    },
    "S5": {
     "harmonisable": false,
     "poolable": false
    },
    "counts": {
     "k_extracted": 5,
     "k_harmonised": 4,
     "k_pooled": 2
    }
   },
   "id": "flags_counts",
   "observed": {
    "S1": {
     "harmonisable": true,
     "poolable": true
    },
    "S2": {
     "harmonisable": true,
     "poolable": false
    },
    "S3": {
     "harmonisable": true,
     "poolable": false
    },
    "S4": {
     "harmonisable": true,
     "poolable": true
    },
    "S5": {
     "harmonisable": false,
     "poolable": false
    },
    "counts": {
     "k_extracted": 5,
     "k_harmonised": 4,
     "k_pooled": 2
    }
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "FE/RE estimates, SEs, tau2, I2, Q within tolerance; CIs and pct change consistent",
   "expected": {
    "fe_estimate": -0.18938,
    "fe_se": 0.10608,
    "i2": 0.0,
    "pooled_pct_change_re": -17.252799241919426,
    "q_stat": 0.319,
    "re_estimate": -0.18938,
    "re_se": 0.10608,
    "tau2": 0.0
   },
   "id": "pooled",
   "observed": {
    "fe_estimate": -0.1881433287,
    "fe_se": 0.106264831,
    "i2": 0.0,
    "pooled_pct_change_re": -17.1504048535,
    "q_stat": 0.3236860523,
    "re_estimate": -0.1881433287,
    "re_se": 0.106264831,
    "tau2": 0.0
   },
   "passed": true,
   "tolerance": {
    "i2": {
     "abs": 5.0
    },
    "pooled_est": {
     "abs": 0.004
    },
    "pooled_se": {
     "abs": 0.003
    },
    "q_stat": {
     "abs": 0.05
    },
    "tau2": {
     "abs": 0.002
    }
   }
  },
  {
   "description": "heterogeneity_interpretation = low; pooled_significant_5pct = false",
   "expected": {
    "heterogeneity_interpretation": "low",
    "pooled_significant_5pct": false
   },
   "id": "interpretation",
   "observed": {
    "heterogeneity_interpretation": "low",
    "pooled_significant_5pct": false
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "table1.csv (5 rows, required columns) and figure1.png present",
   "expected": null,
   "id": "artifacts",
   "observed": {
    "figure": true,
    "table1_rows": 5
   },
   "passed": true,
   "tolerance": null
  }
 ],
 "run_id": "RT0006",
 "schema_error": "",
 "schema_valid": true,
 "scientific_success": true,
 "task_id": "T3-synthesis"
}