{
  "task_id": "T3-synthesis",
  "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to a generative-AI assistant to mean completion time without it, for the same tasks, from a randomised comparison; negative values mean faster with AI.",
  "items": {
    "S1": {
      "estimate_as_printed": "ExpCon coefficient -0.24. (Model 2, logged time on task); text: 'effect of AI = 21%' faster, 'about 21%' in abstract",
      "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086 (NS); marker '.' p<.1",
      "sample_as_printed": "n = 96 (Table III); N=47 AI, N=46 no-AI (Sec VI-B)",
      "definition": "OLS coefficient of the AI-exposure indicator (ExpCon) on log time on task (LogToT), Model 2 with developer-level covariates; one complex enterprise coding task, randomised AI vs no-AI",
      "locator": "arxiv-2410.12944v3, Table III, row ExpCon (+ SE row below), column Model (2) 'Dev'; Sec VI-C sentence 'estimate of experimental condition on best-fit Model 2'",
      "harmonisable": true,
      "harmonised_estimate": -0.24,
      "harmonised_se": 0.14,
      "poolable": true,
      "transformation": "theta = b = -0.24 (log-time coefficient); SE(theta) = SE(b) = 0.14 (printed). Cross-check from CI: (0.03-(-0.51))/(2*1.96) = 0.1378",
      "reason_if_not": null
    },
    "S2": {
      "estimate_as_printed": "+19% (allowing AI increases completion time by 19%; 'implement issues in 19% more time on average')",
      "uncertainty_as_printed": "none printed (95% CI shown only in Figure 1 / Figure 15, not reproduced in the extract)",
      "sample_as_printed": "16 developers, 246 tasks (136 AI-allowed, 110 AI-disallowed issues)",
      "definition": "S = E[time | AI allowed]/E[time | AI disallowed] - 1 = exp(beta) - 1, beta from OLS of log implementation time on AI indicator and log forecast time; issue-level randomisation",
      "locator": "arxiv-2507.09089v2, Abstract and Section 3 opening paragraph ('19% more time'); estimator Sec 2.3 and App. D.1-D.3",
      "harmonisable": true,
      "harmonised_estimate": 0.173953,
      "harmonised_se": null,
      "poolable": false,
      "transformation": "theta = ln(1 + c) = ln(1 + 0.19) = 0.173953 (equivalently theta = beta, since S = exp(beta) - 1)",
      "reason_if_not": "Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; the confidence interval appears only in Figure 1 (not reproduced), so SE(theta) cannot be obtained."
    },
    "S3": {
      "estimate_as_printed": "All Content: Overall 53.84, Treatment 44.29, Control 63.10 (minutes), Improvement -29.79%***",
      "uncertainty_as_printed": "significance stars only (*** = 0<p<=0.01 per the paper's definition); no SE or CI printed",
      "sample_as_printed": "181 subjects (treatment/control split ~half each; per-arm N not printed in extract)",
      "definition": "difference in task duration (minutes) between Copilot and control groups, holding accuracy constant (control duration predicted at treatment accuracy, App. B), all scenarios and tasks",
      "locator": "arxiv-2411.01067v2, Section 4.2, Table 3, block 'Aggregate Results', row 'All Content', columns Treatment / Control / Improvement",
      "harmonisable": true,
      "harmonised_estimate": -0.353679,
      "harmonised_se": null,
      "poolable": false,
      "transformation": "theta = ln(1 + c) = ln(1 - 0.2979) = -0.353679 (ratio of printed times: ln(44.29/63.10) = -0.353962)",
      "reason_if_not": "Not poolable: only significance stars (p<=0.01) are printed; no SE, CI or exact p-value, so SE(theta) cannot be obtained."
    },
    "S4": {
      "estimate_as_printed": "Full Corpus: Aware Time per Email (s) 50.5, Control Time per Email (s) 56.9, % Time Savings 11.1",
      "uncertainty_as_printed": "p-value 0.471",
      "sample_as_printed": "Aware N=56, Control N=52 (167 analysts across three arms after filtering)",
      "definition": "percentage time saving (Control vs Aware arm) in email-processing time over the full 25-email queue, without the resolve-benign protocol; Aware arm sees the agent's verdicts/rationales and an agent-ordered queue",
      "locator": "arxiv-2511.13860v1, Section 3.3, Table 3, column 'Full Corpus', rows Aware Time / Control Time / % Time Savings / p-value",
      "harmonisable": true,
      "harmonised_estimate": -0.117658,
      "harmonised_se": 0.163221,
      "poolable": true,
      "transformation": "time saving 11.1% => c = -0.111; theta = ln(1 - 0.111) = -0.117658; approximate SE = |theta| / z_(1-0.471/2) = 0.117658/0.720853 = 0.163221 (p assumed two-sided). APPROXIMATE SE.",
      "reason_if_not": null
    },
    "S5": {
      "estimate_as_printed": "Treat x Post -0.278 (Output, column 3, panel A)",
      "uncertainty_as_printed": "SE (0.295); no stars",
      "sample_as_printed": "N 18332 (user-day observations, users with affiliation)",
      "definition": "difference-in-differences coefficient of Italy x post-ban on the aggregate Output measure (count-based GitHub activity: pushes, PRs, comments, creates, issues), pre 27/28.03 vs post 03/04.04 2023, user FE",
      "locator": "arxiv-2304.09339v1, Section 4, Table 1, panel A, row 'Treat x Post' (+ SE row), column (3) Output, N row",
      "harmonisable": false,
      "harmonised_estimate": null,
      "harmonised_se": null,
      "poolable": false,
      "transformation": "none (not harmonisable)",
      "reason_if_not": "Not harmonisable: the effect is on output counts (GitHub events), not on task-completion time for the same tasks, and the contrast is a non-randomised difference-in-differences around the withdrawal of ChatGPT access (ban), not a randomised AI vs no-AI comparison."
    }
  },
  "k_extracted": 5,
  "k_harmonised": 4,
  "k_pooled": 2,
  "fe_estimate": -0.188143,
  "fe_se": 0.106265,
  "fe_ci_low": -0.396422,
  "fe_ci_high": 0.020136,
  "re_estimate": -0.188143,
  "re_se": 0.106265,
  "re_ci_low": -0.396422,
  "re_ci_high": 0.020136,
  "q_stat": 0.323686,
  "i2": 0.0,
  "tau2": 0.0,
  "heterogeneity_interpretation": "low",
  "pooled_significant_5pct": false,
  "pooled_pct_change_re": -17.150405,
  "feasibility": "complete",
  "feasibility_reason": null,
  "data_issues": [
    {
      "issue": "S1: Sec VI-B prints Mean LogToT = 4.6 for the AI group and 4.3 for the no-AI group, contradicting the minute means (96 vs 114) and the stated direction (AI faster).",
      "handling": "Not used; theta taken from the Table III Model 2 coefficient as instructed."
    },
    {
      "issue": "S1: Table III caption says n = 96 while Sec VI-B reports N=47 + N=46 = 93.",
      "handling": "Both reported verbatim in sample_as_printed; no effect on theta/SE."
    },
    {
      "issue": "S1: SE implied by the printed 95% CI [-0.51, 0.03] is 0.1378, vs printed SE 0.14 (rounding).",
      "handling": "Printed SE 0.14 used (first convention); CI-based SE reported in transformation as a cross-check."
    },
    {
      "issue": "S1: Table III model columns collapsed into one line per row (rendering); stray 'p<'/'p=' lines.",
      "handling": "Model 2 read as second value left to right per editorial note; consistent with -0.24 quoted in text. Stray lines ignored (README)."
    },
    {
      "issue": "S2: the 95% CI for the headline +19% is shown only in Figure 1/15, not reproduced; the raw ratio estimator gives 34% (footnote 12), a different estimator.",
      "handling": "Regression-based 19% used; SE set to null and study not pooled; 34% not used."
    },
    {
      "issue": "S3: printed improvement -29.79% differs slightly from the ratio of printed times (44.29/63.10 - 1 = -29.81%).",
      "handling": "Printed -29.79% used per the percentage-change convention; difference in theta < 0.0003."
    },
    {
      "issue": "S3: only significance stars printed, with unusual definitions (*** 0<p<=0.01, ** 0.01<p<=0.05, * 0.05<p<=0.1); 25% of control subjects used other AI tools.",
      "handling": "Not poolable; contamination noted as a limitation (estimate is incremental over other AI use)."
    },
    {
      "issue": "S4: Table 3 unit label '(s)' is implausible: per-email malicious/non-malicious values (2.8/1.8, 1.9/2.4) cannot average to the Full Corpus values (50.5/56.9); values look like per-email minutes and a full-queue total in minutes.",
      "handling": "theta is a unit-free log ratio, so unaffected; numbers reported as printed with the printed unit label."
    },
    {
      "issue": "S4: printed % time savings 11.1 vs 1 - 50.5/56.9 = 11.25%; p-value sidedness not stated.",
      "handling": "Printed 11.1% used; p treated as two-sided for the approximate SE."
    },
    {
      "issue": "S5: footnote star legend reads '** p<0.5' (apparent typo for 0.05); render-time date 'August 24, 2026'; text says April 11th excluded while panel B header lists 11.04.",
      "handling": "Irrelevant to the harmonisation (S5 not harmonisable); recorded only."
    }
  ],
  "n_input_rows": {
    "studies.json": 5,
    "README.md": 17,
    "arxiv-2410.12944v3.txt": 112,
    "arxiv-2507.09089v2.txt": 132,
    "arxiv-2411.01067v2.txt": 109,
    "arxiv-2511.13860v1.txt": 162,
    "arxiv-2304.09339v1.txt": 108
  },
  "n_analysed": 5,
  "random_seed": null,
  "baseline_unweighted_mean_theta_harmonised": -0.134346,
  "baseline_unweighted_mean_theta_poolable": -0.178829,
  "pooled_studies": [
    "S1",
    "S4"
  ],
  "q_df": 1,
  "q_pvalue": 0.569401,
  "s1_se_from_ci": 0.137755,
  "sensitivity_fe_s1_se_from_ci": [
    -0.189107,
    -0.395442,
    0.017228
  ],
  "sensitivity_fe_s4_theta_from_time_ratio": [
    -0.189675,
    -0.399187,
    0.019838
  ],
  "pooled_pct_change_re_ci": [
    -32.727752,
    2.033983
  ]
}