{
  "task_id": "T3-synthesis",
  "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to a generative-AI assistant to mean completion time without it, on the same tasks, from a randomised comparison; negative = faster with AI.",
  "items": {
    "S1": {
      "estimate_as_printed": "-0.24. (ExpCon coefficient, Model 2, LogToT; marker '.' = p<.1); text: beta_1=-0.24, 'about 21%' faster",
      "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086",
      "sample_as_printed": "n = 96 (Table III caption); Section VI-B: N=47 (AI), N=46 (no AI)",
      "definition": "OLS coefficient of the experimental-condition (AI) indicator on natural-log time on task, with developer-level covariates (Model 2); RCT, one complex enterprise coding task",
      "locator": "arxiv-2410.12944v3: Table III, row ExpCon, column (2) 'Dev' (+ SE line below); Section VI-C (CI and p for Model 2)",
      "harmonisable": true,
      "harmonised_estimate": -0.24,
      "harmonised_se": 0.14,
      "poolable": true,
      "transformation": "log-time regression coefficient: theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed CI gives (0.03-(-0.51))/(2*1.96) = 0.1378, consistent)",
      "reason_if_not": null
    },
    "S2": {
      "estimate_as_printed": "+19% ('allowing AI actually increases completion time by 19%'; 'implement issues in 19% more time on average')",
      "uncertainty_as_printed": "none printed in the extract (95% CI shown only in Figure 1, not reproduced)",
      "sample_as_printed": "16 developers, 246 tasks; 136 issues AI-allowed, 110 issues AI-disallowed",
      "definition": "S = E[time AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time (Sections 2.3, D.1-D.3); issue-level RCT",
      "locator": "arxiv-2507.09089v2: Abstract and Section 3 'Results' opening paragraph (estimator Sections 2.3, D.1, D.3)",
      "harmonisable": true,
      "harmonised_estimate": 0.1739533071,
      "harmonised_se": null,
      "poolable": false,
      "transformation": "percentage change: theta = ln(1 + 0.19) = 0.173953",
      "reason_if_not": "Not poolable: no SE, CI or p-value for the headline estimate is printed in the extract; its CI appears only in Figure 1 (and Figure 15), which is not reproduced, so SE(theta) cannot be obtained."
    },
    "S3": {
      "estimate_as_printed": "Treatment 44.29 min, Control 63.10 min, Overall 53.84 min, Improvement -29.79%***",
      "uncertainty_as_printed": "significance stars only: *** (defined as 0<p<=0.01); no SE or CI printed",
      "sample_as_printed": "181 subjects (randomised half to treatment, half to control)",
      "definition": "percentage difference in mean task duration (minutes) for Copilot vs control, holding accuracy constant (control duration predicted at treatment accuracy, Appendix B); RCT",
      "locator": "arxiv-2411.01067v2: Table 3, block 'Aggregate Results', row 'All Content', columns Treatment/Control/Improvement",
      "harmonisable": true,
      "harmonised_estimate": -0.353679435,
      "harmonised_se": null,
      "poolable": false,
      "transformation": "percentage change: theta = ln(1 + (-0.2979)) = -0.353679 (ratio of printed durations ln(44.29/63.10) = -0.353962)",
      "reason_if_not": "Not poolable: the only uncertainty information is significance stars (***, p<=0.01); SE(theta) cannot be obtained."
    },
    "S4": {
      "estimate_as_printed": "Aware 50.5 s, Control 56.9 s per email; % Time Savings 11.1",
      "uncertainty_as_printed": "p-value 0.471",
      "sample_as_printed": "Aware N=56, Control N=52 (Blind N=59 not used); 167 analysts in total",
      "definition": "percentage time saving per email (full 25-email queue) of the Aware arm (agent verdicts, rationale, agent-prioritised queue) vs Control (no AI); RCT",
      "locator": "arxiv-2511.13860v1: Section 3.3, Table 3, column 'Full Corpus', rows Aware/Control time, % Time Savings, p-value",
      "harmonisable": true,
      "harmonised_estimate": -0.1176580435,
      "harmonised_se": 0.163220592,
      "poolable": true,
      "transformation": "percentage change c = -0.111 (11.1% saving): theta = ln(1 - 0.111) = -0.117658; APPROXIMATE SE from two-sided p: SE = |theta|/z(1-0.471/2) = 0.117658/0.720853 = 0.163221 (ratio of printed times ln(50.5/56.9) = -0.119322)",
      "reason_if_not": null
    },
    "S5": {
      "estimate_as_printed": "-0.278 (Treat x Post, Output, Panel A column (3))",
      "uncertainty_as_printed": "SE (0.295) (clustered at user level); not marked significant",
      "sample_as_printed": "N 18332",
      "definition": "difference-in-differences coefficient of Italy x post-ban on GitHub Output (count of push, pull requests, PR comments, commit comments, create, issues events; users with affiliation), Pre 27/28.03 vs Post 03/04.04 2023",
      "locator": "arxiv-2304.09339v1: Table 1, panel A, row Treat x Post (+ SE line), column (3) Output; row N",
      "harmonisable": false,
      "harmonised_estimate": null,
      "harmonised_se": null,
      "poolable": false,
      "transformation": "none (not harmonisable)",
      "reason_if_not": "Not harmonisable: the effect is on output counts (GitHub events) around the withdrawal of ChatGPT access in a non-randomised difference-in-differences design, not a comparison of task-completion times with vs without AI on the same tasks."
    }
  },
  "k_extracted": 5,
  "k_harmonised": 4,
  "k_pooled": 2,
  "fe_estimate": -0.1881433287,
  "fe_se": 0.106264831,
  "fe_ci_low": -0.3964223974,
  "fe_ci_high": 0.02013574,
  "re_estimate": -0.1881433287,
  "re_se": 0.106264831,
  "re_ci_low": -0.3964223974,
  "re_ci_high": 0.02013574,
  "q_stat": 0.3236860523,
  "i2": 0.0,
  "tau2": 0.0,
  "heterogeneity_interpretation": "low",
  "pooled_significant_5pct": false,
  "pooled_pct_change_re": -17.1504048535,
  "feasibility": "complete",
  "feasibility_reason": null,
  "data_issues": [
    {
      "issue": "S1: Section VI-B prints Mean LogToT=4.6 for the AI group and 4.3 for the no-AI group while stating the AI group was faster (96 vs 114 min); the two values appear swapped.",
      "handling": "Not used; the harmonised value is the Table III Model 2 coefficient, which agrees with the text of VI-C (CI, p, 21%)."
    },
    {
      "issue": "S1: Table III caption says n = 96 whereas Section VI-B reports N=47 + N=46 = 93.",
      "handling": "Both reported as printed in sample_as_printed; no effect on theta or SE."
    },
    {
      "issue": "S1: printed SE (0.14) and SE implied by the printed 95% CI [-0.51, 0.03] (0.1378) differ by rounding.",
      "handling": "Printed SE(b)=0.14 used (first convention); CI-based SE used as sensitivity analysis."
    },
    {
      "issue": "S2: the confidence interval of the headline +19% estimate is displayed only in Figure 1/15, which is not reproduced.",
      "handling": "harmonised_se = null, poolable = false; no remembered value substituted."
    },
    {
      "issue": "S3: printed improvement -29.79% differs slightly from the ratio of printed durations (44.29/63.10 - 1 = -29.81%).",
      "handling": "Printed percentage used (theta=-0.35368); duration-ratio value (-0.35396) reported in the transformation string."
    },
    {
      "issue": "S3: uncertainty only as significance stars with a non-standard key (*** 0<p<=0.01, * 0.05<p<=0.1).",
      "handling": "Harmonisable but not poolable per convention."
    },
    {
      "issue": "S4: printed % Time Savings 11.1 differs from the ratio of printed full-corpus times (50.5/56.9 -> 11.25%).",
      "handling": "Printed percentage and its p-value used; times-based theta used as sensitivity analysis."
    },
    {
      "issue": "S4: Table 3 Malicious/Non-Malicious columns (2.8, 1.8 vs 1.9, 2.4, labelled seconds) cannot be reconciled with Full Corpus times of 50.5/56.9 s; units of those columns ambiguous.",
      "handling": "Those columns are not used; only the Full Corpus column (a unit-free ratio) enters."
    },
    {
      "issue": "S4: SE recovered from a two-sided p-value (0.471) - approximate, assumes a Wald-type test on the log-ratio scale; the Aware arm also bundles agent queue prioritisation.",
      "handling": "Marked poolable with approximate SE, flagged in transformation and report."
    },
    {
      "issue": "S5: Table 1 notes print '** p<0.5' (presumably p<0.05); row label rendered as '\\times TreatPost'; header date 'August 24, 2026' is a rendering artefact; whether Output is logged is not stated unambiguously.",
      "handling": "Value reported as printed; S5 is not harmonisable for substantive reasons regardless."
    }
  ],
  "n_input_rows": {
    "studies.json": 5,
    "README.md": 17,
    "arxiv-2410.12944v3.txt": 112,
    "arxiv-2507.09089v2.txt": 132,
    "arxiv-2411.01067v2.txt": 109,
    "arxiv-2511.13860v1.txt": 162,
    "arxiv-2304.09339v1.txt": 108
  },
  "n_analysed": 5,
  "random_seed": null,
  "baseline_unweighted_mean_theta_harmonised": -0.1343460428,
  "baseline_unweighted_mean_theta_poolable": -0.1788290217,
  "pooled_studies": [
    "S1",
    "S4"
  ],
  "sensitivity_fe_with_S1_ci_se": -0.1891067723,
  "sensitivity_fe_with_S4_from_times": -0.1896745175,
  "fe_weights_pct": {
    "S1": 57.613338254,
    "S4": 42.386661746
  }
}