Repository Public Research preview
1{2 "name": "pangram_opus",3 "created_at": "2026-11-10T17:19:20Z",4 "detector": {5 "vendor": "Pangram Labs",6 "model": "pangram-4.1",7 "served_versions": [8 "4.1"9 ],10 "ai_likelihood_definition": "word-weighted mean of windows[].ai_assistance_score",11 "flag_definition": "prediction_short == 'AI' (API's own label)"12 },13 "inputs": {14 "pairs": "data/pairs/pilot_opus/validated/pairs.jsonl",15 "outputs_dir": "artifacts/predictions/heldout_opus",16 "systems": [17 "B0_identity",18 "htr-m1-tiny-s3"19 ],20 "passages": "data/parsed/pilot_pmc/passages.jsonl,data/parsed/medium_v1/passages.jsonl"21 },22 "sample": {23 "n_pairs": 85,24 "n_docs": 40,25 "seed": 20261110,26 "requested_n": 85,27 "by_discipline": {28 "finance": 10,29 "mathematics": 6,30 "statistics": 7,31 "life_sciences": 18,32 "biomedicine": 40,33 "social_sciences": 3,34 "engineering": 135 },36 "mean_words": {37 "original": 153.72941176470587,38 "rewrite": 146.0,39 "B0_identity": 146.0,40 "htr-m1-tiny-s3": 146.023529411764741 }42 },43 "restoration": {44 "passages_found": 85,45 "protect_reproduces_target": 85,46 "restored": 575,47 "neutral": 0,48 "texts_with_neutral": 0,49 "neutral_by_kind": {}50 },51 "cost": {52 "model": "pangram-4.1",53 "texts_submitted": 254,54 "cache_hits": 0,55 "est_units": 487,56 "est_usd": 19.48,57 "ledger_run_texts": 254,58 "ledger_run_units": 487,59 "ledger_run_usd": 19.48,60 "ledger_run_requests": 161 },62 "n_boot": 2000,63 "caveat": "Detector scores measure one commercial classifier's behaviour; they are not evidence of authorship.",64 "conditions": {65 "original": {66 "n": 85,67 "n_docs": 40,68 "n_errors": 0,69 "ai_likelihood_mean": {70 "mean": 1.6580215275188334e-20,71 "ci_low": 2.0948476601038593e-29,72 "ci_high": 5.726106840824526e-20,73 "n_items": 85,74 "n_groups": 4075 },76 "ai_likelihood_median": 0.0,77 "frac_label_ai": {78 "mean": 0.0,79 "ci_low": 0.0,80 "ci_high": 0.0,81 "n_items": 85,82 "n_groups": 4083 },84 "frac_label_not_human": {85 "mean": 0.0,86 "ci_low": 0.0,87 "ci_high": 0.0,88 "n_items": 85,89 "n_groups": 4090 },91 "labels": {92 "Human": 8593 },94 "frac_score_positive": 0.18823529411764706,95 "fraction_ai_assisted_mean": 7.881817795083862e-2196 },97 "rewrite": {98 "n": 85,99 "n_docs": 40,100 "n_errors": 0,101 "ai_likelihood_mean": {102 "mean": 0.1209784359077529,103 "ci_low": 0.07834710618408053,104 "ci_high": 0.176873416893178,105 "n_items": 85,106 "n_groups": 40107 },108 "ai_likelihood_median": 1.4929447063525916e-12,109 "frac_label_ai": {110 "mean": 0.0,111 "ci_low": 0.0,112 "ci_high": 0.0,113 "n_items": 85,114 "n_groups": 40115 },116 "frac_label_not_human": {117 "mean": 0.32941176470588235,118 "ci_low": 0.2247191011235955,119 "ci_high": 0.45454545454545453,120 "n_items": 85,121 "n_groups": 40122 },123 "labels": {124 "Human": 57,125 "Mixed": 28126 },127 "frac_score_positive": 0.9176470588235294,128 "fraction_ai_assisted_mean": 0.24315256511871464129 },130 "B0_identity": {131 "n": 85,132 "n_docs": 40,133 "n_errors": 0,134 "ai_likelihood_mean": {135 "mean": 0.1209784359077529,136 "ci_low": 0.07834710618408053,137 "ci_high": 0.176873416893178,138 "n_items": 85,139 "n_groups": 40140 },141 "ai_likelihood_median": 1.4929447063525916e-12,142 "frac_label_ai": {143 "mean": 0.0,144 "ci_low": 0.0,145 "ci_high": 0.0,146 "n_items": 85,147 "n_groups": 40148 },149 "frac_label_not_human": {150 "mean": 0.32941176470588235,151 "ci_low": 0.2247191011235955,152 "ci_high": 0.45454545454545453,153 "n_items": 85,154 "n_groups": 40155 },156 "labels": {157 "Human": 57,158 "Mixed": 28159 },160 "frac_score_positive": 0.9176470588235294,161 "fraction_ai_assisted_mean": 0.24315256511871464162 },163 "htr-m1-tiny-s3": {164 "n": 85,165 "n_docs": 40,166 "n_errors": 0,167 "ai_likelihood_mean": {168 "mean": 7.19491930564766e-10,169 "ci_low": 6.026678964620597e-21,170 "ci_high": 2.3736597176145193e-09,171 "n_items": 85,172 "n_groups": 40173 },174 "ai_likelihood_median": 0.0,175 "frac_label_ai": {176 "mean": 0.0,177 "ci_low": 0.0,178 "ci_high": 0.0,179 "n_items": 85,180 "n_groups": 40181 },182 "frac_label_not_human": {183 "mean": 0.0,184 "ci_low": 0.0,185 "ci_high": 0.0,186 "n_items": 85,187 "n_groups": 40188 },189 "labels": {190 "Human": 85191 },192 "frac_score_positive": 0.36470588235294116,193 "fraction_ai_assisted_mean": 1.4358303775021826e-09194 }195 },196 "paired_differences": {197 "B0_identity - rewrite": {198 "n": 85,199 "ai_likelihood": {200 "mean": 0.0,201 "ci_low": 0.0,202 "ci_high": 0.0,203 "n_items": 85,204 "n_groups": 40,205 "p_value_two_sided": null206 },207 "frac_label_ai": {208 "mean": 0.0,209 "ci_low": 0.0,210 "ci_high": 0.0,211 "n_items": 85,212 "n_groups": 40,213 "p_value_two_sided": null214 },215 "n_label_changed": 0,216 "label_transitions": {217 "Human->Human": 57,218 "Mixed->Mixed": 28219 }220 },221 "B0_identity - original": {222 "n": 85,223 "ai_likelihood": {224 "mean": 0.1209784359077529,225 "ci_low": 0.07834710618408053,226 "ci_high": 0.176873416893178,227 "n_items": 85,228 "n_groups": 40,229 "p_value_two_sided": 0.0230 },231 "frac_label_ai": {232 "mean": 0.0,233 "ci_low": 0.0,234 "ci_high": 0.0,235 "n_items": 85,236 "n_groups": 40,237 "p_value_two_sided": null238 },239 "n_label_changed": 28,240 "label_transitions": {241 "Human->Human": 57,242 "Human->Mixed": 28243 }244 },245 "htr-m1-tiny-s3 - rewrite": {246 "n": 85,247 "ai_likelihood": {248 "mean": -0.12097843518826096,249 "ci_low": -0.17687341596142453,250 "ci_high": -0.07834710615606748,251 "n_items": 85,252 "n_groups": 40,253 "p_value_two_sided": 0.0254 },255 "frac_label_ai": {256 "mean": 0.0,257 "ci_low": 0.0,258 "ci_high": 0.0,259 "n_items": 85,260 "n_groups": 40,261 "p_value_two_sided": null262 },263 "n_label_changed": 28,264 "label_transitions": {265 "Human->Human": 57,266 "Mixed->Human": 28267 }268 },269 "htr-m1-tiny-s3 - original": {270 "n": 85,271 "ai_likelihood": {272 "mean": 7.194919305481858e-10,273 "ci_low": 5.941632114378319e-21,274 "ci_high": 2.3736597176123538e-09,275 "n_items": 85,276 "n_groups": 40,277 "p_value_two_sided": 0.001278 },279 "frac_label_ai": {280 "mean": 0.0,281 "ci_low": 0.0,282 "ci_high": 0.0,283 "n_items": 85,284 "n_groups": 40,285 "p_value_two_sided": null286 },287 "n_label_changed": 0,288 "label_transitions": {289 "Human->Human": 85290 }291 }292 }293}294