Repository Public Research preview
1{2 "name": "pangram_test",3 "created_at": "2026-11-10T17:15:49Z",4 "detector": {5 "vendor": "Pangram Labs",6 "model": "pangram-4.1",7 "served_versions": [8 "4.1"9 ],10 "ai_likelihood_definition": "word-weighted mean of windows[].ai_assistance_score",11 "flag_definition": "prediction_short == 'AI' (API's own label)"12 },13 "inputs": {14 "pairs": "data/pairs/medium_v1_haiku/validated/pairs.jsonl",15 "outputs_dir": "artifacts/predictions/medium_test",16 "systems": [17 "B0_identity",18 "htr-m1-tiny-s3",19 "htr-m1-small",20 "B2_t5-small"21 ],22 "passages": "data/parsed/medium_v1/passages.jsonl"23 },24 "sample": {25 "n_pairs": 54,26 "n_docs": 50,27 "seed": 20261110,28 "requested_n": 54,29 "by_discipline": {30 "other": 2,31 "engineering": 4,32 "social_sciences": 5,33 "chemistry": 3,34 "life_sciences": 7,35 "finance": 7,36 "biomedicine": 7,37 "humanities": 1,38 "economics": 7,39 "statistics": 3,40 "computer_science": 5,41 "mathematics": 2,42 "general_reference": 143 },44 "mean_words": {45 "original": 138.25925925925927,46 "rewrite": 133.33333333333334,47 "B0_identity": 133.33333333333334,48 "htr-m1-tiny-s3": 134.42592592592592,49 "htr-m1-small": 134.90740740740742,50 "B2_t5-small": 130.6666666666666651 }52 },53 "restoration": {54 "passages_found": 54,55 "protect_reproduces_target": 54,56 "restored": 504,57 "neutral": 0,58 "texts_with_neutral": 0,59 "neutral_by_kind": {}60 },61 "cost": {62 "model": "pangram-4.1",63 "texts_submitted": 0,64 "cache_hits": 269,65 "est_units": 0,66 "est_usd": 0.0,67 "ledger_run_texts": 269,68 "ledger_run_units": 485,69 "ledger_run_usd": 19.4,70 "ledger_run_requests": 171 },72 "n_boot": 2000,73 "caveat": "Detector scores measure one commercial classifier's behaviour; they are not evidence of authorship.",74 "conditions": {75 "original": {76 "n": 54,77 "n_docs": 50,78 "n_errors": 0,79 "ai_likelihood_mean": {80 "mean": 9.15551651970516e-18,81 "ci_low": 1.1095681003283814e-23,82 "ci_high": 2.9274231825918986e-17,83 "n_items": 54,84 "n_groups": 5085 },86 "ai_likelihood_median": 0.0,87 "frac_label_ai": {88 "mean": 0.0,89 "ci_low": 0.0,90 "ci_high": 0.0,91 "n_items": 54,92 "n_groups": 5093 },94 "frac_label_not_human": {95 "mean": 0.0,96 "ci_low": 0.0,97 "ci_high": 0.0,98 "n_items": 54,99 "n_groups": 50100 },101 "labels": {102 "Human": 54103 },104 "frac_score_positive": 0.2777777777777778,105 "fraction_ai_assisted_mean": 1.5625868731460758e-19106 },107 "rewrite": {108 "n": 54,109 "n_docs": 50,110 "n_errors": 0,111 "ai_likelihood_mean": {112 "mean": 0.187471966687167,113 "ci_low": 0.12452513542082204,114 "ci_high": 0.25324247635759917,115 "n_items": 54,116 "n_groups": 50117 },118 "ai_likelihood_median": 1.0992218079808609e-05,119 "frac_label_ai": {120 "mean": 0.0,121 "ci_low": 0.0,122 "ci_high": 0.0,123 "n_items": 54,124 "n_groups": 50125 },126 "frac_label_not_human": {127 "mean": 0.46296296296296297,128 "ci_low": 0.32075471698113206,129 "ci_high": 0.6,130 "n_items": 54,131 "n_groups": 50132 },133 "labels": {134 "Human": 29,135 "Mixed": 25136 },137 "frac_score_positive": 0.9629629629629629,138 "fraction_ai_assisted_mean": 0.3424673229154049139 },140 "B0_identity": {141 "n": 54,142 "n_docs": 50,143 "n_errors": 0,144 "ai_likelihood_mean": {145 "mean": 0.187471966687167,146 "ci_low": 0.12452513542082204,147 "ci_high": 0.25324247635759917,148 "n_items": 54,149 "n_groups": 50150 },151 "ai_likelihood_median": 1.0992218079808609e-05,152 "frac_label_ai": {153 "mean": 0.0,154 "ci_low": 0.0,155 "ci_high": 0.0,156 "n_items": 54,157 "n_groups": 50158 },159 "frac_label_not_human": {160 "mean": 0.46296296296296297,161 "ci_low": 0.32075471698113206,162 "ci_high": 0.6,163 "n_items": 54,164 "n_groups": 50165 },166 "labels": {167 "Human": 29,168 "Mixed": 25169 },170 "frac_score_positive": 0.9629629629629629,171 "fraction_ai_assisted_mean": 0.3424673229154049172 },173 "htr-m1-tiny-s3": {174 "n": 54,175 "n_docs": 50,176 "n_errors": 0,177 "ai_likelihood_mean": {178 "mean": 1.5418536763722546e-06,179 "ci_low": 6.380651662964253e-18,180 "ci_high": 4.900101680106405e-06,181 "n_items": 54,182 "n_groups": 50183 },184 "ai_likelihood_median": 2.0379784215907977e-41,185 "frac_label_ai": {186 "mean": 0.0,187 "ci_low": 0.0,188 "ci_high": 0.0,189 "n_items": 54,190 "n_groups": 50191 },192 "frac_label_not_human": {193 "mean": 0.0,194 "ci_low": 0.0,195 "ci_high": 0.0,196 "n_items": 54,197 "n_groups": 50198 },199 "labels": {200 "Human": 54201 },202 "frac_score_positive": 0.5185185185185185,203 "fraction_ai_assisted_mean": 3.9936382926675196e-16204 },205 "htr-m1-small": {206 "n": 54,207 "n_docs": 50,208 "n_errors": 0,209 "ai_likelihood_mean": {210 "mean": 3.572504718555094e-10,211 "ci_low": 1.9937892269757927e-20,212 "ci_high": 1.1347952672240593e-09,213 "n_items": 54,214 "n_groups": 50215 },216 "ai_likelihood_median": 0.0,217 "frac_label_ai": {218 "mean": 0.0,219 "ci_low": 0.0,220 "ci_high": 0.0,221 "n_items": 54,222 "n_groups": 50223 },224 "frac_label_not_human": {225 "mean": 0.0,226 "ci_low": 0.0,227 "ci_high": 0.0,228 "n_items": 54,229 "n_groups": 50230 },231 "labels": {232 "Human": 54233 },234 "frac_score_positive": 0.48148148148148145,235 "fraction_ai_assisted_mean": 2.319852197262697e-11236 },237 "B2_t5-small": {238 "n": 54,239 "n_docs": 50,240 "n_errors": 0,241 "ai_likelihood_mean": {242 "mean": 0.03183072881153848,243 "ci_low": 0.003870415004513381,244 "ci_high": 0.06684536261663526,245 "n_items": 54,246 "n_groups": 50247 },248 "ai_likelihood_median": 2.43245862490614e-35,249 "frac_label_ai": {250 "mean": 0.0,251 "ci_low": 0.0,252 "ci_high": 0.0,253 "n_items": 54,254 "n_groups": 50255 },256 "frac_label_not_human": {257 "mean": 0.07407407407407407,258 "ci_low": 0.018518518518518517,259 "ci_high": 0.14545454545454545,260 "n_items": 54,261 "n_groups": 50262 },263 "labels": {264 "Human": 50,265 "Mixed": 4266 },267 "frac_score_positive": 0.6851851851851852,268 "fraction_ai_assisted_mean": 0.04349885287660183269 }270 },271 "paired_differences": {272 "B0_identity - rewrite": {273 "n": 54,274 "ai_likelihood": {275 "mean": 0.0,276 "ci_low": 0.0,277 "ci_high": 0.0,278 "n_items": 54,279 "n_groups": 50,280 "p_value_two_sided": null281 },282 "frac_label_ai": {283 "mean": 0.0,284 "ci_low": 0.0,285 "ci_high": 0.0,286 "n_items": 54,287 "n_groups": 50,288 "p_value_two_sided": null289 },290 "n_label_changed": 0,291 "label_transitions": {292 "Human->Human": 29,293 "Mixed->Mixed": 25294 }295 },296 "B0_identity - original": {297 "n": 54,298 "ai_likelihood": {299 "mean": 0.187471966687167,300 "ci_low": 0.12452513542082204,301 "ci_high": 0.25324247635759917,302 "n_items": 54,303 "n_groups": 50,304 "p_value_two_sided": 0.0305 },306 "frac_label_ai": {307 "mean": 0.0,308 "ci_low": 0.0,309 "ci_high": 0.0,310 "n_items": 54,311 "n_groups": 50,312 "p_value_two_sided": null313 },314 "n_label_changed": 25,315 "label_transitions": {316 "Human->Human": 29,317 "Human->Mixed": 25318 }319 },320 "htr-m1-tiny-s3 - rewrite": {321 "n": 54,322 "ai_likelihood": {323 "mean": -0.18747042483349063,324 "ci_low": -0.2532408646281885,325 "ci_high": -0.12452509539192837,326 "n_items": 54,327 "n_groups": 50,328 "p_value_two_sided": 0.0329 },330 "frac_label_ai": {331 "mean": 0.0,332 "ci_low": 0.0,333 "ci_high": 0.0,334 "n_items": 54,335 "n_groups": 50,336 "p_value_two_sided": null337 },338 "n_label_changed": 25,339 "label_transitions": {340 "Human->Human": 29,341 "Mixed->Human": 25342 }343 },344 "htr-m1-tiny-s3 - original": {345 "n": 54,346 "ai_likelihood": {347 "mean": 1.5418536763630988e-06,348 "ci_low": 6.14732264401941e-18,349 "ci_high": 4.900101680087751e-06,350 "n_items": 54,351 "n_groups": 50,352 "p_value_two_sided": 0.001353 },354 "frac_label_ai": {355 "mean": 0.0,356 "ci_low": 0.0,357 "ci_high": 0.0,358 "n_items": 54,359 "n_groups": 50,360 "p_value_two_sided": null361 },362 "n_label_changed": 0,363 "label_transitions": {364 "Human->Human": 54365 }366 },367 "htr-m1-small - rewrite": {368 "n": 54,369 "ai_likelihood": {370 "mean": -0.18747196632991653,371 "ci_low": -0.2532424763390495,372 "ci_high": -0.12452513506593088,373 "n_items": 54,374 "n_groups": 50,375 "p_value_two_sided": 0.0376 },377 "frac_label_ai": {378 "mean": 0.0,379 "ci_low": 0.0,380 "ci_high": 0.0,381 "n_items": 54,382 "n_groups": 50,383 "p_value_two_sided": null384 },385 "n_label_changed": 25,386 "label_transitions": {387 "Human->Human": 29,388 "Mixed->Human": 25389 }390 },391 "htr-m1-small - original": {392 "n": 54,393 "ai_likelihood": {394 "mean": 3.572504626999928e-10,395 "ci_low": 1.9672069756805572e-20,396 "ci_high": 1.1347952669842607e-09,397 "n_items": 54,398 "n_groups": 50,399 "p_value_two_sided": 0.018400 },401 "frac_label_ai": {402 "mean": 0.0,403 "ci_low": 0.0,404 "ci_high": 0.0,405 "n_items": 54,406 "n_groups": 50,407 "p_value_two_sided": null408 },409 "n_label_changed": 0,410 "label_transitions": {411 "Human->Human": 54412 }413 },414 "B2_t5-small - rewrite": {415 "n": 54,416 "ai_likelihood": {417 "mean": -0.15564123787562853,418 "ci_low": -0.22023283796490484,419 "ci_high": -0.09287872028737068,420 "n_items": 54,421 "n_groups": 50,422 "p_value_two_sided": 0.0423 },424 "frac_label_ai": {425 "mean": 0.0,426 "ci_low": 0.0,427 "ci_high": 0.0,428 "n_items": 54,429 "n_groups": 50,430 "p_value_two_sided": null431 },432 "n_label_changed": 23,433 "label_transitions": {434 "Human->Human": 28,435 "Mixed->Human": 22,436 "Human->Mixed": 1,437 "Mixed->Mixed": 3438 }439 },440 "B2_t5-small - original": {441 "n": 54,442 "ai_likelihood": {443 "mean": 0.031830728811538464,444 "ci_low": 0.0038704150045133718,445 "ci_high": 0.06684536261663525,446 "n_items": 54,447 "n_groups": 50,448 "p_value_two_sided": 0.0449 },450 "frac_label_ai": {451 "mean": 0.0,452 "ci_low": 0.0,453 "ci_high": 0.0,454 "n_items": 54,455 "n_groups": 50,456 "p_value_two_sided": null457 },458 "n_label_changed": 4,459 "label_transitions": {460 "Human->Human": 50,461 "Human->Mixed": 4462 }463 }464 }465}466