1{2 "targets": {3 "markdown30": {4 "chronological": {5 "n_train": 31050,6 "n_test": 12953,7 "base_rate_train": 0.1801610305958132,8 "models": {9 "logistic": {10 "log_loss": 0.42750443531874166,11 "log_loss_ci": [12 0.4166018773449265,13 0.438370233532485714 ],15 "brier": 0.13450365317427446,16 "brier_ci": [17 0.13045599567318253,18 0.1383837053495182719 ],20 "auc": 0.7484693951027384,21 "auc_ci": [22 0.737909568140902,23 0.759309313615871224 ],25 "ece": 0.01730073078587064,26 "n_test": 12953,27 "base_rate": 0.1927738747780437,28 "n_boot": 20029 },30 "lightgbm": {31 "log_loss": 0.4056075176546041,32 "log_loss_ci": [33 0.39458102948911755,34 0.415770824810822835 ],36 "brier": 0.12850630504527635,37 "brier_ci": [38 0.12492909051145196,39 0.1320113778614877840 ],41 "auc": 0.7844642338978158,42 "auc_ci": [43 0.7750294037667133,44 0.793143769594169345 ],46 "ece": 0.021023207499034996,47 "n_test": 12953,48 "base_rate": 0.1927738747780437,49 "n_boot": 200,50 "best_iteration": 22351 }52 }53 },54 "dealer_holdout": {55 "n_train": 35202,56 "n_test": 44003,57 "base_rate_train": 0.18379591843568377,58 "models": {59 "logistic": {60 "log_loss": 0.47603195914870594,61 "log_loss_ci": [62 0.4691448407916578,63 0.482717902744631764 ],65 "brier": 0.14956133723719078,66 "brier_ci": [67 0.14681886000410743,68 0.152199715949352869 ],70 "auc": 0.5802647906902381,71 "auc_ci": [72 0.5733687749143543,73 0.586044510661207174 ],75 "ece": 0.03259713949229643,76 "n_test": 44003,77 "base_rate": 0.18387382678453743,78 "n_boot": 20079 },80 "lightgbm": {81 "log_loss": 0.4842918455473671,82 "log_loss_ci": [83 0.47716753090790887,84 0.4919310367303529385 ],86 "brier": 0.15125919820462633,87 "brier_ci": [88 0.1485053191385561,89 0.1541219678494926790 ],91 "auc": 0.5777586319092147,92 "auc_ci": [93 0.5708631268857159,94 0.584062099303777795 ],96 "ece": 0.04930848852593494,97 "n_test": 44003,98 "base_rate": 0.18387382678453743,99 "n_boot": 200,100 "best_iteration": 187101 }102 },103 "folds": [104 {105 "logistic": {106 "log_loss": 0.40943258253874,107 "brier": 0.12207271739318214,108 "auc": 0.5600725787112404,109 "n_test": 9442110 },111 "lightgbm": {112 "log_loss": 0.41500853560226947,113 "brier": 0.12277600070829686,114 "auc": 0.5210200038568125,115 "n_test": 9442116 }117 },118 {119 "logistic": {120 "log_loss": 0.5096047931597615,121 "brier": 0.16441712850163426,122 "auc": 0.6540259577054304,123 "n_test": 7394124 },125 "lightgbm": {126 "log_loss": 0.5356907310215666,127 "brier": 0.17055387540055642,128 "auc": 0.64076435156967,129 "n_test": 7394130 }131 },132 {133 "logistic": {134 "log_loss": 0.4876614647256647,135 "brier": 0.1547650294125288,136 "auc": 0.597862423607253,137 "n_test": 7548138 },139 "lightgbm": {140 "log_loss": 0.4905092226450667,141 "brier": 0.15543860362866624,142 "auc": 0.5759085843982019,143 "n_test": 7548144 }145 },146 {147 "logistic": {148 "log_loss": 0.4512451295449607,149 "brier": 0.14049666686090417,150 "auc": 0.6265568341220527,151 "n_test": 6812152 },153 "lightgbm": {154 "log_loss": 0.462625649150589,155 "brier": 0.14338613128650887,156 "auc": 0.6187942317190578,157 "n_test": 6812158 }159 },160 {161 "logistic": {162 "log_loss": 0.5120796014099209,163 "brier": 0.16300514874821193,164 "auc": 0.5871099994686383,165 "n_test": 12807166 },167 "lightgbm": {168 "log_loss": 0.5135563900017123,169 "brier": 0.16284338554586145,170 "auc": 0.580390038485771,171 "n_test": 12807172 }173 }174 ],175 "fold_mean": {176 "logistic": {177 "log_loss": 0.4740047142758096,178 "brier": 0.14895133818329226,179 "auc": 0.605125558722923180 },181 "lightgbm": {182 "log_loss": 0.48347810568424077,183 "brier": 0.15099959931397797,184 "auc": 0.5873754420059026185 }186 },187 "fold_sd": {188 "logistic": {189 "log_loss": 0.03896441549377764,190 "brier": 0.01590245886661948,191 "auc": 0.032427251196387105192 },193 "lightgbm": {194 "log_loss": 0.04195999287062199,195 "brier": 0.01671458140990917,196 "auc": 0.04103862782584155197 }198 }199 },200 "region_holdout": {201 "n_train": 39178,202 "n_test": 28947,203 "base_rate_train": 0.18223023652950746,204 "models": {205 "logistic": {206 "log_loss": 0.5081708580019036,207 "log_loss_ci": [208 0.4998529808058603,209 0.5160353180522947210 ],211 "brier": 0.16346334388335346,212 "brier_ci": [213 0.16002606678557083,214 0.1666491253509428215 ],216 "auc": 0.6055050483473974,217 "auc_ci": [218 0.5965395478483017,219 0.6143427404060463220 ],221 "ece": 0.04061943442091372,222 "n_test": 28947,223 "base_rate": 0.19818979514284726,224 "n_boot": 200225 },226 "lightgbm": {227 "log_loss": 0.5289470665959981,228 "log_loss_ci": [229 0.5180102275706995,230 0.5394076117917554231 ],232 "brier": 0.16682471314062872,233 "brier_ci": [234 0.1628695751406138,235 0.17063898538049768236 ],237 "auc": 0.580770005467191,238 "auc_ci": [239 0.5725672848885526,240 0.5884021487002266241 ],242 "ece": 0.07691417888302861,243 "n_test": 28947,244 "base_rate": 0.19818979514284726,245 "n_boot": 200,246 "best_iteration": 224247 }248 },249 "per_region": {250 "Mont\u00e9r\u00e9gie": {251 "logistic": {252 "log_loss": 0.452161698876822,253 "brier": 0.14212071145105978,254 "auc": 0.65250309504921,255 "n_test": 9137,256 "base_rate": 0.17883331509248113257 },258 "lightgbm": {259 "log_loss": 0.47820728589395944,260 "brier": 0.14797285357542334,261 "auc": 0.6354813439781166,262 "n_test": 9137,263 "base_rate": 0.17883331509248113264 }265 },266 "Capitale-Nationale": {267 "logistic": {268 "log_loss": 0.40566669445413134,269 "brier": 0.12195921361556367,270 "auc": 0.629125458852056,271 "n_test": 5358,272 "base_rate": 0.1452034341172079273 },274 "lightgbm": {275 "log_loss": 0.4185644898170062,276 "brier": 0.12378175745704309,277 "auc": 0.5542907578495976,278 "n_test": 5358,279 "base_rate": 0.1452034341172079280 }281 },282 "Montr\u00e9al": {283 "logistic": {284 "log_loss": 0.5003663356596327,285 "brier": 0.1584746243976379,286 "auc": 0.5132099653689886,287 "n_test": 4290,288 "base_rate": 0.18578088578088578289 },290 "lightgbm": {291 "log_loss": 0.5280538188369651,292 "brier": 0.1621894265953359,293 "auc": 0.43915039255783483,294 "n_test": 4290,295 "base_rate": 0.18578088578088578296 }297 },298 "Laurentides": {299 "logistic": {300 "log_loss": 0.5846016300361694,301 "brier": 0.19553131756639106,302 "auc": 0.5726448340104598,303 "n_test": 4033,304 "base_rate": 0.25861641457971735305 },306 "lightgbm": {307 "log_loss": 0.5977312717847972,308 "brier": 0.1982952655437102,309 "auc": 0.5860367091327114,310 "n_test": 4033,311 "base_rate": 0.25861641457971735312 }313 },314 "Laval": {315 "logistic": {316 "log_loss": 0.6317090846314307,317 "brier": 0.2141206161279708,318 "auc": 0.7005024301121044,319 "n_test": 3309,320 "base_rate": 0.2901178603807797321 },322 "lightgbm": {323 "log_loss": 0.6444428089532214,324 "brier": 0.21358889094630226,325 "auc": 0.6911103572442174,326 "n_test": 3309,327 "base_rate": 0.2901178603807797328 }329 },330 "Chaudi\u00e8re-Appalaches": {331 "logistic": {332 "log_loss": 0.474519704353236,333 "brier": 0.1485735801414975,334 "auc": 0.5650445066915655,335 "n_test": 2820,336 "base_rate": 0.18617021276595744337 },338 "lightgbm": {339 "log_loss": 0.5066827242900391,340 "brier": 0.15512008472595737,341 "auc": 0.5785504720406681,342 "n_test": 2820,343 "base_rate": 0.18617021276595744344 }345 }346 },347 "aggregation": "unweighted mean over the 6 held-out regions; CI = bootstrap within region, then averaged"348 },349 "drift": {350 "weeks_4_5": {351 "n_train": 24354,352 "n_test": 16259,353 "weeks_observable": [354 4,355 5356 ],357 "models": {358 "logistic": {359 "log_loss": 0.4172980500943417,360 "log_loss_ci": [361 0.4078892333968611,362 0.42792501293356267363 ],364 "brier": 0.13028210191911407,365 "brier_ci": [366 0.1269175676153395,367 0.13395263408367583368 ],369 "auc": 0.7492457271971177,370 "auc_ci": [371 0.7397635617914534,372 0.7587462270210206373 ],374 "ece": 0.02287873370320918,375 "n_test": 16259,376 "base_rate": 0.18420567070545543,377 "n_boot": 200378 },379 "lightgbm": {380 "log_loss": 0.39306320013350626,381 "log_loss_ci": [382 0.3829517825666116,383 0.4033021206623044384 ],385 "brier": 0.12300596416427913,386 "brier_ci": [387 0.1196597142669703,388 0.12673548980656985389 ],390 "auc": 0.7919337189445215,391 "auc_ci": [392 0.7825396642943132,393 0.8007247515269758394 ],395 "ece": 0.024650995703471436,396 "n_test": 16259,397 "base_rate": 0.18420567070545543,398 "n_boot": 200399 }400 }401 },402 "weeks_6_7": {403 "n_train": 24354,404 "n_test": 3390,405 "weeks_observable": [406 6407 ],408 "models": {409 "logistic": {410 "log_loss": 0.4574664935898489,411 "log_loss_ci": [412 0.4386567768372843,413 0.47863900911457274414 ],415 "brier": 0.1461414692248857,416 "brier_ci": [417 0.13886963450397957,418 0.15414744839870803419 ],420 "auc": 0.7333942190486362,421 "auc_ci": [422 0.7139021379287034,423 0.7511810377810034424 ],425 "ece": 0.029979682967721875,426 "n_test": 3390,427 "base_rate": 0.20914454277286135,428 "n_boot": 200429 },430 "lightgbm": {431 "log_loss": 0.44687234982908236,432 "log_loss_ci": [433 0.42755382743884957,434 0.4716631221365267435 ],436 "brier": 0.14386771600720633,437 "brier_ci": [438 0.13655056999498,439 0.15186071829326037440 ],441 "auc": 0.7575876104583843,442 "auc_ci": [443 0.7365552953121464,444 0.7791821192820314445 ],446 "ece": 0.038623798444173775,447 "n_test": 3390,448 "base_rate": 0.20914454277286135,449 "n_boot": 200450 }451 }452 }453 },454 "chronological_oct1_naive": {455 "n_train": 49629,456 "n_test": 3317,457 "base_rate_train": 0.18525458905075662,458 "models": {459 "logistic": {460 "log_loss": 0.46509032297028385,461 "log_loss_ci": [462 0.4470978564068478,463 0.48600377941085987464 ],465 "brier": 0.1496902465009174,466 "brier_ci": [467 0.14264398202210207,468 0.158060585967047469 ],470 "auc": 0.7375578538066375,471 "auc_ci": [472 0.7195049645973914,473 0.7571505747637189474 ],475 "ece": 0.03005696934895857,476 "n_test": 3317,477 "base_rate": 0.21977690684353332,478 "n_boot": 200479 },480 "lightgbm": {481 "log_loss": 0.4439707299774289,482 "log_loss_ci": [483 0.42205230403691435,484 0.46176621078250085485 ],486 "brier": 0.1428246998812338,487 "brier_ci": [488 0.13594568743272048,489 0.1499526613238938490 ],491 "auc": 0.7745026639783066,492 "auc_ci": [493 0.7566072005085992,494 0.7920219022181805495 ],496 "ece": 0.02937900920685068,497 "n_test": 3317,498 "base_rate": 0.21977690684353332,499 "n_boot": 200,500 "best_iteration": 261501 }502 }503 }504 },505 "markdown30cr": {506 "chronological": {507 "n_train": 14276,508 "n_test": 6063,509 "base_rate_train": 0.3918464555898011,510 "models": {511 "logistic": {512 "log_loss": 0.5637219907792722,513 "log_loss_ci": [514 0.5482431704065615,515 0.5778146612256625516 ],517 "brier": 0.18898139694973967,518 "brier_ci": [519 0.18269645765385287,520 0.19467250569282127521 ],522 "auc": 0.7755413057643372,523 "auc_ci": [524 0.7640938113944955,525 0.7874854542227134526 ],527 "ece": 0.031394109039069536,528 "n_test": 6063,529 "base_rate": 0.41184232228269835,530 "n_boot": 200531 },532 "lightgbm": {533 "log_loss": 0.5467246174577667,534 "log_loss_ci": [535 0.5310146598184058,536 0.5629300267524441537 ],538 "brier": 0.183276689115341,539 "brier_ci": [540 0.17681122219478695,541 0.18987126943664784542 ],543 "auc": 0.7909485212877999,544 "auc_ci": [545 0.77861711147321,546 0.8027709748473271547 ],548 "ece": 0.04323603074207272,549 "n_test": 6063,550 "base_rate": 0.41184232228269835,551 "n_boot": 200,552 "best_iteration": 115553 }554 }555 },556 "dealer_holdout": {557 "n_train": 16271,558 "n_test": 20339,559 "base_rate_train": 0.39771721138450267,560 "models": {561 "logistic": {562 "log_loss": 0.7005190170382144,563 "log_loss_ci": [564 0.6928866102612254,565 0.7064217330317188566 ],567 "brier": 0.25111431277069857,568 "brier_ci": [569 0.24763056400761974,570 0.2537672066965328571 ],572 "auc": 0.5661423583840284,573 "auc_ci": [574 0.5578319444843819,575 0.5747562525757856576 ],577 "ece": 0.12054954569707432,578 "n_test": 20339,579 "base_rate": 0.39780716849402625,580 "n_boot": 200581 },582 "lightgbm": {583 "log_loss": 0.7144309406244321,584 "log_loss_ci": [585 0.7061210144206004,586 0.7205447668953259587 ],588 "brier": 0.25622138352776636,589 "brier_ci": [590 0.25244333927177787,591 0.25889654354664343592 ],593 "auc": 0.5504099817062947,594 "auc_ci": [595 0.5431619587061016,596 0.557388232181443597 ],598 "ece": 0.13487619846537036,599 "n_test": 20339,600 "base_rate": 0.39780716849402625,601 "n_boot": 200,602 "best_iteration": 82603 }604 },605 "folds": [606 {607 "logistic": {608 "log_loss": 0.6721499004851819,609 "brier": 0.2381456264337999,610 "auc": 0.5368595331702526,611 "n_test": 3605612 },613 "lightgbm": {614 "log_loss": 0.6684538336359788,615 "brier": 0.23642654790540482,616 "auc": 0.5436638069862091,617 "n_test": 3605618 }619 },620 {621 "logistic": {622 "log_loss": 0.6962175691625236,623 "brier": 0.2505438892292713,624 "auc": 0.6260159483373768,625 "n_test": 3713626 },627 "lightgbm": {628 "log_loss": 0.7661854033534757,629 "brier": 0.2771699388038127,630 "auc": 0.5730483801912373,631 "n_test": 3713632 }633 },634 {635 "logistic": {636 "log_loss": 0.6997896602220616,637 "brier": 0.25179983844287573,638 "auc": 0.5565209933060266,639 "n_test": 3554640 },641 "lightgbm": {642 "log_loss": 0.6916168049669729,643 "brier": 0.24814415377160262,644 "auc": 0.5629023324149636,645 "n_test": 3554646 }647 },648 {649 "logistic": {650 "log_loss": 0.6788168191305408,651 "brier": 0.24193390204383972,652 "auc": 0.6029888495507757,653 "n_test": 3068654 },655 "lightgbm": {656 "log_loss": 0.6984075958591713,657 "brier": 0.24998879197692284,658 "auc": 0.6054765679128055,659 "n_test": 3068660 }661 },662 {663 "logistic": {664 "log_loss": 0.7298074087308966,665 "brier": 0.26277226537656256,666 "auc": 0.5462240524840558,667 "n_test": 6399668 },669 "lightgbm": {670 "log_loss": 0.7306560086736379,671 "brier": 0.26269215429039283,672 "auc": 0.5493163768940164,673 "n_test": 6399674 }675 }676 ],677 "fold_mean": {678 "logistic": {679 "log_loss": 0.6953562715462409,680 "brier": 0.24903910430526985,681 "auc": 0.5737218753696975682 },683 "lightgbm": {684 "log_loss": 0.7110639292978473,685 "brier": 0.2548843173496272,686 "auc": 0.5668814928798463687 }688 },689 "fold_sd": {690 "logistic": {691 "log_loss": 0.020096600538824665,692 "brier": 0.008574817816935252,693 "auc": 0.03464695866858239694 },695 "lightgbm": {696 "log_loss": 0.03398888097451171,697 "brier": 0.013911840049987833,698 "auc": 0.021867959537952723699 }700 }701 },702 "region_holdout": {703 "n_train": 18132,704 "n_test": 13239,705 "base_rate_train": 0.3935066693678861,706 "models": {707 "logistic": {708 "log_loss": 0.7279169836539531,709 "log_loss_ci": [710 0.7203545423861841,711 0.7368465337081728712 ],713 "brier": 0.26405975781893554,714 "brier_ci": [715 0.26080416474510393,716 0.26817906166815014717 ],718 "auc": 0.5689278880282912,719 "auc_ci": [720 0.559730577457717,721 0.5785767350181213722 ],723 "ece": 0.15263668744259273,724 "n_test": 13239,725 "base_rate": 0.4333408867739255,726 "n_boot": 200727 },728 "lightgbm": {729 "log_loss": 0.7754028335160541,730 "log_loss_ci": [731 0.7653136294660776,732 0.7869190462821098733 ],734 "brier": 0.2807395283646377,735 "brier_ci": [736 0.27656324074565614,737 0.28561108153143416738 ],739 "auc": 0.5660288981673771,740 "auc_ci": [741 0.5567816213441923,742 0.5760181596090554743 ],744 "ece": 0.1983107356543753,745 "n_test": 13239,746 "base_rate": 0.4333408867739255,747 "n_boot": 200,748 "best_iteration": 112749 }750 },751 "per_region": {752 "Mont\u00e9r\u00e9gie": {753 "logistic": {754 "log_loss": 0.6979735344282783,755 "brier": 0.25091931128131195,756 "auc": 0.6274596487032731,757 "n_test": 3968,758 "base_rate": 0.4117943548387097759 },760 "lightgbm": {761 "log_loss": 0.7308769612041666,762 "brier": 0.26401534301799795,763 "auc": 0.6247866932231636,764 "n_test": 3968,765 "base_rate": 0.4117943548387097766 }767 },768 "Capitale-Nationale": {769 "logistic": {770 "log_loss": 0.6501257364476918,771 "brier": 0.2272923471771655,772 "auc": 0.5837718405413049,773 "n_test": 2205,774 "base_rate": 0.3528344671201814775 },776 "lightgbm": {777 "log_loss": 0.6684309174003459,778 "brier": 0.23459433465335439,779 "auc": 0.6002426576689371,780 "n_test": 2205,781 "base_rate": 0.3528344671201814782 }783 },784 "Montr\u00e9al": {785 "logistic": {786 "log_loss": 0.789015108743636,787 "brier": 0.2933939906949219,788 "auc": 0.48283471025001756,789 "n_test": 1721,790 "base_rate": 0.463102847181871791 },792 "lightgbm": {793 "log_loss": 0.8101831753581833,794 "brier": 0.3005070692199156,795 "auc": 0.4491185832151955,796 "n_test": 1721,797 "base_rate": 0.463102847181871798 }799 },800 "Laurentides": {801 "logistic": {802 "log_loss": 0.751779913083233,803 "brier": 0.27640311102869236,804 "auc": 0.5589441980695804,805 "n_test": 2175,806 "base_rate": 0.47954022988505746807 },808 "lightgbm": {809 "log_loss": 0.9070770692950642,810 "brier": 0.3267051727647063,811 "auc": 0.5495216299814683,812 "n_test": 2175,813 "base_rate": 0.47954022988505746814 }815 },816 "Laval": {817 "logistic": {818 "log_loss": 0.8222275938779959,819 "brier": 0.305515719695642,820 "auc": 0.6248827077747989,821 "n_test": 1706,822 "base_rate": 0.5627198124267292823 },824 "lightgbm": {825 "log_loss": 0.8209667529522688,826 "brier": 0.3067649326755811,827 "auc": 0.6293363214924039,828 "n_test": 1706,829 "base_rate": 0.5627198124267292830 }831 },832 "Chaudi\u00e8re-Appalaches": {833 "logistic": {834 "log_loss": 0.6563800153428839,835 "brier": 0.23083406703587958,836 "auc": 0.5356742228307724,837 "n_test": 1464,838 "base_rate": 0.35860655737704916839 },840 "lightgbm": {841 "log_loss": 0.7148821248862958,842 "brier": 0.2518503178562711,843 "auc": 0.5431675034230945,844 "n_test": 1464,845 "base_rate": 0.35860655737704916846 }847 }848 },849 "aggregation": "unweighted mean over the 6 held-out regions; CI = bootstrap within region, then averaged"850 },851 "drift": {852 "weeks_4_5": {853 "n_train": 11235,854 "n_test": 7443,855 "weeks_observable": [856 4,857 5858 ],859 "models": {860 "logistic": {861 "log_loss": 0.5567006521667089,862 "log_loss_ci": [863 0.5459277692610365,864 0.5710694199383016865 ],866 "brier": 0.18571018269335923,867 "brier_ci": [868 0.18136510458658878,869 0.19133924524398507870 ],871 "auc": 0.7776112165359532,872 "auc_ci": [873 0.7666936968640178,874 0.7873250821328512875 ],876 "ece": 0.023385648247175744,877 "n_test": 7443,878 "base_rate": 0.40239150880021496,879 "n_boot": 200880 },881 "lightgbm": {882 "log_loss": 0.5410008625221233,883 "log_loss_ci": [884 0.5288858758028351,885 0.5532435601578369886 ],887 "brier": 0.18063948478616662,888 "brier_ci": [889 0.17574997585801025,890 0.18566945358616768891 ],892 "auc": 0.7905054587381848,893 "auc_ci": [894 0.7802982958186815,895 0.8005679840226058896 ],897 "ece": 0.025709115696372607,898 "n_test": 7443,899 "base_rate": 0.40239150880021496,900 "n_boot": 200901 }902 }903 },904 "weeks_6_7": {905 "n_train": 11235,906 "n_test": 1661,907 "weeks_observable": [908 6909 ],910 "models": {911 "logistic": {912 "log_loss": 0.5962064347610395,913 "log_loss_ci": [914 0.5699809905795714,915 0.6170593535347667916 ],917 "brier": 0.20476278962038852,918 "brier_ci": [919 0.19385436783402532,920 0.21351496914610565921 ],922 "auc": 0.7441271289898188,923 "auc_ci": [924 0.7227945242067729,925 0.7683391033064382926 ],927 "ece": 0.059024879967338455,928 "n_test": 1661,929 "base_rate": 0.42685129440096325,930 "n_boot": 200931 },932 "lightgbm": {933 "log_loss": 0.5887291970118351,934 "log_loss_ci": [935 0.5611034342098475,936 0.6110330619225628937 ],938 "brier": 0.20188682991095178,939 "brier_ci": [940 0.19022287997094836,941 0.21088155568241712942 ],943 "auc": 0.7539520392077848,944 "auc_ci": [945 0.7337742472107264,946 0.7784943330270407947 ],948 "ece": 0.055579280547829485,949 "n_test": 1661,950 "base_rate": 0.42685129440096325,951 "n_boot": 200952 }953 }954 }955 }956 },957 "exit30": {958 "chronological": {959 "n_train": 31050,960 "n_test": 12953,961 "base_rate_train": 0.5936876006441224,962 "models": {963 "logistic": {964 "log_loss": 0.6088472387814978,965 "log_loss_ci": [966 0.6031527702410103,967 0.6159728743003138968 ],969 "brier": 0.21178355794941933,970 "brier_ci": [971 0.20923576034582778,972 0.21470020946952184973 ],974 "auc": 0.7051763801959573,975 "auc_ci": [976 0.6967221086575787,977 0.7130210455398301978 ],979 "ece": 0.01002959582650733,980 "n_test": 12953,981 "base_rate": 0.591523199258859,982 "n_boot": 200983 },984 "lightgbm": {985 "log_loss": 0.5848254122682515,986 "log_loss_ci": [987 0.579572637852006,988 0.5917882119276924989 ],990 "brier": 0.20417535681077367,991 "brier_ci": [992 0.20197740695201732,993 0.2071859275181045994 ],995 "auc": 0.7266748680217747,996 "auc_ci": [997 0.7180928270188718,998 0.7345726612979446999 ],1000 "ece": 0.01353355318051327,1001 "n_test": 12953,1002 "base_rate": 0.591523199258859,1003 "n_boot": 200,1004 "best_iteration": 1061005 }1006 }1007 },1008 "dealer_holdout": {1009 "n_train": 35202,1010 "n_test": 44003,1011 "base_rate_train": 0.5932610167639272,1012 "models": {1013 "logistic": {1014 "log_loss": 0.6953075532604525,1015 "log_loss_ci": [1016 0.6939756215804797,1017 0.69687235352750851018 ],1019 "brier": 0.25069102088297,1020 "brier_ci": [1021 0.25005532404300757,1022 0.25142063402178071023 ],1024 "auc": 0.5661785513729787,1025 "auc_ci": [1026 0.5610973794101907,1027 0.57102823373994551028 ],1029 "ece": 0.10358565468474734,1030 "n_test": 44003,1031 "base_rate": 0.5930504738313297,1032 "n_boot": 2001033 },1034 "lightgbm": {1035 "log_loss": 0.6821109296285043,1036 "log_loss_ci": [1037 0.6795523967337413,1038 0.68438753907356311039 ],1040 "brier": 0.24422247387862409,1041 "brier_ci": [1042 0.24295783220673198,1043 0.24525748455241721044 ],1045 "auc": 0.6059890624432378,1046 "auc_ci": [1047 0.6016595848855064,1048 0.61166641915286111049 ],1050 "ece": 0.09144264800675378,1051 "n_test": 44003,1052 "base_rate": 0.5930504738313297,1053 "n_boot": 200,1054 "best_iteration": 4491055 }1056 },1057 "folds": [1058 {1059 "logistic": {1060 "log_loss": 0.7642903638085777,1061 "brier": 0.2842491164669829,1062 "auc": 0.5138150525496235,1063 "n_test": 94421064 },1065 "lightgbm": {1066 "log_loss": 0.7304352031135112,1067 "brier": 0.2664534489927506,1068 "auc": 0.5533726768883124,1069 "n_test": 94421070 }1071 },1072 {1073 "logistic": {1074 "log_loss": 0.6720080012327938,1075 "brier": 0.2394538905841713,1076 "auc": 0.6175266332948275,1077 "n_test": 73941078 },1079 "lightgbm": {1080 "log_loss": 0.6385625871818302,1081 "brier": 0.22435440481556582,1082 "auc": 0.674373084029575,1083 "n_test": 73941084 }1085 },1086 {1087 "logistic": {1088 "log_loss": 0.67169022792453,1089 "brier": 0.23939251100123296,1090 "auc": 0.6130220004181246,1091 "n_test": 75481092 },1093 "lightgbm": {1094 "log_loss": 0.6783879532657491,1095 "brier": 0.24246094778639626,1096 "auc": 0.6211780048465038,1097 "n_test": 75481098 }1099 },1100 {1101 "logistic": {1102 "log_loss": 0.6763314074965648,1103 "brier": 0.24149337517820746,1104 "auc": 0.6043619492049606,1105 "n_test": 68121106 },1107 "lightgbm": {1108 "log_loss": 0.6769938281467315,1109 "brier": 0.24219782232526701,1110 "auc": 0.5784028279993041,1111 "n_test": 68121112 }1113 },1114 {1115 "logistic": {1116 "log_loss": 0.6819141174110152,1117 "brier": 0.2439890077693026,1118 "auc": 0.5977789695616047,1119 "n_test": 128071120 },1121 "lightgbm": {1122 "log_loss": 0.6765418950915469,1123 "brier": 0.24141834808415819,1124 "auc": 0.6199872578629079,1125 "n_test": 128071126 }1127 }1128 ],1129 "fold_mean": {1130 "logistic": {1131 "log_loss": 0.6932468235746964,1132 "brier": 0.24971558019997944,1133 "auc": 0.58930092100582821134 },1135 "lightgbm": {1136 "log_loss": 0.6801842933598737,1137 "brier": 0.24337699440082758,1138 "auc": 0.60946277032532061139 }1140 },1141 "fold_sd": {1142 "logistic": {1143 "log_loss": 0.03571420033977364,1144 "brier": 0.017348380015330414,1145 "auc": 0.038356769042823321146 },1147 "lightgbm": {1148 "log_loss": 0.029271806665782482,1149 "brier": 0.013419743325710736,1150 "auc": 0.04140574593475561151 }1152 }1153 },1154 "region_holdout": {1155 "n_train": 39178,1156 "n_test": 28947,1157 "base_rate_train": 0.5919150624764695,1158 "models": {1159 "logistic": {1160 "log_loss": 0.6898424179937633,1161 "log_loss_ci": [1162 0.6876116232562756,1163 0.69288968501780581164 ],1165 "brier": 0.24805033270420537,1166 "brier_ci": [1167 0.2470147822821151,1168 0.249505606102326921169 ],1170 "auc": 0.5843639938572319,1171 "auc_ci": [1172 0.5765915195861516,1173 0.59063795165397571174 ],1175 "ece": 0.11595122463170887,1176 "n_test": 28947,1177 "base_rate": 0.6009949217535496,1178 "n_boot": 2001179 },1180 "lightgbm": {1181 "log_loss": 0.6798344332722573,1182 "log_loss_ci": [1183 0.6751019194075778,1184 0.68363718031147341185 ],1186 "brier": 0.24298869257240452,1187 "brier_ci": [1188 0.24087326534193323,1189 0.244696855254386281190 ],1191 "auc": 0.6133305977264796,1192 "auc_ci": [1193 0.6071406896202246,1194 0.62194107224617711195 ],1196 "ece": 0.10448533817805425,1197 "n_test": 28947,1198 "base_rate": 0.6009949217535496,1199 "n_boot": 200,1200 "best_iteration": 4731201 }1202 },1203 "per_region": {1204 "Mont\u00e9r\u00e9gie": {1205 "logistic": {1206 "log_loss": 0.7188528093306531,1207 "brier": 0.2624094644893386,1208 "auc": 0.5509636330525092,1209 "n_test": 9137,1210 "base_rate": 0.61322096968370371211 },1212 "lightgbm": {1213 "log_loss": 0.6751915484947962,1214 "brier": 0.24098619437238528,1215 "auc": 0.6164142349967947,1216 "n_test": 9137,1217 "base_rate": 0.61322096968370371218 }1219 },1220 "Capitale-Nationale": {1221 "logistic": {1222 "log_loss": 0.6828446854680117,1223 "brier": 0.2444565449905255,1224 "auc": 0.6109676179358625,1225 "n_test": 5358,1226 "base_rate": 0.63624486748786861227 },1228 "lightgbm": {1229 "log_loss": 0.6604653600398963,1230 "brier": 0.23358603589076554,1231 "auc": 0.6573706066743618,1232 "n_test": 5358,1233 "base_rate": 0.63624486748786861234 }1235 },1236 "Montr\u00e9al": {1237 "logistic": {1238 "log_loss": 0.6946656128794362,1239 "brier": 0.250569480296554,1240 "auc": 0.5517556120672836,1241 "n_test": 4290,1242 "base_rate": 0.65081585081585081243 },1244 "lightgbm": {1245 "log_loss": 0.7148603276129286,1246 "brier": 0.26040523393997345,1247 "auc": 0.5211963611462848,1248 "n_test": 4290,1249 "base_rate": 0.65081585081585081250 }1251 },1252 "Laurentides": {1253 "logistic": {1254 "log_loss": 0.683805325162613,1255 "brier": 0.24530426447232903,1256 "auc": 0.5789858356492006,1257 "n_test": 4033,1258 "base_rate": 0.53830895115298781259 },1260 "lightgbm": {1261 "log_loss": 0.678479763842318,1262 "brier": 0.24183514013061563,1263 "auc": 0.6295728381294092,1264 "n_test": 4033,1265 "base_rate": 0.53830895115298781266 }1267 },1268 "Laval": {1269 "logistic": {1270 "log_loss": 0.6681485669888422,1271 "brier": 0.23771971448543755,1272 "auc": 0.6306599161513359,1273 "n_test": 3309,1274 "base_rate": 0.58114233907524941275 },1276 "lightgbm": {1277 "log_loss": 0.6677104327303682,1278 "brier": 0.23724406407731877,1279 "auc": 0.6422776535881061,1280 "n_test": 3309,1281 "base_rate": 0.58114233907524941282 }1283 },1284 "Chaudi\u00e8re-Appalaches": {1285 "logistic": {1286 "log_loss": 0.6907375081330245,1287 "brier": 0.24784252749104724,1288 "auc": 0.5828513482872003,1289 "n_test": 2820,1290 "base_rate": 0.53156028368794331291 },1292 "lightgbm": {1293 "log_loss": 0.6822991669132373,1294 "brier": 0.24387548702336842,1295 "auc": 0.613151891823921,1296 "n_test": 2820,1297 "base_rate": 0.53156028368794331298 }1299 }1300 },1301 "aggregation": "unweighted mean over the 6 held-out regions; CI = bootstrap within region, then averaged"1302 },1303 "drift": {1304 "weeks_4_5": {1305 "n_train": 24354,1306 "n_test": 16259,1307 "weeks_observable": [1308 4,1309 51310 ],1311 "models": {1312 "logistic": {1313 "log_loss": 0.6180448625667474,1314 "log_loss_ci": [1315 0.6122545852471974,1316 0.62485153866428631317 ],1318 "brier": 0.2143300681453042,1319 "brier_ci": [1320 0.21199396407553625,1321 0.21723104590212581322 ],1323 "auc": 0.6961149574713865,1324 "auc_ci": [1325 0.6874773333231922,1326 0.70310135451551281327 ],1328 "ece": 0.020812454782359588,1329 "n_test": 16259,1330 "base_rate": 0.5975767267359616,1331 "n_boot": 2001332 },1333 "lightgbm": {1334 "log_loss": 0.5961430484020771,1335 "log_loss_ci": [1336 0.5885803073722725,1337 0.60385202063758411338 ],1339 "brier": 0.2060214424618318,1340 "brier_ci": [1341 0.20320126544077627,1342 0.208959018759708041343 ],1344 "auc": 0.7262358107656184,1345 "auc_ci": [1346 0.7187771600735826,1347 0.73232301817541981348 ],1349 "ece": 0.03818442246045094,1350 "n_test": 16259,1351 "base_rate": 0.5975767267359616,1352 "n_boot": 2001353 }1354 }1355 },1356 "weeks_6_7": {1357 "n_train": 24354,1358 "n_test": 3390,1359 "weeks_observable": [1360 61361 ],1362 "models": {1363 "logistic": {1364 "log_loss": 0.6333550648257135,1365 "log_loss_ci": [1366 0.6209306764410181,1367 0.64490924944711581368 ],1369 "brier": 0.2218456776799865,1370 "brier_ci": [1371 0.21630831587905686,1372 0.226784674080364541373 ],1374 "auc": 0.6753586317599491,1375 "auc_ci": [1376 0.6587602871104657,1377 0.69154433076270261378 ],1379 "ece": 0.02274087552905669,1380 "n_test": 3390,1381 "base_rate": 0.58023598820059,1382 "n_boot": 2001383 },1384 "lightgbm": {1385 "log_loss": 0.6218107135111449,1386 "log_loss_ci": [1387 0.6075731463716002,1388 0.6364792572168471389 ],1390 "brier": 0.21818203376834822,1391 "brier_ci": [1392 0.2124429283828738,1393 0.224222548761608831394 ],1395 "auc": 0.691144038261676,1396 "auc_ci": [1397 0.6741500674625492,1398 0.70988875556244321399 ],1400 "ece": 0.035984583133177,1401 "n_test": 3390,1402 "base_rate": 0.58023598820059,1403 "n_boot": 2001404 }1405 }1406 }1407 },1408 "chronological_oct1_naive": {1409 "n_train": 49629,1410 "n_test": 3317,1411 "base_rate_train": 0.5878014870337908,1412 "models": {1413 "logistic": {1414 "log_loss": 0.6322307351973232,1415 "log_loss_ci": [1416 0.6197181733045866,1417 0.64473371240536971418 ],1419 "brier": 0.22085203765615868,1420 "brier_ci": [1421 0.21528512815956186,1422 0.226339719265520521423 ],1424 "auc": 0.7127147572782868,1425 "auc_ci": [1426 0.6945614223520684,1427 0.73326390291927711428 ],1429 "ece": 0.08234808302947866,1430 "n_test": 3317,1431 "base_rate": 0.4615616520952668,1432 "n_boot": 2001433 },1434 "lightgbm": {1435 "log_loss": 0.6260313192136273,1436 "log_loss_ci": [1437 0.6139812011725851,1438 0.63910324205065951439 ],1440 "brier": 0.22117522310144752,1441 "brier_ci": [1442 0.215729811481472,1443 0.226451500715404431444 ],1445 "auc": 0.7302575807335229,1446 "auc_ci": [1447 0.7164698938679159,1448 0.74541289380630271449 ],1450 "ece": 0.12055348417954814,1451 "n_test": 3317,1452 "base_rate": 0.4615616520952668,1453 "n_boot": 200,1454 "best_iteration": 2751455 }1456 }1457 }1458 }1459 },1460 "notes": {1461 "region_holdout_regions": [1462 "Mont\u00e9r\u00e9gie",1463 "Capitale-Nationale",1464 "Montr\u00e9al",1465 "Laurentides",1466 "Laval",1467 "Chaudi\u00e8re-Appalaches"1468 ],1469 "drift": "train = entries in weeks 0-3 (days 0-27); the week 6-7 window only contains observable entries up to day 44 (week 6), week 7 has no determinable 30-day outcome",1470 "chronological_oct1_naive": "Analysis-plan cutoff (entry before vs on/after 2026-10-01 UTC) applied to the looser rule entry day <= 51. Test entries (days 50-51) have only ~22-24 days of confirmable follow-up: exits at ages between ~23 and 30 days are coded 0 (exit30 base rate is a lower bound) and the last-day markdowns may be unclassified. Reported as a sensitivity only.",1471 "chronological": "primary chronological split: train entries before 2026-09-15 UTC, test 2026-09-15 to day 44 (2026-09-25); the plan's 2026-10-01 cutoff leaves no listing with a determinable 30-day outcome in the test fold",1472 "targets": {1473 "markdown30": "first sustained markdown (>=1 % and >=100 $) at listing age <= 30 d; exits before any cut count as 0",1474 "markdown30cr": "competing-risk-aware: listings that exit within 30 d without a prior cut are removed; event = markdown within 30 d among the rest",1475 "exit30": "disappearance confirmed by >= 7 d of absence at listing age <= 30 d (NOT a sale)"1476 },1477 "features": {1478 "numeric": [1479 "age_years_entry",1480 "km10k",1481 "year",1482 "logp",1483 "resid_A2",1484 "week",1485 "log_dens_10",1486 "log_dens_25",1487 "log_dens_50",1488 "log_own_stock",1489 "pricepos_25"1490 ],1491 "dealer_target_encoding": [1492 "dealer_rate_markdown30",1493 "dealer_rate_exit30",1494 "dealer_log_n_train",1495 "(training fold only; inner 5-fold OOF for training rows; smoothing m=20)"1496 ],1497 "categorical_lightgbm": [1498 "powertrain",1499 "body",1500 "region",1501 "source_class",1502 "make",1503 "model",1504 "trim",1505 "transmission",1506 "drivetrain"1507 ],1508 "one_hot_logistic": [1509 "powertrain",1510 "body",1511 "region",1512 "source_class"1513 ],1514 "competition_at_entry": "risk_set row with listing_age = 0 (dens_10/25/50, own_stock, pricepos_25; NaN when no competitor or no coordinates)",1515 "excluded": "dealer_id (identity), anything measured after entry"1516 },1517 "models": {1518 "logistic": "sklearn LogisticRegression L2, C=1, standardised numerics + missing indicators + one-hot coarse categoricals",1519 "lightgbm": {1520 "n_estimators": 2000,1521 "learning_rate": 0.05,1522 "num_leaves": 31,1523 "min_child_samples": 50,1524 "subsample": 0.8,1525 "subsample_freq": 1,1526 "colsample_bytree": 0.8,1527 "reg_lambda": 1.0,1528 "cat_l2": 10.0,1529 "cat_smooth": 50.0,1530 "max_cat_to_onehot": 4,1531 "n_jobs": 8,1532 "verbose": -1,1533 "random_state": 36,1534 "early_stopping": "100 rounds on inner validation (latest 15 % of training entries for chronological splits, 15 % of training dealers otherwise)"1535 }1536 },1537 "metrics": {1538 "ci": "200 bootstrap resamples of test rows, percentile 2.5/97.5",1539 "calibration": "10 equal-width probability bins; ECE = n-weighted |frac_pos - mean_pred|"1540 }1541 },1542 "samples": {1543 "n_S2": 80856,1544 "n_strict_observable": 44003,1545 "strict_rule": "entry_ts <= SNAPSHOT_TS - 30 - 7 days (entry day <= 44); every listing has a determined 30-day exit and markdown outcome",1546 "n_naive_observable": 52946,1547 "naive_rule": "entry day <= 51 (analysis-plan rule; exit at ages > snapshot-7d-entry cannot be confirmed)",1548 "n_without_coordinates": 574,1549 "base_rates_strict": {1550 "markdown30": 0.18387382678453743,1551 "markdown30cr": 0.39780716849402625,1552 "exit30": 0.59305047383132971553 },1554 "n_markdown30cr": 20339,1555 "n_exit_before_cut_removed_in_cr": 236641556 },1557 "survival": {1558 "model": "CoxPHFitter(penalizer=0.01)",1559 "split": "chronological, entry before vs on/after 2026-10-01 UTC",1560 "n_train": 49629,1561 "events_train": 34694,1562 "n_test": 31227,1563 "events_test": 9040,1564 "c_index_train": 0.6001165372154094,1565 "c_index_test": 0.6581351276518386,1566 "dynamic_auc_30d": null,1567 "dynamic_auc_note": "scikit-survival (sksurv) is not installed in the project environment; the 30-day cumulative/dynamic AUC was not computed.",1568 "covariates": [1569 "age_years_entry",1570 "logp",1571 "resid_A2",1572 "km10k",1573 "log_dens_25",1574 "pricepos_25f",1575 "pricepos_missing",1576 "coords_missing",1577 "powertrain=BEV",1578 "powertrain=Diesel",1579 "powertrain=HEV",1580 "powertrain=PHEV",1581 "powertrain=unknown",1582 "body=Convertible",1583 "body=Coupe",1584 "body=Hatchback",1585 "body=Pickup",1586 "body=Sedan",1587 "body=Van",1588 "body=Wagon",1589 "body=unknown",1590 "region=(missing)",1591 "region=Abitibi-T\u00e9miscamingue",1592 "region=Bas-Saint-Laurent",1593 "region=Capitale-Nationale",1594 "region=Centre-du-Qu\u00e9bec",1595 "region=Chaudi\u00e8re-Appalaches",1596 "region=C\u00f4te-Nord",1597 "region=Estrie",1598 "region=Gasp\u00e9sie\u2013\u00celes-de-la-Madeleine",1599 "region=Lanaudi\u00e8re",1600 "region=Laurentides",1601 "region=Laval",1602 "region=Mauricie",1603 "region=Mont\u00e9r\u00e9gie",1604 "region=Outaouais",1605 "region=Saguenay\u2013Lac-Saint-Jean"1606 ],1607 "seconds": 1.91608 },1609 "seconds_total": 91.21610}