1{2 "targets": {3 "markdown30": {4 "chronological": {5 "n_train": 31050,6 "n_test": 12953,7 "base_rate_train": 0.1801610305958132,8 "models": {9 "logistic": {10 "log_loss": 0.42750623257152703,11 "log_loss_ci": [12 0.4166106768427018,13 0.438373981527184814 ],15 "brier": 0.1345025681231389,16 "brier_ci": [17 0.1304564359112002,18 0.138385537075825819 ],20 "auc": 0.7484582110621498,21 "auc_ci": [22 0.7379166219959526,23 0.759319149774091724 ],25 "ece": 0.017449892406937298,26 "n_test": 12953,27 "base_rate": 0.1927738747780437,28 "n_boot": 20029 },30 "lightgbm": {31 "log_loss": 0.405069395922176,32 "log_loss_ci": [33 0.39518869462086986,34 0.415174341161424135 ],36 "brier": 0.12834546775673097,37 "brier_ci": [38 0.12500161895416048,39 0.1320970791582154540 ],41 "auc": 0.7849142957777335,42 "auc_ci": [43 0.7759402630086811,44 0.793036311970565145 ],46 "ece": 0.017154048273368354,47 "n_test": 12953,48 "base_rate": 0.1927738747780437,49 "n_boot": 200,50 "best_iteration": 20751 }52 }53 },54 "dealer_holdout": {55 "n_train": 35202,56 "n_test": 44003,57 "base_rate_train": 0.18379591843568377,58 "models": {59 "logistic": {60 "log_loss": 0.4761785736116495,61 "log_loss_ci": [62 0.46930154146077574,63 0.482877142796650364 ],65 "brier": 0.14959691330267216,66 "brier_ci": [67 0.14685355336351175,68 0.1522382873482736669 ],70 "auc": 0.5797519260404436,71 "auc_ci": [72 0.5728929876742087,73 0.585524872756838174 ],75 "ece": 0.032841922762235826,76 "n_test": 44003,77 "base_rate": 0.18387382678453743,78 "n_boot": 20079 },80 "lightgbm": {81 "log_loss": 0.49356532371936024,82 "log_loss_ci": [83 0.48578858607171627,84 0.501526241991285685 ],86 "brier": 0.15282816957870576,87 "brier_ci": [88 0.14996576207828186,89 0.155683094088074790 ],91 "auc": 0.5818531774577216,92 "auc_ci": [93 0.5755689074469532,94 0.58876583781580995 ],96 "ece": 0.06444814346386876,97 "n_test": 44003,98 "base_rate": 0.18387382678453743,99 "n_boot": 200,100 "best_iteration": 328101 }102 },103 "folds": [104 {105 "logistic": {106 "log_loss": 0.40957112977000665,107 "brier": 0.12211555325696961,108 "auc": 0.5596656690240565,109 "n_test": 9442110 },111 "lightgbm": {112 "log_loss": 0.4126903470581999,113 "brier": 0.12122175786673123,114 "auc": 0.5521863552436166,115 "n_test": 9442116 }117 },118 {119 "logistic": {120 "log_loss": 0.509415081001042,121 "brier": 0.1643607764970009,122 "auc": 0.6536422756538248,123 "n_test": 7394124 },125 "lightgbm": {126 "log_loss": 0.5415009680094971,127 "brier": 0.17174478939410306,128 "auc": 0.6357532231481676,129 "n_test": 7394130 }131 },132 {133 "logistic": {134 "log_loss": 0.4877034794825335,135 "brier": 0.15477553508252154,136 "auc": 0.5975975969204793,137 "n_test": 7548138 },139 "lightgbm": {140 "log_loss": 0.5112177945402372,141 "brier": 0.15970768972798888,142 "auc": 0.5538139327022289,143 "n_test": 7548144 }145 },146 {147 "logistic": {148 "log_loss": 0.4512466370992524,149 "brier": 0.14050170798545783,150 "auc": 0.6264868547689653,151 "n_test": 6812152 },153 "lightgbm": {154 "log_loss": 0.4719536685507126,155 "brier": 0.14508390645374936,156 "auc": 0.608342530011907,157 "n_test": 6812158 }159 },160 {161 "logistic": {162 "log_loss": 0.5125651677495925,163 "brier": 0.16311946335492622,164 "auc": 0.5870307886167137,165 "n_test": 12807166 },167 "lightgbm": {168 "log_loss": 0.526606872950738,169 "brier": 0.16527336008310423,170 "auc": 0.5770475644655639,171 "n_test": 12807172 }173 }174 ],175 "fold_mean": {176 "logistic": {177 "log_loss": 0.4741002990204854,178 "brier": 0.1489746072353752,179 "auc": 0.6048846369968078180 },181 "lightgbm": {182 "log_loss": 0.49279393022187695,183 "brier": 0.15260630070513534,184 "auc": 0.5854287211142968185 }186 },187 "fold_sd": {188 "logistic": {189 "log_loss": 0.038982124156417344,190 "brier": 0.01589755154350599,191 "auc": 0.032436334748729166192 },193 "lightgbm": {194 "log_loss": 0.046273167050968046,195 "brier": 0.017993008714973422,196 "auc": 0.032349317349656244197 }198 }199 },200 "region_holdout": {201 "n_train": 39178,202 "n_test": 28947,203 "base_rate_train": 0.18223023652950746,204 "models": {205 "logistic": {206 "log_loss": 0.5082798451338891,207 "log_loss_ci": [208 0.4999467599238686,209 0.5161637208266595210 ],211 "brier": 0.16349444465660634,212 "brier_ci": [213 0.16004604145352064,214 0.16668660811281166215 ],216 "auc": 0.6053941458953844,217 "auc_ci": [218 0.5965430127854012,219 0.6142298883566174220 ],221 "ece": 0.040782211420796806,222 "n_test": 28947,223 "base_rate": 0.19818979514284726,224 "n_boot": 200225 },226 "lightgbm": {227 "log_loss": 0.526177806707393,228 "log_loss_ci": [229 0.5155238809523972,230 0.5370475139043628231 ],232 "brier": 0.16632842942292825,233 "brier_ci": [234 0.16240351139704695,235 0.17012147700278982236 ],237 "auc": 0.5825639152877317,238 "auc_ci": [239 0.5740309209915405,240 0.5915368817008297241 ],242 "ece": 0.07425126606132011,243 "n_test": 28947,244 "base_rate": 0.19818979514284726,245 "n_boot": 200,246 "best_iteration": 210247 }248 },249 "per_region": {250 "Mont\u00e9r\u00e9gie": {251 "logistic": {252 "log_loss": 0.452183978916403,253 "brier": 0.14212819997670179,254 "auc": 0.6525672064915363,255 "n_test": 9137,256 "base_rate": 0.17883331509248113257 },258 "lightgbm": {259 "log_loss": 0.4694169326468643,260 "brier": 0.14619473862795498,261 "auc": 0.6475888224881406,262 "n_test": 9137,263 "base_rate": 0.17883331509248113264 }265 },266 "Capitale-Nationale": {267 "logistic": {268 "log_loss": 0.40564772383211145,269 "brier": 0.12194928702609958,270 "auc": 0.6289856984093128,271 "n_test": 5358,272 "base_rate": 0.1452034341172079273 },274 "lightgbm": {275 "log_loss": 0.42090224285450073,276 "brier": 0.12450967030785759,277 "auc": 0.5374883532964381,278 "n_test": 5358,279 "base_rate": 0.1452034341172079280 }281 },282 "Montr\u00e9al": {283 "logistic": {284 "log_loss": 0.5003961930923267,285 "brier": 0.1584830115644028,286 "auc": 0.513379869615553,287 "n_test": 4290,288 "base_rate": 0.18578088578088578289 },290 "lightgbm": {291 "log_loss": 0.5314319585060785,292 "brier": 0.16286581390154248,293 "auc": 0.4369829459959532,294 "n_test": 4290,295 "base_rate": 0.18578088578088578296 }297 },298 "Laurentides": {299 "logistic": {300 "log_loss": 0.5847515805129399,301 "brier": 0.19557273284142074,302 "auc": 0.5727015907932161,303 "n_test": 4033,304 "base_rate": 0.25861641457971735305 },306 "lightgbm": {307 "log_loss": 0.5980627041516525,308 "brier": 0.19829779375715464,309 "auc": 0.587789756202362,310 "n_test": 4033,311 "base_rate": 0.25861641457971735312 }313 },314 "Laval": {315 "logistic": {316 "log_loss": 0.6319761628571743,317 "brier": 0.21420536450691405,318 "auc": 0.7003383531999432,319 "n_test": 3309,320 "base_rate": 0.2901178603807797321 },322 "lightgbm": {323 "log_loss": 0.6356260942226282,324 "brier": 0.21162689959842576,325 "auc": 0.7173203136086279,326 "n_test": 3309,327 "base_rate": 0.2901178603807797328 }329 },330 "Chaudi\u00e8re-Appalaches": {331 "logistic": {332 "log_loss": 0.47472343159237923,333 "brier": 0.1486280720240991,334 "auc": 0.564392156862745,335 "n_test": 2820,336 "base_rate": 0.18617021276595744337 },338 "lightgbm": {339 "log_loss": 0.5016269078626335,340 "brier": 0.154475660344634,341 "auc": 0.5682133001348687,342 "n_test": 2820,343 "base_rate": 0.18617021276595744344 }345 }346 },347 "aggregation": "unweighted mean over the 6 held-out regions; CI = bootstrap within region, then averaged"348 },349 "drift": {350 "weeks_4_5": {351 "n_train": 24354,352 "n_test": 16259,353 "weeks_observable": [354 4,355 5356 ],357 "models": {358 "logistic": {359 "log_loss": 0.4173164063193048,360 "log_loss_ci": [361 0.4079098851335305,362 0.4279503479487398363 ],364 "brier": 0.13028631906067592,365 "brier_ci": [366 0.1269213130200807,367 0.13395893293138275368 ],369 "auc": 0.7492247835656936,370 "auc_ci": [371 0.7397275196136489,372 0.7587138500361528373 ],374 "ece": 0.02302921647331614,375 "n_test": 16259,376 "base_rate": 0.18420567070545543,377 "n_boot": 200378 },379 "lightgbm": {380 "log_loss": 0.39085408923589154,381 "log_loss_ci": [382 0.3806536103761275,383 0.4008621103469145384 ],385 "brier": 0.12244771285551145,386 "brier_ci": [387 0.11906715996781063,388 0.12593964124051688389 ],390 "auc": 0.7930861850571217,391 "auc_ci": [392 0.7832040018509864,393 0.8016148414656868394 ],395 "ece": 0.022256217822746907,396 "n_test": 16259,397 "base_rate": 0.18420567070545543,398 "n_boot": 200399 }400 }401 },402 "weeks_6_7": {403 "n_train": 24354,404 "n_test": 3390,405 "weeks_observable": [406 6407 ],408 "models": {409 "logistic": {410 "log_loss": 0.45746415296206383,411 "log_loss_ci": [412 0.4386145776645392,413 0.47866413632902705414 ],415 "brier": 0.14614020537123143,416 "brier_ci": [417 0.1388593954311711,418 0.1541542182372055419 ],420 "auc": 0.7333957973073855,421 "auc_ci": [422 0.7138865158474841,423 0.7512470827884635424 ],425 "ece": 0.030072903928325463,426 "n_test": 3390,427 "base_rate": 0.20914454277286135,428 "n_boot": 200429 },430 "lightgbm": {431 "log_loss": 0.44093076204043397,432 "log_loss_ci": [433 0.4221044052371359,434 0.46530112434948756435 ],436 "brier": 0.14186272625131685,437 "brier_ci": [438 0.1351799481247074,439 0.15016632049135298440 ],441 "auc": 0.7617605265912926,442 "auc_ci": [443 0.7412496672929431,444 0.7814590762040565445 ],446 "ece": 0.027534424095712027,447 "n_test": 3390,448 "base_rate": 0.20914454277286135,449 "n_boot": 200450 }451 }452 }453 },454 "chronological_oct1_naive": {455 "n_train": 49629,456 "n_test": 3317,457 "base_rate_train": 0.18525458905075662,458 "models": {459 "logistic": {460 "log_loss": 0.46477813693145725,461 "log_loss_ci": [462 0.4467499148680626,463 0.48575071640365114464 ],465 "brier": 0.1495821223634504,466 "brier_ci": [467 0.1425540124551429,468 0.15797526967050968469 ],470 "auc": 0.7381234059063356,471 "auc_ci": [472 0.7199324331863326,473 0.7578027950645022474 ],475 "ece": 0.028775694245302394,476 "n_test": 3317,477 "base_rate": 0.21977690684353332,478 "n_boot": 200479 },480 "lightgbm": {481 "log_loss": 0.44148274747445687,482 "log_loss_ci": [483 0.42094055498237765,484 0.46008129743042775485 ],486 "brier": 0.14211547935097907,487 "brier_ci": [488 0.135030975028993,489 0.14876458809296433490 ],491 "auc": 0.7739797800548274,492 "auc_ci": [493 0.7561496618054185,494 0.791151076318268495 ],496 "ece": 0.02495589959856237,497 "n_test": 3317,498 "base_rate": 0.21977690684353332,499 "n_boot": 200,500 "best_iteration": 107501 }502 }503 }504 },505 "markdown30cr": {506 "chronological": {507 "n_train": 14276,508 "n_test": 6063,509 "base_rate_train": 0.3918464555898011,510 "models": {511 "logistic": {512 "log_loss": 0.5638895964021118,513 "log_loss_ci": [514 0.5483626307956806,515 0.577927533586226516 ],517 "brier": 0.18903990112235444,518 "brier_ci": [519 0.18272317763360166,520 0.19476213095498135521 ],522 "auc": 0.7754598844468663,523 "auc_ci": [524 0.764022742728931,525 0.7874068050710149526 ],527 "ece": 0.03134502988231817,528 "n_test": 6063,529 "base_rate": 0.41184232228269835,530 "n_boot": 200531 },532 "lightgbm": {533 "log_loss": 0.5498899704305041,534 "log_loss_ci": [535 0.5339328498306773,536 0.5653421016847111537 ],538 "brier": 0.18445368542000976,539 "brier_ci": [540 0.17783996205881983,541 0.1907892839784524542 ],543 "auc": 0.7891945376515757,544 "auc_ci": [545 0.776838015085632,546 0.8006678073418573547 ],548 "ece": 0.046678520429866505,549 "n_test": 6063,550 "base_rate": 0.41184232228269835,551 "n_boot": 200,552 "best_iteration": 119553 }554 }555 },556 "dealer_holdout": {557 "n_train": 16271,558 "n_test": 20339,559 "base_rate_train": 0.39771721138450267,560 "models": {561 "logistic": {562 "log_loss": 0.7004408019712537,563 "log_loss_ci": [564 0.6927949528978081,565 0.7063050061476126566 ],567 "brier": 0.2510870643928481,568 "brier_ci": [569 0.24759670467434686,570 0.25374216367416264571 ],572 "auc": 0.5666387429533795,573 "auc_ci": [574 0.5584058545681978,575 0.5750732232194316576 ],577 "ece": 0.12062944203570568,578 "n_test": 20339,579 "base_rate": 0.39780716849402625,580 "n_boot": 200581 },582 "lightgbm": {583 "log_loss": 0.7215868383461882,584 "log_loss_ci": [585 0.7127664466090898,586 0.728208219422887587 ],588 "brier": 0.2588563766483554,589 "brier_ci": [590 0.25491963852221505,591 0.26171929106903613592 ],593 "auc": 0.547630097944503,594 "auc_ci": [595 0.539325096641866,596 0.5554351076584109597 ],598 "ece": 0.1426202792097802,599 "n_test": 20339,600 "base_rate": 0.39780716849402625,601 "n_boot": 200,602 "best_iteration": 93603 }604 },605 "folds": [606 {607 "logistic": {608 "log_loss": 0.6722112165488539,609 "brier": 0.2381702248050217,610 "auc": 0.5368242640737004,611 "n_test": 3605612 },613 "lightgbm": {614 "log_loss": 0.6688961859985117,615 "brier": 0.2365822445398945,616 "auc": 0.5497617006891449,617 "n_test": 3605618 }619 },620 {621 "logistic": {622 "log_loss": 0.6962910007523655,623 "brier": 0.25057353449799624,624 "auc": 0.6267437919223633,625 "n_test": 3713626 },627 "lightgbm": {628 "log_loss": 0.7734497207514515,629 "brier": 0.2796694183822182,630 "auc": 0.5578928511964226,631 "n_test": 3713632 }633 },634 {635 "logistic": {636 "log_loss": 0.6993752197836073,637 "brier": 0.25163414146456065,638 "auc": 0.5570526351627401,639 "n_test": 3554640 },641 "lightgbm": {642 "log_loss": 0.7134279994928003,643 "brier": 0.25641196704432834,644 "auc": 0.5570179344011565,645 "n_test": 3554646 }647 },648 {649 "logistic": {650 "log_loss": 0.6792352647423426,651 "brier": 0.2421105185365461,652 "auc": 0.6041964972387335,653 "n_test": 3068654 },655 "lightgbm": {656 "log_loss": 0.7026935489677363,657 "brier": 0.2516061494307656,658 "auc": 0.5922801132758284,659 "n_test": 3068660 }661 },662 {663 "logistic": {664 "log_loss": 0.7295112090171477,665 "brier": 0.26266194702888007,666 "auc": 0.545933196571475,667 "n_test": 6399668 },669 "lightgbm": {670 "log_loss": 0.7347676547926113,671 "brier": 0.2641619792627401,672 "auc": 0.5450476296868018,673 "n_test": 6399674 }675 }676 ],677 "fold_mean": {678 "logistic": {679 "log_loss": 0.6953247821688634,680 "brier": 0.24903007326660095,681 "auc": 0.5741500769938024682 },683 "lightgbm": {684 "log_loss": 0.7186470220006222,685 "brier": 0.25768635173198934,686 "auc": 0.5604000458498708687 }688 },689 "fold_sd": {690 "logistic": {691 "log_loss": 0.019895531611715905,692 "brier": 0.008494798082283636,693 "auc": 0.03507311616495651694 },695 "lightgbm": {696 "log_loss": 0.034699013740005225,697 "brier": 0.014206795754122764,698 "auc": 0.016630901790150834699 }700 }701 },702 "region_holdout": {703 "n_train": 18132,704 "n_test": 13239,705 "base_rate_train": 0.3935066693678861,706 "models": {707 "logistic": {708 "log_loss": 0.7279608067404245,709 "log_loss_ci": [710 0.7203742749774137,711 0.7368864583023297712 ],713 "brier": 0.2640763912276577,714 "brier_ci": [715 0.2608133543346487,716 0.2681929903062917717 ],718 "auc": 0.5688706878616209,719 "auc_ci": [720 0.5596895234505151,721 0.5784831830436126722 ],723 "ece": 0.1526930321622381,724 "n_test": 13239,725 "base_rate": 0.4333408867739255,726 "n_boot": 200727 },728 "lightgbm": {729 "log_loss": 0.7818190166598273,730 "log_loss_ci": [731 0.7724134411125672,732 0.7934154517619193733 ],734 "brier": 0.2832273181033232,735 "brier_ci": [736 0.2790347507462463,737 0.28801411460875653738 ],739 "auc": 0.5650308367917641,740 "auc_ci": [741 0.5565529569482552,742 0.5755462049661195743 ],744 "ece": 0.20579920615717145,745 "n_test": 13239,746 "base_rate": 0.4333408867739255,747 "n_boot": 200,748 "best_iteration": 114749 }750 },751 "per_region": {752 "Mont\u00e9r\u00e9gie": {753 "logistic": {754 "log_loss": 0.6977659234475921,755 "brier": 0.2508394634609104,756 "auc": 0.6278325094736001,757 "n_test": 3968,758 "base_rate": 0.4117943548387097759 },760 "lightgbm": {761 "log_loss": 0.7702040862365774,762 "brier": 0.27550500372094566,763 "auc": 0.6179970611648988,764 "n_test": 3968,765 "base_rate": 0.4117943548387097766 }767 },768 "Capitale-Nationale": {769 "logistic": {770 "log_loss": 0.6501193396178676,771 "brier": 0.2272922814678233,772 "auc": 0.5837421163279608,773 "n_test": 2205,774 "base_rate": 0.3528344671201814775 },776 "lightgbm": {777 "log_loss": 0.6672586285350506,778 "brier": 0.23457832139684207,779 "auc": 0.5989847830042353,780 "n_test": 2205,781 "base_rate": 0.3528344671201814782 }783 },784 "Montr\u00e9al": {785 "logistic": {786 "log_loss": 0.7891428397946212,787 "brier": 0.29345044090977257,788 "auc": 0.4827450884539969,789 "n_test": 1721,790 "base_rate": 0.463102847181871791 },792 "lightgbm": {793 "log_loss": 0.835099741329755,794 "brier": 0.30923354426516786,795 "auc": 0.4641960924896935,796 "n_test": 1721,797 "base_rate": 0.463102847181871798 }799 },800 "Laurentides": {801 "logistic": {802 "log_loss": 0.75136153101177,803 "brier": 0.27623483562522655,804 "auc": 0.5588103764284189,805 "n_test": 2175,806 "base_rate": 0.47954022988505746807 },808 "lightgbm": {809 "log_loss": 0.8836214775570168,810 "brier": 0.320908073081427,811 "auc": 0.5556342298818642,812 "n_test": 2175,813 "base_rate": 0.47954022988505746814 }815 },816 "Laval": {817 "logistic": {818 "log_loss": 0.8228116735297076,819 "brier": 0.3057353206640463,820 "auc": 0.6248701407506702,821 "n_test": 1706,822 "base_rate": 0.5627198124267292823 },824 "lightgbm": {825 "log_loss": 0.8304382506811459,826 "brier": 0.31035892011487787,827 "auc": 0.6258866733690795,828 "n_test": 1706,829 "base_rate": 0.5627198124267292830 }831 },832 "Chaudi\u00e8re-Appalaches": {833 "logistic": {834 "log_loss": 0.6565635330409889,835 "brier": 0.23090600523816723,836 "auc": 0.5352238957350778,837 "n_test": 1464,838 "base_rate": 0.35860655737704916839 },840 "lightgbm": {841 "log_loss": 0.7042919156194186,842 "brier": 0.2487800460406789,843 "auc": 0.5274861808408133,844 "n_test": 1464,845 "base_rate": 0.35860655737704916846 }847 }848 },849 "aggregation": "unweighted mean over the 6 held-out regions; CI = bootstrap within region, then averaged"850 },851 "drift": {852 "weeks_4_5": {853 "n_train": 11235,854 "n_test": 7443,855 "weeks_observable": [856 4,857 5858 ],859 "models": {860 "logistic": {861 "log_loss": 0.5567346950343365,862 "log_loss_ci": [863 0.5459619762943217,864 0.5711023503576953865 ],866 "brier": 0.1857267695506284,867 "brier_ci": [868 0.18136538792881438,869 0.1913884690976126870 ],871 "auc": 0.7775743595440843,872 "auc_ci": [873 0.7666601355569772,874 0.7873221685819515875 ],876 "ece": 0.023230589739597435,877 "n_test": 7443,878 "base_rate": 0.40239150880021496,879 "n_boot": 200880 },881 "lightgbm": {882 "log_loss": 0.540345996534645,883 "log_loss_ci": [884 0.5284937882754046,885 0.5527355994169363886 ],887 "brier": 0.18046266423549073,888 "brier_ci": [889 0.17568413706146446,890 0.18558142599518676891 ],892 "auc": 0.7916780890813226,893 "auc_ci": [894 0.7810892071064042,895 0.8016050571557458896 ],897 "ece": 0.030500070604068515,898 "n_test": 7443,899 "base_rate": 0.40239150880021496,900 "n_boot": 200901 }902 }903 },904 "weeks_6_7": {905 "n_train": 11235,906 "n_test": 1661,907 "weeks_observable": [908 6909 ],910 "models": {911 "logistic": {912 "log_loss": 0.5962059284240813,913 "log_loss_ci": [914 0.5699806837231501,915 0.6170598349569482916 ],917 "brier": 0.2047625817692497,918 "brier_ci": [919 0.1938543246185836,920 0.2135147288087788921 ],922 "auc": 0.7441330551966907,923 "auc_ci": [924 0.7227975088465454,925 0.7683478203641556926 ],927 "ece": 0.059025521446466354,928 "n_test": 1661,929 "base_rate": 0.42685129440096325,930 "n_boot": 200931 },932 "lightgbm": {933 "log_loss": 0.5908720086067301,934 "log_loss_ci": [935 0.5631193930386329,936 0.6150010738855946937 ],938 "brier": 0.2027885639936886,939 "brier_ci": [940 0.1905904507442143,941 0.21239676546081865942 ],943 "auc": 0.7534349776582001,944 "auc_ci": [945 0.7327823501519576,946 0.7781418314211761947 ],948 "ece": 0.06154124457206232,949 "n_test": 1661,950 "base_rate": 0.42685129440096325,951 "n_boot": 200952 }953 }954 }955 }956 },957 "exit30": {958 "chronological": {959 "n_train": 31050,960 "n_test": 12953,961 "base_rate_train": 0.5936876006441224,962 "models": {963 "logistic": {964 "log_loss": 0.6088466861373079,965 "log_loss_ci": [966 0.6031441325132438,967 0.6159819995416025968 ],969 "brier": 0.21178979444355045,970 "brier_ci": [971 0.20923553708383189,972 0.21471718872259998973 ],974 "auc": 0.7051544263760394,975 "auc_ci": [976 0.6967140688020557,977 0.712991299199323978 ],979 "ece": 0.011098029585027088,980 "n_test": 12953,981 "base_rate": 0.591523199258859,982 "n_boot": 200983 },984 "lightgbm": {985 "log_loss": 0.5846151980680317,986 "log_loss_ci": [987 0.5797723430156764,988 0.5913731168711103989 ],990 "brier": 0.20399341239740035,991 "brier_ci": [992 0.20185712890520965,993 0.20689501441673702994 ],995 "auc": 0.7274472971418939,996 "auc_ci": [997 0.7188675564699939,998 0.7353464481143868999 ],1000 "ece": 0.015319677918253553,1001 "n_test": 12953,1002 "base_rate": 0.591523199258859,1003 "n_boot": 200,1004 "best_iteration": 941005 }1006 }1007 },1008 "dealer_holdout": {1009 "n_train": 35202,1010 "n_test": 44003,1011 "base_rate_train": 0.5932610167639272,1012 "models": {1013 "logistic": {1014 "log_loss": 0.6950997863455771,1015 "log_loss_ci": [1016 0.6937704930116179,1017 0.69666213402739911018 ],1019 "brier": 0.25059085167120265,1020 "brier_ci": [1021 0.24996162086867182,1022 0.251320748616889241023 ],1024 "auc": 0.5665666202452024,1025 "auc_ci": [1026 0.5615437854107324,1027 0.57142861648289171028 ],1029 "ece": 0.1033253895507352,1030 "n_test": 44003,1031 "base_rate": 0.5930504738313297,1032 "n_boot": 2001033 },1034 "lightgbm": {1035 "log_loss": 0.6806059620538215,1036 "log_loss_ci": [1037 0.6776759680404983,1038 0.6832472144242051039 ],1040 "brier": 0.2435928870014518,1041 "brier_ci": [1042 0.2422043961349471,1043 0.24477263419627081044 ],1045 "auc": 0.6079275140203402,1046 "auc_ci": [1047 0.6033111847021673,1048 0.61365841764173421049 ],1050 "ece": 0.08997247704185676,1051 "n_test": 44003,1052 "base_rate": 0.5930504738313297,1053 "n_boot": 200,1054 "best_iteration": 4191055 }1056 },1057 "folds": [1058 {1059 "logistic": {1060 "log_loss": 0.7637357356049415,1061 "brier": 0.28398737448997663,1062 "auc": 0.5140435150820424,1063 "n_test": 94421064 },1065 "lightgbm": {1066 "log_loss": 0.7177019425885123,1067 "brier": 0.26042194193329476,1068 "auc": 0.5695193486374643,1069 "n_test": 94421070 }1071 },1072 {1073 "logistic": {1074 "log_loss": 0.6721068516854943,1075 "brier": 0.23950253821992448,1076 "auc": 0.6174046231041781,1077 "n_test": 73941078 },1079 "lightgbm": {1080 "log_loss": 0.6344290341443435,1081 "brier": 0.22266291838611618,1082 "auc": 0.677458225501697,1083 "n_test": 73941084 }1085 },1086 {1087 "logistic": {1088 "log_loss": 0.6715285377518726,1089 "brier": 0.23931232679543418,1090 "auc": 0.6130534721638573,1091 "n_test": 75481092 },1093 "lightgbm": {1094 "log_loss": 0.6775903836629404,1095 "brier": 0.2421437152997418,1096 "auc": 0.6231382450092808,1097 "n_test": 75481098 }1099 },1100 {1101 "logistic": {1102 "log_loss": 0.6761618503418108,1103 "brier": 0.24140912447270493,1104 "auc": 0.6045811871317802,1105 "n_test": 68121106 },1107 "lightgbm": {1108 "log_loss": 0.6823981214925687,1109 "brier": 0.2447753577894644,1110 "auc": 0.5783874876058118,1111 "n_test": 68121112 }1113 },1114 {1115 "logistic": {1116 "log_loss": 0.6817375727446296,1117 "brier": 0.2439017951116925,1118 "auc": 0.5984227059886065,1119 "n_test": 128071120 },1121 "lightgbm": {1122 "log_loss": 0.6807406815315877,1123 "brier": 0.24349447271800476,1124 "auc": 0.612751425914652,1125 "n_test": 128071126 }1127 }1128 ],1129 "fold_mean": {1130 "logistic": {1131 "log_loss": 0.6930541096257498,1132 "brier": 0.24962263181794656,1133 "auc": 0.58950110069409291134 },1135 "lightgbm": {1136 "log_loss": 0.6785720326839906,1137 "brier": 0.24269968122532443,1138 "auc": 0.61225094653378121139 }1140 },1141 "fold_sd": {1142 "logistic": {1143 "log_loss": 0.03552869978333845,1144 "brier": 0.017261772206438004,1145 "auc": 0.0382992742467505451146 },1147 "lightgbm": {1148 "log_loss": 0.0264577391226201,1149 "brier": 0.012006661605143302,1150 "auc": 0.038323612270879861151 }1152 }1153 },1154 "region_holdout": {1155 "n_train": 39178,1156 "n_test": 28947,1157 "base_rate_train": 0.5919150624764695,1158 "models": {1159 "logistic": {1160 "log_loss": 0.6895086811014098,1161 "log_loss_ci": [1162 0.6873192581802143,1163 0.6925412854139071164 ],1165 "brier": 0.2478891684784745,1166 "brier_ci": [1167 0.24687389591644682,1168 0.249335936311534721169 ],1170 "auc": 0.5843988559292631,1171 "auc_ci": [1172 0.5766574148498544,1173 0.59069972898824321174 ],1175 "ece": 0.11532407638989386,1176 "n_test": 28947,1177 "base_rate": 0.6009949217535496,1178 "n_boot": 2001179 },1180 "lightgbm": {1181 "log_loss": 0.6890728325227832,1182 "log_loss_ci": [1183 0.6844833481593796,1184 0.69242816257173681185 ],1186 "brier": 0.24725910342919488,1187 "brier_ci": [1188 0.24514076666320075,1189 0.24881711910906921190 ],1191 "auc": 0.6065048609213427,1192 "auc_ci": [1193 0.6001358647505153,1194 0.61357458650892861195 ],1196 "ece": 0.11224426208325944,1197 "n_test": 28947,1198 "base_rate": 0.6009949217535496,1199 "n_boot": 200,1200 "best_iteration": 5001201 }1202 },1203 "per_region": {1204 "Mont\u00e9r\u00e9gie": {1205 "logistic": {1206 "log_loss": 0.7185548741531423,1207 "brier": 0.2622670755352521,1208 "auc": 0.5507198070077464,1209 "n_test": 9137,1210 "base_rate": 0.61322096968370371211 },1212 "lightgbm": {1213 "log_loss": 0.6821032233112184,1214 "brier": 0.24412024729940518,1215 "auc": 0.6097070996710166,1216 "n_test": 9137,1217 "base_rate": 0.61322096968370371218 }1219 },1220 "Capitale-Nationale": {1221 "logistic": {1222 "log_loss": 0.6826432948911425,1223 "brier": 0.2443618673736587,1224 "auc": 0.6112885021555081,1225 "n_test": 5358,1226 "base_rate": 0.63624486748786861227 },1228 "lightgbm": {1229 "log_loss": 0.692498946330963,1230 "brier": 0.2488109699935438,1231 "auc": 0.6229425293653461,1232 "n_test": 5358,1233 "base_rate": 0.63624486748786861234 }1235 },1236 "Montr\u00e9al": {1237 "logistic": {1238 "log_loss": 0.6939403574317395,1239 "brier": 0.2502112164586107,1240 "auc": 0.5520071413269267,1241 "n_test": 4290,1242 "base_rate": 0.65081585081585081243 },1244 "lightgbm": {1245 "log_loss": 0.7407256390973135,1246 "brier": 0.2715057800974201,1247 "auc": 0.5115864849407614,1248 "n_test": 4290,1249 "base_rate": 0.65081585081585081250 }1251 },1252 "Laurentides": {1253 "logistic": {1254 "log_loss": 0.6836822032908833,1255 "brier": 0.24524372752759388,1256 "auc": 0.5790217054117824,1257 "n_test": 4033,1258 "base_rate": 0.53830895115298781259 },1260 "lightgbm": {1261 "log_loss": 0.6760231711527256,1262 "brier": 0.24097788216521682,1263 "auc": 0.6283442369165659,1264 "n_test": 4033,1265 "base_rate": 0.53830895115298781266 }1267 },1268 "Laval": {1269 "logistic": {1270 "log_loss": 0.6679218283231959,1271 "brier": 0.23761150726507252,1272 "auc": 0.6305263465949894,1273 "n_test": 3309,1274 "base_rate": 0.58114233907524941275 },1276 "lightgbm": {1277 "log_loss": 0.6602450076615012,1278 "brier": 0.23403708273939391,1279 "auc": 0.6507309931646904,1280 "n_test": 3309,1281 "base_rate": 0.58114233907524941282 }1283 },1284 "Chaudi\u00e8re-Appalaches": {1285 "logistic": {1286 "log_loss": 0.6903095285183557,1287 "brier": 0.24763961671065904,1288 "auc": 0.5828296330786258,1289 "n_test": 2820,1290 "base_rate": 0.53156028368794331291 },1292 "lightgbm": {1293 "log_loss": 0.6828410075829774,1294 "brier": 0.2441026582801894,1295 "auc": 0.6157178214696752,1296 "n_test": 2820,1297 "base_rate": 0.53156028368794331298 }1299 }1300 },1301 "aggregation": "unweighted mean over the 6 held-out regions; CI = bootstrap within region, then averaged"1302 },1303 "drift": {1304 "weeks_4_5": {1305 "n_train": 24354,1306 "n_test": 16259,1307 "weeks_observable": [1308 4,1309 51310 ],1311 "models": {1312 "logistic": {1313 "log_loss": 0.618162464970206,1314 "log_loss_ci": [1315 0.6123170701522876,1316 0.62488518433617141317 ],1318 "brier": 0.21437804280362754,1319 "brier_ci": [1320 0.21201373045830557,1321 0.21730361454276391322 ],1323 "auc": 0.6959144125378385,1324 "auc_ci": [1325 0.687254674723287,1326 0.70289427453808951327 ],1328 "ece": 0.020602736684108267,1329 "n_test": 16259,1330 "base_rate": 0.5975767267359616,1331 "n_boot": 2001332 },1333 "lightgbm": {1334 "log_loss": 0.5954846295525256,1335 "log_loss_ci": [1336 0.5882372598982487,1337 0.603742427298791338 ],1339 "brier": 0.20552943836890286,1340 "brier_ci": [1341 0.20264678625904206,1342 0.208680246845170031343 ],1344 "auc": 0.7275086552544345,1345 "auc_ci": [1346 0.720227611541182,1347 0.73429701607433881348 ],1349 "ece": 0.03206067711276024,1350 "n_test": 16259,1351 "base_rate": 0.5975767267359616,1352 "n_boot": 2001353 }1354 }1355 },1356 "weeks_6_7": {1357 "n_train": 24354,1358 "n_test": 3390,1359 "weeks_observable": [1360 61361 ],1362 "models": {1363 "logistic": {1364 "log_loss": 0.6335406964925955,1365 "log_loss_ci": [1366 0.6212458123830099,1367 0.64507653834882111368 ],1369 "brier": 0.2219334974369145,1370 "brier_ci": [1371 0.21648779673683372,1372 0.226849948865309571373 ],1374 "auc": 0.674931342556254,1375 "auc_ci": [1376 0.6582693643726644,1377 0.69103792760718711378 ],1379 "ece": 0.022569791027503453,1380 "n_test": 3390,1381 "base_rate": 0.58023598820059,1382 "n_boot": 2001383 },1384 "lightgbm": {1385 "log_loss": 0.6185171938975021,1386 "log_loss_ci": [1387 0.6036703816435322,1388 0.63300472228419041389 ],1390 "brier": 0.21666892021735712,1391 "brier_ci": [1392 0.2107951018556213,1393 0.223166746784112971394 ],1395 "auc": 0.6953947798549575,1396 "auc_ci": [1397 0.6785078625186961,1398 0.71344337221398461399 ],1400 "ece": 0.02787863043036501,1401 "n_test": 3390,1402 "base_rate": 0.58023598820059,1403 "n_boot": 2001404 }1405 }1406 }1407 },1408 "chronological_oct1_naive": {1409 "n_train": 49629,1410 "n_test": 3317,1411 "base_rate_train": 0.5878014870337908,1412 "models": {1413 "logistic": {1414 "log_loss": 0.6322114387021451,1415 "log_loss_ci": [1416 0.6197245122439214,1417 0.64472691818427531418 ],1419 "brier": 0.22084303889969914,1420 "brier_ci": [1421 0.21527341133434813,1422 0.22632630651337491423 ],1424 "auc": 0.7127074429684979,1425 "auc_ci": [1426 0.6945860057032724,1427 0.73328956639499471428 ],1429 "ece": 0.08236096055258645,1430 "n_test": 3317,1431 "base_rate": 0.4615616520952668,1432 "n_boot": 2001433 },1434 "lightgbm": {1435 "log_loss": 0.629292096351042,1436 "log_loss_ci": [1437 0.6157108059959672,1438 0.64227840209256221439 ],1440 "brier": 0.22257599524033,1441 "brier_ci": [1442 0.21679288730122043,1443 0.22783562905879681444 ],1445 "auc": 0.728343608719535,1446 "auc_ci": [1447 0.7147762501849771,1448 0.74325687383753311449 ],1450 "ece": 0.12296025739657858,1451 "n_test": 3317,1452 "base_rate": 0.4615616520952668,1453 "n_boot": 200,1454 "best_iteration": 4161455 }1456 }1457 }1458 }1459 },1460 "notes": {1461 "region_holdout_regions": [1462 "Mont\u00e9r\u00e9gie",1463 "Capitale-Nationale",1464 "Montr\u00e9al",1465 "Laurentides",1466 "Laval",1467 "Chaudi\u00e8re-Appalaches"1468 ],1469 "drift": "train = entries in weeks 0-3 (days 0-27); the week 6-7 window only contains observable entries up to day 44 (week 6), week 7 has no determinable 30-day outcome",1470 "chronological_oct1_naive": "Analysis-plan cutoff (entry before vs on/after 2026-10-01 UTC) applied to the looser rule entry day <= 51. Test entries (days 50-51) have only ~22-24 days of confirmable follow-up: exits at ages between ~23 and 30 days are coded 0 (exit30 base rate is a lower bound) and the last-day markdowns may be unclassified. Reported as a sensitivity only.",1471 "chronological": "primary chronological split: train entries before 2026-09-15 UTC, test 2026-09-15 to day 44 (2026-09-25); the plan's 2026-10-01 cutoff leaves no listing with a determinable 30-day outcome in the test fold",1472 "targets": {1473 "markdown30": "first sustained markdown (>=1 % and >=100 $) at listing age <= 30 d; exits before any cut count as 0",1474 "markdown30cr": "competing-risk-aware: listings that exit within 30 d without a prior cut are removed; event = markdown within 30 d among the rest",1475 "exit30": "disappearance confirmed by >= 7 d of absence at listing age <= 30 d (NOT a sale)"1476 },1477 "features": {1478 "numeric": [1479 "age_years_entry",1480 "km10k",1481 "year",1482 "logp",1483 "resid_A2",1484 "week",1485 "log_dens_10",1486 "log_dens_25",1487 "log_dens_50",1488 "log_own_stock",1489 "pricepos_25"1490 ],1491 "dealer_target_encoding": [1492 "dealer_rate_markdown30",1493 "dealer_rate_exit30",1494 "dealer_log_n_train",1495 "(training fold only; inner 5-fold OOF for training rows; smoothing m=20)"1496 ],1497 "categorical_lightgbm": [1498 "powertrain",1499 "body",1500 "region",1501 "source_class",1502 "make",1503 "model",1504 "trim",1505 "transmission",1506 "drivetrain"1507 ],1508 "one_hot_logistic": [1509 "powertrain",1510 "body",1511 "region",1512 "source_class"1513 ],1514 "competition_at_entry": "risk_set row with listing_age = 0 (dens_10/25/50, own_stock, pricepos_25; NaN when no competitor or no coordinates)",1515 "excluded": "dealer_id (identity), anything measured after entry"1516 },1517 "models": {1518 "logistic": "sklearn LogisticRegression L2, C=1, standardised numerics + missing indicators + one-hot coarse categoricals",1519 "lightgbm": {1520 "n_estimators": 2000,1521 "learning_rate": 0.05,1522 "num_leaves": 31,1523 "min_child_samples": 50,1524 "subsample": 0.8,1525 "subsample_freq": 1,1526 "colsample_bytree": 0.8,1527 "reg_lambda": 1.0,1528 "cat_l2": 10.0,1529 "cat_smooth": 50.0,1530 "max_cat_to_onehot": 4,1531 "n_jobs": 8,1532 "verbose": -1,1533 "random_state": 36,1534 "early_stopping": "100 rounds on inner validation (latest 15 % of training entries for chronological splits, 15 % of training dealers otherwise)"1535 }1536 },1537 "metrics": {1538 "ci": "200 bootstrap resamples of test rows, percentile 2.5/97.5",1539 "calibration": "10 equal-width probability bins; ECE = n-weighted |frac_pos - mean_pred|"1540 }1541 },1542 "samples": {1543 "n_S2": 80856,1544 "n_strict_observable": 44003,1545 "strict_rule": "entry_ts <= SNAPSHOT_TS - 30 - 7 days (entry day <= 44); every listing has a determined 30-day exit and markdown outcome",1546 "n_naive_observable": 52946,1547 "naive_rule": "entry day <= 51 (analysis-plan rule; exit at ages > snapshot-7d-entry cannot be confirmed)",1548 "n_without_coordinates": 573,1549 "base_rates_strict": {1550 "markdown30": 0.18387382678453743,1551 "markdown30cr": 0.39780716849402625,1552 "exit30": 0.59305047383132971553 },1554 "n_markdown30cr": 20339,1555 "n_exit_before_cut_removed_in_cr": 236641556 },1557 "survival": {1558 "model": "CoxPHFitter(penalizer=0.01)",1559 "split": "chronological, entry before vs on/after 2026-10-01 UTC",1560 "n_train": 49629,1561 "events_train": 34694,1562 "n_test": 31227,1563 "events_test": 9040,1564 "c_index_train": 0.6001305839001478,1565 "c_index_test": 0.6581044283827092,1566 "dynamic_auc_30d": null,1567 "dynamic_auc_note": "scikit-survival (sksurv) is not installed in the project environment; the 30-day cumulative/dynamic AUC was not computed.",1568 "covariates": [1569 "age_years_entry",1570 "logp",1571 "resid_A2",1572 "km10k",1573 "log_dens_25",1574 "pricepos_25f",1575 "pricepos_missing",1576 "coords_missing",1577 "powertrain=BEV",1578 "powertrain=Diesel",1579 "powertrain=HEV",1580 "powertrain=PHEV",1581 "powertrain=unknown",1582 "body=Convertible",1583 "body=Coupe",1584 "body=Hatchback",1585 "body=Pickup",1586 "body=Sedan",1587 "body=Van",1588 "body=Wagon",1589 "body=unknown",1590 "region=(missing)",1591 "region=Abitibi-T\u00e9miscamingue",1592 "region=Bas-Saint-Laurent",1593 "region=Capitale-Nationale",1594 "region=Centre-du-Qu\u00e9bec",1595 "region=Chaudi\u00e8re-Appalaches",1596 "region=C\u00f4te-Nord",1597 "region=Estrie",1598 "region=Gasp\u00e9sie\u2013\u00celes-de-la-Madeleine",1599 "region=Lanaudi\u00e8re",1600 "region=Laurentides",1601 "region=Laval",1602 "region=Mauricie",1603 "region=Mont\u00e9r\u00e9gie",1604 "region=Outaouais",1605 "region=Saguenay\u2013Lac-Saint-Jean"1606 ],1607 "seconds": 2.11608 },1609 "seconds_total": 133.31610}