Repository Public Research preview
1{2 "max_seq_length": 256,3 "content_tokens": 254,4 "model_max_length_tokenizer": 512,5 "pipeline_override": "none: src/embeddings/models.py calls SentenceTransformer(model_name, device) without setting max_seq_length; config/embeddings.yaml lists max_tokens: 256 for all-MiniLM-L6-v2 (informational only)",6 "chunker": "512 words, overlap 64, min 50 (src/parsing/chunker.py)",7 "seed": 20261105,8 "frames": {9 "K": {10 "n": 10000,11 "n_full_512w": 9759,12 "tokens_median": 634.0,13 "tokens_mean": 643.4061,14 "tokens_p05": 587.0,15 "tokens_p95": 736.0,16 "tokens_per_word_median": 1.240234375,17 "share_truncated": 0.9976,18 "share_truncated_full_512w": 1.0,19 "share_of_token_mass_dropped": 0.6023935738253026,20 "mean_share_tokens_dropped_if_truncated": 0.5992429518937146,21 "mean_share_chars_dropped": 0.6016807732572541,22 "words_in_window_median": 205.0,23 "share_with_ai_hit": 1.0,24 "hits_total": 40522,25 "hits_beyond_window": 24800,26 "share_hits_beyond_window": 0.6120132273826563,27 "share_hit_chunks_all_hits_beyond": 0.5081,28 "hit_class_counts": {29 "beyond_only": 5081,30 "inside_only": 3019,31 "both": 190032 },33 "mean_score_adopt_by_hit_class": {34 "beyond_only": 0.17960000038146973,35 "both": 0.4318999946117401,36 "inside_only": 0.3436000049114227337 },38 "mean_score_infra_by_hit_class": {39 "beyond_only": 0.13860000669956207,40 "both": 0.3650999963283539,41 "inside_only": 0.2829000055789947542 },43 "median_score_adopt_by_hit_class": {44 "beyond_only": 0.1745000034570694,45 "both": 0.45190000534057617,46 "inside_only": 0.343499988317489647 },48 "p90_score_adopt_by_hit_class": {49 "beyond_only": 0.3004,50 "both": 0.5853,51 "inside_only": 0.491852 }53 },54 "R": {55 "n": 10000,56 "n_full_512w": 8577,57 "tokens_median": 661.0,58 "tokens_mean": 662.2819,59 "tokens_p05": 250.0,60 "tokens_p95": 914.0,61 "tokens_per_word_median": 1.316481139174246,62 "share_truncated": 0.9481,63 "share_truncated_full_512w": 0.999883409117407,64 "share_of_token_mass_dropped": 0.6223162674383823,65 "mean_share_tokens_dropped_if_truncated": 0.612973350385886,66 "mean_share_chars_dropped": 0.5837520740841418,67 "words_in_window_median": 195.0,68 "share_with_ai_hit": 0.0074,69 "hits_total": 280,70 "hits_beyond_window": 187,71 "share_hits_beyond_window": 0.6678571428571428,72 "share_hit_chunks_all_hits_beyond": 0.4594594594594595,73 "hit_class_counts": {74 "none": 9926,75 "beyond_only": 34,76 "inside_only": 21,77 "both": 1978 },79 "mean_score_adopt_by_hit_class": {80 "beyond_only": 0.17489999532699585,81 "both": 0.392300009727478,82 "inside_only": 0.3077999949455261,83 "none": 0.0833000019192695684 },85 "mean_score_infra_by_hit_class": {86 "beyond_only": 0.13830000162124634,87 "both": 0.33219999074935913,88 "inside_only": 0.24070000648498535,89 "none": 0.0628999993205070590 },91 "median_score_adopt_by_hit_class": {92 "beyond_only": 0.17569999396800995,93 "both": 0.4325999915599823,94 "inside_only": 0.31459999084472656,95 "none": 0.0759000033140182596 },97 "p90_score_adopt_by_hit_class": {98 "beyond_only": 0.256,99 "both": 0.551,100 "inside_only": 0.4242,101 "none": 0.1767102 }103 },104 "T_adopt": {105 "n": 5000,106 "n_full_512w": 4880,107 "tokens_median": 620.0,108 "tokens_mean": 622.9938,109 "tokens_p05": 585.0,110 "tokens_p95": 689.0,111 "tokens_per_word_median": 1.212890625,112 "share_truncated": 0.996,113 "share_truncated_full_512w": 1.0,114 "share_of_token_mass_dropped": 0.5896485647208688,115 "mean_share_tokens_dropped_if_truncated": 0.5878734312579401,116 "mean_share_chars_dropped": 0.5850239008305681,117 "words_in_window_median": 210.0,118 "share_with_ai_hit": 0.9792,119 "hits_total": 59264,120 "hits_beyond_window": 25801,121 "share_hits_beyond_window": 0.4353570464362851,122 "share_hit_chunks_all_hits_beyond": 0.00428921568627451,123 "hit_class_counts": {124 "both": 3461,125 "inside_only": 1414,126 "none": 104,127 "beyond_only": 21128 },129 "mean_score_adopt_by_hit_class": {130 "beyond_only": 0.5184999704360962,131 "both": 0.5640000104904175,132 "inside_only": 0.5449000000953674,133 "none": 0.5318999886512756134 },135 "mean_score_infra_by_hit_class": {136 "beyond_only": 0.40389999747276306,137 "both": 0.4657000005245209,138 "inside_only": 0.4456999897956848,139 "none": 0.4147000014781952140 },141 "median_score_adopt_by_hit_class": {142 "beyond_only": 0.517799973487854,143 "both": 0.5547999739646912,144 "inside_only": 0.5371999740600586,145 "none": 0.5257999897003174146 },147 "p90_score_adopt_by_hit_class": {148 "beyond_only": 0.527,149 "both": 0.6279,150 "inside_only": 0.5896,151 "none": 0.5575152 },153 "share_top_chunks_truncated": 0.996,154 "share_top_chunks_short_lt_256_tokens": 0.004,155 "words_median": 512.0156 },157 "T_infra": {158 "n": 5000,159 "n_full_512w": 4903,160 "tokens_median": 626.0,161 "tokens_mean": 635.5158,162 "tokens_p05": 588.0,163 "tokens_p95": 722.0,164 "tokens_per_word_median": 1.22265625,165 "share_truncated": 0.9972,166 "share_truncated_full_512w": 1.0,167 "share_of_token_mass_dropped": 0.5975432868860223,168 "mean_share_tokens_dropped_if_truncated": 0.5952051251748295,169 "mean_share_chars_dropped": 0.5927375936333942,170 "words_in_window_median": 208.0,171 "share_with_ai_hit": 0.7938,172 "hits_total": 40443,173 "hits_beyond_window": 17339,174 "share_hits_beyond_window": 0.4287268501347576,175 "share_hit_chunks_all_hits_beyond": 0.02091206853111615,176 "hit_class_counts": {177 "both": 2581,178 "inside_only": 1305,179 "none": 1031,180 "beyond_only": 83181 },182 "mean_score_adopt_by_hit_class": {183 "beyond_only": 0.38179999589920044,184 "both": 0.5375999808311462,185 "inside_only": 0.48260000348091125,186 "none": 0.3538999855518341187 },188 "mean_score_infra_by_hit_class": {189 "beyond_only": 0.48510000109672546,190 "both": 0.5097000002861023,191 "inside_only": 0.5038999915122986,192 "none": 0.48730000853538513193 },194 "median_score_adopt_by_hit_class": {195 "beyond_only": 0.3788999915122986,196 "both": 0.5422000288963318,197 "inside_only": 0.4846999943256378,198 "none": 0.34549999237060547199 },200 "p90_score_adopt_by_hit_class": {201 "beyond_only": 0.4424,202 "both": 0.6339,203 "inside_only": 0.5797,204 "none": 0.434205 },206 "share_top_chunks_truncated": 0.9972,207 "share_top_chunks_short_lt_256_tokens": 0.0028,208 "words_median": 512.0209 }210 },211 "random_by_form": {212 "10-K": {213 "n": 4210,214 "tokens_median": 650.0,215 "share_truncated": 0.965083135391924,216 "share_token_mass_dropped": 0.6198667529840062217 },218 "10-Q": {219 "n": 5053,220 "tokens_median": 669.0,221 "share_truncated": 0.9477538096180487,222 "share_token_mass_dropped": 0.6264975535097554223 },224 "8-K": {225 "n": 737,226 "tokens_median": 672.0,227 "share_truncated": 0.8534599728629579,228 "share_token_mass_dropped": 0.6063548469784571229 }230 },231 "random_by_section": {232 "business": {233 "n": 457,234 "tokens_median": 640.0,235 "share_truncated": 0.973741794310722236 },237 "changes_in_accounting": {238 "n": 223,239 "tokens_median": 689.0,240 "share_truncated": 0.9820627802690582241 },242 "controls_and_procedures": {243 "n": 212,244 "tokens_median": 517.0,245 "share_truncated": 0.7830188679245284246 },247 "financial_statements": {248 "n": 3168,249 "tokens_median": 689.0,250 "share_truncated": 0.9892676767676768251 },252 "full_body": {253 "n": 737,254 "tokens_median": 672.0,255 "share_truncated": 0.8534599728629579256 },257 "full_document": {258 "n": 57,259 "tokens_median": 721.0,260 "share_truncated": 1.0261 },262 "legal_proceedings": {263 "n": 135,264 "tokens_median": 178.0,265 "share_truncated": 0.37037037037037035266 },267 "market_for_equity": {268 "n": 59,269 "tokens_median": 635.0,270 "share_truncated": 0.8305084745762712271 },272 "md_and_a": {273 "n": 2534,274 "tokens_median": 675.0,275 "share_truncated": 0.978689818468824276 },277 "mine_safety": {278 "n": 15,279 "tokens_median": 638.0,280 "share_truncated": 0.7333333333333333281 },282 "other_information": {283 "n": 489,284 "tokens_median": 703.0,285 "share_truncated": 0.9815950920245399286 },287 "properties": {288 "n": 57,289 "tokens_median": 623.0,290 "share_truncated": 0.8070175438596491291 },292 "quantitative_disclosures": {293 "n": 262,294 "tokens_median": 611.5,295 "share_truncated": 0.7099236641221374296 },297 "risk_factors": {298 "n": 1328,299 "tokens_median": 611.0,300 "share_truncated": 0.9653614457831325301 },302 "selected_financial_data": {303 "n": 66,304 "tokens_median": 709.0,305 "share_truncated": 0.9848484848484849306 },307 "unregistered_sales": {308 "n": 194,309 "tokens_median": 691.5,310 "share_truncated": 0.9175257731958762311 },312 "unresolved_staff_comments": {313 "n": 7,314 "tokens_median": 622.0,315 "share_truncated": 0.5714285714285714316 }317 },318 "random_tokens_hist": [319 94,320 117,321 158,322 146,323 129,324 97,325 101,326 118,327 2921,328 4315,329 1596,330 208331 ],332 "random_tokens_hist_bins": [333 0,334 64,335 128,336 192,337 256,338 320,339 384,340 448,341 512,342 640,343 768,344 1024,345 "inf"346 ],347 "K_first_hit_beyond_window_share": 0.5081,348 "K_first_hit_token_quantiles": {349 "0.1": 27.0,350 "0.25": 103.0,351 "0.5": 260.0,352 "0.75": 437.0,353 "0.9": 561.0354 },355 "pooled_RK_spearman_adopt_vs_hits_inside": 0.6829415413008677,356 "pooled_RK_spearman_adopt_vs_hits_beyond": 0.37790264254319395,357 "pooled_RK_1to3hits_mean_adopt_inside_only": 0.32244035601615906,358 "pooled_RK_1to3hits_mean_adopt_beyond_only": 0.17989477515220642,359 "pooled_RK_nohit_mean_adopt": 0.083250992000103,360 "pooled_RK_1to3hits_n": {361 "beyond_only": 4086,362 "inside_only": 2573,363 "both": 350364 }365}