spb/wpai0002_uqo

1{2  "created_at": "2026-11-09T17:57:10Z",3  "git": {4    "commit": "c27a8dad9b8875ec7e0b6eb6a79001b306b96ca4",5    "branch": "main",6    "dirty": true7  },8  "libraries": {9    "python": "3.14.4",10    "torch": "2.12.0",11    "numpy": "2.5.4",12    "anthropic": "2.3.0",13    "transformers": "5.8.1",14    "sentencepiece": "0.2.2",15    "lxml": "6.1.3",16    "pyarrow": "24.0.0",17    "sacrebleu": "2.6.0",18    "datasketch": "2.0.0",19    "rapidfuzz": "3.14.6"20  },21  "hardware": {22    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",23    "machine": "arm64",24    "cpu_count": 18,25    "chip": "Apple M5 Max",26    "ram_gb": 48.0,27    "torch_mps": true,28    "torch_cuda": false29  },30  "script": "discover_arxiv",31  "args": {32    "sets": [33      "q-fin",34      "econ"35    ],36    "date_from": "2008-01-01",37    "date_until": "2017-12-31",38    "datestamp_until": null,39    "max_docs": 50,40    "seed": 1,41    "out": "data/raw/arxiv",42    "codes": "CC0,CC-BY,CC-BY-SA,CC-BY-NC,CC-BY-NC-SA",43    "metadata_only": true,44    "metadata_out": null,45    "report": null,46    "max_minutes": 25.047  },48  "stats": {49    "records_seen": 25499,50    "duplicates": 0,51    "license_excluded": 18486,52    "no_versions": 0,53    "before_window": 2,54    "not_pre2018": 6849,55    "eligible": 162,56    "pdf_only": 0,57    "fetched": 0,58    "errors": 0,59    "by_license": {60      "UNKNOWN": 1416,61      "ARXIV-NONEXCLUSIVE": 15567,62      "CC-BY-NC-SA": 804,63      "CC-BY": 5676,64      "CC0": 220,65      "CC-BY-SA": 313,66      "CC-BY-NC-ND": 150367    },68    "by_set": {69      "q-fin": 16270    },71    "by_v1_year_eligible": {72      "2008": 2,73      "2009": 7,74      "2010": 17,75      "2011": 5,76      "2012": 8,77      "2013": 21,78      "2014": 27,79      "2015": 17,80      "2016": 30,81      "2017": 2882    },83    "by_license_eligible": {84      "CC-BY-NC-SA": 55,85      "CC-BY": 96,86      "CC0": 9,87      "CC-BY-SA": 288    },89    "windows_harvested": [90      {91        "set": "q-fin",92        "from": "2008-01-01",93        "until": "2008-12-31",94        "pages_total": 1,95        "records_seen_total": 94696      },97      {98        "set": "q-fin",99        "from": "2009-01-01",100        "until": "2009-12-31",101        "pages_total": 2,102        "records_seen_total": 1725103      },104      {105        "set": "q-fin",106        "from": "2010-01-01",107        "until": "2010-12-31",108        "pages_total": 3,109        "records_seen_total": 2129110      },111      {112        "set": "q-fin",113        "from": "2011-01-01",114        "until": "2011-12-31",115        "pages_total": 4,116        "records_seen_total": 2543117      },118      {119        "set": "q-fin",120        "from": "2012-01-01",121        "until": "2012-12-31",122        "pages_total": 5,123        "records_seen_total": 3011124      },125      {126        "set": "q-fin",127        "from": "2013-01-01",128        "until": "2013-12-31",129        "pages_total": 6,130        "records_seen_total": 3629131      },132      {133        "set": "q-fin",134        "from": "2014-01-01",135        "until": "2014-12-31",136        "pages_total": 7,137        "records_seen_total": 4302138      },139      {140        "set": "q-fin",141        "from": "2015-01-01",142        "until": "2015-12-31",143        "pages_total": 8,144        "records_seen_total": 5265145      },146      {147        "set": "q-fin",148        "from": "2016-01-01",149        "until": "2016-12-31",150        "pages_total": 9,151        "records_seen_total": 6062152      },153      {154        "set": "q-fin",155        "from": "2017-01-01",156        "until": "2017-12-31",157        "pages_total": 10,158        "records_seen_total": 6723159      },160      {161        "set": "q-fin",162        "from": "2018-01-01",163        "until": "2018-12-31",164        "pages_total": 11,165        "records_seen_total": 7644166      },167      {168        "set": "q-fin",169        "from": "2019-01-01",170        "until": "2019-12-31",171        "pages_total": 12,172        "records_seen_total": 8805173      },174      {175        "set": "q-fin",176        "from": "2020-01-01",177        "until": "2020-12-31",178        "pages_total": 14,179        "records_seen_total": 10476180      },181      {182        "set": "q-fin",183        "from": "2021-01-01",184        "until": "2021-12-31",185        "pages_total": 16,186        "records_seen_total": 12461187      },188      {189        "set": "q-fin",190        "from": "2022-01-01",191        "until": "2022-12-31",192        "pages_total": 18,193        "records_seen_total": 14229194      },195      {196        "set": "q-fin",197        "from": "2023-01-01",198        "until": "2023-12-31",199        "pages_total": 20,200        "records_seen_total": 16201201      },202      {203        "set": "q-fin",204        "from": "2024-01-01",205        "until": "2024-12-31",206        "pages_total": 22,207        "records_seen_total": 18596208      },209      {210        "set": "q-fin",211        "from": "2025-01-01",212        "until": "2025-12-31",213        "pages_total": 25,214        "records_seen_total": 21599215      }216    ],217    "truncated_at": {218      "set": "q-fin",219      "window": [220        "2026-01-01",221        "2026-11-09"222      ],223      "last_datestamp": "2026-09-15",224      "sets_not_started": [225        "econ"226      ]227    },228    "metadata_out": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/ia/wp-ai-0002/data/manifests/arxiv_eligible_q-fin-econ_2008-2017.jsonl",229    "datestamp_window": [230      "2008-01-01",231      "2026-11-09"232    ],233    "oai_endpoint": "https://oaipmh.arxiv.org/oai",234    "oai_pages": 29,235    "http": {236      "requests": 30,237      "retries": 0,238      "bytes": 64118631239    },240    "elapsed_s": 1506.1241  }242}243