Repository Public Research preview
1{2 "created_at": "2026-11-09T18:18:41Z",3 "git": {4 "commit": "e5b9872e8ec134616aca832cf77f5ed5f73008af",5 "branch": "main",6 "dirty": false7 },8 "libraries": {9 "python": "3.14.4",10 "torch": "2.12.0",11 "numpy": "2.5.4",12 "anthropic": "2.3.0",13 "transformers": "5.8.1",14 "sentencepiece": "0.2.2",15 "lxml": "6.1.3",16 "pyarrow": "24.0.0",17 "sacrebleu": "2.6.0",18 "datasketch": "2.0.0",19 "rapidfuzz": "3.14.6"20 },21 "hardware": {22 "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",23 "machine": "arm64",24 "cpu_count": 18,25 "chip": "Apple M5 Max",26 "ram_gb": 48.0,27 "torch_mps": true,28 "torch_cuda": false29 },30 "script": "discover_arxiv",31 "args": {32 "sets": [33 "q-fin",34 "econ"35 ],36 "date_from": "2008-01-01",37 "date_until": "2017-12-31",38 "datestamp_until": null,39 "max_docs": 170,40 "seed": 2,41 "out": "data/raw/arxiv",42 "codes": "CC0,CC-BY,CC-BY-SA,CC-BY-NC,CC-BY-NC-SA",43 "metadata_only": false,44 "metadata_out": null,45 "report": null,46 "max_minutes": 90.047 },48 "stats": {49 "records_seen": 6797,50 "duplicates": 2,51 "license_excluded": 6672,52 "no_versions": 0,53 "before_window": 0,54 "not_pre2018": 0,55 "eligible": 123,56 "pdf_only": 28,57 "fetched": 92,58 "errors": 3,59 "by_license": {60 "ARXIV-NONEXCLUSIVE": 5274,61 "CC-BY-NC-SA": 45,62 "UNKNOWN": 1398,63 "CC-BY": 71,64 "CC0": 6,65 "CC-BY-SA": 166 },67 "by_set": {68 "q-fin": 121,69 "econ": 270 },71 "by_v1_year_eligible": {72 "2012": 8,73 "2013": 20,74 "2011": 5,75 "2017": 16,76 "2016": 21,77 "2010": 15,78 "2014": 20,79 "2015": 9,80 "2009": 7,81 "2008": 282 },83 "by_license_eligible": {84 "CC-BY-NC-SA": 45,85 "CC-BY": 71,86 "CC0": 6,87 "CC-BY-SA": 188 },89 "windows_harvested": [90 {91 "set": "q-fin",92 "from": "2013-01-01",93 "until": "2013-12-31",94 "pages_total": 1,95 "records_seen_total": 61896 },97 {98 "set": "q-fin",99 "from": "2017-01-01",100 "until": "2017-12-31",101 "pages_total": 2,102 "records_seen_total": 1279103 },104 {105 "set": "q-fin",106 "from": "2011-01-01",107 "until": "2011-12-31",108 "pages_total": 3,109 "records_seen_total": 1693110 },111 {112 "set": "q-fin",113 "from": "2012-01-01",114 "until": "2012-12-31",115 "pages_total": 4,116 "records_seen_total": 2161117 },118 {119 "set": "q-fin",120 "from": "2014-01-01",121 "until": "2014-12-31",122 "pages_total": 5,123 "records_seen_total": 2834124 },125 {126 "set": "q-fin",127 "from": "2015-01-01",128 "until": "2015-12-31",129 "pages_total": 6,130 "records_seen_total": 3797131 },132 {133 "set": "q-fin",134 "from": "2010-01-01",135 "until": "2010-12-31",136 "pages_total": 7,137 "records_seen_total": 4201138 },139 {140 "set": "q-fin",141 "from": "2016-01-01",142 "until": "2016-12-31",143 "pages_total": 8,144 "records_seen_total": 4998145 },146 {147 "set": "q-fin",148 "from": "2009-01-01",149 "until": "2009-12-31",150 "pages_total": 9,151 "records_seen_total": 5777152 },153 {154 "set": "q-fin",155 "from": "2008-01-01",156 "until": "2008-12-31",157 "pages_total": 10,158 "records_seen_total": 6723159 },160 {161 "set": "econ",162 "from": "2017-09-01",163 "until": "2017-12-31",164 "pages_total": 11,165 "records_seen_total": 6797166 }167 ],168 "truncated_at": null,169 "candidate_pool": 123,170 "oai_endpoint": "https://oaipmh.arxiv.org/oai",171 "oai_pages": 11,172 "http": {173 "requests": 211,174 "retries": 79,175 "bytes": 100219165176 },177 "elapsed_s": 847.4178 }179}180