spb/wpai0002_uqo
Public
1{2 "stratum": "books",3 "description": "English reference / scientific / scholarly books published before 2018 with verifiable licences; adapter htr.ingest.books, parser htr.parse.books (txt/html/epub) and htr.parse.jats (BITS).",4 "verified_at": "2026-11-09T17:30:00Z",5 "sub_sources": {6 "gutenberg": {7 "status": "implemented, verified live",8 "catalog_url": "https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv",9 "text_url_pattern": [10 "https://gutenberg.pglaf.org/cache/epub/<id>/pg<id>.txt",11 "https://gutenberg.pglaf.org/<d1>/<d2>/.../<id>/<id>-0.txt",12 "https://gutenberg.pglaf.org/<d1>/<d2>/.../<id>/<id>.txt"13 ],14 "access_policy": "PG 'Robot Access' policy (https://www.gutenberg.org/policy/robot_access.html, read 2026-11-09): the www.gutenberg.org website is for human users only; robots may use the catalog feeds, the /robot/harvest endpoint or mirrors. www.gutenberg.org/robots.txt only disallows /ebooks/search. Texts are fetched from the official PG mirror gutenberg.pglaf.org (listed in MIRRORS.ALL, includes cache files, no robots.txt: HTTP 404). mirror.csclub.uwaterloo.ca disallows /gutenberg for all agents; mirror.cs.odu.edu disallows ClaudeBot/anthropic-ai/GPTBot: both NOT used. >= 2 s between requests per host.",15 "licence_basis": "PUBLIC-DOMAIN (US): PG catalog + the eBook header 'This eBook is for the use of anyone anywhere in the United States ... at no cost'. Headers declaring a copyrighted eBook are excluded. Status outside the US is not verified (policy condition). PG header/footer and trademark text are stripped.",16 "filter": "Type=Text, Language=en (exactly), LoCC prefix in Q,R,T,H,G,B,L,Z, excluding BL-BX (religion), GV (recreation), TX (home economics); any literature class PN/PQ/PR/PS/PT/PZ or fiction/poetry/drama/juvenile subjects or fiction bookshelves excludes the record.",17 "date_evidence_rule": "(1) printed front matter (first 5000 chars after the PG start marker): copyright / 'Entered according to Act of Congress' lines, then published/edition/printed lines, then a publisher imprint line ending with a year or a roman-numeral year before the first CONTENTS/PREFACE/CHAPTER heading; PG-added lines ignored; latest year on the matched line used -> evidence 'gutenberg front matter: <matched string>'. (2) else latest death year over ALL catalog contributors (none may be undated) -> evidence 'author death year (catalog); upper bound' + note (posthumous editions possible, era approximate). (3) else undated -> excluded by the corpus builder.",18 "eligible_pool_estimate": {"catalog_rows": 79769, "text_en": 61977, "eligible_nonfiction_locc": 7799, "by_letter": {"Q": 2186, "H": 1653, "T": 1051, "B": 1016, "R": 620, "Z": 490, "G": 481, "L": 302}, "with_any_author_life_dates": 5929, "catalog_last_modified": "Sun, 08 Nov 2026 22:43:51 GMT"},19 "smoke_run_seed1": {"downloaded": 12, "date_from_frontmatter": 10, "date_from_death_year": 1, "no_date": 1, "documents_kept": 11, "passages_eligible": 2037}20 },21 "bookshelf": {22 "status": "implemented, verified live (small trial)",23 "file_list_url": "ftp://ftp.ncbi.nlm.nih.gov/pub/litarch/file_list.txt",24 "book_url_pattern": "ftp://ftp.ncbi.nlm.nih.gov/pub/litarch/<xx>/<yy>/<name>_<NBK>.tar.gz",25 "access_policy": "The Literature Archive is still online (9 733 books on 2026-11-09). https://ftp.ncbi.nlm.nih.gov/robots.txt is 'User-agent: * / Disallow: /', so HTTPS is NOT used; the FTP protocol (NCBI's documented bulk channel) is used with >= 2 s between transfers. https://www.ncbi.nlm.nih.gov/books/about/ftp/ returned a reCAPTCHA page to curl (not readable by a script).",26 "licence_basis": "Per book, from the BITS <permissions>: <license> xlink:href / text -> canonical code; per the archive's license.txt, 'If there is no license or copyright statement in the XML or PDF files, then the document is in the public domain' -> PUBLIC-DOMAIN with that evidence; a copyright statement without a machine-readable licence -> UNKNOWN (excluded). The file list has no licence column, so each candidate tarball must be downloaded to read its licence.",27 "date_evidence_rule": "BITS <book-meta><pub-date> (parsed by htr.parse.jats; takes precedence) with the file_list 'Publication Year' as a stub fallback; candidates pre-filtered on file_list year < 2018.",28 "unit": "one stub per body chapter file (raw_format=bits, doc_type=book_chapter); source_ids.nbk groups the chapters of a book for splitting",29 "eligible_pool_estimate": {"list_rows": 9733, "pre2018_by_list_year": 3941, "licence_mix": "unknown until downloaded; trial 4 books: PUBLIC-DOMAIN 2 (AHRQ), CC-BY 1 (Springer), UNKNOWN 1"},30 "trial_run_seed1": {"attempted": 4, "books_kept": 3, "chapters_written": 17, "passages_eligible": 299}31 },32 "openstax": {33 "status": "implemented for Internet Archive items with in-file dates; openstax.org itself not usable for pre-2018 editions",34 "search_url": "https://archive.org/advancedsearch.php",35 "access_policy": "openstax.org serves current (post-2018) revisions only and its robots.txt disallows /apps/archive for all agents and /books/ for GPTBot; archive.cnx.org timed out (offline); the osbooks GitHub repositories start after the 2020-2021 migration (not checked for older history). Internet Archive (robots.txt only disallows /control/ and /report/) holds 6 OpenStax College items uploaded before 2018 (A&P 1.6 507 MB, Chemistry 2015, Introduction to Sociology 2013, and three 2017 'LR' PDFs) and the 'ost-*' Open Source Textbook snapshot (2 179 items, 164 matching identifier:ost-*col* i.e. Connexions collections).",36 "licence_basis": "Creative Commons statement read from the PDF front matter (URI, or the text 'Creative Commons Attribution [4.0 International] License' with no NC/ND/SA mention). IA 'licenseurl' metadata is recorded but not used (uploader-supplied; e.g. A&P is tagged BY-SA on IA).",37 "date_evidence_rule": "Only from the PDF itself: 'PDF generated: <date>' / 'Collection structure revised: <date>' (CNX) or '© <year> Rice University' / '© <year>' (OpenStax College copyright page). IA upload dates are recorded in notes and never used as evidence. Items whose PDF has no date or a date > 2017 are rejected.",38 "text": "PyMuPDF block extraction to <stem>.txt (raw_format=txt); PDF kept next to it; ligature loss (e.g. 'rst' for 'first') occurs in some CNX PDFs.",39 "eligible_pool_estimate": {"openstax_college_items": 6, "cnx_collection_items": 164, "note": "most CNX collections are short course packs or K-12 material; many ost-* items are scanned pre-1923 books with no licence statement and are rejected"},40 "trial_run_seed1": {"attempted": 8, "too_large_over_60MB": 3, "downloaded": 5, "openstax_college": 3, "cnx": 2}41 }42 }43}44