Repository Public Research preview
1"""Books stratum: English reference / scientific / scholarly books published before 2018 with verifiable2licences. Three sub-sources (verified live 2026-11-09; details in ``configs/data/books_sources.json``):34(a) **Project Gutenberg** (public domain in the US). Metadata-first: the official machine-readable catalog5 ``https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv`` (public domain; permitted for robots by the PG6 "Robot Access" policy) → filter Type=Text, Language=en, non-fiction LoCC classes → plain-text UTF-8 download7 from the official PG mirror ``gutenberg.pglaf.org`` (the www.gutenberg.org website itself is "intended for8 human users only"; robots must use mirrors, the harvest endpoint or the catalog feeds). Publication date:9 printed front matter (copyright / published / imprint / roman numerals) → else the latest death year of all10 contributors in the catalog (an UPPER BOUND, flagged in notes) → else no date (excluded by the builder).1112(b) **NCBI Bookshelf** Literature Archive (``ftp.ncbi.nlm.nih.gov/pub/litarch``): ``file_list.txt`` (title,13 publisher, year, NBK accession) → per-book ``.tar.gz`` with BITS ``book-part-wrapper`` XML files. The HTTPS14 host's robots.txt is ``Disallow: /``, so we use the FTP protocol (NCBI's documented bulk channel). Licences15 are NOT in the file list: they are read from each book's XML ``<permissions>``; per the archive's16 ``license.txt``, a book with no licence and no copyright statement is public domain. One stub per body17 chapter (``raw_format="bits"``, ``doc_type="book_chapter"``, ``source_ids.nbk`` groups chapters of a book).1819(c) **OpenStax / Connexions** via Internet Archive: dated pre-2018 editions are not retrievable from openstax.org20 (current revisions only; ``/apps/archive`` disallowed by robots.txt) and archive.cnx.org is offline. The21 Internet Archive holds (i) a few OpenStax College textbooks uploaded before 2018 and (ii) the ``ost-*`` bulk22 snapshot of 2 179 Connexions collections (uploaded 2012-11). Only items whose PDF carries its own date23 ("PDF generated: <date>", "Collection structure revised: <date>", "© <year>") and a Creative Commons licence24 statement are kept; IA upload timestamps are recorded but never used as date evidence. Text is extracted with25 PyMuPDF to ``<stem>.txt`` (``raw_format="txt"``); the PDF is kept next to it for provenance.26"""27from __future__ import annotations2829import csv30import ftplib31import html32import io33import json34import random35import re36import tarfile37import time38from dataclasses import dataclass39from pathlib import Path4041from htr.data.taxonomy import era_of42from htr.licenses.policy import LicenseRecord, normalize_license43from htr.parse.books import detect_frontmatter_year, gutenberg_header_status, strip_gutenberg_boilerplate44from htr.utils.hashing import md5_bytes, sha256_bytes, short_id45from htr.utils.http import HostPolicy, PoliteClient46from htr.utils.io import atomic_write_bytes, atomic_write_text, ensure_dir, write_json47from htr.utils.provenance import utc_now4849# --------------------------------------------------------------------------------------------------------------50# Politeness: >= 2 s between requests to the same host; robots.txt checked for gutenberg / archive / openstax51# --------------------------------------------------------------------------------------------------------------52BOOK_HOST_POLICIES = {53 "www.gutenberg.org": HostPolicy(2.0, True),54 "gutenberg.pglaf.org": HostPolicy(2.0, True),55 "archive.org": HostPolicy(2.0, True),56 "openstax.org": HostPolicy(2.0, True),57}585960def make_client(timeout: float = 120.0) -> PoliteClient:61 c = PoliteClient(timeout=timeout)62 c.limiter.policies.update(BOOK_HOST_POLICIES)63 return c646566def _get(client: PoliteClient, url: str, **kw):67 host = url.split("/")[2]68 # archive.org redirects downloads to ia*.us.archive.org data nodes: same politeness, robots checked on archive.org69 if host not in client.limiter.policies and host.endswith(".archive.org"):70 client.limiter.policies[host] = HostPolicy(2.0, False)71 return client.get(url, check_robots=client.limiter.policies.get(host, HostPolicy()).check_robots, **kw)727374def _inc(d: dict, k, n: int = 1) -> None:75 k = str(k)76 d[k] = d.get(k, 0) + n777879# ==============================================================================================================80# (a) Project Gutenberg81# ==============================================================================================================82PG_CATALOG_URL = "https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv"83PG_MIRROR = "https://gutenberg.pglaf.org"84PG_LICENSE_URI = "https://www.gutenberg.org/policy/license.html"85PG_LICENSE_EVIDENCE = "Project Gutenberg catalog: public-domain in the US; Gutenberg License header"8687# Library of Congress classes for scholarly / reference non-fiction (prefix match on each LoCC code)88DEFAULT_LOCC_PREFIXES = ("Q", "R", "T", "H", "G", "B", "L", "Z")89# Excluded inside the allowed letters: religion (BL-BX), recreation/games (GV), home economics/cookery (TX)90EXCLUDED_LOCC_PREFIXES = ("BL", "BM", "BP", "BQ", "BR", "BS", "BT", "BV", "BX", "GV", "TX")91# Any literature class on the record excludes it (fiction, poetry, drama, juvenile belles-lettres)92FICTION_LOCC_PREFIXES = ("PN", "PQ", "PR", "PS", "PT", "PZ")93FICTION_SUBJECT_RE = re.compile(r"\b(?:fiction|poetry|poems|drama|juvenile|short stories|stories|humor|humour|plays|fairy tales|novels?)\b", re.IGNORECASE)94FICTION_SHELF_RE = re.compile(r"Category: (?:Novels|Short Stories|Poetry|Plays/Films/Dramas|Children & Young Adult Reading|Humour|Science-Fiction & Fantasy|Adventure|Romance|Crime, Thrillers & Mystery|Mythology, Legends & Folklore)", re.IGNORECASE)9596_LOCC_DISCIPLINE = [97 (r"^QA7[56]", "computer_science"), (r"^QA", "mathematics"), (r"^Q[BC]", "physics"), (r"^QD", "chemistry"), (r"^QE", "earth_sciences"),98 (r"^Q[HKLMPR]", "life_sciences"), (r"^Q", "general_reference"), (r"^R", "biomedicine"), (r"^T", "engineering"), (r"^HA", "statistics"),99 (r"^HG", "finance"), (r"^H[B-J]", "economics"), (r"^H", "social_sciences"), (r"^GN", "social_sciences"), (r"^G", "earth_sciences"),100 (r"^BF", "social_sciences"), (r"^B", "humanities"), (r"^L", "social_sciences"), (r"^Z", "general_reference"),101]102103104def locc_codes(field_: str) -> list[str]:105 return [c.strip() for c in (field_ or "").split(";") if c.strip()]106107108def discipline_from_locc(codes: list[str]) -> str:109 for c in codes:110 for pat, d in _LOCC_DISCIPLINE:111 if re.match(pat, c):112 return d113 return "other"114115116def gutenberg_row_eligible(row: dict, prefixes: tuple[str, ...] = DEFAULT_LOCC_PREFIXES) -> tuple[bool, str | None]:117 """(eligible, primary LoCC code). English text, non-fiction scholarly/reference LoCC classes."""118 if row.get("Type") != "Text" or (row.get("Language") or "").strip() != "en":119 return False, None120 codes = locc_codes(row.get("LoCC", ""))121 if any(c.startswith(FICTION_LOCC_PREFIXES) for c in codes):122 return False, None123 if FICTION_SUBJECT_RE.search(row.get("Subjects") or "") or FICTION_SHELF_RE.search(row.get("Bookshelves") or ""):124 return False, None125 allowed = [c for c in codes if c.startswith(prefixes) and not c.startswith(EXCLUDED_LOCC_PREFIXES)]126 if not allowed:127 return False, None128 return True, allowed[0]129130131@dataclass132class Contributor:133 name: str134 birth: int | None135 death: int | None136 role: str | None137 has_dates: bool138139140141142def parse_pg_authors(field_: str) -> list[Contributor]:143 out: list[Contributor] = []144 for part in (field_ or "").split(";"):145 part = part.strip()146 if not part:147 continue148 role = None149 rm = re.search(r"\[([^\]]+)\]\s*$", part)150 if rm:151 role = rm.group(1).strip()152 part = part[: rm.start()].strip()153 dm = re.search(r",\s*((?:\d{1,4}\??\s*(?:BCE|BC)?)?\s*-\s*(?:\d{1,4}\??\s*(?:BCE|BC)?)?)\s*$", part)154 birth = death = None155 has_dates = False156 if dm and re.search(r"\d", dm.group(1)):157 has_dates = True158 b, _, d = dm.group(1).partition("-")159 bn, dn = re.search(r"\d{1,4}", b), re.search(r"\d{1,4}", d)160 if bn and "BC" not in b:161 birth = int(bn.group())162 if dn and "BC" not in d:163 death = int(dn.group())164 part = part[: dm.start()].strip()165 name = part166 if "," in name: # "Darwin, Charles" → "Charles Darwin" (keep extra qualifiers in parentheses out)167 last, _, first = name.partition(",")168 first = re.sub(r"\s*\(.*?\)", "", first).strip().strip(",")169 name = f"{first} {last.strip()}".strip()170 out.append(Contributor(name, birth, death, role, has_dates))171 return out172173174def death_year_upper_bound(contribs: list[Contributor]) -> int | None:175 """Latest death year over ALL contributors (authors, editors, translators...). None when any contributor has176 no recorded death year (living, unknown, 'Various', 'Anonymous'), since the bound would then be invalid."""177 if not contribs:178 return None179 if any(c.death is None for c in contribs):180 return None181 return max(c.death for c in contribs) # type: ignore[type-var]182183184def pg_mirror_paths(ebook_id: int) -> list[str]:185 s = str(ebook_id)186 d = "/".join(s[:-1]) if len(s) > 1 else "0"187 return [f"{PG_MIRROR}/cache/epub/{s}/pg{s}.txt", f"{PG_MIRROR}/{d}/{s}/{s}-0.txt", f"{PG_MIRROR}/{d}/{s}/{s}.txt"]188189190def load_pg_catalog(cache_dir: Path, client: PoliteClient, max_age_days: float = 7.0) -> tuple[list[dict], dict]:191 """Download (or reuse a cached copy of) the PG catalog CSV. Returns rows and provenance."""192 ensure_dir(cache_dir)193 p = cache_dir / "pg_catalog.csv"194 info = {"url": PG_CATALOG_URL, "path": str(p)}195 if p.exists() and (time.time() - p.stat().st_mtime) < max_age_days * 86400:196 info["cached"] = True197 else:198 r = client.get(PG_CATALOG_URL, check_robots=True)199 r.raise_for_status()200 atomic_write_bytes(p, r.content)201 info["cached"] = False202 info["last_modified"] = r.headers.get("last-modified")203 data = p.read_bytes()204 info["sha256"] = sha256_bytes(data)205 info["retrieved_or_cached_at"] = utc_now()206 rows = list(csv.DictReader(io.StringIO(data.decode("utf-8"))))207 return rows, info208209210def pg_pool(rows: list[dict], prefixes: tuple[str, ...] = DEFAULT_LOCC_PREFIXES) -> tuple[list[tuple[dict, str]], dict]:211 pool: list[tuple[dict, str]] = []212 by_class: dict = {}213 for r in rows:214 ok, code = gutenberg_row_eligible(r, prefixes)215 if ok and code:216 pool.append((r, code))217 _inc(by_class, re.match(r"[A-Z]+", code).group(0) if re.match(r"[A-Z]+", code) else code) # type: ignore[union-attr]218 return pool, by_class219220221def _stratified_order(pool: list[tuple[dict, str]], seed: int) -> list[tuple[dict, str]]:222 """Round-robin over LoCC first letters (seeded shuffle inside each class) so small samples span classes."""223 rng = random.Random(seed)224 groups: dict[str, list[tuple[dict, str]]] = {}225 for item in pool:226 groups.setdefault(item[1][0], []).append(item)227 keys = sorted(groups)228 rng.shuffle(keys)229 for k in keys:230 rng.shuffle(groups[k])231 out: list[tuple[dict, str]] = []232 i = 0233 while any(groups.values()):234 k = keys[i % len(keys)]235 if groups[k]:236 out.append(groups[k].pop())237 i += 1238 return out239240241def date_gutenberg(body: str, row: dict, contribs: list[Contributor]) -> dict:242 """Publication-date decision for one Gutenberg text: front matter → death-year upper bound → none."""243 issued_year = int(row["Issued"][:4]) if re.match(r"\d{4}", row.get("Issued") or "") else 2017244 births = [c.birth for c in contribs if c.birth]245 min_year = (min(births) + 10) if births else 1450246 fm = detect_frontmatter_year(body, max_chars=5000, min_year=max(1450, min_year), max_year=min(2017, issued_year))247 if fm:248 return {"pub_year": fm.year, "pub_date": str(fm.year), "pub_date_evidence": f"gutenberg front matter: {fm.matched}", "date_rule": f"frontmatter:{fm.rule}", "notes": [f"front-matter years found on the matched line: {fm.all_years}; the latest is used (upper bound on the reproduced edition)"]}249 dy = death_year_upper_bound(contribs)250 if dy is not None and dy <= 2017:251 return {"pub_year": dy, "pub_date": str(dy), "pub_date_evidence": "author death year (catalog); upper bound", "date_rule": "death_year", "notes": [f"pub_year {dy} is the latest death year of all catalog contributors: an UPPER BOUND, not an observed publication date (posthumous editions are possible); era assignment is approximate"]}252 return {"pub_year": None, "pub_date": None, "pub_date_evidence": None, "date_rule": "none", "notes": ["no publication year found in the front matter and no complete contributor death years: undated (excluded by the corpus builder)"]}253254255def discover_gutenberg(256 out_dir: Path,257 max_docs: int,258 seed: int,259 locc_prefixes: tuple[str, ...] = DEFAULT_LOCC_PREFIXES,260 client: PoliteClient | None = None,261 log_path: Path | None = None,262 max_bytes: int = 8_000_000,263 max_attempts: int | None = None,264) -> dict:265 """Catalog → eligible pool (English, non-fiction LoCC) → stratified seeded sample → mirror download →266 PG header licence check → front-matter / death-year dating → ``<stem>.txt`` + ``<stem>.meta.json`` stubs."""267 client = client or make_client()268 out_dir = ensure_dir(Path(out_dir) / "gutenberg")269 stats: dict = {"catalog_rows": 0, "candidates": 0, "attempted": 0, "downloaded": 0, "already_present": 0, "not_found": 0, "too_large": 0, "copyrighted_header": 0, "no_pd_header": 0, "no_date": 0, "date_from_frontmatter": 0, "date_from_death_year": 0, "errors": 0, "by_locc": {}, "by_era": {}, "by_date_rule": {}, "pool_by_class": {}}270 rows, cat_info = load_pg_catalog(Path(out_dir).parent / "_catalog", client)271 stats["catalog"] = cat_info272 stats["catalog_rows"] = len(rows)273 pool, by_class = pg_pool(rows, locc_prefixes)274 stats["candidates"] = len(pool)275 stats["pool_by_class"] = dict(sorted(by_class.items(), key=lambda kv: -kv[1]))276 log = open(log_path, "a", encoding="utf-8") if log_path else None277 for row, code in _stratified_order(pool, seed):278 if stats["downloaded"] >= max_docs or (max_attempts and stats["attempted"] >= max_attempts):279 break280 ebook_id = int(row["Text#"])281 stem = f"pg{ebook_id}"282 if (out_dir / f"{stem}.meta.json").exists():283 stats["already_present"] += 1284 stats["downloaded"] += 1285 continue286 stats["attempted"] += 1287 data, url = None, None288 try:289 for u in pg_mirror_paths(ebook_id):290 r = _get(client, u)291 if r.status_code == 200 and r.content:292 data, url = r.content, u293 break294 except Exception as e: # noqa: BLE001295 stats["errors"] += 1296 if log:297 log.write(json.dumps({"source": "gutenberg", "id": ebook_id, "error": repr(e)[:300]}) + "\n")298 continue299 if data is None:300 stats["not_found"] += 1301 continue302 if len(data) > max_bytes:303 stats["too_large"] += 1304 continue305 text = data.decode("utf-8", errors="replace").replace("\r\n", "\n").replace("\r", "\n")306 sp = strip_gutenberg_boilerplate(text)307 hs = gutenberg_header_status(sp.header)308 if hs == "copyrighted":309 stats["copyrighted_header"] += 1310 continue311 if hs != "public_domain_us":312 stats["no_pd_header"] += 1313 continue314 contribs = parse_pg_authors(row.get("Authors", ""))315 dated = date_gutenberg(sp.body, row, contribs)316 rule = dated.pop("date_rule")317 _inc(stats["by_date_rule"], rule)318 if rule.startswith("frontmatter"):319 stats["date_from_frontmatter"] += 1320 elif rule == "death_year":321 stats["date_from_death_year"] += 1322 else:323 stats["no_date"] += 1324 atomic_write_bytes(out_dir / f"{stem}.txt", data)325 header_line = " ".join(sp.header.split())326 m = re.search(r"This e-?book is for the use of anyone anywhere in the United States[^.]*\.", header_line, re.IGNORECASE)327 lic = LicenseRecord(code="PUBLIC-DOMAIN", uri=PG_LICENSE_URI, evidence=PG_LICENSE_EVIDENCE, raw=(m.group(0) if m else header_line[:300]))328 authors = [c.name for c in contribs if (c.role or "Author").lower() in ("author", "creator")] or [c.name for c in contribs]329 notes = list(dated.pop("notes")) + ["licence: public domain in the USA per Project Gutenberg; status elsewhere not verified (policy condition: US works only unless stated)", "PG header/footer and trademark text stripped by the parser"]330 stub = {331 "doc_id": short_id("doc", "gutenberg", str(ebook_id)),332 "source": "gutenberg",333 "source_ids": {"gutenberg_id": str(ebook_id)},334 "title": " ".join((row.get("Title") or "").split()) or None,335 "authors": authors,336 "source_url": url,337 "license": lic.to_dict(),338 "acquired_at": utc_now(),339 "raw_path": str(out_dir / f"{stem}.txt"),340 "raw_sha256": sha256_bytes(data),341 "raw_bytes": len(data),342 "raw_format": "txt",343 **dated,344 "subject_area": discipline_from_locc(locc_codes(row.get("LoCC", ""))),345 "subject_raw": [s.strip() for s in (row.get("Subjects") or "").split(";") if s.strip()],346 "doc_type": "book",347 "language": "en",348 "provider_meta": {"catalog_row": row, "catalog_sha256": cat_info.get("sha256"), "contributors": [c.__dict__ for c in contribs], "mirror": PG_MIRROR},349 "notes": notes,350 }351 write_json(out_dir / f"{stem}.meta.json", stub)352 stats["downloaded"] += 1353 _inc(stats["by_locc"], re.match(r"[A-Z]+", code).group(0)) # type: ignore[union-attr]354 _inc(stats["by_era"], era_of(stub.get("pub_year")))355 if log:356 log.close()357 stats["http"] = client.stats358 return stats359360361# ==============================================================================================================362# (b) NCBI Bookshelf (Literature Archive, FTP)363# ==============================================================================================================364NCBI_FTP_HOST = "ftp.ncbi.nlm.nih.gov"365LITARCH_DIR = "/pub/litarch"366_HEALTH_PUBLISHER_RE = re.compile(r"health|medic|disease|drug|nurs|clinical|cancer|NICE|NIHR|AHRQ|Healthcare|Surgeon General|Veterans|Mental|Substance Abuse|Biotechnology|Microbiology|Genetic|Heart|Lung|Blood|Child Health|Environmental Health|World Health", re.IGNORECASE)367BOOKSHELF_PD_RULE = "bookshelf litarch license.txt: no license or copyright statement in the XML → public domain"368_NONBODY_PART_RE = re.compile(r"(?:^|_)(?:toc|ack|bio|rl|ref|refs|gl|glossary|abbr|abbrev|app\w*|appendix\w*|fm|front\w*|cover|about|pref?ace_?notes|copyright|disclaimer|index|license|ded|dedication|contrib\w*|fu|editors?)\d*$", re.IGNORECASE)369370371class FtpFetcher:372 """Minimal polite FTP client (anonymous), >= min_interval seconds between commands that transfer data."""373374 def __init__(self, host: str = NCBI_FTP_HOST, min_interval: float = 2.0, timeout: float = 120.0, retries: int = 4):375 self.host, self.min_interval, self.timeout, self.retries = host, min_interval, timeout, retries376 self._ftp: ftplib.FTP | None = None377 self._last = 0.0378 self.stats = {"requests": 0, "retries": 0, "bytes": 0}379380 def _conn(self) -> ftplib.FTP:381 if self._ftp is None:382 self._ftp = ftplib.FTP(self.host, timeout=self.timeout)383 self._ftp.login()384 return self._ftp385386 def _wait(self) -> None:387 d = self.min_interval - (time.monotonic() - self._last)388 if d > 0:389 time.sleep(d)390 self._last = time.monotonic()391392 def _do(self, fn):393 last = None394 for attempt in range(self.retries + 1):395 self._wait()396 try:397 self.stats["requests"] += 1398 return fn(self._conn())399 except (ftplib.error_temp, OSError, EOFError) as e: # pragma: no cover - network400 last = e401 self.stats["retries"] += 1402 self.close()403 time.sleep(min(60.0, 2.0**attempt))404 raise RuntimeError(f"FTP failed after retries: {last}")405406 def size(self, path: str) -> int | None:407 def f(ftp: ftplib.FTP):408 ftp.voidcmd("TYPE I")409 return ftp.size(path)410 try:411 return self._do(f)412 except ftplib.error_perm:413 return None414415 def get(self, path: str) -> bytes:416 def f(ftp: ftplib.FTP):417 buf = io.BytesIO()418 ftp.retrbinary(f"RETR {path}", buf.write)419 return buf.getvalue()420 data = self._do(f)421 self.stats["bytes"] += len(data)422 return data423424 def close(self) -> None:425 if self._ftp is not None:426 try:427 self._ftp.quit()428 except Exception: # noqa: BLE001429 pass430 self._ftp = None431432433def parse_litarch_file_list(data: bytes) -> list[dict]:434 rows = []435 for i, raw in enumerate(data.split(b"\n")):436 if i == 0 or not raw.strip(): # first line is a timestamp437 continue438 try:439 line = raw.decode("utf-8")440 except UnicodeDecodeError:441 line = raw.decode("cp1252", errors="replace")442 parts = line.rstrip("\r").split("\t")443 if len(parts) < 5:444 continue445 rows.append({"file": parts[0], "title": html.unescape(parts[1]), "publisher": html.unescape(parts[2]), "year": parts[3].strip(), "nbk": parts[4].strip(), "last_updated": parts[5].strip() if len(parts) > 5 else None})446 return rows447448449def _bits_license(xml: bytes) -> tuple[str, str | None, str, str | None]:450 """(code, uri, evidence, raw) from a BITS file: licence element → copyright statement → PD by absence."""451 from lxml import etree452453 from htr.parse.jats import _license # reuse the JATS licence logic454455 root = etree.fromstring(xml, etree.XMLParser(recover=True, huge_tree=True, resolve_entities=False, no_network=True))456 code, uri, ev = _license(None, root)457 perms = root.xpath("//permissions")458 raw = " ".join(" ".join("".join(x.itertext()) for x in perms).split())[:500] if perms else None459 if code != "UNKNOWN":460 return code, uri, ev or "bits permissions", raw461 has_lic = bool(root.xpath("//permissions/license"))462 has_cr = bool(root.xpath("//permissions/copyright-statement|//permissions/copyright-holder"))463 if not has_lic and not has_cr:464 return "PUBLIC-DOMAIN", None, BOOKSHELF_PD_RULE, raw465 return "UNKNOWN", uri, "bits permissions present but no machine-readable licence", raw466467468def discover_bookshelf(469 out_dir: Path,470 max_docs: int,471 seed: int,472 allowed_codes: set[str],473 ftp: FtpFetcher | None = None,474 log_path: Path | None = None,475 max_mb: float = 25.0,476 max_attempts: int | None = None,477) -> dict:478 """``max_docs`` counts BOOKS kept (each book yields one stub per body chapter)."""479 from htr.parse.jats import EXCLUDED_SECTION_TITLES, parse_jats480481 ftp = ftp or FtpFetcher()482 out_dir = ensure_dir(Path(out_dir) / "bookshelf")483 stats: dict = {"list_rows": 0, "pre2018_rows": 0, "attempted": 0, "too_large": 0, "books_kept": 0, "chapters_written": 0, "license_excluded": 0, "license_unknown": 0, "not_pre2018_xml": 0, "no_body_chapters": 0, "errors": 0, "by_license": {}, "by_publisher": {}, "by_era": {}}484 cache = ensure_dir(Path(out_dir).parent / "_catalog") / "litarch_file_list.txt"485 if not cache.exists() or time.time() - cache.stat().st_mtime > 7 * 86400:486 atomic_write_bytes(cache, ftp.get(f"{LITARCH_DIR}/file_list.txt"))487 rows = parse_litarch_file_list(cache.read_bytes())488 stats["list_rows"] = len(rows)489 pre = [r for r in rows if r["year"].isdigit() and int(r["year"]) < 2018]490 stats["pre2018_rows"] = len(pre)491 rng = random.Random(seed)492 rng.shuffle(pre)493 log = open(log_path, "a", encoding="utf-8") if log_path else None494 try:495 for row in pre:496 if stats["books_kept"] >= max_docs or (max_attempts and stats["attempted"] >= max_attempts):497 break498 stats["attempted"] += 1499 path = f"{LITARCH_DIR}/{row['file']}"500 try:501 size = ftp.size(path)502 if size and size > max_mb * 1e6:503 stats["too_large"] += 1504 continue505 blob = ftp.get(path)506 members: dict[str, bytes] = {}507 with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tf:508 for m in tf.getmembers():509 if m.isfile() and m.name.endswith((".nxml", ".xml", "license.txt")):510 f = tf.extractfile(m)511 if f is not None:512 members[Path(m.name).name] = f.read()513 except Exception as e: # noqa: BLE001514 stats["errors"] += 1515 if log:516 log.write(json.dumps({"source": "bookshelf", "nbk": row["nbk"], "error": repr(e)[:300]}) + "\n")517 continue518 xml_names = sorted(n for n in members if n.endswith((".nxml", ".xml")))519 if not xml_names:520 stats["errors"] += 1521 continue522 probe = members.get("TOC.nxml") or members[xml_names[0]]523 code, uri, ev, raw_perm = _bits_license(probe)524 _inc(stats["by_license"], code)525 if code == "UNKNOWN":526 stats["license_unknown"] += 1527 continue528 if code not in allowed_codes:529 stats["license_excluded"] += 1530 continue531 written = 0532 for name in xml_names:533 stem_part = name.rsplit(".", 1)[0]534 if name == "TOC.nxml" or _NONBODY_PART_RE.search(stem_part):535 continue536 xml = members[name]537 head = xml[:4000].decode("utf-8", errors="replace")538 if 'from-where="body"' not in head and "<book-part-wrapper" in head:539 continue540 try:541 parsed = parse_jats(xml, "bookshelf")542 except Exception: # noqa: BLE001543 continue544 if not parsed.paragraphs:545 continue546 if parsed.pub_year is not None and parsed.pub_year >= 2018:547 stats["not_pre2018_xml"] += 1548 break549 from lxml import etree550551 r0 = etree.fromstring(xml, etree.XMLParser(recover=True, huge_tree=True, resolve_entities=False, no_network=True))552 ct = r0.xpath("string(//book-part-meta/title-group/title)").strip() if r0 is not None else ""553 if ct and EXCLUDED_SECTION_TITLES.search(ct):554 continue555 stem = f"{row['nbk']}_{stem_part}"556 atomic_write_bytes(out_dir / f"{stem}.nxml", xml)557 lic = LicenseRecord(code=code, uri=uri, evidence=ev, raw=raw_perm)558 stub = {559 "doc_id": short_id("doc", "bookshelf", row["nbk"], stem_part),560 "source": "bookshelf",561 "source_ids": {"nbk": row["nbk"], "book_part": stem_part, **({"doi": parsed.ids["doi"]} if parsed.ids.get("doi") else {})},562 "title": f"{row['title']} — {ct}" if ct else row["title"],563 "authors": parsed.authors,564 "publisher": row["publisher"],565 "source_url": f"ftp://{NCBI_FTP_HOST}{path}#{name}",566 "license": lic.to_dict(),567 "acquired_at": utc_now(),568 "raw_path": str(out_dir / f"{stem}.nxml"),569 "raw_sha256": sha256_bytes(xml),570 "raw_bytes": len(xml),571 "raw_format": "bits",572 "pub_year": int(row["year"]),573 "pub_date": row["year"],574 "pub_date_evidence": "bookshelf litarch file_list Publication Year (the BITS <pub-date> takes precedence when present)",575 "subject_area": "biomedicine" if _HEALTH_PUBLISHER_RE.search(row["publisher"]) else "other",576 "doc_type": "book_chapter",577 "language": "en",578 "provider_md5": None,579 "provider_md5_ok": None,580 "provider_meta": {"file_list_row": row, "tarball_sha256": sha256_bytes(blob), "tarball_md5": md5_bytes(blob), "license_txt": members.get("license.txt", b"").decode("utf-8", errors="replace")[:2000]},581 "notes": [f"chapter {stem_part} of Bookshelf book {row['nbk']}: split by source_ids.nbk to keep chapters of one book together"],582 }583 write_json(out_dir / f"{stem}.meta.json", stub)584 written += 1585 if written:586 stats["books_kept"] += 1587 stats["chapters_written"] += written588 _inc(stats["by_publisher"], row["publisher"])589 _inc(stats["by_era"], era_of(int(row["year"])))590 else:591 stats["no_body_chapters"] += 1592 finally:593 if log:594 log.close()595 ftp.close()596 stats["ftp"] = ftp.stats597 return stats598599600# ==============================================================================================================601# (c) OpenStax / Connexions via Internet Archive (in-file dates only)602# ==============================================================================================================603IA_SEARCH = "https://archive.org/advancedsearch.php"604IA_QUERIES = {605 "openstax_college": "(creator:(openstax) OR title:(openstax) OR description:(\"openstax college\" OR openstaxcollege)) AND mediatype:texts AND publicdate:[2010-01-01 TO 2017-12-31]",606 "cnx_ost": "identifier:ost-*col* AND mediatype:texts AND publicdate:[2010-01-01 TO 2017-12-31]",607}608_MONTHS = "January|February|March|April|May|June|July|August|September|October|November|December"609PDF_DATE_PATTERNS = [610 (re.compile(rf"PDF generated:\s*((?:{_MONTHS})\s+\d{{1,2}},\s*(\d{{4}}))"), "pdf front matter: PDF generated"),611 (re.compile(rf"Collection structure revised:\s*((?:{_MONTHS})\s+\d{{1,2}},\s*(\d{{4}}))"), "pdf front matter: Collection structure revised"),612 (re.compile(r"(?:©|Copyright\s*©?)\s*((\d{4}))\s+(?:by\s+)?Rice University"), "pdf copyright page: © Rice University"),613 (re.compile(r"(?:©|Copyright\s*©?)\s*((\d{4}))\b"), "pdf copyright page"),614]615CC_URI_RE = re.compile(r"https?://creativecommons\.\s*org/licenses/[a-z\-]+/\d\.\d/?", re.IGNORECASE)616617618def pdf_front_evidence(pdf_bytes: bytes, n_pages: int = 10) -> dict:619 """Licence URI and date read from the PDF itself (first pages + document metadata)."""620 import fitz # PyMuPDF621622 out: dict = {"license_uri": None, "license_code": "UNKNOWN", "date": None, "year": None, "evidence": None, "pdf_metadata": {}}623 with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:624 out["pdf_metadata"] = {k: v for k, v in (doc.metadata or {}).items() if v}625 out["page_count"] = doc.page_count626 front = "\n".join(doc[i].get_text("text") for i in range(min(n_pages, doc.page_count)))627 m = CC_URI_RE.search(front)628 if m:629 uri = re.sub(r"\s+", "", m.group(0))630 out["license_uri"], out["license_code"] = uri, normalize_license(None, uri)631 else:632 tm = re.search(r"Creative\s+Commons\s+Attribution(?:\s+\d\.\d)?(?:\s+International|\s+Unported|\s+Generic)?\s+Licen[sc]e", front, re.IGNORECASE)633 if tm and not re.search(r"Non-?Commercial|No-?Deriv|Share-?Alike", front, re.IGNORECASE):634 out["license_code"], out["license_uri"], out["license_text"] = "CC-BY", None, " ".join(tm.group(0).split())635 for pat, ev in PDF_DATE_PATTERNS:636 dm = pat.search(front)637 if dm:638 out["date"], out["year"], out["evidence"] = dm.group(1), int(dm.group(2)), f"{ev} {dm.group(1).strip()}" if ev.endswith(("generated", "revised")) else f"{ev}: {' '.join(dm.group(0).split())}"639 break640 return out641642643def pdf_to_text(pdf_bytes: bytes, margin: float = 0.06) -> str:644 """Born-digital PDF → blank-line separated blocks (running headers/footers in the top/bottom margins dropped)."""645 import fitz646647 paras: list[str] = []648 with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:649 for page in doc:650 h = page.rect.height651 for b in page.get_text("blocks"):652 y0, y1, txt = b[1], b[3], b[4]653 if y1 < h * margin or y0 > h * (1 - margin) or not txt.strip():654 continue655 paras.append(txt.strip())656 return "\n\n".join(paras) + "\n"657658659def discover_openstax(660 out_dir: Path,661 max_docs: int,662 seed: int,663 allowed_codes: set[str],664 client: PoliteClient | None = None,665 log_path: Path | None = None,666 max_pdf_mb: float = 60.0,667 queries: tuple[str, ...] = ("openstax_college", "cnx_ost"),668 max_attempts: int | None = None,669) -> dict:670 client = client or make_client()671 out_dir = ensure_dir(Path(out_dir) / "openstax")672 stats: dict = {"search_hits": {}, "candidates": 0, "attempted": 0, "no_pdf": 0, "too_large": 0, "md5_fail": 0, "no_infile_date": 0, "date_after_2017": 0, "license_unverified": 0, "license_excluded": 0, "downloaded": 0, "errors": 0, "by_license": {}, "by_era": {}, "by_query": {}}673 items: list[tuple[str, dict]] = []674 for qname in queries:675 r = _get(client, IA_SEARCH, params={"q": IA_QUERIES[qname], "fl[]": ["identifier", "title", "publicdate", "date", "licenseurl", "creator", "subject"], "rows": "5000", "output": "json"})676 r.raise_for_status()677 docs = r.json()["response"]["docs"]678 stats["search_hits"][qname] = len(docs)679 items.extend((qname, d) for d in docs)680 stats["candidates"] = len(items)681 rng = random.Random(seed)682 first = [x for x in items if x[0] == "openstax_college"]683 rest = [x for x in items if x[0] != "openstax_college"]684 rng.shuffle(first)685 rng.shuffle(rest)686 log = open(log_path, "a", encoding="utf-8") if log_path else None687 for qname, it in first + rest:688 if stats["downloaded"] >= max_docs or (max_attempts and stats["attempted"] >= max_attempts):689 break690 ident = it["identifier"]691 if (out_dir / f"{ident}.meta.json").exists():692 stats["downloaded"] += 1693 continue694 stats["attempted"] += 1695 try:696 meta = _get(client, f"https://archive.org/metadata/{ident}").json()697 pdfs = [f for f in meta.get("files", []) if f.get("name", "").lower().endswith(".pdf") and f.get("source") == "original"]698 pdfs = pdfs or [f for f in meta.get("files", []) if f.get("format") == "Text PDF"]699 if not pdfs:700 stats["no_pdf"] += 1701 continue702 f0 = pdfs[0]703 if f0.get("size") and int(f0["size"]) > max_pdf_mb * 1e6:704 stats["too_large"] += 1705 continue706 url = f"https://archive.org/download/{ident}/{f0['name']}"707 r = _get(client, url)708 r.raise_for_status()709 pdf = r.content710 md5_ok = (md5_bytes(pdf) == f0.get("md5")) if f0.get("md5") else None711 if md5_ok is False:712 stats["md5_fail"] += 1713 continue714 ev = pdf_front_evidence(pdf)715 except Exception as e: # noqa: BLE001716 stats["errors"] += 1717 if log:718 log.write(json.dumps({"source": "openstax", "id": ident, "error": repr(e)[:300]}) + "\n")719 continue720 if ev["year"] is None:721 stats["no_infile_date"] += 1722 continue723 if ev["year"] > 2017:724 stats["date_after_2017"] += 1725 continue726 code = ev["license_code"]727 _inc(stats["by_license"], code)728 if code == "UNKNOWN":729 stats["license_unverified"] += 1730 continue731 if code not in allowed_codes:732 stats["license_excluded"] += 1733 continue734 text = pdf_to_text(pdf)735 atomic_write_bytes(out_dir / f"{ident}.pdf", pdf)736 atomic_write_text(out_dir / f"{ident}.txt", text)737 md = meta.get("metadata", {})738 creators = md.get("creator") or []739 creators = [creators] if isinstance(creators, str) else creators740 subj = md.get("subject") or []741 subj = [subj] if isinstance(subj, str) else subj742 lic = LicenseRecord(code=code, uri=ev["license_uri"], evidence="pdf front matter licence statement" + (f" ({ev['license_uri']})" if ev["license_uri"] else f" (text: {ev.get('license_text')})"), raw=md.get("licenseurl"))743 stub = {744 "doc_id": short_id("doc", "openstax", ident),745 "source": "openstax",746 "source_ids": {"ia_identifier": ident, **({"cnx_collection": m.group(0)} if (m := re.search(r"col\d{5}", ident + " " + f0["name"])) else {})},747 "title": md.get("title"),748 "authors": creators,749 "publisher": md.get("publisher") or ("OpenStax College (Rice University)" if qname == "openstax_college" else "Connexions"),750 "source_url": url,751 "license": lic.to_dict(),752 "acquired_at": utc_now(),753 "raw_path": str(out_dir / f"{ident}.txt"),754 "raw_sha256": sha256_bytes(text.encode("utf-8")),755 "raw_bytes": len(text.encode("utf-8")),756 "raw_format": "txt",757 "pub_year": ev["year"],758 "pub_date": str(ev["year"]),759 "pub_date_evidence": ev["evidence"],760 "subject_area": "other",761 "subject_raw": subj,762 "doc_type": "book",763 "language": "en",764 "provider_md5": f0.get("md5"),765 "provider_md5_ok": md5_ok,766 "provider_meta": {"ia_metadata": md, "pdf_file": f0, "pdf_path": str(out_dir / f"{ident}.pdf"), "pdf_sha256": sha256_bytes(pdf), "pdf_metadata": ev["pdf_metadata"], "query": qname},767 "notes": ["text extracted from the born-digital PDF with PyMuPDF (blocks; top/bottom 6% margins dropped)", f"IA upload date {md.get('publicdate')} recorded for information only, NOT used as date evidence"],768 }769 write_json(out_dir / f"{ident}.meta.json", stub)770 stats["downloaded"] += 1771 _inc(stats["by_query"], qname)772 _inc(stats["by_era"], era_of(ev["year"]))773 if log:774 log.close()775 stats["http"] = client.stats776 return stats777