Repository Public Research preview
1"""PMC Article Datasets on AWS Open Data (bucket ``pmc-oa-opendata``), the only official bulk channel2since August 2026 (FTP packages and the OA Web Service were retired; see docs/data_governance).34Layout (bucket README.txt, retrieved 2026-11-09):5 <PMCID>.<version>/<PMCID>.<version>.{json,xml,txt,pdf} and metadata/<PMCID>.<version>.json6The JSON carries ``is_pmc_openaccess``, ``is_retracted``, ``license_code`` (Creative Commons code or7'TDM') and S3 URLs with ``?md5=`` digests. Publication dates are NOT in the JSON: they are verified8from the JATS ``<pub-date>`` elements by the parser.9"""10from __future__ import annotations1112import json13import random14import re15from dataclasses import dataclass16from pathlib import Path17from urllib.parse import parse_qs, urlparse1819from htr.licenses.policy import LicenseRecord, normalize_license20from htr.utils.hashing import md5_bytes, sha256_bytes, short_id21from htr.utils.http import PoliteClient22from htr.utils.io import atomic_write_bytes, ensure_dir, write_json23from htr.utils.provenance import utc_now2425BASE = "https://pmc-oa-opendata.s3.amazonaws.com"26# PMCIDs are assigned roughly in deposit order; PMC5760000 ≈ early January 2018. The range is only a27# sampling prior: the publication date is verified per document from the JATS front matter.28PRE2018_ID_MAX_PRIOR = 5_760_00029PRE2018_ID_MIN_PRIOR = 10_000303132@dataclass33class PmcMeta:34 pmcid: str35 version: int36 raw: dict3738 @property39 def license_code(self) -> str:40 return normalize_license(self.raw.get("license_code"))4142 @property43 def xml_url(self) -> str | None:44 return _s3_to_https(self.raw.get("xml_url"))4546 @property47 def xml_md5(self) -> str | None:48 return _md5_param(self.raw.get("xml_url"))495051def _s3_to_https(u: str | None) -> str | None:52 if not u:53 return None54 p = urlparse(u)55 return f"{BASE}/{p.path.lstrip('/')}" if p.scheme == "s3" else u.split("?")[0]565758def _md5_param(u: str | None) -> str | None:59 if not u:60 return None61 q = parse_qs(urlparse(u).query)62 return (q.get("md5") or [None])[0]636465class PmcClient:66 def __init__(self, client: PoliteClient | None = None):67 self.http = client or PoliteClient(timeout=60)6869 def fetch_meta(self, pmcid_num: int, version: int = 1) -> PmcMeta | None:70 url = f"{BASE}/metadata/PMC{pmcid_num}.{version}.json"71 r = self.http.get(url)72 if r.status_code == 404:73 return None74 r.raise_for_status()75 return PmcMeta(f"PMC{pmcid_num}", version, r.json())7677 def fetch_xml(self, meta: PmcMeta) -> tuple[bytes, bool | None]:78 url = meta.xml_url or f"{BASE}/{meta.pmcid}.{meta.version}/{meta.pmcid}.{meta.version}.xml"79 r = self.http.get(url)80 r.raise_for_status()81 data = r.content82 ok = (md5_bytes(data) == meta.xml_md5) if meta.xml_md5 else None83 return data, ok8485 def list_metadata_prefix(self, prefix: str, max_keys: int = 1000) -> list[str]:86 """List metadata keys under a PMCID prefix (e.g. 'metadata/PMC533') via S3 ListObjects."""87 r = self.http.get(f"{BASE}/", params={"prefix": prefix, "max-keys": str(max_keys)})88 r.raise_for_status()89 return re.findall(r"<Key>([^<]+)</Key>", r.text)909192def sample_pmcids(n: int, seed: int, lo: int = PRE2018_ID_MIN_PRIOR, hi: int = PRE2018_ID_MAX_PRIOR, strata: int = 8) -> list[int]:93 """Stratified random PMCID candidates across the id range (so that no single deposit period dominates)."""94 rng = random.Random(seed)95 edges = [lo + (hi - lo) * i // strata for i in range(strata + 1)]96 out: list[int] = []97 per = max(1, n // strata)98 for i in range(strata):99 out.extend(rng.sample(range(edges[i], edges[i + 1]), min(per, edges[i + 1] - edges[i])))100 rng.shuffle(out)101 return out[:n]102103104def discover_and_fetch(105 out_dir: Path,106 n_candidates: int,107 seed: int,108 allowed_codes: set[str],109 client: PmcClient | None = None,110 max_docs: int | None = None,111 require_open_access: bool = True,112 log_path: Path | None = None,113) -> dict:114 """Metadata-first discovery: sample PMCIDs → metadata JSON → licence filter → XML download (+MD5 check).115116 Writes ``<out_dir>/<PMCID>.<v>.xml`` and ``.meta.json`` (manifest stub with licence evidence) and returns counters.117 Nothing is parsed here; date verification happens in ``htr.parse.jats``.118 """119 client = client or PmcClient()120 out_dir = ensure_dir(out_dir)121 stats = {"candidates": 0, "missing": 0, "not_open_access": 0, "retracted": 0, "license_excluded": 0, "manuscript": 0, "fetched": 0, "md5_fail": 0, "errors": 0, "by_license": {}}122 log = open(log_path, "a", encoding="utf-8") if log_path else None123 for num in sample_pmcids(n_candidates, seed):124 if max_docs and stats["fetched"] >= max_docs:125 break126 stats["candidates"] += 1127 try:128 meta = client.fetch_meta(num)129 except Exception as e: # noqa: BLE001130 stats["errors"] += 1131 if log:132 log.write(json.dumps({"pmcid": f"PMC{num}", "error": repr(e)}) + "\n")133 continue134 if meta is None:135 stats["missing"] += 1136 continue137 code = meta.license_code138 stats["by_license"][code] = stats["by_license"].get(code, 0) + 1139 if require_open_access and not meta.raw.get("is_pmc_openaccess"):140 stats["not_open_access"] += 1141 continue142 if meta.raw.get("is_retracted"):143 stats["retracted"] += 1144 continue145 if meta.raw.get("is_manuscript"):146 stats["manuscript"] += 1147 continue148 if code not in allowed_codes:149 stats["license_excluded"] += 1150 continue151 try:152 data, md5_ok = client.fetch_xml(meta)153 except Exception as e: # noqa: BLE001154 stats["errors"] += 1155 if log:156 log.write(json.dumps({"pmcid": meta.pmcid, "error": repr(e)}) + "\n")157 continue158 if md5_ok is False:159 stats["md5_fail"] += 1160 continue161 stem = f"{meta.pmcid}.{meta.version}"162 atomic_write_bytes(out_dir / f"{stem}.xml", data)163 lic = LicenseRecord(code=code, uri=None, evidence="pmc_json.license_code", raw=meta.raw.get("license_code"))164 stub = {165 "doc_id": short_id("doc", "pmc", stem),166 "source": "pmc",167 "source_ids": {k: str(v) for k, v in (("pmcid", meta.pmcid), ("pmid", meta.raw.get("pmid")), ("doi", meta.raw.get("doi"))) if v},168 "title": meta.raw.get("title"),169 "citation": meta.raw.get("citation"),170 "source_url": f"{BASE}/{stem}/{stem}.xml",171 "license": lic.to_dict(),172 "acquired_at": utc_now(),173 "raw_path": str(out_dir / f"{stem}.xml"),174 "raw_sha256": sha256_bytes(data),175 "raw_bytes": len(data),176 "raw_format": "jats",177 "provider_md5": meta.xml_md5,178 "provider_md5_ok": md5_ok,179 "retracted": bool(meta.raw.get("is_retracted")),180 "provider_meta": meta.raw,181 }182 write_json(out_dir / f"{stem}.meta.json", stub)183 stats["fetched"] += 1184 if log:185 log.close()186 stats["http"] = client.http.stats187 return stats188