Repository Public Research preview
1"""Crossref adapter: offline checks of the item-level rules + a live end-to-end fetch of 2 real articles."""2from __future__ import annotations34import datetime as _dt5import json6from pathlib import Path78import pytest910from htr.ingest import crossref as cx1112ROOT = Path(__file__).resolve().parents[2]13CONFIG = ROOT / "configs/data/crossref_journals.json"141516def test_item_rules_offline() -> None:17 item = {18 "DOI": "10.1186/X-1",19 "published-print": {"date-parts": [[2018, 1]]},20 "published-online": {"date-parts": [[2017, 11, 3]]},21 "issued": {"date-parts": [[2017, 11, 3]]},22 "license": [23 {"content-version": "tdm", "start": {"date-parts": [[2017, 1, 1]]}, "URL": "http://www.springer.com/tdm"},24 {"content-version": "vor", "start": {"date-parts": [[2017, 11, 3]]}, "URL": "http://creativecommons.org/licenses/by/4.0"},25 {"content-version": "vor", "start": {"date-parts": [[2030, 1, 1]]}, "URL": "http://creativecommons.org/licenses/by-nc/4.0"},26 ],27 "link": [28 {"URL": "http://link.springer.com/article/10.1186/X-1/fulltext.html", "content-type": "text/html", "content-version": "vor", "intended-application": "text-mining"},29 {"URL": "http://link.springer.com/content/pdf/10.1186/X-1.pdf", "content-type": "application/pdf", "content-version": "vor", "intended-application": "text-mining"},30 {"URL": "https://link.springer.com/content/pdf/10.1186/X-1.pdf", "content-type": "application/pdf", "content-version": "vor", "intended-application": "similarity-checking"},31 ],32 "resource": {"primary": {"URL": "http://x.springeropen.com/articles/10.1186/X-1"}},33 }34 iso, ev = cx.pick_date(item)35 assert (iso, ev) == ("2018-01", "crossref published-print date-parts") # print date wins → not pre-201836 assert not cx.is_before(iso, "2017-12-31")37 code, lic = cx.pick_license(item, today=_dt.date(2026, 11, 9))38 assert code == "CC-BY" and lic["content-version"] == "vor" # tdm entry ignored, future entry ignored39 routes = cx.fulltext_candidates(item, {"fallback_pdf": ["https://{landing_host}/track/pdf/{doi}"]})40 assert [r[1] for r in routes] == ["pdf", "pdf"] # duplicate http/https link collapsed, html skipped41 assert routes[0][0] == "https://link.springer.com/content/pdf/10.1186/X-1.pdf"42 assert routes[1][0] == "https://x.springeropen.com/track/pdf/10.1186/X-1"43 assert cx.sanitize_doi("10.3390/Risks5040056") == "10.3390_risks5040056"44 assert cx.author_group(["Jane van Doe"]) == "doe_j"454647def test_robots_rfc9309_longest_match() -> None:48 rules = cx.Robots._parse("User-agent: *\nDisallow: /\nAllow: /content/\nDisallow: /*/figures\n\nUser-agent: ClaudeBot\nDisallow: /\n")49 r = cx.Robots.__new__(cx.Robots)50 r.cache = {"https://link.springer.com": rules["*"]}51 assert r.allowed("https://link.springer.com/content/pdf/10.1186/x.pdf")52 assert not r.allowed("https://link.springer.com/article/10.1186/x")53 assert not r.allowed("https://link.springer.com/content/x/figures")545556@pytest.mark.network57def test_crossref_live_two_articles(tmp_path: Path) -> None:58 cfg = json.loads(CONFIG.read_text())59 names = {"Econometrics", "Financial Innovation"} # one MDPI, one SpringerOpen route60 cfg["journals"] = [j for j in cfg["journals"] if j["name"] in names]61 stats = cx.discover_crossref(tmp_path, cfg, {"CC-BY"}, max_docs=2, seed=7, per_journal_cap=1)62 assert stats["fetched"] == 2, stats63 assert set(stats["by_journal"]) == names64 stubs = sorted(tmp_path.glob("*.meta.json"))65 assert len(stubs) == 266 from htr.ingest.corpus import parse_raw6768 for sp in stubs:69 stub = json.loads(sp.read_text())70 assert stub["source"] == "crossref" and stub["license"]["code"] == "CC-BY"71 assert stub["license"]["evidence"] == "crossref license[].URL"72 assert stub["pub_year"] < 2018 and stub["pub_date_evidence"].startswith("crossref ")73 assert stub["raw_format"] in ("pdf", "jats")74 assert Path(stub["raw_path"]).stat().st_size == stub["raw_bytes"]75 parsed = parse_raw(stub)76 assert len(parsed.paragraphs) >= 577 for p in parsed.paragraphs:78 assert parsed.body_text[p.char_start:p.char_end] == p.text79 # resumable: a second run fetches nothing new80 again = cx.discover_crossref(tmp_path, cfg, {"CC-BY"}, max_docs=2, seed=7, per_journal_cap=1)81 assert again["fetched"] == 0 and again["already_present"] == 282