1"""Phase B — full-text retrieval of stage-1 candidates from legal open-access sources only:2Semantic Scholar openAccessPdf, Unpaywall (best_oa_location) by DOI, arXiv. Records every attempt in3data/metadata/retrieval_log.csv; PDFs stored in data/raw/pdfs/<record_id>.pdf (not committed).4Usage: python scripts/extraction/fetch_pdfs.py [--limit N]"""5import argparse, csv, hashlib, json, re, time, urllib.parse, urllib.request6from datetime import date7from pathlib import Path8import pandas as pd910ROOT = Path(__file__).resolve().parents[2]; META = ROOT / "data" / "metadata"; PDFS = ROOT / "data" / "raw" / "pdfs"; PDFS.mkdir(parents=True, exist_ok=True)11MAILTO = "simon-pierre.boucher@uqo.ca"12UA = f"Mozilla/5.0 (compatible; wp21-uqo meta-analysis; mailto:{MAILTO})"131415def fetch(url, timeout=60):16 req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/pdf,*/*"})17 with urllib.request.urlopen(req, timeout=timeout) as r:18 data = r.read(); ctype = r.headers.get("Content-Type", "")19 return data, ctype202122def unpaywall(doi):23 """All OA locations from Unpaywall, repositories first, pdf URLs before landing pages. Returns list of (label, url)."""24 try:25 with urllib.request.urlopen(urllib.request.Request(f"https://api.unpaywall.org/v2/{urllib.parse.quote(doi)}?email={MAILTO}", headers={"User-Agent": UA}), timeout=30) as r:26 d = json.load(r)27 except Exception:28 return []29 locs = d.get("oa_locations") or []30 locs = sorted(locs, key=lambda l: (0 if l.get("host_type") == "repository" else 1))31 out = []32 for l in locs:33 for key in ("url_for_pdf", "url"):34 u = l.get(key)35 if u and (f"unpaywall:{l.get('host_type')}", u) not in out:36 out.append((f"unpaywall:{l.get('host_type')}", u))37 return out[:6]383940def pdf_link_from_html(html, base):41 m = re.search(r'<meta[^>]+name="citation_pdf_url"[^>]+content="([^"]+)"', html, re.I) or re.search(r'content="([^"]+)"[^>]+name="citation_pdf_url"', html, re.I)42 if m:43 return urllib.parse.urljoin(base, m.group(1))44 m = re.search(r'href="([^"]+\.pdf[^"]*)"', html, re.I)45 return urllib.parse.urljoin(base, m.group(1)) if m else None464748def is_pdf(data):49 return data[:5] == b"%PDF-"505152def main():53 ap = argparse.ArgumentParser(); ap.add_argument("--limit", type=int, default=100000); ap.add_argument("--input", default=str(META / "stage1_candidates.csv")); a = ap.parse_args()54 cand = pd.read_csv(a.input)55 logf = META / "retrieval_log.csv"; done = set()56 if logf.exists():57 done = {r["record_id"] for r in csv.DictReader(open(logf)) if r["status"] in ("ok", "no_oa_url")}58 with open(logf, "a", newline="") as lf:59 log = csv.writer(lf)60 if lf.tell() == 0:61 log.writerow(["date", "record_id", "doi", "source_tried", "url", "status", "bytes", "sha256"])62 n_ok = 063 for _, r in cand.head(a.limit).iterrows():64 rid = str(r.get("record_id") or r.sid)65 if rid in done or (PDFS / f"{rid}.pdf").exists():66 continue67 urls = []68 if isinstance(r.get("oa_pdf"), str) and r.oa_pdf.startswith("http"):69 urls.append(("s2_oa", r.oa_pdf))70 if isinstance(r.get("arxiv"), str) and r.arxiv:71 urls.append(("arxiv", f"https://arxiv.org/pdf/{r.arxiv}"))72 if isinstance(r.get("doi"), str) and r.doi:73 urls += unpaywall(r.doi)74 if not urls:75 log.writerow([str(date.today()), rid, r.get("doi", ""), "none", "", "no_oa_url", 0, ""]); continue76 got = False77 for src, u in urls:78 try:79 data, ctype = fetch(u)80 if not is_pdf(data) and b"<html" in data[:2000].lower():81 link = pdf_link_from_html(data.decode("utf-8", "ignore"), u)82 if link and link != u:83 try:84 data, ctype = fetch(link); u = link; src = src + "+html_pdf_link"85 except Exception:86 pass87 if is_pdf(data) and len(data) > 10_000:88 (PDFS / f"{rid}.pdf").write_bytes(data)89 log.writerow([str(date.today()), rid, r.get("doi", ""), src, u, "ok", len(data), hashlib.sha256(data).hexdigest()[:16]]); got = True; n_ok += 1; break90 log.writerow([str(date.today()), rid, r.get("doi", ""), src, u, f"not_pdf:{ctype[:30]}", len(data), ""])91 except Exception as e:92 log.writerow([str(date.today()), rid, r.get("doi", ""), src, u, f"error:{type(e).__name__}", 0, ""])93 time.sleep(0.5)94 lf.flush()95 print("retrieved", n_ok)969798if __name__ == "__main__":99 main()100