1"""WP38 — resolve references that Crossref's bibliographic search cannot verify.23literature/manual_references.yaml holds (a) entries with an explicit DOI, resolved through Crossref /works/{doi}4(raw JSON cached in literature/crossref_cache/<key>__doi.json), and (b) DOI-less items (books, reports, official5web pages) verified by fetching the URL: we record HTTP status, final URL, content type, page <title>, whether the6`must_contain` string is present and, for government pages, the dateModified tag. Raw fetch metadata is cached in7literature/crossref_cache/<key>__url.json. Rows are merged into literature/references_verified.csv (replacing any8seed row with the same key); unverified.csv is refreshed; call counts are appended to literature/LOG.md.9"""10from __future__ import annotations11import datetime as dt, html, json, re, sys, time12from pathlib import Path13import requests, yaml, pandas as pd1415ROOT = Path(__file__).resolve().parents[2]16LIT = ROOT / "literature"; CACHE = LIT / "crossref_cache"; CACHE.mkdir(parents=True, exist_ok=True)17UA = {"User-Agent": "Mozilla/5.0 (Macintosh) WP38-UQO-research/0.1 (mailto:simon-pierre.boucher@uqo.ca)"}18TODAY = dt.date.today().isoformat()19CALLS = {"crossref": 0, "web": 0}2021def log(msg): print(f"{dt.datetime.now().isoformat(timespec='seconds')} INFO resolve: {msg}")2223def by_doi(key, doi):24 f = CACHE / f"{key}__doi.json"25 if f.exists(): return json.loads(f.read_text())["message"]26 r = requests.get(f"https://api.crossref.org/works/{doi}", headers={"User-Agent": UA["User-Agent"].split(') ')[-1]}, timeout=60)27 CALLS["crossref"] += 1; r.raise_for_status(); js = r.json(); js["_wp38"] = {"fetched": TODAY, "key": key}28 f.write_text(json.dumps(js, ensure_ascii=False, indent=1)); time.sleep(0.35)29 return js["message"]3031def fetch_url(key, url):32 f = CACHE / f"{key}__url.json"33 if f.exists(): return json.loads(f.read_text())34 r = requests.get(url, headers=UA, timeout=60, allow_redirects=True); CALLS["web"] += 135 ct = r.headers.get("content-type", "")36 rec = {"key": key, "url": url, "final_url": r.url, "status": r.status_code, "content_type": ct, "fetched": TODAY,37 "bytes": len(r.content), "title": None, "date_modified": None, "text_sample": None}38 if "html" in ct or "text" in ct:39 t = re.search(r"<title[^>]*>(.*?)</title>", r.text, re.S | re.I)40 rec["title"] = html.unescape(re.sub(r"\s+", " ", t.group(1))).strip() if t else None41 dm = re.search(r'dateModified[^>]*?(\d{4}-\d{2}-\d{2})', r.text) or re.search(r'(\d{4}-\d{2}-\d{2})[^<]{0,40}dateModified', r.text)42 rec["date_modified"] = dm.group(1) if dm else None43 rec["text_sample"] = html.unescape(re.sub(r"<[^>]+>", " ", r.text))44 rec["text_sample"] = re.sub(r"\s+", " ", rec["text_sample"])[:4000]45 elif "pdf" in ct:46 rec["title"] = "(PDF document)"47 f.write_text(json.dumps(rec, ensure_ascii=False, indent=1)); time.sleep(0.5)48 return rec4950def main():51 man = yaml.safe_load((LIT / "manual_references.yaml").read_text())52 rows = []53 for m in man:54 base = dict(key=m["key"], stream=m["stream"], author=m.get("author", ""), title=m["title"], year=m.get("year"),55 why_needed=m.get("why_needed", ""), doi=None, url=m.get("url"), crossref_title=None, container=m.get("container", ""),56 volume=m.get("volume"), issue=m.get("issue"), pages=m.get("pages"), pub_year=m.get("year"), type=m.get("type", "report"),57 publisher=m.get("publisher", ""), authors=m.get("author", ""), title_sim=None, verification_method=None, verified_date=TODAY, note="")58 if m.get("doi"):59 it = by_doi(m["key"], m["doi"]); yr = it["issued"]["date-parts"][0][0]60 base.update(doi=it["DOI"], url=it.get("URL"), crossref_title=" ".join(it.get("title") or []), container=" ".join(it.get("container-title") or []),61 volume=it.get("volume"), issue=it.get("issue"), pages=it.get("page"), pub_year=yr, type=it.get("type"), publisher=it.get("publisher"),62 authors="; ".join(f"{a.get('family','')}, {a.get('given','')}" for a in it.get("author", []) or []), title_sim=1.0,63 verification_method="crossref:works/{doi}", verified=True, note="resolved by explicit DOI")64 log(f"{m['key']}: DOI {it['DOI']} -> {base['crossref_title'][:60]} ({base['container']}, {yr})")65 else:66 try:67 rec = fetch_url(m["key"], m["url"])68 except Exception as e:69 base.update(verified=False, verification_method="url:fetch", note=f"fetch error {type(e).__name__}"); rows.append(base); continue70 body = (rec.get("text_sample") or "") + " " + (rec.get("title") or "")71 ok = rec["status"] == 200 and (not m.get("must_contain") or m["must_contain"].lower() in body.lower()) and ("pdf" in rec["content_type"] or rec.get("title"))72 base.update(verified=bool(ok), url=rec["final_url"], verification_method="url:fetch",73 note=f"HTTP {rec['status']}, {rec['content_type'].split(';')[0]}, page title: {rec.get('title')!r}" + (f", dateModified {rec['date_modified']}" if rec.get("date_modified") else "") + ("" if ok else " — must_contain string NOT found"))74 log(f"{m['key']}: {'OK' if ok else 'FAIL'} {rec['status']} {rec['final_url'][:70]} | {rec.get('title')}")75 rows.append(base)76 new = pd.DataFrame(rows)77 ver = pd.read_csv(LIT / "references_verified.csv")78 ver = ver[~ver.key.isin(new.key)]79 out = pd.concat([ver, new[ver.columns]], ignore_index=True)80 out.to_csv(LIT / "references_verified.csv", index=False)81 out[out.verified != True].to_csv(LIT / "unverified.csv", index=False)82 nv = int((out.verified == True).sum())83 log(f"references now: {nv} verified / {len(out)} ; unverified: {out.loc[out.verified != True, 'key'].tolist()}")84 with open(LIT / "LOG.md", "a") as f:85 f.write(f"- {dt.datetime.now().isoformat(timespec='seconds')} resolve_dois.py: {len(man)} manual entries, {CALLS['crossref']} Crossref DOI calls, "86 f"{CALLS['web']} web fetches (rest cached); total {nv} verified / {len(out)}\n")8788if __name__ == "__main__":89 main()90