TeX 59%
Python 40.8%
1#!/usr/bin/env python32"""Verify every literature candidate against primary bibliographic records.34- Crossref (query.bibliographic) for articles/chapters/proceedings; raw JSON saved per key in5 docs/verification/<key>.crossref.json. A candidate is VERIFIED when the best hit's title matches6 (normalised similarity >= 0.80) and the year is within ±1; the Crossref metadata (authors, container,7 volume, pages, DOI, year) then replaces our own typing.8- arXiv API for preprints (id taken from the container field "arXiv:ID"); raw Atom saved.9- Items that fail are listed in docs/verification/unverified.csv and are NOT written to references.bib.1011Outputs: paper/references.bib (verified only), docs/verification/verified.csv, docs/verification/unverified.csv12"""13from __future__ import annotations1415import csv16import difflib17import json18import re19import sys20import time21import unicodedata22import urllib.parse23import urllib.request24import xml.etree.ElementTree as ET25from pathlib import Path2627ROOT = Path(__file__).resolve().parents[1]28VER = ROOT / "docs" / "verification"29VER.mkdir(parents=True, exist_ok=True)30UA = "WP20-UQO-literature-verifier/1.0 (mailto:simon-pierre.boucher@uqo.ca)"313233def norm(s: str) -> str:34 s = unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode()35 s = re.sub(r"[^a-z0-9 ]", " ", s.lower())36 return re.sub(r"\s+", " ", s).strip()373839def sim(a: str, b: str) -> float:40 return difflib.SequenceMatcher(None, norm(a), norm(b)).ratio()414243def get(url: str, tries: int = 4) -> bytes:44 for t in range(tries):45 try:46 req = urllib.request.Request(url, headers={"User-Agent": UA})47 with urllib.request.urlopen(req, timeout=60) as r:48 return r.read()49 except Exception as e: # noqa: BLE00150 if t == tries - 1:51 raise52 time.sleep(2 * (t + 1))53 return b""545556def crossref(c: dict) -> dict | None:57 key = c["key"]58 out = VER / f"{key}.crossref.json"59 if c["doi_hint"]:60 try:61 data = json.loads(get(f"https://api.crossref.org/works/{urllib.parse.quote(c['doi_hint'])}"))62 out.write_text(json.dumps(data, ensure_ascii=False))63 item = data["message"]64 fam = (item.get("author") or [{}])[0].get("family", "")65 y = year_of(item)66 if sim(item.get("title", [""])[0], c["title"]) >= 0.6 or (67 norm(fam) == norm(c["first_author"]) and y and abs(int(y) - int(c["year"])) <= 1):68 return item69 except Exception: # noqa: BLE00170 pass71 q = urllib.parse.quote(f"{c['title']} {c['first_author']} {c['year']}")72 data = json.loads(get(f"https://api.crossref.org/works?query.bibliographic={q}&rows=6"))73 out.write_text(json.dumps(data, ensure_ascii=False))74 best, best_s = None, 0.075 for item in data["message"]["items"]:76 t = (item.get("title") or [""])[0]77 s = sim(t, c["title"])78 if s > best_s:79 best, best_s = item, s80 if best is None or best_s < 0.80:81 return None82 y = year_of(best)83 if y and abs(int(y) - int(c["year"])) > 1:84 return None85 return best868788def year_of(item: dict) -> int | None:89 for k in ("published-print", "published-online", "issued", "created"):90 dp = item.get(k, {}).get("date-parts")91 if dp and dp[0] and dp[0][0]:92 return int(dp[0][0])93 return None949596def arxiv(c: dict) -> dict | None:97 m = re.search(r"arXiv:([\d.]+)", c["container"])98 if not m:99 return None100 aid = m.group(1)101 time.sleep(3.5) # arXiv API rate limit102 raw = get(f"https://export.arxiv.org/api/query?id_list={aid}")103 (VER / f"{c['key']}.arxiv.xml").write_bytes(raw)104 ns = {"a": "http://www.w3.org/2005/Atom"}105 root = ET.fromstring(raw)106 e = root.find("a:entry", ns)107 if e is None:108 return None109 title = " ".join((e.findtext("a:title", default="", namespaces=ns) or "").split())110 if sim(title, c["title"]) < 0.8:111 return None112 authors = [a.findtext("a:name", default="", namespaces=ns) for a in e.findall("a:author", ns)]113 year = (e.findtext("a:published", default="", namespaces=ns) or "")[:4]114 return {"title": title, "authors": authors, "year": year, "id": aid,115 "url": f"https://arxiv.org/abs/{aid}", "doi": e.findtext("{http://arxiv.org/schemas/atom}doi")}116117118def bib_escape(s: str) -> str:119 return s.replace("&", r"\&").replace("%", r"\%").replace("_", r"\_")120121122def protect(title: str) -> str:123 # protect acronyms / capitalised words after the first124 return re.sub(r"\b([A-Z][A-Z0-9]+|[A-Z][a-z]*[A-Z][A-Za-z]*)\b", r"{\1}", title)125126127def crossref_bib(c: dict, it: dict) -> tuple[str, dict]:128 authors = " and ".join(f"{a.get('family','')}, {a.get('given','')}".strip(", ") for a in it.get("author", []) if a.get("family"))129 title = (it.get("title") or [c["title"]])[0]130 cont = (it.get("container-title") or [c["container"]])[0] if it.get("container-title") else c["container"]131 year = year_of(it) or c["year"]132 typ = it.get("type", "")133 doi = it.get("DOI", "")134 vol = it.get("volume", ""); issue = it.get("issue", ""); pages = it.get("page", "")135 publisher = it.get("publisher", "")136 if typ == "journal-article":137 kind = "article"138 elif typ in ("proceedings-article",):139 kind = "inproceedings"140 elif typ in ("book-chapter",):141 kind = "incollection"142 elif typ in ("report", "posted-content", "report-component"):143 kind = "techreport"144 else:145 kind = c["kind"] if c["kind"] in ("article", "inproceedings", "incollection", "techreport") else "misc"146 f = {"author": authors, "title": protect(bib_escape(title)), "year": str(year), "doi": doi}147 if kind == "article":148 f["journal"] = bib_escape(cont)149 if vol: f["volume"] = vol150 if issue: f["number"] = issue151 if pages: f["pages"] = pages.replace("-", "--")152 elif kind == "inproceedings":153 f["booktitle"] = bib_escape(cont)154 if pages: f["pages"] = pages.replace("-", "--")155 if publisher: f["publisher"] = bib_escape(publisher)156 elif kind == "incollection":157 f["booktitle"] = bib_escape(cont)158 if pages: f["pages"] = pages.replace("-", "--")159 if publisher: f["publisher"] = bib_escape(publisher)160 else:161 f["institution"] = bib_escape(publisher or cont)162 if "NBER" in c["container"]:163 f["type"] = "NBER Working Paper"164 f["number"] = re.sub(r"\D", "", c["container"])165 entry = "@%s{%s,\n%s\n}\n" % (kind, c["key"], ",\n".join(f" {k} = {{{v}}}" for k, v in f.items() if v))166 meta = {"key": c["key"], "authors": authors, "year": year, "title": title, "container": cont, "doi": doi,167 "verified_via": "Crossref works API (DOI lookup or query.bibliographic), JSON in docs/verification/"}168 return entry, meta169170171def arxiv_bib(c: dict, a: dict) -> tuple[str, dict]:172 authors = " and ".join(a["authors"])173 f = {"author": authors, "title": protect(bib_escape(a["title"])), "year": a["year"],174 "eprint": a["id"], "archivePrefix": "arXiv", "url": a["url"], "note": f"arXiv preprint {a['id']}"}175 if a.get("doi"):176 f["doi"] = a["doi"]177 entry = "@misc{%s,\n%s\n}\n" % (c["key"], ",\n".join(f" {k} = {{{v}}}" for k, v in f.items() if v))178 meta = {"key": c["key"], "authors": authors, "year": a["year"], "title": a["title"], "container": f"arXiv:{a['id']}",179 "doi": a.get("doi") or "", "verified_via": "arXiv API (export.arxiv.org), Atom in docs/verification/"}180 return entry, meta181182183def main() -> None:184 cands = list(csv.DictReader(open(ROOT / "docs" / "literature_candidates.csv", encoding="utf-8")))185 verified, unverified, bib = [], [], []186 for c in cands:187 try:188 if c["kind"] == "preprint":189 a = arxiv(c)190 if a:191 e, m = arxiv_bib(c, a); bib.append(e); m["topic"] = c["topic"]; verified.append(m)192 else:193 unverified.append({**c, "reason": "arXiv title mismatch or not found"})194 else:195 it = crossref(c)196 if it:197 e, m = crossref_bib(c, it); bib.append(e); m["topic"] = c["topic"]; verified.append(m)198 else:199 unverified.append({**c, "reason": "no Crossref record with title similarity >= 0.80 and year ±1"})200 except Exception as ex: # noqa: BLE001201 unverified.append({**c, "reason": f"error: {ex}"})202 print(f"{c['key']:34s} {'OK' if verified and verified[-1]['key']==c['key'] else '--'}", flush=True)203 time.sleep(0.6)204 (ROOT / "paper" / "references.bib").write_text("\n".join(bib), encoding="utf-8")205 with open(VER / "verified.csv", "w", newline="", encoding="utf-8") as fh:206 w = csv.DictWriter(fh, fieldnames=list(verified[0].keys())); w.writeheader(); w.writerows(verified)207 with open(VER / "unverified.csv", "w", newline="", encoding="utf-8") as fh:208 if unverified:209 w = csv.DictWriter(fh, fieldnames=list(unverified[0].keys())); w.writeheader(); w.writerows(unverified)210 print(f"\nverified {len(verified)} / {len(cands)}; unverified {len(unverified)}")211212213if __name__ == "__main__":214 sys.exit(main())215