1"""WP38 — verify every seed reference against Crossref (public API, polite pool).23Self-contained (no src.common): ROOT = project root, simple stdout+file logging.4For each seed (author words, title, year) in literature/seed_references.yaml we query Crossref, keep the best5candidate only if title similarity >= 0.90, year within ±1 and the first author's family name matches.6Raw Crossref responses are cached in literature/crossref_cache/<key>.json; the number of HTTP calls made is appended7to literature/LOG.md. Unverified seeds go to literature/unverified.csv and must NOT be cited.8"""9from __future__ import annotations10import json, logging, re, sys, time, datetime as dt11from pathlib import Path12import requests, yaml, pandas as pd13from rapidfuzz import fuzz1415ROOT = Path(__file__).resolve().parents[2]16LIT = ROOT / "literature"; CACHE = LIT / "crossref_cache"; CACHE.mkdir(parents=True, exist_ok=True)17UA = "WP38-UQO-research/0.1 (mailto:simon-pierre.boucher@uqo.ca)"18API = "https://api.crossref.org/works"19SELECT = "DOI,title,author,issued,container-title,volume,issue,page,type,publisher,URL,ISBN"20TODAY = dt.date.today().isoformat()2122def get_logger(name):23 log = logging.getLogger(name)24 if not log.handlers:25 fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%Y-%m-%dT%H:%M:%S")26 sh = logging.StreamHandler(sys.stdout); sh.setFormatter(fmt); log.addHandler(sh)27 (ROOT / "logs").mkdir(exist_ok=True)28 fh = logging.FileHandler(ROOT / "logs" / "literature.log"); fh.setFormatter(fmt); log.addHandler(fh)29 log.setLevel(logging.INFO)30 return log31log = get_logger("crossref")32CALLS = {"crossref": 0}3334def norm(s): return re.sub(r"[^a-z0-9 ]", " ", (s or "").lower().replace("’", "'"))3536PUBLISHED = {"journal-article": 3, "book-chapter": 2, "book": 2, "monograph": 2, "edited-book": 2, "report": 0, "posted-content": 0}3738def query(key, author, title, year, refresh=False, filt=None):39 f = CACHE / (f"{key}.json" if not filt else f"{key}__{filt.replace(':', '-')}.json")40 if f.exists() and not refresh:41 return json.loads(f.read_text())["message"]["items"]42 params = {"query.bibliographic": f"{author} {title} {year}", "rows": 6, "select": SELECT}43 if filt: params["filter"] = filt44 r = requests.get(API, params=params, headers={"User-Agent": UA}, timeout=60); CALLS["crossref"] += 145 r.raise_for_status()46 js = r.json(); js["_wp38"] = {"fetched": dt.datetime.now(dt.timezone.utc).isoformat(), "url": r.url, "key": key}47 f.write_text(json.dumps(js, ensure_ascii=False, indent=1))48 time.sleep(0.35)49 return js["message"]["items"]5051def fam_of(it):52 return " ".join(f"{a.get('family') or ''} {a.get('given') or ''} {a.get('name') or ''}" for a in it.get("author", []) or []) # given included: Crossref sometimes swaps given/family5354def title_sim(a, b):55 """Token-set similarity, guarded against subset false positives (e.g. 'Reflections' vs a long title containing it):56 when one title has fewer than 60% of the other's tokens, fall back to the stricter token-sort ratio."""57 na, nb = norm(a).split(), norm(b).split()58 if min(len(na), len(nb)) < 0.6 * max(len(na), len(nb)):59 short, long_ = (na, nb) if len(na) <= len(nb) else (nb, na)60 if len(short) >= 2 and long_[:len(short)] == short: # main title without its subtitle (books, reports)61 return 1.062 return fuzz.token_sort_ratio(" ".join(na), " ".join(nb)) / 10063 return fuzz.token_set_ratio(" ".join(na), " ".join(nb)) / 1006465def rank(it, s):66 """Return (sim, published-priority, not-SSRN) for a candidate, or None if year/author checks fail."""67 t = " ".join(it.get("title") or [])68 if not t: return None69 sim = title_sim(t, s["title"])70 yr = (it.get("issued", {}).get("date-parts") or [[None]])[0][0]71 first_author = norm(s["author"].split()[0]) if s["author"] else ""72 fam = norm(fam_of(it))73 auth_ok = (not first_author) or (first_author in fam) or (not it.get("author")) # Crossref sometimes omits authors74 if not (yr and abs(int(yr) - int(s["year"])) <= int(s.get("year_tol", 1)) and auth_ok): return None75 ssrn = str(it.get("DOI", "")).startswith("10.2139/")76 return (sim, PUBLISHED.get(it.get("type"), 1), 0 if ssrn else 1)7778def best_match(items, s):79 """Among candidates with title similarity >= 0.90, prefer published versions (journal article / chapter / book)80 over NBER reports and SSRN postings; otherwise return the highest-similarity candidate."""81 scored = [(rank(it, s), it) for it in items]82 scored = [(r, it) for r, it in scored if r]83 if not scored: return None, 0.084 good = [(r, it) for r, it in scored if r[0] >= 0.90]85 if good:86 r, it = max(good, key=lambda x: (x[0][1], x[0][2], x[0][0]))87 return it, r[0]88 r, it = max(scored, key=lambda x: x[0][0])89 return it, r[0]9091def row_from(s, it, sim, method):92 yr = it["issued"]["date-parts"][0][0]93 return {**s, "verified": True, "doi": it.get("DOI"), "url": it.get("URL"), "crossref_title": " ".join(it.get("title") or []),94 "container": " ".join(it.get("container-title") or []), "volume": it.get("volume"), "issue": it.get("issue"),95 "pages": it.get("page"), "pub_year": yr, "type": it.get("type"), "publisher": it.get("publisher"),96 "authors": "; ".join(f"{a.get('family','')}, {a.get('given','')}" for a in it.get("author", []) or []),97 "title_sim": round(sim, 3), "verification_method": method, "verified_date": TODAY, "note": ""}9899def main(refresh=False):100 seeds = yaml.safe_load((LIT / "seed_references.yaml").read_text())101 rows = []102 for s in seeds:103 try:104 items = query(s["key"], s["author"], s["title"], s["year"], refresh)105 except Exception as e:106 rows.append({**s, "verified": False, "note": f"crossref error {type(e).__name__}"}); continue107 best, sim = best_match(items, s)108 if best and sim >= 0.90 and PUBLISHED.get(best.get("type"), 1) == 0:109 # working-paper version found: look for the published version with a type filter110 for filt in ("type:journal-article", "type:book-chapter"):111 try:112 alt, asim = best_match(query(s["key"], s["author"], s["title"], s["year"], refresh, filt), s)113 if alt and asim >= 0.90 and PUBLISHED.get(alt.get("type"), 1) > 0:114 best, sim = alt, asim; break115 except Exception as e:116 log.warning("%s: filtered query failed %s", s["key"], type(e).__name__)117 if best and sim >= 0.90:118 rows.append(row_from(s, best, sim, "crossref:query.bibliographic"))119 else:120 cand = " | ".join(f"{' '.join(it.get('title') or [])[:70]} ({(it.get('issued',{}).get('date-parts') or [[None]])[0][0]}, {it.get('DOI')})" for it in items[:3])121 rows.append({**s, "verified": False, "note": f"no confident Crossref match (best sim {sim:.2f}); candidates: {cand}"})122 df = pd.DataFrame(rows)123 cols = ["key", "stream", "author", "title", "year", "why_needed", "verified", "doi", "url", "crossref_title", "container", "volume", "issue",124 "pages", "pub_year", "type", "publisher", "authors", "title_sim", "verification_method", "verified_date", "note"]125 for c in cols:126 if c not in df.columns: df[c] = None127 df = df[cols]128 df.to_csv(LIT / "references_verified.csv", index=False)129 df[~df.verified.astype(bool)].to_csv(LIT / "unverified.csv", index=False)130 nv = int(df.verified.astype(bool).sum())131 log.info("verified %d / %d ; unverified: %s", nv, len(df), df.loc[~df.verified.astype(bool), "key"].tolist())132 with open(LIT / "LOG.md", "a") as f:133 f.write(f"- {dt.datetime.now().isoformat(timespec='seconds')} verify_refs.py: {len(seeds)} seeds, {CALLS['crossref']} Crossref HTTP calls "134 f"(rest served from crossref_cache/), {nv} verified, {len(df)-nv} unverified\n")135136if __name__ == "__main__":137 main(refresh="--refresh" in sys.argv)138