"""Verify every intended reference against Crossref (public API, polite pool) and build the verified bibliography. For each seed (author words, title, year) we query Crossref, keep the best candidate only if the title similarity is high and the year matches (±1), and record DOI, container title, volume, pages. Unverified entries are listed in literature/unverified.csv and are NOT cited in the paper. """ import time, re, json, requests, yaml, pandas as pd from rapidfuzz import fuzz from src.common import ROOT, get_logger, ledger log = get_logger("crossref") UA = "WP37-UQO-research/0.1 (mailto:simon-pierre.boucher@uqo.ca)" LIT = ROOT / "literature" def norm(s): return re.sub(r"[^a-z0-9 ]", " ", (s or "").lower()) def query(author, title, year): params = {"query.bibliographic": f"{author} {title} {year}", "rows": 5, "select": "DOI,title,author,issued,container-title,volume,issue,page,type,publisher"} r = requests.get("https://api.crossref.org/works", params=params, headers={"User-Agent": UA}, timeout=60) r.raise_for_status() return r.json()["message"]["items"] def main(): seeds = yaml.safe_load((LIT / "seed_references.yaml").read_text()) rows = [] for s in seeds: if not s["author"]: rows.append({**s, "verified": False, "note": "placeholder topic query, needs a specific paper"}); continue try: items = query(s["author"], s["title"], s["year"]) except Exception as e: rows.append({**s, "verified": False, "note": f"crossref error {type(e).__name__}"}); continue best, best_sim = None, 0 for it in items: t = " ".join(it.get("title") or []) sim = fuzz.token_set_ratio(norm(t), norm(s["title"])) / 100 yr = (it.get("issued", {}).get("date-parts") or [[None]])[0][0] fam = " ".join(a.get("family", "") for a in it.get("author", []) or []) auth_ok = all(norm(w) in norm(fam) for w in s["author"].split()[:1]) if sim > best_sim and yr and abs(int(yr) - int(s["year"])) <= 1 and auth_ok: best, best_sim = it, sim if best and best_sim >= 0.90: yr = best["issued"]["date-parts"][0][0] rows.append({**s, "verified": True, "doi": best["DOI"], "crossref_title": " ".join(best.get("title") or []), "journal": " ".join(best.get("container-title") or []), "volume": best.get("volume"), "issue": best.get("issue"), "pages": best.get("page"), "pub_year": yr, "type": best.get("type"), "publisher": best.get("publisher"), "authors": "; ".join(f"{a.get('family','')}, {a.get('given','')}" for a in best.get("author", []) or []), "title_sim": round(best_sim, 3), "note": ""}) else: rows.append({**s, "verified": False, "note": f"no confident Crossref match (best sim {best_sim:.2f})"}) time.sleep(0.3) df = pd.DataFrame(rows) df.to_csv(LIT / "references_verified.csv", index=False) df[~df.verified].to_csv(LIT / "unverified.csv", index=False) log.info("verified %d / %d ; unverified: %s", df.verified.sum(), len(df), df.loc[~df.verified, "key"].tolist()) ledger("2", "verify_refs", f"{int(df.verified.sum())}/{len(df)} verified via Crossref") if __name__ == "__main__": main()