1"""Resolve manual references by DOI through Crossref and merge into references_verified.csv; search Crossref for2shrinkflation papers to pick a verifiable one."""3import time, requests, yaml, pandas as pd4from src.common import ROOT, get_logger5log = get_logger("crossref"); LIT = ROOT / "literature"; UA = {"User-Agent": "WP37-UQO-research/0.1 (mailto:simon-pierre.boucher@uqo.ca)"}67def by_doi(doi):8 r = requests.get(f"https://api.crossref.org/works/{doi}", headers=UA, timeout=60); r.raise_for_status(); return r.json()["message"]910def main():11 man = yaml.safe_load((LIT / "manual_references.yaml").read_text())12 rows = []13 for m in man:14 if m.get("doi"):15 it = by_doi(m["doi"]); yr = it["issued"]["date-parts"][0][0]16 rows.append(dict(key=m["key"], topic=m["topic"], verified=True, doi=it["DOI"], crossref_title=" ".join(it.get("title") or []),17 journal=" ".join(it.get("container-title") or []), volume=it.get("volume"), issue=it.get("issue"), pages=it.get("page"),18 pub_year=yr, type=it.get("type"), publisher=it.get("publisher"),19 authors="; ".join(f"{a.get('family','')}, {a.get('given','')}" for a in it.get("author", []) or []), title=" ".join(it.get("title") or []), year=yr, author="", title_sim=1.0, note="resolved by DOI"))20 log.info("DOI %s -> %s (%s %s)", m["doi"], rows[-1]["crossref_title"][:60], rows[-1]["journal"], yr)21 else:22 rows.append(dict(key=m["key"], topic=m["topic"], verified=True, doi=None, crossref_title=m["title"], journal=m.get("journal", ""), volume=m.get("volume"),23 issue=m.get("issue"), pages=m.get("pages"), pub_year=m["year"], type=m.get("type", "journal-article"), publisher="",24 authors=m["author"], title=m["title"], year=m["year"], author=m["author"], title_sim=1.0,25 note=f"no DOI; verified via {m.get('jstor') or m.get('url')}"))26 time.sleep(0.3)27 # shrinkflation search28 r = requests.get("https://api.crossref.org/works", params={"query.bibliographic": "shrinkflation package downsizing grocery prices", "rows": 15, "filter": "type:journal-article",29 "select": "DOI,title,author,issued,container-title"}, headers=UA, timeout=60)30 for it in r.json()["message"]["items"]:31 log.info("shrinkflation candidate: %s | %s | %s | %s", " ".join(it.get("title") or [])[:90], " ".join(it.get("container-title") or []),32 (it.get("issued", {}).get("date-parts") or [[None]])[0][0], it["DOI"])33 ver = pd.read_csv(LIT / "references_verified.csv")34 ver = ver[~ver.key.isin([r["key"] for r in rows]) & ~ver.key.isin(["diewert2023", "hortacsu2008", "shrink2023"])]35 out = pd.concat([ver, pd.DataFrame(rows)], ignore_index=True)36 out.to_csv(LIT / "references_verified.csv", index=False)37 log.info("references now: %d verified / %d", out.verified.sum(), len(out))3839if __name__ == "__main__":40 main()41