1"""WP38 — build literature/references.bib from literature/references_verified.csv (verified entries ONLY).23Entry types follow the Crossref `type` (journal-article -> @article, book-chapter / handbook chapters -> @incollection,4book / monograph / edited-book -> @book, report (NBER) -> @techreport, posted-content -> @unpublished) and, for5URL-verified grey literature, the `type` given in manual_references.yaml (article/book/report). Titles are escaped6for LaTeX (no bare '&', no '$'); grey literature carries `url` + `note` with the verification date.7"""8from __future__ import annotations9import re, sys, datetime as dt10from pathlib import Path11import pandas as pd1213ROOT = Path(__file__).resolve().parents[2]; LIT = ROOT / "literature"1415def tex(s):16 if not isinstance(s, str): return ""17 s = s.replace("&", "&").replace("\\&", "&").replace("&", "\\&").replace("$", "\\$").replace("%", "\\%").replace("#", "\\#")18 s = s.replace("‐", "-").replace("—", "---").replace("–", "--")19 return s.strip()2021def protect_caps(title):22 """Keep acronyms / proper nouns capitalised under title-casing styles: wrap tokens with inner capitals or all-caps."""23 out = []24 for tok in title.split(" "):25 core = re.sub(r"[^A-Za-z]", "", tok)26 if len(core) > 1 and (core.isupper() or (core[0].isupper() and any(c.isupper() for c in core[1:])) or core in ("Quebec", "Québec", "Canada", "China", "Ireland", "eBay", "MercadoLibre", "Amazon", "Airbnb", "Google", "Pareto", "Balassa", "Internet", "Hollywood")):27 m = re.match(r"^(.*?)([.,:;?!)]*)$", tok)28 tok = "{" + m.group(1) + "}" + m.group(2)29 out.append(tok)30 return " ".join(out)3132def num(x):33 if pd.isna(x) or str(x) in ("nan", "None", ""): return None34 s = str(x); return s[:-2] if s.endswith(".0") else s3536def authors_of(r):37 a = r.authors if isinstance(r.authors, str) and r.authors.strip() else r.author38 a = str(a)39 if ";" in a: return " and ".join(x.strip() for x in a.split(";") if x.strip())40 return a4142def main():43 df = pd.read_csv(LIT / "references_verified.csv"); df = df[df.verified == True].copy()44 out = []45 for _, r in df.sort_values("key").iterrows():46 title = r.crossref_title if isinstance(r.crossref_title, str) and r.crossref_title else r.title47 title = re.sub(r"^Chapter \d+\.\s*", "", title)48 title = protect_caps(tex(title))49 au = authors_of(r); yr = num(r.pub_year) or num(r.year) or "n.d."50 typ = str(r.type) if isinstance(r.type, str) else "article"51 f = [f" author = {{{tex(au)}}}", f" title = {{{title}}}", f" year = {{{yr}}}"]52 doi = r.doi if isinstance(r.doi, str) and r.doi else None53 url = r.url if isinstance(r.url, str) and r.url else None54 cont = tex(r.container) if isinstance(r.container, str) else ""55 if typ in ("journal-article", "article"):56 kind = "article"; f.append(f" journal = {{{cont}}}")57 if num(r.volume): f.append(f" volume = {{{num(r.volume)}}}")58 if num(r.issue): f.append(f" number = {{{num(r.issue)}}}")59 if num(r.pages): f.append(f" pages = {{{num(r.pages).replace('--', '-').replace('-', '--')}}}")60 elif typ in ("book-chapter",) or (typ in ("book", "edited-book") and cont):61 kind = "incollection"; f.append(f" booktitle = {{{cont}}}")62 if isinstance(r.publisher, str) and r.publisher: f.append(f" publisher = {{{tex(r.publisher)}}}")63 if num(r.volume): f.append(f" volume = {{{num(r.volume)}}}")64 if num(r.pages): f.append(f" pages = {{{num(r.pages).replace('--', '-').replace('-', '--')}}}")65 elif typ in ("book", "monograph", "edited-book"):66 kind = "book"67 if isinstance(r.publisher, str) and r.publisher: f.append(f" publisher = {{{tex(r.publisher)}}}")68 elif typ == "report":69 kind = "techreport"70 inst = tex(r.publisher) if isinstance(r.publisher, str) and r.publisher else ("National Bureau of Economic Research" if doi and doi.startswith("10.3386/") else "")71 if inst: f.append(f" institution = {{{inst}}}")72 if doi and doi.startswith("10.3386/"): f.append(f" type = {{NBER Working Paper}}"); f.append(f" number = {{{doi.split('/w')[-1]}}}")73 elif typ == "posted-content":74 kind = "unpublished"; f.append(" note = {Working paper / preprint}")75 else:76 kind = "misc"77 if doi: f.append(f" doi = {{{doi}}}")78 if r.verification_method == "url:fetch" and url:79 f.append(f" url = {{{url}}}")80 f.append(f" note = {{Accessed {r.verified_date}}}")81 elif url and not doi:82 f.append(f" url = {{{url}}}")83 out.append(f"@{kind}{{{r.key},\n" + ",\n".join(f) + "\n}\n")84 head = (f"% WP38 bibliography — generated {dt.date.today().isoformat()} by src/literature/make_bib.py from literature/references_verified.csv.\n"85 "% Every DOI comes from a Crossref response cached in literature/crossref_cache/; URL entries were fetched on the stated date.\n"86 "% Unverified seeds (literature/unverified.csv) are deliberately absent and must not be cited.\n\n")87 (LIT / "references.bib").write_text(head + "\n".join(out))88 print(f"wrote {len(out)} entries to literature/references.bib")89 # sanity: parse back90 import bibtexparser91 txt = (LIT / "references.bib").read_text()92 if hasattr(bibtexparser, "parse_string"): # v293 db = bibtexparser.parse_string(txt); entries = [(e.key, e.fields_dict["title"].value) for e in db.entries]; failed = len(db.failed_blocks)94 else: # v195 db = bibtexparser.loads(txt); entries = [(e["ID"], e["title"]) for e in db.entries]; failed = 096 bad = [k for k, t in entries if "$" in t or re.search(r"(?<!\\)&", t)]97 print(f"bibtexparser: {len(entries)} entries parsed, {failed} failed blocks, bad titles: {bad}")9899if __name__ == "__main__":100 main()101