1"""Merge the verified literature parts into research/literature_matrix.csv and generate paper/references.bib.23Every entry is built from the verified CSV rows and cross-checked against the raw Crossref JSON saved in4research/verification/ (title similarity, year, DOI). Mismatches are reported, never silently fixed.5"""6from __future__ import annotations78import csv9import json10import re11import sys12import unicodedata13from difflib import SequenceMatcher14from pathlib import Path1516ROOT = Path(__file__).resolve().parents[1]17RES = ROOT / "research"18VER = RES / "verification"1920# B-side keys that duplicate an A-side DOI -> canonical key21ALIASES = {"fisher2003liquidity": "fisher2003controlling", "glower1998selling": "glower1998motivation",22 "han2024cashisking": "han2024cash", "reher2024mortgagecash": "reher2024mortgage"}2324BOOK_HINTS = ("MIT Press", "Princeton", "Cambridge University Press", "Oxford University Press", "Elsevier", "North-Holland")252627def norm(s: str) -> str:28 s = unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode()29 return re.sub(r"[^a-z0-9 ]", " ", s.lower()).strip()303132def tex_escape(s: str) -> str:33 return (s.replace("&", r"\&").replace("%", r"\%").replace("#", r"\#").replace("_", r"\_")34 .replace("$", r"\$"))353637def protect_caps(title: str) -> str:38 """Brace words with internal capitals / acronyms so aer.bst keeps them."""39 out = []40 for w in title.split(" "):41 core = re.sub(r"[^A-Za-z]", "", w)42 if len(core) > 1 and (core.isupper() or (core[0].isupper() and any(ch.isupper() for ch in core[1:]))):43 out.append("{" + w + "}")44 else:45 out.append(w)46 return " ".join(out)474849def load_parts():50 rows = {}51 order = []52 for part in ("A", "B"):53 with open(RES / f"lit_part_{part}.csv", newline="", encoding="utf-8") as fh:54 for r in csv.DictReader(fh):55 key = ALIASES.get(r["key"], r["key"])56 r = dict(r); r["key"] = key; r["part"] = part57 if key in rows:58 # keep A's row, merge B's overlap statement59 rows[key]["overlap_with_wp19"] += " | B: " + r["overlap_with_wp19"]60 rows[key]["part"] = "A+B"61 continue62 rows[key] = r; order.append(key)63 return [rows[k] for k in order]646566def crossref_json(key: str, part: str):67 for p in ("A", "B"):68 f = VER / f"{p}_{key}.crossref.json"69 if f.exists():70 return json.loads(f.read_text())71 for alias, canon in ALIASES.items():72 if canon == key:73 for p in ("A", "B"):74 f = VER / f"{p}_{alias}.crossref.json"75 if f.exists():76 return json.loads(f.read_text())77 return None787980def cr_item(j):81 if j is None:82 return None83 if "message" in j:84 m = j["message"]85 if "items" in m:86 return m["items"][0] if m["items"] else None87 return m88 if "items" in j:89 return j["items"][0] if j["items"] else None90 return j919293def check_against_crossref(row, item):94 issues = []95 if item is None:96 return ["no crossref json (evidence file only)"]97 t = " ".join(item.get("title", []) or [])98 sim = SequenceMatcher(None, norm(t), norm(row["title"])).ratio()99 if sim < 0.6:100 issues.append(f"title similarity {sim:.2f}: crossref='{t[:60]}'")101 yr = None102 for k in ("published-print", "published-online", "issued", "created"):103 dp = item.get(k, {}).get("date-parts", [[None]])104 if dp and dp[0] and dp[0][0]:105 yr = yr or dp[0][0]106 yrs = set()107 for k in ("published-print", "published-online", "issued", "created"):108 dp = item.get(k, {}).get("date-parts", [[None]])109 if dp and dp[0] and dp[0][0]:110 yrs.add(int(dp[0][0]))111 if yrs and int(row["year"]) not in yrs and abs(int(row["year"]) - min(yrs)) > 1:112 issues.append(f"year {row['year']} vs crossref {sorted(yrs)}")113 doi = (item.get("DOI") or "").lower()114 if doi and doi not in row["doi_url"].lower():115 issues.append(f"doi {doi} vs {row['doi_url']}")116 return issues117118119def parse_vp(vp: str):120 """'48(2): 641-664' -> volume, number, pages."""121 vol = num = pages = ""122 m = re.match(r"\s*([\w\-]+)\s*(?:\(([^)]*)\))?\s*[:,]?\s*(.*)$", vp or "")123 if m:124 vol, num, pages = m.group(1), m.group(2) or "", (m.group(3) or "").strip()125 pages = pages.replace("--", "-").replace("–", "-").replace("-", "--")126 return vol, num, pages127128129def to_bib(row, item):130 key = row["key"]; outlet = row["outlet"]; year = row["year"]131 authors = tex_escape(row["authors"])132 title = protect_caps(tex_escape(row["title"]))133 doi = row["doi_url"].replace("https://doi.org/", "") if "doi.org" in row["doi_url"] else ""134 url = "" if doi else row["doi_url"]135 typ = (item or {}).get("type", "")136 vol, num, pages = parse_vp(row.get("volume_pages", ""))137 lines = []138 if typ == "book" or (typ in ("monograph",) ) or any(h in outlet for h in BOOK_HINTS) and typ not in ("journal-article", "book-chapter"):139 lines.append(f"@book{{{key},")140 lines.append(f" author = {{{authors}}},"); lines.append(f" title = {{{title}}},")141 lines.append(f" publisher = {{{tex_escape(outlet)}}},"); lines.append(f" year = {{{year}}},")142 elif typ == "book-chapter" or "Handbook" in outlet:143 lines.append(f"@incollection{{{key},")144 lines.append(f" author = {{{authors}}},"); lines.append(f" title = {{{title}}},")145 lines.append(f" booktitle = {{{tex_escape(outlet)}}},"); lines.append(f" year = {{{year}}},")146 if vol: lines.append(f" volume = {{{vol}}},")147 if pages: lines.append(f" pages = {{{pages}}},")148 pub = ((item or {}).get("publisher") or "")149 if pub: lines.append(f" publisher = {{{tex_escape(pub)}}},")150 elif typ in ("posted-content", "report") or re.search(r"working paper|discussion paper|staff|NBER|SSRN|CEPR|mimeo", outlet, re.I):151 lines.append(f"@techreport{{{key},")152 lines.append(f" author = {{{authors}}},"); lines.append(f" title = {{{title}}},")153 lines.append(f" institution = {{{tex_escape(outlet)}}},"); lines.append(f" year = {{{year}}},")154 if vol: lines.append(f" number = {{{vol}}},")155 else:156 lines.append(f"@article{{{key},")157 lines.append(f" author = {{{authors}}},"); lines.append(f" title = {{{title}}},")158 lines.append(f" journal = {{{tex_escape(outlet)}}},"); lines.append(f" year = {{{year}}},")159 if vol: lines.append(f" volume = {{{vol}}},")160 if num: lines.append(f" number = {{{num}}},")161 if pages: lines.append(f" pages = {{{pages}}},")162 if doi: lines.append(f" doi = {{{doi}}},")163 if url: lines.append(f" url = {{{url}}},")164 lines[-1] = lines[-1].rstrip(",")165 lines.append("}")166 return "\n".join(lines)167168169def main():170 rows = load_parts()171 report = []172 bib = []173 seen_doi = {}174 with open(RES / "literature_matrix.csv", "w", newline="", encoding="utf-8") as fh:175 cols = ["key", "authors", "year", "title", "outlet", "volume_pages", "doi_url", "mechanism", "approach",176 "key_finding", "overlap_with_wp19", "gaps", "confidence", "verified_via", "part"]177 w = csv.DictWriter(fh, fieldnames=cols, quoting=csv.QUOTE_ALL, extrasaction="ignore")178 w.writeheader()179 for r in rows:180 item = cr_item(crossref_json(r["key"], r["part"]))181 issues = check_against_crossref(r, item)182 d = r["doi_url"].lower()183 if d in seen_doi:184 issues.append(f"duplicate DOI with {seen_doi[d]}")185 seen_doi[d] = r["key"]186 if issues:187 report.append((r["key"], issues))188 w.writerow(r)189 bib.append(to_bib(r, item))190 (ROOT / "paper" / "references.bib").write_text("\n\n".join(bib) + "\n", encoding="utf-8")191 (RES / "literature_merge_report.md").write_text(192 "# Literature merge report\n\n" + f"{len(rows)} references ({sum(r['part']=='A' for r in rows)} A, "193 f"{sum(r['part']=='B' for r in rows)} B, {sum(r['part']=='A+B' for r in rows)} both; aliases {ALIASES}).\n\n"194 + "## Cross-check issues (CSV vs raw Crossref JSON)\n\n"195 + ("\n".join(f"- `{k}`: " + "; ".join(v) for k, v in report) if report else "- none") + "\n")196 print(f"{len(rows)} references merged; {len(report)} rows with cross-check notes")197 for k, v in report:198 print(" ", k, "->", v)199200201if __name__ == "__main__":202 main()203