1"""Build paper/references.bib from verified DOIs (Crossref transform to BibTeX) plus hand-written entries for books2and DOI-less items listed in research/bib_overrides.csv. Only keys in research/bib_keys_used.txt are emitted when that3file exists (keeps the bibliography to cited works). Usage: python -m wp35.validation.build_bib4"""5from __future__ import annotations6import csv, re, sys, time7from pathlib import Path8import requests910ROOT = Path(__file__).resolve().parents[3]11UA = {"User-Agent": "WP35-UQO-bib/0.1 (mailto:simon-pierre.boucher@uqo.ca)", "Accept": "application/x-bibtex"}1213MANUAL = {14 "greif1993": "@article{greif1993,\n author = {Greif, Avner},\n title = {Contract Enforceability and Economic Institutions in Early Trade: The {M}aghribi Traders' Coalition},\n journal = {American Economic Review},\n volume = {83},\n number = {3},\n pages = {525--548},\n year = {1993}\n}",15 "hayek1945": "@article{hayek1945,\n author = {Hayek, Friedrich A.},\n title = {The Use of Knowledge in Society},\n journal = {American Economic Review},\n volume = {35},\n number = {4},\n pages = {519--530},\n year = {1945}\n}",16 "simon1962": "@article{simon1962,\n author = {Simon, Herbert A.},\n title = {The Architecture of Complexity},\n journal = {Proceedings of the American Philosophical Society},\n volume = {106},\n number = {6},\n pages = {467--482},\n year = {1962}\n}",17 "johnson1982": "@incollection{johnson1982,\n author = {Johnson, Gregory A.},\n title = {Organizational Structure and Scalar Stress},\n booktitle = {Theory and Explanation in Archaeology: The Southampton Conference},\n editor = {Renfrew, Colin and Rowlands, Michael J. and Segraves, Barbara Abbott},\n publisher = {Academic Press},\n address = {New York},\n pages = {389--421},\n year = {1982}\n}",18 "turchin2009gavrilets": "@article{turchin2009gavrilets,\n author = {Turchin, Peter and Gavrilets, Sergey},\n title = {Evolution of Complex Hierarchical Societies},\n journal = {Social Evolution \\& History},\n volume = {8},\n number = {2},\n pages = {167--198},\n year = {2009}\n}",19 "marschak1972": "@book{marschak1972,\n author = {Marschak, Jacob and Radner, Roy},\n title = {Economic Theory of Teams},\n publisher = {Yale University Press},\n address = {New Haven},\n year = {1972}\n}",20 "schelling1960": "@book{schelling1960,\n author = {Schelling, Thomas C.},\n title = {The Strategy of Conflict},\n publisher = {Harvard University Press},\n address = {Cambridge, MA},\n year = {1960}\n}",21 "boyd1985": "@book{boyd1985,\n author = {Boyd, Robert and Richerson, Peter J.},\n title = {Culture and the Evolutionary Process},\n publisher = {University of Chicago Press},\n address = {Chicago},\n year = {1985}\n}",22 "yates1989": "@book{yates1989,\n author = {Yates, JoAnne},\n title = {Control through Communication: The Rise of System in {A}merican Management},\n publisher = {Johns Hopkins University Press},\n address = {Baltimore},\n year = {1989}\n}",23 "chandler1977": "@book{chandler1977,\n author = {Chandler, Alfred D.},\n title = {The Visible Hand: The Managerial Revolution in {A}merican Business},\n publisher = {Harvard University Press},\n address = {Cambridge, MA},\n year = {1977}\n}",24 "eisenstadt1963": "@book{eisenstadt1963,\n author = {Eisenstadt, Shmuel N.},\n title = {The Political Systems of Empires},\n publisher = {Free Press},\n address = {New York},\n year = {1963}\n}",25 "weber1922": "@book{weber1922,\n author = {Weber, Max},\n title = {Economy and Society: An Outline of Interpretive Sociology},\n publisher = {University of California Press},\n address = {Berkeley},\n year = {1978},\n note = {Edited by G. Roth and C. Wittich; first published 1922}\n}",26 "turchin2016": "@book{turchin2016,\n author = {Turchin, Peter},\n title = {Ultrasociety: How 10,000 Years of War Made Humans the Greatest Cooperators on Earth},\n publisher = {Beresta Books},\n address = {Chaplin, CT},\n year = {2016}\n}",27 "turchin2023": "@book{turchin2023,\n author = {Turchin, Peter},\n title = {End Times: Elites, Counter-Elites, and the Path of Political Disintegration},\n publisher = {Penguin Press},\n address = {New York},\n year = {2023}\n}",28 "hidalgo2015": "@book{hidalgo2015,\n author = {Hidalgo, C{\\'e}sar},\n title = {Why Information Grows: The Evolution of Order, from Atoms to Economies},\n publisher = {Basic Books},\n address = {New York},\n year = {2015}\n}",29 "fukuyama2011": "@book{fukuyama2011,\n author = {Fukuyama, Francis},\n title = {The Origins of Political Order: From Prehuman Times to the {F}rench Revolution},\n publisher = {Farrar, Straus and Giroux},\n address = {New York},\n year = {2011}\n}",30 "graeber2021": "@book{graeber2021,\n author = {Graeber, David and Wengrow, David},\n title = {The Dawn of Everything: A New History of Humanity},\n publisher = {Farrar, Straus and Giroux},\n address = {New York},\n year = {2021}\n}",31 "koyama2022": "@book{koyama2022,\n author = {Koyama, Mark and Rubin, Jared},\n title = {How the World Became Rich: The Historical Origins of Economic Growth},\n publisher = {Polity},\n address = {Cambridge},\n year = {2022}\n}",32 "arrow1974": "@book{arrow1974,\n author = {Arrow, Kenneth J.},\n title = {The Limits of Organization},\n publisher = {W. W. Norton},\n address = {New York},\n year = {1974}\n}",33 "williamson1985": "@book{williamson1985,\n author = {Williamson, Oliver E.},\n title = {The Economic Institutions of Capitalism},\n publisher = {Free Press},\n address = {New York},\n year = {1985}\n}",34 "tainter1988": "@book{tainter1988,\n author = {Tainter, Joseph A.},\n title = {The Collapse of Complex Societies},\n publisher = {Cambridge University Press},\n address = {Cambridge},\n year = {1988}\n}",35 "scott1998": "@book{scott1998,\n author = {Scott, James C.},\n title = {Seeing Like a State: How Certain Schemes to Improve the Human Condition Have Failed},\n publisher = {Yale University Press},\n address = {New Haven},\n year = {1998}\n}",36 "goldstone1991": "@book{goldstone1991,\n author = {Goldstone, Jack A.},\n title = {Revolution and Rebellion in the Early Modern World},\n publisher = {University of California Press},\n address = {Berkeley},\n year = {1991}\n}",37 "tilly1992": "@book{tilly1992,\n author = {Tilly, Charles},\n title = {Coercion, Capital, and {E}uropean States, {AD} 990--1992},\n publisher = {Blackwell},\n address = {Oxford},\n year = {1992}\n}",38 "beniger1986": "@book{beniger1986,\n author = {Beniger, James R.},\n title = {The Control Revolution: Technological and Economic Origins of the Information Society},\n publisher = {Harvard University Press},\n address = {Cambridge, MA},\n year = {1986}\n}",39 "innis1950": "@book{innis1950,\n author = {Innis, Harold A.},\n title = {Empire and Communications},\n publisher = {Oxford University Press},\n address = {Oxford},\n year = {1950}\n}",40 "lewis1969": "@book{lewis1969,\n author = {Lewis, David K.},\n title = {Convention: A Philosophical Study},\n publisher = {Harvard University Press},\n address = {Cambridge, MA},\n year = {1969}\n}",41 "johnson1978": "@incollection{johnson1978,\n author = {Johnson, Gregory A.},\n title = {Information Sources and the Development of Decision-Making Organizations},\n booktitle = {Social Archeology: Beyond Subsistence and Dating},\n editor = {Redman, Charles L. and others},\n publisher = {Academic Press},\n address = {New York},\n pages = {87--112},\n year = {1978}\n}",42 "rubinstein1989": "@article{rubinstein1989,\n author = {Rubinstein, Ariel},\n title = {The Electronic Mail Game: Strategic Behavior under ``Almost Common Knowledge''},\n journal = {American Economic Review},\n volume = {79},\n number = {3},\n pages = {385--391},\n year = {1989}\n}",43 "chwe2001": "@book{chwe2001,\n author = {Chwe, Michael Suk-Young},\n title = {Rational Ritual: Culture, Coordination, and Common Knowledge},\n publisher = {Princeton University Press},\n address = {Princeton},\n year = {2001}\n}",44 "seshat2026": "@misc{seshat2026,\n author = {{Seshat: Global History Databank}},\n title = {Polaris-2026 data release (repository build\\_polaris\\_dataset, commit 55ca5fbc2b)},\n year = {2026},\n howpublished = {\\url{https://seshatdatabank.info/data/}},\n note = {Accessed 2026-11-01}\n}",45 "vdem2026": "@misc{vdem2026,\n author = {Coppedge, Michael and Gerring, John and Knutsen, Carl Henrik and Lindberg, Staffan I. and Teorell, Jan and others},\n title = {V-Dem Country-Year Dataset v16},\n year = {2026},\n howpublished = {Varieties of Democracy (V-Dem) Project, \\url{https://www.v-dem.net/}},\n note = {Obtained from the official vdemdata R package (commit f4dd26922e, 2026-03-17); accessed 2026-11-01}\n}",46 "glottolog2026": "@misc{glottolog2026,\n author = {Hammarstr{\\\"o}m, Harald and Forkel, Robert and Haspelmath, Martin and Bank, Sebastian},\n title = {Glottolog 5.3},\n year = {2026},\n howpublished = {Max Planck Institute for Evolutionary Anthropology, \\url{https://glottolog.org}},\n note = {CLDF languages table, accessed 2026-11-01}\n}",47}484950def crossref_bibtex(doi: str) -> str | None:51 r = requests.get(f"https://api.crossref.org/works/{doi}/transform/application/x-bibtex", headers=UA, timeout=60)52 return r.text if r.status_code == 200 else None535455def rekey(bib: str, key: str) -> str:56 return re.sub(r"^(@\w+\{)[^,]*,", lambda m: m.group(1) + key + ",", bib.strip(), count=1, flags=re.M)575859def main():60 cands = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/reference_candidates.csv"), delimiter="|") if r["type"] != "skip"}61 prior = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/prior_art.csv"), delimiter="|")}62 over = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/bib_overrides.csv"), delimiter="|")}63 used_f = ROOT / "research/bib_keys_used.txt"64 used = set(used_f.read_text().split()) if used_f.exists() else None65 keys = sorted(set(cands) | set(over) | set(MANUAL))66 out, log = [], []67 for k in keys:68 if used is not None and k not in used:69 continue70 o = over.get(k)71 if o and o["doi"] == "drop":72 continue73 if o and o["doi"] == "manual" or (k in MANUAL and not (o and o["doi"] not in ("manual", "drop"))):74 if k in MANUAL:75 out.append(MANUAL[k]); log.append((k, "manual")); continue76 log.append((k, "MISSING_MANUAL_ENTRY")); continue77 doi = o["doi"] if o else (prior[k]["doi"] if prior.get(k, {}).get("status") == "VERIFIED_CROSSREF" else None)78 if not doi:79 log.append((k, "NO_DOI")); continue80 b = crossref_bibtex(doi); time.sleep(0.2)81 if b:82 b = rekey(b, k)83 b = b.replace("{\\&}amp;", "\\&").replace("&", "\\&")84 out.append(b); log.append((k, f"crossref:{doi}"))85 else:86 log.append((k, f"CROSSREF_FAIL:{doi}"))87 (ROOT / "paper/references.bib").write_text("% Generated by src/wp35/validation/build_bib.py from Crossref (by DOI) and verified manual entries.\n\n" + "\n\n".join(out) + "\n")88 (ROOT / "research/bib_build_log.csv").write_text("key,source\n" + "\n".join(f"{k},{s}" for k, s in log))89 bad = [l for l in log if "FAIL" in l[1] or "MISSING" in l[1] or "NO_DOI" in l[1]]90 print(f"{len(out)} entries written; problems: {bad}")919293if __name__ == "__main__":94 main()95