1"""Build paper/references.bib from verified DOIs (Crossref transform to BibTeX) plus hand-written entries for books2and DOI-less items listed in research/bib_overrides.csv. Only keys in research/bib_keys_used.txt are emitted when that3file exists (keeps the bibliography to cited works). Usage: python -m wp35.validation.build_bib4"""5from __future__ import annotations6import csv, re, sys, time7from pathlib import Path8import requests910ROOT = Path(__file__).resolve().parents[3]11UA = {"User-Agent": "WP35-UQO-bib/0.1 (mailto:simon-pierre.boucher@uqo.ca)", "Accept": "application/x-bibtex"}1213MANUAL = {14 "greif1993": "@article{greif1993,\n author = {Greif, Avner},\n title = {Contract Enforceability and Economic Institutions in Early Trade: The {M}aghribi Traders' Coalition},\n journal = {American Economic Review},\n volume = {83},\n number = {3},\n pages = {525--548},\n year = {1993}\n}",15 "hayek1945": "@article{hayek1945,\n author = {Hayek, Friedrich A.},\n title = {The Use of Knowledge in Society},\n journal = {American Economic Review},\n volume = {35},\n number = {4},\n pages = {519--530},\n year = {1945}\n}",16 "simon1962": "@article{simon1962,\n author = {Simon, Herbert A.},\n title = {The Architecture of Complexity},\n journal = {Proceedings of the American Philosophical Society},\n volume = {106},\n number = {6},\n pages = {467--482},\n year = {1962}\n}",17 "johnson1982": "@incollection{johnson1982,\n author = {Johnson, Gregory A.},\n title = {Organizational Structure and Scalar Stress},\n booktitle = {Theory and Explanation in Archaeology: The Southampton Conference},\n editor = {Renfrew, Colin and Rowlands, Michael J. and Segraves, Barbara Abbott},\n publisher = {Academic Press},\n address = {New York},\n pages = {389--421},\n year = {1982}\n}",18 "turchin2009gavrilets": "@article{turchin2009gavrilets,\n author = {Turchin, Peter and Gavrilets, Sergey},\n title = {Evolution of Complex Hierarchical Societies},\n journal = {Social Evolution \\& History},\n volume = {8},\n number = {2},\n pages = {167--198},\n year = {2009}\n}",19 "marschak1972": "@book{marschak1972,\n author = {Marschak, Jacob and Radner, Roy},\n title = {Economic Theory of Teams},\n publisher = {Yale University Press},\n address = {New Haven},\n year = {1972}\n}",20 "schelling1960": "@book{schelling1960,\n author = {Schelling, Thomas C.},\n title = {The Strategy of Conflict},\n publisher = {Harvard University Press},\n address = {Cambridge, MA},\n year = {1960}\n}",21 "boyd1985": "@book{boyd1985,\n author = {Boyd, Robert and Richerson, Peter J.},\n title = {Culture and the Evolutionary Process},\n publisher = {University of Chicago Press},\n address = {Chicago},\n year = {1985}\n}",22 "yates1989": "@book{yates1989,\n author = {Yates, JoAnne},\n title = {Control through Communication: The Rise of System in {A}merican Management},\n publisher = {Johns Hopkins University Press},\n address = {Baltimore},\n year = {1989}\n}",23 "chandler1977": "@book{chandler1977,\n author = {Chandler, Alfred D.},\n title = {The Visible Hand: The Managerial Revolution in {A}merican Business},\n publisher = {Harvard University Press},\n address = {Cambridge, MA},\n year = {1977}\n}",24 "eisenstadt1963": "@book{eisenstadt1963,\n author = {Eisenstadt, Shmuel N.},\n title = {The Political Systems of Empires},\n publisher = {Free Press},\n address = {New York},\n year = {1963}\n}",25 "weber1922": "@book{weber1922,\n author = {Weber, Max},\n title = {Economy and Society: An Outline of Interpretive Sociology},\n publisher = {University of California Press},\n address = {Berkeley},\n year = {1978},\n note = {Edited by G. Roth and C. Wittich; first published 1922}\n}",26 "turchin2016": "@book{turchin2016,\n author = {Turchin, Peter},\n title = {Ultrasociety: How 10,000 Years of War Made Humans the Greatest Cooperators on Earth},\n publisher = {Beresta Books},\n address = {Chaplin, CT},\n year = {2016}\n}",27 "turchin2023": "@book{turchin2023,\n author = {Turchin, Peter},\n title = {End Times: Elites, Counter-Elites, and the Path of Political Disintegration},\n publisher = {Penguin Press},\n address = {New York},\n year = {2023}\n}",28 "hidalgo2015": "@book{hidalgo2015,\n author = {Hidalgo, C{\\'e}sar},\n title = {Why Information Grows: The Evolution of Order, from Atoms to Economies},\n publisher = {Basic Books},\n address = {New York},\n year = {2015}\n}",29 "fukuyama2011": "@book{fukuyama2011,\n author = {Fukuyama, Francis},\n title = {The Origins of Political Order: From Prehuman Times to the {F}rench Revolution},\n publisher = {Farrar, Straus and Giroux},\n address = {New York},\n year = {2011}\n}",30 "graeber2021": "@book{graeber2021,\n author = {Graeber, David and Wengrow, David},\n title = {The Dawn of Everything: A New History of Humanity},\n publisher = {Farrar, Straus and Giroux},\n address = {New York},\n year = {2021}\n}",31 "koyama2022": "@book{koyama2022,\n author = {Koyama, Mark and Rubin, Jared},\n title = {How the World Became Rich: The Historical Origins of Economic Growth},\n publisher = {Polity},\n address = {Cambridge},\n year = {2022}\n}",32 "arrow1974": "@book{arrow1974,\n author = {Arrow, Kenneth J.},\n title = {The Limits of Organization},\n publisher = {W. W. Norton},\n address = {New York},\n year = {1974}\n}",33 "williamson1985": "@book{williamson1985,\n author = {Williamson, Oliver E.},\n title = {The Economic Institutions of Capitalism},\n publisher = {Free Press},\n address = {New York},\n year = {1985}\n}",34 "tainter1988": "@book{tainter1988,\n author = {Tainter, Joseph A.},\n title = {The Collapse of Complex Societies},\n publisher = {Cambridge University Press},\n address = {Cambridge},\n year = {1988}\n}",35 "scott1998": "@book{scott1998,\n author = {Scott, James C.},\n title = {Seeing Like a State: How Certain Schemes to Improve the Human Condition Have Failed},\n publisher = {Yale University Press},\n address = {New Haven},\n year = {1998}\n}",36 "goldstone1991": "@book{goldstone1991,\n author = {Goldstone, Jack A.},\n title = {Revolution and Rebellion in the Early Modern World},\n publisher = {University of California Press},\n address = {Berkeley},\n year = {1991}\n}",37 "tilly1992": "@book{tilly1992,\n author = {Tilly, Charles},\n title = {Coercion, Capital, and {E}uropean States, {AD} 990--1992},\n publisher = {Blackwell},\n address = {Oxford},\n year = {1992}\n}",38 "beniger1986": "@book{beniger1986,\n author = {Beniger, James R.},\n title = {The Control Revolution: Technological and Economic Origins of the Information Society},\n publisher = {Harvard University Press},\n address = {Cambridge, MA},\n year = {1986}\n}",39 "innis1950": "@book{innis1950,\n author = {Innis, Harold A.},\n title = {Empire and Communications},\n publisher = {Oxford University Press},\n address = {Oxford},\n year = {1950}\n}",40 "lewis1969": "@book{lewis1969,\n author = {Lewis, David K.},\n title = {Convention: A Philosophical Study},\n publisher = {Harvard University Press},\n address = {Cambridge, MA},\n year = {1969}\n}",41 "johnson1978": "@incollection{johnson1978,\n author = {Johnson, Gregory A.},\n title = {Information Sources and the Development of Decision-Making Organizations},\n booktitle = {Social Archeology: Beyond Subsistence and Dating},\n editor = {Redman, Charles L. and others},\n publisher = {Academic Press},\n address = {New York},\n pages = {87--112},\n year = {1978}\n}",42 "rubinstein1989": "@article{rubinstein1989,\n author = {Rubinstein, Ariel},\n title = {The Electronic Mail Game: Strategic Behavior under ``Almost Common Knowledge''},\n journal = {American Economic Review},\n volume = {79},\n number = {3},\n pages = {385--391},\n year = {1989}\n}",43 "hoyer2024": "@unpublished{hoyer2024,\n author = {Hoyer, Daniel and Holder, Samantha and Bennett, James S. and Fran{\\c{c}}ois, Pieter and Whitehouse, Harvey and Covey, Alan and Feinman, Gary and Korotayev, Andrey and Vustiuzhanin, Vadim and Preiser-Kapeller, Johannes and Bard, Kathryn and Levine, Jenny and Reddish, Jennifer and Orlandi, Georg and Ainsworth, Rachel and Turchin, Peter},\n title = {All Crises are Unhappy in their Own Way: The Role of Societal Instability in Shaping the Past},\n note = {SocArXiv preprint, DOI 10.31235/osf.io/rk4gd; no journal version found as of November 2026},\n year = {2024}\n}",44 "pemstein2018": "@techreport{pemstein2018,\n author = {Pemstein, Daniel and Marquardt, Kyle L. and Tzelgov, Eitan and Wang, Yi-ting and Medzihorsky, Juraj and Krusell, Joshua and Miri, Farhad and von Roemer, Johannes},\n title = {The V-Dem Measurement Model: Latent Variable Analysis for Cross-National and Cross-Temporal Expert-Coded Data},\n institution = {Varieties of Democracy Institute},\n type = {V-Dem Working Paper},\n number = {21},\n year = {2018}\n}",45 "slingerland2020": "@unpublished{slingerland2020,\n author = {Slingerland, Edward and Monroe, M. Willis and Sullivan, Brenton and Walsh, Robyn Faith and Veidlinger, Daniel and Alderson-Day, Ben and others},\n title = {Historians Respond to Whitehouse et al. (2019), ``Complex Societies Precede Moralizing Gods Throughout World History''},\n note = {PsyArXiv preprint, DOI 10.31234/osf.io/2amjz; published in Journal of Cognitive Historiography},\n year = {2020}\n}",46 "kohler2018": "@book{kohler2018,\n editor = {Kohler, Timothy A. and Smith, Michael E.},\n title = {Ten Thousand Years of Inequality: The Archaeology of Wealth Differences},\n publisher = {University of Arizona Press},\n address = {Tucson},\n year = {2018}\n}",47 "chwe2001": "@book{chwe2001,\n author = {Chwe, Michael Suk-Young},\n title = {Rational Ritual: Culture, Coordination, and Common Knowledge},\n publisher = {Princeton University Press},\n address = {Princeton},\n year = {2001}\n}",48 "seshat2026": "@misc{seshat2026,\n author = {{Seshat: Global History Databank}},\n title = {Polaris-2026 data release (repository build\\_polaris\\_dataset, commit 55ca5fbc2b)},\n year = {2026},\n howpublished = {\\url{https://seshatdatabank.info/data/}},\n note = {Accessed 2026-11-01}\n}",49 "vdem2026": "@misc{vdem2026,\n author = {Coppedge, Michael and Gerring, John and Knutsen, Carl Henrik and Lindberg, Staffan I. and Teorell, Jan and others},\n title = {V-Dem Country-Year Dataset v16},\n year = {2026},\n howpublished = {Varieties of Democracy (V-Dem) Project, \\url{https://www.v-dem.net/}},\n note = {Obtained from the official vdemdata R package (commit f4dd26922e, 2026-03-17); accessed 2026-11-01}\n}",50 "glottolog2026": "@misc{glottolog2026,\n author = {Hammarstr{\\\"o}m, Harald and Forkel, Robert and Haspelmath, Martin and Bank, Sebastian},\n title = {Glottolog 5.3},\n year = {2026},\n howpublished = {Max Planck Institute for Evolutionary Anthropology, \\url{https://glottolog.org}},\n note = {CLDF languages table, accessed 2026-11-01}\n}",51}525354def crossref_bibtex(doi: str) -> str | None:55 r = requests.get(f"https://api.crossref.org/works/{doi}/transform/application/x-bibtex", headers=UA, timeout=60)56 return r.content.decode("utf-8", errors="replace") if r.status_code == 200 else None575859def rekey(bib: str, key: str) -> str:60 return re.sub(r"^(@\w+\{)[^,]*,", lambda m: m.group(1) + key + ",", bib.strip(), count=1, flags=re.M)616263def main():64 cands = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/reference_candidates.csv"), delimiter="|") if r["type"] != "skip"}65 prior = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/prior_art.csv"), delimiter="|")}66 over = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/bib_overrides.csv"), delimiter="|")}67 used_f = ROOT / "research/bib_keys_used.txt"68 used = set(used_f.read_text().split()) if used_f.exists() else None69 keys = sorted(set(cands) | set(over) | set(MANUAL))70 out, log = [], []71 for k in keys:72 if used is not None and k not in used:73 continue74 o = over.get(k)75 if o and o["doi"] == "drop":76 continue77 if (o and o["doi"] == "manual") or k in MANUAL:78 if k in MANUAL:79 out.append(MANUAL[k]); log.append((k, "manual")); continue80 log.append((k, "MISSING_MANUAL_ENTRY")); continue81 doi = o["doi"] if o else (prior[k]["doi"] if prior.get(k, {}).get("status") == "VERIFIED_CROSSREF" else None)82 if not doi:83 log.append((k, "NO_DOI")); continue84 b = crossref_bibtex(doi); time.sleep(0.2)85 if b:86 b = rekey(b, k)87 b = b.replace("{\\&}amp;", "\\&").replace("&", "\\&")88 b = re.sub(r",\s*month\s*=\s*[^,}]*", "", b) # aer.bst lacks month strings89 b = b.replace("\ufffd", " ")90 b = re.sub(r"(title|number|pages|journal|booktitle|publisher)=\{([^}]*)\}", lambda m: f"{m.group(1)}={{{m.group(2).replace('_', chr(92) + '_')}}}", b)91 b = b.replace("@inbook{", "@incollection{").replace("p.~", "").replace("\u2013", "--").replace("–", "--")92 out.append(b); log.append((k, f"crossref:{doi}"))93 else:94 log.append((k, f"CROSSREF_FAIL:{doi}"))95 (ROOT / "paper/references.bib").write_text("\n\n".join(out) + "\n") # no leading comment line: aer.bst rejects it96 (ROOT / "research/bib_build_log.csv").write_text("key,source\n" + "\n".join(f"{k},{s}" for k, s in log))97 bad = [l for l in log if "FAIL" in l[1] or "MISSING" in l[1] or "NO_DOI" in l[1]]98 print(f"{len(out)} entries written; problems: {bad}")99100101if __name__ == "__main__":102 main()103