"""Build paper/references.bib from verified DOIs (Crossref transform to BibTeX) plus hand-written entries for books
and DOI-less items listed in research/bib_overrides.csv. Only keys in research/bib_keys_used.txt are emitted when that
file exists (keeps the bibliography to cited works). Usage: python -m wp35.validation.build_bib
"""
from __future__ import annotations
import csv, re, sys, time
from pathlib import Path
import requests

ROOT = Path(__file__).resolve().parents[3]
UA = {"User-Agent": "WP35-UQO-bib/0.1 (mailto:simon-pierre.boucher@uqo.ca)", "Accept": "application/x-bibtex"}

MANUAL = {
    "greif1993": "@article{greif1993,\n  author = {Greif, Avner},\n  title = {Contract Enforceability and Economic Institutions in Early Trade: The {M}aghribi Traders' Coalition},\n  journal = {American Economic Review},\n  volume = {83},\n  number = {3},\n  pages = {525--548},\n  year = {1993}\n}",
    "hayek1945": "@article{hayek1945,\n  author = {Hayek, Friedrich A.},\n  title = {The Use of Knowledge in Society},\n  journal = {American Economic Review},\n  volume = {35},\n  number = {4},\n  pages = {519--530},\n  year = {1945}\n}",
    "simon1962": "@article{simon1962,\n  author = {Simon, Herbert A.},\n  title = {The Architecture of Complexity},\n  journal = {Proceedings of the American Philosophical Society},\n  volume = {106},\n  number = {6},\n  pages = {467--482},\n  year = {1962}\n}",
    "johnson1982": "@incollection{johnson1982,\n  author = {Johnson, Gregory A.},\n  title = {Organizational Structure and Scalar Stress},\n  booktitle = {Theory and Explanation in Archaeology: The Southampton Conference},\n  editor = {Renfrew, Colin and Rowlands, Michael J. and Segraves, Barbara Abbott},\n  publisher = {Academic Press},\n  address = {New York},\n  pages = {389--421},\n  year = {1982}\n}",
    "turchin2009gavrilets": "@article{turchin2009gavrilets,\n  author = {Turchin, Peter and Gavrilets, Sergey},\n  title = {Evolution of Complex Hierarchical Societies},\n  journal = {Social Evolution \\& History},\n  volume = {8},\n  number = {2},\n  pages = {167--198},\n  year = {2009}\n}",
    "marschak1972": "@book{marschak1972,\n  author = {Marschak, Jacob and Radner, Roy},\n  title = {Economic Theory of Teams},\n  publisher = {Yale University Press},\n  address = {New Haven},\n  year = {1972}\n}",
    "schelling1960": "@book{schelling1960,\n  author = {Schelling, Thomas C.},\n  title = {The Strategy of Conflict},\n  publisher = {Harvard University Press},\n  address = {Cambridge, MA},\n  year = {1960}\n}",
    "boyd1985": "@book{boyd1985,\n  author = {Boyd, Robert and Richerson, Peter J.},\n  title = {Culture and the Evolutionary Process},\n  publisher = {University of Chicago Press},\n  address = {Chicago},\n  year = {1985}\n}",
    "yates1989": "@book{yates1989,\n  author = {Yates, JoAnne},\n  title = {Control through Communication: The Rise of System in {A}merican Management},\n  publisher = {Johns Hopkins University Press},\n  address = {Baltimore},\n  year = {1989}\n}",
    "chandler1977": "@book{chandler1977,\n  author = {Chandler, Alfred D.},\n  title = {The Visible Hand: The Managerial Revolution in {A}merican Business},\n  publisher = {Harvard University Press},\n  address = {Cambridge, MA},\n  year = {1977}\n}",
    "eisenstadt1963": "@book{eisenstadt1963,\n  author = {Eisenstadt, Shmuel N.},\n  title = {The Political Systems of Empires},\n  publisher = {Free Press},\n  address = {New York},\n  year = {1963}\n}",
    "weber1922": "@book{weber1922,\n  author = {Weber, Max},\n  title = {Economy and Society: An Outline of Interpretive Sociology},\n  publisher = {University of California Press},\n  address = {Berkeley},\n  year = {1978},\n  note = {Edited by G. Roth and C. Wittich; first published 1922}\n}",
    "turchin2016": "@book{turchin2016,\n  author = {Turchin, Peter},\n  title = {Ultrasociety: How 10,000 Years of War Made Humans the Greatest Cooperators on Earth},\n  publisher = {Beresta Books},\n  address = {Chaplin, CT},\n  year = {2016}\n}",
    "turchin2023": "@book{turchin2023,\n  author = {Turchin, Peter},\n  title = {End Times: Elites, Counter-Elites, and the Path of Political Disintegration},\n  publisher = {Penguin Press},\n  address = {New York},\n  year = {2023}\n}",
    "hidalgo2015": "@book{hidalgo2015,\n  author = {Hidalgo, C{\\'e}sar},\n  title = {Why Information Grows: The Evolution of Order, from Atoms to Economies},\n  publisher = {Basic Books},\n  address = {New York},\n  year = {2015}\n}",
    "fukuyama2011": "@book{fukuyama2011,\n  author = {Fukuyama, Francis},\n  title = {The Origins of Political Order: From Prehuman Times to the {F}rench Revolution},\n  publisher = {Farrar, Straus and Giroux},\n  address = {New York},\n  year = {2011}\n}",
    "graeber2021": "@book{graeber2021,\n  author = {Graeber, David and Wengrow, David},\n  title = {The Dawn of Everything: A New History of Humanity},\n  publisher = {Farrar, Straus and Giroux},\n  address = {New York},\n  year = {2021}\n}",
    "koyama2022": "@book{koyama2022,\n  author = {Koyama, Mark and Rubin, Jared},\n  title = {How the World Became Rich: The Historical Origins of Economic Growth},\n  publisher = {Polity},\n  address = {Cambridge},\n  year = {2022}\n}",
    "arrow1974": "@book{arrow1974,\n  author = {Arrow, Kenneth J.},\n  title = {The Limits of Organization},\n  publisher = {W. W. Norton},\n  address = {New York},\n  year = {1974}\n}",
    "williamson1985": "@book{williamson1985,\n  author = {Williamson, Oliver E.},\n  title = {The Economic Institutions of Capitalism},\n  publisher = {Free Press},\n  address = {New York},\n  year = {1985}\n}",
    "tainter1988": "@book{tainter1988,\n  author = {Tainter, Joseph A.},\n  title = {The Collapse of Complex Societies},\n  publisher = {Cambridge University Press},\n  address = {Cambridge},\n  year = {1988}\n}",
    "scott1998": "@book{scott1998,\n  author = {Scott, James C.},\n  title = {Seeing Like a State: How Certain Schemes to Improve the Human Condition Have Failed},\n  publisher = {Yale University Press},\n  address = {New Haven},\n  year = {1998}\n}",
    "goldstone1991": "@book{goldstone1991,\n  author = {Goldstone, Jack A.},\n  title = {Revolution and Rebellion in the Early Modern World},\n  publisher = {University of California Press},\n  address = {Berkeley},\n  year = {1991}\n}",
    "tilly1992": "@book{tilly1992,\n  author = {Tilly, Charles},\n  title = {Coercion, Capital, and {E}uropean States, {AD} 990--1992},\n  publisher = {Blackwell},\n  address = {Oxford},\n  year = {1992}\n}",
    "beniger1986": "@book{beniger1986,\n  author = {Beniger, James R.},\n  title = {The Control Revolution: Technological and Economic Origins of the Information Society},\n  publisher = {Harvard University Press},\n  address = {Cambridge, MA},\n  year = {1986}\n}",
    "innis1950": "@book{innis1950,\n  author = {Innis, Harold A.},\n  title = {Empire and Communications},\n  publisher = {Oxford University Press},\n  address = {Oxford},\n  year = {1950}\n}",
    "lewis1969": "@book{lewis1969,\n  author = {Lewis, David K.},\n  title = {Convention: A Philosophical Study},\n  publisher = {Harvard University Press},\n  address = {Cambridge, MA},\n  year = {1969}\n}",
    "johnson1978": "@incollection{johnson1978,\n  author = {Johnson, Gregory A.},\n  title = {Information Sources and the Development of Decision-Making Organizations},\n  booktitle = {Social Archeology: Beyond Subsistence and Dating},\n  editor = {Redman, Charles L. and others},\n  publisher = {Academic Press},\n  address = {New York},\n  pages = {87--112},\n  year = {1978}\n}",
    "rubinstein1989": "@article{rubinstein1989,\n  author = {Rubinstein, Ariel},\n  title = {The Electronic Mail Game: Strategic Behavior under ``Almost Common Knowledge''},\n  journal = {American Economic Review},\n  volume = {79},\n  number = {3},\n  pages = {385--391},\n  year = {1989}\n}",
    "hoyer2024": "@unpublished{hoyer2024,\n  author = {Hoyer, Daniel and Holder, Samantha and Bennett, James S. and Fran{\\c{c}}ois, Pieter and Whitehouse, Harvey and Covey, Alan and Feinman, Gary and Korotayev, Andrey and Vustiuzhanin, Vadim and Preiser-Kapeller, Johannes and Bard, Kathryn and Levine, Jenny and Reddish, Jennifer and Orlandi, Georg and Ainsworth, Rachel and Turchin, Peter},\n  title = {All Crises are Unhappy in their Own Way: The Role of Societal Instability in Shaping the Past},\n  note = {SocArXiv preprint, DOI 10.31235/osf.io/rk4gd; no journal version found as of November 2026},\n  year = {2024}\n}",
    "pemstein2018": "@techreport{pemstein2018,\n  author = {Pemstein, Daniel and Marquardt, Kyle L. and Tzelgov, Eitan and Wang, Yi-ting and Medzihorsky, Juraj and Krusell, Joshua and Miri, Farhad and von Roemer, Johannes},\n  title = {The V-Dem Measurement Model: Latent Variable Analysis for Cross-National and Cross-Temporal Expert-Coded Data},\n  institution = {Varieties of Democracy Institute},\n  type = {V-Dem Working Paper},\n  number = {21},\n  year = {2018}\n}",
    "slingerland2020": "@unpublished{slingerland2020,\n  author = {Slingerland, Edward and Monroe, M. Willis and Sullivan, Brenton and Walsh, Robyn Faith and Veidlinger, Daniel and Alderson-Day, Ben and others},\n  title = {Historians Respond to Whitehouse et al. (2019), ``Complex Societies Precede Moralizing Gods Throughout World History''},\n  note = {PsyArXiv preprint, DOI 10.31234/osf.io/2amjz; published in Journal of Cognitive Historiography},\n  year = {2020}\n}",
    "kohler2018": "@book{kohler2018,\n  editor = {Kohler, Timothy A. and Smith, Michael E.},\n  title = {Ten Thousand Years of Inequality: The Archaeology of Wealth Differences},\n  publisher = {University of Arizona Press},\n  address = {Tucson},\n  year = {2018}\n}",
    "chwe2001": "@book{chwe2001,\n  author = {Chwe, Michael Suk-Young},\n  title = {Rational Ritual: Culture, Coordination, and Common Knowledge},\n  publisher = {Princeton University Press},\n  address = {Princeton},\n  year = {2001}\n}",
    "seshat2026": "@misc{seshat2026,\n  author = {{Seshat: Global History Databank}},\n  title = {Polaris-2026 data release (repository build\\_polaris\\_dataset, commit 55ca5fbc2b)},\n  year = {2026},\n  howpublished = {\\url{https://seshatdatabank.info/data/}},\n  note = {Accessed 2026-11-01}\n}",
    "vdem2026": "@misc{vdem2026,\n  author = {Coppedge, Michael and Gerring, John and Knutsen, Carl Henrik and Lindberg, Staffan I. and Teorell, Jan and others},\n  title = {V-Dem Country-Year Dataset v16},\n  year = {2026},\n  howpublished = {Varieties of Democracy (V-Dem) Project, \\url{https://www.v-dem.net/}},\n  note = {Obtained from the official vdemdata R package (commit f4dd26922e, 2026-03-17); accessed 2026-11-01}\n}",
    "glottolog2026": "@misc{glottolog2026,\n  author = {Hammarstr{\\\"o}m, Harald and Forkel, Robert and Haspelmath, Martin and Bank, Sebastian},\n  title = {Glottolog 5.3},\n  year = {2026},\n  howpublished = {Max Planck Institute for Evolutionary Anthropology, \\url{https://glottolog.org}},\n  note = {CLDF languages table, accessed 2026-11-01}\n}",
}


def crossref_bibtex(doi: str) -> str | None:
    r = requests.get(f"https://api.crossref.org/works/{doi}/transform/application/x-bibtex", headers=UA, timeout=60)
    return r.content.decode("utf-8", errors="replace") if r.status_code == 200 else None


def rekey(bib: str, key: str) -> str:
    return re.sub(r"^(@\w+\{)[^,]*,", lambda m: m.group(1) + key + ",", bib.strip(), count=1, flags=re.M)


def main():
    cands = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/reference_candidates.csv"), delimiter="|") if r["type"] != "skip"}
    prior = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/prior_art.csv"), delimiter="|")}
    over = {r["key"]: r for r in csv.DictReader(open(ROOT / "research/bib_overrides.csv"), delimiter="|")}
    used_f = ROOT / "research/bib_keys_used.txt"
    used = set(used_f.read_text().split()) if used_f.exists() else None
    keys = sorted(set(cands) | set(over) | set(MANUAL))
    out, log = [], []
    for k in keys:
        if used is not None and k not in used:
            continue
        o = over.get(k)
        if o and o["doi"] == "drop":
            continue
        if (o and o["doi"] == "manual") or k in MANUAL:
            if k in MANUAL:
                out.append(MANUAL[k]); log.append((k, "manual")); continue
            log.append((k, "MISSING_MANUAL_ENTRY")); continue
        doi = o["doi"] if o else (prior[k]["doi"] if prior.get(k, {}).get("status") == "VERIFIED_CROSSREF" else None)
        if not doi:
            log.append((k, "NO_DOI")); continue
        b = crossref_bibtex(doi); time.sleep(0.2)
        if b:
            b = rekey(b, k)
            b = b.replace("{\\&}amp;", "\\&").replace("&", "\\&")
            b = re.sub(r",\s*month\s*=\s*[^,}]*", "", b)                        # aer.bst lacks month strings
            b = b.replace("\ufffd", " ")
            b = re.sub(r"(title|number|pages|journal|booktitle|publisher)=\{([^}]*)\}", lambda m: f"{m.group(1)}={{{m.group(2).replace('_', chr(92) + '_')}}}", b)
            b = b.replace("@inbook{", "@incollection{").replace("p.~", "").replace("\u2013", "--").replace("–", "--")
            out.append(b); log.append((k, f"crossref:{doi}"))
        else:
            log.append((k, f"CROSSREF_FAIL:{doi}"))
    (ROOT / "paper/references.bib").write_text("\n\n".join(out) + "\n")   # no leading comment line: aer.bst rejects it
    (ROOT / "research/bib_build_log.csv").write_text("key,source\n" + "\n".join(f"{k},{s}" for k, s in log))
    bad = [l for l in log if "FAIL" in l[1] or "MISSING" in l[1] or "NO_DOI" in l[1]]
    print(f"{len(out)} entries written; problems: {bad}")


if __name__ == "__main__":
    main()