WP35 Phase 2: harmonized Seshat/D-PLACE/modern panels, coverage + missingness diagnostics, data dictionary, parser tests

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 2 days ago (Nov 1, 2026) parent abaa84c Browse files

32 changed files +1,939 −0

modified configs/sources.yaml +11 −0

		`@@ -90,3 +90,14 @@ sources:`
90	90	filename: "vdem_v16_codebook.RData"
91	91	units: "variable metadata"
92	92	role: "data dictionary for V-Dem"
	93	+ glottolog_cldf_languages:
	94	+ dataset: "Glottolog CLDF languages table (language family for Galton controls)"
	95	+ publisher: "Glottolog (Hammarström, Forkel, Haspelmath, Bank), MPI-EVA"
	96	+ release: "glottolog-cldf v5.3 (2026-03-02)"
	97	+ url: "https://raw.githubusercontent.com/glottolog/glottolog-cldf/v5.3/cldf/languages.csv"
	98	+ landing_page: "https://glottolog.org/"
	99	+ license: "CC BY 4.0"
	100	+ citation: "Hammarström, H., Forkel, R., Haspelmath, M. & Bank, S. Glottolog 5.3. Leipzig: Max Planck Institute for Evolutionary Anthropology. https://glottolog.org"
	101	+ filename: "glottolog_v5.3_languages.csv"
	102	+ units: "one row per languoid; Family_ID"
	103	+ role: "map D-PLACE Glottocodes to language families (non-independence control)"

added data/metadata/crosswalk_vdem_maddison.csv +209 −0

		`@@ -0,0 +1,209 @@`
	1	+code,in_vdem,in_maddison,vdem_name
	2	+AFG,True,True,Afghanistan
	3	+AGO,True,True,Angola
	4	+ALB,True,True,Albania
	5	+ARE,True,True,United Arab Emirates
	6	+ARG,True,True,Argentina
	7	+ARM,True,True,Armenia
	8	+AUS,True,True,Australia
	9	+AUT,True,True,Austria
	10	+AZE,True,True,Azerbaijan
	11	+BDI,True,True,Burundi
	12	+BDN,True,False,Baden
	13	+BEL,True,True,Belgium
	14	+BEN,True,True,Benin
	15	+BFA,True,True,Burkina Faso
	16	+BGD,True,True,Bangladesh
	17	+BGR,True,True,Bulgaria
	18	+BHR,True,True,Bahrain
	19	+BIH,True,True,Bosnia and Herzegovina
	20	+BLR,True,True,Belarus
	21	+BOL,True,True,Bolivia
	22	+BRA,True,True,Brazil
	23	+BRB,True,True,Barbados
	24	+BRW,True,False,Brunswick
	25	+BTN,True,False,Bhutan
	26	+BVR,True,False,Bavaria
	27	+BWA,True,True,Botswana
	28	+CAF,True,True,Central African Republic
	29	+CAN,True,True,Canada
	30	+CHE,True,True,Switzerland
	31	+CHL,True,True,Chile
	32	+CHN,True,True,China
	33	+CIV,True,True,Ivory Coast
	34	+CMR,True,True,Cameroon
	35	+COD,True,True,Democratic Republic of the Congo
	36	+COG,True,True,Republic of the Congo
	37	+COL,True,True,Colombia
	38	+COM,True,True,Comoros
	39	+CPV,True,True,Cape Verde
	40	+CRI,True,True,Costa Rica
	41	+CSK,False,True,
	42	+CUB,True,True,Cuba
	43	+CYP,True,True,Cyprus
	44	+CZE,True,True,Czechia
	45	+DDR,True,False,German Democratic Republic
	46	+DEU,True,True,Germany
	47	+DJI,True,True,Djibouti
	48	+DMA,False,True,
	49	+DNK,True,True,Denmark
	50	+DOM,True,True,Dominican Republic
	51	+DZA,True,True,Algeria
	52	+ECU,True,True,Ecuador
	53	+EGY,True,True,Egypt
	54	+ERI,True,False,Eritrea
	55	+ESP,True,True,Spain
	56	+EST,True,True,Estonia
	57	+ETH,True,True,Ethiopia
	58	+FIN,True,True,Finland
	59	+FJI,True,False,Fiji
	60	+FRA,True,True,France
	61	+GAB,True,True,Gabon
	62	+GBR,True,True,United Kingdom
	63	+GEO,True,True,Georgia
	64	+GHA,True,True,Ghana
	65	+GIN,True,True,Guinea
	66	+GMB,True,True,The Gambia
	67	+GNB,True,True,Guinea-Bissau
	68	+GNQ,True,True,Equatorial Guinea
	69	+GRC,True,True,Greece
	70	+GTM,True,True,Guatemala
	71	+GUY,True,False,Guyana
	72	+HDM,True,False,Hesse-Darmstadt
	73	+HKG,True,True,Hong Kong
	74	+HKS,True,False,Hesse-Kassel
	75	+HND,True,True,Honduras
	76	+HRG,True,False,Hamburg
	77	+HRV,True,True,Croatia
	78	+HTI,True,True,Haiti
	79	+HUN,True,True,Hungary
	80	+HVR,True,False,Hanover
	81	+IDN,True,True,Indonesia
	82	+IND,True,True,India
	83	+IRL,True,True,Ireland
	84	+IRN,True,True,Iran
	85	+IRQ,True,True,Iraq
	86	+ISL,True,True,Iceland
	87	+ISR,True,True,Israel
	88	+ITA,True,True,Italy
	89	+JAM,True,True,Jamaica
	90	+JOR,True,True,Jordan
	91	+JPN,True,True,Japan
	92	+KAZ,True,True,Kazakhstan
	93	+KEN,True,True,Kenya
	94	+KGZ,True,True,Kyrgyzstan
	95	+KHM,True,True,Cambodia
	96	+KOR,True,True,South Korea
	97	+KWT,True,True,Kuwait
	98	+LAO,True,True,Laos
	99	+LBN,True,True,Lebanon
	100	+LBR,True,True,Liberia
	101	+LBY,True,True,Libya
	102	+LCA,False,True,
	103	+LKA,True,True,Sri Lanka
	104	+LSO,True,True,Lesotho
	105	+LTU,True,True,Lithuania
	106	+LUX,True,True,Luxembourg
	107	+LVA,True,True,Latvia
	108	+MAR,True,True,Morocco
	109	+MCL,True,False,Mecklenburg Schwerin
	110	+MDA,True,True,Moldova
	111	+MDG,True,True,Madagascar
	112	+MDN,True,False,Modena
	113	+MDV,True,False,Maldives
	114	+MEX,True,True,Mexico
	115	+MKD,True,True,North Macedonia
	116	+MLI,True,True,Mali
	117	+MLT,True,True,Malta
	118	+MMR,True,True,Burma/Myanmar
	119	+MNE,True,True,Montenegro
	120	+MNG,True,True,Mongolia
	121	+MOZ,True,True,Mozambique
	122	+MRT,True,True,Mauritania
	123	+MUS,True,True,Mauritius
	124	+MWI,True,True,Malawi
	125	+MYS,True,True,Malaysia
	126	+NAM,True,True,Namibia
	127	+NER,True,True,Niger
	128	+NGA,True,True,Nigeria
	129	+NIC,True,True,Nicaragua
	130	+NLD,True,True,Netherlands
	131	+NOR,True,True,Norway
	132	+NPL,True,True,Nepal
	133	+NSS,True,False,Nassau
	134	+NZL,True,True,New Zealand
	135	+OLD,True,False,Oldenburg
	136	+OMN,True,True,Oman
	137	+PAK,True,True,Pakistan
	138	+PAN,True,True,Panama
	139	+PER,True,True,Peru
	140	+PHL,True,True,Philippines
	141	+PNG,True,False,Papua New Guinea
	142	+POL,True,True,Poland
	143	+PPS,True,False,Papal States
	144	+PRI,False,True,
	145	+PRK,True,True,North Korea
	146	+PRM,True,False,Parma
	147	+PRT,True,True,Portugal
	148	+PRY,True,True,Paraguay
	149	+PSB,True,False,Palestine/British Mandate
	150	+PSE,True,True,Palestine/West Bank
	151	+PSG,True,False,Palestine/Gaza
	152	+QAT,True,True,Qatar
	153	+ROU,True,True,Romania
	154	+RUS,True,True,Russia
	155	+RWA,True,True,Rwanda
	156	+SAU,True,True,Saudi Arabia
	157	+SAX,True,False,Saxe-Weimar-Eisenach
	158	+SDN,True,True,Sudan
	159	+SEN,True,True,Senegal
	160	+SGP,True,True,Singapore
	161	+SLB,True,False,Solomon Islands
	162	+SLE,True,True,Sierra Leone
	163	+SLV,True,True,El Salvador
	164	+SML,True,False,Somaliland
	165	+SOM,True,False,Somalia
	166	+SPD,True,False,Piedmont-Sardinia
	167	+SRB,True,True,Serbia
	168	+SSD,True,False,South Sudan
	169	+STP,True,True,Sao Tome and Principe
	170	+SUN,False,True,
	171	+SUR,True,False,Suriname
	172	+SVK,True,True,Slovakia
	173	+SVN,True,True,Slovenia
	174	+SWE,True,True,Sweden
	175	+SWZ,True,True,Eswatini
	176	+SXN,True,False,Saxony
	177	+SYC,True,True,Seychelles
	178	+SYR,True,True,Syria
	179	+TCD,True,True,Chad
	180	+TGO,True,True,Togo
	181	+THA,True,True,Thailand
	182	+TJK,True,True,Tajikistan
	183	+TKM,True,True,Turkmenistan
	184	+TLS,True,False,Timor-Leste
	185	+TSC,True,False,Tuscany
	186	+TTO,True,True,Trinidad and Tobago
	187	+TUN,True,True,Tunisia
	188	+TUR,True,True,Türkiye
	189	+TWN,True,True,Taiwan
	190	+TWS,True,False,Two Sicilies
	191	+TZA,True,True,Tanzania
	192	+UGA,True,True,Uganda
	193	+UKR,True,True,Ukraine
	194	+URY,True,True,Uruguay
	195	+USA,True,True,United States of America
	196	+UZB,True,True,Uzbekistan
	197	+VDR,True,False,Republic of Vietnam
	198	+VEN,True,True,Venezuela
	199	+VNM,True,True,Vietnam
	200	+VUT,True,False,Vanuatu
	201	+WRG,True,False,Würtemberg
	202	+XKX,True,False,Kosovo
	203	+YEM,True,True,Yemen
	204	+YMD,True,False,South Yemen
	205	+YUG,False,True,
	206	+ZAF,True,True,South Africa
	207	+ZMB,True,True,Zambia
	208	+ZWE,True,True,Zimbabwe
	209	+ZZB,True,False,Zanzibar

added data/metadata/data_dictionary.md +65 −0

		`@@ -0,0 +1,65 @@`
	1	+# WP35 data dictionary (processed files)
	2	+
	3	+Built by `python -m wp35.harmonization.build_panels` (2026-11-01). Raw provenance and SHA-256: `source_registry.json`.
	4	+Counts below are copied from `results/diagnostics/coverage_summary.json` and the coverage CSVs produced by the same
	5	+run; they are not typed by hand elsewhere. Missing values are `NaN` everywhere; no zero stands for "unknown".
	6	+
	7	+## A. Seshat Polaris-2026 (historical polities)
	8	+
	9	+### `seshat_facts.parquet` — one row per fact (56,274 rows; 2,503 dated; 132 unparsed values logged)
	10	+\| column \| meaning \|
	11	+\|---\|---\|
	12	+\| sheet, section \| Polaris sheet (Social complexity / Warfare / Religion / General) and section code (sc, wf, rt, general) \|
	13	+\| seshat_region, polity_number, polity_id \| Seshat identifiers (42 regions, 626 polities) \|
	14	+\| variable \| original Polaris `variable_name` \|
	15	+\| value \| presence code mapped P/IP→1, A/IA→0, disputed (A~P)→0.5, SU/U/unknown→NaN; numeric variables → midpoint (geometric for population/territory/largest settlement; arithmetic for levels) \|
	16	+\| value_lo, value_hi \| numeric bounds from `value_from`/`value_to` (ranges kept) \|
	17	+\| text \| raw text for General-sheet categorical facts \|
	18	+\| year_from, year_to, dated \| fact date interval; `dated=False` ⇒ fact describes the whole polity period (24,401 of 26,033 SC facts) \|
	19	+\| inferred, disputed, unparsed \| IP/IA flag; A~P-type flag; value not in the code vocabulary (logged to `results/diagnostics/seshat_unparsed_values.csv`) \|
	20	+\| is_uncertain, is_disputed \| Polaris flags \|
	21	+
	22	+### `seshat_polity_agg.parquet` — polity × variable aggregates (all facts)
	23	+`value_mean` (mean of coded values across facts), `value_min/max`, `logv_mean` (mean log midpoint for geometric
	24	+variables), `lo_min`, `hi_max`, `n_facts`, `n_dated`, `n_undated`, `n_inferred`, `n_disputed`, `n_flag_uncertain`.
	25	+`seshat_polity_wide.parquet` = pivot of `value_mean`. `seshat_precrisis_agg.parquet` = same aggregates computed per
	26	+CrisisDB record using only facts with `year_to < crisis_start` or undated facts (`crisis_id` column).
	27	+
	28	+### `seshat_polity_meta.parquet` — one row per polity (626)
	29	+`long_name, seshat_region, homeNGA, start_year, end_year, macro_region` (10, `configs/regions.yaml`), `midpoint`,
	30	+`era` (6 blocks), `duration`, `censored` (end_year ≥ 1900: 30 polities), `centralization`
	31	+(`polity_degree_of_centralization`), `relationship_to_preceding`, `preceding_entity`, `succeeding_entity`,
	32	+`suprapolity_relations`, `utm_zone`, `language_family`, `successor_id` (matched for 285 polities),
	33	+`successor_relationship`, `y3_discontinuous` (coded for 207 polities; 37 = 1).
	34	+
	35	+### `seshat_crisis.parquet` — CrisisDB "Crisis consequences" (166 records, 150 polities)
	36	+`crisis_id, polity_id, other_polity_id, crisis_start, crisis_end` (= `year_from`, `year_to`), item values `<item>_v`
	37	+(same code mapping), `y1_n_coded`, `y1_n_present`, `y1_severity` (= present / coded over the 14 instability items
	38	+listed in the pre-registration; coded for all 166), `y1b_severe` (any of collapse, fragmentation, century_plus,
	39	+conquest, civil_war present; defined when all five coded: 158 records, 122 positive), `y_reform`, `epidemic_v`.
	40	+Codes in Polaris: P, A, IP, IA, SU, DIS (treated as missing).
	41	+
	42	+Coverage of the pre-registered indicators (share of the 626 polities missing): `results/diagnostics/seshat_missingness_by_indicator.csv`;
	43	+e.g. written_record 12%, settlement_hierarchy 16%, administrative_level 21%, polity_population 52%, measurement systems 82–86%.
	44	+Polities with ≥ 50% of the K indicators coded: 376; with any D scale variable: 561; with population: 299; with territory: 355.
	45	+
	46	+## B. D-PLACE Ethnographic Atlas v3.3 — `dplace_ea_societies.parquet` (1,291 societies)
	47	+`soc_id, Name, Latitude, Longitude, Glottocode, region` (50), `main_focal_year` (−2000…1965), `xd_id`,
	48	+`Family_ID` (Glottolog 5.3 family; isolates keep their own glottocode; 173 families), `Macroarea`,
	49	+`EA030, EA032, EA033, EA034, EA042, EA066, EA067, EA068, EA070, EA072, EA073, EA090, EA005, EA028, EA113` (ordinal code
	50	+numbers from `codes.csv` `ord`; "Missing data" → NaN), `ea031_code`, `EA031_mid` (geometric midpoint of the category
	51	+range parsed from the code label; open categories use the bound, see `results/diagnostics/dplace_code_midpoints.csv`),
	52	+`EA202_mid` = `EA202` (continuous population size, 953 coded), `log_community_size` (611 coded), `log_population`,
	53	+`jurisdictional_hierarchy_beyond` (= EA033 ord − 1 : 0 acephalous … 4 four levels; 1,155 coded), `jurisdictional_hierarchy_local`.
	54	+
	55	+## C. Modern panel — `modern_panel.parquet` (28,092 country-years, 202 countries, 1789–2025)
	56	+V-Dem v16 columns kept verbatim (`country_name, country_text_id, country_id, year, e_regionpol_6C, e_regionpol,
	57	+v2x_polyarchy, e_civil_war, e_miinterc, e_coups, e_pt_coup, v2regendtype, v2regint, v2regdur, v2regidnr, e_pop, e_area,
	58	+e_gdppc, v2x_regime, v2x_rule`, K items `v2clrspct, v2stcritrecadm, v2cltrnslw, v2stfisccap, v2svstterr, v2elembcap`).
	59	+Maddison 2023 merged by ISO3 (`country_text_id` = `countrycode`; 163 matched, 39 V-Dem countries unmatched, list in
	60	+`crosswalk_vdem_maddison.csv`): `maddison_pop` (thousands; 15,821 country-years, from 1820), `maddison_gdppc`.
	61	+Derived: `log_pop`, `log_area`, `log_gdppc`, `y4_civilwar_onset` (e_civil_war 0→1; 176 onsets / 13,608 coded,
	62	+1817–2006), `y5_coup` (e_coups ≥ 1; 599 / 13,813, 1789–2008), `regime_end_year` (last year of a `v2regidnr`),
	63	+`y6_irregular_regime_end` (regime end year with `v2regendtype` ∈ {0 military coup, 1 other coup, 2 self-coup,
	64	+3 assassination, 5 loss in civil war, 6 loss in inter-state war, 7 foreign intervention, 8 popular uprising};
	65	+1,041 / 27,846). Codebook readings are in `research/measurement.md`.

added data/metadata/dplace_ea_codes_used.csv +112 −0

		`@@ -0,0 +1,112 @@`
	1	+Var_ID,ID,Name,Description
	2	+EA005,EA005-0,0-5%,Zero to 5 percent dependence
	3	+EA005,EA005-1,6-15%,6 to 15 percent dependence
	4	+EA005,EA005-2,16-25%,16 to 25 percent dependence
	5	+EA005,EA005-3,26-35%,26 to 35 percent dependence
	6	+EA005,EA005-4,36-45%,36 to 45 percent dependence
	7	+EA005,EA005-5,46-55%,46 to 55 percent dependence
	8	+EA005,EA005-6,56-65%,56 to 65 percent dependence
	9	+EA005,EA005-7,66-75%,66 to 75 percent dependence
	10	+EA005,EA005-8,76-85%,76 to 85 percent dependence
	11	+EA005,EA005-9,86-100%,86 to 100 percent dependence
	12	+EA005,EA005-NA,Missing data,Missing data
	13	+EA028,EA028-NA,Missing data,Missing data
	14	+EA028,EA028-1,No agriculture,Complete absence of agriculture
	15	+EA028,EA028-2,Casual,"Casual agriculture, i.e., the slight or sporadic cultivation of food or other plants incidental to a primary dependence upon other subsistence practices"
	16	+EA028,EA028-3,Extensive/shifting,"Extensive or shifting cultivation, as where new fields are cleared annually, cultivated for a year or two, and then allowed to revert to forest or brush for a long fallow period"
	17	+EA028,EA028-4,Horticulture,"Horticulture, i.e., semi-intensive agriculture limited mainly to vegetable gardens or groves of fruit trees rather than the cultivation of field crops"
	18	+EA028,EA028-5,Intensive,"Intensive agriculture on permanent fields, utilizing fertilization by compost or animal manure, crop rotation, or other techniques so that fallowing is either unnecessary or is confined to relatively short periods"
	19	+EA028,EA028-6,Intensive irrigated,Intensive cultivation where it is largely dependent upon irrigation
	20	+EA030,EA030-NA,Missing data,Missing data
	21	+EA030,EA030-1,Nomadic,Fully migratory or nomadic bands
	22	+EA030,EA030-2,Seminomadic,"Seminomadic communities whose members wander in bands for at least half of the year but occupy a fixed settlement at some season or seasons, e.g., recurrently occupied winter quarters"
	23	+EA030,EA030-3,Semisedentary,"Semisedentary communities whose members shift from one to another fixed settlement at different seasons or who occupy more or less permanently a single settlement from which a substantial proportion of the population departs seasonally to occupy shifting camps, e.g., during transhumance"
	24	+EA030,EA030-4,Impermanent,"Compact but impermanent settlements, i.e., villages whose location is shifted every few years"
	25	+EA030,EA030-5,Dispersed homesteads,Neighborhoods of dispersed family homesteads
	26	+EA030,EA030-6,Hamlets,Separated hamlets where several such form a more or less permanent single community
	27	+EA030,EA030-7,Villages/towns,"Compact and relatively permanent settlements, i.e., nucleated villages or towns"
	28	+EA030,EA030-8,Complex permanent,Complex settlements consisting of a nucleated village or town with outlying homesteads or satellite hamlets. Urban aggregations of population are not separately indicated since EA031 deals with community size
	29	+EA031,EA031-NA,Missing data,Missing data
	30	+EA031,EA031-1,<50,Fewer than 50 persons
	31	+EA031,EA031-2,50-99,From 50 to 99 persons
	32	+EA031,EA031-3,100-199,From 100 to 199 persons
	33	+EA031,EA031-4,200-399,From 200 to 399 persons
	34	+EA031,EA031-5,400-1000,"From 400 to 1,000 persons"
	35	+EA031,EA031-6,1000-5000,"More than 1,000 persons in the absence of indigenous urban aggregations of more than 5,000"
	36	+EA031,EA031-7,5000-50000,"One or more indigenous towns of more than 5,000 inhabitants but none of more than 50,000"
	37	+EA031,EA031-8,50000+,"One or more indigenous cities with more than 50,000 inhabitants"
	38	+EA032,EA032-NA,Missing data,Missing data
	39	+EA032,EA032-2,Independent families,Independent families (may be nuclear or polygynous)
	40	+EA032,EA032-3,Extended families,Extended families
	41	+EA032,EA032-4,Clan-barrios,Clan-barrios
	42	+EA033,EA033-NA,Missing data,Missing data
	43	+EA033,EA033-1,Acephalous,"No political authority beyond community (e.g., autonomous bands and villages)"
	44	+EA033,EA033-2,One level,"One level (e.g., petty chiefdoms)"
	45	+EA033,EA033-3,Two levels,"Two levels (e.g., larger chiefdoms)"
	46	+EA033,EA033-4,Three levels,"Three levels (e.g., states)"
	47	+EA033,EA033-5,Four levels,"Four levels (e.g., large states)"
	48	+EA034,EA034-NA,Missing data,Missing data
	49	+EA034,EA034-1,Absent,A high god absent or not reported in substantial descriptions of religious beliefs
	50	+EA034,EA034-2,Otiose,A high god present but otiose or not concerned with human affairs
	51	+EA034,EA034-3,"Active, but not supporting morality",A high god present and active in human affairs but not offering positive support to human morality
	52	+EA034,EA034-4,"Active, supporting morality","A high god present, active, and specifically supportive of human morality"
	53	+EA042,EA042-NA,Missing for at least 1 activity,Missing data for at least one subsistence activity
	54	+EA042,EA042-1,Gathering,Gathering contributes most
	55	+EA042,EA042-2,Fishing,Fishing contributes most
	56	+EA042,EA042-3,Hunting,Hunting contributes most
	57	+EA042,EA042-4,Pastoralism,Pastoralism contributes most
	58	+EA042,EA042-5,Casual agriculture,Casual agriculture contributes most
	59	+EA042,EA042-6,Extensive agriculture,Extensive agriculture contributes most
	60	+EA042,EA042-7,Intensive agriculture,Intensive agriculture contributes most
	61	+EA042,EA042-8,Two or more sources,Two or more sources contribute equally
	62	+EA042,EA042-9,"Agriculture, type unknown","Agriculture contributes most, type unknown"
	63	+EA066,EA066-NA,Missing data,Missing data
	64	+EA066,EA066-1,Absence of distinctions,"Absence of significant class distinctions among freemen (slavery is treated in EA070), ignoring variations in individual repute achieved through skill, valor, piety, or wisdom"
	65	+EA066,EA066-2,Wealth distinctions,"Wealth distinctions, based on the possession or distribution of property, present and socially important but not crystallized into distinct and hereditary social classes"
	66	+EA066,EA066-3,Elite stratification,"Elite stratification, in which an elite class derives its superior status from, and perpetuates it through, control over scarce resources, particularly land, and is thereby differentiated from a property-less proletariat or serf class"
	67	+EA066,EA066-4,Dual stratification,"Dual stratification into a hereditary aristocracy and a lower class of ordinary commoners or freemen, where traditionally ascribed noble status is at least as decisive as control over scarce resources"
	68	+EA066,EA066-5,Complex stratification,Complex stratification into social classes correlated in large measure with extensive differentiation of occupational statuses
	69	+EA067,EA067-NA,Missing data,Missing data
	70	+EA067,EA067-1,Absence of distinctions,"Absence of significant class distinctions among freemen (slavery is treated in EA070), ignoring variations in individual repute achieved through skill, valor, piety, or wisdom"
	71	+EA067,EA067-2,Wealth distinctions,"Wealth distinctions, based on the possession or distribution of property, present and socially important but not crystallized into distinct and hereditary social classes"
	72	+EA067,EA067-3,Elite stratification,"Elite stratification, in which an elite class derives its superior status from, and perpetuates it through, control over scarce resources, particularly land, and is thereby differentiated from a property-less proletariat or serf class"
	73	+EA067,EA067-4,Dual stratification,"Dual stratification into a hereditary aristocracy and a lower class of ordinary commoners or freemen, where traditionally ascribed noble status is at least as decisive as control over scarce resources"
	74	+EA067,EA067-5,Complex stratification,Complex stratification into social classes correlated in large measure with extensive differentiation of occupational statuses
	75	+EA067,EA067-9,No secondary type,No secondary type
	76	+EA068,EA068-NA,Missing data,Missing data
	77	+EA068,EA068-1,Absence of distinctions,Caste distinctions absent or insignificant
	78	+EA068,EA068-2,Despised occupation groups,"One or more despised occupational groups, e.g., smiths or leather workers, distinguished from the general population, regarded as outcastes by the latter, and characterized by strict endogamy"
	79	+EA068,EA068-3,Ethnic stratification,"Ethnic stratification, in which a superordinate caste withholds privileges from and refuses to intermarry with a subordinate caste (or castes) which it stigmatizes as ethnically alien, e.g., as descended from a conquered and culturally inferior indigenous population, from former slaves, or from foreign immigrants of different race and/or culture"
	80	+EA068,EA068-4,Complex caste stratification,Complex caste stratification in which occupational differentiation emphasizes hereditary ascription and endogamy to the near exclusion of achievable class statuses
	81	+EA070,EA070-NA,Missing data,Missing data
	82	+EA070,EA070-1,Absence of slavery,Absence or near absence of slavery
	83	+EA070,EA070-2,"Slavery, nonhereditary","Incipient or nonhereditary slavery, i.e., where slave status is temporary and not transmitted to the children of slaves"
	84	+EA070,EA070-3,"Slavery, no info on heritability",Slavery reported but not identified as hereditary or nonhereditary
	85	+EA070,EA070-4,"Slavery, hereditary",Hereditary slavery present and of at least modest social significance
	86	+EA072,EA072-NA,Missing data,Missing data
	87	+EA072,EA072-1,Patrilineal heir,Patrilineal heir
	88	+EA072,EA072-2,Matrilineal heir,Matrilineal heir
	89	+EA072,EA072-3,Nonhered: appointed by auth.,Nonhereditary succession through appointment by some higher authority
	90	+EA072,EA072-4,Nonhered: seniority,Nonhereditary succession on the basis primarily of seniority or age
	91	+EA072,EA072-5,Nonhered: influence,"Nonhereditary succession through influence, e.g., of wealth or social status"
	92	+EA072,EA072-6,Nonhered: election,Nonhereditary succession through election or some other mode of formal consensus
	93	+EA072,EA072-7,Nonhered: inf. consensus,Nonhereditary succession through informal consensus
	94	+EA072,EA072-9,Absence of office,Absence of any office resembling that of a local headman
	95	+EA073,EA073-NA,Missing data,Missing data
	96	+EA073,EA073-1,Son,Hereditary succession by a son
	97	+EA073,EA073-2,Patrilineal heir,Hereditary succession by a patrilineal heir who takes precedence over a son
	98	+EA073,EA073-3,Sister's son,Hereditary succession by a sister's son
	99	+EA073,EA073-4,Matrilineal heir,"Hereditary succession by a matrilineal heir who takes precedence over a sister's son, e.g., a younger brother"
	100	+EA073,EA073-5,Nonhereditary,Nonhereditary
	101	+EA073,EA073-9,Absence of office,Absence of any office resembling that of a local headman
	102	+EA090,EA090-NA,Missing data,Insufficient information or not coded
	103	+EA090,EA090-1,"Absence, even locally","Absence of any politicdal organization even at local level, e.g., where family heads acknowledge no higher political authority"
	104	+EA090,EA090-2,Autonomous communities,"Autonomous local communities, i.e., politically independent groups which do not exceed 1500 in average population"
	105	+EA090,EA090-3,Peace groups among communities,"Peace groups transcending the local community where the basis of unity is other than political, e.g., derived from reciprocal trade relations, defensive military agreements, or a common cult or age-grade organization"
	106	+EA090,EA090-4,Minimal states,"Minimal states, i.e., political integration even at the local level, e.g., where family heads acknowledge no higher political authority (1500 ﾖ 10,000)"
	107	+EA090,EA090-5,Little states,"Little states, i.e., political integration in independent units averaging between 10,000 and 100,000 in population"
	108	+EA090,EA090-6,States,"States, i.e., political integration in large independent units averaging at least 100,000 in population"
	109	+EA090,EA090-8,Dependent societies,"Dependent societies lacking any political organization of their own, e.g., those forming an integral part of some larger political system and those governed exclusively and directly by agents of another and politically dominant society. Colonial governments operating through indirect rule are ignored."
	110	+EA113,EA113-NA,Missing data,Missing data
	111	+EA113,EA113-1,Rigid,"Rigid, characterized as: non-egalitarian, ascriptive status distinctions, autocratic, hierarchical political system, fixed residence and group membership, central authority, fixed religious rites."
	112	+EA113,EA113-2,Flexible,"Flexible, characterized as: egalitarian, achieved status distinctions, autocratic, democratic, federated or stateless political system, ease in residence and group changes, individualized or flexible religious rites."

modified data/metadata/source_registry.json +22 −0

		`@@ -174,5 +174,27 @@`
174	174	"size_bytes": 148633,
175	175	"sha256": "2840d879cb9071b1f742f5f18c5373518a586ac90454b58dbf5f4309077f0700",
176	176	"status": "downloaded"
	177	+ },
	178	+ "glottolog_cldf_languages": {
	179	+ "dataset": "Glottolog CLDF languages table (language family for Galton controls)",
	180	+ "publisher": "Glottolog (Hammarström, Forkel, Haspelmath, Bank), MPI-EVA",
	181	+ "release": "glottolog-cldf v5.3 (2026-03-02)",
	182	+ "url": "https://raw.githubusercontent.com/glottolog/glottolog-cldf/v5.3/cldf/languages.csv",
	183	+ "landing_page": "https://glottolog.org/",
	184	+ "license": "CC BY 4.0",
	185	+ "citation": "Hammarström, H., Forkel, R., Haspelmath, M. & Bank, S. Glottolog 5.3. Leipzig: Max Planck Institute for Evolutionary Anthropology. https://glottolog.org",
	186	+ "filename": "glottolog_v5.3_languages.csv",
	187	+ "units": "one row per languoid; Family_ID",
	188	+ "role": "map D-PLACE Glottocodes to language families (non-independence control)",
	189	+ "key": "glottolog_cldf_languages",
	190	+ "local_path": "data/raw/glottolog_cldf_languages/glottolog_v5.3_languages.csv",
	191	+ "http_status": 200,
	192	+ "final_url": "https://raw.githubusercontent.com/glottolog/glottolog-cldf/v5.3/cldf/languages.csv",
	193	+ "content_type": "text/plain; charset=utf-8",
	194	+ "retrieved_at_utc": "2026-11-01T08:06:56+00:00",
	195	+ "elapsed_s": 0.48,
	196	+ "size_bytes": 2477017,
	197	+ "sha256": "1a50a393bc81568b656f9522be18aa4f80f38e94309ba6c863d583234adfbb89",
	198	+ "status": "downloaded"
177	199	}
178	200	}
		\ No newline at end of file

added data/processed/dplace_ea_societies.parquet +0 −0

Binary file not shown.

added data/processed/modern_panel.parquet +0 −0

Binary file not shown.

added data/processed/seshat_crisis.parquet +0 −0

Binary file not shown.

added data/processed/seshat_facts.parquet +0 −0

Binary file not shown.

added data/processed/seshat_polity_agg.parquet +0 −0

Binary file not shown.

added data/processed/seshat_polity_meta.parquet +0 −0

Binary file not shown.

added data/processed/seshat_polity_wide.parquet +0 −0

Binary file not shown.

added data/processed/seshat_precrisis_agg.parquet +0 −0

Binary file not shown.

added paper/figures/fig_dplace_focal_years.pdf +0 −0

Binary file not shown.

added paper/figures/fig_modern_coverage.pdf +0 −0

Binary file not shown.

added paper/figures/fig_seshat_missingness.pdf +0 −0

Binary file not shown.

added paper/figures/fig_seshat_temporal_coverage.pdf +0 −0

Binary file not shown.

added results/diagnostics/coverage_summary.json +76 −0

		`@@ -0,0 +1,76 @@`
	1	+{
	2	+ "built_at_utc": "2026-11-01T08:13:28+00:00",
	3	+ "seshat": {
	4	+ "polities": 626,
	5	+ "regions": 42,
	6	+ "macro_regions": 10,
	7	+ "facts_total": 56274,
	8	+ "facts_dated": 2503,
	9	+ "facts_unparsed": 132,
	10	+ "facts_inferred": 8094,
	11	+ "facts_disputed_code": 106,
	12	+ "polities_with_population": 299,
	13	+ "polities_with_territory": 355,
	14	+ "polities_with_any_D": 561,
	15	+ "polities_with_ge50pct_K": 376,
	16	+ "crisis_records": 166,
	17	+ "crisis_polities": 150,
	18	+ "crisis_y1_coded": 166,
	19	+ "crisis_y1b_coded": 158,
	20	+ "crisis_y1b_positive": 122,
	21	+ "y3_coded": 207,
	22	+ "y3_positive": 37,
	23	+ "duration_censored_ge1900": 30,
	24	+ "warfare_indicators": 49,
	25	+ "era_counts": {
	26	+ "1-1000": 148,
	27	+ "1001-1500": 144,
	28	+ "1501-1800": 103,
	29	+ "pre-1000BCE": 95,
	30	+ "1000BCE-0": 95,
	31	+ "post-1800": 41
	32	+ },
	33	+ "macro_region_counts": {
	34	+ "Sub_Saharan_Africa": 111,
	35	+ "Southwest_Asia": 105,
	36	+ "Europe": 98,
	37	+ "South_Asia": 78,
	38	+ "North_America": 53,
	39	+ "East_Asia": 52,
	40	+ "Central_Asia": 44,
	41	+ "North_Africa": 41,
	42	+ "Southeast_Asia_Oceania": 33,
	43	+ "South_America": 11
	44	+ }
	45	+ },
	46	+ "dplace": {
	47	+ "societies": 1291,
	48	+ "with_EA031": 611,
	49	+ "with_EA202": 953,
	50	+ "with_EA033": 1155,
	51	+ "with_glottolog_family": 1291,
	52	+ "families": 173,
	53	+ "regions": 50,
	54	+ "focal_year_range": [
	55	+ -2000,
	56	+ 1965
	57	+ ]
	58	+ },
	59	+ "modern": {
	60	+ "country_years": 28092,
	61	+ "countries": 202,
	62	+ "years": [
	63	+ 1789,
	64	+ 2025
	65	+ ],
	66	+ "with_maddison_pop": 15821,
	67	+ "countries_matched_maddison": 163,
	68	+ "vdem_countries_unmatched": 39,
	69	+ "y4_onsets": 176,
	70	+ "y4_coded": 13608,
	71	+ "y5_coups": 599,
	72	+ "y5_coded": 13813,
	73	+ "y6_irregular_ends": 1041,
	74	+ "y6_coded": 27846
	75	+ }
	76	+}
		\ No newline at end of file

added results/diagnostics/dplace_code_midpoints.csv +10 −0

		`@@ -0,0 +1,10 @@`
	1	+Var_ID,ID,Name,Description,mid,lo,hi,kind
	2	+EA031,EA031-NA,Missing data,Missing data,,,,none
	3	+EA031,EA031-1,<50,Fewer than 50 persons,25.0,,50.0,open_lower
	4	+EA031,EA031-2,50-99,From 50 to 99 persons,70.35623639735144,50.0,99.0,range
	5	+EA031,EA031-3,100-199,From 100 to 199 persons,141.06735979665885,100.0,199.0,range
	6	+EA031,EA031-4,200-399,From 200 to 399 persons,282.4889378365107,200.0,399.0,range
	7	+EA031,EA031-5,400-1000,"From 400 to 1,000 persons",632.4555320336759,400.0,1000.0,range
	8	+EA031,EA031-6,1000-5000,"More than 1,000 persons in the absence of indigenous urban aggregations of more than 5,000",2236.06797749979,1000.0,5000.0,range
	9	+EA031,EA031-7,5000-50000,"One or more indigenous towns of more than 5,000 inhabitants but none of more than 50,000",15811.388300841896,5000.0,50000.0,range
	10	+EA031,EA031-8,50000+,"One or more indigenous cities with more than 50,000 inhabitants",50000.0,50000.0,,open_upper

added results/diagnostics/dplace_coverage_by_variable.csv +18 −0

		`@@ -0,0 +1,18 @@`
	1	+variable,societies_coded
	2	+EA005,1290
	3	+EA028,1188
	4	+EA030,1187
	5	+EA032,1167
	6	+EA033,1155
	7	+EA034,775
	8	+EA042,1290
	9	+EA066,1109
	10	+EA067,1109
	11	+EA068,1103
	12	+EA070,1121
	13	+EA072,937
	14	+EA073,937
	15	+EA090,335
	16	+EA113,32
	17	+EA202_mid,953
	18	+EA031_mid,611

added results/diagnostics/modern_coverage_by_variable.csv +32 −0

		`@@ -0,0 +1,32 @@`
	1	+variable,country_years_coded,first_year,last_year,countries
	2	+e_regionpol_6C,28092,1789,2025,202
	3	+e_regionpol,28092,1789,2025,202
	4	+v2x_polyarchy,26954,1789,2025,202
	5	+e_civil_war,13776,1816,2006,163
	6	+e_miinterc,21218,1789,2000,171
	7	+e_coups,13813,1789,2008,169
	8	+e_pt_coup,11062,1950,2025,178
	9	+v2regendtype,28052,1789,2025,202
	10	+v2regint,28051,1789,2025,202
	11	+v2regdur,28052,1789,2025,202
	12	+e_pop,22191,1789,2019,191
	13	+e_area,13806,1886,2019,178
	14	+e_gdppc,22191,1789,2019,191
	15	+v2x_regime,19760,1900,2025,183
	16	+v2x_rule,27704,1789,2025,200
	17	+v2regidnr,28052,1789,2025,202
	18	+v2clrspct,27751,1789,2025,200
	19	+v2stcritrecadm,26344,1789,2025,197
	20	+v2cltrnslw,27631,1789,2025,200
	21	+v2stfisccap,26591,1789,2025,197
	22	+v2svstterr,19977,1789,2025,195
	23	+v2elembcap,25404,1789,2025,199
	24	+maddison_pop,15821,1820,2022,163
	25	+maddison_gdppc,15953,1789,2022,163
	26	+y4_civilwar_onset,13608,1817,2006,163
	27	+y5_coup,13813,1789,2008,169
	28	+regime_end_year,28092,1789,2025,202
	29	+y6_irregular_regime_end,27846,1789,2024,202
	30	+log_pop,15821,1820,2022,163
	31	+log_area,13806,1886,2019,178
	32	+log_gdppc,15953,1789,2022,163

added results/diagnostics/seshat_coverage_by_variable.csv +140 −0

		`@@ -0,0 +1,140 @@`
	1	+variable,polities_coded,facts,dated_facts,inferred_facts,disputed_facts,section,block
	2	+settlement_hierarchy,518,538,34,0,0,sc,D_scale
	3	+population_of_the_largest_settlement,361,497,250,0,0,sc,D_scale
	4	+polity_territory,355,504,261,0,0,sc,D_scale
	5	+polity_population,299,459,291,0,0,sc,D_scale
	6	+administrative_level,490,514,41,0,0,sc,admin
	7	+full_time_bureaucrat,416,434,19,147,2,sc,admin
	8	+specialized_government_building,393,405,7,142,0,sc,admin
	9	+examination_system,255,259,2,115,0,sc,admin
	10	+merit_promotion,252,262,11,120,0,sc,admin
	11	+road,396,401,12,133,0,sc,comm_infra
	12	+port,383,389,12,81,0,sc,comm_infra
	13	+courier,309,311,2,91,0,sc,comm_infra
	14	+bridge,276,276,3,74,0,sc,comm_infra
	15	+canal,258,259,4,67,0,sc,comm_infra
	16	+general_postal_service,254,257,8,95,1,sc,comm_infra
	17	+postal_station,251,254,5,80,2,sc,comm_infra
	18	+written_record,543,561,28,68,2,sc,records
	19	+script,538,553,26,53,1,sc,records
	20	+calendar,467,479,17,113,0,sc,records
	21	+phonetic_alphabetic_writing,456,469,21,36,0,sc,records
	22	+practical_literature,434,446,19,110,0,sc,records
	23	+lists_tables_and_classification,363,372,18,79,1,sc,records
	24	+non_phonetic_writing,314,322,8,35,2,sc,records
	25	+nonwritten_record,252,253,1,36,0,sc,records
	26	+mnemonic_device,122,122,2,26,0,sc,records
	27	+indigenous_coin,459,479,29,71,2,sc,rules_money
	28	+formal_legal_code,407,428,23,132,0,sc,rules_money
	29	+article,403,411,9,122,1,sc,rules_money
	30	+paper_currency,387,393,15,82,3,sc,rules_money
	31	+foreign_coin,378,384,12,114,1,sc,rules_money
	32	+judge,361,377,16,144,0,sc,rules_money
	33	+court,330,338,7,128,0,sc,rules_money
	34	+precious_metal,311,313,3,95,0,sc,rules_money
	35	+professional_lawyer,309,315,7,124,0,sc,rules_money
	36	+token,229,238,5,69,1,sc,rules_money
	37	+weight_measurement_system,110,112,5,54,0,sc,standards
	38	+length_measurement_system,102,104,5,56,0,sc,standards
	39	+volume_measurement_system,93,95,6,52,0,sc,standards
	40	+time_measurement_system,92,92,0,35,1,sc,standards
	41	+area_measurement_system,86,88,5,50,0,sc,standards
	42	+religious_literature,510,524,21,115,3,sc,
	43	+sacred_text,478,495,25,66,1,sc,
	44	+fiction,424,434,13,91,1,sc,
	45	+market,423,427,7,154,0,sc,
	46	+history,420,430,9,80,2,sc,
	47	+irrigation_system,400,403,5,110,0,sc,
	48	+professional_priesthood,397,407,10,115,1,sc,
	49	+professional_soldier,394,433,51,144,0,sc,
	50	+scientific_literature,389,399,14,85,3,sc,
	51	+philosophy,371,381,12,94,2,sc,
	52	+professional_military_officer,369,396,31,146,0,sc,
	53	+food_storage_site,368,375,4,132,0,sc,
	54	+military_level,367,378,24,0,0,sc,
	55	+religious_level,364,368,10,0,0,sc,
	56	+iron,359,369,10,34,0,wf,
	57	+moralizing_enforcement_is_targeted,357,362,10,0,9,rt,
	58	+moralizing_supernatural_punishment_and_reward,356,361,9,0,3,rt,
	59	+moralizing_enforcement_is_agentic,354,359,10,0,3,rt,
	60	+moralizing_supernatural_concern_is_primary,352,357,11,0,3,rt,
	61	+moralizing_enforcement_is_broad,351,354,9,0,5,rt,
	62	+sword,349,355,2,78,0,wf,
	63	+handheld_firearm,346,359,22,14,0,wf,
	64	+moralizing_enforcement_of_rulers,345,348,7,0,6,rt,
	65	+modern_fortification,344,345,2,14,0,wf,
	66	+horse,343,350,9,30,0,wf,
	67	+moralizing_enforcement_is_certain,343,348,11,0,13,rt,
	68	+gunpowder_siege_artillery,341,350,18,15,0,wf,
	69	+moralizing_enforcement_in_this_life,338,343,10,0,3,rt,
	70	+moralizing_enforcement_in_afterlife,333,337,10,0,5,rt,
	71	+bronze,330,332,5,67,0,wf,
	72	+atlatl,329,332,4,49,0,wf,
	73	+helmet,327,331,4,89,0,wf,
	74	+spear,327,332,5,75,0,wf,
	75	+steel,326,340,6,50,0,wf,
	76	+dagger,325,333,4,79,0,wf,
	77	+composite_bow,315,319,4,89,0,wf,
	78	+moralizing_religion_adopted_by_elites,309,314,8,0,4,rt,
	79	+chainmail,308,312,3,60,0,wf,
	80	+copper,306,308,5,105,0,wf,
	81	+camel,305,310,5,59,0,wf,
	82	+sling_siege_engine,304,320,9,36,0,wf,
	83	+moralizing_religion_adopted_by_commoners,303,310,6,0,8,rt,
	84	+mines_or_quarry,302,304,5,76,0,sc,
	85	+battle_axe,300,302,3,106,0,wf,
	86	+crossbow,300,302,2,53,0,wf,
	87	+self_bow,295,307,7,105,0,wf,
	88	+plate_armor,294,302,8,81,0,wf,
	89	+shield,292,297,6,71,0,wf,
	90	+tension_siege_engine,291,294,0,59,0,wf,
	91	+human_sacrifice,286,316,314,0,5,rt,
	92	+leather_cloth,286,289,3,90,0,wf,
	93	+elephant,285,288,5,29,0,wf,
	94	+drinking_water_supply_system,284,289,4,93,0,sc,
	95	+settlements_in_a_defensive_position,282,283,3,73,0,wf,
	96	+earth_rampart,272,279,8,93,0,wf,
	97	+war_club,272,275,4,120,0,wf,
	98	+specialized_military_vessel,268,271,4,58,0,wf,
	99	+javelin,267,280,4,132,0,wf,
	100	+complex_fortification,263,266,2,60,0,wf,
	101	+limb_protection,262,266,3,98,0,wf,
	102	+stone_walls_mortared,261,269,4,62,0,wf,
	103	+scaled_armor,260,262,7,52,0,wf,
	104	+donkey,257,262,1,79,0,wf,
	105	+sling,256,260,1,108,0,wf,
	106	+breastplate,252,257,4,85,0,wf,
	107	+laminar_armor,250,251,2,49,0,wf,
	108	+small_vessels_canoes_etc,250,252,1,76,0,wf,
	109	+polearm,242,246,4,91,0,wf,
	110	+ditch,234,241,3,89,0,wf,
	111	+moat,232,240,3,83,0,wf,
	112	+stone_walls_non_mortared,217,220,5,69,0,wf,
	113	+wood_bark_etc,214,216,4,80,0,wf,
	114	+sync_rel_pra_ind_beli,211,217,8,0,0,rt,
	115	+wooden_palisade,210,219,8,80,0,wf,
	116	+fortified_camp,204,207,5,48,0,wf,
	117	+merchant_ships_pressed_into_service,179,179,0,40,0,wf,
	118	+dog,165,165,1,34,0,wf,
	119	+utilitarian_public_building,151,153,0,38,0,sc,
	120	+communal_building,123,123,3,15,0,sc,
	121	+symbolic_building,115,116,1,18,1,sc,
	122	+knowledge_or_information_building,103,106,6,33,1,sc,
	123	+special_purpose_site,97,97,1,19,0,sc,
	124	+burial_site,89,90,1,7,0,sc,
	125	+ceremonial_site,88,91,0,8,1,sc,
	126	+religious_fragmentation,82,84,3,0,0,rt,
	127	+trading_emporia,74,74,4,38,1,sc,
	128	+special_purpose_house,62,62,1,6,0,sc,
	129	+store_of_wealth,58,58,1,17,0,sc,
	130	+debt_and_credit_structure,51,53,5,10,1,sc,
	131	+enclosure,50,50,0,4,0,sc,
	132	+geometrical_measurement_system,50,51,3,29,1,sc,
	133	+entertainment_building,45,49,6,8,0,sc,
	134	+largest_communication_distance,27,27,0,0,0,sc,
	135	+other_measurement_system,17,18,2,8,0,sc,
	136	+other_special_purpose_site,16,16,0,2,0,sc,
	137	+fastest_individual_communication,11,12,4,0,0,sc,
	138	+other_utilitarian_public_building,10,10,0,0,0,sc,
	139	+occupational_complexity,6,8,3,4,0,sc,
	140	+source_of_support,1,2,0,2,0,sc,

added results/diagnostics/seshat_missingness_by_indicator.csv +49 −0

		`@@ -0,0 +1,49 @@`
	1	+indicator,share_polities_missing
	2	+full_time_bureaucrat,0.32794830371567046
	3	+specialized_government_building,0.3651050080775444
	4	+merit_promotion,0.592891760904685
	5	+examination_system,0.5880452342487884
	6	+administrative_level,0.2084006462035541
	7	+formal_legal_code,0.34248788368336025
	8	+judge,0.4168012924071082
	9	+court,0.4668820678513732
	10	+professional_lawyer,0.5008077544426495
	11	+indigenous_coin,0.25848142164781907
	12	+foreign_coin,0.3893376413570275
	13	+paper_currency,0.37479806138933763
	14	+token,0.630048465266559
	15	+precious_metal,0.4975767366720517
	16	+article,0.34894991922455576
	17	+courier,0.5008077544426495
	18	+postal_station,0.5945072697899838
	19	+general_postal_service,0.5896607431340872
	20	+road,0.3602584814216478
	21	+bridge,0.5541195476575121
	22	+canal,0.5831987075928917
	23	+port,0.38126009693053314
	24	+written_record,0.12277867528271405
	25	+script,0.1308562197092084
	26	+phonetic_alphabetic_writing,0.2633279483037157
	27	+non_phonetic_writing,0.4927302100161551
	28	+lists_tables_and_classification,0.4135702746365105
	29	+calendar,0.2455573505654281
	30	+practical_literature,0.2988691437802908
	31	+nonwritten_record,0.592891760904685
	32	+mnemonic_device,0.802907915993538
	33	+polity_population,0.5169628432956381
	34	+polity_territory,0.42649434571890144
	35	+population_of_the_largest_settlement,0.4168012924071082
	36	+settlement_hierarchy,0.1631663974151858
	37	+length_measurement_system,0.8352180936995154
	38	+area_measurement_system,0.8610662358642972
	39	+volume_measurement_system,0.8497576736672051
	40	+weight_measurement_system,0.8222940226171244
	41	+time_measurement_system,0.851373182552504
	42	+irrigation_system,0.35379644588045234
	43	+drinking_water_supply_system,0.5411954765751211
	44	+market,0.3166397415185784
	45	+food_storage_site,0.40549273021001614
	46	+mines_or_quarry,0.5121163166397416
	47	+professional_soldier,0.36348949919224555
	48	+professional_military_officer,0.40387722132471726
	49	+military_level,0.407108239095315

added results/diagnostics/seshat_missingness_by_polity.csv +620 −0

		`@@ -0,0 +1,620 @@`
	1	+polity_id,n_K_indicators_coded,n_D_coded,macro_region,era,start_year,end_year
	2	+Early Niynginya,23,2,Sub_Saharan_Africa,1501-1800,1650,1897
	3	+af_durrani_emp,28,2,Central_Asia,1501-1800,1747,1826
	4	+af_ghaznavid_emp,2,0,Central_Asia,1001-1500,998,1040
	5	+af_ghur_principality,18,3,Central_Asia,1001-1500,1025,1215
	6	+af_greco_bactrian_k,23,4,Central_Asia,1000BCE-0,-256,-125
	7	+af_hephthalite_emp,19,3,Central_Asia,1-1000,408,561
	8	+af_kidarite_k,17,3,Central_Asia,1-1000,388,477
	9	+af_kushan_emp,25,4,Central_Asia,1-1000,35,319
	10	+af_tocharian,14,3,Central_Asia,1000BCE-0,-129,29
	11	+am_artaxiad_dyn,12,2,Southwest_Asia,1000BCE-0,-188,6
	12	+ao_kongo_2,20,3,Sub_Saharan_Africa,1501-1800,1491,1568
	13	+at_austro_hungarian_emp,11,1,Europe,post-1800,1867,1918
	14	+at_habsburg_1,23,4,Europe,1501-1800,1454,1648
	15	+at_habsburg_2,25,4,Europe,1501-1800,1649,1918
	16	+bd_bengal_sultanate,12,2,South_Asia,1001-1500,1338,1538
	17	+bd_chandra_dyn,9,0,South_Asia,1-1000,900,1050
	18	+bd_deva_dyn,6,0,South_Asia,1001-1500,1150,1300
	19	+bd_khadga_dyn,7,0,South_Asia,1-1000,650,700
	20	+bd_nawabs_of_bengal,8,1,South_Asia,1501-1800,1717,1757
	21	+bd_sena_dyn,9,0,South_Asia,1001-1500,1095,1245
	22	+bd_twelve_bhuyans,5,0,South_Asia,1501-1800,1538,1612
	23	+bd_yadava_varman_dyn,6,0,South_Asia,1001-1500,1080,1150
	24	+bf_mossi_k_1,9,0,Sub_Saharan_Africa,1001-1500,1100,1897
	25	+bf_west_burkina_faso_red_1,2,0,Sub_Saharan_Africa,1-1000,701,1100
	26	+bf_west_burkina_faso_red_2,7,1,Sub_Saharan_Africa,1001-1500,1100,1400
	27	+bf_west_burkina_faso_red_4,7,1,Sub_Saharan_Africa,1001-1500,1401,1500
	28	+bf_west_burkina_faso_yellow_5,6,1,Sub_Saharan_Africa,1-1000,100,500
	29	+bf_west_burkina_faso_yellow_6,1,1,Sub_Saharan_Africa,1-1000,501,700
	30	+bg_bulgaria_early,17,2,Europe,1-1000,681,864
	31	+bg_bulgaria_medieval,17,2,Europe,1-1000,865,1018
	32	+bo_tiwanaku_2,2,3,South_America,1-1000,800,1149
	33	+bu_burundi_k,19,2,Sub_Saharan_Africa,1501-1800,1680,1903
	34	+cd_kanem,2,0,Sub_Saharan_Africa,1001-1500,800,1379
	35	+cn_chu_dyn_spring_autumn,27,2,East_Asia,1000BCE-0,-740,-489
	36	+cn_chu_k_warring_states,20,2,East_Asia,1000BCE-0,-488,-223
	37	+cn_eastern_han_dyn,30,4,East_Asia,1-1000,25,220
	38	+cn_eastern_zhou_warring_states,20,1,East_Asia,1000BCE-0,-475,-256
	39	+cn_erligang,20,4,East_Asia,pre-1000BCE,-1650,-1250
	40	+cn_erlitou,21,4,East_Asia,pre-1000BCE,-1850,-1600
	41	+cn_five_dyn,14,0,East_Asia,1-1000,906,970
	42	+cn_hmong_1,19,4,East_Asia,1501-1800,1701,1895
	43	+cn_hmong_2,28,1,East_Asia,post-1800,1895,1941
	44	+cn_jin_spring_and_autumn,28,2,East_Asia,1000BCE-0,-780,-404
	45	+cn_late_shang_dyn,24,4,East_Asia,pre-1000BCE,-1250,-1045
	46	+cn_later_great_jin,28,2,East_Asia,1001-1500,1115,1234
	47	+cn_later_qin_dyn,18,2,East_Asia,1-1000,386,417
	48	+cn_later_yan_dyn,14,1,East_Asia,1-1000,385,409
	49	+cn_longshan,18,3,East_Asia,pre-1000BCE,-3000,-1900
	50	+cn_ming_dyn,31,4,East_Asia,1501-1800,1368,1644
	51	+cn_northern_song_dyn,29,4,East_Asia,1001-1500,960,1127
	52	+cn_northern_wei_dyn,28,4,East_Asia,1-1000,386,534
	53	+cn_peiligang,1,2,East_Asia,pre-1000BCE,-7000,-5001
	54	+cn_qin_emp,26,4,East_Asia,1000BCE-0,-338,-207
	55	+cn_qing_dyn_1,30,4,East_Asia,1501-1800,1644,1796
	56	+cn_qing_dyn_2,30,4,East_Asia,post-1800,1796,1912
	57	+cn_southern_qi_dyn,13,1,East_Asia,1-1000,479,502
	58	+cn_southern_song_dyn,25,0,East_Asia,1001-1500,1127,1279
	59	+cn_sui_dyn,28,4,East_Asia,1-1000,581,618
	60	+cn_tang_dyn_1,29,4,East_Asia,1-1000,617,763
	61	+cn_tang_dyn_2,29,4,East_Asia,1-1000,763,907
	62	+cn_tuyuhun,8,2,Central_Asia,1-1000,300,663
	63	+cn_viet_baiyu_k,7,0,Southeast_Asia_Oceania,1000BCE-0,-332,-109
	64	+cn_wei_dyn_warring_states,25,2,East_Asia,1000BCE-0,-445,-225
	65	+cn_wei_k,11,1,East_Asia,1-1000,220,265
	66	+cn_western_han_dyn,30,4,East_Asia,1000BCE-0,-202,9
	67	+cn_western_jin_dyn,28,4,East_Asia,1-1000,265,317
	68	+cn_western_zhou_dyn,26,4,East_Asia,1000BCE-0,-1122,-771
	69	+cn_wu_confederacy,1,1,East_Asia,1000BCE-0,-585,-477
	70	+cn_xixia,15,1,Central_Asia,1001-1500,1032,1227
	71	+cn_yangshao,17,3,East_Asia,pre-1000BCE,-5000,-3000
	72	+cn_yuan_dyn,29,4,East_Asia,1001-1500,1271,1368
	73	+cn_yue_dyn,1,1,East_Asia,1000BCE-0,-510,-334
	74	+co_neguanje,13,2,North_America,1-1000,250,1050
	75	+co_tairona,20,4,North_America,1001-1500,1050,1524
	76	+cz_bohemian_k_2,21,4,Europe,1001-1500,1310,1526
	77	+de_east_francia,28,4,Europe,1-1000,842,919
	78	+de_empire_1,24,4,Europe,1001-1500,919,1125
	79	+de_empire_2,1,0,Europe,1001-1500,1126,1254
	80	+dz_numidia,16,4,North_Africa,1000BCE-0,-220,-46
	81	+dz_tahert,17,2,North_Africa,1-1000,761,909
	82	+dz_tlemcen,19,4,North_Africa,1001-1500,1235,1554
	83	+ec_shuar_1,30,4,South_America,1501-1800,1534,1830
	84	+ec_shuar_2,30,2,South_America,post-1800,1831,1931
	85	+eg_ayyubid_sultanate,30,4,North_Africa,1001-1500,1171,1250
	86	+eg_badarian,28,2,North_Africa,pre-1000BCE,-4400,-3800
	87	+eg_dynasty_1,30,4,North_Africa,pre-1000BCE,-3100,-2900
	88	+eg_dynasty_2,29,4,North_Africa,pre-1000BCE,-2900,-2687
	89	+eg_inter_occupation,29,4,North_Africa,1000BCE-0,-404,-342
	90	+eg_kushite,25,4,North_Africa,1000BCE-0,-747,-656
	91	+eg_mamluk_sultanate_1,30,4,North_Africa,1001-1500,1260,1348
	92	+eg_mamluk_sultanate_2,30,4,North_Africa,1001-1500,1348,1412
	93	+eg_mamluk_sultanate_3,30,4,North_Africa,1001-1500,1412,1517
	94	+eg_middle_k,27,4,North_Africa,pre-1000BCE,-2016,-1700
	95	+eg_naqada_1,29,4,North_Africa,pre-1000BCE,-3800,-3550
	96	+eg_naqada_2,27,4,North_Africa,pre-1000BCE,-3550,-3300
	97	+eg_naqada_3,26,4,North_Africa,pre-1000BCE,-3300,-3100
	98	+eg_new_k_1,27,4,North_Africa,pre-1000BCE,-1550,-1293
	99	+eg_new_k_2,26,4,North_Africa,pre-1000BCE,-1293,-1070
	100	+eg_old_k_1,29,4,North_Africa,pre-1000BCE,-2650,-2350
	101	+eg_old_k_2,30,4,North_Africa,pre-1000BCE,-2350,-2150
	102	+eg_ptolemaic_k_1,31,4,North_Africa,1000BCE-0,-305,-217
	103	+eg_ptolemaic_k_2,30,4,North_Africa,1000BCE-0,-217,-30
	104	+eg_regions,27,3,North_Africa,pre-1000BCE,-2150,-2016
	105	+eg_saite,27,4,North_Africa,1000BCE-0,-664,-525
	106	+eg_thebes_hyksos,27,3,North_Africa,pre-1000BCE,-1720,-1567
	107	+eg_thebes_libyan,24,3,North_Africa,1000BCE-0,-1069,-747
	108	+eg_thulunid_ikhshidid,29,4,North_Africa,1-1000,868,969
	109	+er_medri_bahri,5,0,Sub_Saharan_Africa,1501-1800,1310,1889
	110	+es_castile_crown,8,0,Europe,1001-1500,1231,1515
	111	+es_castile_k,2,1,Europe,1001-1500,1065,1230
	112	+es_leon_castille,0,2,Europe,1001-1500,1230,1715
	113	+es_spanish_emp_1,29,4,Europe,1501-1800,1516,1715
	114	+es_spanish_emp_2,22,4,Europe,1501-1800,1716,1814
	115	+et_aksum_emp_1,16,4,Sub_Saharan_Africa,1-1000,-149,349
	116	+et_aksum_emp_2,16,4,Sub_Saharan_Africa,1-1000,350,599
	117	+et_aksum_emp_3,16,4,Sub_Saharan_Africa,1-1000,600,800
	118	+et_aussa_sultanate,6,0,Sub_Saharan_Africa,post-1800,1734,1895
	119	+et_funj_sultanate,14,1,Sub_Saharan_Africa,1501-1800,1504,1820
	120	+et_gomma_k,7,1,Sub_Saharan_Africa,post-1800,1780,1886
	121	+et_gumma_k,7,0,Sub_Saharan_Africa,post-1800,1800,1897
	122	+et_hadiya_sultanate,4,0,Sub_Saharan_Africa,1001-1500,1300,1680
	123	+et_harar_emirate,12,2,Sub_Saharan_Africa,1501-1800,1650,1875
	124	+et_harla_k,8,0,Sub_Saharan_Africa,1-1000,500,1500
	125	+et_jimma_k,12,3,Sub_Saharan_Africa,post-1800,1790,1932
	126	+et_kaffa_k,10,1,Sub_Saharan_Africa,1501-1800,1390,1897
	127	+et_showa_sultanate,10,0,Sub_Saharan_Africa,1001-1500,1108,1285
	128	+et_zagwe,14,1,Sub_Saharan_Africa,1001-1500,1137,1269
	129	+fm_truk_1,31,2,Southeast_Asia_Oceania,post-1800,1775,1886
	130	+fm_truk_2,29,3,Southeast_Asia_Oceania,post-1800,1886,1948
	131	+fr_aquitaine_duc_1,9,2,Europe,1-1000,602,768
	132	+fr_atlantic_complex,4,3,Europe,pre-1000BCE,-2200,-1000
	133	+fr_beaker_eba,5,3,Europe,pre-1000BCE,-3200,-2000
	134	+fr_bourbon_k_1,24,4,Europe,1501-1800,1589,1660
	135	+fr_bourbon_k_2,24,4,Europe,1501-1800,1660,1815
	136	+fr_capetian_k_1,20,4,Europe,1001-1500,987,1150
	137	+fr_capetian_k_2,24,4,Europe,1001-1500,1150,1328
	138	+fr_carolingian_emp_1,21,4,Europe,1-1000,752,840
	139	+fr_carolingian_emp_2,23,4,Europe,1-1000,840,987
	140	+fr_france_napoleonic,30,4,Europe,post-1800,1816,1870
	141	+fr_hallstatt_a_b1,4,4,Europe,1000BCE-0,-1000,-900
	142	+fr_hallstatt_b2_3,3,3,Europe,1000BCE-0,-900,-700
	143	+fr_hallstatt_c,3,4,Europe,1000BCE-0,-700,-600
	144	+fr_hallstatt_d,7,3,Europe,1000BCE-0,-600,-475
	145	+fr_la_tene_a_b1,13,3,Europe,1000BCE-0,-475,-325
	146	+fr_la_tene_b2_c1,17,4,Europe,1000BCE-0,-325,-175
	147	+fr_la_tene_c2_d,17,4,Europe,1000BCE-0,-175,-27
	148	+fr_merovingian_emp_1,21,4,Europe,1-1000,481,543
	149	+fr_merovingian_emp_2,21,4,Europe,1-1000,543,687
	150	+fr_merovingian_emp_3,22,4,Europe,1-1000,687,751
	151	+fr_valois_k_1,21,4,Europe,1001-1500,1328,1450
	152	+fr_valois_k_2,22,4,Europe,1501-1800,1450,1589
	153	+gb_anglo_saxon_1,20,3,Europe,1-1000,410,926
	154	+gb_anglo_saxon_2,19,4,Europe,1-1000,927,1065
	155	+gb_british_emp_1,30,4,Europe,1501-1800,1690,1849
	156	+gb_british_emp_2,30,4,Europe,post-1800,1850,1968
	157	+gb_england_norman,31,4,Europe,1001-1500,1066,1153
	158	+gb_england_plantagenet,29,4,Europe,1001-1500,1154,1485
	159	+gb_tudor_stuart,28,4,Europe,1501-1800,1486,1689
	160	+ge_georgia_k_2,17,1,Southwest_Asia,1001-1500,975,1243
	161	+gh_akan,30,1,Sub_Saharan_Africa,1501-1800,1501,1701
	162	+gh_ashanti_emp,28,4,Sub_Saharan_Africa,1501-1800,1701,1895
	163	+gm_kaabu_emp,13,0,Sub_Saharan_Africa,1501-1800,1500,1867
	164	+gr_crete_archaic,27,3,Europe,1000BCE-0,-710,-500
	165	+gr_crete_classical,27,4,Europe,1000BCE-0,-500,-323
	166	+gr_crete_emirate,25,4,Europe,1-1000,824,961
	167	+gr_crete_geometric,18,2,Europe,1000BCE-0,-1000,-710
	168	+gr_crete_hellenistic,25,4,Europe,1000BCE-0,-323,-69
	169	+gr_crete_mono_palace,26,4,Europe,pre-1000BCE,-1450,-1300
	170	+gr_crete_new_palace,24,2,Europe,pre-1000BCE,-1700,-1450
	171	+gr_crete_nl,23,2,Europe,pre-1000BCE,-7000,-3000
	172	+gr_crete_old_palace,23,2,Europe,pre-1000BCE,-1900,-1700
	173	+gr_crete_post_palace_1,26,2,Europe,pre-1000BCE,-1300,-1200
	174	+gr_crete_post_palace_2,21,2,Europe,pre-1000BCE,-1200,-1000
	175	+gr_crete_pre_palace,25,2,Europe,pre-1000BCE,-3000,-1900
	176	+gr_macedonian_emp,17,4,Europe,1000BCE-0,-330,-312
	177	+gt_tikal_early_classic,0,1,North_America,1-1000,250,554
	178	+gt_tikal_late_classic,7,3,North_America,1-1000,555,869
	179	+gu_futa_jallon,11,1,Sub_Saharan_Africa,post-1800,1725,1896
	180	+hu_avar_khaganate,11,2,Europe,1-1000,586,822
	181	+hu_hun_k,5,2,Europe,1-1000,376,469
	182	+ib_banu_ghaniya,15,1,North_Africa,1001-1500,1126,1227
	183	+id_buni,1,1,Southeast_Asia_Oceania,1-1000,-400,500
	184	+id_iban_1,30,3,Southeast_Asia_Oceania,1501-1800,1650,1841
	185	+id_iban_2,30,3,Southeast_Asia_Oceania,post-1800,1841,1987
	186	+id_kalingga_k,8,1,Southeast_Asia_Oceania,1-1000,500,732
	187	+id_kediri_k,13,3,Southeast_Asia_Oceania,1001-1500,1049,1222
	188	+id_majapahit_k,19,4,Southeast_Asia_Oceania,1001-1500,1292,1518
	189	+id_mataram_k,18,3,Southeast_Asia_Oceania,1501-1800,1568,1755
	190	+id_medang_k,16,4,Southeast_Asia_Oceania,1-1000,732,1019
	191	+il_canaan,22,4,Southwest_Asia,pre-1000BCE,-2000,-1175
	192	+il_judea,16,3,Southwest_Asia,1000BCE-0,-141,-63
	193	+il_yisrael,17,4,Southwest_Asia,1000BCE-0,-1030,-722
	194	+in_achik_1,29,3,South_Asia,post-1800,1775,1867
	195	+in_achik_2,26,4,South_Asia,post-1800,1867,1956
	196	+in_ayodhya_k,9,0,South_Asia,1000BCE-0,-64,34
	197	+in_badami_chalukya_emp,14,4,South_Asia,1-1000,543,753
	198	+in_carnatic_sul,8,1,South_Asia,1501-1800,1710,1801
	199	+in_chandela_k,9,1,South_Asia,1001-1500,950,1308
	200	+in_chauhana_dyn,12,1,South_Asia,1001-1500,973,1192
	201	+in_chaulukya_dyn,15,1,South_Asia,1001-1500,941,1245
	202	+in_chola_emp,1,0,South_Asia,1001-1500,849,1280
	203	+in_cholas_1,14,1,South_Asia,1000BCE-0,-300,300
	204	+in_deccan_ia,4,3,South_Asia,1000BCE-0,-1200,-300
	205	+in_deccan_nl,12,1,South_Asia,pre-1000BCE,-2700,-1200
	206	+in_delhi_sultanate,24,3,South_Asia,1001-1500,1206,1526
	207	+in_east_india_co,17,3,South_Asia,post-1800,1757,1858
	208	+in_gahadavala_dyn,12,2,South_Asia,1001-1500,1085,1193
	209	+in_ganga_ca,0,1,South_Asia,pre-1000BCE,-3000,-601
	210	+in_ganga_nl,0,1,South_Asia,pre-1000BCE,-7000,-3001
	211	+in_gauda_k,6,0,South_Asia,1-1000,600,625
	212	+in_gupta_emp,15,3,South_Asia,1-1000,320,550
	213	+in_gurjara_pratihara_dyn,13,2,South_Asia,1-1000,730,1030
	214	+in_hoysala_k,11,2,South_Asia,1001-1500,1108,1346
	215	+in_kadamba_emp,12,3,South_Asia,1-1000,345,550
	216	+in_kakatiya_dyn,16,2,South_Asia,1001-1500,1175,1324
	217	+in_kalabhra_dyn,8,1,South_Asia,1-1000,200,600
	218	+in_kalachuri_emp,1,3,South_Asia,1001-1500,1157,1184
	219	+in_kalyani_chalukya_emp,11,4,South_Asia,1001-1500,973,1189
	220	+in_kamarupa_k,15,0,South_Asia,1-1000,350,1130
	221	+in_kampili_k,15,2,South_Asia,1001-1500,1280,1327
	222	+in_kannauj_varman_dyn,11,1,South_Asia,1-1000,650,780
	223	+in_karkota_dyn,8,0,South_Asia,1-1000,625,1339
	224	+in_madurai_nayaks,10,1,South_Asia,1501-1800,1529,1736
	225	+in_magadha_k,8,0,South_Asia,1-1000,450,605
	226	+in_mahajanapada,21,2,South_Asia,1000BCE-0,-600,-324
	227	+in_maitraka_dyn,2,1,South_Asia,1-1000,470,790
	228	+in_maukhari_dyn,9,1,South_Asia,1-1000,550,605
	229	+in_mauryan_emp,29,4,South_Asia,1000BCE-0,-324,-187
	230	+in_mughal_emp,27,4,South_Asia,1501-1800,1526,1858
	231	+in_pallava_emp_2,11,1,South_Asia,1-1000,300,890
	232	+in_pandya_emp_1,13,1,South_Asia,1000BCE-0,-300,300
	233	+in_pandya_emp_2,9,1,South_Asia,1-1000,590,915
	234	+in_pandya_emp_3,7,1,South_Asia,1001-1500,1216,1323
	235	+in_paramara_dyn,7,1,South_Asia,1001-1500,974,1235
	236	+in_post_mauryan_k,8,1,South_Asia,1000BCE-0,-205,-101
	237	+in_rashtrakuta_emp,19,4,South_Asia,1-1000,753,973
	238	+in_satavahana_emp,15,4,South_Asia,1-1000,-100,200
	239	+in_sharqi_dyn,7,2,South_Asia,1001-1500,1394,1479
	240	+in_sunga_emp,11,4,South_Asia,1000BCE-0,-187,-65
	241	+in_thanjavur_maratha_k,10,1,South_Asia,1501-1800,1675,1799
	242	+in_thanjavur_nayaks,7,1,South_Asia,1501-1800,1532,1676
	243	+in_vakataka_k,15,3,South_Asia,1-1000,255,550
	244	+in_vijayanagara_emp,19,4,South_Asia,1001-1500,1336,1646
	245	+in_yadava_dyn,6,1,South_Asia,1001-1500,1190,1318
	246	+iq_abbasid_cal_1,30,4,Southwest_Asia,1-1000,750,946
	247	+iq_abbasid_cal_2,23,4,Southwest_Asia,1001-1500,1191,1258
	248	+iq_akkad_emp,26,3,Southwest_Asia,pre-1000BCE,-2270,-2083
	249	+iq_babylonia_1,11,2,Southwest_Asia,pre-1000BCE,-2000,-1600
	250	+iq_babylonia_2,24,2,Southwest_Asia,pre-1000BCE,-1595,-1150
	251	+iq_bazi_dyn,24,1,Southwest_Asia,1000BCE-0,-1005,-986
	252	+iq_dynasty_e,25,1,Southwest_Asia,1000BCE-0,-979,-732
	253	+iq_early_dynastic,12,3,Southwest_Asia,pre-1000BCE,-2900,-2500
	254	+iq_isin_dynasty2,24,1,Southwest_Asia,pre-1000BCE,-1153,-1027
	255	+iq_isin_larsa,25,2,Southwest_Asia,pre-1000BCE,-2004,-1763
	256	+iq_lakhmid_k,5,0,Southwest_Asia,1-1000,400,611
	257	+iq_neo_assyrian_emp,27,4,Southwest_Asia,1000BCE-0,-911,-612
	258	+iq_neo_babylonian_emp,19,2,Southwest_Asia,1000BCE-0,-626,-539
	259	+iq_parthian_emp_2,25,4,Southwest_Asia,1-1000,41,226
	260	+iq_so_mesopotamia_nl,1,1,Southwest_Asia,pre-1000BCE,-9000,-5501
	261	+iq_ubaid,14,2,Southwest_Asia,pre-1000BCE,-5500,-4000
	262	+iq_ur_dyn_3,18,3,Southwest_Asia,pre-1000BCE,-2112,-2004
	263	+iq_uruk,14,4,Southwest_Asia,pre-1000BCE,-4000,-2900
	264	+ir_achaemenid_emp,29,4,Southwest_Asia,1000BCE-0,-550,-331
	265	+ir_ak_koyunlu,20,2,Southwest_Asia,1001-1500,1339,1501
	266	+ir_buyid_confederation,22,4,Southwest_Asia,1-1000,932,1062
	267	+ir_elam_1,21,3,Southwest_Asia,pre-1000BCE,-2675,-2100
	268	+ir_elam_2,17,2,Southwest_Asia,pre-1000BCE,-2028,-1940
	269	+ir_elam_3,19,2,Southwest_Asia,pre-1000BCE,-1900,-1701
	270	+ir_elam_4,19,2,Southwest_Asia,pre-1000BCE,-1700,-1500
	271	+ir_elam_5,19,3,Southwest_Asia,pre-1000BCE,-1500,-1400
	272	+ir_elam_6,18,3,Southwest_Asia,pre-1000BCE,-1399,-1200
	273	+ir_elam_7,21,2,Southwest_Asia,pre-1000BCE,-1199,-1100
	274	+ir_elam_8,9,0,Southwest_Asia,1000BCE-0,-1100,-900
	275	+ir_elymais_2,22,2,Southwest_Asia,1-1000,25,215
	276	+ir_il_khanate,27,4,Southwest_Asia,1001-1500,1256,1339
	277	+ir_jayarid_khanate,12,2,Southwest_Asia,1001-1500,1336,1393
	278	+ir_median_emp,4,2,Southwest_Asia,1000BCE-0,-715,-550
	279	+ir_neo_elam_1,10,1,Southwest_Asia,1000BCE-0,-900,-744
	280	+ir_neo_elam_2,20,2,Southwest_Asia,1000BCE-0,-743,-647
	281	+ir_neo_elam_3,16,0,Southwest_Asia,1000BCE-0,-612,-539
	282	+ir_parthian_emp_1,25,4,Southwest_Asia,1000BCE-0,-247,40
	283	+ir_qajar_dyn,17,4,Southwest_Asia,post-1800,1794,1925
	284	+ir_safavid_emp,27,4,Southwest_Asia,1501-1800,1501,1722
	285	+ir_saffarid_emp,17,1,Southwest_Asia,1-1000,861,1003
	286	+ir_sassanid_emp_1,23,4,Southwest_Asia,1-1000,205,487
	287	+ir_sassanid_emp_2,26,4,Southwest_Asia,1-1000,488,642
	288	+ir_seleucid_emp,19,4,Southwest_Asia,1000BCE-0,-312,-63
	289	+ir_seljuk_sultanate,27,4,Southwest_Asia,1001-1500,1037,1157
	290	+ir_susa_1,18,4,Southwest_Asia,pre-1000BCE,-4300,-3800
	291	+ir_susa_2,17,3,Southwest_Asia,pre-1000BCE,-3800,-3100
	292	+ir_susa_3,18,2,Southwest_Asia,pre-1000BCE,-3100,-2675
	293	+ir_susiana_a,19,2,Southwest_Asia,pre-1000BCE,-6000,-5700
	294	+ir_susiana_archaic,21,2,Southwest_Asia,pre-1000BCE,-7000,-6000
	295	+ir_susiana_b,21,2,Southwest_Asia,pre-1000BCE,-5700,-5100
	296	+ir_susiana_formative,20,2,Southwest_Asia,pre-1000BCE,-7200,-7000
	297	+ir_susiana_pre_ceramic,19,2,Southwest_Asia,pre-1000BCE,-7800,-7200
	298	+ir_susiana_ubaid_1,20,2,Southwest_Asia,pre-1000BCE,-5100,-4700
	299	+ir_susiana_ubaid_2,19,2,Southwest_Asia,pre-1000BCE,-4700,-4300
	300	+is_icelandic_commonwealth,27,4,Europe,1001-1500,930,1262
	301	+it_italy_k,14,2,Europe,1-1000,476,489
	302	+it_latium_ba,28,3,Europe,pre-1000BCE,-1800,-900
	303	+it_latium_ca,30,1,Europe,pre-1000BCE,-3600,-1800
	304	+it_latium_ia,22,2,Europe,1000BCE-0,-1000,-580
	305	+it_lombard_k,20,3,Europe,1-1000,568,774
	306	+it_ostrogoth_k,29,4,Europe,1-1000,489,554
	307	+it_papal_state_1,30,4,Europe,1001-1500,1198,1309
	308	+it_papal_state_2,31,4,Europe,1001-1500,1378,1527
	309	+it_papal_state_3,31,4,Europe,1501-1800,1527,1648
	310	+it_papal_state_4,31,4,Europe,1501-1800,1648,1809
	311	+it_ravenna_exarchate,29,4,Europe,1-1000,568,751
	312	+it_roman_k,27,4,Europe,1000BCE-0,-716,-509
	313	+it_roman_principate,30,4,Europe,1-1000,-31,284
	314	+it_roman_rep_1,30,4,Europe,1000BCE-0,-509,-264
	315	+it_roman_rep_2,30,4,Europe,1000BCE-0,-264,-133
	316	+it_roman_rep_3,30,4,Europe,1000BCE-0,-133,-31
	317	+it_sicily_k_2,27,4,Europe,1001-1500,1194,1281
	318	+it_st_peter_rep_1,27,4,Europe,1-1000,752,904
	319	+it_st_peter_rep_2,31,4,Europe,1001-1500,904,1198
	320	+it_venetian_rep_3,22,3,Europe,1001-1500,1204,1563
	321	+it_venetian_rep_4,23,3,Europe,1501-1800,1564,1797
	322	+it_western_roman_emp,30,3,Europe,1-1000,395,476
	323	+jp_ashikaga,26,4,East_Asia,1001-1500,1336,1467
	324	+jp_asuka,22,3,East_Asia,1-1000,538,710
	325	+jp_azuchi_momoyama,23,4,East_Asia,1501-1800,1568,1603
	326	+jp_heian,26,4,East_Asia,1-1000,794,1185
	327	+jp_jomon_1,20,1,East_Asia,pre-1000BCE,-13600,-9200
	328	+jp_jomon_2,20,1,East_Asia,pre-1000BCE,-9200,-5300
	329	+jp_jomon_3,20,3,East_Asia,pre-1000BCE,-5300,-3500
	330	+jp_jomon_4,21,2,East_Asia,pre-1000BCE,-3500,-2500
	331	+jp_jomon_5,19,3,East_Asia,pre-1000BCE,-2500,-1200
	332	+jp_jomon_6,19,3,East_Asia,1000BCE-0,-1200,-300
	333	+jp_kamakura,26,4,East_Asia,1001-1500,1185,1333
	334	+jp_kofun,15,3,East_Asia,1-1000,250,537
	335	+jp_nara,25,3,East_Asia,1-1000,710,794
	336	+jp_sengoku_jidai,16,4,East_Asia,1501-1800,1467,1568
	337	+jp_tokugawa_shogunate,29,4,East_Asia,1501-1800,1603,1868
	338	+jp_yayoi,24,3,East_Asia,1000BCE-0,-300,250
	339	+kg_kara_khanid_dyn,26,4,Central_Asia,1001-1500,950,1212
	340	+kg_western_turk_khaganate,17,3,Central_Asia,1-1000,582,630
	341	+kh_angkor_1,30,3,Southeast_Asia_Oceania,1-1000,802,1100
	342	+kh_angkor_2,30,4,Southeast_Asia_Oceania,1001-1500,1100,1220
	343	+kh_angkor_3,29,4,Southeast_Asia_Oceania,1001-1500,1220,1432
	344	+kh_cambodia_ba,0,2,Southeast_Asia_Oceania,1000BCE-0,-1200,-501
	345	+kh_cambodia_ia,0,2,Southeast_Asia_Oceania,1000BCE-0,-500,224
	346	+kh_chenla,21,3,Southeast_Asia_Oceania,1-1000,550,825
	347	+kh_funan_1,23,3,Southeast_Asia_Oceania,1-1000,225,540
	348	+kh_funan_2,23,4,Southeast_Asia_Oceania,1-1000,540,640
	349	+kh_khmer_k,28,3,Southeast_Asia_Oceania,1501-1800,1432,1594
	350	+kz_andronovo,12,3,Central_Asia,pre-1000BCE,-1800,-1200
	351	+kz_chionite,1,0,Central_Asia,1-1000,300,388
	352	+kz_oirat,7,1,Central_Asia,1001-1500,1368,1630
	353	+kz_yueban,2,1,Central_Asia,1-1000,350,450
	354	+lb_phoenician_emp,18,4,Southwest_Asia,1000BCE-0,-1200,-332
	355	+ma_almoravid_dyn,21,3,North_Africa,1001-1500,1035,1150
	356	+ma_idrisid_dyn,14,2,North_Africa,1-1000,789,917
	357	+ma_mauretania,14,3,North_Africa,1000BCE-0,-125,44
	358	+ma_saadi_sultanate,16,4,North_Africa,1501-1800,1554,1659
	359	+ma_wattasid_dyn,13,3,North_Africa,1501-1800,1465,1554
	360	+ml_bamana_k,10,4,Sub_Saharan_Africa,1501-1800,1712,1861
	361	+ml_jenne_jeno_1,9,4,Sub_Saharan_Africa,1000BCE-0,-250,49
	362	+ml_jenne_jeno_2,10,4,Sub_Saharan_Africa,1-1000,50,399
	363	+ml_jenne_jeno_3,16,4,Sub_Saharan_Africa,1-1000,400,899
	364	+ml_jenne_jeno_4,14,3,Sub_Saharan_Africa,1001-1500,900,1300
	365	+ml_mali_emp,22,4,Sub_Saharan_Africa,1001-1500,1230,1410
	366	+ml_segou_k,15,3,Sub_Saharan_Africa,1501-1800,1650,1712
	367	+ml_songhai_1,27,2,North_Africa,1001-1500,1376,1493
	368	+ml_songhai_2,24,4,Sub_Saharan_Africa,1501-1800,1493,1591
	369	+mm_bagan,16,3,Southeast_Asia_Oceania,1001-1500,1044,1287
	370	+mn_hunnu_early,23,3,Central_Asia,1000BCE-0,-1400,-300
	371	+mn_hunnu_emp,20,4,Central_Asia,1000BCE-0,-209,-60
	372	+mn_hunnu_late,17,4,Central_Asia,1-1000,-60,100
	373	+mn_khitan_1,22,3,Central_Asia,1001-1500,907,1125
	374	+mn_mongol_early,27,4,Central_Asia,1001-1500,1000,1206
	375	+mn_mongol_emp,23,2,Central_Asia,1001-1500,1206,1270
	376	+mn_mongol_late,25,3,Central_Asia,1501-1800,1368,1690
	377	+mn_rouran_khaganate,24,3,Central_Asia,1-1000,300,555
	378	+mn_shiwei,11,0,Central_Asia,1-1000,600,1000
	379	+mn_turk_khaganate_1,15,3,Central_Asia,1-1000,583,630
	380	+mn_turk_khaganate_2,17,3,Central_Asia,1-1000,682,744
	381	+mn_uygur_khaganate,14,3,Central_Asia,1-1000,745,840
	382	+mn_xianbei,24,2,Central_Asia,1-1000,100,250
	383	+mn_zungharian_emp,24,3,Central_Asia,1501-1800,1670,1757
	384	+mo_alawi_dyn_1,13,0,North_Africa,1501-1800,1631,1727
	385	+mr_wagadu_1,8,1,Sub_Saharan_Africa,1-1000,250,700
	386	+mr_wagadu_2,25,4,Sub_Saharan_Africa,1-1000,700,1077
	387	+mr_wagadu_3,26,4,Sub_Saharan_Africa,1001-1500,1078,1203
	388	+mw_early_maravi,8,1,Sub_Saharan_Africa,1001-1500,1400,1499
	389	+mw_maravi_emp,9,1,Sub_Saharan_Africa,1501-1800,1622,1870
	390	+mw_northern_maravi_k,9,1,Sub_Saharan_Africa,1501-1800,1500,1621
	391	+mw_pre_maravi,6,0,Sub_Saharan_Africa,1001-1500,1151,1399
	392	+mx_aztec_emp,17,4,North_America,1001-1500,1427,1526
	393	+mx_basin_of_mexico_1,12,1,North_America,pre-1000BCE,-6000,-2001
	394	+mx_basin_of_mexico_10,12,3,North_America,1001-1500,1200,1426
	395	+mx_basin_of_mexico_2,12,2,North_America,pre-1000BCE,-2000,-1201
	396	+mx_basin_of_mexico_3,24,2,North_America,pre-1000BCE,-1200,-801
	397	+mx_basin_of_mexico_4,23,2,North_America,1000BCE-0,-800,-401
	398	+mx_basin_of_mexico_5,19,2,North_America,1000BCE-0,-400,-101
	399	+mx_basin_of_mexico_6,19,2,North_America,1000BCE-0,-100,99
	400	+mx_basin_of_mexico_7,0,2,North_America,1-1000,100,649
	401	+mx_basin_of_mexico_8,15,2,North_America,1-1000,650,899
	402	+mx_mexico_1,23,4,North_America,post-1800,1810,1920
	403	+mx_monte_alban_1_early,27,4,North_America,1000BCE-0,-500,-300
	404	+mx_monte_alban_1_late,24,4,North_America,1000BCE-0,-300,-100
	405	+mx_monte_alban_2,24,4,North_America,1-1000,-100,200
	406	+mx_monte_alban_3_a,25,4,North_America,1-1000,200,500
	407	+mx_monte_alban_3_b_4,24,3,North_America,1-1000,500,900
	408	+mx_monte_alban_5,22,3,North_America,1001-1500,900,1520
	409	+mx_monte_alban_5_a,23,3,North_America,1-1000,900,1099
	410	+mx_monte_alban_5_b,23,3,North_America,1001-1500,1101,1520
	411	+mx_rosario,28,4,North_America,1000BCE-0,-700,-500
	412	+mx_san_jose,29,2,North_America,1000BCE-0,-1150,-700
	413	+mx_tierras_largas,27,2,North_America,pre-1000BCE,-1400,-1150
	414	+mx_toltec,13,4,North_America,1001-1500,900,1199
	415	+my_malacca_sultanate,14,4,Southeast_Asia_Oceania,1001-1500,1396,1511
	416	+my_malacca_sultanate_22222,19,1,Southeast_Asia_Oceania,1001-1500,1270,1415
	417	+ni_allada_k,13,4,Sub_Saharan_Africa,1001-1500,1100,1724
	418	+ni_aro,8,1,Sub_Saharan_Africa,1501-1800,1690,1902
	419	+ni_benin_emp,20,3,Sub_Saharan_Africa,1501-1800,1140,1897
	420	+ni_bornu_emp,16,2,Sub_Saharan_Africa,1501-1800,1380,1893
	421	+ni_dahomey_k,20,4,Sub_Saharan_Africa,post-1800,1715,1894
	422	+ni_formative_yoruba,9,1,Sub_Saharan_Africa,1-1000,650,1049
	423	+ni_hausa_k,15,2,Sub_Saharan_Africa,1001-1500,900,1808
	424	+ni_igala_k,9,2,Sub_Saharan_Africa,1501-1800,1600,1900
	425	+ni_igodomingodo,1,1,Sub_Saharan_Africa,1001-1500,900,1450
	426	+ni_kwararafa,0,1,Sub_Saharan_Africa,1001-1500,596,1820
	427	+ni_nok_1,16,3,Sub_Saharan_Africa,pre-1000BCE,-1500,-901
	428	+ni_nok_2,16,3,Sub_Saharan_Africa,1000BCE-0,-900,0
	429	+ni_nri_k,12,1,Sub_Saharan_Africa,1001-1500,1043,1911
	430	+ni_oyo_emp_1,7,0,Sub_Saharan_Africa,1001-1500,1300,1535
	431	+ni_oyo_emp_2,12,3,Sub_Saharan_Africa,1501-1800,1601,1835
	432	+ni_proto_yoruba,7,1,Sub_Saharan_Africa,1-1000,301,649
	433	+ni_proto_yoruboid,7,1,Sub_Saharan_Africa,1000BCE-0,-300,300
	434	+ni_sokoto_cal,15,4,Sub_Saharan_Africa,post-1800,1804,1904
	435	+ni_whydah_k,14,2,Sub_Saharan_Africa,1501-1800,1671,1727
	436	+ni_wukari_fed,3,2,Sub_Saharan_Africa,post-1800,1820,1899
	437	+ni_yoruba_classic,15,1,Sub_Saharan_Africa,1001-1500,1000,1400
	438	+nl_dutch_emp_1,21,2,Europe,1501-1800,1648,1795
	439	+no_norway_k_2,29,4,Europe,1001-1500,1262,1396
	440	+om_busaidi_imamate_1,12,0,Southwest_Asia,post-1800,1749,1895
	441	+pa_cocle_1,13,0,North_America,1-1000,200,700
	442	+pa_cocle_2,13,1,North_America,1-1000,700,1000
	443	+pa_cocle_3,15,2,North_America,1001-1500,1000,1515
	444	+pa_la_mula_sarigua,10,1,North_America,1000BCE-0,-1300,200
	445	+pa_monagrillo,8,1,North_America,pre-1000BCE,-3000,-1300
	446	+pe_cuzco_1,11,1,South_America,1000BCE-0,-500,200
	447	+pe_cuzco_2,15,2,South_America,1-1000,200,499
	448	+pe_cuzco_3,11,2,South_America,1-1000,500,649
	449	+pe_cuzco_5,16,2,South_America,1001-1500,1000,1250
	450	+pe_cuzco_6,17,2,South_America,1001-1500,1250,1400
	451	+pe_inca_emp,31,4,South_America,1001-1500,1375,1532
	452	+pe_wari_emp,19,4,South_America,1-1000,650,999
	453	+pg_orokaiva_colonial,30,4,Southeast_Asia_Oceania,post-1800,1884,1942
	454	+pg_orokaiva_pre_colonial,30,2,Southeast_Asia_Oceania,post-1800,1734,1883
	455	+pk_indo_greek_k,16,3,South_Asia,1000BCE-0,-180,-10
	456	+pk_kachi_ca,24,2,South_Asia,pre-1000BCE,-4000,-3200
	457	+pk_kachi_enl,24,2,South_Asia,pre-1000BCE,-7500,-5500
	458	+pk_kachi_lnl,24,2,South_Asia,pre-1000BCE,-5500,-4000
	459	+pk_kachi_post_urban,28,2,South_Asia,pre-1000BCE,-1800,-1300
	460	+pk_kachi_pre_urban,24,2,South_Asia,pre-1000BCE,-3200,-2500
	461	+pk_kachi_proto_historic,18,2,South_Asia,1000BCE-0,-1300,-500
	462	+pk_kachi_urban_1,17,3,South_Asia,pre-1000BCE,-2500,-2100
	463	+pk_kachi_urban_2,17,3,South_Asia,pre-1000BCE,-2100,-1800
	464	+pk_samma_dyn,15,3,South_Asia,1001-1500,1335,1521
	465	+pk_sind_abbasid_fatimid,18,3,South_Asia,1001-1500,854,1193
	466	+pt_portuguese_emp_1,17,1,Europe,1501-1800,1495,1579
	467	+pt_portuguese_emp_2,25,3,Europe,1501-1800,1640,1806
	468	+ru_crimean_khanate,15,1,Europe,1501-1800,1440,1783
	469	+ru_golden_horde,28,4,Europe,1001-1500,1240,1440
	470	+ru_kazan_khanate,12,2,Europe,1001-1500,1438,1552
	471	+ru_moskva_rurik_dyn,23,4,Europe,1501-1800,1480,1613
	472	+ru_novgorod_land,14,3,Europe,1001-1500,880,1240
	473	+ru_romanov_dyn_1,22,3,Europe,1501-1800,1614,1775
	474	+ru_romanov_dyn_2,24,4,Europe,post-1800,1776,1917
	475	+ru_sakha_early,30,2,Central_Asia,1501-1800,1400,1632
	476	+ru_sakha_late,30,1,Central_Asia,1501-1800,1632,1900
	477	+ru_soviet_union,22,3,Europe,post-1800,1918,1991
	478	+rw_bugesera_k,18,1,Sub_Saharan_Africa,1501-1800,1700,1799
	479	+rw_gisaka_k,18,1,Sub_Saharan_Africa,1501-1800,1700,1867
	480	+rw_mubari_k,18,1,Sub_Saharan_Africa,1501-1800,1700,1896
	481	+rw_ndorwa_k,18,1,Sub_Saharan_Africa,1501-1800,1700,1800
	482	+sd_makuria_k_1,13,3,North_Africa,1-1000,568,618
	483	+sd_makuria_k_2,16,3,North_Africa,1-1000,619,849
	484	+sd_makuria_k_3,16,3,North_Africa,1-1000,850,1099
	485	+se_baol_k,6,1,Sub_Saharan_Africa,1501-1800,1550,1890
	486	+se_cayor_k,7,1,Sub_Saharan_Africa,1501-1800,1549,1864
	487	+se_futa_toro_imamate,10,0,Sub_Saharan_Africa,post-1800,1776,1860
	488	+se_great_fulo_emp,7,0,Sub_Saharan_Africa,1501-1800,1490,1776
	489	+se_jolof_emp,5,0,Sub_Saharan_Africa,1001-1500,1360,1549
	490	+se_jolof_k,8,1,Sub_Saharan_Africa,1501-1800,1549,1865
	491	+se_saloum_k,6,1,Sub_Saharan_Africa,1501-1800,1490,1863
	492	+se_sine_k,7,1,Sub_Saharan_Africa,1501-1800,1350,1887
	493	+se_waalo_k,7,1,Sub_Saharan_Africa,1501-1800,1287,1855
	494	+si_early_modern_interior,19,3,Sub_Saharan_Africa,1501-1800,1650,1896
	495	+si_freetown_1,6,3,Sub_Saharan_Africa,1501-1800,1787,1808
	496	+si_mane_emp,9,0,Sub_Saharan_Africa,1501-1800,1550,1650
	497	+si_pre_sape,13,0,Sub_Saharan_Africa,1-1000,600,1400
	498	+si_sape,13,0,Sub_Saharan_Africa,1001-1500,1400,1550
	499	+sl_anuradhapura_1,5,1,South_Asia,1000BCE-0,-300,70
	500	+sl_anuradhapura_2,3,1,South_Asia,1-1000,70,428
	501	+sl_anuradhapura_3,9,1,South_Asia,1-1000,428,614
	502	+sl_anuradhapura_4,8,1,South_Asia,1-1000,614,1017
	503	+sl_dambadeniya,4,0,South_Asia,1001-1500,1232,1293
	504	+sl_jaffa_k,5,0,South_Asia,1001-1500,1310,1591
	505	+sl_polonnaruva,7,1,South_Asia,1001-1500,1070,1255
	506	+so_adal_sultanate,12,0,Sub_Saharan_Africa,1001-1500,1375,1543
	507	+so_ajuran_sultanate,10,0,Sub_Saharan_Africa,1001-1500,1250,1700
	508	+so_geledi_sultanate,10,2,Sub_Saharan_Africa,post-1800,1750,1911
	509	+so_habr_yunis,10,1,Sub_Saharan_Africa,1501-1800,1300,1886
	510	+so_ifat_sultanate,12,1,Sub_Saharan_Africa,1001-1500,1280,1375
	511	+so_isaaq_sultanate,8,0,Sub_Saharan_Africa,1501-1800,1300,1886
	512	+so_majeerteen_sultanate,8,0,Sub_Saharan_Africa,post-1800,1750,1926
	513	+so_tunni_sultanate,9,0,Sub_Saharan_Africa,1-1000,800,1200
	514	+sy_umayyad_cal,29,4,Southwest_Asia,1-1000,661,750
	515	+th_ayutthaya,26,2,Southeast_Asia_Oceania,1501-1800,1593,1767
	516	+th_rattanakosin,27,4,Southeast_Asia_Oceania,post-1800,1782,1873
	517	+th_sukhotai,14,1,Southeast_Asia_Oceania,1001-1500,1238,1419
	518	+tj_sarasm,11,2,Central_Asia,pre-1000BCE,-3500,-2000
	519	+tm_khwarezmid_emp,20,3,Central_Asia,1001-1500,1157,1231
	520	+tn_fatimid_cal,30,4,North_Africa,1001-1500,909,1171
	521	+tn_zirid_dyn,15,3,North_Africa,1001-1500,973,1148
	522	+tr_byzantine_emp_1,31,4,Southwest_Asia,1-1000,632,866
	523	+tr_byzantine_emp_2,30,4,Southwest_Asia,1-1000,867,1072
	524	+tr_byzantine_emp_3,31,4,Southwest_Asia,1001-1500,1073,1204
	525	+tr_cappadocia_2,20,3,Southwest_Asia,1000BCE-0,-330,16
	526	+tr_central_anatolia_mba,13,4,Southwest_Asia,pre-1000BCE,-2000,-1700
	527	+tr_east_roman_emp,30,4,Southwest_Asia,1-1000,395,631
	528	+tr_hatti_new_k,29,4,Southwest_Asia,pre-1000BCE,-1400,-1180
	529	+tr_hatti_old_k,25,4,Southwest_Asia,pre-1000BCE,-1650,-1500
	530	+tr_konya_eba,16,1,Southwest_Asia,pre-1000BCE,-3000,-2000
	531	+tr_konya_eca,28,3,Southwest_Asia,pre-1000BCE,-6000,-5500
	532	+tr_konya_enl,19,1,Southwest_Asia,pre-1000BCE,-9600,-7000
	533	+tr_konya_lba,23,4,Southwest_Asia,pre-1000BCE,-1500,-1400
	534	+tr_konya_lca,20,3,Southwest_Asia,pre-1000BCE,-5500,-3000
	535	+tr_konya_lnl,22,3,Southwest_Asia,pre-1000BCE,-6600,-6000
	536	+tr_konya_mnl,20,1,Southwest_Asia,pre-1000BCE,-7000,-6600
	537	+tr_lydia_k,17,3,Southwest_Asia,1000BCE-0,-670,-546
	538	+tr_lysimachus_k,21,3,Southwest_Asia,1000BCE-0,-323,-281
	539	+tr_neo_hittite_k,7,4,Southwest_Asia,pre-1000BCE,-1180,-900
	540	+tr_ottoman_emirate,27,4,Southwest_Asia,1001-1500,1299,1402
	541	+tr_ottoman_emp_1,28,4,Southwest_Asia,1001-1500,1402,1517
	542	+tr_ottoman_emp_2,29,4,Southwest_Asia,1501-1800,1517,1683
	543	+tr_ottoman_emp_3,28,4,Southwest_Asia,1501-1800,1683,1839
	544	+tr_ottoman_emp_4,30,4,Southwest_Asia,post-1800,1839,1922
	545	+tr_ottoman_emp_4_copy,28,1,Southwest_Asia,post-1800,1873,1920
	546	+tr_pergamon_k,5,3,Southwest_Asia,1000BCE-0,-282,-133
	547	+tr_phrygian_k,12,4,Southwest_Asia,1000BCE-0,-900,-695
	548	+tr_roman_dominate,30,4,Southwest_Asia,1-1000,285,394
	549	+tr_rum_sultanate,28,4,Southwest_Asia,1001-1500,1077,1307
	550	+tr_tabal_k,9,2,Southwest_Asia,1000BCE-0,-900,-730
	551	+tz_buhayo_k,18,1,Sub_Saharan_Africa,1501-1800,1700,1890
	552	+tz_early_tana_1,3,1,Sub_Saharan_Africa,1-1000,500,749
	553	+tz_early_tana_2,12,0,Sub_Saharan_Africa,1-1000,750,999
	554	+tz_east_africa_ia_1,5,1,Sub_Saharan_Africa,1-1000,200,499
	555	+tz_east_africa_ia_2,6,0,Sub_Saharan_Africa,1-1000,800,1150
	556	+tz_karagwe_k,18,0,Sub_Saharan_Africa,1501-1800,1500,1916
	557	+tz_milansi_k,12,1,Sub_Saharan_Africa,1501-1800,1600,1890
	558	+tz_tana,13,1,Sub_Saharan_Africa,1001-1500,1000,1498
	559	+ua_kievan_rus,19,1,Europe,1001-1500,880,1242
	560	+ua_russian_principate,8,0,Europe,1001-1500,1133,1240
	561	+ua_skythian_k_3,14,3,Europe,1000BCE-0,-429,-225
	562	+ug_buganda_k_1,14,0,Sub_Saharan_Africa,1501-1800,1408,1716
	563	+ug_buganda_k_2,22,2,Sub_Saharan_Africa,post-1800,1717,1894
	564	+ug_bunyoro_k_1,14,1,Sub_Saharan_Africa,1501-1800,1450,1699
	565	+ug_bunyoro_k_2,19,3,Sub_Saharan_Africa,1501-1800,1700,1894
	566	+ug_early_nyoro,14,1,Sub_Saharan_Africa,1001-1500,900,1449
	567	+ug_nkore_k_1,12,1,Sub_Saharan_Africa,1501-1800,1450,1749
	568	+ug_nkore_k_2,20,1,Sub_Saharan_Africa,post-1800,1750,1901
	569	+ug_toro_k,18,1,Sub_Saharan_Africa,post-1800,1830,1896
	570	+us_antebellum,27,4,North_America,post-1800,1776,1865
	571	+us_cahokia_1,22,4,North_America,1001-1500,1050,1199
	572	+us_cahokia_2,22,4,North_America,1001-1500,1200,1275
	573	+us_cahokia_3,21,4,North_America,1001-1500,1275,1400
	574	+us_chaco_bonito_3,12,4,North_America,1001-1500,1101,1140
	575	+us_early_illinois_confederation,12,4,North_America,1501-1800,1640,1717
	576	+us_emergent_mississippian_1,26,4,North_America,1-1000,750,900
	577	+us_emergent_mississippian_2,24,4,North_America,1-1000,900,1049
	578	+us_haudenosaunee_1,30,4,North_America,1501-1800,1566,1713
	579	+us_haudenosaunee_2,30,4,North_America,1501-1800,1714,1848
	580	+us_hawaii_1,18,2,Southeast_Asia_Oceania,1001-1500,1000,1200
	581	+us_hawaii_2,27,3,Southeast_Asia_Oceania,1001-1500,1200,1580
	582	+us_hawaii_3,26,3,Southeast_Asia_Oceania,1501-1800,1580,1778
	583	+us_hawaii_k,29,4,Southeast_Asia_Oceania,post-1800,1820,1898
	584	+us_hohokam_culture,15,3,North_America,1-1000,300,1500
	585	+us_kamehameha_k,23,3,Southeast_Asia_Oceania,1501-1800,1778,1819
	586	+us_oneota,19,3,North_America,1501-1800,1400,1650
	587	+us_proto_haudenosaunee,1,4,North_America,1001-1500,1300,1565
	588	+us_united_states_of_america_reconstruction,29,3,North_America,post-1800,1866,1933
	589	+us_woodland_1,26,2,North_America,1000BCE-0,-600,-150
	590	+us_woodland_2,26,2,North_America,1-1000,-150,300
	591	+us_woodland_3,26,2,North_America,1-1000,300,450
	592	+us_woodland_4,26,2,North_America,1-1000,450,600
	593	+us_woodland_5,26,2,North_America,1-1000,600,750
	594	+uz_chagatai_khanate,20,3,Central_Asia,1001-1500,1227,1402
	595	+uz_janid_dyn,20,3,Central_Asia,1501-1800,1599,1747
	596	+uz_kangju,16,3,Central_Asia,1-1000,-150,350
	597	+uz_khwarasm_1,11,3,Central_Asia,1000BCE-0,-1000,-521
	598	+uz_koktepe_1,8,1,Central_Asia,pre-1000BCE,-1400,-1000
	599	+uz_koktepe_2,18,2,Central_Asia,1000BCE-0,-750,-550
	600	+uz_samanid_emp,28,4,Central_Asia,1-1000,819,999
	601	+uz_shaybanid_k,2,0,Central_Asia,1501-1800,1500,1598
	602	+uz_sogdiana_city_states,26,4,Central_Asia,1-1000,604,711
	603	+uz_timurid_emp,21,4,Central_Asia,1001-1500,1370,1526
	604	+ye_himyar_1,20,4,Southwest_Asia,1-1000,270,340
	605	+ye_himyar_2,22,4,Southwest_Asia,1-1000,378,525
	606	+ye_qasimid_dyn,16,1,Southwest_Asia,1501-1800,1637,1805
	607	+ye_qasimid_dyn_222222,2,2,Southwest_Asia,1501-1800,1637,1805
	608	+ye_qatabanian_commonwealth,20,2,Southwest_Asia,1000BCE-0,-450,-111
	609	+ye_rasulid_dyn,21,2,Southwest_Asia,1001-1500,1229,1453
	610	+ye_saba_k,7,1,Southwest_Asia,1-1000,-110,149
	611	+ye_sabaean_commonwealth,6,2,Southwest_Asia,1000BCE-0,-800,-451
	612	+ye_tahirid_dyn,22,1,Southwest_Asia,1001-1500,1454,1517
	613	+ye_warlords,16,2,Southwest_Asia,1001-1500,1038,1174
	614	+ye_yemen_lba,0,1,Southwest_Asia,pre-1000BCE,-1200,-801
	615	+ye_yemen_lnl,0,1,Southwest_Asia,pre-1000BCE,-3500,-1201
	616	+ye_ziyad_dyn,18,2,Southwest_Asia,1-1000,822,1037
	617	+zi_great_zimbabwe,7,3,Sub_Saharan_Africa,1001-1500,1270,1550
	618	+zi_mutapa,3,2,Sub_Saharan_Africa,1501-1800,1450,1880
	619	+zi_torwa_rozvi,3,0,Sub_Saharan_Africa,1501-1800,1494,1850
	620	+zi_toutswe,2,4,Sub_Saharan_Africa,1-1000,700,1250

added results/diagnostics/seshat_unparsed_values.csv +1 −0

		`@@ -0,0 +1 @@`
	1	+sheet,polity_id,variable,value_from

added results/diagnostics/seshat_zero_scale_values.csv +1 −0

		`@@ -0,0 +1 @@`
	1	+polity_id,variable,value_from,value_to,year_from

modified results/logs/execution_log.md +3 −0

		`@@ -14,3 +14,6 @@`
14	14	\| 2026-11-01 03:10 \| 1 \| Subagent verification pass on the 18 closest works (abstracts/results read from primary sources) \| `docs/novelty_search_report.md`; "Social Compression Hypothesis" has no prior academic use; closest precursor Shin et al. 2020 \|
15	15	\| 2026-11-01 03:20 \| 1 \| Structure-only inspection of CrisisDB codes, Seshat date coverage, V-Dem variables, EA variables (no predictor–outcome association) \| 24,401/26,033 SC facts undated; CrisisDB 166 rows/150 polities \|
16	16	\| 2026-11-01 03:40 \| 1 \| Pre-registration written and frozen (D-003, SHA-256 ecda8804e17f931789be8e274ba1ff989000de356700a85b638b043928182e3a) ; literature review + novelty matrix \| `research/preregistration.md`, `research/literature_review.md`, `research/decisions.md`, `configs/regions.yaml` \|
	17	+\| 2026-11-01 03:55 \| 2 \| Glottolog 5.3 languages.csv registered + fetched (language families for Galton controls) \| registry 9 sources \|
	18	+\| 2026-11-01 04:10 \| 2 \| `python -m wp35.harmonization.build_panels` (3 runs: EA033 code parsing fixed via `ord`; EA202 continuous; Y6 redefined as last regime year with irregular end type after inspecting France 1789–1960) \| `data/processed/.parquet`, `results/diagnostics/`, `paper/figures/fig_seshat_temporal_coverage.pdf`, `fig_seshat_missingness.pdf`, `fig_dplace_focal_years.pdf`, `fig_modern_coverage.pdf`, `coverage_summary.json` \|
	19	+\| 2026-11-01 04:15 \| 2 \| Data dictionary + parser tests (`pytest`: 6 passed) \| `data/metadata/data_dictionary.md`, `tests/test_harmonization.py` \|

added src/wp35/harmonization/build_panels.py +151 −0

		`@@ -0,0 +1,151 @@`
	1	+"""Phase 2 orchestrator: build the three panels, write processed files, coverage tables, missingness diagnostics,
	2	+temporal-distribution figures and the data dictionary. Usage: python -m wp35.harmonization.build_panels
	3	+No predictor-outcome association is computed here.
	4	+"""
	5	+from __future__ import annotations
	6	+import json
	7	+from datetime import datetime, timezone
	8	+from pathlib import Path
	9	+import matplotlib
	10	+matplotlib.use("Agg")
	11	+import matplotlib.pyplot as plt
	12	+import numpy as np
	13	+import pandas as pd
	14	+from wp35.harmonization import seshat as S, dplace as D, modern as M
	15	+
	16	+ROOT = Path(__file__).resolve().parents[3]
	17	+PROC = ROOT / "data" / "processed"; META = ROOT / "data" / "metadata"; DIAG = ROOT / "results" / "diagnostics"
	18	+FIG = ROOT / "paper" / "figures"; TAB = ROOT / "paper" / "tables"
	19	+for p in (PROC, META, DIAG, FIG, TAB):
	20	+ p.mkdir(parents=True, exist_ok=True)
	21	+
	22	+K_BLOCKS = {
	23	+ "admin": ["full_time_bureaucrat", "specialized_government_building", "merit_promotion", "examination_system", "administrative_level"],
	24	+ "rules_money": ["formal_legal_code", "judge", "court", "professional_lawyer", "indigenous_coin", "foreign_coin",
	25	+ "paper_currency", "token", "precious_metal", "article"],
	26	+ "comm_infra": ["courier", "postal_station", "general_postal_service", "road", "bridge", "canal", "port"],
	27	+ "records": ["written_record", "script", "phonetic_alphabetic_writing", "non_phonetic_writing",
	28	+ "lists_tables_and_classification", "calendar", "practical_literature", "nonwritten_record", "mnemonic_device"],
	29	+}
	30	+STANDARDS = ["length_measurement_system", "area_measurement_system", "volume_measurement_system", "weight_measurement_system", "time_measurement_system"]
	31	+D_VARS = ["polity_population", "polity_territory", "population_of_the_largest_settlement", "settlement_hierarchy"]
	32	+PROD_INFRA = ["irrigation_system", "drinking_water_supply_system", "market", "food_storage_site", "mines_or_quarry"]
	33	+MILITARY = ["professional_soldier", "professional_military_officer", "military_level"]
	34	+
	35	+
	36	+def seshat_panels():
	37	+ out = S.build_all()
	38	+ facts, agg, meta, crisis, pre = out["facts"], out["polity_agg"], out["polity_meta"], out["crisis"], out["precrisis_agg"]
	39	+ facts.to_parquet(PROC / "seshat_facts.parquet", index=False)
	40	+ agg.to_parquet(PROC / "seshat_polity_agg.parquet", index=False)
	41	+ meta.to_parquet(PROC / "seshat_polity_meta.parquet", index=False)
	42	+ crisis.to_parquet(PROC / "seshat_crisis.parquet", index=False)
	43	+ pre.to_parquet(PROC / "seshat_precrisis_agg.parquet", index=False)
	44	+ wide = S.wide(agg, "value_mean"); wide_log = S.wide(agg, "logv_mean")
	45	+ wide.to_parquet(PROC / "seshat_polity_wide.parquet")
	46	+ # coverage table: per variable, polities coded, facts, dated facts, inferred share
	47	+ cov = agg.groupby("variable").agg(polities_coded=("polity_id", "nunique"), facts=("n_facts", "sum"),
	48	+ dated_facts=("n_dated", "sum"), inferred_facts=("n_inferred", "sum"),
	49	+ disputed_facts=("n_disputed", "sum")).reset_index()
	50	+ sect = facts.drop_duplicates("variable").set_index("variable").section
	51	+ cov["section"] = cov.variable.map(sect)
	52	+ cov["block"] = cov.variable.map({v: b for b, vs in K_BLOCKS.items() for v in vs}).fillna(
	53	+ cov.variable.map({v: "D_scale" for v in D_VARS}).fillna(cov.variable.map({v: "standards" for v in STANDARDS})))
	54	+ cov = cov.sort_values(["block", "polities_coded"], ascending=[True, False])
	55	+ cov.to_csv(DIAG / "seshat_coverage_by_variable.csv", index=False)
	56	+ # Warfare and other sheets: coverage
	57	+ warfare_vars = facts[facts.sheet == "Warfare"].variable.unique().tolist()
	58	+ # missingness among the pre-registered indicator set, per polity
	59	+ ind = sum(K_BLOCKS.values(), []) + D_VARS + STANDARDS + PROD_INFRA + MILITARY
	60	+ pres = wide.reindex(columns=ind)
	61	+ miss = pd.DataFrame({"indicator": ind, "share_polities_missing": pres.isna().mean().values})
	62	+ miss.to_csv(DIAG / "seshat_missingness_by_indicator.csv", index=False)
	63	+ per_pol = pd.DataFrame({"polity_id": pres.index, "n_K_indicators_coded": pres[sum(K_BLOCKS.values(), [])].notna().sum(axis=1).values,
	64	+ "n_D_coded": pres[D_VARS].notna().sum(axis=1).values})
	65	+ per_pol = per_pol.merge(meta[["polity_id", "macro_region", "era", "start_year", "end_year"]], on="polity_id")
	66	+ per_pol.to_csv(DIAG / "seshat_missingness_by_polity.csv", index=False)
	67	+ # temporal distribution figure (polity time spans by macro-region) and crisis start years
	68	+ fig, axes = plt.subplots(1, 2, figsize=(11, 4.2), gridspec_kw={"width_ratios": [2, 1]})
	69	+ regs = sorted(meta.macro_region.unique())
	70	+ for i, r in enumerate(regs):
	71	+ sub = meta[meta.macro_region == r]
	72	+ axes[0].hlines([i + np.random.RandomState(1).uniform(-0.35, 0.35) for _ in range(len(sub))], sub.start_year, sub.end_year, lw=1.2, alpha=0.6)
	73	+ axes[0].set_yticks(range(len(regs))); axes[0].set_yticklabels([r.replace("_", " ") for r in regs], fontsize=8)
	74	+ axes[0].set_xlabel("Year (negative = BCE)"); axes[0].set_title(f"Polity time spans, Polaris-2026 (n = {len(meta)})", fontsize=10)
	75	+ axes[1].hist(crisis.crisis_start.dropna(), bins=30, color="#555"); axes[1].set_xlabel("Crisis start year"); axes[1].set_title(f"CrisisDB records (n = {len(crisis)})", fontsize=10)
	76	+ fig.tight_layout(); fig.savefig(FIG / "fig_seshat_temporal_coverage.pdf"); plt.close(fig)
	77	+ # missingness heatmap for pre-registered indicators
	78	+ fig, ax = plt.subplots(figsize=(10, 5))
	79	+ order = pres[sum(K_BLOCKS.values(), []) + D_VARS].notna()
	80	+ order = order.loc[order.sum(axis=1).sort_values(ascending=False).index]
	81	+ ax.imshow(order.values.T, aspect="auto", cmap="Greys", interpolation="nearest")
	82	+ ax.set_yticks(range(order.shape[1])); ax.set_yticklabels(order.columns, fontsize=6); ax.set_xlabel("Polities (sorted by number of coded indicators)")
	83	+ ax.set_title("Coded (dark) vs missing (white) indicators, Seshat Polaris-2026", fontsize=10)
	84	+ fig.tight_layout(); fig.savefig(FIG / "fig_seshat_missingness.pdf"); plt.close(fig)
	85	+ # coverage summary numbers
	86	+ summary = {
	87	+ "polities": int(len(meta)), "regions": int(meta.seshat_region.nunique()), "macro_regions": int(meta.macro_region.nunique()),
	88	+ "facts_total": int(len(facts)), "facts_dated": int(facts.dated.sum()), "facts_unparsed": int(facts.unparsed.sum()),
	89	+ "facts_inferred": int(facts.inferred.sum()), "facts_disputed_code": int(facts.disputed.sum()),
	90	+ "polities_with_population": int(wide.polity_population.notna().sum()), "polities_with_territory": int(wide.polity_territory.notna().sum()),
	91	+ "polities_with_any_D": int(pres[D_VARS].notna().any(axis=1).sum()),
	92	+ "polities_with_ge50pct_K": int((pres[sum(K_BLOCKS.values(), [])].notna().mean(axis=1) >= 0.5).sum()),
	93	+ "crisis_records": int(len(crisis)), "crisis_polities": int(crisis.polity_id.nunique()),
	94	+ "crisis_y1_coded": int(crisis.y1_severity.notna().sum()), "crisis_y1b_coded": int(crisis.y1b_severe.notna().sum()),
	95	+ "crisis_y1b_positive": int((crisis.y1b_severe == 1).sum()),
	96	+ "y3_coded": int(meta.y3_discontinuous.notna().sum()), "y3_positive": int((meta.y3_discontinuous == 1).sum()),
	97	+ "duration_censored_ge1900": int(meta.censored.sum()), "warfare_indicators": len(warfare_vars),
	98	+ "era_counts": meta.era.value_counts().to_dict(), "macro_region_counts": meta.macro_region.value_counts().to_dict(),
	99	+ }
	100	+ return summary, cov
	101	+
	102	+
	103	+def dplace_panel():
	104	+ out = D.build()
	105	+ soc = out["societies"]
	106	+ soc.to_parquet(PROC / "dplace_ea_societies.parquet", index=False)
	107	+ out["codes"].to_csv(META / "dplace_ea_codes_used.csv", index=False)
	108	+ cov = pd.DataFrame({"variable": [c for c in soc.columns if c.startswith("EA")], })
	109	+ cov["societies_coded"] = [int(soc[c].notna().sum()) for c in cov.variable]
	110	+ cov.to_csv(DIAG / "dplace_coverage_by_variable.csv", index=False)
	111	+ fig, ax = plt.subplots(figsize=(6, 3.5))
	112	+ ax.hist(soc.main_focal_year.dropna(), bins=40, color="#555"); ax.set_xlabel("Main focal year of the ethnographic description"); ax.set_title(f"D-PLACE EA v3.3 societies (n = {len(soc)})", fontsize=10)
	113	+ fig.tight_layout(); fig.savefig(FIG / "fig_dplace_focal_years.pdf"); plt.close(fig)
	114	+ return {"societies": int(len(soc)), "with_EA031": int(soc.ea031_code.notna().sum()), "with_EA202": int(soc.ea202_code.notna().sum()),
	115	+ "with_EA033": int(soc.EA033.notna().sum()), "with_glottolog_family": int(soc.Family_ID.notna().sum()),
	116	+ "families": int(soc.Family_ID.nunique()), "regions": int(soc.region.nunique()),
	117	+ "focal_year_range": [int(soc.main_focal_year.min()), int(soc.main_focal_year.max())]}
	118	+
	119	+
	120	+def modern_panel():
	121	+ out = M.build(); p = out["panel"]
	122	+ p.to_parquet(PROC / "modern_panel.parquet", index=False)
	123	+ cov = pd.DataFrame({"variable": [c for c in p.columns if c not in ("country_name", "country_text_id", "country_id", "year")]})
	124	+ cov["country_years_coded"] = [int(p[c].notna().sum()) for c in cov.variable]
	125	+ cov["first_year"] = [int(p.year[p[c].notna()].min()) if p[c].notna().any() else None for c in cov.variable]
	126	+ cov["last_year"] = [int(p.year[p[c].notna()].max()) if p[c].notna().any() else None for c in cov.variable]
	127	+ cov["countries"] = [int(p.country_text_id[p[c].notna()].nunique()) for c in cov.variable]
	128	+ cov.to_csv(DIAG / "modern_coverage_by_variable.csv", index=False)
	129	+ fig, ax = plt.subplots(figsize=(7, 3.5))
	130	+ for c, lab in [("maddison_pop", "Maddison population"), ("v2clrspct", "V-Dem K items"), ("e_civil_war", "civil war (COW)"), ("e_coups", "coups"), ("v2regendtype", "regime end type")]:
	131	+ ax.plot(p.groupby("year")[c].apply(lambda s: s.notna().sum()), label=lab, lw=1.2)
	132	+ ax.set_ylabel("countries coded"); ax.set_xlabel("year"); ax.legend(fontsize=7); ax.set_title("Modern panel coverage by year", fontsize=10)
	133	+ fig.tight_layout(); fig.savefig(FIG / "fig_modern_coverage.pdf"); plt.close(fig)
	134	+ cw = out["crosswalk"]
	135	+ return {"country_years": int(len(p)), "countries": int(p.country_text_id.nunique()), "years": [int(p.year.min()), int(p.year.max())],
	136	+ "with_maddison_pop": int(p.maddison_pop.notna().sum()), "countries_matched_maddison": int(cw[cw.in_vdem & cw.in_maddison].shape[0]),
	137	+ "vdem_countries_unmatched": int(cw[cw.in_vdem & ~cw.in_maddison].shape[0]),
	138	+ "y4_onsets": int((p.y4_civilwar_onset == 1).sum()), "y4_coded": int(p.y4_civilwar_onset.notna().sum()),
	139	+ "y5_coups": int((p.y5_coup == 1).sum()), "y5_coded": int(p.y5_coup.notna().sum()),
	140	+ "y6_irregular_ends": int((p.y6_irregular_regime_end == 1).sum()), "y6_coded": int(p.y6_irregular_regime_end.notna().sum())}
	141	+
	142	+
	143	+def main():
	144	+ s, cov = seshat_panels(); d = dplace_panel(); m = modern_panel()
	145	+ summary = {"built_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), "seshat": s, "dplace": d, "modern": m}
	146	+ (DIAG / "coverage_summary.json").write_text(json.dumps(summary, indent=2))
	147	+ print(json.dumps(summary, indent=2))
	148	+
	149	+
	150	+if __name__ == "__main__":
	151	+ main()

added src/wp35/harmonization/dplace.py +83 −0

		`@@ -0,0 +1,83 @@`
	1	+"""Parse the D-PLACE Ethnographic Atlas CLDF release (v3.3) into one row per society with the variables fixed in
	2	+research/preregistration.md §2.2, plus Glottolog language family for Galton controls.
	3	+
	4	+Ordinal categories of EA031 (mean size of local communities) and EA202 (population size) are converted to
	5	+log-midpoints of the category ranges printed in `codes.csv` (parsed from the code descriptions, not typed by hand);
	6	+the parsed table is written to results/diagnostics/dplace_code_midpoints.csv for inspection.
	7	+"""
	8	+from __future__ import annotations
	9	+import io, re, zipfile
	10	+from pathlib import Path
	11	+import numpy as np
	12	+import pandas as pd
	13	+
	14	+ROOT = Path(__file__).resolve().parents[3]
	15	+ZIP = ROOT / "data" / "raw" / "dplace_ea_v3_3" / "dplace-dataset-ea-v3.3.zip"
	16	+GLOTTO = ROOT / "data" / "raw" / "glottolog_cldf_languages" / "glottolog_v5.3_languages.csv"
	17	+DIAG = ROOT / "results" / "diagnostics"
	18	+
	19	+VARS = ["EA031", "EA202", "EA030", "EA033", "EA032", "EA072", "EA073", "EA090", "EA066", "EA067", "EA068",
	20	+ "EA070", "EA005", "EA028", "EA042", "EA034", "EA113"]
	21	+
	22	+
	23	+def _read(z: zipfile.ZipFile, name: str) -> pd.DataFrame:
	24	+ path = [n for n in z.namelist() if n.endswith("cldf/" + name)][0]
	25	+ return pd.read_csv(io.BytesIO(z.read(path)), low_memory=False)
	26	+
	27	+
	28	+def _range_midpoint(desc: str):
	29	+ """'1,000-5,000' -> geometric midpoint; '50,000 or more' -> 50,000 (lower bound, flagged); '<50' -> 25."""
	30	+ s = desc.replace(",", "")
	31	+ m = re.search(r"(\d+)\s[-–]\s(\d+)", s)
	32	+ if m:
	33	+ lo, hi = float(m.group(1)), float(m.group(2))
	34	+ return np.sqrt(lo * hi), lo, hi, "range"
	35	+ m = re.search(r"(\d+)\s*(or more\|\+\|and over\|or over)", s)
	36	+ if m:
	37	+ lo = float(m.group(1)); return lo, lo, np.nan, "open_upper"
	38	+ m = re.search(r"(fewer than\|less than\|under\|<)\s*(\d+)", s, flags=re.I)
	39	+ if m:
	40	+ hi = float(m.group(2)); return hi / 2, np.nan, hi, "open_lower"
	41	+ return np.nan, np.nan, np.nan, "none"
	42	+
	43	+
	44	+def build() -> dict[str, pd.DataFrame]:
	45	+ z = zipfile.ZipFile(ZIP)
	46	+ soc = _read(z, "societies.csv"); var = _read(z, "variables.csv"); codes = _read(z, "codes.csv"); data = _read(z, "data.csv")
	47	+ d = data[data.Var_ID.isin(VARS)].copy()
	48	+ # numeric code value (the ordinal code number) and its description
	49	+ # ordinal code number = `ord` column of codes.csv; the "Missing data" code (ord 99) is NaN, never a value
	50	+ codes["code_num"] = pd.to_numeric(codes["ord"], errors="coerce")
	51	+ codes.loc[codes.Name.eq("Missing data") \| codes.ID.str.endswith("-NA"), "code_num"] = np.nan
	52	+ cmap = codes.set_index("ID")[["code_num", "Description"]]
	53	+ d = d.join(cmap, on="Code_ID")
	54	+ wide = d.pivot_table(index="Soc_ID", columns="Var_ID", values="code_num", aggfunc="first")
	55	+ desc = d.pivot_table(index="Soc_ID", columns="Var_ID", values="Description", aggfunc="first")
	56	+ # midpoints for EA031 and EA202 from code descriptions
	57	+ mids = []
	58	+ # EA202 (population size) is a continuous variable in the CLDF release: numeric Value, no codes
	59	+ ea202 = d[d.Var_ID == "EA202"].set_index("Soc_ID").Value.map(lambda x: pd.to_numeric(x, errors="coerce"))
	60	+ wide["EA202_mid"] = ea202.reindex(wide.index) if len(ea202) else np.nan
	61	+ wide["EA202"] = wide["EA202_mid"]
	62	+ for v in ["EA031"]:
	63	+ cc = codes[codes.Var_ID == v].copy()
	64	+ parsed = [(_range_midpoint(n) if _range_midpoint(n)[3] != "none" else _range_midpoint(desc)) for n, desc in zip(cc.Name.astype(str), cc.Description.astype(str))]
	65	+ cc["mid"] = [p[0] for p in parsed]; cc["lo"] = [p[1] for p in parsed]; cc["hi"] = [p[2] for p in parsed]; cc["kind"] = [p[3] for p in parsed]
	66	+ mids.append(cc[["Var_ID", "ID", "Name", "Description", "mid", "lo", "hi", "kind"]])
	67	+ m = cc.set_index("ID")["mid"]
	68	+ wide[v + "_mid"] = d[d.Var_ID == v].set_index("Soc_ID").Code_ID.map(m).reindex(wide.index)
	69	+ DIAG.mkdir(parents=True, exist_ok=True)
	70	+ pd.concat(mids).to_csv(DIAG / "dplace_code_midpoints.csv", index=False)
	71	+ wide.columns = [str(c) for c in wide.columns]
	72	+ out = soc.set_index("ID")[["Name", "Latitude", "Longitude", "Glottocode", "region", "main_focal_year", "xd_id"]].join(wide, how="left")
	73	+ g = pd.read_csv(GLOTTO, low_memory=False)[["Glottocode", "Family_ID", "Macroarea"]].drop_duplicates("Glottocode")
	74	+ out = out.join(g.set_index("Glottocode"), on="Glottocode")
	75	+ out["Family_ID"] = out.Family_ID.fillna(out.Glottocode) # isolates: own family
	76	+ out.index.name = "soc_id"
	77	+ out = out.rename(columns={"EA031": "ea031_code", "EA202": "ea202_code"})
	78	+ out["log_community_size"] = np.log(out["EA031_mid"])
	79	+ out["log_population"] = np.log(out["EA202_mid"])
	80	+ out["jurisdictional_hierarchy_beyond"] = out["EA033"] - 1 # ord 1 = acephalous -> 0 levels beyond the community
	81	+ out["jurisdictional_hierarchy_local"] = out["EA032"]
	82	+ return {"societies": out.reset_index(), "variables": var[var.ID.isin(VARS)][["ID", "Name", "Description", "type"]],
	83	+ "codes": codes[codes.Var_ID.isin(VARS)][["Var_ID", "ID", "Name", "Description"]], "desc": desc.reset_index()}

added src/wp35/harmonization/modern.py +64 −0

		`@@ -0,0 +1,64 @@`
	1	+"""Modern country-year panel: V-Dem v16 (institutions, outcomes) + Maddison 2023 (population, GDP per capita).
	2	+
	3	+Variables fixed in research/preregistration.md §3 and the codebook readings recorded in research/measurement.md:
	4	+ K items: v2clrspct, v2stcritrecadm, v2cltrnslw, v2stfisccap, v2svstterr, v2elembcap
	5	+ D: Maddison `pop` (thousands) ; e_area (km2, 1886+)
	6	+ Outcomes (events in year t): Y4 civil war onset = e_civil_war 0->1 ; Y5 coup = e_coups >= 1 ;
	7	+ Y6 irregular regime end = last year of a regime (v2regidnr changes) whose v2regendtype is in {0,1,2,3,5,6,7,8}
	8	+ Covariates: Maddison gdppc, v2x_polyarchy, e_regionpol_6C
	9	+Crosswalk: ISO3 (`country_text_id` == `countrycode`); unmatched countries are listed in
	10	+data/metadata/crosswalk_vdem_maddison.csv and kept without Maddison values (no substitution from e_pop).
	11	+"""
	12	+from __future__ import annotations
	13	+from pathlib import Path
	14	+import numpy as np
	15	+import pandas as pd
	16	+import pyreadr
	17	+
	18	+ROOT = Path(__file__).resolve().parents[3]
	19	+VDEM = ROOT / "data" / "raw" / "vdem_v16_rdata" / "vdem_v16.RData"
	20	+MADD = ROOT / "data" / "raw" / "maddison_2023_xlsx" / "mpd2023_web.xlsx"
	21	+META = ROOT / "data" / "metadata"
	22	+
	23	+K_ITEMS = ["v2clrspct", "v2stcritrecadm", "v2cltrnslw", "v2stfisccap", "v2svstterr", "v2elembcap"]
	24	+IRREGULAR_END = {0, 1, 2, 3, 5, 6, 7, 8}
	25	+KEEP = ["country_name", "country_text_id", "country_id", "year", "e_regionpol_6C", "e_regionpol", "v2x_polyarchy",
	26	+ "e_civil_war", "e_miinterc", "e_coups", "e_pt_coup", "v2regendtype", "v2regint", "v2regdur", "e_pop", "e_area",
	27	+ "e_gdppc", "v2x_regime", "v2x_rule", "v2regidnr"] + K_ITEMS
	28	+
	29	+
	30	+def build() -> dict[str, pd.DataFrame]:
	31	+ vd = pyreadr.read_r(VDEM)["vdem"]
	32	+ cols = [c for c in KEEP if c in vd.columns]
	33	+ v = vd[cols].copy()
	34	+ v["year"] = v.year.astype(int)
	35	+ md = pd.read_excel(MADD, sheet_name="Full data")
	36	+ md = md.rename(columns={"countrycode": "country_text_id", "pop": "maddison_pop", "gdppc": "maddison_gdppc"})
	37	+ md["year"] = md.year.astype(int)
	38	+ m = v.merge(md[["country_text_id", "year", "maddison_pop", "maddison_gdppc"]], on=["country_text_id", "year"], how="left")
	39	+ # crosswalk audit
	40	+ vset, mset = set(v.country_text_id.unique()), set(md.country_text_id.unique())
	41	+ cw = pd.DataFrame({"code": sorted(vset \| mset)})
	42	+ cw["in_vdem"] = cw.code.isin(vset); cw["in_maddison"] = cw.code.isin(mset)
	43	+ names = v.drop_duplicates("country_text_id").set_index("country_text_id").country_name
	44	+ cw["vdem_name"] = cw.code.map(names)
	45	+ META.mkdir(parents=True, exist_ok=True)
	46	+ cw.to_csv(META / "crosswalk_vdem_maddison.csv", index=False)
	47	+ m = m.sort_values(["country_text_id", "year"]).reset_index(drop=True)
	48	+ g = m.groupby("country_text_id")
	49	+ # outcomes in year t
	50	+ cw_prev = g.e_civil_war.shift(1)
	51	+ m["y4_civilwar_onset"] = np.where(m.e_civil_war.isna() \| cw_prev.isna(), np.nan,
	52	+ ((m.e_civil_war == 1) & (cw_prev == 0)).astype(float))
	53	+ m["y5_coup"] = np.where(m.e_coups.isna(), np.nan, (m.e_coups >= 1).astype(float))
	54	+ # v2regendtype is carried on every year of a regime (how that regime eventually ended; 13 = still exists).
	55	+ # The event is placed in the regime's LAST year: v2regidnr changes between t and t+1 (within country).
	56	+ next_id = g.v2regidnr.shift(-1)
	57	+ last_year = m.v2regidnr.notna() & next_id.notna() & (m.v2regidnr != next_id)
	58	+ m["regime_end_year"] = last_year.astype(float)
	59	+ m["y6_irregular_regime_end"] = np.where(m.v2regendtype.isna() \| next_id.isna(), np.nan,
	60	+ (last_year & m.v2regendtype.isin(IRREGULAR_END)).astype(float))
	61	+ m["log_pop"] = np.log(m.maddison_pop.where(m.maddison_pop > 0))
	62	+ m["log_area"] = np.log(m.e_area.where(m.e_area > 0))
	63	+ m["log_gdppc"] = np.log(m.maddison_gdppc.where(m.maddison_gdppc > 0))
	64	+ return {"panel": m, "crosswalk": cw}

added src/wp35/harmonization/seshat.py +226 −0

		`@@ -0,0 +1,226 @@`
	1	+"""Parse Seshat Polaris-2026 into (i) a long facts table with explicit uncertainty, (ii) polity-level
	2	+aggregates, (iii) the CrisisDB outcome table, (iv) polity metadata (macro-region, era, duration, succession).
	3	+
	4	+Coding rules are those frozen in research/preregistration.md §4:
	5	+ P, IP -> 1 (IP flagged inferred); A, IA -> 0 (IA flagged inferred); A~P / P~A -> 0.5 (disputed);
	6	+ SU, U, unknown, blank, anything else -> NaN (logged in results/diagnostics/seshat_unparsed_values.csv).
	7	+Numeric ranges: midpoint (geometric for scale variables that are logged later; arithmetic for levels), bounds kept.
	8	+Nothing is forward-filled; nothing is imputed; undated facts are kept as "whole-polity" facts.
	9	+"""
	10	+from __future__ import annotations
	11	+import json
	12	+from pathlib import Path
	13	+import numpy as np
	14	+import pandas as pd
	15	+import yaml
	16	+
	17	+ROOT = Path(__file__).resolve().parents[3]
	18	+RAW = ROOT / "data" / "raw" / "seshat_polaris2026" / "Polaris2026.xlsx"
	19	+DIAG = ROOT / "results" / "diagnostics"
	20	+
	21	+# Variables that are numeric in Polaris (everything else in the SC/WF/RT sheets is a presence code)
	22	+GEOMETRIC = ["polity_population", "polity_territory", "population_of_the_largest_settlement"]
	23	+ARITHMETIC = ["settlement_hierarchy", "administrative_level", "military_level", "religious_level",
	24	+ "largest_communication_distance", "fastest_individual_communication"]
	25	+NUMERIC = GEOMETRIC + ARITHMETIC
	26	+
	27	+PRESENT = {"P": 1.0, "IP": 1.0, "present": 1.0, "inferred present": 1.0, "inferred P": 1.0}
	28	+ABSENT = {"A": 0.0, "IA": 0.0, "absent": 0.0, "inferred absent": 0.0, "inferred A": 0.0}
	29	+DISPUTED = {"A~P": 0.5, "P~A": 0.5, "IA~IP": 0.5, "IP~IA": 0.5, "A~IP": 0.5, "IP~A": 0.5, "P~IA": 0.5, "IA~P": 0.5}
	30	+INFERRED = {"IP", "IA", "inferred present", "inferred absent", "inferred P", "inferred A"}
	31	+
	32	+# CrisisDB items (pre-registration §3.5)
	33	+INSTABILITY_ITEMS = ["decline", "collapse", "downward_mobility", "extermination", "uprising", "revolution",
	34	+ "successful_revolution", "civil_war", "century_plus", "fragmentation", "capital", "conquest",
	35	+ "assassination", "depose"]
	36	+SEVERE_ITEMS = ["collapse", "fragmentation", "century_plus", "conquest", "civil_war"]
	37	+REFORM_ITEMS = ["constitution", "labor", "unfree_labor", "suffrage", "public_goods", "religion"]
	38	+
	39	+
	40	+def load_sheets() -> dict[str, pd.DataFrame]:
	41	+ xl = pd.ExcelFile(RAW)
	42	+ return {s: xl.parse(s) for s in xl.sheet_names}
	43	+
	44	+
	45	+def code_presence(v) -> tuple[float, bool, bool, bool]:
	46	+ """-> (value, inferred, disputed, unparsed)"""
	47	+ if v is None or (isinstance(v, float) and np.isnan(v)):
	48	+ return np.nan, False, False, False
	49	+ s = str(v).strip()
	50	+ if s in PRESENT:
	51	+ return 1.0, s in INFERRED, False, False
	52	+ if s in ABSENT:
	53	+ return 0.0, s in INFERRED, False, False
	54	+ if s in DISPUTED:
	55	+ return 0.5, False, True, False
	56	+ if s in {"SU", "U", "unknown", "suspected unknown", "SA", "SP", "DIS", "uncoded", ""}:
	57	+ return np.nan, False, False, False
	58	+ return np.nan, False, False, True
	59	+
	60	+
	61	+def _num(v):
	62	+ try:
	63	+ x = float(str(v).replace(",", ""))
	64	+ return x if np.isfinite(x) else np.nan
	65	+ except Exception:
	66	+ return np.nan
	67	+
	68	+
	69	+def build_facts(sheets: dict[str, pd.DataFrame]) -> pd.DataFrame:
	70	+ frames = []
	71	+ for sheet in ["Social complexity", "Warfare", "Religion", "General"]:
	72	+ df = sheets[sheet].copy()
	73	+ df["sheet"] = sheet
	74	+ frames.append(df)
	75	+ f = pd.concat(frames, ignore_index=True)
	76	+ f = f.rename(columns={"variable_name": "variable"})
	77	+ f["dated"] = f.year_from.notna()
	78	+ f["year_from"] = pd.to_numeric(f.year_from, errors="coerce")
	79	+ f["year_to"] = pd.to_numeric(f.year_to, errors="coerce")
	80	+ f.loc[f.year_to.isna() & f.year_from.notna(), "year_to"] = f.year_from
	81	+ f["is_numeric"] = f.variable.isin(NUMERIC)
	82	+ # presence codes
	83	+ coded = f.loc[~f.is_numeric, "value_from"].map(code_presence)
	84	+ f["value"] = np.nan; f["inferred"] = False; f["disputed"] = False; f["unparsed"] = False
	85	+ f.loc[~f.is_numeric, "value"] = [c[0] for c in coded]
	86	+ f.loc[~f.is_numeric, "inferred"] = [c[1] for c in coded]
	87	+ f.loc[~f.is_numeric, "disputed"] = [c[2] for c in coded]
	88	+ f.loc[~f.is_numeric, "unparsed"] = [c[3] for c in coded]
	89	+ # General sheet is categorical text (centralization, succession...): keep raw text in `text`
	90	+ f["text"] = np.where(f.sheet == "General", f.value_from.astype(str), None)
	91	+ f.loc[f.sheet == "General", ["value", "unparsed"]] = [np.nan, False]
	92	+ # numeric
	93	+ lo = f.loc[f.is_numeric, "value_from"].map(_num)
	94	+ hi = f.loc[f.is_numeric, "value_to"].map(_num)
	95	+ hi = hi.where(hi.notna(), lo)
	96	+ lo2, hi2 = np.minimum(lo, hi), np.maximum(lo, hi)
	97	+ f.loc[f.is_numeric, "value_lo"] = lo2
	98	+ f.loc[f.is_numeric, "value_hi"] = hi2
	99	+ geo = f.is_numeric & f.variable.isin(GEOMETRIC)
	100	+ ari = f.is_numeric & f.variable.isin(ARITHMETIC)
	101	+ with np.errstate(divide="ignore", invalid="ignore"):
	102	+ f.loc[geo, "value"] = np.sqrt(f.loc[geo, "value_lo"].clip(lower=0) * f.loc[geo, "value_hi"].clip(lower=0))
	103	+ f.loc[ari, "value"] = (f.loc[ari, "value_lo"] + f.loc[ari, "value_hi"]) / 2
	104	+ # a strictly positive scale variable with value 0 is not a measurement -> NaN (logged)
	105	+ zero_scale = geo & (f.value <= 0)
	106	+ f.loc[zero_scale, "value"] = np.nan
	107	+ f["is_uncertain"] = f.is_uncertain.fillna(False).astype(bool)
	108	+ f["is_disputed"] = f.is_disputed.fillna(False).astype(bool)
	109	+ DIAG.mkdir(parents=True, exist_ok=True)
	110	+ f[f.unparsed][["sheet", "polity_id", "variable", "value_from"]].to_csv(DIAG / "seshat_unparsed_values.csv", index=False)
	111	+ f[zero_scale][["polity_id", "variable", "value_from", "value_to", "year_from"]].to_csv(DIAG / "seshat_zero_scale_values.csv", index=False)
	112	+ keep = ["sheet", "section", "seshat_region", "polity_number", "polity_id", "variable", "value", "value_lo", "value_hi",
	113	+ "text", "year_from", "year_to", "dated", "is_numeric", "inferred", "disputed", "unparsed", "is_uncertain", "is_disputed"]
	114	+ return f[keep]
	115	+
	116	+
	117	+def aggregate_polity(facts: pd.DataFrame, before_year: pd.Series \| None = None) -> pd.DataFrame:
	118	+ """One row per polity x variable. If `before_year` (indexed by polity_id) is given, dated facts are kept only
	119	+ when year_to < before_year[polity]; undated facts are always kept (flagged via `n_undated`)."""
	120	+ f = facts[facts.value.notna()].copy()
	121	+ if before_year is not None:
	122	+ by = f.polity_id.map(before_year)
	123	+ f = f[(~f.dated) \| (f.year_to < by)]
	124	+ f["logv"] = np.where(f.variable.isin(GEOMETRIC), np.log(f.value.where(f.value > 0)), np.nan)
	125	+ g = f.groupby(["polity_id", "variable"])
	126	+ out = g.agg(value_mean=("value", "mean"), value_min=("value", "min"), value_max=("value", "max"),
	127	+ logv_mean=("logv", "mean"), lo_min=("value_lo", "min"), hi_max=("value_hi", "max"),
	128	+ n_facts=("value", "size"), n_dated=("dated", "sum"), n_inferred=("inferred", "sum"),
	129	+ n_disputed=("disputed", "sum"), n_flag_uncertain=("is_uncertain", "sum")).reset_index()
	130	+ out["n_undated"] = out.n_facts - out.n_dated
	131	+ return out
	132	+
	133	+
	134	+def wide(agg: pd.DataFrame, stat: str = "value_mean") -> pd.DataFrame:
	135	+ return agg.pivot(index="polity_id", columns="variable", values=stat)
	136	+
	137	+
	138	+def crisis_table(sheets: dict[str, pd.DataFrame]) -> pd.DataFrame:
	139	+ cc = sheets["CrisisDB - Crisis consequences"].copy()
	140	+ cc["crisis_id"] = [f"{p}_{i}" for i, p in enumerate(cc.polity_id)]
	141	+ items = INSTABILITY_ITEMS + REFORM_ITEMS + ["epidemic"]
	142	+ for it in items:
	143	+ coded = cc[it].map(code_presence)
	144	+ cc[it + "_v"] = [c[0] for c in coded]
	145	+ inst = cc[[i + "_v" for i in INSTABILITY_ITEMS]]
	146	+ cc["y1_n_coded"] = inst.notna().sum(axis=1)
	147	+ cc["y1_n_present"] = (inst == 1).sum(axis=1)
	148	+ cc["y1_severity"] = np.where(cc.y1_n_coded > 0, cc.y1_n_present / cc.y1_n_coded, np.nan)
	149	+ sev = cc[[i + "_v" for i in SEVERE_ITEMS]]
	150	+ cc["y1b_severe"] = np.where(sev.notna().all(axis=1), (sev == 1).any(axis=1).astype(float), np.nan)
	151	+ ref = cc[[i + "_v" for i in REFORM_ITEMS]]
	152	+ cc["y_reform"] = np.where(ref.notna().sum(axis=1) > 0, (ref == 1).sum(axis=1) / ref.notna().sum(axis=1), np.nan)
	153	+ cc["epidemic_v"] = cc["epidemic_v"]
	154	+ cc["crisis_start"] = pd.to_numeric(cc.year_from, errors="coerce")
	155	+ cc["crisis_end"] = pd.to_numeric(cc.year_to, errors="coerce")
	156	+ keep = ["crisis_id", "polity_id", "seshat_region", "other_polity_id", "crisis_start", "crisis_end", "y1_n_coded",
	157	+ "y1_n_present", "y1_severity", "y1b_severe", "y_reform", "epidemic_v"] + [i + "_v" for i in INSTABILITY_ITEMS]
	158	+ return cc[keep]
	159	+
	160	+
	161	+def polity_metadata(sheets: dict[str, pd.DataFrame], facts: pd.DataFrame) -> pd.DataFrame:
	162	+ pol = sheets["Polities"].copy()
	163	+ cfg = yaml.safe_load((ROOT / "configs" / "regions.yaml").read_text())
	164	+ r2m = {r: m for m, rs in cfg["macro_regions"].items() for r in rs}
	165	+ pol["macro_region"] = pol.seshat_region.map(r2m)
	166	+ pol["midpoint"] = (pol.start_year + pol.end_year) / 2
	167	+ def era(y):
	168	+ for b in cfg["era_blocks"]:
	169	+ if y >= b.get("min", -1e9) and y <= b.get("max", 1e9):
	170	+ return b["name"]
	171	+ pol["era"] = np.floor(pol.midpoint).map(era)
	172	+ pol["duration"] = pol.end_year - pol.start_year
	173	+ pol["censored"] = pol.end_year >= 1900
	174	+ gen = facts[facts.sheet == "General"]
	175	+ def first_text(var):
	176	+ s = gen[gen.variable == var].groupby("polity_id").text.first()
	177	+ return pol.polity_id.map(s)
	178	+ pol["centralization"] = first_text("polity_degree_of_centralization")
	179	+ pol["relationship_to_preceding"] = first_text("polity_relationship_to_preceding_entity")
	180	+ pol["preceding_entity"] = first_text("polity_preceding_entity")
	181	+ pol["succeeding_entity"] = first_text("polity_succeeding_entity")
	182	+ pol["suprapolity_relations"] = first_text("polity_suprapolity_relations")
	183	+ pol["utm_zone"] = first_text("polity_utm_zone")
	184	+ pol["language_family"] = first_text("polity_linguistic_family")
	185	+ # Y3: discontinuous succession = relationship_to_preceding of the successor polity (matched by name or id)
	186	+ name2id = {str(n).strip().lower(): i for n, i in zip(pol.long_name, pol.polity_id)}
	187	+ id_set = set(pol.polity_id)
	188	+ def succ_id(s):
	189	+ if s is None or str(s) == "nan" or str(s).strip() == "":
	190	+ return None
	191	+ key = str(s).strip()
	192	+ if key in id_set:
	193	+ return key
	194	+ return name2id.get(key.lower())
	195	+ pol["successor_id"] = pol.succeeding_entity.map(succ_id)
	196	+ # fallback: the polity that names this one as preceding entity
	197	+ prec_map = {}
	198	+ for pid, prec in zip(pol.polity_id, pol.preceding_entity):
	199	+ sid = succ_id(prec)
	200	+ if sid and sid not in prec_map:
	201	+ prec_map[sid] = pid
	202	+ pol["successor_id"] = pol.successor_id.where(pol.successor_id.notna(), pol.polity_id.map(prec_map))
	203	+ rel = dict(zip(pol.polity_id, pol.relationship_to_preceding))
	204	+ succ_rel = pol.successor_id.map(rel)
	205	+ cont = {"continuity", "continuation"}
	206	+ pol["successor_relationship"] = succ_rel
	207	+ pol["y3_discontinuous"] = np.where(succ_rel.isna() \| succ_rel.isin(["SU", "suspected unknown", "nan", "None"]), np.nan,
	208	+ (~succ_rel.isin(cont)).astype(float))
	209	+ return pol
	210	+
	211	+
	212	+def build_all() -> dict[str, pd.DataFrame]:
	213	+ sheets = load_sheets()
	214	+ facts = build_facts(sheets)
	215	+ agg = aggregate_polity(facts)
	216	+ meta = polity_metadata(sheets, facts)
	217	+ crisis = crisis_table(sheets)
	218	+ # pre-crisis aggregates: one per crisis record
	219	+ pre = []
	220	+ for _, r in crisis.iterrows():
	221	+ sub = facts[facts.polity_id == r.polity_id]
	222	+ a = aggregate_polity(sub, before_year=pd.Series({r.polity_id: r.crisis_start}))
	223	+ a["crisis_id"] = r.crisis_id
	224	+ pre.append(a)
	225	+ pre_agg = pd.concat(pre, ignore_index=True) if pre else pd.DataFrame()
	226	+ return {"facts": facts, "polity_agg": agg, "polity_meta": meta, "crisis": crisis, "precrisis_agg": pre_agg}

added tests/test_harmonization.py +46 −0

		`@@ -0,0 +1,46 @@`
	1	+import numpy as np, pandas as pd, pytest
	2	+from pathlib import Path
	3	+from wp35.harmonization import seshat as S
	4	+ROOT = Path(__file__).resolve().parents[1]
	5	+PROC = ROOT / "data" / "processed"
	6	+needs_data = pytest.mark.skipif(not (PROC / "seshat_facts.parquet").exists(), reason="processed data not built")
	7	+
	8	+def test_presence_coding_rules():
	9	+ assert S.code_presence("P")[0] == 1 and S.code_presence("IP") == (1.0, True, False, False)
	10	+ assert S.code_presence("A")[0] == 0 and S.code_presence("IA")[1] is True
	11	+ assert S.code_presence("A~P") == (0.5, False, True, False)
	12	+ for v in ["SU", "U", "unknown", None, np.nan, ""]:
	13	+ assert np.isnan(S.code_presence(v)[0]) and not S.code_presence(v)[3]
	14	+ assert S.code_presence("land")[3] is True # unparsed, logged, never zero
	15	+
	16	+def test_geometric_midpoint_and_bounds():
	17	+ df = pd.DataFrame({"sheet": ["Social complexity"], "section": ["sc"], "seshat_region": ["x"], "polity_number": [1],
	18	+ "polity_id": ["p"], "variable_name": ["polity_population"], "value_from": [100], "value_to": [400],
	19	+ "year_from": [np.nan], "year_to": [np.nan], "is_disputed": [False], "is_uncertain": [False]})
	20	+ f = S.build_facts({"Social complexity": df, "Warfare": df.iloc[0:0], "Religion": df.iloc[0:0], "General": df.iloc[0:0]})
	21	+ assert f.value.iloc[0] == pytest.approx(200.0) and f.value_lo.iloc[0] == 100 and f.value_hi.iloc[0] == 400 and not f.dated.iloc[0]
	22	+
	23	+@needs_data
	24	+def test_no_zero_for_missing_and_counts_match_registry():
	25	+ facts = pd.read_parquet(PROC / "seshat_facts.parquet")
	26	+ assert facts.value.isna().sum() > 0 # unknowns stay missing
	27	+ assert (facts.loc[facts.variable == "polity_population", "value"].dropna() > 0).all()
	28	+ crisis = pd.read_parquet(PROC / "seshat_crisis.parquet")
	29	+ assert len(crisis) == 166 and crisis.polity_id.nunique() == 150
	30	+ assert crisis.y1_severity.between(0, 1).all()
	31	+ meta = pd.read_parquet(PROC / "seshat_polity_meta.parquet")
	32	+ assert len(meta) == 626 and meta.macro_region.notna().all() and meta.era.notna().all()
	33	+
	34	+@needs_data
	35	+def test_modern_events_are_sparse_and_in_last_regime_year():
	36	+ p = pd.read_parquet(PROC / "modern_panel.parquet")
	37	+ assert 0 < p.y6_irregular_regime_end.mean() < 0.1
	38	+ fra = p[p.country_text_id == "FRA"].set_index("year")
	39	+ assert fra.loc[1939, "y6_irregular_regime_end"] == 1.0 and fra.loc[1938, "y6_irregular_regime_end"] == 0.0
	40	+ assert fra.loc[1958, "y6_irregular_regime_end"] == 0.0 # Fourth Republic ended by a directed process (type 10)
	41	+
	42	+@needs_data
	43	+def test_dplace_levels():
	44	+ d = pd.read_parquet(PROC / "dplace_ea_societies.parquet")
	45	+ assert len(d) == 1291 and d.jurisdictional_hierarchy_beyond.dropna().between(0, 4).all()
	46	+ assert d.Family_ID.notna().all()
	47

spb/wp35_uqo

WP35 Phase 2: harmonized Seshat/D-PLACE/modern panels, coverage + missingness diagnostics, data dictionary, parser tests

Theme