N10: universe artefacts shipped with SHA-256; holdings CSV / company_tickers.json not archived; 2,573 -> 2,555 reconciled; missing filings; annual_panel = 2,499 CIKs
Items: FIRC0001-ADVE-15 AI_C-12 DATA-01 ECON-10 EDIT-16 LITE-09 THEO-14 (THEO-07, ECON-18).
data/metadata/{universe,russell3000_constituents,sp500_constituents}.json + SHA256SUMS; 18 dual-class CIKs explain 2,573 lines -> 2,555 CIKs;
7 unmatched tickers absent from ticker_to_cik (10,430 entries); 2,587 and the 14/21 drop counts not reconstructible;
2 filings missing on disk (XEL 10-K 2014, LNT 10-Q 2012) + 14 parse errors; annual_panel/quarterly_panel carry 2,499 CIKs (paper: 2,501).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +20,001 −0
added
code/audit/a10_universe_artefacts.py
+77 −0
@@ -0,0 +1,77 @@ | |||||
| 1 | +"""N10 - universe artefacts: hashes, reconstructible counts, missing files, 'missing' filings, annual_panel CIKs (cluster H, I). Read-only.""" | ||||
| 2 | +from __future__ import annotations | ||||
| 3 | +import json, sys, subprocess | ||||
| 4 | +from pathlib import Path | ||||
| 5 | +sys.path.insert(0, str(Path(__file__).resolve().parent)) | ||||
| 6 | +from common import * | ||||
| 7 | + | ||||
| 8 | +r = Run("universe") | ||||
| 9 | +META = RDB / "data/metadata" | ||||
| 10 | +U = {} | ||||
| 11 | +files = ["universe.json", "russell3000_constituents.json", "sp500_constituents.json", "cik_mapping.json", "shards.json"] | ||||
| 12 | +U["sha256"] = {f: {"sha256": sha256_file(META / f), "bytes": (META / f).stat().st_size, "mtime": dt.datetime.fromtimestamp((META / f).stat().st_mtime).isoformat()} for f in files if (META / f).exists()} | ||||
| 13 | +with open(r.out / "SHA256SUMS", "w") as fh: | ||||
| 14 | + for f, v in U["sha256"].items(): | ||||
| 15 | + fh.write(f"{v['sha256']} {f}\n") | ||||
| 16 | +# not archived? | ||||
| 17 | +found = subprocess.run(["find", str(RDB), "-maxdepth", "6", "(", "-iname", "*holding*", "-o", "-iname", "company_tickers*", "-o", "-iname", "*IWV*", ")", "-not", "-path", "*/.venv/*"], capture_output=True, text=True).stdout.strip().splitlines() | ||||
| 18 | +U["holdings_or_company_tickers_files_found"] = found | ||||
| 19 | +uni = json.load(open(META / "universe.json")); ru = json.load(open(META / "russell3000_constituents.json")); sp = json.load(open(META / "sp500_constituents.json")); cm = json.load(open(META / "cik_mapping.json")) | ||||
| 20 | +U["universe_keys"] = sorted(uni.keys()); U["russell_keys"] = sorted(ru.keys()); U["sp500_keys"] = sorted(sp.keys()) | ||||
| 21 | +U["cik_mapping_type"] = type(cm).__name__; U["cik_mapping_size"] = len(cm) | ||||
| 22 | +U["cik_mapping_scalars"] = {k: v for k, v in cm.items() if not isinstance(v, (list, dict))} if isinstance(cm, dict) else None | ||||
| 23 | +U["cik_mapping_dict_sizes"] = {k: len(v) for k, v in cm.items() if isinstance(v, (list, dict))} if isinstance(cm, dict) else None | ||||
| 24 | +U["universe_scalars"] = {k: v for k, v in uni.items() if not isinstance(v, (list, dict))} | ||||
| 25 | +U["russell_scalars"] = {k: v for k, v in ru.items() if not isinstance(v, (list, dict))} | ||||
| 26 | +U["sp500_scalars"] = {k: v for k, v in sp.items() if not isinstance(v, (list, dict))} | ||||
| 27 | +cons = ru.get("constituents", []) | ||||
| 28 | +U["russell_constituent_lines"] = len(cons) | ||||
| 29 | +U["russell_distinct_ciks"] = len({c.get("cik") for c in cons}) | ||||
| 30 | +U["russell_constituent_keys"] = sorted(cons[0].keys()) if cons else [] | ||||
| 31 | +U["russell_unmatched"] = ru.get("unmatched") | ||||
| 32 | +dup = {} | ||||
| 33 | +for c in cons: dup.setdefault(c.get("cik"), []).append(c.get("ticker")) | ||||
| 34 | +U["russell_ciks_with_multiple_lines"] = {k: v for k, v in dup.items() if len(v) > 1} | ||||
| 35 | +# mapper view of the unmatched tickers | ||||
| 36 | +T2C = cm.get("ticker_to_cik", {}) if isinstance(cm, dict) else {} | ||||
| 37 | +def lookup(tkr): | ||||
| 38 | + out = {"in_ticker_to_cik": T2C.get(tkr) or T2C.get(tkr.upper()) or T2C.get(tkr.replace(".", "-")) or T2C.get(tkr.replace("-", "."))} | ||||
| 39 | + for k, v in (cm.items() if isinstance(cm, dict) else []): | ||||
| 40 | + if isinstance(v, dict) and k != "ticker_to_cik" and (tkr in v or tkr.upper() in v): | ||||
| 41 | + out[k] = v.get(tkr, v.get(tkr.upper())) | ||||
| 42 | + if isinstance(v, list) and any(str(x).upper().startswith(tkr.upper()) for x in v[:100000] if isinstance(x, str)): | ||||
| 43 | + out[k] = [x for x in v if isinstance(x, str) and str(x).upper().startswith(tkr.upper())][:3] | ||||
| 44 | + return out | ||||
| 45 | +U["unmatched_mapper_view"] = {str(u): lookup(str(u).split(" ")[0]) for u in (ru.get("unmatched") or [])} | ||||
| 46 | +# is the unmatched ticker's company present in the universe under another ticker/CIK (e.g. HOLX)? | ||||
| 47 | +U["unmatched_in_universe_by_name"] = {str(u): [ (c.get("ticker"), c.get("cik"), c.get("company_name") or c.get("name")) for c in ucons0 if str(u).split("(")[-1].rstrip(")").split(" ")[0].upper() in str(c.get("company_name") or c.get("name") or "").upper()][:3] for u in (ru.get("unmatched") or [])} if (ucons0 := uni.get("constituents", [])) is not None else None | ||||
| 48 | +U["unmatched_in_edgar_companies"] = {} | ||||
| 49 | +ucons = uni.get("constituents", []) | ||||
| 50 | +U["universe_counts"] = {"n": len(ucons), "in_sp500": sum(1 for c in ucons if c.get("in_sp500")), "in_russell3000": sum(1 for c in ucons if c.get("in_russell3000")), | ||||
| 51 | + "both": sum(1 for c in ucons if c.get("in_sp500") and c.get("in_russell3000")), "sp_only": sum(1 for c in ucons if c.get("in_sp500") and not c.get("in_russell3000")), | ||||
| 52 | + "ru_only": sum(1 for c in ucons if c.get("in_russell3000") and not c.get("in_sp500")), "keys": sorted(ucons[0].keys()) if ucons else []} | ||||
| 53 | +U["sp500_rows"] = len(sp.get("constituents", [])); U["sp500_distinct_ciks"] = len({c.get("cik") for c in sp.get("constituents", [])}) | ||||
| 54 | +U["reconciliation"] = {"iwv_equity_lines_reported_in_paper": 2587, "reconstructible": False, "reason": "latest-holdings.csv / company_tickers.json not archived; universe.py drops non-company lines with a bare continue", | ||||
| 55 | + "lines_mapped": ru.get("count"), "distinct_ciks": U["russell_distinct_ciks"], "unmatched": len(ru.get("unmatched") or []), "universe_n": uni.get("count")} | ||||
| 56 | +# 'missing' filings and annual_panel CIKs | ||||
| 57 | +con = connect(SPID_DB); attach_edgar(con) | ||||
| 58 | +U["download_parse_status"] = con.execute("select form_type, download_status, parse_status, count(*) from edgar.filings group by 1,2,3 order by 1,2,3").fetchall() | ||||
| 59 | +r.dump("filings_status_missing.csv", con, "select accession_number, cik, ticker, form_type, filing_date, download_status, parse_status, local_path, file_size_bytes from edgar.filings where parse_status in ('missing','error') or download_status <> 'complete' order by parse_status, 1") | ||||
| 60 | +U["filings_missing_on_disk"] = con.execute("select form_type, count(*) from edgar.filings where parse_status='missing' group by 1").fetchall() | ||||
| 61 | +U["filings_parse_error"] = con.execute("select form_type, count(*) from edgar.filings where parse_status='error' group by 1").fetchall() | ||||
| 62 | +U["sections_for_error_filings"] = con.execute("select count(distinct accession_number) from edgar.filing_sections where accession_number in (select accession_number from edgar.filings where parse_status in ('error','missing'))").fetchone()[0] | ||||
| 63 | +U["paper_annual_panel_firms"] = 2501 | ||||
| 64 | +U["annual_panel_distinct_ciks"] = con.execute("select count(distinct cik) from edgar.annual_panel").fetchone()[0] | ||||
| 65 | +U["filings_distinct_ciks"] = con.execute("select count(distinct cik) from edgar.filings").fetchone()[0] | ||||
| 66 | +U["companies_rows"] = con.execute("select count(*) from edgar.companies").fetchone()[0] | ||||
| 67 | +U["annual_panel_ciks_not_in_filings"] = con.execute("select list(distinct cik) from edgar.annual_panel where cik not in (select cik from edgar.filings)").fetchone()[0] | ||||
| 68 | +U["filings_ciks_not_in_annual_panel"] = con.execute("select list(distinct cik) from edgar.filings where cik not in (select cik from edgar.annual_panel)").fetchone()[0] | ||||
| 69 | +U["annual_panel_ciks_not_in_companies"] = con.execute("select list(distinct cik) from edgar.annual_panel where cik not in (select cik from edgar.companies)").fetchone()[0] | ||||
| 70 | +U["annual_panel_rows"] = con.execute("select count(*), min(fiscal_year), max(fiscal_year) from edgar.annual_panel").fetchone() | ||||
| 71 | +U["quarterly_panel_distinct_ciks"] = con.execute("select count(distinct cik) from edgar.quarterly_panel").fetchone()[0] | ||||
| 72 | +for u in (ru.get("unmatched") or []): | ||||
| 73 | + tk = str(u).split(" ")[0]; nm = str(u).split("(")[-1].rstrip(")") | ||||
| 74 | + U["unmatched_in_edgar_companies"][str(u)] = con.execute("select ticker, cik, company_name, in_sp500, in_russell3000 from edgar.companies where upper(ticker)=? or upper(company_name) like ? limit 3", [tk.upper(), "%" + nm.split(" ")[0].upper() + "%"]).fetchall() | ||||
| 75 | +U["companies_in_sp500_in_russell"] = con.execute("select count(*) filter (where in_sp500), count(*) filter (where in_russell3000), count(*) filter (where in_sp500 and in_russell3000), count(*) from edgar.companies").fetchone() | ||||
| 76 | +r.write_json("universe_artefacts.json", U) | ||||
| 77 | +r.finish() | ||||
added
data/metadata/SHA256SUMS
+5 −0
@@ -0,0 +1,5 @@ | |||||
| 1 | +3967b29b378e50fbbac139bfc905120749ff09dcf8ec5769c3fbb12cb1c016e2 universe.json | ||||
| 2 | +758cbebf58540230ede17fd35890336ba6bf9e29fad58e3792ce7134997c6b41 russell3000_constituents.json | ||||
| 3 | +5084075260692130896c3c9a187223e2870b8e73774b35d1e28763fe8ffddb7d sp500_constituents.json | ||||
| 4 | +7ec59b276cef779f8f714d29d7b77938ee3731f11ea5bd95526006fd03ed20fc cik_mapping.json | ||||
| 5 | +8351ff54bea877175ca4bba51fc043ba279b2bfbcc7ddb80db1da534ff922771 shards.json | ||||
added
data/metadata/russell3000_constituents.json
+19919 −0
Large diff (19,919 lines) is not loaded by default to keep the page fast.Load diff
Diff truncated — file too large.