#!/usr/bin/env python3 """ 01_download_data.py — key-less download of every raw input used by the WP 24 calibration ("From Labor to Capital: AGI, General Equilibrium, and Asset Valuation"). Run: python3 -I scripts/01_download_data.py [--force] [--only fred,shiller,...] [--series GDP,COE] Rules implemented here ---------------------- * No API keys. Only public, key-less endpoints. * Idempotent: an existing non-empty raw file is skipped unless --force. * Each download is attempted at most MAX_ATTEMPTS (=2) times; on failure the manifest records status=NOT_RETRIEVED with the URL and the error text. Nothing is ever substituted from memory. * Everything is written under data/raw// and recorded in data/raw/manifest.json (url, retrieval date, size, sha256, status, http code). * Downloaded files are treated as untrusted data: they are only read by pandas / openpyxl / xlrd / PyMuPDF in 02_build_calibration_targets.py, never executed. * Only stdlib + requests are needed here (pandas/openpyxl/xlrd/fitz are used in script 02). """ from __future__ import annotations import argparse import hashlib import io import json import sys import time import zipfile from datetime import date from pathlib import Path import requests import urllib.request ROOT = Path(__file__).resolve().parents[1] RAW = ROOT / "data" / "raw" MANIFEST = RAW / "manifest.json" RETRIEVED = date.today().isoformat() MAX_ATTEMPTS = 2 FRED_TIMEOUT = 30 # seconds; FRED's Akamai edge silently hangs non-browser clients from some networks UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0 Safari/537.36") HEADERS = {"User-Agent": UA, "Accept": "*/*", "Connection": "close"} # ---------------------------------------------------------------------------- # FRED series (one call per series; fredgraph.csv = data, data/.txt = metadata) # ---------------------------------------------------------------------------- FRED_SERIES = { # 1. labor share "PRS85006173": "BLS nonfarm business sector labor share, index 2017=100, quarterly", "COE": "Compensation of employees, paid (NIPA), $bn SAAR, quarterly", "GDI": "Gross domestic income, $bn SAAR, quarterly", "W256RC1Q027SBEA": "Net domestic income, $bn SAAR, quarterly", "LABSHPUSA156NRUG": "PWT 10.01 share of labour compensation in GDP, USA, annual", "LABSHPCAA156NRUG": "PWT 10.01 share of labour compensation in GDP, Canada, annual", # 2. profits, dividends, GDP "CP": "Corporate profits after tax (with IVA and CCAdj), $bn SAAR, quarterly", "CPATAX": "Corporate profits after tax (without IVA and CCAdj), $bn SAAR, quarterly", "DIVIDEND": "Corporate profits after tax with IVA and CCAdj: net dividends, $bn SAAR, quarterly", "B056RC1A027NBEA": "Net corporate dividend payments, $bn, annual", "GDP": "Gross domestic product, $bn SAAR, quarterly", "GDPA": "Gross domestic product, $bn, annual", "GDPC1": "Real GDP, chained 2017 $bn SAAR, quarterly", "CPIAUCSL": "CPI-U all items, index 1982-84=100, monthly SA", # 3. household balance sheet (Z.1 via FRED) "TNWBSHNO": "Households and nonprofit organizations; net worth, level, $mn, quarterly", "HNOREMQ027S": "Households and nonprofit organizations; real estate at market value, level, $mn, quarterly", "OEHRENWBSHNO": "Households; owners' equity in real estate, level, $mn, quarterly", "HOEREPHRE": "Households; owners' equity in real estate as % of household real estate, quarterly", # 4. DFA wealth shares (also taken from the Fed DFA zip) "WFRBST01134": "DFA share of net worth held by the top 1%, quarterly", "WFRBSN09161": "DFA share of net worth held by the 90th-99th percentiles, quarterly", "WFRBSN40188": "DFA share of net worth held by the 50th-90th percentiles, quarterly", "WFRBSB50215": "DFA share of net worth held by the bottom 50%, quarterly", # 5. investment "GPDI": "Gross private domestic investment, $bn SAAR, quarterly", "PNFI": "Private nonresidential fixed investment, $bn SAAR, quarterly", "Y034RC1Q027SBEA": "Private fixed investment: nonresidential: equipment: information processing equipment, $bn SAAR, quarterly", "B985RC1Q027SBEA": "Private fixed investment: nonresidential: IPP: software, $bn SAAR, quarterly", "A679RC1Q027SBEA": "Private fixed investment in information processing equipment and software, $bn SAAR, quarterly", "Y001RC1Q027SBEA": "FRED id suggested by the brief for nonresidential fixed investment (title verified from metadata file)", # 6. capital stock and depreciation (BEA Fixed Assets via FRED) "K1PTOTL1ES000": "Current-cost net stock of fixed assets: private (total), $mn, annual end-of-year", "K1NTOTL1ES000": "Current-cost net stock of fixed assets: private: nonresidential, $mn, annual end-of-year", "M1PTOTL1ES000": "Current-cost depreciation of fixed assets: private (total), $mn, annual", "M1NTOTL1ES000": "Current-cost depreciation of fixed assets: private: nonresidential, $mn, annual", "A024RC1A027NBEA": "Consumption of fixed capital (NIPA), $bn, annual", "COFC": "Consumption of fixed capital (NIPA), $bn SAAR, quarterly", # 7. real interest rates "DFII10": "10-year TIPS constant-maturity yield, % p.a., daily", "FII10": "10-year TIPS constant-maturity yield, % p.a., monthly", "REAINTRATREARAT10Y": "10-year real interest rate (Cleveland Fed model), %, monthly", "DGS10": "10-year Treasury constant-maturity yield, %, daily", "GS10": "10-year Treasury constant-maturity yield, %, monthly", # 9. housing "CSUSHPISA": "S&P CoreLogic Case-Shiller US national home price index, Jan-2000=100, monthly SA", "CUSR0000SEHA": "CPI-U rent of primary residence, index 1982-84=100, monthly SA", "CUUR0000SEHA": "CPI-U rent of primary residence, index 1982-84=100, monthly NSA", # 13. electricity price "APU000072610": "Average price: electricity per kWh, US city average, $/kWh, monthly NSA", } # ---------------------------------------------------------------------------- # Other key-less sources # ---------------------------------------------------------------------------- OTHER_FILES = [ # (source dir, file name, url, description) ("shiller", "ie_data.xls", "http://www.econ.yale.edu/~shiller/data/ie_data.xls", "Shiller 'Irrational Exuberance' monthly data: S&P price, dividends, earnings, CPI, GS10, CAPE"), ("shiller", "Fig3-1.xls", "http://www.econ.yale.edu/~shiller/data/Fig3-1.xls", "Shiller Fig. 3.1 data: US real home price index since 1890, building costs, population, interest rate"), ("fed_dfa", "dfa.zip", "https://www.federalreserve.gov/releases/z1/dataviz/download/zips/dfa.zip", "Federal Reserve Distributional Financial Accounts (all csv tables)"), ("bea_io", "AllTablesIO.zip", "https://apps.bea.gov/industry/iTables%20Static%20Files/AllTablesIO.zip", "BEA Input-Output accounts static files (Make/Use before/after redefinitions, 1997-2023 Summary/Sector, 2017 Detail)"), ("bea_io", "AllTablesSUP.zip", "https://apps.bea.gov/industry/iTables%20Static%20Files/AllTablesSUP.zip", "BEA Supply-Use tables static files 1997-2023"), ("pwt", "pwt1001.xlsx", "https://dataverse.nl/api/access/datafile/354095", "Penn World Table 10.01 (Groningen Growth and Development Centre, DataverseNL doi:10.34894/QT5BCC)"), ("epoch", "ml_hardware.csv", "https://epoch.ai/data/ml_hardware.csv", "Epoch AI 'Machine Learning Hardware' dataset (release date, price, FLOP/s, price-performance)"), ("eia", "epa_01_01.xlsx", "https://www.eia.gov/electricity/annual/xls/epa_01_01.xlsx", "EIA Electric Power Annual Table 1.1: total electric power industry summary statistics (net generation, sales)"), ("eia", "table_es1a.xlsx", "https://www.eia.gov/electricity/monthly/xls/table_es1a.xlsx", "EIA Electric Power Monthly Table ES1.A: summary statistics for the latest month"), ("eia", "epa_02_01.xlsx", "https://www.eia.gov/electricity/annual/xls/epa_02_01.xlsx", "EIA Electric Power Annual Table 2.1: retail sales of electricity to ultimate customers (if available)"), ("clevelandfed", "inflation-expectations.xlsx", "https://www.clevelandfed.org/-/media/files/webcharts/inflationexpectations/inflation-expectations.xlsx", "Cleveland Fed model-based inflation expectations and real interest rates (1-30 years)"), ("statcan", "36100103-eng.zip", "https://www150.statcan.gc.ca/n1/tbl/csv/36100103-eng.zip", "Statistics Canada table 36-10-0103-01 GDP income-based, quarterly (link obtained from WDS getFullTableDownloadCSV/36100103/en)"), ("statcan", "36100580-eng.zip", "https://www150.statcan.gc.ca/n1/tbl/csv/36100580-eng.zip", "Statistics Canada table 36-10-0580-01 national balance sheet accounts, quarterly (link from WDS getFullTableDownloadCSV/36100580/en)"), ] LBNL_URLS = [ "https://eta-publications.lbl.gov/sites/default/files/2024-12/lbnl-2024-united-states-data-center-energy-usage-report.pdf", "https://escholarship.org/content/qt32d6m0d1/qt32d6m0d1.pdf", ] TREASURY_REAL_YIELD_YEARS = range(2003, date.today().year + 1) def sha256(path: Path) -> str: h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def load_manifest() -> dict: if MANIFEST.exists(): return json.loads(MANIFEST.read_text()) return {} def save_manifest(m: dict) -> None: MANIFEST.write_text(json.dumps(m, indent=2, sort_keys=True)) def _get(url: str, timeout: int = 90, headers: dict | None = None) -> tuple[int, bytes, str]: """GET with requests; fall back to urllib (system OpenSSL) on connection errors.""" hdr = dict(HEADERS) if headers: hdr.update(headers) try: r = requests.get(url, headers=hdr, timeout=timeout, allow_redirects=True) return r.status_code, r.content, r.headers.get("content-type", "") except requests.RequestException as e: req = urllib.request.Request(url, headers=hdr) with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 (public http(s) URLs only) return resp.status, resp.read(), resp.headers.get("content-type", "") def looks_valid(content: bytes, ctype: str, kind: str) -> str | None: """Return an error string if the payload is obviously not the expected file.""" if not content: return "empty body" if kind == "csv": head = content[:200].lower() if b" bool: """Download url -> dest with MAX_ATTEMPTS attempts; record in manifest.""" dest.parent.mkdir(parents=True, exist_ok=True) if dest.exists() and dest.stat().st_size > 0 and not force: entry = manifest.get(key, {}) if entry.get("status") != "RETRIEVED": manifest[key] = {"url": url, "file": str(dest.relative_to(ROOT)), "description": desc, "retrieved": entry.get("retrieved", RETRIEVED), "size": dest.stat().st_size, "sha256": sha256(dest), "status": "RETRIEVED", "note": "file already present"} print(f" [skip] {key} (exists)") return True urls = [url] + (alt_urls or []) errors = [] for u in urls: for attempt in range(1, MAX_ATTEMPTS + 1): try: code, content, ctype = _get(u, headers=headers, timeout=timeout) err = None if code == 200 else f"HTTP {code}" if err is None: err = looks_valid(content, ctype, kind) if err is None: dest.write_bytes(content) manifest[key] = {"url": u, "requested_url": url, "file": str(dest.relative_to(ROOT)), "description": desc, "retrieved": RETRIEVED, "size": len(content), "sha256": sha256(dest), "status": "RETRIEVED", "content_type": ctype, "attempts": attempt} print(f" [ok] {key} <- {u} ({len(content):,} bytes)") if pause: time.sleep(pause) return True errors.append(f"{u} attempt {attempt}: {err}") except Exception as e: # noqa: BLE001 errors.append(f"{u} attempt {attempt}: {type(e).__name__}: {str(e)[:160]}") time.sleep(5 * attempt) manifest[key] = {"url": url, "file": str(dest.relative_to(ROOT)), "description": desc, "retrieved": RETRIEVED, "status": "NOT_RETRIEVED", "errors": errors} print(f" [FAIL] {key}: {errors[-1]}") return False def extract_zip(zpath: Path, outdir: Path, members: list[str] | None = None) -> None: """Extract (data only, no execution) into its own directory, with path-traversal guard.""" outdir.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(zpath) as z: for info in z.infolist(): name = info.filename if members and name not in members: continue target = (outdir / name).resolve() if not str(target).startswith(str(outdir.resolve())): raise RuntimeError(f"unsafe path in zip: {name}") if info.is_dir(): continue if target.exists() and target.stat().st_size == info.file_size: continue target.parent.mkdir(parents=True, exist_ok=True) with z.open(info) as src, open(target, "wb") as dst: dst.write(src.read()) # ---------------------------------------------------------------------------- def do_fred(manifest, force, only_series): print("== FRED (one fredgraph.csv request per series, paced)") print(" NOTE 2026-10-27: fred.stlouisfed.org (Akamai) accepted TLS but never answered curl/requests/urllib") print(" from this network (laptop and OVH BHS64). Files present in data/raw/fred/ were obtained verbatim") print(" through the Claude Code WebFetch tool the same day (see data/raw/fred/_RETRIEVAL_NOTE.md and") print(" manifest 'retrieval_method'); this block re-downloads them when FRED is reachable (--force).") d = RAW / "fred" for sid, desc in FRED_SERIES.items(): if only_series and sid not in only_series: continue fetch(f"fred:{sid}:csv", f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={sid}", d / f"{sid}.csv", "csv", desc, manifest, force, pause=2.0, timeout=FRED_TIMEOUT) save_manifest(manifest) def do_other(manifest, force, only): print("== Other key-less files") for src, fname, url, desc in OTHER_FILES: if only and src not in only: continue kind = fname.rsplit(".", 1)[-1].lower() fetch(f"{src}:{fname}", url, RAW / src / fname, kind, desc, manifest, force) save_manifest(manifest) # extraction of zip archives (data only) if (not only) or "fed_dfa" in only: z = RAW / "fed_dfa" / "dfa.zip" if z.exists(): extract_zip(z, RAW / "fed_dfa" / "unzipped") if (not only) or "bea_io" in only: z = RAW / "bea_io" / "AllTablesIO.zip" if z.exists(): extract_zip(z, RAW / "bea_io" / "unzipped", members=[ "IOUse_After_Redefinitions_PRO_1997-2023_Summary.xlsx", "IOUse_After_Redefinitions_PRO_1997-2023_Sector.xlsx", "IOMake_After_Redefinitions_PRO_1997-2023_Summary.xlsx", ]) if (not only) or "statcan" in only: for pid in ("36100103", "36100580"): z = RAW / "statcan" / f"{pid}-eng.zip" if z.exists(): extract_zip(z, RAW / "statcan" / "unzipped") def do_lbnl(manifest, force): print("== LBNL 2024 United States Data Center Energy Usage Report (PDF)") fetch("lbnl:report_pdf", LBNL_URLS[0], RAW / "lbnl" / "lbnl-2024-united-states-data-center-energy-usage-report.pdf", "pdf", "LBNL-2001637, Shehabi et al. (Dec. 2024), 2024 United States Data Center Energy Usage Report; " "eScholarship mirror (DOI 10.71468/P1WC7Q) used if the LBNL host blocks non-browser clients", manifest, force, headers={"Accept": "application/pdf,*/*"}, alt_urls=LBNL_URLS[1:]) save_manifest(manifest) def do_treasury(manifest, force): print("== US Treasury daily real yield curve (TIPS), one CSV per year") for y in TREASURY_REAL_YIELD_YEARS: url = ("https://home.treasury.gov/resource-center/data-chart-center/interest-rates/" f"daily-treasury-rates.csv/{y}/all?type=daily_treasury_real_yield_curve&field_tdr_date_value={y}&page&_format=csv") fetch(f"treasury:real_yield_{y}", url, RAW / "treasury" / f"daily_treasury_real_yield_curve_{y}.csv", "csv", f"Daily Treasury par real yield curve rates (TIPS), {y}", manifest, force, pause=0.5) save_manifest(manifest) def do_bls(manifest, force): """BLS public API v1 (no key; max 10 years per query, 25 queries/day). Used as an independent cross-check of PRS85006173, APU000072610 and CUSR0000SEHA.""" print("== BLS public API v1 (cross-check series)") series = ["PRS85006173", "APU000072610", "CUSR0000SEHA"] windows = [(1990, 1999), (2000, 2009), (2010, 2019), (2020, date.today().year)] for sid in series: for (a, b) in windows: key = f"bls:{sid}:{a}-{b}" dest = RAW / "bls" / f"{sid}_{a}-{b}.json" dest.parent.mkdir(parents=True, exist_ok=True) if dest.exists() and dest.stat().st_size > 0 and not force: print(f" [skip] {key}") continue url = "https://api.bls.gov/publicAPI/v1/timeseries/data/" payload = json.dumps({"seriesid": [sid], "startyear": str(a), "endyear": str(b)}) errors = [] ok = False for attempt in range(1, MAX_ATTEMPTS + 1): try: r = requests.post(url, data=payload, headers={**HEADERS, "Content-Type": "application/json"}, timeout=60) js = r.json() if r.status_code == 200 and js.get("status") == "REQUEST_SUCCEEDED": dest.write_bytes(r.content) manifest[key] = {"url": url, "post_payload": json.loads(payload), "file": str(dest.relative_to(ROOT)), "description": f"BLS series {sid}, {a}-{b}", "retrieved": RETRIEVED, "size": len(r.content), "sha256": sha256(dest), "status": "RETRIEVED"} print(f" [ok] {key}") ok = True break errors.append(f"attempt {attempt}: HTTP {r.status_code} {js.get('status')} {js.get('message')}") except Exception as e: # noqa: BLE001 errors.append(f"attempt {attempt}: {type(e).__name__}: {str(e)[:160]}") time.sleep(5) if not ok: manifest[key] = {"url": url, "post_payload": json.loads(payload), "file": str(dest.relative_to(ROOT)), "retrieved": RETRIEVED, "status": "NOT_RETRIEVED", "errors": errors} print(f" [FAIL] {key}: {errors[-1]}") time.sleep(1) save_manifest(manifest) def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--force", action="store_true", help="re-download even if the raw file exists") ap.add_argument("--only", default="", help="comma list of sources: fred,shiller,fed_dfa,bea_io,pwt,epoch,eia,clevelandfed,statcan,lbnl,treasury,bls") ap.add_argument("--series", default="", help="comma list of FRED series ids to restrict the FRED block") args = ap.parse_args() only = {s.strip() for s in args.only.split(",") if s.strip()} only_series = {s.strip() for s in args.series.split(",") if s.strip()} RAW.mkdir(parents=True, exist_ok=True) manifest = load_manifest() if not only or "fred" in only: do_fred(manifest, args.force, only_series) do_other(manifest, args.force, only) if not only or "lbnl" in only: do_lbnl(manifest, args.force) if not only or "treasury" in only: do_treasury(manifest, args.force) if not only or "bls" in only: do_bls(manifest, args.force) save_manifest(manifest) nr = [k for k, v in manifest.items() if v.get("status") == "NOT_RETRIEVED"] print(f"\nManifest: {len(manifest)} entries, {len(nr)} NOT_RETRIEVED") for k in nr: print(" NOT_RETRIEVED:", k, "<-", manifest[k].get("url")) return 0 if __name__ == "__main__": sys.exit(main())