1#!/usr/bin/env python32"""301_download_data.py — key-less download of every raw input used by the WP 244calibration ("From Labor to Capital: AGI, General Equilibrium, and Asset Valuation").56Run: python3 -I scripts/01_download_data.py [--force] [--only fred,shiller,...] [--series GDP,COE]78Rules implemented here9----------------------10* No API keys. Only public, key-less endpoints.11* Idempotent: an existing non-empty raw file is skipped unless --force.12* Each download is attempted at most MAX_ATTEMPTS (=2) times; on failure the13 manifest records status=NOT_RETRIEVED with the URL and the error text.14 Nothing is ever substituted from memory.15* Everything is written under data/raw/<source>/ and recorded in16 data/raw/manifest.json (url, retrieval date, size, sha256, status, http code).17* Downloaded files are treated as untrusted data: they are only read by pandas /18 openpyxl / xlrd / PyMuPDF in 02_build_calibration_targets.py, never executed.19* Only stdlib + requests are needed here (pandas/openpyxl/xlrd/fitz are used in20 script 02).21"""22from __future__ import annotations2324import argparse25import hashlib26import io27import json28import sys29import time30import zipfile31from datetime import date32from pathlib import Path3334import requests35import urllib.request3637ROOT = Path(__file__).resolve().parents[1]38RAW = ROOT / "data" / "raw"39MANIFEST = RAW / "manifest.json"40RETRIEVED = date.today().isoformat()41MAX_ATTEMPTS = 242FRED_TIMEOUT = 30 # seconds; FRED's Akamai edge silently hangs non-browser clients from some networks43UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 "44 "(KHTML, like Gecko) Chrome/120.0 Safari/537.36")45HEADERS = {"User-Agent": UA, "Accept": "*/*", "Connection": "close"}4647# ----------------------------------------------------------------------------48# FRED series (one call per series; fredgraph.csv = data, data/<ID>.txt = metadata)49# ----------------------------------------------------------------------------50FRED_SERIES = {51 # 1. labor share52 "PRS85006173": "BLS nonfarm business sector labor share, index 2017=100, quarterly",53 "COE": "Compensation of employees, paid (NIPA), $bn SAAR, quarterly",54 "GDI": "Gross domestic income, $bn SAAR, quarterly",55 "W256RC1Q027SBEA": "Net domestic income, $bn SAAR, quarterly",56 "LABSHPUSA156NRUG": "PWT 10.01 share of labour compensation in GDP, USA, annual",57 "LABSHPCAA156NRUG": "PWT 10.01 share of labour compensation in GDP, Canada, annual",58 # 2. profits, dividends, GDP59 "CP": "Corporate profits after tax (with IVA and CCAdj), $bn SAAR, quarterly",60 "CPATAX": "Corporate profits after tax (without IVA and CCAdj), $bn SAAR, quarterly",61 "DIVIDEND": "Corporate profits after tax with IVA and CCAdj: net dividends, $bn SAAR, quarterly",62 "B056RC1A027NBEA": "Net corporate dividend payments, $bn, annual",63 "GDP": "Gross domestic product, $bn SAAR, quarterly",64 "GDPA": "Gross domestic product, $bn, annual",65 "GDPC1": "Real GDP, chained 2017 $bn SAAR, quarterly",66 "CPIAUCSL": "CPI-U all items, index 1982-84=100, monthly SA",67 # 3. household balance sheet (Z.1 via FRED)68 "TNWBSHNO": "Households and nonprofit organizations; net worth, level, $mn, quarterly",69 "HNOREMQ027S": "Households and nonprofit organizations; real estate at market value, level, $mn, quarterly",70 "OEHRENWBSHNO": "Households; owners' equity in real estate, level, $mn, quarterly",71 "HOEREPHRE": "Households; owners' equity in real estate as % of household real estate, quarterly",72 # 4. DFA wealth shares (also taken from the Fed DFA zip)73 "WFRBST01134": "DFA share of net worth held by the top 1%, quarterly",74 "WFRBSN09161": "DFA share of net worth held by the 90th-99th percentiles, quarterly",75 "WFRBSN40188": "DFA share of net worth held by the 50th-90th percentiles, quarterly",76 "WFRBSB50215": "DFA share of net worth held by the bottom 50%, quarterly",77 # 5. investment78 "GPDI": "Gross private domestic investment, $bn SAAR, quarterly",79 "PNFI": "Private nonresidential fixed investment, $bn SAAR, quarterly",80 "Y034RC1Q027SBEA": "Private fixed investment: nonresidential: equipment: information processing equipment, $bn SAAR, quarterly",81 "B985RC1Q027SBEA": "Private fixed investment: nonresidential: IPP: software, $bn SAAR, quarterly",82 "A679RC1Q027SBEA": "Private fixed investment in information processing equipment and software, $bn SAAR, quarterly",83 "Y001RC1Q027SBEA": "FRED id suggested by the brief for nonresidential fixed investment (title verified from metadata file)",84 # 6. capital stock and depreciation (BEA Fixed Assets via FRED)85 "K1PTOTL1ES000": "Current-cost net stock of fixed assets: private (total), $mn, annual end-of-year",86 "K1NTOTL1ES000": "Current-cost net stock of fixed assets: private: nonresidential, $mn, annual end-of-year",87 "M1PTOTL1ES000": "Current-cost depreciation of fixed assets: private (total), $mn, annual",88 "M1NTOTL1ES000": "Current-cost depreciation of fixed assets: private: nonresidential, $mn, annual",89 "A024RC1A027NBEA": "Consumption of fixed capital (NIPA), $bn, annual",90 "COFC": "Consumption of fixed capital (NIPA), $bn SAAR, quarterly",91 # 7. real interest rates92 "DFII10": "10-year TIPS constant-maturity yield, % p.a., daily",93 "FII10": "10-year TIPS constant-maturity yield, % p.a., monthly",94 "REAINTRATREARAT10Y": "10-year real interest rate (Cleveland Fed model), %, monthly",95 "DGS10": "10-year Treasury constant-maturity yield, %, daily",96 "GS10": "10-year Treasury constant-maturity yield, %, monthly",97 # 9. housing98 "CSUSHPISA": "S&P CoreLogic Case-Shiller US national home price index, Jan-2000=100, monthly SA",99 "CUSR0000SEHA": "CPI-U rent of primary residence, index 1982-84=100, monthly SA",100 "CUUR0000SEHA": "CPI-U rent of primary residence, index 1982-84=100, monthly NSA",101 # 13. electricity price102 "APU000072610": "Average price: electricity per kWh, US city average, $/kWh, monthly NSA",103}104105# ----------------------------------------------------------------------------106# Other key-less sources107# ----------------------------------------------------------------------------108OTHER_FILES = [109 # (source dir, file name, url, description)110 ("shiller", "ie_data.xls", "http://www.econ.yale.edu/~shiller/data/ie_data.xls",111 "Shiller 'Irrational Exuberance' monthly data: S&P price, dividends, earnings, CPI, GS10, CAPE"),112 ("shiller", "Fig3-1.xls", "http://www.econ.yale.edu/~shiller/data/Fig3-1.xls",113 "Shiller Fig. 3.1 data: US real home price index since 1890, building costs, population, interest rate"),114 ("fed_dfa", "dfa.zip", "https://www.federalreserve.gov/releases/z1/dataviz/download/zips/dfa.zip",115 "Federal Reserve Distributional Financial Accounts (all csv tables)"),116 ("bea_io", "AllTablesIO.zip", "https://apps.bea.gov/industry/iTables%20Static%20Files/AllTablesIO.zip",117 "BEA Input-Output accounts static files (Make/Use before/after redefinitions, 1997-2023 Summary/Sector, 2017 Detail)"),118 ("bea_io", "AllTablesSUP.zip", "https://apps.bea.gov/industry/iTables%20Static%20Files/AllTablesSUP.zip",119 "BEA Supply-Use tables static files 1997-2023"),120 ("pwt", "pwt1001.xlsx", "https://dataverse.nl/api/access/datafile/354095",121 "Penn World Table 10.01 (Groningen Growth and Development Centre, DataverseNL doi:10.34894/QT5BCC)"),122 ("epoch", "ml_hardware.csv", "https://epoch.ai/data/ml_hardware.csv",123 "Epoch AI 'Machine Learning Hardware' dataset (release date, price, FLOP/s, price-performance)"),124 ("eia", "epa_01_01.xlsx", "https://www.eia.gov/electricity/annual/xls/epa_01_01.xlsx",125 "EIA Electric Power Annual Table 1.1: total electric power industry summary statistics (net generation, sales)"),126 ("eia", "table_es1a.xlsx", "https://www.eia.gov/electricity/monthly/xls/table_es1a.xlsx",127 "EIA Electric Power Monthly Table ES1.A: summary statistics for the latest month"),128 ("eia", "epa_02_01.xlsx", "https://www.eia.gov/electricity/annual/xls/epa_02_01.xlsx",129 "EIA Electric Power Annual Table 2.1: retail sales of electricity to ultimate customers (if available)"),130 ("clevelandfed", "inflation-expectations.xlsx",131 "https://www.clevelandfed.org/-/media/files/webcharts/inflationexpectations/inflation-expectations.xlsx",132 "Cleveland Fed model-based inflation expectations and real interest rates (1-30 years)"),133 ("statcan", "36100103-eng.zip", "https://www150.statcan.gc.ca/n1/tbl/csv/36100103-eng.zip",134 "Statistics Canada table 36-10-0103-01 GDP income-based, quarterly (link obtained from WDS getFullTableDownloadCSV/36100103/en)"),135 ("statcan", "36100580-eng.zip", "https://www150.statcan.gc.ca/n1/tbl/csv/36100580-eng.zip",136 "Statistics Canada table 36-10-0580-01 national balance sheet accounts, quarterly (link from WDS getFullTableDownloadCSV/36100580/en)"),137]138139LBNL_URLS = [140 "https://eta-publications.lbl.gov/sites/default/files/2024-12/lbnl-2024-united-states-data-center-energy-usage-report.pdf",141 "https://escholarship.org/content/qt32d6m0d1/qt32d6m0d1.pdf",142]143144TREASURY_REAL_YIELD_YEARS = range(2003, date.today().year + 1)145146147def sha256(path: Path) -> str:148 h = hashlib.sha256()149 with open(path, "rb") as f:150 for chunk in iter(lambda: f.read(1 << 20), b""):151 h.update(chunk)152 return h.hexdigest()153154155def load_manifest() -> dict:156 if MANIFEST.exists():157 return json.loads(MANIFEST.read_text())158 return {}159160161def save_manifest(m: dict) -> None:162 MANIFEST.write_text(json.dumps(m, indent=2, sort_keys=True))163164165def _get(url: str, timeout: int = 90, headers: dict | None = None) -> tuple[int, bytes, str]:166 """GET with requests; fall back to urllib (system OpenSSL) on connection errors."""167 hdr = dict(HEADERS)168 if headers:169 hdr.update(headers)170 try:171 r = requests.get(url, headers=hdr, timeout=timeout, allow_redirects=True)172 return r.status_code, r.content, r.headers.get("content-type", "")173 except requests.RequestException as e:174 req = urllib.request.Request(url, headers=hdr)175 with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 (public http(s) URLs only)176 return resp.status, resp.read(), resp.headers.get("content-type", "")177178179def looks_valid(content: bytes, ctype: str, kind: str) -> str | None:180 """Return an error string if the payload is obviously not the expected file."""181 if not content:182 return "empty body"183 if kind == "csv":184 head = content[:200].lower()185 if b"<html" in head or b"<!doctype" in head:186 return "HTML page instead of CSV"187 if kind == "txt":188 if b"<html" in content[:200].lower():189 return "HTML page instead of text"190 if kind == "xls" and content[:4] != b"\xd0\xcf\x11\xe0":191 return "not an OLE2 .xls file"192 if kind == "xlsx" and content[:2] != b"PK":193 return "not a zip-based .xlsx file (got %s)" % ctype194 if kind == "zip" and content[:2] != b"PK":195 return "not a zip file (got %s)" % ctype196 if kind == "pdf" and content[:4] != b"%PDF":197 return "not a PDF (got %s)" % ctype198 if kind == "json":199 try:200 json.loads(content.decode("utf-8"))201 except Exception as e: # noqa: BLE001202 return "invalid JSON: %s" % e203 return None204205206def fetch(key: str, url: str, dest: Path, kind: str, desc: str, manifest: dict,207 force: bool, pause: float = 0.0, headers: dict | None = None,208 alt_urls: list[str] | None = None, timeout: int = 90) -> bool:209 """Download url -> dest with MAX_ATTEMPTS attempts; record in manifest."""210 dest.parent.mkdir(parents=True, exist_ok=True)211 if dest.exists() and dest.stat().st_size > 0 and not force:212 entry = manifest.get(key, {})213 if entry.get("status") != "RETRIEVED":214 manifest[key] = {"url": url, "file": str(dest.relative_to(ROOT)), "description": desc,215 "retrieved": entry.get("retrieved", RETRIEVED), "size": dest.stat().st_size,216 "sha256": sha256(dest), "status": "RETRIEVED", "note": "file already present"}217 print(f" [skip] {key} (exists)")218 return True219 urls = [url] + (alt_urls or [])220 errors = []221 for u in urls:222 for attempt in range(1, MAX_ATTEMPTS + 1):223 try:224 code, content, ctype = _get(u, headers=headers, timeout=timeout)225 err = None if code == 200 else f"HTTP {code}"226 if err is None:227 err = looks_valid(content, ctype, kind)228 if err is None:229 dest.write_bytes(content)230 manifest[key] = {"url": u, "requested_url": url, "file": str(dest.relative_to(ROOT)),231 "description": desc, "retrieved": RETRIEVED, "size": len(content),232 "sha256": sha256(dest), "status": "RETRIEVED", "content_type": ctype,233 "attempts": attempt}234 print(f" [ok] {key} <- {u} ({len(content):,} bytes)")235 if pause:236 time.sleep(pause)237 return True238 errors.append(f"{u} attempt {attempt}: {err}")239 except Exception as e: # noqa: BLE001240 errors.append(f"{u} attempt {attempt}: {type(e).__name__}: {str(e)[:160]}")241 time.sleep(5 * attempt)242 manifest[key] = {"url": url, "file": str(dest.relative_to(ROOT)), "description": desc,243 "retrieved": RETRIEVED, "status": "NOT_RETRIEVED", "errors": errors}244 print(f" [FAIL] {key}: {errors[-1]}")245 return False246247248def extract_zip(zpath: Path, outdir: Path, members: list[str] | None = None) -> None:249 """Extract (data only, no execution) into its own directory, with path-traversal guard."""250 outdir.mkdir(parents=True, exist_ok=True)251 with zipfile.ZipFile(zpath) as z:252 for info in z.infolist():253 name = info.filename254 if members and name not in members:255 continue256 target = (outdir / name).resolve()257 if not str(target).startswith(str(outdir.resolve())):258 raise RuntimeError(f"unsafe path in zip: {name}")259 if info.is_dir():260 continue261 if target.exists() and target.stat().st_size == info.file_size:262 continue263 target.parent.mkdir(parents=True, exist_ok=True)264 with z.open(info) as src, open(target, "wb") as dst:265 dst.write(src.read())266267268# ----------------------------------------------------------------------------269def do_fred(manifest, force, only_series):270 print("== FRED (one fredgraph.csv request per series, paced)")271 print(" NOTE 2026-10-27: fred.stlouisfed.org (Akamai) accepted TLS but never answered curl/requests/urllib")272 print(" from this network (laptop and OVH BHS64). Files present in data/raw/fred/ were obtained verbatim")273 print(" through the Claude Code WebFetch tool the same day (see data/raw/fred/_RETRIEVAL_NOTE.md and")274 print(" manifest 'retrieval_method'); this block re-downloads them when FRED is reachable (--force).")275 d = RAW / "fred"276 for sid, desc in FRED_SERIES.items():277 if only_series and sid not in only_series:278 continue279 fetch(f"fred:{sid}:csv", f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={sid}",280 d / f"{sid}.csv", "csv", desc, manifest, force, pause=2.0, timeout=FRED_TIMEOUT)281 save_manifest(manifest)282283284def do_other(manifest, force, only):285 print("== Other key-less files")286 for src, fname, url, desc in OTHER_FILES:287 if only and src not in only:288 continue289 kind = fname.rsplit(".", 1)[-1].lower()290 fetch(f"{src}:{fname}", url, RAW / src / fname, kind, desc, manifest, force)291 save_manifest(manifest)292 # extraction of zip archives (data only)293 if (not only) or "fed_dfa" in only:294 z = RAW / "fed_dfa" / "dfa.zip"295 if z.exists():296 extract_zip(z, RAW / "fed_dfa" / "unzipped")297 if (not only) or "bea_io" in only:298 z = RAW / "bea_io" / "AllTablesIO.zip"299 if z.exists():300 extract_zip(z, RAW / "bea_io" / "unzipped", members=[301 "IOUse_After_Redefinitions_PRO_1997-2023_Summary.xlsx",302 "IOUse_After_Redefinitions_PRO_1997-2023_Sector.xlsx",303 "IOMake_After_Redefinitions_PRO_1997-2023_Summary.xlsx",304 ])305 if (not only) or "statcan" in only:306 for pid in ("36100103", "36100580"):307 z = RAW / "statcan" / f"{pid}-eng.zip"308 if z.exists():309 extract_zip(z, RAW / "statcan" / "unzipped")310311312def do_lbnl(manifest, force):313 print("== LBNL 2024 United States Data Center Energy Usage Report (PDF)")314 fetch("lbnl:report_pdf", LBNL_URLS[0], RAW / "lbnl" / "lbnl-2024-united-states-data-center-energy-usage-report.pdf",315 "pdf", "LBNL-2001637, Shehabi et al. (Dec. 2024), 2024 United States Data Center Energy Usage Report; "316 "eScholarship mirror (DOI 10.71468/P1WC7Q) used if the LBNL host blocks non-browser clients",317 manifest, force, headers={"Accept": "application/pdf,*/*"}, alt_urls=LBNL_URLS[1:])318 save_manifest(manifest)319320321def do_treasury(manifest, force):322 print("== US Treasury daily real yield curve (TIPS), one CSV per year")323 for y in TREASURY_REAL_YIELD_YEARS:324 url = ("https://home.treasury.gov/resource-center/data-chart-center/interest-rates/"325 f"daily-treasury-rates.csv/{y}/all?type=daily_treasury_real_yield_curve&field_tdr_date_value={y}&page&_format=csv")326 fetch(f"treasury:real_yield_{y}", url, RAW / "treasury" / f"daily_treasury_real_yield_curve_{y}.csv",327 "csv", f"Daily Treasury par real yield curve rates (TIPS), {y}", manifest, force, pause=0.5)328 save_manifest(manifest)329330331def do_bls(manifest, force):332 """BLS public API v1 (no key; max 10 years per query, 25 queries/day). Used as an333 independent cross-check of PRS85006173, APU000072610 and CUSR0000SEHA."""334 print("== BLS public API v1 (cross-check series)")335 series = ["PRS85006173", "APU000072610", "CUSR0000SEHA"]336 windows = [(1990, 1999), (2000, 2009), (2010, 2019), (2020, date.today().year)]337 for sid in series:338 for (a, b) in windows:339 key = f"bls:{sid}:{a}-{b}"340 dest = RAW / "bls" / f"{sid}_{a}-{b}.json"341 dest.parent.mkdir(parents=True, exist_ok=True)342 if dest.exists() and dest.stat().st_size > 0 and not force:343 print(f" [skip] {key}")344 continue345 url = "https://api.bls.gov/publicAPI/v1/timeseries/data/"346 payload = json.dumps({"seriesid": [sid], "startyear": str(a), "endyear": str(b)})347 errors = []348 ok = False349 for attempt in range(1, MAX_ATTEMPTS + 1):350 try:351 r = requests.post(url, data=payload, headers={**HEADERS, "Content-Type": "application/json"}, timeout=60)352 js = r.json()353 if r.status_code == 200 and js.get("status") == "REQUEST_SUCCEEDED":354 dest.write_bytes(r.content)355 manifest[key] = {"url": url, "post_payload": json.loads(payload), "file": str(dest.relative_to(ROOT)),356 "description": f"BLS series {sid}, {a}-{b}", "retrieved": RETRIEVED,357 "size": len(r.content), "sha256": sha256(dest), "status": "RETRIEVED"}358 print(f" [ok] {key}")359 ok = True360 break361 errors.append(f"attempt {attempt}: HTTP {r.status_code} {js.get('status')} {js.get('message')}")362 except Exception as e: # noqa: BLE001363 errors.append(f"attempt {attempt}: {type(e).__name__}: {str(e)[:160]}")364 time.sleep(5)365 if not ok:366 manifest[key] = {"url": url, "post_payload": json.loads(payload), "file": str(dest.relative_to(ROOT)),367 "retrieved": RETRIEVED, "status": "NOT_RETRIEVED", "errors": errors}368 print(f" [FAIL] {key}: {errors[-1]}")369 time.sleep(1)370 save_manifest(manifest)371372373def main() -> int:374 ap = argparse.ArgumentParser()375 ap.add_argument("--force", action="store_true", help="re-download even if the raw file exists")376 ap.add_argument("--only", default="", help="comma list of sources: fred,shiller,fed_dfa,bea_io,pwt,epoch,eia,clevelandfed,statcan,lbnl,treasury,bls")377 ap.add_argument("--series", default="", help="comma list of FRED series ids to restrict the FRED block")378 args = ap.parse_args()379 only = {s.strip() for s in args.only.split(",") if s.strip()}380 only_series = {s.strip() for s in args.series.split(",") if s.strip()}381382 RAW.mkdir(parents=True, exist_ok=True)383 manifest = load_manifest()384 if not only or "fred" in only:385 do_fred(manifest, args.force, only_series)386 do_other(manifest, args.force, only)387 if not only or "lbnl" in only:388 do_lbnl(manifest, args.force)389 if not only or "treasury" in only:390 do_treasury(manifest, args.force)391 if not only or "bls" in only:392 do_bls(manifest, args.force)393 save_manifest(manifest)394395 nr = [k for k, v in manifest.items() if v.get("status") == "NOT_RETRIEVED"]396 print(f"\nManifest: {len(manifest)} entries, {len(nr)} NOT_RETRIEVED")397 for k in nr:398 print(" NOT_RETRIEVED:", k, "<-", manifest[k].get("url"))399 return 0400401402if __name__ == "__main__":403 sys.exit(main())404