#!/usr/bin/env python3 """01_download_data.py -- fetch raw public series for WP25 calibration into data/raw/. Every attempted download is logged in data/raw/MANIFEST.json with URL, UTC timestamp, HTTP status, bytes, sha256 and a status/notes field. Failures are recorded, never hidden. No API key is required for any source. Run: python3 -I scripts/01_download_data.py Standard library only (urllib, hashlib, zipfile, json). """ from __future__ import annotations import hashlib import json import re import sys import urllib.error import urllib.request import zipfile from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] RAW = ROOT / "data" / "raw" RAW.mkdir(parents=True, exist_ok=True) MANIFEST = RAW / "MANIFEST.json" FETCH_DATE = "2026-10-27" UA = "UQO-WP25-calibration/1.0 (research; contact spbou4@icloud.com)" TIMEOUT = 120 # ---------------------------------------------------------------------------- # FRED series (CSV endpoint, no key). cosd forces the full history where FRED # allows it; ICE BofA series are contractually limited to a 3-year window since # April 2026 (recorded in notes), SP500 to 10 years. # ---------------------------------------------------------------------------- FRED = { "PRS85006173": "Nonfarm Business Sector: Labor Share for All Workers (index 2017=100, quarterly, BLS)", "LABSHPUSA156NRUG": "Share of Labour Compensation in GDP at Current National Prices for United States (PWT 10.x, annual ratio) -- level anchor for the BLS index", "A4002E1A156NBEA": "Shares of gross domestic income: Compensation of employees, paid (percent, annual, BEA)", "W270RE1A156NBEA": "Shares of gross domestic income: Compensation of employees, paid: Wage and salary accruals: Disbursements: to persons (percent, annual, BEA)", "GDPC1": "Real Gross Domestic Product (bn chained 2017 $, SAAR, quarterly)", "PCECC96": "Real Personal Consumption Expenditures (bn chained 2017 $, SAAR, quarterly)", "GPDIC1": "Real Gross Private Domestic Investment (bn chained 2017 $, SAAR, quarterly)", "A191RL1Q225SBEA": "Real GDP: percent change from preceding period, SAAR (quarterly)", "DFII10": "Market Yield on U.S. Treasury Securities at 10-Year Constant Maturity, Inflation-Indexed (TIPS, percent, daily, 2003-)", "DGS10": "Market Yield on U.S. Treasury Securities at 10-Year Constant Maturity (percent, daily)", "DFF": "Federal Funds Effective Rate (percent, daily)", "TB3MS": "3-Month Treasury Bill Secondary Market Rate, Discount Basis (percent, monthly)", "CPIAUCSL": "Consumer Price Index for All Urban Consumers: All Items in U.S. City Average (1982-84=100, monthly, SA)", "K1TTOTL1ES000": "Current-Cost Net Stock of Fixed Assets: Private (millions $, annual, end of year, BEA Fixed Assets)", "M1TTOTL1ES000": "Current-Cost Depreciation of Fixed Assets: Private (millions $, annual, BEA Fixed Assets)", "SP500": "S&P 500 index level (daily; FRED licence limits download to the most recent 10 years)", "BAMLC0A0CM": "ICE BofA US Corporate Index Option-Adjusted Spread (percent, daily; 3-year window only since April 2026)", "BAMLH0A0HYM2": "ICE BofA US High Yield Index Option-Adjusted Spread (percent, daily; 3-year window only since April 2026)", "DRTSCILM": "Net Percentage of Domestic Banks Tightening Standards for C&I Loans to Large and Middle-Market Firms (SLOOS, quarterly)", "TOTBKCR": "Bank Credit, All Commercial Banks (bn $, weekly, H.8)", "TLAACBW027SBOG": "Total Assets, All Commercial Banks (bn $, weekly, H.8)", "RALACBW027SBOG": "Residual (Assets Less Liabilities), All Commercial Banks (bn $, weekly, H.8) -- book equity proxy", "BOGZ1FL073164003Q": "Interest Rates and Price Indexes; NYSE Composite Index, Level (Z.1, quarterly, end of period)", "WILL5000INDFC": "Wilshire 5000 Full Cap Price Index -- EXPECTED TO FAIL (series withdrawn from FRED; recorded for completeness)", } OTHER = [ # Federal Reserve Distributional Financial Accounts -- full CSV bundle { "id": "DFA_zip", "url": "https://www.federalreserve.gov/releases/z1/dataviz/download/zips/dfa.zip", "file": "dfa/dfa.zip", "notes": "Fed DFA complete download (networth/income/age/... levels and shares CSVs). Landing page: https://www.federalreserve.gov/releases/z1/dataviz/dfa/distribute/table/", "unzip_to": "dfa", }, # Fernald utilization-adjusted TFP (FRBSF) { "id": "Fernald_quarterly_tfp", "url": "https://www.frbsf.org/wp-content/uploads/quarterly_tfp.xlsx", "file": "fernald/quarterly_tfp.xlsx", "notes": "John Fernald, 'A Quarterly, Utilization-Adjusted Series on Total Factor Productivity', FRBSF WP 2012-19; sheets quarterly/annual. Landing page: https://www.frbsf.org/research-and-insights/data-and-indicators/total-factor-productivity-tfp/", }, # Shiller long-run stock market data (current file hosted on shillerdata.com) { "id": "Shiller_ie_data", "url": "https://img1.wsimg.com/blobby/go/e5e77e0b-59d1-44d9-ab25-4763ac982e53/downloads/1449043e-a1ed-4e2e-8e9c-7b9a2a064493/ie_data.xls?ver=1791297298731", "file": "shiller/ie_data.xls", "notes": "Robert Shiller, U.S. Stock Markets 1871-Present and CAPE Ratio (monthly S&P price, dividends, earnings, CPI, GS10). Landing page: https://shillerdata.com/ . Legacy Yale copy (ends 2023-09): http://www.econ.yale.edu/~shiller/data/ie_data.xls", }, # SEC XBRL company facts: capital expenditures (PaymentsToAcquirePropertyPlantAndEquipment) { "id": "SEC_XBRL_capex_MSFT", "url": "https://data.sec.gov/api/xbrl/companyconcept/CIK0000789019/us-gaap/PaymentsToAcquirePropertyPlantAndEquipment.json", "file": "sec_xbrl/capex_MSFT.json", "notes": "Microsoft (CIK 789019) 'Additions to property and equipment', all filed periods (10-K/10-Q).", }, { "id": "SEC_XBRL_capex_GOOGL", "url": "https://data.sec.gov/api/xbrl/companyconcept/CIK0001652044/us-gaap/PaymentsToAcquirePropertyPlantAndEquipment.json", "file": "sec_xbrl/capex_GOOGL.json", "notes": "Alphabet (CIK 1652044) purchases of property and equipment.", }, { "id": "SEC_XBRL_capex_AMZN", "url": "https://data.sec.gov/api/xbrl/companyconcept/CIK0001018724/us-gaap/PaymentsToAcquirePropertyPlantAndEquipment.json", "file": "sec_xbrl/capex_AMZN.json", "notes": "Amazon (CIK 1018724) purchases of property and equipment.", }, { "id": "SEC_XBRL_capex_META", "url": "https://data.sec.gov/api/xbrl/companyconcept/CIK0001326801/us-gaap/PaymentsToAcquirePropertyPlantAndEquipment.json", "file": "sec_xbrl/capex_META.json", "notes": "Meta Platforms (CIK 1326801) purchases of property and equipment (excludes finance-lease principal payments).", }, { "id": "AMZN_Q4_2025_release", "url": "https://s2.q4cdn.com/299287126/files/doc_earnings/2025/q4/earnings-result/AMZN-Q4-2025-Earnings-Release.pdf", "file": "sec_xbrl/AMZN-Q4-2025-Earnings-Release.pdf", "notes": "Amazon Q4/FY2025 earnings release (cash-flow statement: purchases of property and equipment, twelve months 2025).", }, { "id": "META_Q4_2025_exhibit991", "url": "https://www.sec.gov/Archives/edgar/data/1326801/000162828026003832/meta-12312025xexhibit991.htm", "file": "sec_xbrl/meta-12312025xexhibit991.htm", "notes": "Meta 8-K exhibit 99.1, Q4/FY2025 results (capex incl. finance leases = $72.22bn FY2025).", }, # EIA average retail electricity price table (HTML) { "id": "EIA_EPM_table_5_6_A", "url": "https://www.eia.gov/electricity/monthly/epm_table_grapher.php?t=epmt_5_6_a", "file": "eia/epm_table_5_6_a.html", "notes": "EIA Electric Power Monthly, Table 5.6.A Average Price of Electricity to Ultimate Customers by End-Use Sector (cents/kWh), latest month vs year earlier.", }, # Open working-paper PDFs used for ledger parameters { "id": "Floden_Linde_1998_WP", "url": "http://su.diva-portal.org/smash/get/diva2:328071/FULLTEXT01", "file": "papers/floden_linde_1998_iies654.pdf", "notes": "Floden & Linde, 'Idiosyncratic risk in the U.S. and Sweden', IIES Seminar Paper 654 (1998); published RED 2001, doi:10.1006/redy.2000.0121. Table 4 earnings-process estimates.", }, { "id": "Gorton_Metrick_w15223", "url": "https://www.nber.org/system/files/working_papers/w15223/w15223.pdf", "file": "papers/gorton_metrick_w15223.pdf", "notes": "Gorton & Metrick, 'Securitized banking and the run on repo', NBER WP 15223 (2009); published JFE 2012, doi:10.1016/j.jfineco.2011.03.016. Repo-haircut index.", }, ] def sha256_of(path: Path) -> str: h = hashlib.sha256() with open(path, "rb") as fh: for chunk in iter(lambda: fh.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def fetch(url: str, dest: Path, *, expect_csv: bool = False) -> dict: dest.parent.mkdir(parents=True, exist_ok=True) rec = { "url": url, "file": str(dest.relative_to(ROOT)), "timestamp_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), "fetch_date": FETCH_DATE, } req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "*/*"}) try: with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: data = resp.read() rec["http_status"] = resp.status rec["content_type"] = resp.headers.get("Content-Type", "") except urllib.error.HTTPError as e: rec.update(http_status=e.code, status="failed", error=f"HTTPError {e.code} {e.reason}") return rec except Exception as e: # noqa: BLE001 rec.update(http_status=None, status="failed", error=f"{type(e).__name__}: {e}") return rec if expect_csv and not data[:40].lstrip().lower().startswith(b"observation_date"): rec.update(status="failed", error="response is not a FRED CSV (probably an HTML error page)", bytes=len(data)) return rec dest.write_bytes(data) rec.update(bytes=len(data), sha256=sha256_of(dest), status="ok") return rec def safe_unzip(zpath: Path, outdir: Path) -> list[str]: """Extract only flat, plainly named members (untrusted archive).""" outdir.mkdir(parents=True, exist_ok=True) names = [] with zipfile.ZipFile(zpath) as z: for m in z.infolist(): if m.is_dir() or not re.fullmatch(r"[A-Za-z0-9._-]+", m.filename): continue if m.file_size > 50_000_000: continue (outdir / m.filename).write_bytes(z.read(m)) names.append(m.filename) return names def main() -> int: manifest = {"fetch_date": FETCH_DATE, "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), "entries": []} n_ok = n_fail = 0 for sid, desc in FRED.items(): url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={sid}&cosd=1900-01-01" rec = fetch(url, RAW / "fred" / f"{sid}.csv", expect_csv=True) rec.update(id=f"FRED:{sid}", description=desc, source="FRED (Federal Reserve Bank of St. Louis), https://fred.stlouisfed.org/series/" + sid) if rec["status"] == "ok": txt = (RAW / "fred" / f"{sid}.csv").read_text(errors="replace").strip().splitlines() rec["n_rows"] = len(txt) - 1 rec["first_obs"] = txt[1].split(",")[0] if len(txt) > 1 else None rec["last_obs"] = txt[-1].split(",")[0] if len(txt) > 1 else None if sid.startswith("BAML"): rec["notes"] = "FRED notes: 'Starting in April 2026, this series will only include 3 years of observations.' 2008/2020 peaks NOT obtainable here." if sid == "SP500": rec["notes"] = "FRED/S&P licence: only the most recent 10 years are downloadable." n_ok += 1 else: n_fail += 1 manifest["entries"].append(rec) print(f"[{rec['status']:6s}] {rec['id']}: {rec.get('bytes', 0)} bytes {rec.get('error', '')}") for item in OTHER: rec = fetch(item["url"], RAW / item["file"]) rec.update(id=item["id"], notes=item["notes"]) if rec["status"] == "ok": n_ok += 1 if item.get("unzip_to"): try: rec["extracted"] = safe_unzip(RAW / item["file"], RAW / item["unzip_to"]) except zipfile.BadZipFile as e: rec["extract_error"] = str(e) else: n_fail += 1 manifest["entries"].append(rec) print(f"[{rec['status']:6s}] {rec['id']}: {rec.get('bytes', 0)} bytes {rec.get('error', '')}") # Things that are documented but not downloadable programmatically manifest["not_downloaded"] = [ {"id": "WID_USA_wealth_shares", "url": "https://wid.world/country/usa/", "reason": "page renders charts client-side; no CSV fetched. DFA (Fed) used instead for top shares."}, {"id": "SCF_2022_bulletin", "url": "https://www.federalreserve.gov/publications/october-2023-changes-in-us-family-finances-from-2019-to-2022.htm", "reason": "HTML report read on 2026-10-27 (median/mean net worth recorded in ledger); no machine-readable series downloaded."}, {"id": "Census_BTOS_AI_use", "url": "https://www.census.gov/hfp/btos/data", "reason": "data page is JavaScript-driven; values taken from the Census story https://census.gov/library/stories/2026/05/ai-use-businesses.html (read 2026-10-27)."}, {"id": "FDIC_QBP_2Q2026_capital_ratio", "url": "https://www.fdic.gov/quarterly-banking-profile/quarterly-banking-profile-2q-2026", "reason": "equity/assets ratio not on the HTML summary page; bank equity ratio computed from FRED H.8 RALACBW027SBOG/TLAACBW027SBOG instead."}, {"id": "ICE_BofA_OAS_history", "url": "https://fred.stlouisfed.org/series/BAMLH0A0HYM2", "reason": "FRED limited to 3-year window since April 2026; full history requires ICE Data Indices licence."}, {"id": "WILL5000INDFC", "url": "https://fred.stlouisfed.org/series/WILL5000INDFC", "reason": "HTTP 404 on 2026-10-27 (Wilshire indices no longer distributed by FRED). SP500 (10-year window) and Shiller data used instead."}, {"id": "Cooley_Prescott_1995_chapter", "url": "https://press.princeton.edu/books/paperback/9780691043234/frontiers-of-business-cycle-research", "reason": "book chapter not available online; publisher page returned HTTP 403 to scripted access on 2026-10-27."}, ] manifest["summary"] = {"ok": n_ok, "failed": n_fail} MANIFEST.write_text(json.dumps(manifest, indent=2)) print(f"\nManifest written: {MANIFEST} ({n_ok} ok, {n_fail} failed)") return 0 if __name__ == "__main__": sys.exit(main())