1#!/usr/bin/env python32"""01_download_data.py -- fetch raw public series for WP25 calibration into data/raw/.34Every attempted download is logged in data/raw/MANIFEST.json with URL, UTC5timestamp, HTTP status, bytes, sha256 and a status/notes field. Failures are6recorded, never hidden. No API key is required for any source.78Run: python3 -I scripts/01_download_data.py9Standard library only (urllib, hashlib, zipfile, json).10"""11from __future__ import annotations1213import hashlib14import json15import re16import sys17import urllib.error18import urllib.request19import zipfile20from datetime import datetime, timezone21from pathlib import Path2223ROOT = Path(__file__).resolve().parents[1]24RAW = ROOT / "data" / "raw"25RAW.mkdir(parents=True, exist_ok=True)26MANIFEST = RAW / "MANIFEST.json"2728FETCH_DATE = "2026-10-27"29UA = "UQO-WP25-calibration/1.0 (research; contact spbou4@icloud.com)"30TIMEOUT = 1203132# ----------------------------------------------------------------------------33# FRED series (CSV endpoint, no key). cosd forces the full history where FRED34# allows it; ICE BofA series are contractually limited to a 3-year window since35# April 2026 (recorded in notes), SP500 to 10 years.36# ----------------------------------------------------------------------------37FRED = {38 "PRS85006173": "Nonfarm Business Sector: Labor Share for All Workers (index 2017=100, quarterly, BLS)",39 "LABSHPUSA156NRUG": "Share of Labour Compensation in GDP at Current National Prices for United States (PWT 10.x, annual ratio) -- level anchor for the BLS index",40 "A4002E1A156NBEA": "Shares of gross domestic income: Compensation of employees, paid (percent, annual, BEA)",41 "W270RE1A156NBEA": "Shares of gross domestic income: Compensation of employees, paid: Wage and salary accruals: Disbursements: to persons (percent, annual, BEA)",42 "GDPC1": "Real Gross Domestic Product (bn chained 2017 $, SAAR, quarterly)",43 "PCECC96": "Real Personal Consumption Expenditures (bn chained 2017 $, SAAR, quarterly)",44 "GPDIC1": "Real Gross Private Domestic Investment (bn chained 2017 $, SAAR, quarterly)",45 "A191RL1Q225SBEA": "Real GDP: percent change from preceding period, SAAR (quarterly)",46 "DFII10": "Market Yield on U.S. Treasury Securities at 10-Year Constant Maturity, Inflation-Indexed (TIPS, percent, daily, 2003-)",47 "DGS10": "Market Yield on U.S. Treasury Securities at 10-Year Constant Maturity (percent, daily)",48 "DFF": "Federal Funds Effective Rate (percent, daily)",49 "TB3MS": "3-Month Treasury Bill Secondary Market Rate, Discount Basis (percent, monthly)",50 "CPIAUCSL": "Consumer Price Index for All Urban Consumers: All Items in U.S. City Average (1982-84=100, monthly, SA)",51 "K1TTOTL1ES000": "Current-Cost Net Stock of Fixed Assets: Private (millions $, annual, end of year, BEA Fixed Assets)",52 "M1TTOTL1ES000": "Current-Cost Depreciation of Fixed Assets: Private (millions $, annual, BEA Fixed Assets)",53 "SP500": "S&P 500 index level (daily; FRED licence limits download to the most recent 10 years)",54 "BAMLC0A0CM": "ICE BofA US Corporate Index Option-Adjusted Spread (percent, daily; 3-year window only since April 2026)",55 "BAMLH0A0HYM2": "ICE BofA US High Yield Index Option-Adjusted Spread (percent, daily; 3-year window only since April 2026)",56 "DRTSCILM": "Net Percentage of Domestic Banks Tightening Standards for C&I Loans to Large and Middle-Market Firms (SLOOS, quarterly)",57 "TOTBKCR": "Bank Credit, All Commercial Banks (bn $, weekly, H.8)",58 "TLAACBW027SBOG": "Total Assets, All Commercial Banks (bn $, weekly, H.8)",59 "RALACBW027SBOG": "Residual (Assets Less Liabilities), All Commercial Banks (bn $, weekly, H.8) -- book equity proxy",60 "BOGZ1FL073164003Q": "Interest Rates and Price Indexes; NYSE Composite Index, Level (Z.1, quarterly, end of period)",61 "WILL5000INDFC": "Wilshire 5000 Full Cap Price Index -- EXPECTED TO FAIL (series withdrawn from FRED; recorded for completeness)",62}6364OTHER = [65 # Federal Reserve Distributional Financial Accounts -- full CSV bundle66 {67 "id": "DFA_zip",68 "url": "https://www.federalreserve.gov/releases/z1/dataviz/download/zips/dfa.zip",69 "file": "dfa/dfa.zip",70 "notes": "Fed DFA complete download (networth/income/age/... levels and shares CSVs). Landing page: https://www.federalreserve.gov/releases/z1/dataviz/dfa/distribute/table/",71 "unzip_to": "dfa",72 },73 # Fernald utilization-adjusted TFP (FRBSF)74 {75 "id": "Fernald_quarterly_tfp",76 "url": "https://www.frbsf.org/wp-content/uploads/quarterly_tfp.xlsx",77 "file": "fernald/quarterly_tfp.xlsx",78 "notes": "John Fernald, 'A Quarterly, Utilization-Adjusted Series on Total Factor Productivity', FRBSF WP 2012-19; sheets quarterly/annual. Landing page: https://www.frbsf.org/research-and-insights/data-and-indicators/total-factor-productivity-tfp/",79 },80 # Shiller long-run stock market data (current file hosted on shillerdata.com)81 {82 "id": "Shiller_ie_data",83 "url": "https://img1.wsimg.com/blobby/go/e5e77e0b-59d1-44d9-ab25-4763ac982e53/downloads/1449043e-a1ed-4e2e-8e9c-7b9a2a064493/ie_data.xls?ver=1791297298731",84 "file": "shiller/ie_data.xls",85 "notes": "Robert Shiller, U.S. Stock Markets 1871-Present and CAPE Ratio (monthly S&P price, dividends, earnings, CPI, GS10). Landing page: https://shillerdata.com/ . Legacy Yale copy (ends 2023-09): http://www.econ.yale.edu/~shiller/data/ie_data.xls",86 },87 # SEC XBRL company facts: capital expenditures (PaymentsToAcquirePropertyPlantAndEquipment)88 {89 "id": "SEC_XBRL_capex_MSFT",90 "url": "https://data.sec.gov/api/xbrl/companyconcept/CIK0000789019/us-gaap/PaymentsToAcquirePropertyPlantAndEquipment.json",91 "file": "sec_xbrl/capex_MSFT.json",92 "notes": "Microsoft (CIK 789019) 'Additions to property and equipment', all filed periods (10-K/10-Q).",93 },94 {95 "id": "SEC_XBRL_capex_GOOGL",96 "url": "https://data.sec.gov/api/xbrl/companyconcept/CIK0001652044/us-gaap/PaymentsToAcquirePropertyPlantAndEquipment.json",97 "file": "sec_xbrl/capex_GOOGL.json",98 "notes": "Alphabet (CIK 1652044) purchases of property and equipment.",99 },100 {101 "id": "SEC_XBRL_capex_AMZN",102 "url": "https://data.sec.gov/api/xbrl/companyconcept/CIK0001018724/us-gaap/PaymentsToAcquirePropertyPlantAndEquipment.json",103 "file": "sec_xbrl/capex_AMZN.json",104 "notes": "Amazon (CIK 1018724) purchases of property and equipment.",105 },106 {107 "id": "SEC_XBRL_capex_META",108 "url": "https://data.sec.gov/api/xbrl/companyconcept/CIK0001326801/us-gaap/PaymentsToAcquirePropertyPlantAndEquipment.json",109 "file": "sec_xbrl/capex_META.json",110 "notes": "Meta Platforms (CIK 1326801) purchases of property and equipment (excludes finance-lease principal payments).",111 },112 {113 "id": "AMZN_Q4_2025_release",114 "url": "https://s2.q4cdn.com/299287126/files/doc_earnings/2025/q4/earnings-result/AMZN-Q4-2025-Earnings-Release.pdf",115 "file": "sec_xbrl/AMZN-Q4-2025-Earnings-Release.pdf",116 "notes": "Amazon Q4/FY2025 earnings release (cash-flow statement: purchases of property and equipment, twelve months 2025).",117 },118 {119 "id": "META_Q4_2025_exhibit991",120 "url": "https://www.sec.gov/Archives/edgar/data/1326801/000162828026003832/meta-12312025xexhibit991.htm",121 "file": "sec_xbrl/meta-12312025xexhibit991.htm",122 "notes": "Meta 8-K exhibit 99.1, Q4/FY2025 results (capex incl. finance leases = $72.22bn FY2025).",123 },124 # EIA average retail electricity price table (HTML)125 {126 "id": "EIA_EPM_table_5_6_A",127 "url": "https://www.eia.gov/electricity/monthly/epm_table_grapher.php?t=epmt_5_6_a",128 "file": "eia/epm_table_5_6_a.html",129 "notes": "EIA Electric Power Monthly, Table 5.6.A Average Price of Electricity to Ultimate Customers by End-Use Sector (cents/kWh), latest month vs year earlier.",130 },131 # Open working-paper PDFs used for ledger parameters132 {133 "id": "Floden_Linde_1998_WP",134 "url": "http://su.diva-portal.org/smash/get/diva2:328071/FULLTEXT01",135 "file": "papers/floden_linde_1998_iies654.pdf",136 "notes": "Floden & Linde, 'Idiosyncratic risk in the U.S. and Sweden', IIES Seminar Paper 654 (1998); published RED 2001, doi:10.1006/redy.2000.0121. Table 4 earnings-process estimates.",137 },138 {139 "id": "Gorton_Metrick_w15223",140 "url": "https://www.nber.org/system/files/working_papers/w15223/w15223.pdf",141 "file": "papers/gorton_metrick_w15223.pdf",142 "notes": "Gorton & Metrick, 'Securitized banking and the run on repo', NBER WP 15223 (2009); published JFE 2012, doi:10.1016/j.jfineco.2011.03.016. Repo-haircut index.",143 },144]145146147def sha256_of(path: Path) -> str:148 h = hashlib.sha256()149 with open(path, "rb") as fh:150 for chunk in iter(lambda: fh.read(1 << 20), b""):151 h.update(chunk)152 return h.hexdigest()153154155def fetch(url: str, dest: Path, *, expect_csv: bool = False) -> dict:156 dest.parent.mkdir(parents=True, exist_ok=True)157 rec = {158 "url": url,159 "file": str(dest.relative_to(ROOT)),160 "timestamp_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),161 "fetch_date": FETCH_DATE,162 }163 req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "*/*"})164 try:165 with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:166 data = resp.read()167 rec["http_status"] = resp.status168 rec["content_type"] = resp.headers.get("Content-Type", "")169 except urllib.error.HTTPError as e:170 rec.update(http_status=e.code, status="failed", error=f"HTTPError {e.code} {e.reason}")171 return rec172 except Exception as e: # noqa: BLE001173 rec.update(http_status=None, status="failed", error=f"{type(e).__name__}: {e}")174 return rec175 if expect_csv and not data[:40].lstrip().lower().startswith(b"observation_date"):176 rec.update(status="failed", error="response is not a FRED CSV (probably an HTML error page)", bytes=len(data))177 return rec178 dest.write_bytes(data)179 rec.update(bytes=len(data), sha256=sha256_of(dest), status="ok")180 return rec181182183def safe_unzip(zpath: Path, outdir: Path) -> list[str]:184 """Extract only flat, plainly named members (untrusted archive)."""185 outdir.mkdir(parents=True, exist_ok=True)186 names = []187 with zipfile.ZipFile(zpath) as z:188 for m in z.infolist():189 if m.is_dir() or not re.fullmatch(r"[A-Za-z0-9._-]+", m.filename):190 continue191 if m.file_size > 50_000_000:192 continue193 (outdir / m.filename).write_bytes(z.read(m))194 names.append(m.filename)195 return names196197198def main() -> int:199 manifest = {"fetch_date": FETCH_DATE, "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), "entries": []}200 n_ok = n_fail = 0201202 for sid, desc in FRED.items():203 url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={sid}&cosd=1900-01-01"204 rec = fetch(url, RAW / "fred" / f"{sid}.csv", expect_csv=True)205 rec.update(id=f"FRED:{sid}", description=desc, source="FRED (Federal Reserve Bank of St. Louis), https://fred.stlouisfed.org/series/" + sid)206 if rec["status"] == "ok":207 txt = (RAW / "fred" / f"{sid}.csv").read_text(errors="replace").strip().splitlines()208 rec["n_rows"] = len(txt) - 1209 rec["first_obs"] = txt[1].split(",")[0] if len(txt) > 1 else None210 rec["last_obs"] = txt[-1].split(",")[0] if len(txt) > 1 else None211 if sid.startswith("BAML"):212 rec["notes"] = "FRED notes: 'Starting in April 2026, this series will only include 3 years of observations.' 2008/2020 peaks NOT obtainable here."213 if sid == "SP500":214 rec["notes"] = "FRED/S&P licence: only the most recent 10 years are downloadable."215 n_ok += 1216 else:217 n_fail += 1218 manifest["entries"].append(rec)219 print(f"[{rec['status']:6s}] {rec['id']}: {rec.get('bytes', 0)} bytes {rec.get('error', '')}")220221 for item in OTHER:222 rec = fetch(item["url"], RAW / item["file"])223 rec.update(id=item["id"], notes=item["notes"])224 if rec["status"] == "ok":225 n_ok += 1226 if item.get("unzip_to"):227 try:228 rec["extracted"] = safe_unzip(RAW / item["file"], RAW / item["unzip_to"])229 except zipfile.BadZipFile as e:230 rec["extract_error"] = str(e)231 else:232 n_fail += 1233 manifest["entries"].append(rec)234 print(f"[{rec['status']:6s}] {rec['id']}: {rec.get('bytes', 0)} bytes {rec.get('error', '')}")235236 # Things that are documented but not downloadable programmatically237 manifest["not_downloaded"] = [238 {"id": "WID_USA_wealth_shares", "url": "https://wid.world/country/usa/", "reason": "page renders charts client-side; no CSV fetched. DFA (Fed) used instead for top shares."},239 {"id": "SCF_2022_bulletin", "url": "https://www.federalreserve.gov/publications/october-2023-changes-in-us-family-finances-from-2019-to-2022.htm", "reason": "HTML report read on 2026-10-27 (median/mean net worth recorded in ledger); no machine-readable series downloaded."},240 {"id": "Census_BTOS_AI_use", "url": "https://www.census.gov/hfp/btos/data", "reason": "data page is JavaScript-driven; values taken from the Census story https://census.gov/library/stories/2026/05/ai-use-businesses.html (read 2026-10-27)."},241 {"id": "FDIC_QBP_2Q2026_capital_ratio", "url": "https://www.fdic.gov/quarterly-banking-profile/quarterly-banking-profile-2q-2026", "reason": "equity/assets ratio not on the HTML summary page; bank equity ratio computed from FRED H.8 RALACBW027SBOG/TLAACBW027SBOG instead."},242 {"id": "ICE_BofA_OAS_history", "url": "https://fred.stlouisfed.org/series/BAMLH0A0HYM2", "reason": "FRED limited to 3-year window since April 2026; full history requires ICE Data Indices licence."},243 {"id": "WILL5000INDFC", "url": "https://fred.stlouisfed.org/series/WILL5000INDFC", "reason": "HTTP 404 on 2026-10-27 (Wilshire indices no longer distributed by FRED). SP500 (10-year window) and Shiller data used instead."},244 {"id": "Cooley_Prescott_1995_chapter", "url": "https://press.princeton.edu/books/paperback/9780691043234/frontiers-of-business-cycle-research", "reason": "book chapter not available online; publisher page returned HTTP 403 to scripted access on 2026-10-27."},245 ]246 manifest["summary"] = {"ok": n_ok, "failed": n_fail}247 MANIFEST.write_text(json.dumps(manifest, indent=2))248 print(f"\nManifest written: {MANIFEST} ({n_ok} ok, {n_fail} failed)")249 return 0250251252if __name__ == "__main__":253 sys.exit(main())254