spb/airiskindex Public

1#  File:    download_oews.py2#  Path:    apps/etl/src/airiskindex_etl/download_oews.py3#  Project: AI Risk Index — airiskindex.io4#  Author:  Simon-Pierre Boucher5#  Contact: contact@spboucher.ai6#  Copyright © 2026 Simon-Pierre Boucher. All rights reserved.7#8#  Description: Fetch BLS OEWS national wage data into data/raw/oews/.910"""Fetch BLS OEWS national wage data (May 2025) into data/raw/oews/.1112BLS blocks non-browser user agents — a descriptive UA with contact info works.13Source: https://www.bls.gov/oes/tables.htm (docs/research/02-data-sources.md).14"""1516from __future__ import annotations1718import sys19import zipfile20from pathlib import Path2122import requests2324OEWS_URL = "https://www.bls.gov/oes/special-requests/oesm25nat.zip"25USER_AGENT = "Mozilla/5.0 (compatible; airiskindex-etl/0.1; +https://www.airiskindex.io)"2627REPO_ROOT = Path(__file__).resolve().parents[4]28RAW_DIR = REPO_ROOT / "data" / "raw" / "oews"293031def download() -> Path:32    RAW_DIR.mkdir(parents=True, exist_ok=True)33    archive = RAW_DIR / "oesm25nat.zip"34    if not archive.exists():35        print(f"downloading {OEWS_URL}")36        response = requests.get(OEWS_URL, headers={"User-Agent": USER_AGENT}, timeout=300)37        response.raise_for_status()38        archive.write_bytes(response.content)39    with zipfile.ZipFile(archive) as zf:40        zf.extractall(RAW_DIR)41    print(f"extracted to {RAW_DIR}")42    return archive434445if __name__ == "__main__":46    try:47        download()48    except requests.RequestException as error:49        print(f"download failed: {error}", file=sys.stderr)50        sys.exit(1)51