spb/airiskindex Public

1#  File:    download_onet.py2#  Path:    apps/etl/src/airiskindex_etl/download_onet.py3#  Project: AI Risk Index — airiskindex.io4#  Author:  Simon-Pierre Boucher5#  Contact: contact@spboucher.ai6#  Copyright © 2026 Simon-Pierre Boucher. All rights reserved.7#8#  Description: Fetch the O*NET database text dump into data/raw/onet/.910"""Fetch the O*NET database text dump into data/raw/onet/ (immutable).1112O*NET 30.x — CC BY 4.0, attribution required. 30.x renamed13"Technology Skills" -> "Software Skills" and split Skills into14Essential/Transferable (see docs/research/02-data-sources.md §O*NET).15Check https://www.onetcenter.org/database.html for the current release16(31.0 expected late Aug 2026) before bumping ONET_VERSION.17"""1819from __future__ import annotations2021import hashlib22import json23import sys24import zipfile25from pathlib import Path2627import requests2829ONET_VERSION = "30_3"30ONET_URL = f"https://www.onetcenter.org/dl_files/database/db_{ONET_VERSION}_text.zip"3132# Some sources (notably BLS OEWS) block non-browser user agents; use a plain33# descriptive UA with contact info everywhere for consistency and courtesy.34USER_AGENT = "airiskindex-etl/0.1 (https://www.airiskindex.io; data pipeline)"3536REPO_ROOT = Path(__file__).resolve().parents[4]37RAW_DIR = REPO_ROOT / "data" / "raw" / "onet"383940def download() -> Path:41    RAW_DIR.mkdir(parents=True, exist_ok=True)42    archive = RAW_DIR / f"db_{ONET_VERSION}_text.zip"43    if archive.exists():44        print(f"already present: {archive}")45        return archive4647    print(f"downloading {ONET_URL}")48    response = requests.get(ONET_URL, headers={"User-Agent": USER_AGENT}, timeout=300, stream=True)49    response.raise_for_status()50    with archive.open("wb") as fh:51        for chunk in response.iter_content(chunk_size=1 << 20):52            fh.write(chunk)5354    with zipfile.ZipFile(archive) as zf:55        zf.extractall(RAW_DIR)5657    manifest = {58        "source": ONET_URL,59        "version": ONET_VERSION,60        "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),61        "files": sorted(p.name for p in RAW_DIR.iterdir()),62    }63    (RAW_DIR / "manifest.json").write_text(json.dumps(manifest, indent=2))64    print(f"downloaded and extracted to {RAW_DIR}")65    return archive666768if __name__ == "__main__":69    try:70        download()71    except requests.RequestException as error:72        print(f"download failed: {error}", file=sys.stderr)73        sys.exit(1)74