#  File:    download_onet.py
#  Path:    apps/etl/src/airiskindex_etl/download_onet.py
#  Project: AI Risk Index — airiskindex.io
#  Author:  Simon-Pierre Boucher
#  Contact: contact@spboucher.ai
#  Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
#
#  Description: Fetch the O*NET database text dump into data/raw/onet/.

"""Fetch the O*NET database text dump into data/raw/onet/ (immutable).

O*NET 30.x — CC BY 4.0, attribution required. 30.x renamed
"Technology Skills" -> "Software Skills" and split Skills into
Essential/Transferable (see docs/research/02-data-sources.md §O*NET).
Check https://www.onetcenter.org/database.html for the current release
(31.0 expected late Aug 2026) before bumping ONET_VERSION.
"""

from __future__ import annotations

import hashlib
import json
import sys
import zipfile
from pathlib import Path

import requests

ONET_VERSION = "30_3"
ONET_URL = f"https://www.onetcenter.org/dl_files/database/db_{ONET_VERSION}_text.zip"

# Some sources (notably BLS OEWS) block non-browser user agents; use a plain
# descriptive UA with contact info everywhere for consistency and courtesy.
USER_AGENT = "airiskindex-etl/0.1 (https://www.airiskindex.io; data pipeline)"

REPO_ROOT = Path(__file__).resolve().parents[4]
RAW_DIR = REPO_ROOT / "data" / "raw" / "onet"


def download() -> Path:
    RAW_DIR.mkdir(parents=True, exist_ok=True)
    archive = RAW_DIR / f"db_{ONET_VERSION}_text.zip"
    if archive.exists():
        print(f"already present: {archive}")
        return archive

    print(f"downloading {ONET_URL}")
    response = requests.get(ONET_URL, headers={"User-Agent": USER_AGENT}, timeout=300, stream=True)
    response.raise_for_status()
    with archive.open("wb") as fh:
        for chunk in response.iter_content(chunk_size=1 << 20):
            fh.write(chunk)

    with zipfile.ZipFile(archive) as zf:
        zf.extractall(RAW_DIR)

    manifest = {
        "source": ONET_URL,
        "version": ONET_VERSION,
        "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),
        "files": sorted(p.name for p in RAW_DIR.iterdir()),
    }
    (RAW_DIR / "manifest.json").write_text(json.dumps(manifest, indent=2))
    print(f"downloaded and extracted to {RAW_DIR}")
    return archive


if __name__ == "__main__":
    try:
        download()
    except requests.RequestException as error:
        print(f"download failed: {error}", file=sys.stderr)
        sys.exit(1)