spb/wp15_uqo

1"""Download the three Stage A pilot months with checksums into data/raw/."""23from __future__ import annotations45import hashlib6import json7import sys8import time9from pathlib import Path1011sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))1213from huggingface_hub import hf_hub_download1415from intraday_fragility import paths1617PILOT_MONTHS = ["1992-06", "2009-06", "2025-12"]181920def main() -> None:21    cfg = paths.base_config()["dataset"]22    records = []23    for m in PILOT_MONTHS:24        y, mm = map(int, m.split("-"))25        t0 = time.time()26        p = hf_hub_download(cfg["id"], cfg["file_pattern"].format(year=y, month=mm),27                            repo_type="dataset", revision=cfg["revision"],28                            local_dir=paths.get("raw"))29        h = hashlib.sha256()30        with open(p, "rb") as f:31            for chunk in iter(lambda: f.read(1 << 20), b""):32                h.update(chunk)33        records.append({"month": m, "bytes": Path(p).stat().st_size,34                        "sha256": h.hexdigest(), "secs": round(time.time() - t0, 1)})35        print(records[-1])36    out = paths.get("manifests") / "pilot_downloads.json"37    out.write_text(json.dumps({"dataset_id": cfg["id"], "revision": cfg["revision"],38                               "files": records}, indent=1))394041if __name__ == "__main__":42    main()43