"""Download the three Stage A pilot months with checksums into data/raw/.""" from __future__ import annotations import hashlib import json import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from huggingface_hub import hf_hub_download from intraday_fragility import paths PILOT_MONTHS = ["1992-06", "2009-06", "2025-12"] def main() -> None: cfg = paths.base_config()["dataset"] records = [] for m in PILOT_MONTHS: y, mm = map(int, m.split("-")) t0 = time.time() p = hf_hub_download(cfg["id"], cfg["file_pattern"].format(year=y, month=mm), repo_type="dataset", revision=cfg["revision"], local_dir=paths.get("raw")) h = hashlib.sha256() with open(p, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) records.append({"month": m, "bytes": Path(p).stat().st_size, "sha256": h.hexdigest(), "secs": round(time.time() - t0, 1)}) print(records[-1]) out = paths.get("manifests") / "pilot_downloads.json" out.write_text(json.dumps({"dataset_id": cfg["id"], "revision": cfg["revision"], "files": records}, indent=1)) if __name__ == "__main__": main()