1"""Download every source listed in configs/sources.yaml into data/raw/<key>/ and record2SHA-256, size, HTTP status, retrieval timestamp in data/metadata/source_registry.json.34Usage: python -m wp35.acquisition.fetch [--only KEY ...] [--force]5Raw files are immutable: a re-run verifies the hash of an existing file and never overwrites6it unless --force is given.7"""8from __future__ import annotations9import argparse, hashlib, json, sys, time10from datetime import datetime, timezone11from pathlib import Path12import requests, yaml1314ROOT = Path(__file__).resolve().parents[3]15RAW = ROOT / "data" / "raw"16META = ROOT / "data" / "metadata"17REGISTRY = META / "source_registry.json"18UA = "WP35-UQO-research-acquisition/0.1 (simon-pierre.boucher@uqo.ca)"192021def sha256(path: Path) -> str:22 h = hashlib.sha256()23 with open(path, "rb") as f:24 for chunk in iter(lambda: f.read(1 << 20), b""):25 h.update(chunk)26 return h.hexdigest()272829def load_registry() -> dict:30 return json.loads(REGISTRY.read_text()) if REGISTRY.exists() else {}313233def fetch(key: str, spec: dict, force: bool, registry: dict) -> dict:34 dest_dir = RAW / key35 dest_dir.mkdir(parents=True, exist_ok=True)36 dest = dest_dir / spec["filename"]37 rec = dict(spec)38 rec["key"] = key39 rec["local_path"] = str(dest.relative_to(ROOT))40 if dest.exists() and not force:41 rec["status"] = "already_present"42 rec["sha256"] = sha256(dest)43 rec["size_bytes"] = dest.stat().st_size44 prev = registry.get(key, {})45 rec["retrieved_at_utc"] = prev.get("retrieved_at_utc")46 rec["http_status"] = prev.get("http_status")47 if prev.get("sha256") and prev["sha256"] != rec["sha256"]:48 rec["status"] = "HASH_MISMATCH_WITH_REGISTRY"49 return rec50 t0 = time.time()51 r = requests.get(spec["url"], headers={"User-Agent": UA, "Accept": "*/*"},52 timeout=300, stream=True, allow_redirects=True)53 rec["http_status"] = r.status_code54 rec["final_url"] = r.url55 rec["content_type"] = r.headers.get("Content-Type")56 if r.status_code != 200:57 rec["status"] = f"download_failed_http_{r.status_code}"58 return rec59 tmp = dest.with_suffix(dest.suffix + ".part")60 with open(tmp, "wb") as f:61 for chunk in r.iter_content(1 << 20):62 f.write(chunk)63 tmp.rename(dest)64 rec["retrieved_at_utc"] = datetime.now(timezone.utc).isoformat(timespec="seconds")65 rec["elapsed_s"] = round(time.time() - t0, 2)66 rec["size_bytes"] = dest.stat().st_size67 rec["sha256"] = sha256(dest)68 rec["status"] = "downloaded"69 return rec707172def main(argv=None):73 ap = argparse.ArgumentParser()74 ap.add_argument("--only", nargs="*")75 ap.add_argument("--force", action="store_true")76 a = ap.parse_args(argv)77 specs = yaml.safe_load((ROOT / "configs" / "sources.yaml").read_text())["sources"]78 META.mkdir(parents=True, exist_ok=True)79 registry = load_registry()80 keys = a.only or list(specs)81 for k in keys:82 rec = fetch(k, specs[k], a.force, registry)83 registry[k] = rec84 print(f"{k:40s} {rec['status']:22s} {rec.get('size_bytes','?'):>10} B sha256={str(rec.get('sha256',''))[:16]}…")85 REGISTRY.write_text(json.dumps(registry, indent=2, ensure_ascii=False))86 bad = [k for k in keys if not registry[k]["status"] in ("downloaded", "already_present")]87 if bad:88 print("FAILED:", bad, file=sys.stderr)89 return 190 return 0919293if __name__ == "__main__":94 sys.exit(main())95