"""Dataset manifests, hashes and code versions (no raw data, no secrets).""" from __future__ import annotations import hashlib import json import os import subprocess import time from pathlib import Path from typing import Any, Iterable def sha256_file(path: str | os.PathLike, chunk: int = 1 << 20) -> str: h = hashlib.sha256() with open(path, "rb") as fh: for block in iter(lambda: fh.read(chunk), b""): h.update(block) return h.hexdigest() def sha256_text(text: str) -> str: return hashlib.sha256(text.encode()).hexdigest() def git_sha(repo: str | os.PathLike | None = None) -> str: try: return subprocess.check_output(["git", "rev-parse", "--short=12", "HEAD"], cwd=repo, text=True, stderr=subprocess.DEVNULL).strip() except Exception: return "nogit" def now_iso() -> str: return time.strftime("%Y-%m-%dT%H:%M:%S%z") def append_jsonl(path: str | os.PathLike, record: dict[str, Any]) -> None: p = Path(path) p.parent.mkdir(parents=True, exist_ok=True) with open(p, "a") as fh: fh.write(json.dumps(record, default=str, sort_keys=True) + "\n") def read_jsonl(path: str | os.PathLike) -> list[dict[str, Any]]: p = Path(path) if not p.exists(): return [] with open(p) as fh: return [json.loads(line) for line in fh if line.strip()] def manifest_index(path: str | os.PathLike, keys: Iterable[str]) -> dict[tuple, dict[str, Any]]: keys = tuple(keys) return {tuple(r.get(k) for k in keys): r for r in read_jsonl(path)} def write_json(path: str | os.PathLike, obj: Any) -> None: p = Path(path) p.parent.mkdir(parents=True, exist_ok=True) with open(p, "w") as fh: json.dump(obj, fh, indent=1, default=str, sort_keys=True)