Repository Public Research preview
1"""Run provenance: git state, environment, hardware, deterministic run IDs and manifests.23Secrets are never read or written here; environment capture is limited to library versions and4hardware descriptors. Personal paths are stored relative to the project root.5"""6from __future__ import annotations78import datetime as _dt9import hashlib10import json11import os12import platform13import subprocess14import sys15from pathlib import Path16from typing import Any1718PROJECT_ROOT = Path(__file__).resolve().parents[3]192021def git_state(root: Path = PROJECT_ROOT) -> dict[str, Any]:22 def _run(args: list[str]) -> str | None:23 try:24 return subprocess.check_output(["git", *args], cwd=root, stderr=subprocess.DEVNULL, text=True).strip()25 except Exception: # noqa: BLE001 - git may be absent or the directory may not be a repo26 return None2728 commit = _run(["rev-parse", "HEAD"])29 dirty = _run(["status", "--porcelain"])30 return {"commit": commit, "dirty": bool(dirty) if dirty is not None else None, "branch": _run(["rev-parse", "--abbrev-ref", "HEAD"])}313233def library_versions() -> dict[str, str | None]:34 out: dict[str, str | None] = {"python": sys.version.split()[0]}35 for name in ["numpy", "torch", "scipy", "pydantic", "mlx", "mlx_lm", "transformers", "safetensors"]:36 try:37 mod = __import__(name)38 out[name] = getattr(mod, "__version__", "?")39 except Exception: # noqa: BLE00140 out[name] = None41 return out424344def hardware() -> dict[str, Any]:45 info: dict[str, Any] = {46 "platform": platform.platform(),47 "machine": platform.machine(),48 "hostname_hash": hashlib.sha256(platform.node().encode()).hexdigest()[:12],49 "cpu_count": os.cpu_count(),50 }51 if sys.platform == "darwin":52 try:53 info["chip"] = subprocess.check_output(["sysctl", "-n", "machdep.cpu.brand_string"], text=True).strip()54 info["memory_gb"] = int(subprocess.check_output(["sysctl", "-n", "hw.memsize"], text=True)) / 2**3055 except Exception: # noqa: BLE00156 pass57 try:58 import torch5960 info["torch_mps"] = bool(torch.backends.mps.is_available())61 info["torch_cuda"] = bool(torch.cuda.is_available())62 except Exception: # noqa: BLE00163 pass64 return info656667def utc_now() -> str:68 return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")697071def sha256_file(path: Path) -> str:72 h = hashlib.sha256()73 with open(path, "rb") as f:74 for chunk in iter(lambda: f.read(1 << 20), b""):75 h.update(chunk)76 return h.hexdigest()777879def sha256_text(text: str) -> str:80 return hashlib.sha256(text.encode()).hexdigest()818283def make_run_id(experiment_id: str, cfg_hash: str, seed: int | None = None) -> str:84 tail = f"_s{seed}" if seed is not None else ""85 return f"{experiment_id}_{cfg_hash}{tail}"868788class Manifest:89 """Mutable run manifest persisted as JSON; status transitions are recorded with timestamps."""9091 def __init__(self, run_id: str, experiment_id: str, config: dict[str, Any], out_dir: Path) -> None:92 self.path = out_dir / f"{run_id}.json"93 self.data: dict[str, Any] = {94 "run_id": run_id,95 "experiment_id": experiment_id,96 "status": "planned",97 "created_utc": utc_now(),98 "git": git_state(),99 "libraries": library_versions(),100 "hardware": hardware(),101 "config": config,102 "data_hashes": {},103 "seeds": {},104 "outputs": {},105 "warnings": [],106 "history": [],107 "interpretation": "",108 }109 out_dir.mkdir(parents=True, exist_ok=True)110 self.save()111112 def set_status(self, status: str, note: str = "") -> None:113 assert status in {"planned", "running", "failed", "completed", "invalidated"}114 self.data["status"] = status115 self.data["history"].append({"utc": utc_now(), "status": status, "note": note})116 self.save()117118 def warn(self, message: str) -> None:119 self.data["warnings"].append({"utc": utc_now(), "message": message})120 self.save()121122 def save(self) -> None:123 self.path.write_text(json.dumps(self.data, indent=2, sort_keys=True, default=str))124