"""Shared helpers: settings, paths, logging, deterministic seeds.""" from __future__ import annotations import hashlib, json, logging, os, random, sys, time from pathlib import Path import numpy as np import yaml ROOT = Path(__file__).resolve().parents[1] SETTINGS = yaml.safe_load((ROOT / "config" / "settings.yaml").read_text()) SEED = int(SETTINGS["seed"]) TZ = SETTINGS.get("timezone_display", "America/Toronto") def path(key: str) -> Path: p = ROOT / SETTINGS["paths"][key] p.mkdir(parents=True, exist_ok=True) return p def snapshot_db() -> Path: return ROOT / SETTINGS["snapshot"]["path"] def verify_snapshot() -> str: h = hashlib.sha256() with open(snapshot_db(), "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) digest = h.hexdigest() if digest != SETTINGS["snapshot"]["sha256"]: raise SystemExit(f"snapshot sha256 mismatch: {digest}") return digest def seed_everything(seed: int = SEED) -> None: random.seed(seed); np.random.seed(seed) os.environ["PYTHONHASHSEED"] = str(seed) def get_logger(name: str) -> logging.Logger: log = logging.getLogger(name) if not log.handlers: fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%Y-%m-%dT%H:%M:%S") sh = logging.StreamHandler(sys.stdout); sh.setFormatter(fmt); log.addHandler(sh) fh = logging.FileHandler(path("logs") / "pipeline.log"); fh.setFormatter(fmt); log.addHandler(fh) log.setLevel(logging.INFO) return log def ledger(phase: str, action: str, result: str) -> None: """Append a machine-readable line to the structured run ledger.""" rec = {"utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "phase": phase, "action": action, "result": result} with open(path("logs") / "run_ledger.jsonl", "a") as f: f.write(json.dumps(rec, ensure_ascii=False) + "\n") def write_manifest(name: str, files: list[Path], extra: dict | None = None) -> None: man = {"name": name, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "seed": SEED, "snapshot": SETTINGS["snapshot"]["id"], "files": {}} for f in files: f = Path(f) if f.exists(): man["files"][str(f.relative_to(ROOT))] = {"bytes": f.stat().st_size, "sha256": hashlib.sha256(f.read_bytes()).hexdigest() if f.stat().st_size < 400_000_000 else None} if extra: man.update(extra) (path("logs") / f"manifest_{name}.json").write_text(json.dumps(man, indent=2, ensure_ascii=False))