"""Shared helpers: project root, settings, logging, run ledger, output manifest.""" from __future__ import annotations import hashlib, json, logging, os, sqlite3, sys, time, pathlib import yaml ROOT = pathlib.Path(__file__).resolve().parents[1] SETTINGS = yaml.safe_load((ROOT / "config" / "settings.yaml").read_text()) SEED = int(SETTINGS["seed"]) REGIONS = list(SETTINGS["regions"]) LABEL = os.environ.get("WP38_LABEL", "rule") # "rule" (platform keyword labels) or "model" (A-002 classifier labels) SUFFIX = "" if LABEL == "rule" else f"_{LABEL}" P = {k: ROOT / v for k, v in SETTINGS["paths"].items()} P["processed"] = ROOT / f"data/processed{SUFFIX}"; P["experiments"] = ROOT / f"experiments{SUFFIX}" P["figures"] = ROOT / f"paper/figures{SUFFIX}"; P["tables"] = ROOT / f"paper/tables{SUFFIX}" for _p in P.values(): _p.mkdir(parents=True, exist_ok=True) PROC_DIR, EXP_DIR = P["processed"], P["experiments"] BASE_PROC, BASE_EXP = ROOT / "data/processed", ROOT / "experiments" # label-independent artefacts (cleaning, audits, matching) BASE_PROC.mkdir(parents=True, exist_ok=True); BASE_EXP.mkdir(parents=True, exist_ok=True) def get_logger(name: str) -> logging.Logger: log = logging.getLogger(name) if not log.handlers: log.setLevel(logging.INFO) fmt = logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s", "%H:%M:%S") h = logging.StreamHandler(sys.stdout); h.setFormatter(fmt); log.addHandler(h) fh = logging.FileHandler(P["logs"] / "pipeline.log"); fh.setFormatter(fmt); log.addHandler(fh) return log def snapshot_conn() -> sqlite3.Connection: path = ROOT / SETTINGS["snapshot"]["path"] return sqlite3.connect(f"file:{path}?mode=ro", uri=True) def sha256_file(path: pathlib.Path) -> str: h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 22), b""): h.update(chunk) return h.hexdigest() def ledger(step: str, **kw) -> None: """Append one JSON line to logs/run_ledger.jsonl (what ran, when, with which inputs/outputs).""" rec = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "step": step, **kw} with open(P["logs"] / "run_ledger.jsonl", "a") as f: f.write(json.dumps(rec, ensure_ascii=False, default=str) + "\n") def register_output(path: pathlib.Path, step: str) -> None: """Record sha256 of a produced file in logs/manifest.json.""" mf = P["logs"] / "manifest.json" d = json.loads(mf.read_text()) if mf.exists() else {} d[str(path.relative_to(ROOT))] = {"sha256": sha256_file(path), "bytes": path.stat().st_size, "step": step, "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} mf.write_text(json.dumps(d, indent=1, sort_keys=True)) def write_json(obj, path: pathlib.Path, step: str = "") -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(obj, indent=1, ensure_ascii=False, default=str)) if step: register_output(path, step)