spb/anomaly-atlas Public License

1#!/usr/bin/env python32# =============================================================================3#  Project   : anomaly-atlas4#  File      : tools/new_experiment.py5#  Purpose   : Scaffold a header-compliant experiment directory (CLAUDE.md §3, §10)6#  Author    : Simon-Pierre Boucher7#  Contact   : contact@spboucher.ai8#  Data src  : hfmarketdata.io (sole data source)9#  Created   : 2026-08-1210#  Modified  : 2026-08-1211#  Platform  : macOS / Apple Silicon (arm64)12#  License   : All rights reserved (research code)13# =============================================================================14"""Scaffold a new experiment directory with the mandatory structure.1516Creates: README.md, hypothesis.md (seven-field scientific block, incl. the17artifact null), benchmark.py, implementation/, results/, analysis.md — all18with conforming author headers.1920Usage:21    python3 tools/new_experiment.py experiments/micro/expX_name "One-line purpose"22    python3 tools/new_experiment.py experiments/candidate_04 "Candidate: ..."23"""2425from __future__ import annotations2627import sys28from datetime import date29from pathlib import Path3031REPO_ROOT = Path(__file__).resolve().parent.parent32TODAY = date.today().isoformat()3334PY_HEADER = """\35# =============================================================================36#  Project   : anomaly-atlas37#  File      : {rel}38#  Purpose   : {purpose}39#  Author    : Simon-Pierre Boucher40#  Contact   : contact@spboucher.ai41#  Data src  : hfmarketdata.io (sole data source)42#  Created   : {today}43#  Modified  : {today}44#  Platform  : macOS / Apple Silicon (arm64)45#  License   : All rights reserved (research code)46# =============================================================================47"""4849MD_HEADER = """\50---51project: anomaly-atlas52document: {doc}53author: Simon-Pierre Boucher54contact: contact@spboucher.ai55data_source: hfmarketdata.io56created: {today}57status: draft58---59"""6061HYPOTHESIS_BODY = """62# Hypothesis — {name}6364```text65Hypothesis66  <what we believe and why — pre-specified BEFORE looking at results>6768Falsification criterion69  <the concrete measurable outcome that would prove this wrong>7071Artifact null(s)72  <the fake-signal baseline(s) this must beat: bounce / staleness /73   non-synchronous timestamps / permuted calendar / random walk>7475Method76  <exact procedure, universe, split (train/validation/holdout), seeds,77   number of hypotheses tested, correction applied>7879Result80  <filled after the run: effect size, bootstrap CIs, corrected p-values,81   OOS status, cost-adjusted effect, credits used>8283Interpretation84  <what the numbers mean, WITH confidence level (0-3); alternative85   explanations considered — artifact first>8687Next experiment88  <the most informative follow-up given this result>89```90"""9192BENCHMARK_BODY = '''93"""Benchmark entry point for {name}.9495Must embed the hardware manifest in all result output96(see benchmarks/hardware_manifest.py) and write results to97results/{name}/<timestamp>/. Uses hfmarketdata.io data ONLY, exclusively98through src/anomaly_atlas/data/hf_client.py.99"""100101import sys102from pathlib import Path103104sys.path.insert(0, str(Path(__file__).resolve().parents[{depth}] / "benchmarks"))105from hardware_manifest import collect_manifest  # noqa: E402106107108def main() -> None:109    collect_manifest()  # embedded in results once implemented110    raise NotImplementedError("experiment not yet implemented")111112113if __name__ == "__main__":114    main()115'''116117118def scaffold(exp_dir: Path, purpose: str) -> None:119    if exp_dir.exists() and any(exp_dir.iterdir()):120        sys.exit(f"error: {exp_dir} already exists and is not empty")121    name = exp_dir.name122    rel = exp_dir.relative_to(REPO_ROOT)123    (exp_dir / "implementation").mkdir(parents=True, exist_ok=True)124    (exp_dir / "results").mkdir(exist_ok=True)125126    def md(doc: str) -> str:127        return MD_HEADER.format(doc=doc, today=TODAY)128129    (exp_dir / "README.md").write_text(130        md(f"{name}/README")131        + f"\n# {name}\n\n{purpose}\n\nStatus: scaffolded {TODAY}, not yet run.\n"132    )133    (exp_dir / "hypothesis.md").write_text(134        md(f"{name}/hypothesis") + HYPOTHESIS_BODY.format(name=name)135    )136    (exp_dir / "analysis.md").write_text(137        md(f"{name}/analysis") + f"\n# Analysis — {name}\n\n*To be written after results exist. "138        "Must include the seven-field block and the evidence standard of CLAUDE.md §10 "139        "(never report an in-sample number as a finding).*\n"140    )141    depth = len(rel.parts)  # parents[] index up to repo root142    (exp_dir / "benchmark.py").write_text(143        PY_HEADER.format(144            rel=rel / "benchmark.py",145            purpose=f"Benchmark runner: {purpose}"[:82],146            today=TODAY,147        )148        + BENCHMARK_BODY.format(name=name, depth=depth)149    )150    print(f"scaffolded {rel} ({purpose})")151152153def main() -> None:154    if len(sys.argv) < 3:155        sys.exit(__doc__)156    exp_dir = (REPO_ROOT / sys.argv[1]).resolve()157    if REPO_ROOT not in exp_dir.parents:158        sys.exit("error: experiment directory must live inside the repository")159    scaffold(exp_dir, sys.argv[2])160161162if __name__ == "__main__":163    main()164