1#!/bin/bash2# Pull raw records, logs and chain messages from the cluster nodes and merge the E1_core Llama shard into runs/raw/E1_core.jsonl3# (idempotent: shard records are appended only if their attempt_id is not already present).4set -e5cd "$(dirname "$0")/.."6mkdir -p runs/raw/remote runs/logs/remote runs/raw/chains7rsync -aq M3U96a:~/wp30-worker/wp30/runs/raw/E1_core.jsonl runs/raw/remote/E1_core_llama_m3u96a.jsonl8rsync -aq M3U96a:~/wp30-worker/wp30/runs/logs/ runs/logs/remote/m3u96a/9for f in ED_hosted_core E4_core E5_core E3_core E3_core_sender; do rsync -aq M3U96b:~/wp30-worker/wp30/runs/raw/$f.jsonl runs/raw/ 2>/dev/null || true; done10rsync -aq M3U96b:~/wp30-worker/wp30/runs/raw/chains/ runs/raw/chains/ 2>/dev/null || true11rsync -aq M3U96b:~/wp30-worker/wp30/runs/logs/ runs/logs/remote/m3u96b/12python3 - <<'PY'13import json14seen = set()15for l in open("runs/raw/E1_core.jsonl"):16 seen.add(json.loads(l)["attempt_id"])17n = 018with open("runs/raw/E1_core.jsonl", "a") as out:19 for l in open("runs/raw/remote/E1_core_llama_m3u96a.jsonl"):20 r = json.loads(l)21 if r["attempt_id"] not in seen:22 r["hardware_id"] = "m3u96a-m3ultra-96g"; out.write(json.dumps(r, ensure_ascii=False) + "\n"); seen.add(r["attempt_id"]); n += 123print("merged", n, "new Llama-shard records; total", len(seen))24# relabel hardware for records produced on M3U96b by code that still carried the laptop constant25for f in ["E4_core", "E5_core", "E3_core", "E3_core_sender"]:26 p = f"runs/raw/{f}.jsonl"27 try:28 rows = [json.loads(l) for l in open(p)]29 except FileNotFoundError:30 continue31 ch = 032 for r in rows:33 if r.get("hardware_id") == "laptop-m5max-48g":34 r["hardware_id"] = "m3u96b-m3ultra-96g"; ch += 135 with open(p, "w") as out:36 for r in rows: out.write(json.dumps(r, ensure_ascii=False) + "\n")37 print(f, "relabelled", ch)38PY39