1#!/bin/bash2# Pull raw records, logs and chain messages from the cluster nodes and merge the E1_core Llama shard into runs/raw/E1_core.jsonl3# (idempotent: shard records are appended only if their attempt_id is not already present).4set -e5cd "$(dirname "$0")/.."6mkdir -p runs/raw/remote runs/logs/remote runs/raw/chains7rsync -aq M3U96a:~/wp30-worker/wp30/runs/raw/E1_core.jsonl runs/raw/remote/E1_core_llama_m3u96a.jsonl8rsync -aq M3U96a:~/wp30-worker/wp30/runs/logs/ runs/logs/remote/m3u96a/9rsync -aq M3U96b:~/wp30-worker/wp30/runs/raw/ED_hosted_core.jsonl runs/raw/ 2>/dev/null || true10rsync -aq M3U96b:~/wp30-worker/wp30/runs/raw/E4_core.jsonl runs/raw/remote/E4_core_shard0_m3u96b.jsonl 2>/dev/null || true11rsync -aq M3U96b:~/wp30-worker/wp30/runs/raw/chains/ runs/raw/chains/ 2>/dev/null || true12rsync -aq M3U96b:~/wp30-worker/wp30/runs/logs/ runs/logs/remote/m3u96b/13python3 - <<'PY'14import json15seen = set()16for l in open("runs/raw/E1_core.jsonl"):17 seen.add(json.loads(l)["attempt_id"])18n = 019with open("runs/raw/E1_core.jsonl", "a") as out:20 for l in open("runs/raw/remote/E1_core_llama_m3u96a.jsonl"):21 r = json.loads(l)22 if r["document_id"].startswith("lb_"):23 continue # public families of the Llama pair were run on the laptop (D-019); the 25 remote public records are dropped24 if r["attempt_id"] not in seen:25 r["hardware_id"] = "m3u96a-m3ultra-96g"; out.write(json.dumps(r, ensure_ascii=False) + "\n"); seen.add(r["attempt_id"]); n += 126print("merged", n, "new Llama-shard records; total", len(seen))27# merge the E4_core chain shard from M3U96b (even documents) into the local file (odd documents)28try:29 seen4 = {json.loads(l)["attempt_id"] for l in open("runs/raw/E4_core.jsonl")} if __import__("os").path.exists("runs/raw/E4_core.jsonl") else set()30 n4 = 031 with open("runs/raw/E4_core.jsonl", "a") as out:32 for l in open("runs/raw/remote/E4_core_shard0_m3u96b.jsonl"):33 r = json.loads(l)34 if r["attempt_id"] not in seen4:35 r["hardware_id"] = "m3u96b-m3ultra-96g"; out.write(json.dumps(r, ensure_ascii=False) + "\n"); seen4.add(r["attempt_id"]); n4 += 136 print("merged", n4, "E4 shard0 records; total", len(seen4))37except FileNotFoundError:38 pass39# (E5/E3 run on the laptop since D-019; the E4 shard0 records are labelled at merge time above)40PY41