spb/firc0001_uqo

1#!/bin/bash2# N13 - re-run the (L1-modified) code/export_stats.py on the master node into a scratch directory, read-only,3# and diff the 29 baseline CSV exports against the regenerated ones (byte identity gate), then the new files.4# usage (on M3U96b): bash scripts/a13_export_stats_check.sh   (expects scripts/export_stats.py = worktree code/export_stats.py,5#        and baseline/ = the 29 CSV + stats.json of commit 678d233)6set -u7cd "$HOME/firc0001_review" || exit 18OUT="results/export_check"; mkdir -p "$OUT/data" logs9PY="$HOME/UQO_RESEARCH_DB/.venv/bin/python"10echo "START $(date -Iseconds)"; T0=$(date +%s)11nice -n 10 env OMP_NUM_THREADS=4 "$PY" -I scripts/export_stats.py "$HOME/UQO_RESEARCH_DB" "$OUT/data" > logs/a13_export_stats.log 2>&112RC=$?13echo "export_stats.py rc=$RC in $(( $(date +%s) - T0 )) s"14{15echo "# N13 export reconciliation — $(date -Iseconds) on $(hostname), rc=$RC, $(( $(date +%s) - T0 )) s"16echo17echo "## Byte identity of the 29 baseline CSV (commit 678d233) vs the regenerated exports"18echo '```'19IDENT=0; DIFF=0; MISSING=020for f in baseline/*.csv; do21  b=$(basename "$f")22  if [ ! -f "$OUT/data/$b" ]; then echo "MISSING   $b"; MISSING=$((MISSING+1)); continue; fi23  if cmp -s "$f" "$OUT/data/$b"; then echo "identical $b"; IDENT=$((IDENT+1)); else echo "DIFFERS   $b"; DIFF=$((DIFF+1)); fi24done25echo "identical=$IDENT differs=$DIFF missing=$MISSING"26echo '```'27if [ "$DIFF" -gt 0 ]; then28  echo; echo "## Differences"; echo '```'29  for f in baseline/*.csv; do b=$(basename "$f"); [ -f "$OUT/data/$b" ] && ! cmp -s "$f" "$OUT/data/$b" && { echo "--- $b"; diff "$f" "$OUT/data/$b" | head -20; }; done30  echo '```'31fi32echo; echo "## New files"; echo '```'33for f in "$OUT"/data/*; do b=$(basename "$f"); [ -f "baseline/$b" ] || echo "new: $b ($(wc -c < "$f") bytes)"; done34echo '```'35echo; echo "## stats.json: baseline keys vs regenerated keys"; echo '```'36"$PY" -I - "$OUT/data/stats.json" baseline/stats.json <<'PYEOF'37import json, sys38new = json.load(open(sys.argv[1])); old = json.load(open(sys.argv[2]))39print("baseline keys:", len(old), "| regenerated keys:", len(new))40print("added keys:", sorted(set(new) - set(old)))41print("removed keys:", sorted(set(old) - set(new)))42for k in sorted(set(old) & set(new)):43    if old[k] != new[k]: print("CHANGED", k, ":", json.dumps(old[k], default=str)[:200], "->", json.dumps(new[k], default=str)[:200])44PYEOF45echo '```'46echo; echo "## Tail of the run log"; echo '```'; tail -15 logs/a13_export_stats.log; echo '```'47} > "$OUT/export_reconciliation.md"48cat "$OUT/export_reconciliation.md"49echo "END $(date -Iseconds)"50