WP44: multi-node embedding orchestration from the laptop (LAN IPs)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
7 changed files +15 −14
modified
embeddings/collect_multinode.sh
+7 −3
@@ -1,4 +1,8 @@ | |||||
| 1 | #!/bin/sh | 1 | #!/bin/sh | ||
| 2 | −cd ~/wp44_uqo; MODEL=${1:-allenai/specter2_base}; TAG=$(echo $MODEL | sed 's#/#__#g') | 2 | +# Run from the laptop: pull finished shards from the three auxiliary nodes to M3U96b (via LAN IPs, agent forwarding). | ||
| 3 | −for node in M3U96a M4M64a M4M64c; do rsync -aq $node:~/wp44_uqo/data/processed/embeddings/$TAG/ data/processed/embeddings/$TAG/ && echo "$node collected"; done | 3 | +MODEL=${1:-allenai/specter2_base}; TAG=$(echo $MODEL | sed 's#/#__#g') | ||
| 4 | −ls data/processed/embeddings/$TAG/shard_*.npy | wc -l | 4 | +for spec in "M3U96a:192.168.2.80" "M4M64a:192.168.2.70" "M4M64c:192.168.2.103"; do | ||
| 5 | + node=${spec%%:*}; ip=${spec#*:} | ||||
| 6 | + ssh -A M3U96b "mkdir -p ~/wp44_uqo/data/processed/embeddings/$TAG; rsync -aq -e 'ssh -o StrictHostKeyChecking=no' $ip:~/wp44_uqo/data/processed/embeddings/$TAG/ ~/wp44_uqo/data/processed/embeddings/$TAG/" && echo "$node collected" | ||||
| 7 | +done | ||||
| 8 | +ssh M3U96b "ls ~/wp44_uqo/data/processed/embeddings/$TAG/shard_*.npy | grep -v ids | wc -l" | ||||
modified
embeddings/run_multinode.sh
+7 −10
@@ -1,17 +1,14 @@ | |||||
| 1 | #!/bin/sh | 1 | #!/bin/sh | ||
| 2 | −# Distribute embedding shards over the nodes (run from M3U96b after `embed_corpus.py --prepare`). | 2 | +# Distribute embedding shards over four nodes. Run from the LAPTOP (which can reach every node by alias); transfers go | ||
| 3 | −# Each node gets a contiguous shard range proportional to its measured throughput; results are rsynced back to M3U96b. | 3 | +# node-to-node over the LAN IPs with agent forwarding during this session; each node then runs its range under nohup. | ||
| 4 | # Usage: sh embeddings/run_multinode.sh allenai/specter2_base | 4 | # Usage: sh embeddings/run_multinode.sh allenai/specter2_base | ||
| 5 | −cd ~/wp44_uqo; MODEL=${1:-allenai/specter2_base}; TAG=$(echo $MODEL | sed 's#/#__#g') | 5 | +MODEL=${1:-allenai/specter2_base}; TAG=$(echo $MODEL | sed 's#/#__#g'); SRC_IP=192.168.2.85 | ||
| 6 | −NS=$(ls -d data/processed/embeddings/input/shard=* | wc -l | tr -d ' ') | 6 | +NS=$(ssh M3U96b 'ls -d ~/wp44_uqo/data/processed/embeddings/input/shard=* | wc -l' | tr -d ' ') | ||
| 7 | −# weights: M3U96b 3, M3U96a 3, M4M64a 2, M4M64c 2 (M3 Ultra ≈ 300 texts/s, M4 Max ≈ 200 texts/s measured/estimated) | 7 | +A=$((NS*3/10)); B=$((NS*3/10)); C=$((NS*2/10)) | ||
| 8 | −A=$((NS*3/10)); B=$((NS*3/10)); C=$((NS*2/10)); D=$((NS - A - B - C)) | ||||
| 9 | echo "shards: $NS → M3U96b 0-$A, M3U96a $A-$((A+B)), M4M64a $((A+B))-$((A+B+C)), M4M64c $((A+B+C))-$NS" | 8 | echo "shards: $NS → M3U96b 0-$A, M3U96a $A-$((A+B)), M4M64a $((A+B))-$((A+B+C)), M4M64c $((A+B+C))-$NS" | ||
| 10 | for spec in "M3U96a:$A:$((A+B))" "M4M64a:$((A+B)):$((A+B+C))" "M4M64c:$((A+B+C)):$NS"; do | 9 | for spec in "M3U96a:$A:$((A+B))" "M4M64a:$((A+B)):$((A+B+C))" "M4M64c:$((A+B+C)):$NS"; do | ||
| 11 | node=${spec%%:*}; rest=${spec#*:}; from=${rest%%:*}; to=${rest#*:} | 10 | node=${spec%%:*}; rest=${spec#*:}; from=${rest%%:*}; to=${rest#*:} | ||
| 12 | − ssh $node "mkdir -p ~/wp44_uqo/data/processed/embeddings/input" | 11 | + ssh -A $node "mkdir -p ~/wp44_uqo/data/processed/embeddings/input ~/wp44_uqo/logs; k=$from; while [ \$k -lt $to ]; do rsync -aq -e 'ssh -o StrictHostKeyChecking=no' $SRC_IP:~/wp44_uqo/data/processed/embeddings/input/shard=\$k ~/wp44_uqo/data/processed/embeddings/input/; k=\$((k+1)); done; ls ~/wp44_uqo/data/processed/embeddings/input | wc -l" | ||
| 13 | − k=$from; while [ $k -lt $to ]; do rsync -aq data/processed/embeddings/input/shard=$k $node:~/wp44_uqo/data/processed/embeddings/input/ ; k=$((k+1)); done | ||||
| 14 | ssh $node "cd ~/wp44_uqo; . .venv/bin/activate; nohup python embeddings/embed_corpus.py --root ~/wp44_uqo --run --model $MODEL --shard-from $from --shard-to $to > logs/embed_$TAG.log 2>&1 &" && echo "$node started $from-$to" | 12 | ssh $node "cd ~/wp44_uqo; . .venv/bin/activate; nohup python embeddings/embed_corpus.py --root ~/wp44_uqo --run --model $MODEL --shard-from $from --shard-to $to > logs/embed_$TAG.log 2>&1 &" && echo "$node started $from-$to" | ||
| 15 | done | 13 | done | ||
| 16 | −. .venv/bin/activate; nohup python embeddings/embed_corpus.py --root ~/wp44_uqo --run --model $MODEL --shard-from 0 --shard-to $A > logs/embed_$TAG.log 2>&1 & | 14 | +ssh M3U96b "cd ~/wp44_uqo; . .venv/bin/activate; nohup python embeddings/embed_corpus.py --root ~/wp44_uqo --run --model $MODEL --shard-from 0 --shard-to $A > logs/embed_$TAG.log 2>&1 &" && echo "M3U96b started 0-$A" | ||
| 17 | −echo "M3U96b started 0-$A; collect with: sh embeddings/collect_multinode.sh $MODEL" | ||||
modified
figures/make_figures.py
+1 −1
@@ -190,7 +190,7 @@ def fig_its_total(R, out): | |||||
| 190 | its = res["its_chatgpt"] | 190 | its = res["its_chatgpt"] | ||
| 191 | ax.text(0.02, 0.97, f"level change {its['level_change']:+.3f} (p = {its['level_p']:.2f}; placebo p = {res['placebo_chatgpt']['level_t_placebo_p']:.2f})\nslope change {its['slope_change']:+.4f}/month (p = {its['slope_p']:.3f}; placebo p = {res['placebo_chatgpt']['slope_t_placebo_p']:.2f})", | 191 | ax.text(0.02, 0.97, f"level change {its['level_change']:+.3f} (p = {its['level_p']:.2f}; placebo p = {res['placebo_chatgpt']['level_t_placebo_p']:.2f})\nslope change {its['slope_change']:+.4f}/month (p = {its['slope_p']:.3f}; placebo p = {res['placebo_chatgpt']['slope_t_placebo_p']:.2f})", | ||
| 192 | transform=ax.transAxes, fontsize=7, va="top", color=style.INK2) | 192 | transform=ax.transAxes, fontsize=7, va="top", color=style.INK2) | ||
| 193 | − ax.set_ylabel("Works per month (thousands)"); ax.set_title("Interrupted time series of monthly CS output at the 2022-11 reference date", loc="left"); ax.legend(loc="lower right") | 193 | + ax.set_ylabel("Works per month (thousands)"); ax.set_title("Interrupted time series of monthly CS output at the 2022-11 reference date", loc="left"); ax.legend(loc="center left", bbox_to_anchor=(0.0, 0.62)) | ||
| 194 | style.save(fig, f"{out}/fig_its_total.pdf"); reg("fig_its_total", "Monthly output with the interrupted-time-series fit; level and slope changes with HAC and placebo p-values.", ["results/econometrics/fit_total.csv", "results/econometrics/its_results.json"]) | 194 | style.save(fig, f"{out}/fig_its_total.pdf"); reg("fig_its_total", "Monthly output with the interrupted-time-series fit; level and slope changes with HAC and placebo p-values.", ["results/econometrics/fit_total.csv", "results/econometrics/its_results.json"]) | ||
| 195 | 195 | ||||
| 196 | 196 | ||||
added
manuscript/figures/fig_prisma_flow.pdf
+0 −0
Binary file not shown.
added
manuscript/figures/fig_types_openness.pdf
+0 −0
Binary file not shown.
added
manuscript/figures/fig_volume_monthly_raw.pdf
+0 −0
Binary file not shown.
added
manuscript/figures/fig_volume_yearly.pdf
+0 −0
Binary file not shown.