sshd wrapper: forward stop to the remote process and wait (no orphan workers), kill leftovers on start
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 changed file +30 −5
modified
infra/scripts/run-under-sshd.sh
+30 −5
@@ -1,14 +1,39 @@ | |||||
| 1 | #!/bin/bash | 1 | #!/bin/bash | ||
| 2 | # macOS TCC workaround (M3U96a): processes started by PM2/launchd have no "Full Disk Access" and block on the | 2 | # macOS TCC workaround (M3U96a): processes started by PM2/launchd have no "Full Disk Access" and block on the | ||
| 3 | # external SSD (/Volumes/ssd). sshd-keygen-wrapper does have Full Disk Access, so PM2 runs this wrapper, which | 3 | # external SSD (/Volumes/ssd). sshd-keygen-wrapper does have Full Disk Access, so PM2 runs this wrapper, which | ||
| 4 | −# re-executes the real command on localhost through sshd. The ssh client is what PM2 supervises; when it stops, | 4 | +# executes the real command on localhost through sshd. PM2 supervises this wrapper; on stop/restart the wrapper | ||
| 5 | −# the pty hangs up and the worker receives SIGHUP (handled as a graceful stop). | 5 | +# sends SIGTERM to the *remote* process (graceful: the worker finishes its current job) and waits for it, so no | ||
| 6 | +# orphan workers are left behind. A leftover remote process from a previous incarnation (same PM2 name) is | ||||
| 7 | +# terminated first. | ||||
| 6 | # Usage: run-under-sshd.sh <workdir> <command...> | 8 | # Usage: run-under-sshd.sh <workdir> <command...> | ||
| 7 | set -u | 9 | set -u | ||
| 8 | WORKDIR="$1"; shift | 10 | WORKDIR="$1"; shift | ||
| 9 | KEY="$HOME/.ssh/id_ed25519_maclustr" | 11 | KEY="$HOME/.ssh/id_ed25519_maclustr" | ||
| 10 | REMOTE_PATH="/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$HOME/.local/bin:/usr/bin:/bin:/usr/sbin:/sbin" | 12 | REMOTE_PATH="/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$HOME/.local/bin:/usr/bin:/bin:/usr/sbin:/sbin" | ||
| 13 | +NAME="${name:-findata-$$}" # PM2 exports the process name as $name | ||||
| 14 | +PIDFILE="$WORKDIR/logs/sshd-$NAME.pid" | ||||
| 15 | +mkdir -p "$WORKDIR/logs" | ||||
| 16 | +SSH=(ssh -T -q -o BatchMode=yes -o IdentitiesOnly=yes -i "$KEY" -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=3 localhost) | ||||
| 17 | +if [ -f "$PIDFILE" ]; then | ||||
| 18 | + OLD=$(cat "$PIDFILE" 2>/dev/null) | ||||
| 19 | + if [ -n "$OLD" ] && kill -0 "$OLD" 2>/dev/null; then | ||||
| 20 | + echo "run-under-sshd: terminating leftover remote process $OLD for $NAME" >&2 | ||||
| 21 | + kill -TERM "$OLD" 2>/dev/null | ||||
| 22 | + for _ in $(seq 1 3600); do kill -0 "$OLD" 2>/dev/null || break; sleep 1; done | ||||
| 23 | + fi | ||||
| 24 | +fi | ||||
| 11 | CMD=$(printf '%q ' "$@") | 25 | CMD=$(printf '%q ' "$@") | ||
| 12 | −exec env -u SSH_AUTH_SOCK ssh -tt -q -o BatchMode=yes -o IdentitiesOnly=yes -i "$KEY" -o StrictHostKeyChecking=accept-new \ | 26 | +"${SSH[@]}" "cd $(printf '%q' "$WORKDIR") && export PATH=$REMOTE_PATH FINDATA_ROOT=$(printf '%q' "$WORKDIR") PYTHONUNBUFFERED=1 && echo \$\$ > $(printf '%q' "$PIDFILE") && exec $CMD" & | ||
| 13 | − -o ServerAliveInterval=30 -o ServerAliveCountMax=3 localhost \ | 27 | +SSHPID=$! | ||
| 14 | − "cd $(printf '%q' "$WORKDIR") && export PATH=$REMOTE_PATH FINDATA_ROOT=$(printf '%q' "$WORKDIR") PYTHONUNBUFFERED=1 && exec $CMD" | 28 | +stop() { | ||
| 29 | + REMOTE=$(cat "$PIDFILE" 2>/dev/null) | ||||
| 30 | + [ -n "$REMOTE" ] && kill -TERM "$REMOTE" 2>/dev/null | ||||
| 31 | + wait "$SSHPID" 2>/dev/null | ||||
| 32 | + rm -f "$PIDFILE" | ||||
| 33 | + exit 0 | ||||
| 34 | +} | ||||
| 35 | +trap stop TERM INT HUP | ||||
| 36 | +wait "$SSHPID" | ||||
| 37 | +RC=$? | ||||
| 38 | +rm -f "$PIDFILE" | ||||
| 39 | +exit $RC | ||||
| 15 | 40 | ||||