sshd wrapper: forward stop to the remote process and wait (no orphan workers), kill leftovers on start
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 changed file +30 −5
modified
infra/scripts/run-under-sshd.sh
+30 −5
@@ -1,14 +1,39 @@ | ||
| 1 | 1 | #!/bin/bash |
| 2 | 2 | # macOS TCC workaround (M3U96a): processes started by PM2/launchd have no "Full Disk Access" and block on the |
| 3 | 3 | # external SSD (/Volumes/ssd). sshd-keygen-wrapper does have Full Disk Access, so PM2 runs this wrapper, which |
| 4 | −# re-executes the real command on localhost through sshd. The ssh client is what PM2 supervises; when it stops, | |
| 5 | −# the pty hangs up and the worker receives SIGHUP (handled as a graceful stop). | |
| 4 | +# executes the real command on localhost through sshd. PM2 supervises this wrapper; on stop/restart the wrapper | |
| 5 | +# sends SIGTERM to the *remote* process (graceful: the worker finishes its current job) and waits for it, so no | |
| 6 | +# orphan workers are left behind. A leftover remote process from a previous incarnation (same PM2 name) is | |
| 7 | +# terminated first. | |
| 6 | 8 | # Usage: run-under-sshd.sh <workdir> <command...> |
| 7 | 9 | set -u |
| 8 | 10 | WORKDIR="$1"; shift |
| 9 | 11 | KEY="$HOME/.ssh/id_ed25519_maclustr" |
| 10 | 12 | REMOTE_PATH="/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$HOME/.local/bin:/usr/bin:/bin:/usr/sbin:/sbin" |
| 13 | +NAME="${name:-findata-$$}" # PM2 exports the process name as $name | |
| 14 | +PIDFILE="$WORKDIR/logs/sshd-$NAME.pid" | |
| 15 | +mkdir -p "$WORKDIR/logs" | |
| 16 | +SSH=(ssh -T -q -o BatchMode=yes -o IdentitiesOnly=yes -i "$KEY" -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=3 localhost) | |
| 17 | +if [ -f "$PIDFILE" ]; then | |
| 18 | + OLD=$(cat "$PIDFILE" 2>/dev/null) | |
| 19 | + if [ -n "$OLD" ] && kill -0 "$OLD" 2>/dev/null; then | |
| 20 | + echo "run-under-sshd: terminating leftover remote process $OLD for $NAME" >&2 | |
| 21 | + kill -TERM "$OLD" 2>/dev/null | |
| 22 | + for _ in $(seq 1 3600); do kill -0 "$OLD" 2>/dev/null || break; sleep 1; done | |
| 23 | + fi | |
| 24 | +fi | |
| 11 | 25 | CMD=$(printf '%q ' "$@") |
| 12 | −exec env -u SSH_AUTH_SOCK ssh -tt -q -o BatchMode=yes -o IdentitiesOnly=yes -i "$KEY" -o StrictHostKeyChecking=accept-new \ | |
| 13 | − -o ServerAliveInterval=30 -o ServerAliveCountMax=3 localhost \ | |
| 14 | − "cd $(printf '%q' "$WORKDIR") && export PATH=$REMOTE_PATH FINDATA_ROOT=$(printf '%q' "$WORKDIR") PYTHONUNBUFFERED=1 && exec $CMD" | |
| 26 | +"${SSH[@]}" "cd $(printf '%q' "$WORKDIR") && export PATH=$REMOTE_PATH FINDATA_ROOT=$(printf '%q' "$WORKDIR") PYTHONUNBUFFERED=1 && echo \$\$ > $(printf '%q' "$PIDFILE") && exec $CMD" & | |
| 27 | +SSHPID=$! | |
| 28 | +stop() { | |
| 29 | + REMOTE=$(cat "$PIDFILE" 2>/dev/null) | |
| 30 | + [ -n "$REMOTE" ] && kill -TERM "$REMOTE" 2>/dev/null | |
| 31 | + wait "$SSHPID" 2>/dev/null | |
| 32 | + rm -f "$PIDFILE" | |
| 33 | + exit 0 | |
| 34 | +} | |
| 35 | +trap stop TERM INT HUP | |
| 36 | +wait "$SSHPID" | |
| 37 | +RC=$? | |
| 38 | +rm -f "$PIDFILE" | |
| 39 | +exit $RC | |
| 15 | 40 | |