SPB Git
spboucher.ai

spb/findata

Public

macOS TCC workaround: run PM2 processes through sshd (Full Disk Access) for the external SSD; SIGHUP graceful stop; EIO-safe logging

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 2, 2026) parent fe29072 Browse files

4 changed files +66 −14

4 changed files
Unified Split
modified infra/mld/findata.json +31 −13
@@ -50,8 +50,10 @@
50 { 50 {
51 "name": "findata-api", 51 "name": "findata-api",
52 "manager": "pm2", 52 "manager": "pm2",
53− "script": "{{HOME}}/apps/findata/.venv/bin/python", 53+ "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
54 "args": [ 54 "args": [
55+ "{{HOME}}/apps/findata",
56+ "{{HOME}}/apps/findata/.venv/bin/python",
55 "-m", 57 "-m",
56 "uvicorn", 58 "uvicorn",
57 "findata_api.main:app", 59 "findata_api.main:app",
@@ -70,13 +72,16 @@
70 }, 72 },
71 "cron_restart": null, 73 "cron_restart": null,
72 "autorestart": true, 74 "autorestart": true,
73− "max_memory_restart": "6G" 75+ "max_memory_restart": "6G",
76+ "kill_timeout": 30000
74 }, 77 },
75 { 78 {
76 "name": "findata-worker", 79 "name": "findata-worker",
77 "manager": "pm2", 80 "manager": "pm2",
78− "script": "{{HOME}}/apps/findata/.venv/bin/python", 81+ "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
79 "args": [ 82 "args": [
83+ "{{HOME}}/apps/findata",
84+ "{{HOME}}/apps/findata/.venv/bin/python",
80 "-m", 85 "-m",
81 "findata_ingest.cli", 86 "findata_ingest.cli",
82 "worker" 87 "worker"
@@ -89,13 +94,16 @@
89 }, 94 },
90 "cron_restart": null, 95 "cron_restart": null,
91 "autorestart": true, 96 "autorestart": true,
92− "max_memory_restart": "24G" 97+ "max_memory_restart": "24G",
98+ "kill_timeout": 3600000
93 }, 99 },
94 { 100 {
95 "name": "findata-bulk-1", 101 "name": "findata-bulk-1",
96 "manager": "pm2", 102 "manager": "pm2",
97− "script": "{{HOME}}/apps/findata/.venv/bin/python", 103+ "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
98 "args": [ 104 "args": [
105+ "{{HOME}}/apps/findata",
106+ "{{HOME}}/apps/findata/.venv/bin/python",
99 "-m", 107 "-m",
100 "findata_ingest.cli", 108 "findata_ingest.cli",
101 "worker", 109 "worker",
@@ -110,13 +118,16 @@
110 }, 118 },
111 "cron_restart": null, 119 "cron_restart": null,
112 "autorestart": true, 120 "autorestart": true,
113− "max_memory_restart": "12G" 121+ "max_memory_restart": "12G",
122+ "kill_timeout": 3600000
114 }, 123 },
115 { 124 {
116 "name": "findata-bulk-2", 125 "name": "findata-bulk-2",
117 "manager": "pm2", 126 "manager": "pm2",
118− "script": "{{HOME}}/apps/findata/.venv/bin/python", 127+ "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
119 "args": [ 128 "args": [
129+ "{{HOME}}/apps/findata",
130+ "{{HOME}}/apps/findata/.venv/bin/python",
120 "-m", 131 "-m",
121 "findata_ingest.cli", 132 "findata_ingest.cli",
122 "worker", 133 "worker",
@@ -131,13 +142,16 @@
131 }, 142 },
132 "cron_restart": null, 143 "cron_restart": null,
133 "autorestart": true, 144 "autorestart": true,
134− "max_memory_restart": "12G" 145+ "max_memory_restart": "12G",
146+ "kill_timeout": 3600000
135 }, 147 },
136 { 148 {
137 "name": "findata-bulk-3", 149 "name": "findata-bulk-3",
138 "manager": "pm2", 150 "manager": "pm2",
139− "script": "{{HOME}}/apps/findata/.venv/bin/python", 151+ "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
140 "args": [ 152 "args": [
153+ "{{HOME}}/apps/findata",
154+ "{{HOME}}/apps/findata/.venv/bin/python",
141 "-m", 155 "-m",
142 "findata_ingest.cli", 156 "findata_ingest.cli",
143 "worker", 157 "worker",
@@ -152,13 +166,16 @@
152 }, 166 },
153 "cron_restart": null, 167 "cron_restart": null,
154 "autorestart": true, 168 "autorestart": true,
155− "max_memory_restart": "12G" 169+ "max_memory_restart": "12G",
170+ "kill_timeout": 3600000
156 }, 171 },
157 { 172 {
158 "name": "findata-bulk-4", 173 "name": "findata-bulk-4",
159 "manager": "pm2", 174 "manager": "pm2",
160− "script": "{{HOME}}/apps/findata/.venv/bin/python", 175+ "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
161 "args": [ 176 "args": [
177+ "{{HOME}}/apps/findata",
178+ "{{HOME}}/apps/findata/.venv/bin/python",
162 "-m", 179 "-m",
163 "findata_ingest.cli", 180 "findata_ingest.cli",
164 "worker", 181 "worker",
@@ -173,7 +190,8 @@
173 }, 190 },
174 "cron_restart": null, 191 "cron_restart": null,
175 "autorestart": true, 192 "autorestart": true,
176− "max_memory_restart": "12G" 193+ "max_memory_restart": "12G",
194+ "kill_timeout": 3600000
177 } 195 }
178 ], 196 ],
179 "ngrok": null, 197 "ngrok": null,
@@ -195,5 +213,5 @@
195 "gateway": "BHS64", 213 "gateway": "BHS64",
196 "note": "DNS GoDaddy A findata.spboucher.ai → 51.161.112.61 vérifié le 2026-11-01 avant déploiement ; route Caddy posée par mld deploy ; findata-api écoute 0.0.0.0:8600 avec garde de provenance (loopback, wg1 10.67.0.0/16, LAN) ; Caddy → wg1 10.67.0.10:8600" 214 "note": "DNS GoDaddy A findata.spboucher.ai → 51.161.112.61 vérifié le 2026-11-01 avant déploiement ; route Caddy posée par mld deploy ; findata-api écoute 0.0.0.0:8600 avec garde de provenance (loopback, wg1 10.67.0.0/16, LAN) ; Caddy → wg1 10.67.0.10:8600"
197 }, 215 },
198− "notes": "Monorepo uv (Python 3.13) : services/api (FastAPI :8600, sert aussi la console Next.js statique apps/web/out), services/ingest (worker unique = verrou d'écriture du lac + ordonnanceur), lac Parquet ~/findata-lake, Postgres 17 local (DB findata, rôle findata). Secrets uniquement dans ~/apps/findata/.env (600) ; copie ~/Desktop/Cluster/secrets/findata/.env. Sauvegarde quotidienne 04:00 ET → M4M64a:~/backups/findata (rsync + pg_dump zstd). CLI : .venv/bin/findata status|discover|ingest … | 2026-11-01 soir : lac froid FINDATA_LAKE_COLD=/Volumes/ssd/findata (SSD externe 4 To, marqueur .findata-cold-root) pour les archives brutes et les datasets non « hot » ; 4 workers bulk (findata-bulk-1..4) pour backfill/options/fundamentals en parallèle ; téléchargement complet FirstRate lancé via `findata ingest backfill-all`." 216+ "notes": "Monorepo uv (Python 3.13) : services/api (FastAPI :8600, sert aussi la console Next.js statique apps/web/out), services/ingest (worker unique = verrou d'écriture du lac + ordonnanceur), lac Parquet ~/findata-lake, Postgres 17 local (DB findata, rôle findata). Secrets uniquement dans ~/apps/findata/.env (600) ; copie ~/Desktop/Cluster/secrets/findata/.env. Sauvegarde quotidienne 04:00 ET → M4M64a:~/backups/findata (rsync + pg_dump zstd). CLI : .venv/bin/findata status|discover|ingest … | 2026-11-01 soir : lac froid FINDATA_LAKE_COLD=/Volumes/ssd/findata (SSD externe 4 To, marqueur .findata-cold-root) pour les archives brutes et les datasets non « hot » ; 4 workers bulk (findata-bulk-1..4) pour backfill/options/fundamentals en parallèle ; téléchargement complet FirstRate lancé via `findata ingest backfill-all`. | TCC macOS : PM2/launchd n'ont pas l'Accès complet au disque → tous les processus passent par infra/scripts/run-under-sshd.sh (ré-exécution via sshd localhost, clé id_ed25519_maclustr) pour accéder au SSD externe. Alternative propre : accorder FDA à node + python dans Réglages → Confidentialité (GUI)."
199 } 217 }
added infra/scripts/run-under-sshd.sh +14 −0
@@ -0,0 +1,14 @@
1+#!/bin/bash
2+# macOS TCC workaround (M3U96a): processes started by PM2/launchd have no "Full Disk Access" and block on the
3+# external SSD (/Volumes/ssd). sshd-keygen-wrapper does have Full Disk Access, so PM2 runs this wrapper, which
4+# re-executes the real command on localhost through sshd. The ssh client is what PM2 supervises; when it stops,
5+# the pty hangs up and the worker receives SIGHUP (handled as a graceful stop).
6+# Usage: run-under-sshd.sh <workdir> <command...>
7+set -u
8+WORKDIR="$1"; shift
9+KEY="$HOME/.ssh/id_ed25519_maclustr"
10+REMOTE_PATH="/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$HOME/.local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
11+CMD=$(printf '%q ' "$@")
12+exec env -u SSH_AUTH_SOCK ssh -tt -q -o BatchMode=yes -o IdentitiesOnly=yes -i "$KEY" -o StrictHostKeyChecking=accept-new \
13+ -o ServerAliveInterval=30 -o ServerAliveCountMax=3 localhost \
14+ "cd $(printf '%q' "$WORKDIR") && export PATH=$REMOTE_PATH FINDATA_ROOT=$(printf '%q' "$WORKDIR") PYTHONUNBUFFERED=1 && exec $CMD"
modified packages/core/src/findata_core/logging.py +20 −1
@@ -1,5 +1,6 @@
1 from __future__ import annotations 1 from __future__ import annotations
2 2
3+import contextlib
3 import logging 4 import logging
4 import sys 5 import sys
5 6
@@ -24,10 +25,28 @@ def configure_logging(level: str = "INFO") -> None:
24 structlog.processors.JSONRenderer() if settings.is_production else structlog.dev.ConsoleRenderer(), 25 structlog.processors.JSONRenderer() if settings.is_production else structlog.dev.ConsoleRenderer(),
25 ], 26 ],
26 wrapper_class=structlog.make_filtering_bound_logger(logging.getLevelName(level)), 27 wrapper_class=structlog.make_filtering_bound_logger(logging.getLevelName(level)),
27− logger_factory=structlog.PrintLoggerFactory(), 28+ logger_factory=_SafePrintLoggerFactory(),
28 cache_logger_on_first_use=False, 29 cache_logger_on_first_use=False,
29 ) 30 )
30 31
31 32
33+class _SafePrintLogger:
34+ """stdout logger that survives a closed pipe/pty (EIO/EPIPE) so a hang-up never crashes a running job."""
35+
36+ def __init__(self):
37+ self._file = sys.stdout
38+
39+ def msg(self, message: str) -> None:
40+ with contextlib.suppress(OSError, ValueError):
41+ print(message, file=self._file, flush=True)
42+
43+ log = debug = info = warn = warning = error = critical = exception = fatal = msg
44+
45+
46+class _SafePrintLoggerFactory:
47+ def __call__(self, *args):
48+ return _SafePrintLogger()
49+
50+
32 def get_logger(name: str): # type: ignore[no-untyped-def] 51 def get_logger(name: str): # type: ignore[no-untyped-def]
33 return structlog.get_logger(name) 52 return structlog.get_logger(name)
modified services/ingest/src/findata_ingest/worker.py +1 −0
@@ -88,6 +88,7 @@ def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: boo
88 log = get_logger("worker") 88 log = get_logger("worker")
89 signal.signal(signal.SIGTERM, _handle) 89 signal.signal(signal.SIGTERM, _handle)
90 signal.signal(signal.SIGINT, _handle) 90 signal.signal(signal.SIGINT, _handle)
91+ signal.signal(signal.SIGHUP, _handle) # pty hang-up when run under sshd (see infra/scripts/run-under-sshd.sh)
91 lake = Lake() 92 lake = Lake()
92 if role == "bulk": 93 if role == "bulk":
93 _loop(lake, log, once, kinds or BULK_KINDS, False) 94 _loop(lake, log, once, kinds or BULK_KINDS, False)
94 95
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere