SPB Git
spboucher.ai

spb/findata

Public

macOS TCC workaround: run PM2 processes through sshd (Full Disk Access) for the external SSD; SIGHUP graceful stop; EIO-safe logging

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 2, 2026) parent fe29072 Browse files

4 changed files +66 −14

4 changed files
Unified Split
modified infra/mld/findata.json +31 −13
@@ -50,8 +50,10 @@
50 50 {
51 51 "name": "findata-api",
52 52 "manager": "pm2",
53 − "script": "{{HOME}}/apps/findata/.venv/bin/python",
53 + "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
54 54 "args": [
55 + "{{HOME}}/apps/findata",
56 + "{{HOME}}/apps/findata/.venv/bin/python",
55 57 "-m",
56 58 "uvicorn",
57 59 "findata_api.main:app",
@@ -70,13 +72,16 @@
70 72 },
71 73 "cron_restart": null,
72 74 "autorestart": true,
73 − "max_memory_restart": "6G"
75 + "max_memory_restart": "6G",
76 + "kill_timeout": 30000
74 77 },
75 78 {
76 79 "name": "findata-worker",
77 80 "manager": "pm2",
78 − "script": "{{HOME}}/apps/findata/.venv/bin/python",
81 + "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
79 82 "args": [
83 + "{{HOME}}/apps/findata",
84 + "{{HOME}}/apps/findata/.venv/bin/python",
80 85 "-m",
81 86 "findata_ingest.cli",
82 87 "worker"
@@ -89,13 +94,16 @@
89 94 },
90 95 "cron_restart": null,
91 96 "autorestart": true,
92 − "max_memory_restart": "24G"
97 + "max_memory_restart": "24G",
98 + "kill_timeout": 3600000
93 99 },
94 100 {
95 101 "name": "findata-bulk-1",
96 102 "manager": "pm2",
97 − "script": "{{HOME}}/apps/findata/.venv/bin/python",
103 + "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
98 104 "args": [
105 + "{{HOME}}/apps/findata",
106 + "{{HOME}}/apps/findata/.venv/bin/python",
99 107 "-m",
100 108 "findata_ingest.cli",
101 109 "worker",
@@ -110,13 +118,16 @@
110 118 },
111 119 "cron_restart": null,
112 120 "autorestart": true,
113 − "max_memory_restart": "12G"
121 + "max_memory_restart": "12G",
122 + "kill_timeout": 3600000
114 123 },
115 124 {
116 125 "name": "findata-bulk-2",
117 126 "manager": "pm2",
118 − "script": "{{HOME}}/apps/findata/.venv/bin/python",
127 + "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
119 128 "args": [
129 + "{{HOME}}/apps/findata",
130 + "{{HOME}}/apps/findata/.venv/bin/python",
120 131 "-m",
121 132 "findata_ingest.cli",
122 133 "worker",
@@ -131,13 +142,16 @@
131 142 },
132 143 "cron_restart": null,
133 144 "autorestart": true,
134 − "max_memory_restart": "12G"
145 + "max_memory_restart": "12G",
146 + "kill_timeout": 3600000
135 147 },
136 148 {
137 149 "name": "findata-bulk-3",
138 150 "manager": "pm2",
139 − "script": "{{HOME}}/apps/findata/.venv/bin/python",
151 + "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
140 152 "args": [
153 + "{{HOME}}/apps/findata",
154 + "{{HOME}}/apps/findata/.venv/bin/python",
141 155 "-m",
142 156 "findata_ingest.cli",
143 157 "worker",
@@ -152,13 +166,16 @@
152 166 },
153 167 "cron_restart": null,
154 168 "autorestart": true,
155 − "max_memory_restart": "12G"
169 + "max_memory_restart": "12G",
170 + "kill_timeout": 3600000
156 171 },
157 172 {
158 173 "name": "findata-bulk-4",
159 174 "manager": "pm2",
160 − "script": "{{HOME}}/apps/findata/.venv/bin/python",
175 + "script": "{{HOME}}/apps/findata/infra/scripts/run-under-sshd.sh",
161 176 "args": [
177 + "{{HOME}}/apps/findata",
178 + "{{HOME}}/apps/findata/.venv/bin/python",
162 179 "-m",
163 180 "findata_ingest.cli",
164 181 "worker",
@@ -173,7 +190,8 @@
173 190 },
174 191 "cron_restart": null,
175 192 "autorestart": true,
176 − "max_memory_restart": "12G"
193 + "max_memory_restart": "12G",
194 + "kill_timeout": 3600000
177 195 }
178 196 ],
179 197 "ngrok": null,
@@ -195,5 +213,5 @@
195 213 "gateway": "BHS64",
196 214 "note": "DNS GoDaddy A findata.spboucher.ai → 51.161.112.61 vérifié le 2026-11-01 avant déploiement ; route Caddy posée par mld deploy ; findata-api écoute 0.0.0.0:8600 avec garde de provenance (loopback, wg1 10.67.0.0/16, LAN) ; Caddy → wg1 10.67.0.10:8600"
197 215 },
198 − "notes": "Monorepo uv (Python 3.13) : services/api (FastAPI :8600, sert aussi la console Next.js statique apps/web/out), services/ingest (worker unique = verrou d'écriture du lac + ordonnanceur), lac Parquet ~/findata-lake, Postgres 17 local (DB findata, rôle findata). Secrets uniquement dans ~/apps/findata/.env (600) ; copie ~/Desktop/Cluster/secrets/findata/.env. Sauvegarde quotidienne 04:00 ET → M4M64a:~/backups/findata (rsync + pg_dump zstd). CLI : .venv/bin/findata status|discover|ingest … | 2026-11-01 soir : lac froid FINDATA_LAKE_COLD=/Volumes/ssd/findata (SSD externe 4 To, marqueur .findata-cold-root) pour les archives brutes et les datasets non « hot » ; 4 workers bulk (findata-bulk-1..4) pour backfill/options/fundamentals en parallèle ; téléchargement complet FirstRate lancé via `findata ingest backfill-all`."
216 + "notes": "Monorepo uv (Python 3.13) : services/api (FastAPI :8600, sert aussi la console Next.js statique apps/web/out), services/ingest (worker unique = verrou d'écriture du lac + ordonnanceur), lac Parquet ~/findata-lake, Postgres 17 local (DB findata, rôle findata). Secrets uniquement dans ~/apps/findata/.env (600) ; copie ~/Desktop/Cluster/secrets/findata/.env. Sauvegarde quotidienne 04:00 ET → M4M64a:~/backups/findata (rsync + pg_dump zstd). CLI : .venv/bin/findata status|discover|ingest … | 2026-11-01 soir : lac froid FINDATA_LAKE_COLD=/Volumes/ssd/findata (SSD externe 4 To, marqueur .findata-cold-root) pour les archives brutes et les datasets non « hot » ; 4 workers bulk (findata-bulk-1..4) pour backfill/options/fundamentals en parallèle ; téléchargement complet FirstRate lancé via `findata ingest backfill-all`. | TCC macOS : PM2/launchd n'ont pas l'Accès complet au disque → tous les processus passent par infra/scripts/run-under-sshd.sh (ré-exécution via sshd localhost, clé id_ed25519_maclustr) pour accéder au SSD externe. Alternative propre : accorder FDA à node + python dans Réglages → Confidentialité (GUI)."
199 217 }
added infra/scripts/run-under-sshd.sh +14 −0
@@ -0,0 +1,14 @@
1 +#!/bin/bash
2 +# macOS TCC workaround (M3U96a): processes started by PM2/launchd have no "Full Disk Access" and block on the
3 +# external SSD (/Volumes/ssd). sshd-keygen-wrapper does have Full Disk Access, so PM2 runs this wrapper, which
4 +# re-executes the real command on localhost through sshd. The ssh client is what PM2 supervises; when it stops,
5 +# the pty hangs up and the worker receives SIGHUP (handled as a graceful stop).
6 +# Usage: run-under-sshd.sh <workdir> <command...>
7 +set -u
8 +WORKDIR="$1"; shift
9 +KEY="$HOME/.ssh/id_ed25519_maclustr"
10 +REMOTE_PATH="/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$HOME/.local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
11 +CMD=$(printf '%q ' "$@")
12 +exec env -u SSH_AUTH_SOCK ssh -tt -q -o BatchMode=yes -o IdentitiesOnly=yes -i "$KEY" -o StrictHostKeyChecking=accept-new \
13 + -o ServerAliveInterval=30 -o ServerAliveCountMax=3 localhost \
14 + "cd $(printf '%q' "$WORKDIR") && export PATH=$REMOTE_PATH FINDATA_ROOT=$(printf '%q' "$WORKDIR") PYTHONUNBUFFERED=1 && exec $CMD"
modified packages/core/src/findata_core/logging.py +20 −1
@@ -1,5 +1,6 @@
1 1 from __future__ import annotations
2 2
3 +import contextlib
3 4 import logging
4 5 import sys
5 6
@@ -24,10 +25,28 @@ def configure_logging(level: str = "INFO") -> None:
24 25 structlog.processors.JSONRenderer() if settings.is_production else structlog.dev.ConsoleRenderer(),
25 26 ],
26 27 wrapper_class=structlog.make_filtering_bound_logger(logging.getLevelName(level)),
27 − logger_factory=structlog.PrintLoggerFactory(),
28 + logger_factory=_SafePrintLoggerFactory(),
28 29 cache_logger_on_first_use=False,
29 30 )
30 31
31 32
33 +class _SafePrintLogger:
34 + """stdout logger that survives a closed pipe/pty (EIO/EPIPE) so a hang-up never crashes a running job."""
35 +
36 + def __init__(self):
37 + self._file = sys.stdout
38 +
39 + def msg(self, message: str) -> None:
40 + with contextlib.suppress(OSError, ValueError):
41 + print(message, file=self._file, flush=True)
42 +
43 + log = debug = info = warn = warning = error = critical = exception = fatal = msg
44 +
45 +
46 +class _SafePrintLoggerFactory:
47 + def __call__(self, *args):
48 + return _SafePrintLogger()
49 +
50 +
32 51 def get_logger(name: str): # type: ignore[no-untyped-def]
33 52 return structlog.get_logger(name)
modified services/ingest/src/findata_ingest/worker.py +1 −0
@@ -88,6 +88,7 @@ def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: boo
88 88 log = get_logger("worker")
89 89 signal.signal(signal.SIGTERM, _handle)
90 90 signal.signal(signal.SIGINT, _handle)
91 + signal.signal(signal.SIGHUP, _handle) # pty hang-up when run under sshd (see infra/scripts/run-under-sshd.sh)
91 92 lake = Lake()
92 93 if role == "bulk":
93 94 _loop(lake, log, once, kinds or BULK_KINDS, False)
94 95
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere