api: listen on all interfaces behind a source-network guard (tunnel reachability); cli closes the pool
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +26 −3
modified
docs/ARCHITECTURE.md
+1 −1
@@ -60,4 +60,4 @@ See `docs/DISCOVERY.md` §5. Budget: curated ≤ 250 GB soft limit, 60 GB free h | |||||
| 60 | 5. **Backup (daily)**: `pg_dump | zstd` + `rsync -a --delete` of the lake to M4M64a; remote file count verified; `last_backup` recorded. Restore drill: `findata restore-drill`. | 60 | 5. **Backup (daily)**: `pg_dump | zstd` + `rsync -a --delete` of the lake to M4M64a; remote file count verified; `last_backup` recorded. Restore drill: `findata restore-drill`. | ||
| 61 | 61 | ||||
| 62 | ## Security model | 62 | ## Security model | ||
| 63 | −Owner-only by default. Browser: HttpOnly/Secure/SameSite=Strict session cookie, CSRF double-submit header, TOTP MFA (scopes empty until the second factor passes). Programmatic: `Authorization: Bearer fd_<prefix>.<secret>` keys, HMAC-hashed at rest, scoped, expirable, revocable, per-key rate limit and daily usage accounting. Vendor credentials live only in `.env` (mode 600) on the node; every log line and job error passes through a redaction filter; the HTTP client never logs URLs. The API binds to 127.0.0.1 (reached through the node's wg1 proxy by Caddy on BHS64). CSP `default-src 'self'`, HSTS, nosniff, frame-ancestors none. SQL playground: single read-only statement, keyword deny-list, DuckDB `allowed_directories` restricted to the lake, external access disabled, timeout and row caps. Exports are scoped to the requesting principal and expire. | 63 | +Owner-only by default. Browser: HttpOnly/Secure/SameSite=Strict session cookie, CSRF double-submit header, TOTP MFA (scopes empty until the second factor passes). Programmatic: `Authorization: Bearer fd_<prefix>.<secret>` keys, HMAC-hashed at rest, scoped, expirable, revocable, per-key rate limit and daily usage accounting. Vendor credentials live only in `.env` (mode 600) on the node; every log line and job error passes through a redaction filter; the HTTP client never logs URLs. The API listens on the node interfaces behind a source-network guard (loopback, WireGuard 10.67.0.0/16, LAN) so Caddy on BHS64 reaches it over wg1. CSP `default-src 'self'`, HSTS, nosniff, frame-ancestors none. SQL playground: single read-only statement, keyword deny-list, DuckDB `allowed_directories` restricted to the lake, external access disabled, timeout and row caps. Exports are scoped to the requesting principal and expire. | ||
modified
infra/mld/findata.json
+2 −2
@@ -12,7 +12,7 @@ | |||||
| 12 | "size_mb": 20, | 12 | "size_mb": 20, | ||
| 13 | "placement": { "pin": "M3U96a", "prefer": "M3U96a", "avoid": ["M1M32"], "reason": "Storage node chosen in docs/DISCOVERY.md: 1 TB Apple NVMe (508 GB free, SMART verified, 5 GB/s), 32 cores / 96 GB for DuckDB, Postgres 17 + Redis already local, wg1 10.67.0.10. Lake at ~/findata-lake (outside the app dir)." }, | 13 | "placement": { "pin": "M3U96a", "prefer": "M3U96a", "avoid": ["M1M32"], "reason": "Storage node chosen in docs/DISCOVERY.md: 1 TB Apple NVMe (508 GB free, SMART verified, 5 GB/s), 32 cores / 96 GB for DuckDB, Postgres 17 + Redis already local, wg1 10.67.0.10. Lake at ~/findata-lake (outside the app dir)." }, | ||
| 14 | "processes": [ | 14 | "processes": [ | ||
| 15 | − { "name": "findata-api", "manager": "pm2", "script": "{{HOME}}/apps/findata/.venv/bin/python", "args": ["-m", "uvicorn", "findata_api.main:app", "--host", "127.0.0.1", "--port", "8600", "--workers", "1", "--proxy-headers", "--forwarded-allow-ips", "10.67.0.1,127.0.0.1"], "interpreter": null, "cwd": "{{HOME}}/apps/findata", "env": {"FINDATA_ROOT": "{{HOME}}/apps/findata", "PATH": "/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"}, "cron_restart": null, "autorestart": true, "max_memory_restart": "6G" }, | 15 | + { "name": "findata-api", "manager": "pm2", "script": "{{HOME}}/apps/findata/.venv/bin/python", "args": ["-m", "uvicorn", "findata_api.main:app", "--host", "0.0.0.0", "--port", "8600", "--workers", "1", "--proxy-headers", "--forwarded-allow-ips", "10.67.0.1,127.0.0.1"], "interpreter": null, "cwd": "{{HOME}}/apps/findata", "env": {"FINDATA_ROOT": "{{HOME}}/apps/findata", "PATH": "/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"}, "cron_restart": null, "autorestart": true, "max_memory_restart": "6G" }, | ||
| 16 | { "name": "findata-worker", "manager": "pm2", "script": "{{HOME}}/apps/findata/.venv/bin/python", "args": ["-m", "findata_ingest.cli", "worker"], "interpreter": null, "cwd": "{{HOME}}/apps/findata", "env": {"FINDATA_ROOT": "{{HOME}}/apps/findata", "PATH": "/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"}, "cron_restart": null, "autorestart": true, "max_memory_restart": "24G" } | 16 | { "name": "findata-worker", "manager": "pm2", "script": "{{HOME}}/apps/findata/.venv/bin/python", "args": ["-m", "findata_ingest.cli", "worker"], "interpreter": null, "cwd": "{{HOME}}/apps/findata", "env": {"FINDATA_ROOT": "{{HOME}}/apps/findata", "PATH": "/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"}, "cron_restart": null, "autorestart": true, "max_memory_restart": "24G" } | ||
| 17 | ], | 17 | ], | ||
| 18 | "ngrok": null, | 18 | "ngrok": null, | ||
@@ -29,6 +29,6 @@ | |||||
| 29 | "post_start": [] | 29 | "post_start": [] | ||
| 30 | }, | 30 | }, | ||
| 31 | "ka_repo": false, | 31 | "ka_repo": false, | ||
| 32 | − "tunnel": { "domain": "findata.spboucher.ai", "gateway": "BHS64", "note": "DNS GoDaddy A findata.spboucher.ai → 51.161.112.61 vérifié le 2026-11-01 avant déploiement ; route Caddy posée par mld deploy ; findata-api écoute 127.0.0.1 seulement (Caddy → wg1 10.67.0.10:8600 via le proxy du nœud)" }, | 32 | + "tunnel": { "domain": "findata.spboucher.ai", "gateway": "BHS64", "note": "DNS GoDaddy A findata.spboucher.ai → 51.161.112.61 vérifié le 2026-11-01 avant déploiement ; route Caddy posée par mld deploy ; findata-api écoute 0.0.0.0:8600 avec garde de provenance (loopback, wg1 10.67.0.0/16, LAN) ; Caddy → wg1 10.67.0.10:8600" }, | ||
| 33 | "notes": "Monorepo uv (Python 3.13) : services/api (FastAPI :8600, sert aussi la console Next.js statique apps/web/out), services/ingest (worker unique = verrou d'écriture du lac + ordonnanceur), lac Parquet ~/findata-lake, Postgres 17 local (DB findata, rôle findata). Secrets uniquement dans ~/apps/findata/.env (600) ; copie ~/Desktop/Cluster/secrets/findata/.env. Sauvegarde quotidienne 04:00 ET → M4M64a:~/backups/findata (rsync + pg_dump zstd). CLI : .venv/bin/findata status|discover|ingest …" | 33 | "notes": "Monorepo uv (Python 3.13) : services/api (FastAPI :8600, sert aussi la console Next.js statique apps/web/out), services/ingest (worker unique = verrou d'écriture du lac + ordonnanceur), lac Parquet ~/findata-lake, Postgres 17 local (DB findata, rôle findata). Secrets uniquement dans ~/apps/findata/.env (600) ; copie ~/Desktop/Cluster/secrets/findata/.env. Sauvegarde quotidienne 04:00 ET → M4M64a:~/backups/findata (rsync + pg_dump zstd). CLI : .venv/bin/findata status|discover|ingest …" | ||
| 34 | } | 34 | } | ||
modified
services/api/src/findata_api/main.py
+15 −0
@@ -72,6 +72,20 @@ def _looks_id(p: str) -> bool: | |||||
| 72 | return len(p) >= 16 and all(c in "0123456789abcdef-" for c in p) | 72 | return len(p) >= 16 and all(c in "0123456789abcdef-" for c in p) | ||
| 73 | 73 | ||||
| 74 | 74 | ||||
| 75 | +TRUSTED_PREFIXES = ("127.", "::1", "10.67.", "192.168.", "10.", "172.16.", "172.17.", "172.18.", "172.19.", "172.2", "172.30.", "172.31.") | ||||
| 76 | + | ||||
| 77 | + | ||||
| 78 | +class SourceGuard(BaseHTTPMiddleware): | ||||
| 79 | + """The API listens on all interfaces so the WireGuard peer (Caddy on BHS64 via wg1) can reach it; only loopback, | ||||
| 80 | + the tunnel subnet and the LAN may connect. Caddy terminates TLS and forwards the real client in X-Forwarded-For.""" | ||||
| 81 | + | ||||
| 82 | + async def dispatch(self, request: Request, call_next): | ||||
| 83 | + host = request.client.host if request.client else "" | ||||
| 84 | + if not host.startswith(TRUSTED_PREFIXES) and host != "testclient": | ||||
| 85 | + return JSONResponse({"error": {"code": "forbidden", "message": "untrusted network"}}, status_code=403) | ||||
| 86 | + return await call_next(request) | ||||
| 87 | + | ||||
| 88 | + | ||||
| 75 | class RateLimit(BaseHTTPMiddleware): | 89 | class RateLimit(BaseHTTPMiddleware): | ||
| 76 | async def dispatch(self, request: Request, call_next): | 90 | async def dispatch(self, request: Request, call_next): | ||
| 77 | if request.url.path.startswith(API_PREFIX) and not request.url.path.endswith("/health"): | 91 | if request.url.path.startswith(API_PREFIX) and not request.url.path.endswith("/health"): | ||
@@ -93,6 +107,7 @@ def create_app() -> FastAPI: | |||||
| 93 | "Timestamps are UTC ISO-8601; bars are stamped at bar **open**.") | 107 | "Timestamps are UTC ISO-8601; bars are stamped at bar **open**.") | ||
| 94 | app.add_middleware(Observability) | 108 | app.add_middleware(Observability) | ||
| 95 | app.add_middleware(RateLimit) | 109 | app.add_middleware(RateLimit) | ||
| 110 | + app.add_middleware(SourceGuard) | ||||
| 96 | 111 | ||||
| 97 | @app.exception_handler(StarletteHTTPException) | 112 | @app.exception_handler(StarletteHTTPException) | ||
| 98 | async def http_exc(request: Request, exc: StarletteHTTPException): | 113 | async def http_exc(request: Request, exc: StarletteHTTPException): | ||
modified
services/ingest/src/findata_ingest/cli.py
+8 −0
@@ -7,6 +7,14 @@ from pathlib import Path | |||||
| 7 | import typer | 7 | import typer | ||
| 8 | 8 | ||||
| 9 | app = typer.Typer(no_args_is_help=True, help="FinData OS operator CLI") | 9 | app = typer.Typer(no_args_is_help=True, help="FinData OS operator CLI") | ||
| 10 | + | ||||
| 11 | + | ||||
| 12 | +@app.callback() | ||||
| 13 | +def _main() -> None: | ||||
| 14 | + import atexit | ||||
| 15 | + | ||||
| 16 | + from findata_core.db import close_pool | ||||
| 17 | + atexit.register(close_pool) | ||||
| 10 | ingest_app = typer.Typer(no_args_is_help=True) | 18 | ingest_app = typer.Typer(no_args_is_help=True) | ||
| 11 | app.add_typer(ingest_app, name="ingest") | 19 | app.add_typer(ingest_app, name="ingest") | ||
| 12 | 20 | ||||
| 13 | 21 | ||||