SPB Git
spboucher.ai

spb/company-atlas

Public

ops: mld manifest (M2U64), render/first-run scripts, backup, DEPLOY.md, stats/status commands

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 12, 2026) parent df0223c Browse files

7 changed files +436 −0

7 changed files
Unified Split
added deploy/company-atlas.mld.json +175 −0
@@ -0,0 +1,175 @@
1+{
2+ "app": "company-atlas",
3+ "label": "Company Atlas — the live atlas of global companies",
4+ "domain": "www.company-atlas.co",
5+ "port": 8360,
6+ "health_path": "/api/v1/health",
7+ "dir": "~/apps/company-atlas",
8+ "extra_paths": [],
9+ "sync_excludes": [
10+ ".venv/",
11+ "__pycache__/",
12+ ".pytest_cache/",
13+ ".ruff_cache/",
14+ "*.egg-info/",
15+ ".git/",
16+ ".env",
17+ ".env.*",
18+ "!.env.example",
19+ "node_modules/",
20+ "apps/web/.next/",
21+ "apps/web/.next-*/",
22+ "apps/web/next-env.d.ts",
23+ "*.tsbuildinfo",
24+ "apps/web/qa/screens/",
25+ "/data/",
26+ "/tmp/",
27+ "logs/",
28+ ".DS_Store",
29+ ".claude/",
30+ "deploy/.admin-token",
31+ "deploy/.llm-key",
32+ "deploy/rendered/",
33+ "fixtures/live/"
34+ ],
35+ "requires": {
36+ "runtimes": [
37+ "pm2",
38+ "node",
39+ "pnpm",
40+ "uv",
41+ "uv-python@3.12"
42+ ],
43+ "ram_gb": 12,
44+ "ports": [
45+ 8360,
46+ 8361
47+ ]
48+ },
49+ "ram_mb_observed": 4000,
50+ "size_mb": 80,
51+ "placement": {
52+ "pin": "M2U64",
53+ "prefer": null,
54+ "avoid": [
55+ "M3U96b",
56+ "M1M32"
57+ ],
58+ "reason": "Mac Studio M2 Ultra 24 c / 64 Go, 53 Go libres, Postgres 17 Homebrew déjà présent (base companyatlas créée le 2026-09-12), wg1 10.67.0.12 ; crawler continu (25 000+ capteurs) + Postgres = nœud le plus généreux hors M3U96a/b"
59+ },
60+ "processes": [
61+ {
62+ "name": "company-atlas-api",
63+ "manager": "pm2",
64+ "script": "{{HOME}}/apps/company-atlas/.venv/bin/python",
65+ "args": [
66+ "-m",
67+ "uvicorn",
68+ "companyatlas.api.main:app",
69+ "--host",
70+ "127.0.0.1",
71+ "--port",
72+ "8361",
73+ "--no-access-log",
74+ "--proxy-headers",
75+ "--timeout-keep-alive",
76+ "75",
77+ "--workers",
78+ "2"
79+ ],
80+ "interpreter": null,
81+ "cwd": "{{HOME}}/apps/company-atlas",
82+ "env": {
83+ "APP_ENV": "production",
84+ "CA_SITE_URL": "https://www.company-atlas.co",
85+ "DATABASE_URL": "postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas",
86+ "CA_DATA_DIR": "{{HOME}}/company-atlas-data",
87+ "CA_API_HOST": "127.0.0.1",
88+ "CA_API_PORT": "8361",
89+ "CA_ADMIN_TOKEN": "{{ADMIN_TOKEN}}",
90+ "CA_LLM_BASE_URL": "https://www.llm-api.io/v1",
91+ "CA_LLM_API_KEY": "{{LLM_KEY}}",
92+ "CA_DB_POOL_SIZE": "6",
93+ "CA_LOG_JSON": "1",
94+ "PYTHONUNBUFFERED": "1"
95+ },
96+ "cron_restart": null,
97+ "autorestart": true,
98+ "max_memory_restart": "2G"
99+ },
100+ {
101+ "name": "company-atlas-scheduler",
102+ "manager": "pm2",
103+ "script": "{{HOME}}/apps/company-atlas/.venv/bin/catlas",
104+ "args": [
105+ "schedule"
106+ ],
107+ "interpreter": null,
108+ "cwd": "{{HOME}}/apps/company-atlas",
109+ "env": {
110+ "APP_ENV": "production",
111+ "CA_SITE_URL": "https://www.company-atlas.co",
112+ "DATABASE_URL": "postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas",
113+ "CA_DATA_DIR": "{{HOME}}/company-atlas-data",
114+ "CA_ADMIN_TOKEN": "{{ADMIN_TOKEN}}",
115+ "CA_LLM_BASE_URL": "https://www.llm-api.io/v1",
116+ "CA_LLM_API_KEY": "{{LLM_KEY}}",
117+ "CA_FETCH_CONCURRENCY": "32",
118+ "CA_ONBOARDING_CONCURRENCY": "12",
119+ "CA_DB_POOL_SIZE": "12",
120+ "CA_DB_MAX_OVERFLOW": "12",
121+ "CA_WORKER_CONCURRENCY": "1",
122+ "CA_LOG_JSON": "1",
123+ "PATH": "/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:/usr/bin:/bin",
124+ "PYTHONUNBUFFERED": "1"
125+ },
126+ "cron_restart": null,
127+ "autorestart": true,
128+ "max_memory_restart": "6G"
129+ },
130+ {
131+ "name": "company-atlas-web",
132+ "manager": "pm2",
133+ "script": "/opt/homebrew/bin/node",
134+ "args": [
135+ "node_modules/next/dist/bin/next",
136+ "start",
137+ "-p",
138+ "8360",
139+ "-H",
140+ "0.0.0.0"
141+ ],
142+ "interpreter": null,
143+ "cwd": "{{HOME}}/apps/company-atlas/apps/web",
144+ "env": {
145+ "NODE_ENV": "production",
146+ "API_URL": "http://127.0.0.1:8361",
147+ "NEXT_PUBLIC_SITE_URL": "https://www.company-atlas.co",
148+ "NEXT_TELEMETRY_DISABLED": "1"
149+ },
150+ "cron_restart": null,
151+ "autorestart": true,
152+ "max_memory_restart": "2G"
153+ }
154+ ],
155+ "ngrok": null,
156+ "launchd": [],
157+ "env_overrides": {},
158+ "hooks": {
159+ "post_sync": [
160+ "set -o pipefail; mkdir -p $HOME/company-atlas-data/{objects,logs,backups,cache,seed} && echo ' data dirs ok'",
161+ "set -o pipefail; export PATH=\"$HOME/.local/bin:/opt/homebrew/bin:$PATH\"; (test -x .venv/bin/python || uv venv --python 3.12 .venv) && uv pip install -q --python .venv/bin/python -e . && echo ' python deps ok'",
162+ "set -o pipefail; export PATH=\"/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$PATH\"; export DATABASE_URL=postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas CA_DATA_DIR=$HOME/company-atlas-data CA_LOG_JSON=0; .venv/bin/catlas migrate 2>&1 | tail -1 && .venv/bin/catlas seed 2>&1 | tail -2 && echo ' migrate + seed ok'",
163+ "set -o pipefail; export PATH=\"/opt/homebrew/bin:$PATH\"; pnpm install --frozen-lockfile --silent && echo ' web deps ok'",
164+ "set -o pipefail; export PATH=\"/opt/homebrew/bin:$PATH\"; cd apps/web && API_URL=http://127.0.0.1:8361 NEXT_PUBLIC_SITE_URL=https://www.company-atlas.co NEXT_TELEMETRY_DISABLED=1 pnpm build 2>&1 | tail -3 && echo ' web build ok'"
165+ ],
166+ "post_start": []
167+ },
168+ "notes": "v0.1.0 (2026-09-12) : Next 16 :8360 (rewrites /api/v1/* → FastAPI 127.0.0.1:8361, 2 workers uvicorn), scheduler `catlas schedule` (capteurs dus + onboarding/discovery + tâches périodiques : events, enrichissement LLM via llm-api.io, métriques horaires, agrégats quotidiens, signaux, tendances, alertes, réparation, rétention, backup pg_dump). Postgres 17 Homebrew local (base companyatlas, rôle companyatlas) ; pas de Redis. Données hors repo ~/company-atlas-data (objets zstd content-addressed, logs, backups). Secrets : CA_ADMIN_TOKEN (laptop deploy/.admin-token) et CA_LLM_API_KEY (laptop deploy/.llm-key, clé llm-api.io « company-atlas »). Après le premier deploy : `bash deploy/first-run.sh` sur le nœud (onboarding de masse). DNS : www.company-atlas.co A → 51.161.112.61 (OK) ; apex company-atlas.co sans A ; company-atlas.com a ses NS chez Vercel — à repointer avant d'ajouter la route/redirection .com.",
169+ "tunnel": {
170+ "domain": "www.company-atlas.co",
171+ "gateway": "BHS64",
172+ "redirects": [],
173+ "note": "Ajouter company-atlas.co (apex) et www.company-atlas.com / company-atlas.com dans redirects dès que leurs A pointent vers 51.161.112.61."
174+ }
175+}
added deploy/first-run.sh +25 −0
@@ -0,0 +1,25 @@
1+#!/bin/bash
2+# First run on the production node (run ON the node from ~/apps/company-atlas, ideally under nohup):
3+# nohup bash deploy/first-run.sh > ~/company-atlas-data/logs/first-run.log 2>&1 &
4+# 1. migrate + seed (idempotent) 2. mass onboarding (discovery → sensors) 3. first metrics/daily pass.
5+# The PM2 scheduler (`catlas schedule`) keeps running in parallel: it also consumes the discovery queue and the due sensors,
6+# so this script only accelerates the initial onboarding with a dedicated worker.
7+set -euo pipefail
8+cd "$(dirname "$0")/.."
9+export PATH="$HOME/.local/bin:/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$PATH"
10+export DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas}
11+export CA_DATA_DIR=${CA_DATA_DIR:-$HOME/company-atlas-data}
12+export CA_LOG_JSON=0
13+CATLAS=.venv/bin/catlas
14+CONC=${CA_ONBOARDING_CONCURRENCY:-16}
15+
16+$CATLAS migrate | tail -1
17+$CATLAS seed | tail -2
18+echo "== onboarding (discovery + sensors) for every pending company, concurrency $CONC"
19+$CATLAS onboard --concurrency "$CONC" || true
20+echo "== first intelligence pass"
21+$CATLAS process-changes --limit 5000 || true
22+$CATLAS metrics --all || true
23+$CATLAS daily --catch-up || true
24+$CATLAS stats || true
25+echo "== done $(date)"
added deploy/render-manifest.sh +19 −0
@@ -0,0 +1,19 @@
1+#!/bin/bash
2+# Render deploy/company-atlas.mld.json with the real secrets and push it to the mld gateway (M1M32:~/dispatch/apps/).
3+# Usage: deploy/render-manifest.sh [--push]
4+# Secrets (git-ignored): deploy/.admin-token (generated if missing), deploy/.llm-key (llm-api.io key "company-atlas"; optional).
5+set -euo pipefail
6+cd "$(dirname "$0")/.."
7+TOKEN_FILE=deploy/.admin-token
8+LLM_FILE=deploy/.llm-key
9+[ -s "$TOKEN_FILE" ] || { openssl rand -hex 24 > "$TOKEN_FILE"; chmod 600 "$TOKEN_FILE"; echo "generated $TOKEN_FILE"; }
10+TOKEN=$(tr -d '\n' < "$TOKEN_FILE")
11+LLM_KEY=""
12+[ -s "$LLM_FILE" ] && LLM_KEY=$(tr -d '\n' < "$LLM_FILE") || echo "warning: $LLM_FILE missing — LLM enrichment disabled in production"
13+mkdir -p deploy/rendered
14+sed -e "s/{{ADMIN_TOKEN}}/$TOKEN/g" -e "s#{{LLM_KEY}}#$LLM_KEY#g" deploy/company-atlas.mld.json > deploy/rendered/company-atlas.json
15+python3 -c "import json; json.load(open('deploy/rendered/company-atlas.json')); print('manifest ok')"
16+if [ "${1:-}" = "--push" ]; then
17+ scp -q deploy/rendered/company-atlas.json M1M32:~/dispatch/apps/company-atlas.json
18+ ssh M1M32 'chmod 600 ~/dispatch/apps/company-atlas.json && echo "pushed to M1M32:~/dispatch/apps/company-atlas.json"'
19+fi
added docs/DEPLOY.md +68 −0
@@ -0,0 +1,68 @@
1+# Deploying Company Atlas on MacLustr
2+
3+Production runs on **M2U64** (Mac Studio M2 Ultra, 24 c / 64 GB) behind the MacLustr Tunnel (BHS64 Caddy → WireGuard wg1 10.67.0.12).
4+Everything goes through the gateway M1M32 and `mld` (`~/Desktop/cluster-skill/mld`).
5+
6+## Topology
7+
8+```
9+Internet → DNS (A www.company-atlas.co → 51.161.112.61)
10+ → BHS64 Caddy (TLS Let's Encrypt) → WireGuard wg1 → M2U64:8360 (Next.js, PM2 company-atlas-web)
11+ └─ rewrite /api/v1/* → 127.0.0.1:8361 (FastAPI, 2 uvicorn workers, PM2 company-atlas-api)
12+ └─ PM2 company-atlas-scheduler (`catlas schedule`: due sensors + discovery queue + periodic tasks)
13+ └─ Postgres 17 Homebrew (db companyatlas) · ~/company-atlas-data (objects zstd, logs, backups)
14+ LLM enrichment: https://www.llm-api.io/v1 (MacLustr llm-api on M1M64, key "company-atlas") — optional, budgeted
15+```
16+
17+Node prerequisites (done 2026-09-12): node 25 + pnpm + pm2 (Homebrew), `uv` + Python 3.12 (`~/.local/bin/uv`), `postgresql@17` running,
18+role/db `companyatlas` (password `companyatlas`, localhost only), extensions `pg_trgm` + `uuid-ossp`, `~/company-atlas-data`.
19+No Redis is required.
20+
21+## Domains
22+
23+| Host | State | Action |
24+|---|---|---|
25+| `www.company-atlas.co` | A → 51.161.112.61 (GoDaddy) — **canonical today** | route posed by `mld deploy` |
26+| `company-atlas.co` | no A record | add `A @ → 51.161.112.61`, then put `company-atlas.co` in `tunnel.redirects` |
27+| `www.company-atlas.com` / `company-atlas.com` | nameservers at **Vercel** (`ns1.vercel-dns.com`), A → Vercel IPs | point A records (or NS) at the gateway; then add both to `tunnel.redirects` (308 → www.company-atlas.co) or make `.com` canonical: `CA_SITE_URL`, `NEXT_PUBLIC_SITE_URL`, `tunnel.domain` |
28+
29+Caddy only issues certificates for hosts whose authoritative DNS points at the gateway — do not add a route before the DNS is in place.
30+
31+## Release procedure (from the laptop)
32+
33+```bash
34+cd ~/Desktop/Projets/apps-web/company-atlas
35+.venv/bin/pytest -q && .venv/bin/ruff check src tests && pnpm -r typecheck # 1. tests + typecheck
36+deploy/render-manifest.sh --push # 2. manifest with CA_ADMIN_TOKEN + CA_LLM_API_KEY → M1M32:~/dispatch/apps/company-atlas.json
37+~/Desktop/cluster-skill/mld stage ~/Desktop/Projets/apps-web/company-atlas company-atlas # 3. laptop → gateway staging (sync_excludes)
38+~/Desktop/cluster-skill/mld deploy company-atlas --node M2U64 # 4. rsync → node, hooks (venv, migrate, seed, pnpm build), PM2, health, tunnel route, registry
39+ssh M2U64 'cd ~/apps/company-atlas && nohup bash deploy/first-run.sh > ~/company-atlas-data/logs/first-run.log 2>&1 &' # 5. first time: mass onboarding
40+curl -sI https://www.company-atlas.co | head -1 # 6. public check
41+```
42+
43+`mld deploy` re-points `https://www.company-atlas.co → M2U64:8360` on BHS64 and runs the public health check. `mld heal` (every 5 min on
44+M1M32) restarts anything missing — note it also *undoes* a manual `pm2 stop`: to pause a process, remove it from the manifest.
45+
46+## Operations
47+
48+```bash
49+~/Desktop/cluster-skill/mld status | grep company-atlas
50+~/Desktop/cluster-skill/mld logs company-atlas
51+ssh M2U64 'cd ~/apps/company-atlas && export PATH=/opt/homebrew/opt/postgresql@17/bin:$PATH CA_LOG_JSON=0 DATABASE_URL=postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas CA_DATA_DIR=$HOME/company-atlas-data; .venv/bin/catlas status' # heartbeat, queue, failures
52+ssh M2U64 '… .venv/bin/catlas stats' # dataset counters
53+ssh M2U64 '… .venv/bin/catlas sensors --status failing --limit 50'
54+ssh M2U64 '… .venv/bin/catlas discover https://example.com --dry-run'
55+ssh M2U64 '… .venv/bin/catlas backup' # pg_dump → ~/company-atlas-data/backups (nightly 04:35 too)
56+scripts/backup-offnode.sh # second copy (dumps + objects) → M1M32:~/backups/company-atlas
57+curl -s https://www.company-atlas.co/api/v1/stats | python3 -m json.tool | head
58+```
59+
60+Admin console: `https://www.company-atlas.co/admin` (token = `deploy/.admin-token`). Environment variables: `.env.example`.
61+
62+## Scaling
63+
64+- More crawl throughput: raise `CA_FETCH_CONCURRENCY` (scheduler env in the manifest) or run a second `catlas schedule --no-onboarding`
65+ on another node with `DATABASE_URL` pointing at M2U64 (Postgres must then listen on the LAN and the object store must be shared or
66+ per-node — observations reference object keys, so a per-node store needs a shared path; prefer scaling on the same node first).
67+- LLM enrichment: any OpenAI-compatible server (`CA_LLM_BASE_URL`); budget `CA_LLM_DAILY_BUDGET`.
68+- Storage: object store growth ≈ changed snapshots × ~15 kB (zstd). 373 GB free on M2U64 at launch.
added scripts/backup-offnode.sh +11 −0
@@ -0,0 +1,11 @@
1+#!/bin/bash
2+# Second independent copy of the historical archive (spec §154–155): pg_dump files + object store from the production node to the
3+# gateway M1M32 (~/backups/company-atlas). Run from the laptop (or cron on M1M32). Incremental rsync; never deletes on the target.
4+# Usage: scripts/backup-offnode.sh [NODE=M2U64] [TARGET=M1M32]
5+set -euo pipefail
6+NODE=${1:-M2U64}
7+TARGET=${2:-M1M32}
8+ssh "$NODE" 'cd ~/apps/company-atlas && export PATH=/opt/homebrew/opt/postgresql@17/bin:$PATH CA_DATA_DIR=$HOME/company-atlas-data DATABASE_URL=postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas && .venv/bin/catlas backup --keep 14 | tail -1'
9+NODE_IP=$(ssh "$NODE" 'ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1')
10+ssh "$TARGET" "mkdir -p ~/backups/company-atlas/{backups,objects} && rsync -a --ignore-existing simon-pierreboucher@${NODE_IP}:company-atlas-data/backups/ ~/backups/company-atlas/backups/ && rsync -a --ignore-existing simon-pierreboucher@${NODE_IP}:company-atlas-data/objects/ ~/backups/company-atlas/objects/ && du -sh ~/backups/company-atlas/*"
11+echo "off-node copy on $TARGET:~/backups/company-atlas done $(date)"
added src/companyatlas/commands/ops.py +136 −0
@@ -0,0 +1,136 @@
1+"""Operations commands: `catlas stats`, `catlas status`, `catlas backup`."""
2+from __future__ import annotations
3+
4+import json
5+import os
6+import subprocess
7+import time
8+from datetime import UTC, datetime
9+from pathlib import Path
10+
11+import typer
12+from rich.table import Table
13+
14+from companyatlas.config import settings
15+
16+
17+def register(app: typer.Typer) -> None:
18+ app.command()(stats)
19+ app.command()(status)
20+ app.command()(backup)
21+
22+
23+def stats() -> None:
24+ """Dataset counters (companies, sensors, observations, changes, events, archive)."""
25+ from companyatlas.archive import store_stats
26+ from companyatlas.cli import out, run_async
27+ from companyatlas.db import connection, fetch_one
28+
29+ async def go(): # type: ignore[no-untyped-def]
30+ async with connection() as conn:
31+ return await fetch_one(conn, """
32+ select (select count(*) from companies) as companies,
33+ (select count(*) from companies where onboarding_status = 'active') as companies_active,
34+ (select count(*) from sensors) as sensors,
35+ (select count(*) from sensors where status = 'active') as sensors_active,
36+ (select count(*) from observations) as observations,
37+ (select count(*) from snapshots) as snapshots,
38+ (select count(*) from changes) as changes,
39+ (select count(*) from changes where kind in ('meaningful','major','critical')) as meaningful_changes,
40+ (select count(*) from events where status = 'active') as events,
41+ (select count(*) from jobs where status = 'open') as jobs_open,
42+ (select count(*) from observations where fetched_at > now() - interval '24 hours') as observations_24h,
43+ (select count(*) from events where detected_at > now() - interval '24 hours' and status = 'active') as events_24h,
44+ (select value from settings_kv where key = 'dataset_started_at') as dataset_started_at
45+ """)
46+
47+ row = run_async(go()) or {}
48+ row["archive"] = store_stats()
49+ started = row.get("dataset_started_at")
50+ if started:
51+ try:
52+ dt = datetime.fromisoformat(str(started).strip('"').replace("Z", "+00:00"))
53+ row["dataset_age_days"] = (datetime.now(UTC) - dt).days
54+ except ValueError:
55+ pass
56+ out.print(json.dumps(row, default=str, indent=1))
57+
58+
59+def status() -> None:
60+ """Scheduler heartbeat, queue depth, failures by class (last 24 h), periodic task health."""
61+ from companyatlas.cli import out, run_async
62+ from companyatlas.db import connection, fetch_all, fetch_one
63+
64+ async def go(): # type: ignore[no-untyped-def]
65+ async with connection() as conn:
66+ hb = await fetch_one(conn, "select value, updated_at from settings_kv where key = 'scheduler:heartbeat'")
67+ due = await fetch_one(conn, """select count(*) filter (where next_run_at <= now()) as due, count(*) as total,
68+ count(*) filter (where status = 'failing') as failing, count(*) filter (where status = 'stale') as stale,
69+ count(*) filter (where status = 'blocked') as blocked, count(*) filter (where status = 'retired') as retired
70+ from sensors where status <> 'retired' or status = 'retired'""")
71+ queue = await fetch_all(conn, "select kind, status, count(*) as n from queue_jobs group by 1, 2 order by 1, 2")
72+ fails = await fetch_all(conn, "select failure_class, count(*) as n from failures where at > now() - interval '24 hours' group by 1 order by 2 desc")
73+ onboarding = await fetch_all(conn, "select onboarding_status, count(*) as n from companies group by 1 order by 2 desc")
74+ return hb, due, queue, fails, onboarding
75+
76+ hb, due, queue, fails, onboarding = run_async(go())
77+ if hb:
78+ age = (datetime.now(UTC) - hb["updated_at"]).total_seconds() if hb.get("updated_at") else None
79+ out.print(f"[bold]scheduler[/] heartbeat {int(age)}s ago" if age is not None else "[bold]scheduler[/] heartbeat unknown")
80+ val = hb["value"] if isinstance(hb["value"], dict) else json.loads(hb["value"])
81+ out.print(f" worker={val.get('worker')} inflight={val.get('inflight')} due={val.get('due')} tick_ms={val.get('tick_ms')}")
82+ for t in val.get("tasks") or []:
83+ out.print(f" task {t.get('name'):<22} runs={t.get('runs')} failures={t.get('failures')} err={t.get('last_error') or '-'}")
84+ else:
85+ out.print("[yellow]scheduler[/] no heartbeat yet")
86+ out.print(f"[bold]sensors[/] {due}")
87+ t = Table(title="onboarding")
88+ t.add_column("status")
89+ t.add_column("n", justify="right")
90+ for r in onboarding:
91+ t.add_row(str(r["onboarding_status"]), str(r["n"]))
92+ out.print(t)
93+ t = Table(title="queue")
94+ for c in ("kind", "status", "n"):
95+ t.add_column(c)
96+ for r in queue:
97+ t.add_row(r["kind"], r["status"], str(r["n"]))
98+ out.print(t)
99+ t = Table(title="failures 24h")
100+ t.add_column("class")
101+ t.add_column("n", justify="right")
102+ for r in fails:
103+ t.add_row(str(r["failure_class"]), str(r["n"]))
104+ out.print(t)
105+
106+
107+def backup(keep: int = typer.Option(14, help="number of dumps to keep")) -> None:
108+ """pg_dump (custom format, compressed) into CA_DATA_DIR/backups; prunes old dumps. Copy off-node with scripts/backup-offnode.sh."""
109+ from companyatlas.cli import out
110+
111+ settings.ensure_dirs()
112+ ts = time.strftime("%Y%m%d-%H%M%S")
113+ target = settings.backups_dir / f"companyatlas-{ts}.dump"
114+ env = dict(os.environ)
115+ cmd = ["pg_dump", "--format=custom", "--compress=zstd:6", "--no-owner", "--dbname", settings.sync_database_url, "--file", str(target)]
116+ try:
117+ subprocess.run(cmd, check=True, env=env, timeout=3600)
118+ except FileNotFoundError:
119+ cmd[0] = "/opt/homebrew/opt/postgresql@17/bin/pg_dump"
120+ subprocess.run(cmd, check=True, env=env, timeout=3600)
121+ dumps = sorted(Path(settings.backups_dir).glob("companyatlas-*.dump"))
122+ for old in dumps[:-keep]:
123+ old.unlink(missing_ok=True)
124+ out.print(f"[green]backup ok[/] {target} ({target.stat().st_size // 1024} kB), kept {min(len(dumps), keep)}")
125+
126+
127+try: # register the nightly backup as a periodic task when the scheduler imports this module
128+ from companyatlas.services.periodic import periodic
129+
130+ @periodic("backup-nightly", cron=settings.backup_cron)
131+ async def _backup_task() -> None:
132+ import asyncio
133+
134+ await asyncio.to_thread(backup, 14)
135+except Exception: # noqa: BLE001
136+ pass
modified src/companyatlas/services/periodic.py +2 −0
@@ -72,6 +72,8 @@ TASK_MODULES: list[str] = [
72 "companyatlas.services.llm.enrich", 72 "companyatlas.services.llm.enrich",
73 "companyatlas.services.signals", 73 "companyatlas.services.signals",
74 "companyatlas.services.trends", 74 "companyatlas.services.trends",
75+ "companyatlas.services.repair",
76+ "companyatlas.commands.ops",
75 ] 77 ]
76 78
77 79
78 80
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere