SPB Git forge
spboucher.ai ↗

spb/ai-atlas

Public
41commits 1branches 0releases
4.6 MBsize
maindefault branch
12 days agolast push
HTML 77.2% TypeScript 10.5% Python 9.6% JavaScript 2.5%

Deploy: mld manifest for M2M32c, render/first-run scripts, off-node backup, DEPLOY.md; embedding model id

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 13 days ago (Sep 11, 2026) parent 71d8228

7 changed files +226 −2

modified .env.example +1 −1
@@ -16,7 +16,7 @@ AIA_LLM_API_KEY=
16 16 AIA_LLM_SMALL_MODEL=qwen3-4b-instruct-2507-4bit
17 17 AIA_LLM_MEDIUM_MODEL=qwen3.6-35b-a3b-4bit
18 18 AIA_LLM_LARGE_MODEL=qwen3.8-27b-4bit
19 −AIA_EMBEDDING_MODEL=qwen3-embedding-0.6b-4bit
19 +AIA_EMBEDDING_MODEL=qwen3-embedding-0.6b-8bit
20 20 # Optional escalation transports (never required)
21 21 SCRAPFLY_API_KEY=
22 22 FIRECRAWL_API_KEY=
added deploy/ai-atlas.mld.json +117 −0
@@ -0,0 +1,117 @@
1 +{
2 + "app": "ai-atlas",
3 + "label": "AI Atlas — the global intelligence layer for artificial intelligence",
4 + "domain": "www.ai-atlas.co",
5 + "port": 8320,
6 + "health_path": "/api/v1/health",
7 + "dir": "~/apps/ai-atlas",
8 + "extra_paths": [],
9 + "sync_excludes": [
10 + ".venv/", "__pycache__/", ".pytest_cache/", ".ruff_cache/", "*.egg-info/", ".git/", ".env", ".env.*", "!.env.example",
11 + "node_modules/", "apps/web/.next/", "apps/web/next-env.d.ts", "*.tsbuildinfo", "apps/web/qa/screens/", "apps/web/AGENTS.md", "apps/web/CLAUDE.md",
12 + "/data/", "/tmp/", "logs/", ".DS_Store", ".claude/", "deploy/.admin-token", "deploy/.llm-key", "deploy/rendered/", "tests/fixtures/"
13 + ],
14 + "requires": {
15 + "runtimes": ["pm2", "node", "pnpm", "uv", "uv-python@3.12"],
16 + "ram_gb": 8,
17 + "ports": [8320, 8321]
18 + },
19 + "ram_mb_observed": 3000,
20 + "size_mb": 60,
21 + "placement": {
22 + "pin": "M2M32c",
23 + "prefer": null,
24 + "avoid": ["M3U96b", "M1M32"],
25 + "reason": "Mac Studio M2 12 c / 32 Go dédié à AI Atlas (aucune autre app) ; Postgres 17 + pgvector + Redis Homebrew installés le 2026-09-11, raccordé au tunnel wg1 (10.67.0.20)"
26 + },
27 + "processes": [
28 + {
29 + "name": "ai-atlas-api",
30 + "manager": "pm2",
31 + "script": "{{HOME}}/apps/ai-atlas/.venv/bin/python",
32 + "args": ["-m", "uvicorn", "aiatlas.api.main:app", "--host", "127.0.0.1", "--port", "8321", "--no-access-log", "--proxy-headers", "--timeout-keep-alive", "75", "--workers", "2"],
33 + "interpreter": null,
34 + "cwd": "{{HOME}}/apps/ai-atlas",
35 + "env": {
36 + "APP_ENV": "production",
37 + "AIA_SITE_URL": "https://www.ai-atlas.co",
38 + "DATABASE_URL": "postgresql+asyncpg://aiatlas:aiatlas@127.0.0.1:5432/aiatlas",
39 + "REDIS_URL": "redis://127.0.0.1:6379/5",
40 + "AIA_DATA_DIR": "{{HOME}}/ai-atlas-data",
41 + "AIA_API_HOST": "127.0.0.1",
42 + "AIA_API_PORT": "8321",
43 + "AIA_ADMIN_TOKEN": "{{ADMIN_TOKEN}}",
44 + "AIA_LLM_BASE_URL": "https://www.llm-api.io/v1",
45 + "AIA_LLM_API_KEY": "{{LLM_KEY}}",
46 + "AIA_LOG_JSON": "1",
47 + "PYTHONUNBUFFERED": "1"
48 + },
49 + "cron_restart": null,
50 + "autorestart": true,
51 + "max_memory_restart": "3G"
52 + },
53 + {
54 + "name": "ai-atlas-scheduler",
55 + "manager": "pm2",
56 + "script": "{{HOME}}/apps/ai-atlas/.venv/bin/aia",
57 + "args": ["schedule"],
58 + "interpreter": null,
59 + "cwd": "{{HOME}}/apps/ai-atlas",
60 + "env": {
61 + "APP_ENV": "production",
62 + "AIA_SITE_URL": "https://www.ai-atlas.co",
63 + "DATABASE_URL": "postgresql+asyncpg://aiatlas:aiatlas@127.0.0.1:5432/aiatlas",
64 + "REDIS_URL": "redis://127.0.0.1:6379/5",
65 + "AIA_DATA_DIR": "{{HOME}}/ai-atlas-data",
66 + "AIA_ADMIN_TOKEN": "{{ADMIN_TOKEN}}",
67 + "AIA_LLM_BASE_URL": "https://www.llm-api.io/v1",
68 + "AIA_LLM_API_KEY": "{{LLM_KEY}}",
69 + "AIA_WORKER_CONCURRENCY": "2",
70 + "AIA_LOG_JSON": "1",
71 + "PATH": "/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:/usr/bin:/bin",
72 + "PYTHONUNBUFFERED": "1"
73 + },
74 + "cron_restart": null,
75 + "autorestart": true,
76 + "max_memory_restart": "6G"
77 + },
78 + {
79 + "name": "ai-atlas-web",
80 + "manager": "pm2",
81 + "script": "/opt/homebrew/bin/node",
82 + "args": ["node_modules/next/dist/bin/next", "start", "-p", "8320", "-H", "0.0.0.0"],
83 + "interpreter": null,
84 + "cwd": "{{HOME}}/apps/ai-atlas/apps/web",
85 + "env": {
86 + "NODE_ENV": "production",
87 + "API_URL": "http://127.0.0.1:8321",
88 + "NEXT_PUBLIC_SITE_URL": "https://www.ai-atlas.co",
89 + "AIA_ADMIN_TOKEN": "{{ADMIN_TOKEN}}",
90 + "NEXT_TELEMETRY_DISABLED": "1"
91 + },
92 + "cron_restart": null,
93 + "autorestart": true,
94 + "max_memory_restart": "2G"
95 + }
96 + ],
97 + "ngrok": null,
98 + "launchd": [],
99 + "env_overrides": {},
100 + "hooks": {
101 + "post_sync": [
102 + "set -o pipefail; mkdir -p $HOME/ai-atlas-data/{raw,text,logs,backups,cache,seed} && echo ' data dirs ok'",
103 + "set -o pipefail; export PATH=\"$HOME/.local/bin:/opt/homebrew/bin:$PATH\"; (test -x .venv/bin/python || uv venv --python 3.12 .venv) && uv pip install -q --python .venv/bin/python -e . && echo ' python deps ok'",
104 + "set -o pipefail; export PATH=\"/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$PATH\"; export DATABASE_URL=postgresql+asyncpg://aiatlas:aiatlas@127.0.0.1:5432/aiatlas AIA_DATA_DIR=$HOME/ai-atlas-data AIA_LOG_JSON=0; .venv/bin/aia migrate 2>&1 | tail -1 && .venv/bin/aia seed 2>&1 | tail -1 && echo ' migrate + seed ok'",
105 + "set -o pipefail; export PATH=\"/opt/homebrew/bin:$PATH\"; pnpm install --frozen-lockfile --silent && echo ' web deps ok'",
106 + "set -o pipefail; export PATH=\"/opt/homebrew/bin:$PATH\"; cd apps/web && API_URL=http://127.0.0.1:8321 NEXT_PUBLIC_SITE_URL=https://www.ai-atlas.co NEXT_TELEMETRY_DISABLED=1 pnpm build 2>&1 | tail -3 && echo ' web build ok'"
107 + ],
108 + "post_start": []
109 + },
110 + "notes": "v0.1.0 (2026-09-11) : Next 16 :8320 (rewrites /api/v1/* → FastAPI 127.0.0.1:8321, 2 workers uvicorn), scheduler APScheduler + worker de jobs (connecteurs adaptatifs, extraction LLM via llm-api.io, embeddings pgvector, stats/qualité horaires, backup pg_dump 04:40). Postgres 17 + pgvector + Redis Homebrew locaux ; données hors repo ~/ai-atlas-data (archive brute content-addressed, texte, backups, logs). Secrets : AIA_ADMIN_TOKEN (laptop deploy/.admin-token) et AIA_LLM_API_KEY (laptop deploy/.llm-key, clé llm-api.io « ai-atlas »). Après le premier deploy : bash deploy/first-run.sh sur le nœud (corpus initial).",
111 + "tunnel": {
112 + "domain": "www.ai-atlas.co",
113 + "gateway": "BHS64",
114 + "redirects": ["ai-atlas.co"],
115 + "note": "DNS GoDaddy A www + A @ → 51.161.112.61 posés par l'utilisateur le 2026-09-11 (vérifiés par dig)."
116 + }
117 +}
added deploy/first-run.sh +23 −0
@@ -0,0 +1,23 @@
1 +#!/bin/bash
2 +# Initial corpus build on the production node (run ON the node from ~/apps/ai-atlas). Idempotent; connectors are conditional
3 +# (ETag / content hash) so re-running only fetches what changed. Priority 0 first (labs, hub, research, providers), then the rest.
4 +set -euo pipefail
5 +cd "$(dirname "$0")/.."
6 +export PATH="/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:$PATH"
7 +export DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://aiatlas:aiatlas@127.0.0.1:5432/aiatlas}
8 +export REDIS_URL=${REDIS_URL:-redis://127.0.0.1:6379/5}
9 +export AIA_DATA_DIR=${AIA_DATA_DIR:-$HOME/ai-atlas-data}
10 +export AIA_LOG_JSON=0
11 +AIA=.venv/bin/aia
12 +
13 +$AIA migrate | tail -1
14 +$AIA seed | tail -1
15 +echo "== priority 0 connectors"
16 +$AIA crawl --priority 0 || true
17 +echo "== priority 1 connectors"
18 +$AIA crawl --priority 1 || true
19 +echo "== priority 2 connectors"
20 +$AIA crawl --priority 2 || true
21 +$AIA quality | tail -1
22 +$AIA stats | head -40
23 +$AIA status
added deploy/render-manifest.sh +19 −0
@@ -0,0 +1,19 @@
1 +#!/bin/bash
2 +# Render deploy/ai-atlas.mld.json with the real secrets and push it to the mld gateway (M1M32:~/dispatch/apps/).
3 +# Usage: deploy/render-manifest.sh [--push]
4 +# Secrets (git-ignored): deploy/.admin-token (generated if missing), deploy/.llm-key (llm-api.io API key for the LLM factory; optional).
5 +set -euo pipefail
6 +cd "$(dirname "$0")/.."
7 +TOKEN_FILE=deploy/.admin-token
8 +LLM_FILE=deploy/.llm-key
9 +[ -s "$TOKEN_FILE" ] || { openssl rand -hex 24 > "$TOKEN_FILE"; echo "generated $TOKEN_FILE"; }
10 +TOKEN=$(tr -d '\n' < "$TOKEN_FILE")
11 +LLM_KEY=""
12 +[ -s "$LLM_FILE" ] && LLM_KEY=$(tr -d '\n' < "$LLM_FILE") || echo "warning: $LLM_FILE missing — LLM factory disabled in production"
13 +mkdir -p deploy/rendered
14 +sed -e "s/{{ADMIN_TOKEN}}/$TOKEN/g" -e "s#{{LLM_KEY}}#$LLM_KEY#g" deploy/ai-atlas.mld.json > deploy/rendered/ai-atlas.json
15 +python3 -c "import json; json.load(open('deploy/rendered/ai-atlas.json')); print('manifest ok')"
16 +if [ "${1:-}" = "--push" ]; then
17 + scp -q deploy/rendered/ai-atlas.json M1M32:~/dispatch/apps/ai-atlas.json
18 + ssh M1M32 'chmod 600 ~/dispatch/apps/ai-atlas.json && echo "pushed to M1M32:~/dispatch/apps/ai-atlas.json"'
19 +fi
added docs/DEPLOY.md +54 −0
@@ -0,0 +1,54 @@
1 +# Deploying AI Atlas on MacLustr
2 +
3 +Production runs on **M2M32c** (Mac Studio M2, 12 c / 32 GB, dedicated) behind the MacLustr Tunnel (BHS64 Caddy → wg1 10.67.0.20).
4 +Everything goes through the gateway M1M32 and `mld` (`~/Desktop/cluster-skill/mld`).
5 +
6 +## Topology
7 +
8 +```
9 +Internet → GoDaddy DNS (A www.ai-atlas.co + A ai-atlas.co → 51.161.112.61)
10 + → BHS64 Caddy (TLS Let's Encrypt, apex → www redirect) → WireGuard wg1 → M2M32c:8320 (Next.js)
11 + └─ rewrite /api/v1/* → 127.0.0.1:8321 (FastAPI, 2 uvicorn workers)
12 + └─ ai-atlas-scheduler (aia schedule: connectors + jobs worker + backups)
13 + └─ Postgres 17 + pgvector (db aiatlas) · Redis db 5 · ~/ai-atlas-data
14 + LLM factory: https://www.llm-api.io/v1 (MacLustr llm-api on M1M64, key "ai-atlas") — optional, extraction degrades gracefully
15 +```
16 +
17 +Node prerequisites (done 2026-09-11 by `mld bootstrap` + `mld prepare M2M32c` + Homebrew): node 26, pnpm, pm2 (LaunchAgent), uv + Python 3.12,
18 +`postgresql@17`, `pgvector`, `redis` as `brew services`, role/db `aiatlas` (password `aiatlas`, localhost only), extensions `vector`, `pg_trgm`, `uuid-ossp`.
19 +
20 +## Release procedure (from the laptop)
21 +
22 +```bash
23 +cd ~/Desktop/Projets/apps-web/ai-atlas
24 +.venv/bin/pytest -q && (cd apps/web && pnpm typecheck) # 1. tests + typecheck
25 +deploy/render-manifest.sh --push # 2. manifest with AIA_ADMIN_TOKEN + AIA_LLM_API_KEY → M1M32:~/dispatch/apps/ai-atlas.json
26 +~/Desktop/cluster-skill/mld stage ~/Desktop/Projets/apps-web/ai-atlas ai-atlas # 3. laptop → gateway staging (sync_excludes)
27 +~/Desktop/cluster-skill/mld deploy ai-atlas --node M2M32c # 4. rsync → node, hooks (venv, migrate, seed, pnpm build), PM2, health, tunnel route, registry
28 +ssh M2M32c 'cd ~/apps/ai-atlas && nohup bash deploy/first-run.sh > ~/ai-atlas-data/logs/first-run.log 2>&1 &' # 5. first time: initial corpus
29 +curl -sI https://www.ai-atlas.co | head -1 # 6. public check
30 +```
31 +
32 +`mld deploy` re-points `https://www.ai-atlas.co → M2M32c:8320` on BHS64 and runs the public health check. `mld heal` (every 5 min on M1M32)
33 +restarts anything missing after a power cut. Secrets: `deploy/.admin-token`, `deploy/.llm-key` (git-ignored); rendered manifest in `deploy/rendered/`.
34 +
35 +## Operations
36 +
37 +```bash
38 +~/Desktop/cluster-skill/mld status | grep ai-atlas
39 +~/Desktop/cluster-skill/mld logs ai-atlas
40 +ssh M2M32c 'cd ~/apps/ai-atlas && AIA_LOG_JSON=0 .venv/bin/aia status' # connector table + queue depth
41 +ssh M2M32c 'cd ~/apps/ai-atlas && AIA_LOG_JSON=0 .venv/bin/aia run huggingface --force'
42 +ssh M2M32c 'cd ~/apps/ai-atlas && .venv/bin/aia backup' # pg_dump → ~/ai-atlas-data/backups (nightly 04:40 too)
43 +scripts/backup-offnode.sh # copies dumps + raw archive to M1M32:~/backups/ai-atlas
44 +curl -s https://www.ai-atlas.co/api/v1/stats | python3 -m json.tool | head # live counters
45 +```
46 +
47 +Admin dashboard: `https://www.ai-atlas.co/admin` (token = `deploy/.admin-token`). Environment variables the processes read are listed in `.env.example`.
48 +
49 +## Scaling
50 +
51 +- More extraction throughput: run `aia worker` on other nodes with `DATABASE_URL` pointing at M2M32c (Postgres must then listen on the LAN —
52 + not enabled by default) or run additional `aia worker --kind llm_extract` processes on M2M32c.
53 +- The LLM factory can point at any OpenAI-compatible server (`AIA_LLM_BASE_URL`), e.g. a second llm-api node.
54 +- Raw archive grows with the number of changed snapshots (gzip, content-addressed). Disk: 423 GB free on M2M32c at launch.
added scripts/backup-offnode.sh +11 −0
@@ -0,0 +1,11 @@
1 +#!/bin/bash
2 +# Off-node copies of the nightly dumps and of the raw archive (run from the laptop or from M1M32 via cron).
3 +# Usage: scripts/backup-offnode.sh [node=M2M32c] [dest=M1M32:~/backups/ai-atlas]
4 +set -euo pipefail
5 +NODE=${1:-M2M32c}
6 +DEST=${2:-M1M32:~/backups/ai-atlas}
7 +ssh "${DEST%%:*}" "mkdir -p ${DEST#*:}/{db,raw,text}"
8 +rsync -az --delete-after -e ssh "$NODE:~/ai-atlas-data/backups/" "$DEST/db/"
9 +rsync -az -e ssh "$NODE:~/ai-atlas-data/raw/" "$DEST/raw/"
10 +rsync -az -e ssh "$NODE:~/ai-atlas-data/text/" "$DEST/text/"
11 +echo "backups copied to $DEST"
modified src/aiatlas/config.py +1 −1
@@ -45,7 +45,7 @@ class Settings(BaseSettings):
45 45 llm_large_model: str = Field("qwen3.8-27b-4bit", alias="AIA_LLM_LARGE_MODEL")
46 46 llm_timeout_s: float = Field(600.0, alias="AIA_LLM_TIMEOUT")
47 47 llm_enabled: bool = Field(True, alias="AIA_LLM_ENABLED")
48 − embedding_model: str = Field("qwen3-embedding-0.6b-4bit", alias="AIA_EMBEDDING_MODEL")
48 + embedding_model: str = Field("qwen3-embedding-0.6b-8bit", alias="AIA_EMBEDDING_MODEL")
49 49 embedding_dim: int = Field(1024, alias="AIA_EMBEDDING_DIM")
50 50
51 51 # Scheduler
52 52