Repository Public
1#!/usr/bin/env python32"""Generate the mld manifest for Agent OS (agent.spboucher.ai) ON THE GATEWAY (M1M32).34Usage (on M1M32):5 python3 make-manifest.py [--node M4M64c] [--port 8640] > ~/dispatch/apps/agent-os.json.new \6 && mv ~/dispatch/apps/agent-os.json.new ~/dispatch/apps/agent-os.json && chmod 600 ~/dispatch/apps/agent-os.json7(write to a temporary name first: a plain `> agent-os.json` truncates the existing manifest before this script8reads it, and the secrets it holds would be regenerated.)910Secrets (database password, AOS_SECRET_KEY) are generated here, or reused from an existing manifest so that11redeploys keep them, and are never printed anywhere else. They live in ONE place in the manifest: the `.env`12block written (0600) on the node by the `post_sync` hook. The PM2 environment holds no secret; the server and13the worker load the file named by AOS_ENV_FILE themselves at start (variables already set win).1415Model provider keys (ANTHROPIC_API_KEY, OPENAI_API_KEY) never pass through the gateway or this manifest: the16operator creates `~/.config/agent-os/keys.env` (0600) ON THE NODE and `post_sync` appends it to `.env`.17"""18import argparse, json, os, re, secrets, sys1920NAME = "agent-os"21TITLE = "Agent OS"22DB = "agent_os" # role and database23BACKUP_CRON = "5 4 * * *" # 04:05 every night24PROVIDER_KEYS = ("ANTHROPIC_API_KEY", "OPENAI_API_KEY")25SECRET_NAMES = ("DATABASE_URL", "AOS_SECRET_KEY") # .env only, never in the PM2 environment2627ap = argparse.ArgumentParser(description="Generate the mld manifest of Agent OS (JSON on stdout).")28ap.add_argument("--node", default="M4M64c")29ap.add_argument("--port", type=int, default=8640)30ap.add_argument("--domain", default="agent.spboucher.ai")31ap.add_argument("--max-runs", type=int, default=4, help="AOS_MAX_CONCURRENT_RUNS (1–32, default 4)")32ap.add_argument("--existing", default=None,33 help=f"manifest whose secrets are reused; default: ~/dispatch/apps/{NAME}.json")34a = ap.parse_args()35if not 1 <= a.max_runs <= 32:36 ap.error("--max-runs must be between 1 and 32")37if a.existing is None:38 a.existing = os.path.expanduser(f"~/dispatch/apps/{NAME}.json")394041def read_old_secrets(path):42 """Secrets of an existing manifest: the .env block of post_sync, else (older layout) the PM2 environment."""43 found = {}44 if not os.path.exists(path):45 return found46 try:47 with open(path) as f:48 old = json.load(f)49 except Exception:50 sys.stderr.write(f"make-manifest: {path} exists but is not valid JSON; refusing to regenerate secrets\n")51 sys.exit(2)52 try:53 for hook in old.get("hooks", {}).get("post_sync", []):54 m = re.search(r"<<'ENVEOF'\n(.*?)\nENVEOF\n", hook, re.S)55 if not m:56 continue57 for line in m.group(1).splitlines():58 k, sep, v = line.partition("=")59 if sep and k in SECRET_NAMES and v:60 found.setdefault(k, v)61 for p in old.get("processes", []):62 for k in SECRET_NAMES:63 v = (p.get("env") or {}).get(k)64 if v:65 found.setdefault(k, v)66 except Exception:67 pass68 return found697071old = read_old_secrets(a.existing)727374def keep(name, gen):75 return old.get(name) or gen()767778HOME = "/Users/simon-pierreboucher"79APP = f"{HOME}/apps/{NAME}"80DATA = f"{HOME}/{NAME}-data"81WORKSPACE = f"{HOME}/agent-workspace"82KEYS_FILE = f"{HOME}/.config/{NAME}/keys.env"83PATH = "/opt/homebrew/bin:/opt/homebrew/opt/postgresql@17/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"8485db_pass = None86if old.get("DATABASE_URL"):87 database_url = old["DATABASE_URL"]88else:89 db_pass = secrets.token_hex(24)90 database_url = f"postgres://{DB}:{db_pass}@127.0.0.1:5432/{DB}"91secret_key = keep("AOS_SECRET_KEY", lambda: secrets.token_hex(32)) # 64 hex9293# Application settings shared by the server and the worker (no secret here).94app_env = {95 "NODE_ENV": "production",96 "AOS_PORT": str(a.port),97 "AOS_HOST": "0.0.0.0",98 "AOS_PUBLIC_ORIGIN": f"https://{a.domain}",99 "AOS_DATA_DIR": DATA,100 "AOS_WORKSPACE_ROOT": WORKSPACE,101 "AOS_TRUSTED_PROXIES": "127.0.0.1,::1,10.67.0.1",102 "AOS_WEB_DIR": f"{APP}/apps/web/out",103 "AOS_MAX_CONCURRENT_RUNS": str(a.max_runs),104 "AOS_LOG_LEVEL": "info",105 # Folders of this node the agents' sandbox must never read, on top of the built-in deny list106 # (the application directory itself — hence .env — is under ~/apps).107 "AOS_SANDBOX_EXTRA_DENY_READ": ",".join(108 f"{HOME}/{d}" for d in ("apps", "backups", "Desktop", "Documents", "Downloads", "dispatch", "srv")),109 "AOS_ENV_FILE": f"{APP}/.env",110}111# PM2 environment: settings + the process basics. No DATABASE_URL, no AOS_SECRET_KEY, no provider key.112env = {113 **app_env,114 "SHELL": "/bin/zsh",115 "HOME": HOME,116 "LANG": "en_CA.UTF-8",117 "PATH": PATH,118}119# .env (0600 on the node): the secrets + the same settings, so one-off commands can be run from it.120dotenv = {"DATABASE_URL": database_url, "AOS_SECRET_KEY": secret_key, **app_env}121122123def proc(name, script_args, **kw):124 p = {"name": name, "manager": "pm2", "script": "/opt/homebrew/bin/node", "args": script_args,125 "interpreter": None, "cwd": f"{{{{HOME}}}}/apps/{NAME}", "env": env,126 "cron_restart": None, "autorestart": True, "max_memory_restart": "1500M"}127 p.update(kw)128 return p129130131envfile = "\n".join(f"{k}={v}" for k, v in dotenv.items())132db_setup = ""133if db_pass:134 db_setup = (135 f"psql -d postgres -tAc \"select 1 from pg_roles where rolname='{DB}'\" | grep -q 1 || "136 f"psql -d postgres -c \"create role {DB} login password '{db_pass}'\" >/dev/null; "137 f"psql -d postgres -tAc \"select 1 from pg_database where datname='{DB}'\" | grep -q 1 || "138 f"createdb -O {DB} {DB}; "139 f"psql -d {DB} -c 'create extension if not exists pgcrypto' >/dev/null; "140 )141post_sync = (142 "export PATH=/opt/homebrew/bin:/opt/homebrew/opt/postgresql@17/bin:/usr/bin:/bin:/usr/sbin:/sbin; umask 077; "143 f"mkdir -p {DATA}/artifacts {WORKSPACE} {HOME}/backups/{NAME} {HOME}/.config/{NAME}; "144 f"chmod 700 {DATA} {HOME}/.config/{NAME}; "145 + db_setup +146 f"cat > .env <<'ENVEOF'\n{envfile}\nENVEOF\nchmod 600 .env; "147 # Provider keys: read on the node only, appended to .env; their values never reach the gateway.148 f"if [ -f {KEYS_FILE} ]; then chmod 600 {KEYS_FILE}; "149 f"{{ echo; cat {KEYS_FILE}; echo; }} >> .env; "150 f"else echo 'WARNING: {KEYS_FILE} is missing: no model provider key, the worker will refuse to start'; fi; "151 "chmod 600 .env; "152 "(pnpm install --frozen-lockfile 2>&1 | tail -2) && pnpm -r --workspace-concurrency=1 build 2>&1 | tail -6 && "153 "test -f apps/server/dist/main.js && test -f apps/worker/dist/main.js && "154 "test -f packages/kernel/dist/migrate-cli.js && test -f apps/web/out/index.html && "155 # The migration reads DATABASE_URL, taken from .env (only that line: .env is not sourced).156 "DATABASE_URL=\"$(sed -n 's/^DATABASE_URL=//p' .env | head -1)\" node packages/kernel/dist/migrate-cli.js"157)158159manifest = {160 "app": NAME,161 "label": f"{TITLE} — Agent Operating System ({a.domain})",162 "domain": a.domain,163 "port": a.port,164 "health_path": "/api/v1/health",165 "dir": f"~/apps/{NAME}",166 "extra_paths": [],167 "sync_excludes": [".DS_Store", "node_modules/", ".next/", "out/", "dist/", "data/", ".env", ".env.local",168 "coverage/", "test-results/", "playwright-report/", "*.tsbuildinfo", "e2e/screenshots/"],169 "requires": {"runtimes": ["node", "pnpm", "pm2", "postgresql@17"], "ram_gb": 4, "ports": [a.port]},170 "ram_mb_observed": 600,171 "size_mb": 30,172 "placement": {"pin": a.node, "prefer": None, "avoid": ["M1M32"],173 "reason": f"Hôte dédié {TITLE} : PostgreSQL local, artefacts et espace de travail des agents "174 f"(~/agent-workspace) sur ce nœud, clés des fournisseurs dans ~/.config/{NAME}/keys.env "175 "(ne pas déplacer sans recopier la base, les données et ce fichier)"},176 "processes": [177 proc(f"{NAME}-server", ["apps/server/dist/main.js"]),178 proc(f"{NAME}-worker", ["apps/worker/dist/main.js"], kill_timeout=20000),179 {"name": f"{NAME}-backup", "manager": "pm2", "script": "/bin/zsh", "args": ["deploy/backup.sh"],180 "interpreter": None, "cwd": f"{{{{HOME}}}}/apps/{NAME}",181 "env": {"AOS_DATA_DIR": DATA, "AOS_ENV_FILE": f"{APP}/.env", "HOME": HOME, "PATH": PATH},182 "cron_restart": BACKUP_CRON, "autorestart": False, "max_memory_restart": None},183 ],184 "ngrok": None,185 "launchd": [],186 "env_overrides": {},187 "hooks": {"post_sync": [post_sync], "post_start": []},188 "ka_repo": False,189 "tunnel": {"domain": a.domain, "gateway": "BHS64",190 "note": "DNS GoDaddy A → 51.161.112.61 à poser par le propriétaire avant le premier déploiement"},191 "notes": f"{TITLE} : serveur Fastify :{a.port} (API + interface statique) + worker (exécution durable des missions, "192 f"{a.max_runs} runs simultanés, kill_timeout 20 s), PostgreSQL 17 local (rôle/base {DB}). "193 f"Secrets (DATABASE_URL, AOS_SECRET_KEY) uniquement dans ~/apps/{NAME}/.env (0600, écrit par post_sync, "194 f"chargé par les processus via AOS_ENV_FILE) ; clés Anthropic/OpenAI lues sur le nœud dans "195 f"~/.config/{NAME}/keys.env (0600) et ajoutées à .env au déploiement — jamais dans ce manifeste. "196 f"Données ~/{NAME}-data (artefacts), projets ~/agent-workspace. "197 f"Sauvegarde chiffrée nocturne deploy/backup.sh → ~/backups/{NAME} (phrase de passe ~/.config/{NAME}/backup.pass).",198}199out = json.dumps(manifest, indent=1, ensure_ascii=False)200# Belt and braces: a provider key present in this process's environment must never end up in the manifest.201for k in PROVIDER_KEYS:202 v = os.environ.get(k, "")203 if (len(v) >= 8 and v in out) or re.search(rf"(?m)^{k}=", envfile) or k in env:204 sys.stderr.write(f"make-manifest: {k} would be embedded in the manifest; aborting\n")205 sys.exit(3)206sys.stdout.write(out)207