// monitor.js — collecteur Écosystème Groupe KA // Health checks HTTPS (45 s) + certificats TLS, sweep des nœuds (5 min : // charge, RAM, disque, PM2, dernier commit), incidents, historique SQLite, // push temps réel via le WebSocket du backend. import https from 'node:https'; import path from 'node:path'; import { execFile } from 'node:child_process'; import { DatabaseSync } from 'node:sqlite'; import { getSites, getNodes, selfNode, sshTarget } from './registry.js'; // Les sites surveillés et leurs nœuds viennent du REGISTRE mld (registry.js) — // plus aucune table figée ici : une app déplacée par `mld move` est suivie // automatiquement (health checks, sweep du bon nœud, actions pm2/launchd). // Historique (pour mémoire, table figée du 2026-08-19, devenue fausse le 2026-09-04) : const _SITES_LEGACY_2026_08 = [ { app: 'lou-ka', label: 'Lou·Ka', domain: 'www.lou-ka.com', node: 'M3U96b', pm2: 'lou-ka-web', dir: '~/apps/lou-ka', deployed: '2026-08' }, { app: 'immo-ka', label: 'Immo·Ka', domain: 'www.immo-ka.com', node: 'M4M64a', pm2: 'immo-ka-web', dir: '~/apps/immo-ka', deployed: '2026-08' }, { app: 'vrai-prix', label: 'Vrai-Prix', domain: 'www.vrai-prix.com', node: 'M3U96a', pm2: 'vrai-prix', dir: '~/apps/vrai-prix', deployed: '2026-08' }, { app: 'valoplex', label: 'ValoPlex', domain: 'www.valoplex.com', node: 'M3U96a', pm2: 'valoplex', dir: '~/apps/valoplex', deployed: '2026-08' }, { app: 'auto-ka', label: 'Auto·Ka', domain: 'www.auto-ka.com', node: 'M4M64b', pm2: 'auto-ka-web', dir: '~/auto-ka', deployed: '2026-08' }, { app: 'fabri-ka', label: 'Fabri·Ka', domain: 'www.fabri-ka.com', node: 'M4M64a', pm2: 'fabri-ka-web', dir: '~/fabri-ka', deployed: '2026-08' }, { app: 'food-ka', label: 'Food·Ka', domain: 'www.food-ka.com', node: 'M4M64b', pm2: 'food-ka-web', dir: '~/apps/food-ka', deployed: '2026-08' }, { app: 'sorti-ka', label: 'Sorti·Ka', domain: 'www.sorti-ka.com', node: 'M3U96a', pm2: 'sorti-ka-web', dir: '~/apps/sorti-ka', deployed: '2026-08' }, { app: 'job-ka', label: 'Job·Ka', domain: 'www.job-ka.com', node: 'M3U96a', pm2: 'job-ka-web', dir: '~/apps/job-ka', deployed: '2026-08' }, { app: 'groupe-ka', label: 'Groupe·Ka', domain: 'www.groupe-ka.com', node: 'M3U96b', pm2: 'groupe-ka', dir: '~/apps/groupe-ka', deployed: '2026-08' }, { app: 'ka-stats', label: 'Ka·Stats', domain: 'www.ka-stats.com', node: 'M1M32', pm2: 'ka-stats', dir: '~/apps/ka-stats', deployed: '2026-08' }, // Agents gardiens KA Guardian — un seul repo sur M4M36, process launchd (unit), pas de pm2 sur ce nœud { app: 'ka2', label: 'KA·2 Guardian', domain: 'www.ka2.bot', node: 'M4M36', unit: 'com.ka2.guardian', log: '~/cluster-projects/ka-guardian/logs/ka2.log', dir: '~/cluster-projects/ka-guardian', deployed: '2026-08' }, { app: 'ka4', label: 'KA·4 Guardian', domain: 'www.ka4.bot', node: 'M4M36', unit: 'com.ka4.guardian', log: '~/cluster-projects/ka-guardian/logs/ka4.log', dir: '~/cluster-projects/ka-guardian', deployed: '2026-08' }, { app: 'ka6', label: 'KA·6 Guardian', domain: 'www.ka6.bot', node: 'M4M36', unit: 'com.ka6.guardian', log: '~/cluster-projects/ka-guardian/logs/ka6.log', dir: '~/cluster-projects/ka-guardian', deployed: '2026-08' }, ]; void _SITES_LEGACY_2026_08; // conservé comme trace, jamais utilisé const CHECK_MS = 45_000; const SWEEP_MS = 5 * 60_000; const SLOW_MS = 4000; // seuil "lent" (▲) const FAILS_FOR_INCIDENT = 2; // échecs consécutifs avant incident let db, emit; const consecFails = {}; // ---------- logos des sites (favicon/apple-touch-icon, mis en cache) ---------- const ICONS = {}; // app -> { buf: Buffer, type: string } const ICON_CANDIDATES = ['/apple-touch-icon.png', '/apple-touch-icon-precomposed.png', '/favicon.svg', '/icon.svg', '/favicon.png', '/favicon.ico']; function fetchUrl(host, pathname) { return new Promise((resolve) => { const req = https.get({ host, path: pathname, timeout: 10_000, headers: { 'User-Agent': 'admin-ka-monitor/2.0' } }, (res) => { if (res.statusCode !== 200) { res.resume(); return resolve(null); } const ct = res.headers['content-type'] || ''; if (!/image|svg/.test(ct)) { res.resume(); return resolve(null); } const chunks = []; let size = 0; res.on('data', (c) => { chunks.push(c); size += c.length; if (size > 500_000) { req.destroy(); resolve(null); } }); res.on('end', () => resolve({ buf: Buffer.concat(chunks), type: ct.split(';')[0] })); }); req.on('timeout', () => req.destroy()); req.on('error', () => resolve(null)); }); } async function fetchIconFor(site) { // 1) essaie les chemins standards ; 2) sinon parse le HTML pour for (const cand of ICON_CANDIDATES) { const r = await fetchUrl(site.domain, cand); if (r) { ICONS[site.app] = r; return; } } const html = await new Promise((resolve) => { https.get({ host: site.domain, path: '/', timeout: 10_000, headers: { 'User-Agent': 'admin-ka-monitor/2.0' } }, (res) => { let d = ''; res.on('data', (c) => { d += c; if (d.length > 200_000) res.destroy(); }); res.on('end', () => resolve(d)); }).on('error', () => resolve('')).on('timeout', function () { this.destroy(); resolve(''); }); }); const m = html.match(/]+rel=["'][^"']*(?:apple-touch-icon|icon)[^"']*["'][^>]*href=["']([^"']+)["']/i) || html.match(/]+href=["']([^"']+)["'][^>]*rel=["'][^"']*icon[^"']*["']/i); if (m) { let href = m[1]; let pathname = href.startsWith('http') ? new URL(href).pathname : (href.startsWith('/') ? href : '/' + href); const r = await fetchUrl(site.domain, pathname); if (r) ICONS[site.app] = r; } } async function fetchAllIcons() { await Promise.all(getSites().map(s => fetchIconFor(s).catch(() => {}))); emit({ type: 'eco', kind: 'icons', ts: Date.now() }); } export function getIcon(app) { return ICONS[app] || null; } export function startMonitor({ dataDir, onEvent }) { emit = onEvent || (() => {}); db = new DatabaseSync(path.join(dataDir, 'eco.sqlite')); db.exec(` CREATE TABLE IF NOT EXISTS checks(site TEXT, ts INTEGER, ms INTEGER, code INTEGER, ok INTEGER, err TEXT); CREATE INDEX IF NOT EXISTS idx_checks ON checks(site, ts);`); try { db.exec('ALTER TABLE checks ADD COLUMN bytes INTEGER DEFAULT 0'); } catch {} db.exec(` CREATE TABLE IF NOT EXISTS incidents(id INTEGER PRIMARY KEY AUTOINCREMENT, site TEXT, started INTEGER, ended INTEGER, reason TEXT); CREATE TABLE IF NOT EXISTS cert(site TEXT PRIMARY KEY, expires INTEGER, issuer TEXT, checked INTEGER); CREATE TABLE IF NOT EXISTS nodestats(node TEXT, ts INTEGER, load1 REAL, mem_used INTEGER, mem_total INTEGER, disk_pct INTEGER); CREATE INDEX IF NOT EXISTS idx_nodestats ON nodestats(node, ts); CREATE TABLE IF NOT EXISTS appstats(app TEXT, ts INTEGER, cpu REAL, mem INTEGER, status TEXT, restarts INTEGER, last_commit INTEGER); CREATE INDEX IF NOT EXISTS idx_appstats ON appstats(app, ts); `); checkAll(); sweepNodes(); fetchAllIcons(); setInterval(checkAll, CHECK_MS); setInterval(sweepNodes, SWEEP_MS); setInterval(fetchAllIcons, 7 * 24 * 3600_000); setInterval(cleanup, 6 * 3600_000); } // ---------- health checks ---------- function checkOne(site) { return new Promise((resolve) => { const t0 = Date.now(); const req = https.get({ host: site.domain, path: '/', timeout: 12_000, headers: { 'User-Agent': 'admin-ka-monitor/2.0' } }, (res) => { const ms = Date.now() - t0; // ≈ TTFB (réponse reçue avant le corps) const cert = res.socket.getPeerCertificate?.(); if (cert && cert.valid_to) { db.prepare('INSERT INTO cert(site,expires,issuer,checked) VALUES(?,?,?,?) ON CONFLICT(site) DO UPDATE SET expires=excluded.expires,issuer=excluded.issuer,checked=excluded.checked') .run(site.app, Date.parse(cert.valid_to), cert.issuer?.O || cert.issuer?.CN || '', Date.now()); } const bytes = Number(res.headers['content-length'] || 0); res.resume(); resolve({ site: site.app, ts: Date.now(), ms, code: res.statusCode, ok: res.statusCode >= 200 && res.statusCode < 400 ? 1 : 0, err: null, bytes }); }); req.on('timeout', () => { req.destroy(new Error('timeout')); }); req.on('error', (e) => resolve({ site: site.app, ts: Date.now(), ms: Date.now() - t0, code: 0, ok: 0, err: e.message.slice(0, 120), bytes: 0 })); }); } async function checkAll() { const results = await Promise.all(getSites().map(checkOne)); const ins = db.prepare('INSERT INTO checks(site,ts,ms,code,ok,err,bytes) VALUES(?,?,?,?,?,?,?)'); for (const r of results) { ins.run(r.site, r.ts, r.ms, r.code, r.ok, r.err, r.bytes || 0); handleIncident(r); } emit({ type: 'eco', kind: 'checks', items: results }); } function handleIncident(r) { const open = db.prepare('SELECT id FROM incidents WHERE site=? AND ended IS NULL').get(r.site); if (!r.ok) { consecFails[r.site] = (consecFails[r.site] || 0) + 1; if (consecFails[r.site] >= FAILS_FOR_INCIDENT && !open) { db.prepare('INSERT INTO incidents(site,started,reason) VALUES(?,?,?)') .run(r.site, r.ts, r.err || `HTTP ${r.code}`); emit({ type: 'eco_alert', kind: 'down', site: r.site, reason: r.err || `HTTP ${r.code}`, ts: r.ts }); } } else { consecFails[r.site] = 0; if (open) { db.prepare('UPDATE incidents SET ended=? WHERE id=?').run(r.ts, open.id); emit({ type: 'eco_alert', kind: 'up', site: r.site, ts: r.ts }); } } } // ---------- sweep des nœuds (charge, RAM, disque, PM2, dernier commit) ---------- function sweepNode(node) { return new Promise((resolve) => { const gitCmds = getSites().filter(s => s.node === node) .map(s => `echo "GIT ${s.app} $(git -C ${s.repoDir || s.dir} log -1 --format=%ct 2>/dev/null || echo 0)"`).join('; '); // sites sous launchd (KA Guardian) : PID + cpu/rss via ps const lchCmds = getSites().filter(s => s.node === node && s.unit) .map(s => `pid=$(launchctl list | awk '$3=="${s.unit}"{print $1}'); [ -z "$pid" ] && pid=-; echo "LCH ${s.app} $pid $(ps -o %cpu=,rss= -p $pid 2>/dev/null | tr -s ' ')"`).join('; '); const script = `export PATH=/opt/homebrew/bin:/usr/local/bin:$PATH echo "LOAD $(sysctl -n vm.loadavg 2>/dev/null)" echo "MEMTOTAL $(sysctl -n hw.memsize 2>/dev/null)" echo "PAGESIZE $(sysctl -n hw.pagesize 2>/dev/null)" vm_stat 2>/dev/null | egrep "active|wired|occupied" | tr -d '.' echo "DISK $(df -k / | tail -1 | awk '{print $5}' | tr -d '%')" echo "PM2JSON $(pm2 jlist 2>/dev/null | tail -1)" ${lchCmds} ${gitCmds}`; const local = node === selfNode(); const args = local ? ['-c', script] : ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=12', sshTarget(node), script]; const bin = local ? '/bin/bash' : 'ssh'; execFile(bin, args, { timeout: 45_000, maxBuffer: 8 * 1024 * 1024, env: { ...process.env, PATH: '/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin' } }, (err, stdout) => { if (err && !stdout) return resolve(null); try { parseSweep(node, stdout); } catch {} resolve(null); }); }); } function parseSweep(node, out) { const ts = Date.now(); const load = out.match(/^LOAD \{?\s*([\d.]+)/m); const memTotal = Number(out.match(/^MEMTOTAL (\d+)/m)?.[1] || 0); const pageSize = Number(out.match(/^PAGESIZE (\d+)/m)?.[1] || 16384); let pages = 0; for (const m of out.matchAll(/Pages (?:active|wired down|occupied by compressor):\s+(\d+)/g)) pages += Number(m[1]); const memUsed = pages * pageSize; const disk = Number(out.match(/^DISK (\d+)/m)?.[1] || 0); db.prepare('INSERT INTO nodestats(node,ts,load1,mem_used,mem_total,disk_pct) VALUES(?,?,?,?,?,?)') .run(node, ts, load ? Number(load[1]) : 0, memUsed, memTotal, disk); const commits = {}; for (const m of out.matchAll(/^GIT (\S+) (\d+)/gm)) commits[m[1]] = Number(m[2]) * 1000; const pm2raw = out.match(/^PM2JSON (\[.*\])\s*$/m); let list = []; if (pm2raw) { try { list = JSON.parse(pm2raw[1]); } catch {} } // sites launchd : "LCH [<%cpu> ]" const lch = {}; for (const m of out.matchAll(/^LCH (\S+) (\S+) ?([\d.]+)? ?(\d+)?/gm)) { lch[m[1]] = { pid: m[2], cpu: Number(m[3] || 0), rss: Number(m[4] || 0) }; } const ins = db.prepare('INSERT INTO appstats(app,ts,cpu,mem,status,restarts,last_commit) VALUES(?,?,?,?,?,?,?)'); for (const site of getSites().filter(s => s.node === node)) { if (site.unit) { const l = lch[site.app]; const up = l && l.pid !== '-' && /^\d+$/.test(l.pid); ins.run(site.app, ts, l ? l.cpu : 0, l ? l.rss * 1024 : 0, l ? (up ? 'online' : 'arrêté') : 'inconnu', 0, commits[site.app] || 0); } else if (pm2raw) { const p = list.find(x => x.name === site.pm2) || list.find(x => (x.name || '').startsWith(site.app)); ins.run(site.app, ts, p ? (p.monit?.cpu ?? 0) : 0, p ? (p.monit?.memory ?? 0) : 0, p ? (p.pm2_env?.status || 'inconnu') : 'introuvable', p ? (p.pm2_env?.restart_time ?? 0) : 0, commits[site.app] || 0); } else { ins.run(site.app, ts, 0, 0, 'inconnu', 0, commits[site.app] || 0); } } } async function sweepNodes() { await Promise.all(getNodes().map(sweepNode)); emit({ type: 'eco', kind: 'sweep', ts: Date.now() }); } function cleanup() { const cut = Date.now() - 35 * 24 * 3600_000; db.prepare('DELETE FROM checks WHERE ts < ?').run(cut); db.prepare('DELETE FROM nodestats WHERE ts < ?').run(cut); db.prepare('DELETE FROM appstats WHERE ts < ?').run(cut); } // ---------- requêtes ---------- function uptime(site, sinceMs) { const r = db.prepare('SELECT COUNT(*) n, SUM(ok) up FROM checks WHERE site=? AND ts>?').get(site, Date.now() - sinceMs); return r.n ? Math.round((r.up / r.n) * 1000) / 10 : null; } function pct(sorted, p) { if (!sorted.length) return null; return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))]; } function p95(site, sinceMs) { const rows = db.prepare('SELECT ms FROM checks WHERE site=? AND ts>? AND ok=1 ORDER BY ms').all(site, Date.now() - sinceMs); return pct(rows.map(r => r.ms), 0.95); } // paquet de latence : min/p50/p90/p95/p99/max/avg sur une fenêtre function latencyStats(site, sinceMs) { const rows = db.prepare('SELECT ms FROM checks WHERE site=? AND ts>? AND ok=1 ORDER BY ms').all(site, Date.now() - sinceMs).map(r => r.ms); if (!rows.length) return { min: null, p50: null, p90: null, p95: null, p99: null, max: null, avg: null, n: 0 }; const sum = rows.reduce((a, b) => a + b, 0); return { min: rows[0], p50: pct(rows, 0.5), p90: pct(rows, 0.9), p95: pct(rows, 0.95), p99: pct(rows, 0.99), max: rows[rows.length - 1], avg: Math.round(sum / rows.length), n: rows.length }; } // distribution des codes HTTP sur 24h function statusDist(site, sinceMs) { const rows = db.prepare('SELECT code, COUNT(*) n FROM checks WHERE site=? AND ts>? GROUP BY code').all(site, Date.now() - sinceMs); const d = { '2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0, err: 0 }; for (const r of rows) { if (!r.code) d.err += r.n; else if (r.code < 300) d['2xx'] += r.n; else if (r.code < 400) d['3xx'] += r.n; else if (r.code < 500) d['4xx'] += r.n; else d['5xx'] += r.n; } return d; } // tendance latence : moyenne dernière heure vs 24h (pour flèche ↑/↓) function latencyTrend(site) { const now = Date.now(); const h = db.prepare('SELECT AVG(ms) a FROM checks WHERE site=? AND ok=1 AND ts>?').get(site, now - 3600_000)?.a; const d = db.prepare('SELECT AVG(ms) a FROM checks WHERE site=? AND ok=1 AND ts>?').get(site, now - 24 * 3600_000)?.a; if (!h || !d) return null; return Math.round((h - d) / d * 100); // % (négatif = plus rapide) } function checkCounts(site, sinceMs) { const r = db.prepare('SELECT COUNT(*) n, SUM(ok) up, AVG(bytes) b FROM checks WHERE site=? AND ts>?').get(site, Date.now() - sinceMs); return { total: r.n || 0, ok: r.up || 0, fail: (r.n || 0) - (r.up || 0), avgBytes: r.b ? Math.round(r.b) : 0 }; } // incidents + MTTR (temps moyen de rétablissement) sur 30 j function incidentStats(site) { const rows = db.prepare('SELECT started, ended FROM incidents WHERE site=? AND started>?').all(site, Date.now() - 30 * 24 * 3600_000); const closed = rows.filter(r => r.ended); const mttr = closed.length ? Math.round(closed.reduce((a, r) => a + (r.ended - r.started), 0) / closed.length / 60000) : null; const longest = rows.length ? Math.round(Math.max(...rows.map(r => (r.ended || Date.now()) - r.started)) / 60000) : 0; return { count30: rows.length, open: rows.some(r => !r.ended), mttrMin: mttr, longestMin: longest }; } export function ecoSummary() { const out = []; for (const s of getSites()) { const last = db.prepare('SELECT ts,ms,code,ok,err FROM checks WHERE site=? ORDER BY ts DESC LIMIT 1').get(s.app); const spark = db.prepare('SELECT ts,ms,ok FROM checks WHERE site=? ORDER BY ts DESC LIMIT 40').all(s.app).reverse(); const avg = db.prepare('SELECT AVG(ms) a FROM checks WHERE site=? AND ok=1 AND ts>?').get(s.app, Date.now() - 24 * 3600_000); const cert = db.prepare('SELECT expires FROM cert WHERE site=?').get(s.app); const inc = db.prepare('SELECT started,reason FROM incidents WHERE site=? AND ended IS NULL').get(s.app); const stat = db.prepare('SELECT cpu,mem,status,restarts,last_commit,ts FROM appstats WHERE app=? ORDER BY ts DESC LIMIT 1').get(s.app); const nodeStat = db.prepare('SELECT load1,mem_used,mem_total,disk_pct,ts FROM nodestats WHERE node=? ORDER BY ts DESC LIMIT 1').get(s.node); const lat = latencyStats(s.app, 24 * 3600_000); const cnt = checkCounts(s.app, 24 * 3600_000); out.push({ app: s.app, label: s.label, domain: s.domain, node: s.node, last, spark, uptime24: uptime(s.app, 24 * 3600_000), uptime7: uptime(s.app, 7 * 24 * 3600_000), uptime30: uptime(s.app, 30 * 24 * 3600_000), avg24: lat.avg, p50_24: lat.p50, p95_24: lat.p95, p99_24: lat.p99, min24: lat.min, max24: lat.max, checks24: cnt.total, fails24: cnt.fail, avgBytes24: cnt.avgBytes, trend: latencyTrend(s.app), status24: statusDist(s.app, 24 * 3600_000), incidents30: incidentStats(s.app), certExpires: cert?.expires || null, incident: inc || null, proc: stat || null, nodeStat: nodeStat || null, }); } // agrégats globaux pour le command center const upNow = out.filter(x => x.last && x.last.ok).length; const okLat = out.filter(x => x.last?.ok).map(x => x.last.ms); const totalChecks = out.reduce((a, x) => a + (x.checks24 || 0), 0); const totalFails = out.reduce((a, x) => a + (x.fails24 || 0), 0); const upts = out.map(x => x.uptime24).filter(v => v != null); return { sites: out, ts: Date.now(), global: { up: upNow, total: out.length, down: out.length - upNow, avgLatency: okLat.length ? Math.round(okLat.reduce((a, b) => a + b, 0) / okLat.length) : 0, worstLatency: okLat.length ? Math.max(...okLat) : 0, uptime24: upts.length ? Math.round(upts.reduce((a, b) => a + b, 0) / upts.length * 10) / 10 : null, checks24: totalChecks, fails24: totalFails, openIncidents: out.filter(x => x.incident).length, }, }; } export function ecoSite(app) { const s = getSites().find(x => x.app === app); if (!s) return null; const since = Date.now() - 24 * 3600_000; // ~200 seaux sur 24 h pour le graphique const bucket = Math.ceil((24 * 3600_000) / 200); const series = db.prepare(` SELECT (ts/${bucket})*${bucket} AS t, AVG(ms) ms, MIN(ok) ok, COUNT(*) n FROM checks WHERE site=? AND ts>? GROUP BY t ORDER BY t`).all(app, since); const incidents = db.prepare('SELECT started,ended,reason FROM incidents WHERE site=? AND started>? ORDER BY started DESC LIMIT 30') .all(app, Date.now() - 30 * 24 * 3600_000); const errors = db.prepare('SELECT ts,code,err,ms FROM checks WHERE site=? AND ok=0 ORDER BY ts DESC LIMIT 20').all(app); // uptime + latence quotidiens 14 jours const days = []; for (let i = 13; i >= 0; i--) { const d0 = new Date(); d0.setHours(0, 0, 0, 0); const start = d0.getTime() - i * 24 * 3600_000; const r = db.prepare('SELECT COUNT(*) n, SUM(ok) up, AVG(CASE WHEN ok=1 THEN ms END) a FROM checks WHERE site=? AND ts>=? AND ts? ORDER BY ts').all(node, Date.now() - 24 * 3600_000); const apps = getSites().filter(s => s.node === node).map(s => { const p = db.prepare('SELECT cpu,mem,status,restarts FROM appstats WHERE app=? ORDER BY ts DESC LIMIT 1').get(s.app); return { app: s.app, label: s.label, ...p }; }); out.push({ node, last: last || null, hist, apps }); } return { nodes: out, ts: Date.now() }; }