Repository Public
1import { execFile } from "node:child_process";2import { all, get, now, run, setSetting } from "../db";3import { configPath, enabled } from "./config";4import { event, pruneEvents } from "./events";5import * as mtx from "./mtx";6import { ACTIVE, alive, type Relay } from "./relays";7import { expire } from "./sessions";89// StreamHealthMonitor — runs in the worker. Every few seconds it reads what MediaMTX and the operating system really10// report (bytes in and out, readers, processor and memory of each process), stores a sample, closes silent sessions,11// and restarts the ingest of a relay whose source stopped sending data.1213export interface Proc { pid: number; ppid: number; cpu: number; rssMb: number; args: string }14export function processes(): Promise<Proc[]> {15 return new Promise((resolve) => {16 execFile("/bin/ps", ["-axo", "pid=,ppid=,%cpu=,rss=,args="], { maxBuffer: 8 * 1024 * 1024, timeout: 5000 }, (err, out) => {17 if (err) return resolve([]);18 resolve(out.split("\n").map((l) => /^\s*(\d+)\s+(\d+)\s+([\d.,]+)\s+(\d+)\s+(.*)$/.exec(l)).filter(Boolean).map((m) => ({ pid: Number(m![1]), ppid: Number(m![2]), cpu: Number(m![3].replace(",", ".")), rssMb: Number(m![4]) / 1024, args: m![5] })));19 });20 });21}22/** Processor and memory of a process and of everything it started. */23export function usage(procs: Proc[], root: number): { cpu: number; rssMb: number; ffmpeg: number | null } {24 let cpu = 0, rss = 0, ffmpeg: number | null = null; const todo = [root]; const seen = new Set<number>();25 while (todo.length) {26 const pid = todo.pop()!; if (seen.has(pid)) continue; seen.add(pid);27 const p = procs.find((x) => x.pid === pid); if (!p) continue;28 cpu += p.cpu; rss += p.rssMb; if (/(^|\/)ffmpeg\s/.test(p.args)) ffmpeg = p.pid;29 for (const c of procs) if (c.ppid === pid) todo.push(c.pid);30 }31 return { cpu, rssMb: rss, ffmpeg };32}3334const STALL_MS = 20_000;35let lastUp: boolean | null = null;36const prev = new Map<string, { at: number; inB: number; outB: number }>();3738export async function sample(): Promise<void> {39 if (!enabled()) return;40 const t = now();41 const state = await mtx.up();42 if (state.ok !== lastUp) { event(state.ok ? "info" : "error", "engine", state.ok ? `MediaMTX répond${state.version ? ` (${state.version})` : ""}` : "MediaMTX ne répond pas : aucun relais possible"); lastUp = state.ok; }43 setSetting("relay_engine", JSON.stringify({ at: t, up: state.ok, version: state.version }));44 expire();45 const relays = all<Relay>(`SELECT * FROM relays WHERE state IN ${ACTIVE}`);46 if (!state.ok) { for (const r of relays) if (!alive(r.pid)) run("UPDATE relays SET state = 'stopped', stopped_at = ?, clients = 0 WHERE id = ?", t, r.id); return; }4748 const [paths, sessions, procs] = await Promise.all([mtx.paths().catch(() => []), mtx.hlsSessions().catch(() => []), processes()]);49 const engine = procs.find((p) => /(^|\/)mediamtx\s/.test(p.args) && p.args.includes(configPath()));50 let totIn = 0, totOut = 0, totCpu = engine?.cpu ?? 0, totRss = engine?.rssMb ?? 0, totClients = 0;51 for (const r of relays) {52 if (!alive(r.pid)) { run("UPDATE relays SET state = 'stopped', stopped_at = ?, clients = 0, pid = NULL, ffmpeg_pid = NULL WHERE id = ? AND state IN " + ACTIVE, t, r.id); prev.delete(r.id); continue; }53 const p = paths.find((x) => x.name === r.id);54 const clients = get<{ n: number }>("SELECT COUNT(*) n FROM relay_sessions WHERE relay_id = ? AND ended_at IS NULL", r.id)!.n;55 const inB = p?.bytesReceived ?? 0;56 // Bytes really delivered to players, as counted by MediaMTX for the sessions of this path.57 const outB = sessions.filter((s) => s.path === r.id).reduce((n, s) => n + (s.outboundBytes ?? 0), 0);58 const before = prev.get(r.id); const dt = before ? (t - before.at) / 1000 : 0;59 const inKbps = before && dt > 0 && inB >= before.inB ? ((inB - before.inB) * 8) / 1000 / dt : 0;60 const outKbps = before && dt > 0 && outB >= before.outB ? ((outB - before.outB) * 8) / 1000 / dt : 0;61 prev.set(r.id, { at: t, inB, outB });62 const u = usage(procs, r.pid!);63 const moved = inB !== r.bytes_in;64 run("UPDATE relays SET bytes_in = ?, bytes_out = MAX(bytes_out, ?), clients = ?, ffmpeg_pid = ?, in_changed_at = CASE WHEN ? THEN ? ELSE COALESCE(in_changed_at, ?) END, ready_at = CASE WHEN ? AND ready_at IS NULL THEN ? ELSE ready_at END, state = CASE WHEN ? AND state = 'starting' THEN 'running' ELSE state END WHERE id = ?",65 inB, outB, clients, u.ffmpeg, Number(moved), t, t, Number(!!p?.ready), t, Number(!!p?.ready), r.id);66 run("INSERT OR REPLACE INTO relay_samples(at, relay_id, cpu, rss_mb, in_kbps, out_kbps, clients) VALUES(?,?,?,?,?,?,?)", t, r.id, u.cpu, u.rssMb, inKbps, outKbps, clients);67 totIn += inKbps; totOut += outKbps; totCpu += u.cpu; totRss += u.rssMb; totClients += clients;68 // A source that went silent: end the frozen FFmpeg so the ingest process reconnects.69 if (p?.ready && !moved && r.in_changed_at && t - r.in_changed_at > STALL_MS && u.ffmpeg) {70 event("warn", "health", `Aucune donnée reçue depuis ${Math.round((t - r.in_changed_at) / 1000)} s : reconnexion à la source`, r.id);71 try { process.kill(u.ffmpeg, "SIGKILL"); } catch { /* already gone */ }72 run("UPDATE relays SET in_changed_at = ? WHERE id = ?", t, r.id);73 }74 }75 run("INSERT OR REPLACE INTO relay_samples(at, relay_id, cpu, rss_mb, in_kbps, out_kbps, clients) VALUES(?,?,?,?,?,?,?)", t, "*", totCpu, totRss, totIn, totOut, totClients);76 run("DELETE FROM relay_samples WHERE at < ?", t - 6 * 3600_000);77 if (t % 600_000 < 6000) pruneEvents();78}79