SPB Git

Repository Public

ops.ts 36.6 KB 483 lines typescript

1// SP Mail Next — exploitation (administrateurs) : contrôles de santé agrégés, alertes levées / résolues par un moniteur, appareils et2// sessions de tous les utilisateurs, tendances (espace, remises, API) et événements de sécurité. Complète /api/admin/overview, /metrics et3// /audit sans les dupliquer : mêmes sources (Stalwart, PostgreSQL), vues différentes. Une alerte n'envoie jamais de courrier vers l'extérieur.4import type { FastifyInstance } from "fastify"5import fs from "node:fs"6import net from "node:net"7import tls from "node:tls"8import { z } from "zod"9import { dnsReport } from "../admin.js"10import { config } from "../config.js"11import { sql, type Sql } from "../db.js"12import { jmapOk, roleFolder, uploadBlob, type Mbx } from "../mailjmap.js"13import { requireAdminReauth } from "../reauth.js"14import { requireAdmin } from "../session.js"15import { getAll, listDomains } from "../stalwart.js"16import { audit, bad, parse, zUuid } from "../util.js"17import type { NextWorker } from "./index.js"1819// ------------------------------------------------------------------ battements des travailleurs Next (registre en mémoire)20const beats = new Map<string, { at: number; intervalMs: number }>()21/** Appelé par chaque travailleur Next à la fin d'un tour : sert au contrôle « workers » de /api/admin/health. */22export function beat(name: string, intervalMs: number) {23  beats.set(name, { at: Date.now(), intervalMs })24}2526// ------------------------------------------------------------------ contrôles27export type CheckStatus = "ok" | "warn" | "fail" | "unknown"28export type Check = { status: CheckStatus; detail: string; checked_at: string; data?: Record<string, unknown> }29type Result = Omit<Check, "checked_at">30const cached = new Map<string, { at: number; value: Check }>()31async function check(name: string, ttlMs: number, fn: () => Promise<Result>, fresh = false): Promise<Check> {32  const c = cached.get(name)33  if (!fresh && c && Date.now() - c.at < ttlMs) return c.value34  let r: Result35  try { r = await fn() } catch (e) { r = { status: "unknown", detail: `check failed: ${String((e as Error).message ?? e).slice(0, 160)}` } }36  const value = { ...r, checked_at: new Date().toISOString() }37  cached.set(name, { at: Date.now(), value })38  return value39}40const worst = (list: CheckStatus[]): CheckStatus => (list.includes("fail") ? "fail" : list.includes("warn") ? "warn" : list.includes("ok") ? "ok" : "unknown")41const hoursSince = (d: string | Date) => (Date.now() - new Date(d).getTime()) / 3600_0004243let serverFailures = 044async function mailServer(): Promise<Result> {45  const t0 = Date.now()46  try {47    const domains = await listDomains()48    serverFailures = 049    const ms = Date.now() - t050    return { status: ms > 3000 ? "warn" : "ok", detail: `reachable in ${ms} ms, ${domains.length} domain(s)`, data: { reachable: true, latency_ms: ms, domains: domains.length } }51  } catch (e) {52    serverFailures++53    return { status: "fail", detail: `unreachable: ${String((e as Error).message).slice(0, 120)}`, data: { reachable: false, consecutive_failures: serverFailures } }54  }55}5657/** File de Stalwart : taille, ancienneté du plus vieux message, messages en échec (enveloppes seulement, jamais le contenu). */58export function queueStats(queue: Record<string, any>[], now = Date.now()) {59  const created = queue.map((q) => Date.parse(q.createdAt ?? q.created ?? "")).filter((t) => !Number.isNaN(t))60  const failed = queue.filter((q) => /"(status|@type)":"[^"]*(fail|error|reject)/i.test(JSON.stringify(q.recipients ?? q.status ?? {})) || Number(q.retryNum ?? q.attempts ?? 0) >= 3).length61  return { size: queue.length, oldest_minutes: created.length ? Math.round((now - Math.min(...created)) / 60_000) : 0, failures: failed }62}63async function queue(): Promise<Result> {64  let list: Record<string, any>[]65  try { list = await getAll("QueuedMessage") } catch { return { status: "unknown", detail: "queue unreadable (mail server unreachable)" } }66  const s = queueStats(list)67  const status: CheckStatus = s.failures > 0 || s.oldest_minutes > 60 ? "fail" : s.size > 50 ? "warn" : "ok"68  return { status, detail: `${s.size} queued, oldest ${s.oldest_minutes} min, ${s.failures} failing`, data: s }69}7071export const diskStatus = (freePct: number, warn = config.opsDiskWarnPct, fail = config.opsDiskFailPct): CheckStatus => (freePct < fail ? "fail" : freePct < warn ? "warn" : "ok")72async function disk(): Promise<Result> {73  const path = config.opsDiskPath || process.cwd()74  const s = await fs.promises.statfs(path)75  const total = s.blocks * s.bsize, free = s.bavail * s.bsize76  const pct = total ? (free / total) * 100 : 077  return { status: diskStatus(pct), detail: `${pct.toFixed(1)} % free (${(free / 1e9).toFixed(1)} GB of ${(total / 1e9).toFixed(1)} GB) on ${path}`, data: { path, free_pct: Number(pct.toFixed(2)), free_bytes: free, total_bytes: total } }78}7980/** État de la sauvegarde d'après son fichier d'état ({ at, ok, error }) : échec → fail ; plus vieille que le seuil → warn, deux fois le seuil → fail. */81export function backupStatus(s: { at?: string; ok?: boolean; error?: string } | null, maxAgeHours = config.opsBackupMaxAgeHours, now = Date.now()): Result {82  if (!s || !s.at || Number.isNaN(Date.parse(s.at))) return { status: "fail", detail: "backup status file is missing or unreadable", data: { readable: false } }83  const age = (now - Date.parse(s.at)) / 3600_00084  const data = { at: s.at, ok: s.ok === true, age_hours: Number(age.toFixed(1)) }85  if (s.ok !== true) return { status: "fail", detail: `last backup failed${s.error ? `: ${String(s.error).slice(0, 120)}` : ""}`, data }86  if (age > maxAgeHours) return { status: age > 2 * maxAgeHours ? "fail" : "warn", detail: `last successful backup is ${age.toFixed(0)} h old (limit ${maxAgeHours} h)`, data }87  return { status: "ok", detail: `last backup ${age.toFixed(1)} h ago`, data }88}89async function backups(): Promise<Result> {90  if (!config.backupStatusFile) return { status: "unknown", detail: "not configured (SPMAIL_BACKUP_STATUS)" }91  return backupStatus(await fs.promises.readFile(config.backupStatusFile, "utf8").then(JSON.parse).catch(() => null))92}9394// Certificats : origine web (TLS 443), MX (STARTTLS 25 — joignable seulement du côté serveur ; un échec de connexion = « unknown »), magasin de Stalwart.95function tlsExpiry(host: string, port: number): Promise<Date> {96  return new Promise((resolve, reject) => {97    const s = tls.connect({ host, port, servername: net.isIP(host) ? undefined : host, rejectUnauthorized: false, timeout: 5000 }, () => {98      const c = s.getPeerCertificate()99      s.destroy()100      c?.valid_to ? resolve(new Date(c.valid_to)) : reject(new Error("no certificate"))101    })102    s.on("timeout", () => { s.destroy(); reject(new Error("timeout")) })103    s.on("error", reject)104  })105}106function starttlsExpiry(host: string, port = 25): Promise<Date> {107  return new Promise((resolve, reject) => {108    const sock = net.connect({ host, port, timeout: 5000 })109    let step = 0, buf = ""110    const fail = (e: Error) => { sock.destroy(); reject(e) }111    sock.on("timeout", () => fail(new Error("timeout")))112    sock.on("error", reject)113    sock.on("data", (d) => {114      buf += d.toString("latin1")115      const lines = buf.split("\r\n")116      const last = lines[lines.length - 2] ?? ""117      if (!/^\d{3} /.test(last)) return // réponse multiligne incomplète118      buf = ""119      if (step === 0 && last.startsWith("220")) { step = 1; sock.write("EHLO spmail-monitor.invalid\r\n") }120      else if (step === 1 && last.startsWith("250")) { step = 2; sock.write("STARTTLS\r\n") }121      else if (step === 2 && last.startsWith("220")) {122        sock.removeAllListeners("data")123        const t = tls.connect({ socket: sock, servername: host, rejectUnauthorized: false }, () => {124          const c = t.getPeerCertificate()125          t.destroy()126          c?.valid_to ? resolve(new Date(c.valid_to)) : reject(new Error("no certificate"))127        })128        t.on("error", reject)129      } else fail(new Error(`unexpected SMTP answer ${last.slice(0, 3)}`))130    })131  })132}133export function certStatus(days: number, warnDays = config.opsCertWarnDays): CheckStatus { return days < 0 ? "fail" : days < warnDays ? "warn" : "ok" }134async function certificates(): Promise<Result> {135  const items: { name: string; status: CheckStatus; days: number | null; expires_at: string | null; detail?: string }[] = []136  const add = async (name: string, p: () => Promise<Date>) => {137    try {138      const d = await p()139      const days = Math.floor((d.getTime() - Date.now()) / 86400_000)140      items.push({ name, status: certStatus(days), days, expires_at: d.toISOString() })141    } catch (e) { items.push({ name, status: "unknown", days: null, expires_at: null, detail: String((e as Error).message).slice(0, 80) }) }142  }143  const origin = new URL(config.origin)144  const jobs: Promise<void>[] = [add(`smtp:${config.mxHost}`, () => starttlsExpiry(config.mxHost))]145  if (origin.protocol === "https:") jobs.push(add(`web:${origin.hostname}`, () => tlsExpiry(origin.hostname, Number(origin.port || 443))))146  else items.push({ name: `web:${origin.hostname}`, status: "unknown", days: null, expires_at: null, detail: "origin is not https" })147  await Promise.all(jobs)148  for (const c of await getAll("Certificate", ["subjectAlternativeNames", "notValidAfter"]).catch(() => [])) {149    if (!c.notValidAfter) continue150    const days = Math.floor((Date.parse(c.notValidAfter) - Date.now()) / 86400_000)151    items.push({ name: `stalwart:${Object.keys(c.subjectAlternativeNames ?? {}).join(",") || c.id}`, status: certStatus(days), days, expires_at: c.notValidAfter })152  }153  return { status: worst(items.map((i) => i.status)), detail: items.map((i) => `${i.name}: ${i.days === null ? `unknown (${i.detail})` : `${i.days} d`}`).join("; "), data: { certificates: items } }154}155156async function dns(): Promise<Result> {157  const reports = await Promise.all((await listDomains()).map((d) => dnsReport(d.name)))158  const bad = reports.flatMap((r) => (["mx", "spf", "dkim", "dmarc", "mxHost", "ptr"] as const).filter((k) => !r[k].ok).map((k) => `${r.domain}:${k}`))159  return { status: bad.length ? "warn" : "ok", detail: bad.length ? `not aligned: ${bad.join(", ")}` : `${reports.length} domain(s) aligned`, data: { domains: reports.length, problems: bad } }160}161162/** Nombre d'échecs consécutifs du veilleur sur une boîte avant de le déclarer bloqué (≈ 1 min à la cadence par défaut). */163const WATCHER_FAIL_AFTER = 12164async function workers(): Promise<Result> {165  if (!config.workers) return { status: "warn", detail: "workers are disabled (SPMAIL_WORKERS=off)", data: { enabled: false } }166  const now = Date.now()167  const list = [...beats.entries()].map(([name, b]) => ({ name, last_run_at: new Date(b.at).toISOString(), age_ms: now - b.at, stale: now - b.at > 3 * b.intervalMs + 30_000 }))168  const [q] = await sql`select (select count(*)::int from rule_jobs) as rule_jobs, (select extract(epoch from now() - min(created_at))::int from rule_jobs) as rule_oldest_s,169    (select count(*)::int from semantic_queue) as semantic_queue, (select count(*)::int from outbound_messages where status = 'queued' and created_at < now() - interval '10 minutes') as stuck_outbound,170    (select extract(epoch from now() - max(updated_at))::int from mailbox_sync) as watcher_idle_s`171  const stale = list.filter((w) => w.stale).map((w) => w.name)172  const lag = Number(q.rule_oldest_s ?? 0) > 600 || q.stuck_outbound > 0173  // Veilleur de réception : une boîte en échec répété ne produit plus ni push, ni règles, ni webhooks, ni contacts.174  const { watcherFailures } = await import("../workers/inbound.js")175  const stalled = [...watcherFailures.values()].filter((f) => f.failures >= WATCHER_FAIL_AFTER).map((f) => ({ address: f.address, failures: f.failures, since: new Date(f.since).toISOString(), error: f.lastError }))176  return {177    status: stalled.length ? "fail" : stale.length || lag ? "warn" : "ok",178    detail: `${list.length - stale.length}/${list.length} Next workers alive${stale.length ? ` (stale: ${stale.join(", ")})` : ""}; rules queue ${q.rule_jobs}, semantic queue ${q.semantic_queue}, stuck outbound ${q.stuck_outbound}${stalled.length ? `; inbound watcher failing for ${stalled.map((x) => x.address).join(", ")}: ${stalled[0].error}` : ""}`,179    // `watcher_failures` : toute boîte dont le dernier tour du veilleur a échoué (dès le premier échec) ; `watcher_stalled` : celles qui dépassent le seuil.180    data: { enabled: true, workers: list, queues: q, watcher_stalled: stalled, watcher_failures: [...watcherFailures.entries()].map(([mailbox_id, f]) => ({ mailbox_id, address: f.address, failures: f.failures, since: new Date(f.since).toISOString(), error: f.lastError })) },181  }182}183184async function webhooks(): Promise<Result> {185  const [w] = await sql`select count(*) filter (where status = 'failed' and created_at > now() - interval '24 hours')::int as failed_24h, count(*) filter (where status in ('pending','retrying'))::int as backlog from webhook_deliveries`186  return { status: w.failed_24h >= config.opsWebhookFailThreshold ? "fail" : w.failed_24h > 0 ? "warn" : "ok", detail: `${w.failed_24h} failed in 24 h, ${w.backlog} pending`, data: w as never }187}188189async function push(): Promise<Result> {190  const configured = !!(process.env.SPMAIL_APNS_KEY_FILE && process.env.SPMAIL_APNS_KEY_ID && process.env.SPMAIL_APNS_TEAM_ID)191  const [{ t }] = await sql`select to_regclass('public.push_outbox') as t`192  if (!t) return { status: "unknown", detail: "push pipeline not installed", data: { configured } }193  const [p] = await sql`select count(*) filter (where status = 'pending')::int as pending, count(*) filter (where status = 'failed' and created_at > now() - interval '24 hours')::int as failed_24h,194    count(*) filter (where status = 'sent' and created_at > now() - interval '24 hours')::int as sent_24h from push_outbox`195  if (!configured) return { status: "ok", detail: "not configured: push is disabled", data: { configured, ...p } }196  return { status: p.failed_24h > 0 && p.failed_24h >= p.sent_24h ? "warn" : "ok", detail: `${p.sent_24h} sent, ${p.failed_24h} failed in 24 h, ${p.pending} pending`, data: { configured, ...p } }197}198199const MIN = 60_000200async function health(fresh = false) {201  const [database, mail_server, q, d, certs, bk, dn, wk, wh, pu] = await Promise.all([202    check("database", 30_000, async () => { const t0 = Date.now(); await sql`select 1`; return { status: "ok", detail: `answered in ${Date.now() - t0} ms` } }, fresh),203    check("mail_server", 30_000, mailServer, fresh), check("queue", 30_000, queue, fresh), check("disk", MIN, disk, fresh), check("certificates", 30 * MIN, certificates, fresh),204    check("backups", MIN, backups, fresh), check("dns", 15 * MIN, dns, fresh), check("workers", 15_000, workers, fresh), check("webhooks", MIN, webhooks, fresh), check("push", MIN, push, fresh),205  ])206  const checks = { database, mail_server, queue: q, disk: d, certificates: certs, backups: bk, dns: dn, push: pu, workers: wk, webhooks: wh }207  return { status: worst(Object.values(checks).map((c) => c.status)), checks, generated_at: new Date().toISOString() }208}209210// ------------------------------------------------------------------ alertes : conditions, levée, résolution, notification211type Condition = { key: string; severity: "warn" | "fail"; title: string; detail: string }212/** Conditions actives et groupes réellement évalués à ce tour (un contrôle « unknown » ne lève ni ne résout rien). */213async function conditions(): Promise<{ active: Condition[]; evaluated: string[] }> {214  const active: Condition[] = [], evaluated: string[] = []215  const sev = (s: CheckStatus) => (s === "fail" ? "fail" : "warn") as "warn" | "fail"216217  const ms = await check("mail_server", 0, mailServer, true)218  if (ms.status !== "fail") evaluated.push("mail_server")219  else if (serverFailures >= 3) { evaluated.push("mail_server"); active.push({ key: "mail_server", severity: "fail", title: "Mail server unreachable", detail: ms.detail }) }220221  const q = await check("queue", 0, queue, true)222  if (q.status !== "unknown") {223    evaluated.push("queue")224    const s = q.data as ReturnType<typeof queueStats>225    if (s.failures > 0 || s.oldest_minutes > 60) active.push({ key: "queue", severity: "fail", title: "Outbound queue is failing or stuck", detail: q.detail })226  }227  const d = await check("disk", 0, disk, true)228  if (d.status !== "unknown") { evaluated.push("disk"); if (d.status !== "ok") active.push({ key: "disk", severity: sev(d.status), title: "Low disk space on the backend node", detail: d.detail }) }229230  const c = await check("certificates", 30 * MIN, certificates)231  for (const i of ((c.data?.certificates ?? []) as { name: string; status: CheckStatus; days: number | null }[])) {232    if (i.status === "unknown") continue233    evaluated.push(`cert:${i.name}`)234    if (i.status !== "ok") active.push({ key: `cert:${i.name}`, severity: sev(i.status), title: `Certificate ${i.name} ${i.days! < 0 ? "has expired" : "expires soon"}`, detail: `${i.name}: ${i.days} day(s) left (threshold ${config.opsCertWarnDays})` })235  }236  const b = await check("backups", 0, backups, true)237  if (b.status !== "unknown") { evaluated.push("backup"); if (b.status !== "ok") active.push({ key: "backup", severity: sev(b.status), title: "Backup failed or is too old", detail: b.detail }) }238239  const [a] = await sql`select count(*)::int as n from audit_log where action in ('login.failed', 'reauth.failed') and at > now() - interval '10 minutes'`240  evaluated.push("auth_anomaly")241  if (a.n >= 10) active.push({ key: "auth_anomaly", severity: "warn", title: "Authentication anomaly", detail: `${a.n} failed passkey verifications in the last 10 minutes` })242243  const [o] = await sql`select count(*)::int as n, count(*) filter (where status in ('bounced', 'failed'))::int as bad from outbound_messages244    where created_at > now() - interval '1 hour' and status in ('sent', 'delivered', 'deferred', 'bounced', 'failed')`245  evaluated.push("delivery")246  if (o.n >= 10 && o.bad / o.n >= 0.2) active.push({ key: "delivery", severity: "fail", title: "Delivery degradation", detail: `${o.bad} of ${o.n} outbound messages bounced or failed in the last hour` })247248  const wk = await check("workers", 0, workers, true)249  evaluated.push("watcher")250  const stalled = (wk.data?.watcher_stalled ?? []) as { address: string; error: string }[]251  if (stalled.length) active.push({ key: "watcher", severity: "fail", title: "Inbound watcher is failing", detail: `${stalled.map((x) => x.address).join(", ")}: ${stalled[0].error}` })252253  const w = await check("webhooks", 0, webhooks, true)254  evaluated.push("webhooks")255  if (w.status === "fail") active.push({ key: "webhooks", severity: "warn", title: "Webhook deliveries are failing", detail: w.detail })256  return { active, evaluated }257}258259type AlertRow = { id: string; key: string; severity: string; title: string; detail: string; state: string; first_seen: Date; last_seen: Date; count: number; notified_at: Date | null }260type PushApi = { enqueueUserPush?: (q: Sql, userId: string, n: { kind: "snooze" | "reminder" | "alert" | "test"; title: string; body: string; mailboxId?: string; emailId?: string; threadId?: string; dedup: string; category?: string; timeSensitive?: boolean }) => Promise<number> }261const pushApi = async (): Promise<PushApi> => { try { return (await import("./push.js")) as unknown as PushApi } catch { return {} } }262263/** Message RFC 5322 minimal de l'alerte (texte brut, UTF-8 en base64), marqué « auto-generated ». */264export function alertMessage(a: Pick<AlertRow, "id" | "key" | "severity" | "title" | "detail" | "first_seen">, address: string, now = new Date()): string {265  const enc = (s: string) => `=?UTF-8?B?${Buffer.from(s, "utf8").toString("base64")}?=`266  const body = [`${a.severity.toUpperCase()} — ${a.title}`, "", a.detail, "", `Condition: ${a.key}`, `First seen: ${new Date(a.first_seen).toISOString()}`, `Console: ${config.origin}/admin`, "",267    "This message was generated by the SP Mail monitor and delivered internally to your mailbox. It will not repeat more than once every 12 hours while the condition lasts."].join("\r\n")268  return [269    `From: SP Mail Monitor <${address}>`, `To: <${address}>`, `Subject: ${enc(`[SP Mail] ${a.severity === "fail" ? "ALERT" : "Warning"}: ${a.title}`)}`, `Date: ${now.toUTCString().replace("GMT", "+0000")}`,270    `Message-ID: <alert-${a.id}-${now.getTime()}@spmail.invalid>`, "MIME-Version: 1.0", "Content-Type: text/plain; charset=utf-8", "Content-Transfer-Encoding: base64", "Auto-Submitted: auto-generated",271    `X-SPMail-Alert: ${a.key.replace(/[^\x21-\x7e]/g, "_")}`, "", (Buffer.from(body, "utf8").toString("base64").match(/.{1,76}/g) ?? []).join("\r\n"), "",272  ].join("\r\n")273}274275/** Notifie les administrateurs : push « alert » et message INTERNE déposé dans leur boîte principale (Email/import — ni SMTP ni soumission). */276async function notify(a: AlertRow): Promise<{ admins: number; messages: number; pushes: number }> {277  const admins = await sql`select id from users where role = 'admin' and status = 'active'`278  const api = await pushApi()279  let messages = 0, pushes = 0280  for (const u of admins) {281    if (api.enqueueUserPush)282      pushes += await api.enqueueUserPush(sql, u.id, { kind: "alert", title: `SP Mail — ${a.title}`.slice(0, 120), body: a.detail.slice(0, 180), dedup: `alert:${a.id}:${Math.floor(Date.now() / 3600_000)}`, category: "MAIL_ALERT", timeSensitive: a.severity === "fail" }).catch(() => 0)283    const [mb] = (await sql`select id, user_id, address, display_name, stalwart_id, secret_enc from mailboxes where user_id = ${u.id} order by position, created_at limit 1`) as unknown as Mbx[]284    if (!mb) continue285    try {286      const inbox = await roleFolder(mb, "inbox")287      if (!inbox) continue288      const blob = await uploadBlob(mb, Buffer.from(alertMessage(a, mb.address), "utf8"), "message/rfc822")289      const [r] = await jmapOk(mb, [["Email/import", { emails: { a: { blobId: blob.blobId, mailboxIds: { [inbox]: true }, keywords: {}, receivedAt: new Date().toISOString().replace(/\.\d{3}Z$/, "Z") } } }]])290      if (r.created?.a) messages++291    } catch { /* serveur de courrier injoignable : le push reste le canal de secours */ }292  }293  return { admins: admins.length, messages, pushes }294}295296/** Un tour du moniteur : lève ou met à jour une alerte par condition active, résout celles dont la condition a disparu, notifie au plus une fois par 12 h. */297export async function monitorOnce() {298  const { active, evaluated } = await conditions()299  for (const c of active) {300    const [row] = (await sql`insert into alerts (key, severity, title, detail) values (${c.key}, ${c.severity}, ${c.title}, ${c.detail})301      on conflict (key) where state <> 'resolved' do update set last_seen = now(), count = alerts.count + 1, severity = excluded.severity, title = excluded.title, detail = excluded.detail302      returning *`) as unknown as AlertRow[]303    if (row.state !== "open") continue // acquittée : plus de rappel304    const [claimed] = await sql`update alerts set notified_at = now() where id = ${row.id} and state = 'open' and (notified_at is null or notified_at < now() - make_interval(hours => ${config.opsRenotifyHours})) returning id`305    if (!claimed) continue306    const n = await notify(row)307    // Rien n'a pu être remis (ni message ni push) : la notification sera retentée au prochain tour.308    if (n.admins && !n.messages && !n.pushes) await sql`update alerts set notified_at = ${row.notified_at} where id = ${row.id}`309    else await sql`insert into audit_log (action, target, meta) values ('alert.notified', ${row.key}, ${sql.json({ alert: row.id, severity: row.severity, ...n } as never)})`310  }311  const open = (await sql`select id, key from alerts where state <> 'resolved'`) as unknown as { id: string; key: string }[]312  const gone = open.filter((o) => evaluated.includes(o.key) && !active.some((c) => c.key === o.key))313  // Certificat disparu de la liste (renouvelé sous un autre nom) : son alerte se résout aussi dès que les certificats sont lisibles.314  const orphanCerts = open.filter((o) => o.key.startsWith("cert:") && !evaluated.includes(o.key) && evaluated.some((k) => k.startsWith("cert:")))315  for (const o of [...gone, ...orphanCerts]) {316    await sql`update alerts set state = 'resolved', resolved_at = now() where id = ${o.id} and state <> 'resolved'`317    await sql`insert into audit_log (action, target, meta) values ('alert.resolved', ${o.key}, ${sql.json({ alert: o.id } as never)})`318  }319}320321// ------------------------------------------------------------------ relevés quotidiens322/** Relevé du jour (UTC) par boîte : espace utilisé (Stalwart) et compteurs de réception / remise. Réécrit à chaque passage. */323export async function snapshotUsage(): Promise<number> {324  const accounts = await getAll("Account", ["usedDiskQuota"]).catch(() => [] as Record<string, any>[])325  const boxes = await sql`select id, stalwart_id from mailboxes`326  for (const b of boxes) {327    const used = accounts.find((x) => x.id === b.stalwart_id)?.usedDiskQuota ?? null328    await sql`insert into usage_snapshots (day, mailbox_id, used_bytes, received, sent, delivered, bounced, failed)329      select (now() at time zone 'utc')::date, ${b.id}, ${used},330        (select count(*)::int from watched_messages where mailbox_id = ${b.id} and (received_at at time zone 'utc')::date = (now() at time zone 'utc')::date),331        count(*) filter (where submitted_at is not null)::int, count(*) filter (where status = 'delivered')::int, count(*) filter (where status = 'bounced')::int, count(*) filter (where status = 'failed')::int332      from outbound_messages where mailbox_id = ${b.id} and (created_at at time zone 'utc')::date = (now() at time zone 'utc')::date333      on conflict (day, mailbox_id) do update set used_bytes = coalesce(excluded.used_bytes, usage_snapshots.used_bytes), received = excluded.received, sent = excluded.sent,334        delivered = excluded.delivered, bounced = excluded.bounced, failed = excluded.failed, updated_at = now()`335  }336  return boxes.length337}338339// ------------------------------------------------------------------ routes340const SECURITY_ACTIONS = ["login%", "reauth%", "enroll", "passkey.%", "session.%", "admin.session.%", "admin.push.%", "apikey.%", "approval.%", "mailbox.grants", "mailbox.purge", "mailbox.detach", "webhook.allowlist.%", "ai.consent%", "ai.provider", "alert.%", "retention.%"]341342export function opsRoutes(app: FastifyInstance) {343  app.get("/api/admin/health", async (req) => {344    await requireAdmin(req)345    const { fresh } = parse(z.object({ fresh: z.enum(["0", "1"]).default("0") }), req.query)346    return health(fresh === "1")347  })348349  app.get("/api/admin/alerts", async (req) => {350    await requireAdmin(req)351    const q = parse(z.object({ state: z.enum(["open", "all"]).default("open"), limit: z.coerce.number().int().min(1).max(500).default(200) }), req.query)352    const rows = q.state === "open"353      ? await sql`select a.*, u.handle as acked_by_handle from alerts a left join users u on u.id = a.acked_by where a.state <> 'resolved' order by a.first_seen desc limit ${q.limit}`354      : await sql`select a.*, u.handle as acked_by_handle from alerts a left join users u on u.id = a.acked_by order by a.last_seen desc limit ${q.limit}`355    return { data: rows }356  })357  // Acquitter : l'alerte reste visible tant que la condition dure, mais ne déclenche plus de rappel.358  app.post("/api/admin/alerts/:id/ack", async (req) => {359    const a = await requireAdmin(req)360    const { id } = parse(z.object({ id: zUuid }), req.params)361    const [row] = await sql`update alerts set state = 'acked', acked_by = ${a.userId}, acked_at = now() where id = ${id} and state = 'open' returning *`362    if (!row) {363      const [cur] = await sql`select state from alerts where id = ${id}`364      if (!cur) throw bad("not_found", 404)365      throw bad("not_open", 409, `This alert is already ${cur.state}`)366    }367    await audit(req, a.userId, "alert.ack", row.key, { alert: id })368    return row369  })370371  // Appareils de tous les utilisateurs : sessions actives et jetons de push (jamais le jeton APNs entier).372  app.get("/api/admin/devices", async (req) => {373    const a = await requireAdmin(req)374    const sessions = await sql`select s.id, s.user_id, u.handle, s.kind, s.device_name, s.platform, s.ip, s.user_agent, s.created_at, s.last_seen_at, s.expires_at, (s.id = ${a.sessionId}) as current375      from sessions s join users u on u.id = s.user_id where s.revoked_at is null and s.expires_at > now() order by s.last_seen_at desc limit 1000`376    const devices = await sql`select to_jsonb(p) - 'apns_token' as d, right(p.apns_token, 6) as token_tail, u.handle from push_devices p join users u on u.id = p.user_id order by p.created_at desc limit 1000`377    return { sessions, push_devices: devices.map((r) => ({ ...(r.d as object), handle: r.handle, token_tail: r.token_tail })) }378  })379  // Révoquer la session de quelqu'un : réauthentification exigée, consignée. Les jetons de push de la session partent avec elle.380  app.delete("/api/admin/devices/sessions/:id", async (req) => {381    const a = await requireAdminReauth(req)382    const { id } = parse(z.object({ id: zUuid }), req.params)383    const [s] = await sql`update sessions set revoked_at = now() where id = ${id} and revoked_at is null returning user_id, kind, device_name`384    if (!s) throw bad("not_found", 404)385    await sql`delete from push_devices where session_id = ${id}`386    await audit(req, a.userId, "admin.session.revoke", id, { user: s.user_id, kind: s.kind, device: s.device_name })387    return { ok: true }388  })389  app.delete("/api/admin/devices/push/:id", async (req) => {390    const a = await requireAdminReauth(req)391    const { id } = parse(z.object({ id: zUuid }), req.params)392    const [d] = await sql`delete from push_devices where id = ${id} returning user_id, bundle_id`393    if (!d) throw bad("not_found", 404)394    await audit(req, a.userId, "admin.push.remove", id, { user: d.user_id, bundle: d.bundle_id })395    return { ok: true }396  })397398  app.get("/api/admin/trends", async (req) => {399    await requireAdmin(req)400    const q = parse(z.object({ metric: z.enum(["storage", "deliveries", "api"]).default("storage"), days: z.coerce.number().int().min(1).max(365).default(30), refresh: z.enum(["0", "1"]).default("0") }), req.query)401    if (q.metric === "api") {402      const data = await sql`select to_char(date_trunc('day', at at time zone 'utc'), 'YYYY-MM-DD') as day, count(*)::int as requests, count(*) filter (where status >= 400)::int as errors,403        count(*) filter (where status = 429)::int as rate_limited, round(avg(duration_ms))::int as avg_ms from api_usage where at > now() - make_interval(days => ${q.days}) group by 1 order by 1`404      return { metric: q.metric, days: q.days, data }405    }406    // Le relevé du jour est rafraîchi s'il date de plus de dix minutes (ou sur demande, `refresh=1`) : la tendance inclut toujours aujourd'hui.407    const [last] = await sql`select max(updated_at) as at from usage_snapshots where day = (now() at time zone 'utc')::date`408    if (q.refresh === "1" || !last?.at || Date.now() - new Date(last.at).getTime() > 10 * MIN) await snapshotUsage()409    if (q.metric === "storage") {410      const rows = await sql`select to_char(s.day, 'YYYY-MM-DD') as day, s.mailbox_id, m.address, s.used_bytes from usage_snapshots s join mailboxes m on m.id = s.mailbox_id411        where s.day > (now() at time zone 'utc')::date - ${q.days}::int order by s.day, m.address`412      const days = [...new Set(rows.map((r) => r.day as string))]413      return { metric: q.metric, days: q.days, data: days.map((day) => { const l = rows.filter((r) => r.day === day); return { day, used_bytes: l.reduce((n, r) => n + Number(r.used_bytes ?? 0), 0), mailboxes: l.map((r) => ({ mailbox_id: r.mailbox_id, address: r.address, used_bytes: r.used_bytes })) } }) }414    }415    const data = await sql`select to_char(day, 'YYYY-MM-DD') as day, sum(received)::int as received, sum(sent)::int as sent, sum(delivered)::int as delivered, sum(bounced)::int as bounced, sum(failed)::int as failed416      from usage_snapshots where day > (now() at time zone 'utc')::date - ${q.days}::int group by 1 order by 1`417    return { metric: q.metric, days: q.days, data }418  })419420  // Événements de sécurité : la partie du journal d'audit qui touche l'authentification, les clés, les sessions, le consentement et les alertes.421  app.get("/api/admin/security-events", async (req) => {422    await requireAdmin(req)423    const q = parse(z.object({ limit: z.coerce.number().int().min(1).max(500).default(100), before: z.coerce.number().int().positive().optional(), action: z.string().max(60).regex(/^[a-z0-9_.-]+$/).optional(), hours: z.coerce.number().int().min(1).max(24 * 90).optional() }), req.query)424    const rows = await sql`select l.id, l.at, l.action, l.target, l.meta, l.ip, u.handle as actor from audit_log l left join users u on u.id = l.actor425      where l.action like any(${q.action ? [`${q.action}%`] : SECURITY_ACTIONS}) and (${q.before ?? null}::bigint is null or l.id < ${q.before ?? null})426        and (${q.hours ?? null}::int is null or l.at > now() - make_interval(hours => ${q.hours ?? 0})) order by l.id desc limit ${q.limit}`427    const [f] = await sql`select count(*) filter (where action = 'login.failed')::int as login_failed, count(*) filter (where action = 'reauth.failed')::int as reauth_failed from audit_log where at > now() - interval '24 hours'`428    return { data: rows, next_before: rows.length === q.limit ? rows[rows.length - 1].id : null, failed_24h: f }429  })430  opsConsoleRoutes(app)431}432433// ------------------------------------------------------------------ compléments de la console (additifs)434export function opsConsoleRoutes(app: FastifyInstance) {435  // Remises à problème récentes (différées, rebondies, échouées) : enveloppe et réponse SMTP du serveur distant — jamais le contenu.436  app.get("/api/admin/deliveries/problems", async (req) => {437    await requireAdmin(req)438    const q = parse(z.object({ limit: z.coerce.number().int().min(1).max(200).default(50), days: z.coerce.number().int().min(1).max(90).default(14) }), req.query)439    const rows = await sql`select id, from_address, recipients, status, attempts, last_error, external, created_at, updated_at, final_at from outbound_messages440      where status in ('deferred', 'bounced', 'failed') and updated_at > now() - make_interval(days => ${q.days}) order by updated_at desc limit ${q.limit}`441    return {442      data: rows.map((o) => ({443        id: o.id, from: o.from_address, status: o.status, attempts: o.attempts, last_error: o.last_error, external: o.external, created_at: o.created_at, updated_at: o.updated_at, final_at: o.final_at,444        recipients: ((o.recipients ?? []) as { address?: string; status?: string; smtp_reply?: string | null }[]).map((r) => ({ address: r.address ?? "", status: r.status ?? null, smtp_reply: r.smtp_reply ?? null })),445      })),446    }447  })448449  // État par boîte pour l'onglet « Boîtes » : espace utilisé, dernière réception, dernier envoi, veilleur de réception.450  app.get("/api/admin/mailboxes/status", async (req) => {451    await requireAdmin(req)452    const rows = await sql`select m.id as mailbox_id, m.address, m.stalwart_id, m.user_id, u.handle,453        (select s.used_bytes from usage_snapshots s where s.mailbox_id = m.id and s.used_bytes is not null order by s.day desc limit 1) as snapshot_bytes,454        (select max(w.received_at) from watched_messages w where w.mailbox_id = m.id) as last_received_at,455        (select max(o.submitted_at) from outbound_messages o where o.mailbox_id = m.id) as last_sent_at,456        ms.updated_at as watcher_updated_at,457        (select count(*)::int from sessions s where s.user_id = m.user_id and s.revoked_at is null and s.expires_at > now()) as owner_sessions458      from mailboxes m join users u on u.id = m.user_id left join mailbox_sync ms on ms.mailbox_id = m.id order by m.address`459    const accounts = await getAll("Account", ["usedDiskQuota"]).catch(() => [] as Record<string, any>[])460    const { watcherFailures } = await import("../workers/inbound.js")461    return {462      workers: config.workers,463      data: rows.map((r) => {464        const f = watcherFailures.get(r.mailbox_id)465        const state = !config.workers ? "off" : f && f.failures >= WATCHER_FAIL_AFTER ? "stalled" : f ? "failing" : r.watcher_updated_at ? "ok" : "unknown"466        return {467          mailbox_id: r.mailbox_id, address: r.address, user_id: r.user_id, handle: r.handle, owner_sessions: r.owner_sessions,468          used_bytes: accounts.find((x) => x.id === r.stalwart_id)?.usedDiskQuota ?? r.snapshot_bytes ?? null,469          last_received_at: r.last_received_at, last_sent_at: r.last_sent_at,470          watcher: { state, updated_at: r.watcher_updated_at, failures: f?.failures ?? 0, since: f ? new Date(f.since).toISOString() : null, error: f?.lastError ?? null },471        }472      }),473    }474  })475}476477export function opsWorkers(): NextWorker[] {478  return [479    ...(config.opsMonitor ? [{ name: "ops monitor", intervalMs: config.opsMonitorIntervalMs, run: async () => { await monitorOnce(); beat("ops monitor", config.opsMonitorIntervalMs) } }] : []),480    { name: "usage snapshot", intervalMs: 3600_000, run: async () => { await snapshotUsage(); beat("usage snapshot", 3600_000) } },481  ]482}483
Files main
Menu

Theme