Repository Public
1// Package ingest is the telemetry-ingest service: agent enrollment, mTLS2// report intake, transport of signed commands, rollups and retention.3//4// Trust model: the service authenticates agents (client certificates from the5// private CA, checked against the agents table on every request) but holds no6// command-signing key. It transports command envelopes byte for byte; an7// attacker controlling this process cannot make an agent execute anything.8package ingest910import (11 "context"12 "crypto/tls"13 "crypto/x509"14 "encoding/json"15 "errors"16 "fmt"17 "log/slog"18 "net"19 "net/http"20 "net/netip"21 "os"22 "sync"23 "sync/atomic"24 "time"2526 "github.com/google/uuid"27 "github.com/jackc/pgx/v5"28 "github.com/jackc/pgx/v5/pgconn"29 "github.com/jackc/pgx/v5/pgxpool"3031 "clustr.spboucher.ai/mcc/internal/config"32 "clustr.spboucher.ai/mcc/internal/httpx"33 "clustr.spboucher.ai/mcc/internal/pki"34)3536// Options configures a Server.37type Options struct {38 Cfg *config.Config39 Pool *pgxpool.Pool40 CA *pki.CA41 Version string42 // Enrollment rate limit per client IP. Zero values use 6/min, burst 6.43 EnrollPerMinute, EnrollBurst int44 // AuthCacheTTL bounds how long an agent lookup is reused (max 5 s).45 AuthCacheTTL time.Duration46 // ReportInterval handed to agents (seconds). Zero uses 10.47 ReportIntervalS int48 Log *slog.Logger49}5051// Server is one telemetry-ingest instance.52type Server struct {53 cfg *config.Config54 pool *pgxpool.Pool55 ca *pki.CA56 version string57 log *slog.Logger58 started time.Time59 inst string6061 enrollLimiter *httpx.Limiter62 intervalS int63 sigNonces *nonceCache64 sigFailures *failureLimiter6566 authTTL time.Duration67 authMu sync.Mutex68 auth map[string]authEntry6970 certMu sync.Mutex71 serverCert *tls.Certificate7273 wake *waker // command notifications per agent7475 seenMu sync.Mutex76 seen map[uuid.UUID]time.Time // last authenticated request per agent7778 nasMu sync.Mutex79 nasTargets []string80 nasAt time.Time8182 longPolls atomic.Int6483 lastRollup atomic.Pointer[RollupStatus]84 dbOK atomic.Bool85 closing chan struct{}86 closeOnce sync.Once87 reqTimeout time.Duration88 maxLongPoll time.Duration89}9091type authEntry struct {92 agent *agentIdentity // nil: unknown or revoked93 at time.Time94}9596type agentIdentity struct {97 ID uuid.UUID98 NodeID uuid.UUID99 NotAfter time.Time100}101102// New builds a server. It performs no I/O.103func New(o Options) (*Server, error) {104 if o.Cfg == nil || o.Pool == nil || o.CA == nil {105 return nil, errors.New("ingest: config, pool and CA are required")106 }107 if o.EnrollPerMinute <= 0 {108 o.EnrollPerMinute = 6109 }110 if o.EnrollBurst <= 0 {111 o.EnrollBurst = 6112 }113 if o.AuthCacheTTL <= 0 || o.AuthCacheTTL > 5*time.Second {114 o.AuthCacheTTL = 3 * time.Second115 }116 if o.ReportIntervalS <= 0 {117 o.ReportIntervalS = 10118 }119 if o.Log == nil {120 o.Log = slog.Default()121 }122 host, _ := os.Hostname()123 s := &Server{124 cfg: o.Cfg, pool: o.Pool, ca: o.CA, version: o.Version, log: o.Log,125 started: time.Now().UTC(), inst: fmt.Sprintf("%s:%d", host, os.Getpid()),126 enrollLimiter: httpx.NewLimiter(o.EnrollPerMinute, o.EnrollBurst),127 intervalS: o.ReportIntervalS,128 authTTL: o.AuthCacheTTL, auth: map[string]authEntry{},129 sigNonces: newNonceCache(), sigFailures: newFailureLimiter(30, time.Minute),130 wake: newWaker(), seen: map[uuid.UUID]time.Time{},131 closing: make(chan struct{}), reqTimeout: 15 * time.Second, maxLongPoll: 55 * time.Second,132 }133 s.dbOK.Store(true)134 return s, nil135}136137// TLSConfig is the listener configuration: TLS 1.3 only, client certificates138// verified against the private CA when presented (enrollment has none yet).139func (s *Server) TLSConfig() *tls.Config {140 return &tls.Config{141 MinVersion: tls.VersionTLS13,142 ClientAuth: tls.VerifyClientCertIfGiven,143 ClientCAs: s.ca.Pool(),144 GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {145 return s.currentServerCert()146 },147 NextProtos: []string{"http/1.1"},148 }149}150151// currentServerCert re-issues the short-lived server certificate when less152// than two thirds of its life remain.153func (s *Server) currentServerCert() (*tls.Certificate, error) {154 s.certMu.Lock()155 defer s.certMu.Unlock()156 now := time.Now()157 if s.serverCert != nil && s.serverCert.Leaf.NotAfter.Sub(now) > pki.ServerCertTTL*2/3 {158 return s.serverCert, nil159 }160 c, err := s.ca.IssueServer(s.cfg.IngestServerName, now)161 if err != nil {162 if s.serverCert != nil && now.Before(s.serverCert.Leaf.NotAfter) {163 return s.serverCert, nil164 }165 return nil, err166 }167 s.serverCert = c168 return c, nil169}170171// Handler returns the agent-facing HTTP handler of the mTLS transport (to be172// served over TLS). Agents are authenticated by their client certificate.173func (s *Server) Handler() http.Handler {174 // No proxy is ever trusted here: the client IP is the TCP peer.175 return httpx.WithRequestContext(nil, httpx.Recover(s.routes(s.withAgent)))176}177178// routes builds the /agent/v1 mux; auth wraps every route except enrollment.179// Both transports share these handlers: only authentication differs.180func (s *Server) routes(auth func(http.HandlerFunc) http.HandlerFunc) http.Handler {181 mux := http.NewServeMux()182 mux.HandleFunc("POST /agent/v1/enroll", s.handleEnroll)183 mux.HandleFunc("POST /agent/v1/renew", auth(s.handleRenew))184 mux.HandleFunc("POST /agent/v1/report", auth(s.handleReport))185 mux.HandleFunc("GET /agent/v1/commands", auth(s.handleCommands))186 mux.HandleFunc("POST /agent/v1/commands/{id}/ack", auth(s.handleCommandAck))187 mux.HandleFunc("POST /agent/v1/commands/{id}/events", auth(s.handleCommandEvents))188 mux.HandleFunc("POST /agent/v1/commands/{id}/result", auth(s.handleCommandResult))189 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { httpx.WriteError(w, r, httpx.ErrNotFound) })190 return mux191}192193// Serve accepts agent connections on ln (plain TCP; TLS is added here) until194// ctx ends, then shuts down gracefully.195func (s *Server) Serve(ctx context.Context, ln net.Listener) error {196 srv := &http.Server{197 Handler: s.Handler(),198 TLSConfig: s.TLSConfig(),199 ReadHeaderTimeout: 10 * time.Second,200 ReadTimeout: 30 * time.Second,201 WriteTimeout: s.maxLongPoll + 20*time.Second,202 IdleTimeout: 120 * time.Second,203 MaxHeaderBytes: 16 << 10,204 ErrorLog: slog.NewLogLogger(s.log.Handler(), slog.LevelDebug),205 }206 errc := make(chan error, 1)207 go func() { errc <- srv.Serve(tls.NewListener(ln, srv.TLSConfig)) }()208 select {209 case err := <-errc:210 return err211 case <-ctx.Done():212 }213 s.closeOnce.Do(func() { close(s.closing) }) // releases long-polls214 sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)215 defer cancel()216 if err := srv.Shutdown(sctx); err != nil {217 srv.Close()218 }219 return nil220}221222// ─────────────────────────── authentication ───────────────────────────223224type agentCtxKey struct{}225226// authed is the authenticated caller of a request, whatever the transport.227type authed struct {228 agent *agentIdentity229 leaf *x509.Certificate // the certificate that authenticated the request230}231232func agentFrom(ctx context.Context) *agentIdentity {233 a, _ := ctx.Value(agentCtxKey{}).(*authed)234 if a == nil {235 return nil236 }237 return a.agent238}239240func leafFrom(ctx context.Context) *x509.Certificate {241 a, _ := ctx.Value(agentCtxKey{}).(*authed)242 if a == nil {243 return nil244 }245 return a.leaf246}247248func (s *Server) markSeen(id uuid.UUID) {249 s.seenMu.Lock()250 s.seen[id] = time.Now()251 s.seenMu.Unlock()252}253254var (255 errNoClientCert = httpx.Err(http.StatusUnauthorized, "client_certificate_required", "A client certificate issued by the agent CA is required.")256 errAgentRevoked = httpx.Err(http.StatusUnauthorized, "agent_revoked", "This agent certificate is unknown or revoked. Enroll again.")257 errUnavailable = &httpx.Error{Status: http.StatusServiceUnavailable, Code: "unavailable", Message: "The service is temporarily unavailable.", RetryAfter: 5}258)259260// withAgent requires a verified client certificate whose serial belongs to a261// non-revoked agent. The lookup is cached for a few seconds at most, so a262// revocation takes effect within that delay.263func (s *Server) withAgent(next http.HandlerFunc) http.HandlerFunc {264 return func(w http.ResponseWriter, r *http.Request) {265 if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 || len(r.TLS.PeerCertificates) == 0 {266 httpx.WriteError(w, r, errNoClientCert)267 return268 }269 leaf := r.TLS.PeerCertificates[0]270 id, err := pki.AgentIDFromCert(leaf)271 if err != nil {272 httpx.WriteError(w, r, errNoClientCert)273 return274 }275 ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)276 a, err := s.lookupAgent(ctx, leaf)277 cancel()278 if err != nil {279 s.writeErr(w, r, err)280 return281 }282 if a == nil || a.ID != id {283 httpx.WriteError(w, r, errAgentRevoked)284 return285 }286 s.markSeen(a.ID)287 next(w, r.WithContext(context.WithValue(r.Context(), agentCtxKey{}, &authed{agent: a, leaf: leaf})))288 }289}290291func (s *Server) lookupAgent(ctx context.Context, leaf *x509.Certificate) (*agentIdentity, error) {292 serial := pki.SerialString(leaf.SerialNumber)293 now := time.Now()294 s.authMu.Lock()295 if e, ok := s.auth[serial]; ok && now.Sub(e.at) < s.authTTL {296 s.authMu.Unlock()297 return e.agent, nil298 }299 s.authMu.Unlock()300301 var a agentIdentity302 err := s.pool.QueryRow(ctx, `select id, node_id, cert_not_after from agents303 where revoked_at is null304 and (cert_serial = $1 or (prev_cert_serial = $1 and prev_cert_valid_until > now()))`, serial).305 Scan(&a.ID, &a.NodeID, &a.NotAfter)306 var res *agentIdentity307 switch {308 case err == nil:309 res = &a310 case errors.Is(err, pgx.ErrNoRows):311 default:312 return nil, err // never cached: a database outage must not look like a revocation313 }314 s.authMu.Lock()315 if len(s.auth) > 4096 {316 for k, e := range s.auth {317 if now.Sub(e.at) >= s.authTTL {318 delete(s.auth, k)319 }320 }321 }322 s.auth[serial] = authEntry{agent: res, at: now}323 s.authMu.Unlock()324 return res, nil325}326327// forgetAuth drops cached lookups (after enrollment or renewal changed them).328func (s *Server) forgetAuth() {329 s.authMu.Lock()330 s.auth = map[string]authEntry{}331 s.authMu.Unlock()332}333334// connectedAgents counts agents heard from within the offline threshold.335func (s *Server) connectedAgents() int {336 cut := time.Now().Add(-s.cfg.OfflineAfter)337 s.seenMu.Lock()338 defer s.seenMu.Unlock()339 n := 0340 for id, t := range s.seen {341 if t.Before(cut) {342 delete(s.seen, id)343 continue344 }345 n++346 }347 return n348}349350// ─────────────────────────── errors ───────────────────────────351352// dbUnavailable reports whether err means "the database cannot serve us now"353// (as opposed to a statement that is wrong and will stay wrong).354func dbUnavailable(err error) bool {355 if err == nil {356 return false357 }358 var pg *pgconn.PgError359 if errors.As(err, &pg) {360 switch pg.Code[:2] {361 case "08", "53", "57", "58": // connection, resources, operator intervention, system error362 return true363 }364 switch pg.Code {365 case "40001", "40P01", "55P03": // serialization failure, deadlock, lock not available366 return true367 }368 return false369 }370 // Everything that is not a server-reported SQL error: dial failures,371 // broken connections, pool acquisition timeouts.372 return true373}374375// writeErr renders err; database outages become 503 so agents buffer and retry.376func (s *Server) writeErr(w http.ResponseWriter, r *http.Request, err error) {377 var he *httpx.Error378 if errors.As(err, &he) {379 httpx.WriteError(w, r, he)380 return381 }382 var pg *pgconn.PgError383 if errors.As(err, &pg) && pg.Code == "23505" {384 httpx.WriteError(w, r, httpx.Conflict("conflict", "The request conflicts with a concurrent change; retry."))385 return386 }387 if dbUnavailable(err) {388 s.dbOK.Store(false)389 s.log.Warn("ingest: database unavailable", "err", err, "path", r.URL.Path)390 httpx.WriteError(w, r, errUnavailable)391 return392 }393 httpx.WriteError(w, r, err)394}395396func clientIP(r *http.Request) netip.Addr {397 if ip, ok := httpx.ClientIP(r.Context()); ok {398 return ip399 }400 if ap, err := netip.ParseAddrPort(r.RemoteAddr); err == nil {401 return ap.Addr().Unmap()402 }403 return netip.Addr{}404}405406func writeJSON(w http.ResponseWriter, status int, v any) { httpx.WriteJSON(w, status, v) }407408func mustJSON(v any) []byte {409 b, err := json.Marshal(v)410 if err != nil {411 panic(err)412 }413 return b414}415416// inTx runs fn in a transaction, committing on success.417func (s *Server) inTx(ctx context.Context, fn func(pgx.Tx) error) error {418 tx, err := s.pool.Begin(ctx)419 if err != nil {420 return err421 }422 defer tx.Rollback(context.WithoutCancel(ctx))423 if err := fn(tx); err != nil {424 return err425 }426 return tx.Commit(ctx)427}428429type pgconnTag = pgconn.CommandTag430431// ActiveLongPolls is the number of command long-polls currently held open.432func (s *Server) ActiveLongPolls() int { return int(s.longPolls.Load()) }433