SPB Git

Repository Public

server.go 12.6 KB 433 lines go

1// Package ingest is the telemetry-ingest service: agent enrollment, mTLS2// report intake, transport of signed commands, rollups and retention.3//4// Trust model: the service authenticates agents (client certificates from the5// private CA, checked against the agents table on every request) but holds no6// command-signing key. It transports command envelopes byte for byte; an7// attacker controlling this process cannot make an agent execute anything.8package ingest910import (11	"context"12	"crypto/tls"13	"crypto/x509"14	"encoding/json"15	"errors"16	"fmt"17	"log/slog"18	"net"19	"net/http"20	"net/netip"21	"os"22	"sync"23	"sync/atomic"24	"time"2526	"github.com/google/uuid"27	"github.com/jackc/pgx/v5"28	"github.com/jackc/pgx/v5/pgconn"29	"github.com/jackc/pgx/v5/pgxpool"3031	"clustr.spboucher.ai/mcc/internal/config"32	"clustr.spboucher.ai/mcc/internal/httpx"33	"clustr.spboucher.ai/mcc/internal/pki"34)3536// Options configures a Server.37type Options struct {38	Cfg     *config.Config39	Pool    *pgxpool.Pool40	CA      *pki.CA41	Version string42	// Enrollment rate limit per client IP. Zero values use 6/min, burst 6.43	EnrollPerMinute, EnrollBurst int44	// AuthCacheTTL bounds how long an agent lookup is reused (max 5 s).45	AuthCacheTTL time.Duration46	// ReportInterval handed to agents (seconds). Zero uses 10.47	ReportIntervalS int48	Log             *slog.Logger49}5051// Server is one telemetry-ingest instance.52type Server struct {53	cfg     *config.Config54	pool    *pgxpool.Pool55	ca      *pki.CA56	version string57	log     *slog.Logger58	started time.Time59	inst    string6061	enrollLimiter *httpx.Limiter62	intervalS     int63	sigNonces     *nonceCache64	sigFailures   *failureLimiter6566	authTTL time.Duration67	authMu  sync.Mutex68	auth    map[string]authEntry6970	certMu     sync.Mutex71	serverCert *tls.Certificate7273	wake *waker // command notifications per agent7475	seenMu sync.Mutex76	seen   map[uuid.UUID]time.Time // last authenticated request per agent7778	nasMu      sync.Mutex79	nasTargets []string80	nasAt      time.Time8182	longPolls   atomic.Int6483	lastRollup  atomic.Pointer[RollupStatus]84	dbOK        atomic.Bool85	closing     chan struct{}86	closeOnce   sync.Once87	reqTimeout  time.Duration88	maxLongPoll time.Duration89}9091type authEntry struct {92	agent *agentIdentity // nil: unknown or revoked93	at    time.Time94}9596type agentIdentity struct {97	ID       uuid.UUID98	NodeID   uuid.UUID99	NotAfter time.Time100}101102// New builds a server. It performs no I/O.103func New(o Options) (*Server, error) {104	if o.Cfg == nil || o.Pool == nil || o.CA == nil {105		return nil, errors.New("ingest: config, pool and CA are required")106	}107	if o.EnrollPerMinute <= 0 {108		o.EnrollPerMinute = 6109	}110	if o.EnrollBurst <= 0 {111		o.EnrollBurst = 6112	}113	if o.AuthCacheTTL <= 0 || o.AuthCacheTTL > 5*time.Second {114		o.AuthCacheTTL = 3 * time.Second115	}116	if o.ReportIntervalS <= 0 {117		o.ReportIntervalS = 10118	}119	if o.Log == nil {120		o.Log = slog.Default()121	}122	host, _ := os.Hostname()123	s := &Server{124		cfg: o.Cfg, pool: o.Pool, ca: o.CA, version: o.Version, log: o.Log,125		started: time.Now().UTC(), inst: fmt.Sprintf("%s:%d", host, os.Getpid()),126		enrollLimiter: httpx.NewLimiter(o.EnrollPerMinute, o.EnrollBurst),127		intervalS:     o.ReportIntervalS,128		authTTL:       o.AuthCacheTTL, auth: map[string]authEntry{},129		sigNonces: newNonceCache(), sigFailures: newFailureLimiter(30, time.Minute),130		wake: newWaker(), seen: map[uuid.UUID]time.Time{},131		closing: make(chan struct{}), reqTimeout: 15 * time.Second, maxLongPoll: 55 * time.Second,132	}133	s.dbOK.Store(true)134	return s, nil135}136137// TLSConfig is the listener configuration: TLS 1.3 only, client certificates138// verified against the private CA when presented (enrollment has none yet).139func (s *Server) TLSConfig() *tls.Config {140	return &tls.Config{141		MinVersion: tls.VersionTLS13,142		ClientAuth: tls.VerifyClientCertIfGiven,143		ClientCAs:  s.ca.Pool(),144		GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {145			return s.currentServerCert()146		},147		NextProtos: []string{"http/1.1"},148	}149}150151// currentServerCert re-issues the short-lived server certificate when less152// than two thirds of its life remain.153func (s *Server) currentServerCert() (*tls.Certificate, error) {154	s.certMu.Lock()155	defer s.certMu.Unlock()156	now := time.Now()157	if s.serverCert != nil && s.serverCert.Leaf.NotAfter.Sub(now) > pki.ServerCertTTL*2/3 {158		return s.serverCert, nil159	}160	c, err := s.ca.IssueServer(s.cfg.IngestServerName, now)161	if err != nil {162		if s.serverCert != nil && now.Before(s.serverCert.Leaf.NotAfter) {163			return s.serverCert, nil164		}165		return nil, err166	}167	s.serverCert = c168	return c, nil169}170171// Handler returns the agent-facing HTTP handler of the mTLS transport (to be172// served over TLS). Agents are authenticated by their client certificate.173func (s *Server) Handler() http.Handler {174	// No proxy is ever trusted here: the client IP is the TCP peer.175	return httpx.WithRequestContext(nil, httpx.Recover(s.routes(s.withAgent)))176}177178// routes builds the /agent/v1 mux; auth wraps every route except enrollment.179// Both transports share these handlers: only authentication differs.180func (s *Server) routes(auth func(http.HandlerFunc) http.HandlerFunc) http.Handler {181	mux := http.NewServeMux()182	mux.HandleFunc("POST /agent/v1/enroll", s.handleEnroll)183	mux.HandleFunc("POST /agent/v1/renew", auth(s.handleRenew))184	mux.HandleFunc("POST /agent/v1/report", auth(s.handleReport))185	mux.HandleFunc("GET /agent/v1/commands", auth(s.handleCommands))186	mux.HandleFunc("POST /agent/v1/commands/{id}/ack", auth(s.handleCommandAck))187	mux.HandleFunc("POST /agent/v1/commands/{id}/events", auth(s.handleCommandEvents))188	mux.HandleFunc("POST /agent/v1/commands/{id}/result", auth(s.handleCommandResult))189	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { httpx.WriteError(w, r, httpx.ErrNotFound) })190	return mux191}192193// Serve accepts agent connections on ln (plain TCP; TLS is added here) until194// ctx ends, then shuts down gracefully.195func (s *Server) Serve(ctx context.Context, ln net.Listener) error {196	srv := &http.Server{197		Handler:           s.Handler(),198		TLSConfig:         s.TLSConfig(),199		ReadHeaderTimeout: 10 * time.Second,200		ReadTimeout:       30 * time.Second,201		WriteTimeout:      s.maxLongPoll + 20*time.Second,202		IdleTimeout:       120 * time.Second,203		MaxHeaderBytes:    16 << 10,204		ErrorLog:          slog.NewLogLogger(s.log.Handler(), slog.LevelDebug),205	}206	errc := make(chan error, 1)207	go func() { errc <- srv.Serve(tls.NewListener(ln, srv.TLSConfig)) }()208	select {209	case err := <-errc:210		return err211	case <-ctx.Done():212	}213	s.closeOnce.Do(func() { close(s.closing) }) // releases long-polls214	sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)215	defer cancel()216	if err := srv.Shutdown(sctx); err != nil {217		srv.Close()218	}219	return nil220}221222// ─────────────────────────── authentication ───────────────────────────223224type agentCtxKey struct{}225226// authed is the authenticated caller of a request, whatever the transport.227type authed struct {228	agent *agentIdentity229	leaf  *x509.Certificate // the certificate that authenticated the request230}231232func agentFrom(ctx context.Context) *agentIdentity {233	a, _ := ctx.Value(agentCtxKey{}).(*authed)234	if a == nil {235		return nil236	}237	return a.agent238}239240func leafFrom(ctx context.Context) *x509.Certificate {241	a, _ := ctx.Value(agentCtxKey{}).(*authed)242	if a == nil {243		return nil244	}245	return a.leaf246}247248func (s *Server) markSeen(id uuid.UUID) {249	s.seenMu.Lock()250	s.seen[id] = time.Now()251	s.seenMu.Unlock()252}253254var (255	errNoClientCert = httpx.Err(http.StatusUnauthorized, "client_certificate_required", "A client certificate issued by the agent CA is required.")256	errAgentRevoked = httpx.Err(http.StatusUnauthorized, "agent_revoked", "This agent certificate is unknown or revoked. Enroll again.")257	errUnavailable  = &httpx.Error{Status: http.StatusServiceUnavailable, Code: "unavailable", Message: "The service is temporarily unavailable.", RetryAfter: 5}258)259260// withAgent requires a verified client certificate whose serial belongs to a261// non-revoked agent. The lookup is cached for a few seconds at most, so a262// revocation takes effect within that delay.263func (s *Server) withAgent(next http.HandlerFunc) http.HandlerFunc {264	return func(w http.ResponseWriter, r *http.Request) {265		if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 || len(r.TLS.PeerCertificates) == 0 {266			httpx.WriteError(w, r, errNoClientCert)267			return268		}269		leaf := r.TLS.PeerCertificates[0]270		id, err := pki.AgentIDFromCert(leaf)271		if err != nil {272			httpx.WriteError(w, r, errNoClientCert)273			return274		}275		ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)276		a, err := s.lookupAgent(ctx, leaf)277		cancel()278		if err != nil {279			s.writeErr(w, r, err)280			return281		}282		if a == nil || a.ID != id {283			httpx.WriteError(w, r, errAgentRevoked)284			return285		}286		s.markSeen(a.ID)287		next(w, r.WithContext(context.WithValue(r.Context(), agentCtxKey{}, &authed{agent: a, leaf: leaf})))288	}289}290291func (s *Server) lookupAgent(ctx context.Context, leaf *x509.Certificate) (*agentIdentity, error) {292	serial := pki.SerialString(leaf.SerialNumber)293	now := time.Now()294	s.authMu.Lock()295	if e, ok := s.auth[serial]; ok && now.Sub(e.at) < s.authTTL {296		s.authMu.Unlock()297		return e.agent, nil298	}299	s.authMu.Unlock()300301	var a agentIdentity302	err := s.pool.QueryRow(ctx, `select id, node_id, cert_not_after from agents303		where revoked_at is null304		  and (cert_serial = $1 or (prev_cert_serial = $1 and prev_cert_valid_until > now()))`, serial).305		Scan(&a.ID, &a.NodeID, &a.NotAfter)306	var res *agentIdentity307	switch {308	case err == nil:309		res = &a310	case errors.Is(err, pgx.ErrNoRows):311	default:312		return nil, err // never cached: a database outage must not look like a revocation313	}314	s.authMu.Lock()315	if len(s.auth) > 4096 {316		for k, e := range s.auth {317			if now.Sub(e.at) >= s.authTTL {318				delete(s.auth, k)319			}320		}321	}322	s.auth[serial] = authEntry{agent: res, at: now}323	s.authMu.Unlock()324	return res, nil325}326327// forgetAuth drops cached lookups (after enrollment or renewal changed them).328func (s *Server) forgetAuth() {329	s.authMu.Lock()330	s.auth = map[string]authEntry{}331	s.authMu.Unlock()332}333334// connectedAgents counts agents heard from within the offline threshold.335func (s *Server) connectedAgents() int {336	cut := time.Now().Add(-s.cfg.OfflineAfter)337	s.seenMu.Lock()338	defer s.seenMu.Unlock()339	n := 0340	for id, t := range s.seen {341		if t.Before(cut) {342			delete(s.seen, id)343			continue344		}345		n++346	}347	return n348}349350// ─────────────────────────── errors ───────────────────────────351352// dbUnavailable reports whether err means "the database cannot serve us now"353// (as opposed to a statement that is wrong and will stay wrong).354func dbUnavailable(err error) bool {355	if err == nil {356		return false357	}358	var pg *pgconn.PgError359	if errors.As(err, &pg) {360		switch pg.Code[:2] {361		case "08", "53", "57", "58": // connection, resources, operator intervention, system error362			return true363		}364		switch pg.Code {365		case "40001", "40P01", "55P03": // serialization failure, deadlock, lock not available366			return true367		}368		return false369	}370	// Everything that is not a server-reported SQL error: dial failures,371	// broken connections, pool acquisition timeouts.372	return true373}374375// writeErr renders err; database outages become 503 so agents buffer and retry.376func (s *Server) writeErr(w http.ResponseWriter, r *http.Request, err error) {377	var he *httpx.Error378	if errors.As(err, &he) {379		httpx.WriteError(w, r, he)380		return381	}382	var pg *pgconn.PgError383	if errors.As(err, &pg) && pg.Code == "23505" {384		httpx.WriteError(w, r, httpx.Conflict("conflict", "The request conflicts with a concurrent change; retry."))385		return386	}387	if dbUnavailable(err) {388		s.dbOK.Store(false)389		s.log.Warn("ingest: database unavailable", "err", err, "path", r.URL.Path)390		httpx.WriteError(w, r, errUnavailable)391		return392	}393	httpx.WriteError(w, r, err)394}395396func clientIP(r *http.Request) netip.Addr {397	if ip, ok := httpx.ClientIP(r.Context()); ok {398		return ip399	}400	if ap, err := netip.ParseAddrPort(r.RemoteAddr); err == nil {401		return ap.Addr().Unmap()402	}403	return netip.Addr{}404}405406func writeJSON(w http.ResponseWriter, status int, v any) { httpx.WriteJSON(w, status, v) }407408func mustJSON(v any) []byte {409	b, err := json.Marshal(v)410	if err != nil {411		panic(err)412	}413	return b414}415416// inTx runs fn in a transaction, committing on success.417func (s *Server) inTx(ctx context.Context, fn func(pgx.Tx) error) error {418	tx, err := s.pool.Begin(ctx)419	if err != nil {420		return err421	}422	defer tx.Rollback(context.WithoutCancel(ctx))423	if err := fn(tx); err != nil {424		return err425	}426	return tx.Commit(ctx)427}428429type pgconnTag = pgconn.CommandTag430431// ActiveLongPolls is the number of command long-polls currently held open.432func (s *Server) ActiveLongPolls() int { return int(s.longPolls.Load()) }433
Files main
Menu

Theme