SPB Git

Repository Public

integration_test.go 40.2 KB 903 lines go

1package ingest_test23import (4	"context"5	"crypto/ed25519"6	"crypto/rand"7	"encoding/json"8	"errors"9	"fmt"10	"log/slog"11	"math"12	"net/http"13	"net/http/httptest"14	"os"15	"strings"16	"sync"17	"sync/atomic"18	"testing"19	"time"2021	"github.com/google/uuid"2223	"clustr.spboucher.ai/mcc/internal/agent"24	"clustr.spboucher.ai/mcc/internal/agent/client"25	"clustr.spboucher.ai/mcc/internal/agent/state"26	"clustr.spboucher.ai/mcc/internal/pki"27	"clustr.spboucher.ai/mcc/internal/protocol"28)2930func TestEnrollment(t *testing.T) {31	forModes(t, func(t *testing.T, mode string) {32		h := newHarness(t, mode, hopt{})33		fp := h.fingerprint()3435		// Success: identity stored 0600 in a 0700 directory, node created with the real inventory, audit written.36		tok := h.token("alpha", 15*time.Minute)37		e, err := h.enrollErr("alpha", newHW(), tok, fp)38		if err != nil {39			t.Fatal(err)40		}41		for _, f := range []string{"agent.key", "agent.crt", "ca.crt", "identity.json", "command-keys.json"} {42			st, err := os.Stat(e.dir.File(f))43			if err != nil || st.Mode().Perm() != 0o600 {44				t.Errorf("%s: %v mode %v", f, err, st)45			}46		}47		if st, _ := os.Stat(e.dir.Path); st.Mode().Perm() != 0o700 {48			t.Errorf("state dir mode %o", st.Mode().Perm())49		}50		if got := scan[string](h, `select alias::text from nodes where id=$1 and hardware_uuid=$2 and registered_via='agent' and model='TestMac1,1' and jsonb_array_length(cpu_levels)=3`, e.id.NodeID, e.hw); got != "alpha" {51			t.Errorf("node alias %q", got)52		}53		if n := scan[int](h, `select count(*) from audit_log where action='agent.enrolled' and actor_kind='agent' and outcome='success' and target_id=$1`, e.id.NodeID); n != 1 {54			t.Errorf("%d audit entries", n)55		}56		keys, _ := e.dir.LoadCommandKeys()57		if len(keys) != 1 || keys[h.keyID] == nil {58			t.Errorf("command keys %v", keys)59		}60		cert, _, _ := e.dir.LoadCredentials()61		if id, err := pki.AgentIDFromCert(cert.Leaf); err != nil || id.String() != e.id.AgentID || len(cert.Leaf.ExtKeyUsage) != 1 || time.Until(cert.Leaf.NotAfter) > 31*24*time.Hour || time.Until(cert.Leaf.NotAfter) < 29*24*time.Hour {62			t.Errorf("certificate: %v %v", err, cert.Leaf.NotAfter)63		}6465		// Token reuse refused.66		if _, err := h.enrollErr("alpha", newHW(), tok, fp); status(err) != 401 || code(err) != "invalid_enrollment_token" {67			t.Errorf("reused token: %v", err)68		}69		// Expired token refused.70		expired := h.token("beta", time.Minute)71		h.exec(`update agent_enrollment_tokens set expires_at = now() - interval '1 second' where node_alias='beta'`)72		if _, err := h.enrollErr("beta", newHW(), expired, fp); status(err) != 401 {73			t.Errorf("expired token: %v", err)74		}75		// Revoked and malformed tokens refused.76		revoked := h.token("beta2", time.Minute)77		h.exec(`update agent_enrollment_tokens set revoked_at = now() where node_alias='beta2'`)78		if _, err := h.enrollErr("beta2", newHW(), revoked, fp); status(err) != 401 {79			t.Errorf("revoked token: %v", err)80		}81		if _, err := h.enrollErr("x", newHW(), "short", fp); status(err) != 401 {82			t.Errorf("malformed token: %v", err)83		}84		if n := scan[int](h, `select count(*) from nodes`); n != 1 {85			t.Errorf("refused enrollments created nodes: %d", n)86		}8788		// Concurrent double use: exactly one success.89		race := h.token("gamma", 15*time.Minute)90		var ok, refused atomic.Int3291		var wg sync.WaitGroup92		for i := 0; i < 8; i++ {93			wg.Add(1)94			go func() {95				defer wg.Done()96				if _, err := h.enrollErr("gamma", newHW(), race, fp); err == nil {97					ok.Add(1)98				} else if status(err) == 401 {99					refused.Add(1)100				}101			}()102		}103		wg.Wait()104		if ok.Load() != 1 || refused.Load() != 7 {105			t.Errorf("concurrent use: %d successes, %d refusals", ok.Load(), refused.Load())106		}107		if n := scan[int](h, `select count(*) from agents a join nodes n on n.id=a.node_id where n.alias='gamma'`); n != 1 {108			t.Errorf("%d agents for gamma", n)109		}110111		// Wrong CA fingerprint: refused, and the state directory holds no identity.112		bad := make([]byte, 32)113		wtok := h.token("delta", 15*time.Minute)114		we, err := h.enrollErr("delta", newHW(), wtok, bad)115		if err == nil {116			t.Fatal("wrong CA fingerprint accepted")117		}118		if _, lerr := we.dir.LoadIdentity(); !errors.Is(lerr, state.ErrNotEnrolled) {119			t.Errorf("identity written despite the pin failure: %v", lerr)120		}121		if _, _, lerr := we.dir.LoadCredentials(); !errors.Is(lerr, state.ErrNotEnrolled) {122			t.Errorf("credentials written despite the pin failure: %v", lerr)123		}124		if mode == "mtls" && !errors.Is(err, client.ErrPin) {125			t.Errorf("mtls: expected the handshake pin to fail, got %v", err)126		}127128		// A manually registered node (hardware_uuid NULL) is adopted.129		h.exec(`insert into nodes(id, alias, registered_via, site) values (gen_random_uuid(), 'manual-1', 'manual', 'lab')`)130		m := h.enroll("manual-1")131		if got := scan[string](h, `select registered_via || '/' || site from nodes where id=$1 and hardware_uuid=$2`, m.id.NodeID, m.hw); got != "manual/lab" {132			t.Errorf("adoption: %q", got)133		}134		// Same hardware, token for another alias: existing node and alias kept, reported in the audit; previous agent revoked.135		again, err := h.enrollErr("other-alias", m.hw, h.token("other-alias", time.Minute), fp)136		if err != nil {137			t.Fatal(err)138		}139		if again.id.NodeID != m.id.NodeID || scan[int](h, `select count(*) from nodes where alias='other-alias'`) != 0 {140			t.Error("hardware match must keep the existing node and alias")141		}142		if d := scan[string](h, `select detail from audit_log where action='agent.enrolled' order by seq desc limit 1`); !strings.Contains(d, "other-alias") || !strings.Contains(d, "manual-1") || !strings.Contains(d, "revoked") {143			t.Errorf("audit detail %q", d)144		}145		if r := scan[string](h, `select revoked_reason from agents where id=$1`, m.id.AgentID); r != "re-enrolled" {146			t.Errorf("previous agent: %q", r)147		}148		if _, err := h.send(m, report(1, time.Now())); status(err) != 401 && err == nil {149			t.Errorf("revoked previous identity still accepted: %v", err)150		}151		// An alias owned by other hardware is refused and the token is not burnt.152		tk := h.token("alpha", time.Minute)153		if _, err := h.enrollErr("alpha", newHW(), tk, fp); status(err) != 409 {154			t.Errorf("alias in use: %v", err)155		}156		if c := scan[bool](h, `select consumed_at is null from agent_enrollment_tokens where node_alias='alpha' order by created_at desc limit 1`); !c {157			t.Error("token consumed by a refused enrollment")158		}159	})160}161162func TestReportRoundtripAndIdempotence(t *testing.T) {163	forModes(t, func(t *testing.T, mode string) {164		h := newHarness(t, mode, hopt{})165		e := h.enroll("n1")166		now := time.Now().UTC().Truncate(time.Millisecond)167		rep := report(1, now, sample("cpu.usage", 12.5), sample("mem.used", 4e9),168			protocol.Sample{M: "net.rx", L: "en0", V: 100}, protocol.Sample{M: "net.rx", L: "utun1", V: 5})169		rep.Availability = map[string]string{"thermal.temp": "unsupported", "gpu.usage": "unavailable"}170		rep.Processes = []protocol.Process{{PID: 10, Name: "proc", User: "u", CPU: 3.5, RSS: 1 << 20}}171		rep.Capabilities = []string{"diagnostic.run"}172		rep.Volumes = []protocol.Volume{{Mount: "/", Device: "/dev/disk3s1", FS: "apfs", Kind: "internal", Total: 100, Used: 40, SMART: "verified"}}173		events0 := scan[int](h, `select count(*) from events_outbox where topic='node.state'`)174		ack, err := h.send(e, rep)175		if err != nil || ack.Seq != 1 || ack.IntervalS != 10 {176			t.Fatalf("%+v %v", ack, err)177		}178		count := func() int { return scan[int](h, `select count(*) from samples_raw where node_id=$1`, e.id.NodeID) }179		if count() != 4 {180			t.Fatalf("%d samples", count())181		}182		if v := scan[float64](h, `select ("values"->>'cpu.usage')::float8 from node_state where node_id=$1`, e.id.NodeID); v != 12.5 {183			t.Errorf("node_state cpu %v", v)184		}185		if v := scan[float64](h, `select ("values"->'net.rx'->>'utun1')::float8 from node_state where node_id=$1`, e.id.NodeID); v != 5 {186			t.Errorf("labeled value %v", v)187		}188		if a := scan[string](h, `select availability->>'thermal.temp' from node_state where node_id=$1`, e.id.NodeID); a != "unsupported" {189			t.Errorf("availability %q", a)190		}191		if n := scan[int](h, `select count(*) from events_outbox where topic='node.state'`) - events0; n != 1 {192			t.Errorf("%d node.state events for one report", n)193		}194		if got := scan[string](h, `select version || '/' || array_to_string(capabilities, ',') || '/' || last_seq from agents where id=$1`, e.id.AgentID); got != "test/diagnostic.run/1" {195			t.Errorf("agent row %q", got)196		}197		if scan[int](h, `select count(*) from volumes where node_id=$1 and mount='/' and smart_status='verified'`, e.id.NodeID) != 1 {198			t.Error("volume not stored")199		}200201		// Replay of the same seq: acknowledged, nothing re-inserted, no second event.202		ack, err = h.send(e, rep)203		if err != nil || ack.Seq != 1 {204			t.Fatalf("replay must be acknowledged: %v", err)205		}206		rep2 := *rep207		rep2.Samples = []protocol.Sample{sample("cpu.usage", 99), sample("load.1", 7)}208		if _, err := h.send(e, &rep2); err != nil { // same seq, different content: still a duplicate209			t.Fatal(err)210		}211		if count() != 4 || scan[int](h, `select count(*) from events_outbox where topic='node.state'`)-events0 != 1 {212			t.Errorf("duplicate seq was applied again: %d samples", count())213		}214215		// A newer report carrying a subset keeps the other values and drops the ones declared unavailable.216		rep3 := report(2, now.Add(10*time.Second), sample("cpu.usage", 20))217		rep3.Availability = map[string]string{"mem.used": "unavailable"}218		if _, err := h.send(e, rep3); err != nil {219			t.Fatal(err)220		}221		if got := scan[string](h, `select ("values"->>'cpu.usage') || '|' || coalesce("values"->>'mem.used','gone') || '|' || ("values"->'net.rx'->>'en0') from node_state where node_id=$1`, e.id.NodeID); got != "20|gone|100" {222			t.Errorf("merge: %q", got)223		}224225		// A buffered (older) report adds samples but never regresses live state.226		old := report(3, now.Add(-time.Hour), sample("cpu.usage", 77))227		old.Buffered = true228		old.Services = &protocol.ServiceSet{Managers: []string{"pm2"}, Items: []protocol.Service{{Manager: "pm2", Name: "stale", State: "online"}}}229		if _, err := h.send(e, old); err != nil {230			t.Fatal(err)231		}232		// Also an out-of-order report that is not flagged buffered.233		if _, err := h.send(e, report(4, now.Add(-30*time.Minute), sample("cpu.usage", 66))); err != nil {234			t.Fatal(err)235		}236		if v := scan[float64](h, `select ("values"->>'cpu.usage')::float8 from node_state where node_id=$1`, e.id.NodeID); v != 20 {237			t.Errorf("old report regressed node_state: cpu=%v", v)238		}239		if ts := scan[time.Time](h, `select ts from node_state where node_id=$1`, e.id.NodeID); !ts.Equal(now.Add(10 * time.Second)) {240			t.Errorf("node_state.ts regressed to %v", ts)241		}242		if scan[int](h, `select count(*) from services where node_id=$1`, e.id.NodeID) != 0 {243			t.Error("a buffered report changed service state")244		}245		if count() != 7 || scan[int64](h, `select last_seq from agents where id=$1`, e.id.AgentID) != 4 {246			t.Errorf("samples %d", count())247		}248	})249}250251func TestReportValidation(t *testing.T) {252	forModes(t, func(t *testing.T, mode string) {253		h := newHarness(t, mode, hopt{})254		e := h.enroll("n1")255		now := time.Now().UTC()256		ts := now.Format(time.RFC3339Nano)257		many := func(metric string, n int) *protocol.Report {258			r := report(50, now)259			for i := 0; i < n; i++ {260				r.Samples = append(r.Samples, protocol.Sample{M: metric, L: fmt.Sprintf("if%d", i), V: 1})261			}262			return r263		}264		type bad struct {265			name string266			rep  *protocol.Report267			raw  string268			want int269		}270		long := report(50, now, protocol.Sample{M: "net.rx", L: strings.Repeat("x", protocol.MaxLabelLen+1), V: 1})271		procs := report(50, now)272		for i := 0; i < protocol.MaxProcesses+1; i++ {273			procs.Processes = append(procs.Processes, protocol.Process{PID: int32(i), Name: "p"})274		}275		svc := report(50, now)276		svc.Services = &protocol.ServiceSet{Managers: []string{"pm2"}, Items: []protocol.Service{{Manager: "launchd", Name: "x", State: "online"}}}277		creds := report(50, now)278		creds.Volumes = []protocol.Volume{{Mount: "/Volumes/nas", Kind: "network", Remote: "//user:pw@192.0.2.9/share"}}279		cases := []bad{280			{name: "unknown metric", rep: report(50, now, sample("cpu.magic", 1)), want: 400},281			{name: "NaN", raw: `{"v":1,"seq":50,"ts":"` + ts + `","samples":[{"m":"cpu.usage","v":NaN}]}`, want: 400},282			{name: "infinite", raw: `{"v":1,"seq":50,"ts":"` + ts + `","samples":[{"m":"cpu.usage","v":1e999}]}`, want: 400},283			{name: "too many interface labels", rep: many("net.rx", protocol.MaxInterfaces+1), want: 400},284			{name: "too many volume labels", rep: many("disk.used", protocol.MaxVolumes+1), want: 400},285			{name: "label too long", rep: long, want: 400},286			{name: "label on unlabeled metric", rep: report(50, now, protocol.Sample{M: "cpu.usage", L: "x", V: 1}), want: 400},287			{name: "missing label", rep: report(50, now, sample("net.rx", 1)), want: 400},288			{name: "duplicate sample", rep: report(50, now, sample("cpu.usage", 1), sample("cpu.usage", 2)), want: 400},289			{name: "future timestamp", rep: report(50, now.Add(6*time.Minute), sample("cpu.usage", 1)), want: 400},290			{name: "older than retention", rep: report(50, now.Add(-8*24*time.Hour), sample("cpu.usage", 1)), want: 400},291			{name: "bad version", raw: `{"v":2,"seq":50,"ts":"` + ts + `","samples":[]}`, want: 400},292			{name: "seq zero", rep: report(0, now, sample("cpu.usage", 1)), want: 400},293			{name: "too many processes", rep: procs, want: 400},294			{name: "service of unlisted manager", rep: svc, want: 400},295			{name: "credentials in remote", rep: creds, want: 400},296			{name: "bad availability state", raw: `{"v":1,"seq":50,"ts":"` + ts + `","samples":[],"availability":{"cpu.usage":"broken"}}`, want: 400},297			{name: "not json", raw: `hello`, want: 400},298		}299		for _, c := range cases {300			var err error301			if c.rep != nil {302				_, err = h.send(e, c.rep)303			} else {304				err = h.sendRaw(e, c.raw)305			}306			if status(err) != c.want {307				t.Errorf("%s: got %v, want HTTP %d", c.name, err, c.want)308			}309			if !client.Permanent(err) {310				t.Errorf("%s: the agent must treat this as permanent (drop, do not retry)", c.name)311			}312		}313		// Oversize: 1.2 MiB of incompressible data.314		big := make([]byte, 1200<<10)315		rand.Read(big)316		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)317		_, err := e.cl.Report(ctx, big)318		cancel()319		if status(err) != 413 {320			t.Errorf("oversize: %v", err)321		}322		// A gzip bomb is cut at the decompressed limit.323		if err := h.sendRaw(e, `{"v":1,"seq":50,"ts":"`+ts+`","samples":[],"agent_version":"`+strings.Repeat("a", 5<<20)+`"}`); status(err) != 413 {324			t.Errorf("gzip bomb: %v", err)325		}326		if n := scan[int](h, `select (select count(*) from samples_raw) + (select count(*) from agent_report_seqs) + (select count(*) from node_state)`); n != 0 {327			t.Errorf("rejected reports wrote %d rows", n)328		}329		// Sanity: math.NaN cannot even be encoded by the agent.330		if _, err := json.Marshal(sample("cpu.usage", math.NaN())); err == nil {331			t.Error("NaN encodable")332		}333		// Accepted: maximum cardinality, unknown fields (forward compatibility).334		if _, err := h.send(e, many("net.rx", protocol.MaxInterfaces)); err != nil {335			t.Errorf("max interfaces refused: %v", err)336		}337		if err := h.sendRaw(e, `{"v":1,"seq":51,"ts":"`+ts+`","samples":[],"future_field":{"x":1}}`); err != nil {338			t.Errorf("unknown field must be ignored within v1: %v", err)339		}340	})341}342343func TestRevocationAndForeignCA(t *testing.T) {344	forModes(t, func(t *testing.T, mode string) {345		h := newHarness(t, mode, hopt{})346		e := h.enroll("n1")347		if _, err := h.send(e, report(1, time.Now(), sample("cpu.usage", 1))); err != nil {348			t.Fatal(err)349		}350		// A long-poll is open when the agent gets revoked.351		pollErr := make(chan error, 1)352		go func() {353			_, err := e.cl.Commands(context.Background(), 20*time.Second)354			pollErr <- err355		}()356		eventually(t, 3*time.Second, "long-poll open", func() bool { return h.srv.ActiveLongPolls() == 1 })357		revokedAt := time.Now()358		h.exec(`update agents set revoked_at = now(), revoked_reason = 'test' where id=$1`, e.id.AgentID)359		eventually(t, 5*time.Second, "revoked agent refused", func() bool {360			_, err := h.send(e, report(2, time.Now(), sample("cpu.usage", 1)))361			return status(err) == 401 && code(err) == "agent_revoked"362		})363		select {364		case err := <-pollErr:365			if status(err) != 401 {366				t.Errorf("long-poll of a revoked agent ended with %v", err)367			}368		case <-time.After(6 * time.Second):369			t.Error("long-poll of a revoked agent stayed open")370		}371		if d := time.Since(revokedAt); d > 6*time.Second {372			t.Errorf("revocation took %s", d)373		}374		if scan[int](h, `select count(*) from samples_raw`) != 1 {375			t.Error("a revoked agent wrote samples")376		}377378		// An identity issued by another CA (another Command Center) is refused.379		other := newHarness(t, mode, hopt{})380		foreign := other.enroll("stranger")381		// Point the foreign identity at our server.382		id, _ := foreign.dir.LoadIdentity()383		id.Endpoints = []string{h.endpoint}384		foreign.dir.SaveIdentity(id)385		if mode == "mtls" {386			// Trust our CA so the handshake reaches client authentication.387			os.WriteFile(foreign.dir.File("ca.crt"), h.ca.PEM, 0o600)388		}389		fcl, err := client.New(foreign.dir)390		if err != nil {391			t.Fatal(err)392		}393		defer fcl.Close()394		enc, _ := client.EncodeReport(report(1, time.Now(), sample("cpu.usage", 1)))395		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)396		_, err = fcl.Report(ctx, enc)397		cancel()398		switch mode {399		case "mtls":400			if err == nil || status(err) != 0 {401				t.Errorf("mtls: a certificate from another CA must fail the TLS handshake, got %v", err)402			}403		default:404			if status(err) != 401 || code(err) != "certificate_invalid" {405				t.Errorf("signed: %v", err)406			}407		}408		if scan[int](h, `select count(*) from samples_raw`) != 1 {409			t.Error("a foreign identity wrote samples")410		}411		// No certificate at all on the mTLS port.412		if mode == "mtls" {413			resp, err := insecureGet(h.mtlsAddr, "/agent/v1/commands")414			if err != nil || resp != 401 {415				t.Errorf("no client certificate: %d %v", resp, err)416			}417		}418	})419}420421func TestRenewal(t *testing.T) {422	forModes(t, func(t *testing.T, mode string) {423		h := newHarness(t, mode, hopt{})424		e := h.enroll("n1")425		before := scan[string](h, `select cert_serial from agents where id=$1`, e.id.AgentID)426		// A second client still holding the old certificate (as if the renewal response had been lost).427		oldCl, err := client.New(e.dir)428		if err != nil {429			t.Fatal(err)430		}431		defer oldCl.Close()432		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)433		defer cancel()434		if err := e.cl.Renew(ctx); err != nil {435			t.Fatal(err)436		}437		after := scan[string](h, `select cert_serial from agents where id=$1`, e.id.AgentID)438		if after == before || scan[string](h, `select prev_cert_serial from agents where id=$1`, e.id.AgentID) != before {439			t.Fatalf("serial not rotated: %s -> %s", before, after)440		}441		if _, err := h.send(e, report(1, time.Now(), sample("cpu.usage", 1))); err != nil {442			t.Fatalf("new certificate refused: %v", err)443		}444		enc, _ := client.EncodeReport(report(2, time.Now(), sample("cpu.usage", 1)))445		if _, err := oldCl.Report(ctx, enc); err != nil {446			t.Errorf("previous certificate must stay valid during the overlap: %v", err)447		}448		h.exec(`update agents set prev_cert_valid_until = now() - interval '1 second' where id=$1`, e.id.AgentID)449		eventually(t, 5*time.Second, "old certificate refused after the overlap", func() bool {450			enc, _ := client.EncodeReport(report(3, time.Now(), sample("cpu.usage", 1)))451			_, err := oldCl.Report(ctx, enc)452			return status(err) == 401453		})454		if scan[int](h, `select count(*) from audit_log where action='agent.cert_renewed'`) != 1 {455			t.Error("renewal not audited")456		}457	})458}459460func TestServiceGoneSemanticsAndEvents(t *testing.T) {461	forModes(t, func(t *testing.T, mode string) {462		h := newHarness(t, mode, hopt{})463		e := h.enroll("n1")464		now := time.Now()465		seq := int64(0)466		push := func(managers []string, items ...protocol.Service) {467			seq++468			r := report(seq, now.Add(time.Duration(seq)*time.Second), sample("cpu.usage", 1))469			r.Services = &protocol.ServiceSet{Managers: managers, Items: items}470			if _, err := h.send(e, r); err != nil {471				t.Fatal(err)472			}473		}474		st := func(mgr, name string) string {475			return scan[string](h, `select coalesce((select state || case when gone_at is null then '' else '/gone' end from services where node_id=$1 and manager::text=$2 and name=$3), 'absent')`, e.id.NodeID, mgr, name)476		}477		ev := func() int { return scan[int](h, `select count(*) from events_outbox where topic='service.changed'`) }478479		push([]string{"pm2", "launchd"},480			protocol.Service{Manager: "pm2", Name: "api", State: "online", PID: 10, Ports: []int{8080}, RawState: "online"},481			protocol.Service{Manager: "pm2", Name: "worker", State: "stopped"},482			protocol.Service{Manager: "launchd", Name: "io.x", State: "online", Scope: "user"})483		if st("pm2", "api") != "online" || st("launchd", "io.x") != "online" || ev() != 3 {484			t.Fatalf("initial: %s %s events=%d", st("pm2", "api"), st("launchd", "io.x"), ev())485		}486		// Same observation again: no event.487		push([]string{"pm2", "launchd"},488			protocol.Service{Manager: "pm2", Name: "api", State: "online", PID: 10},489			protocol.Service{Manager: "pm2", Name: "worker", State: "stopped"},490			protocol.Service{Manager: "launchd", Name: "io.x", State: "online"})491		if ev() != 3 {492			t.Errorf("unchanged services produced events: %d", ev())493		}494		// pm2 observed with zero services; launchd NOT observed: its services are untouched.495		push([]string{"pm2"})496		if st("pm2", "api") != "online/gone" || st("pm2", "worker") != "stopped/gone" || st("launchd", "io.x") != "online" {497			t.Errorf("gone handling: api=%s worker=%s launchd=%s", st("pm2", "api"), st("pm2", "worker"), st("launchd", "io.x"))498		}499		if ev() != 5 {500			t.Errorf("events after gone: %d", ev())501		}502		// No manager observed at all (adapters failed): nothing changes.503		push([]string{})504		if st("launchd", "io.x") != "online" || ev() != 5 {505			t.Error("an empty observation changed services")506		}507		// The service comes back, in another state.508		push([]string{"pm2"}, protocol.Service{Manager: "pm2", Name: "api", State: "errored"})509		if st("pm2", "api") != "errored" || st("pm2", "worker") != "stopped/gone" || ev() != 6 {510			t.Errorf("return: %s events=%d", st("pm2", "api"), ev())511		}512		// State change only.513		push([]string{"pm2"}, protocol.Service{Manager: "pm2", Name: "api", State: "online"})514		if got := scan[string](h, `select payload->>'previous_state' || '>' || (payload->>'state') from events_outbox where topic='service.changed' order by seq desc limit 1`); got != "errored>online" || ev() != 7 {515			t.Errorf("state change event %q, %d events", got, ev())516		}517		// Inference and volumes follow the same complete-observation rule.518		seq++519		r := report(seq, now.Add(time.Minute), sample("cpu.usage", 1))520		r.Inference = &protocol.InferenceSet{Items: []protocol.InferenceServer{{Kind: "mlx_lm", Port: 8081, Model: "org/model", PID: 5, Health: "ok", HealthMS: 3}}}521		r.Volumes = []protocol.Volume{{Mount: "/", Kind: "internal", Total: 10, Used: 5}, {Mount: "/Volumes/ssd", Kind: "external", Total: 10, Used: 1}}522		h.send(e, r)523		seq++524		r = report(seq, now.Add(2*time.Minute), sample("cpu.usage", 1))525		r.Inference = &protocol.InferenceSet{}526		r.Volumes = []protocol.Volume{{Mount: "/", Kind: "internal", Total: 10, Used: 6}}527		h.send(e, r)528		if got := scan[string](h, `select (select count(*) from inference_servers where gone_at is not null)::text || '/' || (select count(*) from volumes where gone_at is not null and mount='/Volumes/ssd')::text || '/' || (select used_bytes from volumes where mount='/')::text`); got != "1/1/6" {529			t.Errorf("inference/volumes gone handling: %s", got)530		}531		// NAS: only registered targets are recorded; state aggregates observations.532		h.exec(`insert into nas_targets(id, name, host) values (gen_random_uuid(), 'nas-a', '192.0.2.50')`)533		seq++534		r = report(seq, now.Add(3*time.Minute), sample("cpu.usage", 1))535		r.NAS = []protocol.NASProbe{{Host: "192.0.2.50", Reachable: false}, {Host: "192.0.2.99", Reachable: true}}536		h.send(e, r)537		if got := scan[string](h, `select state || '/' || mounted_on from nas_targets where name='nas-a'`); got != "offline/0" {538			t.Errorf("nas: %s", got)539		}540		seq++541		r = report(seq, now.Add(4*time.Minute), sample("cpu.usage", 1))542		r.NAS = []protocol.NASProbe{{Host: "192.0.2.50", Reachable: true, Mounted: true, LatencyMS: 1.5}}543		h.send(e, r)544		if got := scan[string](h, `select state || '/' || mounted_on from nas_targets where name='nas-a'`); got != "online/1" {545			t.Errorf("nas: %s", got)546		}547	})548}549550// startAgent runs the real agent runtime against the harness.551func (h *harness) startAgent(e *enrolled) (stop func() int) {552	a, err := agent.New(agent.Options{Dir: e.dir, Version: "test", Interval: 400 * time.Millisecond, PollWait: 2 * time.Second,553		Log: slog.New(slog.DiscardHandler)})554	if err != nil {555		h.t.Fatal(err)556	}557	ctx, cancel := context.WithCancel(context.Background())558	done := make(chan int, 1)559	go func() { done <- a.Run(ctx) }()560	var once sync.Once561	code := 0562	stop = func() int {563		once.Do(func() {564			cancel()565			select {566			case code = <-done:567			case <-time.After(30 * time.Second):568				h.t.Error("agent did not stop")569			}570		})571		return code572	}573	h.t.Cleanup(func() { stop() })574	return stop575}576577func TestCommandsEndToEnd(t *testing.T) {578	forModes(t, func(t *testing.T, mode string) {579		h := newHarness(t, mode, hopt{})580		e := h.enroll("n1")581		stop := h.startAgent(e)582		eventually(t, 15*time.Second, "first report of the real agent", func() bool {583			return scan[int](h, `select count(*) from node_state where node_id=$1`, e.id.NodeID) == 1584		})585586		// Accepted and executed: a real read-only diagnostic on this machine.587		id := h.command(e, "diagnostic.run", map[string]any{"profile": "basic"}, cmdOpt{})588		r := h.waitCmd(id, "finished", 40*time.Second)589		if r.AckAccepted == nil || !*r.AckAccepted || r.ResultStatus == nil || *r.ResultStatus != "succeeded" {590			t.Fatalf("diagnostic: %+v", r)591		}592		lines := scan[int](h, `select count(*) from action_events e join agent_commands c on c.action_id=e.action_id where c.id=$1 and e.kind='output'`, id)593		if lines < 5 {594			t.Errorf("only %d output lines stored", lines)595		}596		if scan[int](h, `select count(*) from events_outbox where topic='action.output'`) == 0 {597			t.Error("no action.output event")598		}599600		// Compromised transport: none of these may execute.601		_, otherKey, _ := ed25519.GenerateKey(rand.Reader)602		outputBefore := scan[int](h, `select count(*) from action_events`)603		refused := map[string]struct {604			id     string605			reason string606		}{}607		add := func(name, reason string, o cmdOpt, typ string, params any) {608			cid := h.command(e, typ, params, o)609			refused[name] = struct{ id, reason string }{cid.String(), reason}610		}611		add("forged signature (key the agent does not hold)", "bad signature", cmdOpt{key: otherKey, keyID: h.keyID}, "diagnostic.run", nil)612		add("unknown key id", "unknown signing key", cmdOpt{key: otherKey, keyID: "attacker"}, "diagnostic.run", nil)613		add("tampered payload", "bad signature", cmdOpt{tamper: true}, "diagnostic.run", nil)614		add("expired", "expired", cmdOpt{issued: time.Now().Add(-10 * time.Minute)}, "diagnostic.run", nil)615		add("issued in the future", "future", cmdOpt{issued: time.Now().Add(10 * time.Minute)}, "diagnostic.run", nil)616		add("addressed to another agent", "another agent", cmdOpt{agentID: "00000000-0000-7000-8000-000000000001"}, "diagnostic.run", nil)617		add("policy: service not allowed", "policy", cmdOpt{}, "service.restart", map[string]any{"manager": "pm2", "name": "api"})618		add("policy: reboot", "policy", cmdOpt{}, "system.reboot", nil)619		add("unknown type", "policy", cmdOpt{}, "shell.exec", map[string]any{"cmd": "id"})620		add("bad parameters", "unknown diagnostic profile", cmdOpt{}, "diagnostic.run", map[string]any{"profile": "x"})621		for name, c := range refused {622			var row cmdRow623			eventually(t, 30*time.Second, name+" refused", func() bool {624				row = h.cmdRow(mustUUID(c.id))625				return row.State == "finished"626			})627			if row.AckAccepted == nil || *row.AckAccepted || row.AckReason == nil || !strings.Contains(*row.AckReason, c.reason) || row.ResultStatus != nil {628				t.Errorf("%s: %+v (reason %v)", name, row, deref(row.AckReason))629			}630		}631		if n := scan[int](h, `select count(*) from action_events`); n != outputBefore {632			t.Errorf("a refused command produced output (%d -> %d)", outputBefore, n)633		}634635		// Replay: the transport hands the already executed envelope out again.636		h.exec(`update agent_commands set state='pending', delivered_at=null, acked_at=null, finished_at=null, ack_accepted=null, result_status=null where id=$1`, id)637		h.exec(`select pg_notify('mcc_commands', $1)`, e.id.AgentID)638		r = h.waitCmd(id, "finished", 30*time.Second)639		if r.AckAccepted == nil || *r.AckAccepted || !strings.Contains(deref(r.AckReason), "nonce already seen") {640			t.Errorf("replay: %+v %s", r, deref(r.AckReason))641		}642		if n := scan[int](h, `select count(*) from action_events`); n != outputBefore {643			t.Error("a replayed command was executed again")644		}645646		// Kill switch.647		os.WriteFile(e.dir.File(state.FreezeFile), nil, 0o600)648		fid := h.command(e, "diagnostic.run", nil, cmdOpt{})649		r = h.waitCmd(fid, "finished", 30*time.Second)650		if r.AckAccepted == nil || *r.AckAccepted || deref(r.AckReason) != "frozen" {651			t.Errorf("FREEZE: %+v %s", r, deref(r.AckReason))652		}653		eventually(t, 10*time.Second, "frozen agent declares no capability", func() bool {654			return scan[int](h, `select cardinality(capabilities) from agents where id=$1`, e.id.AgentID) == 0655		})656		os.Remove(e.dir.File(state.FreezeFile))657658		// A job allowed by the local policy runs, is cancelled cooperatively, and inventory refresh works.659		os.WriteFile(e.dir.File(state.PolicyFile), []byte(`{"allow":["job.run","inventory.refresh"],"jobs":{"nap":{"argv":["/bin/sleep","{s}"],"params":{"s":{"type":"int","min":1,"max":60}}},"say":{"argv":["/bin/echo","API_TOKEN=abcdef123456","{w}"],"params":{"w":{"type":"enum","values":["hi"]}}}}}`), 0o600)660		say := h.command(e, "job.run", map[string]any{"template": "say", "params": map[string]any{"w": "hi"}}, cmdOpt{})661		r = h.waitCmd(say, "finished", 30*time.Second)662		if deref(r.ResultStatus) != "succeeded" {663			t.Fatalf("job: %+v %s", r, deref(r.AckReason))664		}665		out := scan[string](h, `select string_agg(e.data, ' ') from action_events e join agent_commands c on c.action_id=e.action_id where c.id=$1`, say)666		if strings.Contains(out, "abcdef123456") || !strings.Contains(out, "hi") {667			t.Errorf("job output not redacted or missing: %q", out)668		}669		inj := h.command(e, "job.run", map[string]any{"template": "nap", "params": map[string]any{"s": "1; touch /tmp/x"}}, cmdOpt{})670		if r = h.waitCmd(inj, "finished", 30*time.Second); r.AckAccepted == nil || *r.AckAccepted {671			t.Errorf("injection attempt accepted: %+v", r)672		}673		nap := h.command(e, "job.run", map[string]any{"template": "nap", "params": map[string]any{"s": 30}}, cmdOpt{})674		h.waitCmd(nap, "acked", 30*time.Second)675		same := h.command(e, "job.run", map[string]any{"template": "nap", "params": map[string]any{"s": 1}}, cmdOpt{})676		if r = h.waitCmd(same, "finished", 30*time.Second); !strings.Contains(deref(r.AckReason), "busy") {677			t.Errorf("second command on a busy target: %+v %s", r, deref(r.AckReason))678		}679		h.exec(`update agent_commands set cancel_requested_at = now() where id=$1`, nap)680		h.exec(`select pg_notify('mcc_commands', $1)`, e.id.AgentID)681		if r = h.waitCmd(nap, "finished", 30*time.Second); deref(r.ResultStatus) != "cancelled" {682			t.Errorf("cancel: %+v", r)683		}684		if deref(h.waitCmd(h.command(e, "inventory.refresh", nil, cmdOpt{}), "finished", 40*time.Second).ResultStatus) != "succeeded" {685			t.Error("inventory.refresh failed")686		}687688		// agent.restart: result delivered, then the process exits with the restart code.689		os.WriteFile(e.dir.File(state.PolicyFile), []byte(`{"allow":["agent.restart"]}`), 0o600)690		rid := h.command(e, "agent.restart", nil, cmdOpt{})691		if deref(h.waitCmd(rid, "finished", 30*time.Second).ResultStatus) != "succeeded" {692			t.Error("agent.restart not reported")693		}694		time.Sleep(300 * time.Millisecond)695		if c := stop(); c != 75 {696			t.Errorf("exit code %d after agent.restart, want 75", c)697		}698	})699}700701func TestCommandIsolationBetweenAgents(t *testing.T) {702	forModes(t, func(t *testing.T, mode string) {703		h := newHarness(t, mode, hopt{})704		a, b := h.enroll("a"), h.enroll("b")705		id := h.command(b, "diagnostic.run", nil, cmdOpt{})706		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)707		defer cancel()708		// A's long-poll never receives B's command.709		if resp, err := a.cl.Commands(ctx, time.Second); err != nil || len(resp.Commands) != 0 {710			t.Fatalf("agent A received %d commands (%v)", len(resp.Commands), err)711		}712		resp, err := b.cl.Commands(ctx, time.Second)713		if err != nil || len(resp.Commands) != 1 {714			t.Fatalf("agent B: %v %v", resp, err)715		}716		// Delivered exactly once: no redelivery.717		if again, _ := b.cl.Commands(ctx, 0); len(again.Commands) != 0 {718			t.Error("command delivered twice")719		}720		for name, call := range map[string]func() error{721			"ack": func() error { return a.cl.Ack(ctx, id.String(), protocol.CommandAck{Accepted: true}) },722			"events": func() error {723				return a.cl.Events(ctx, id.String(), []protocol.CommandOutput{{Stream: "stdout", Data: "x"}})724			},725			"result": func() error { return a.cl.Result(ctx, id.String(), protocol.CommandResult{Status: "succeeded"}) },726		} {727			if err := call(); status(err) != 404 {728				t.Errorf("agent A %s on B's command: %v", name, err)729			}730		}731		if h.cmdRow(id).State != "delivered" || scan[int](h, `select count(*) from action_events`) != 0 {732			t.Error("agent A changed B's command")733		}734		// Lifecycle order is enforced for the owner too.735		if err := b.cl.Events(ctx, id.String(), []protocol.CommandOutput{{Stream: "stdout", Data: "x"}}); status(err) != 409 {736			t.Errorf("output before ack: %v", err)737		}738		if err := b.cl.Ack(ctx, id.String(), protocol.CommandAck{Accepted: true}); err != nil {739			t.Fatal(err)740		}741		if err := b.cl.Ack(ctx, id.String(), protocol.CommandAck{Accepted: true}); status(err) != 409 {742			t.Errorf("double ack: %v", err)743		}744		// Server-side second redaction pass and bounds.745		secretLines := []protocol.CommandOutput{746			{Stream: "stdout", Data: "Authorization: Bearer abcdefghijklmnop\nDB_PASSWORD=hunter2\nfine line"},747			{Stream: "stderr", Data: "https://user:pw123@example.org/x"},748		}749		if err := b.cl.Events(ctx, id.String(), secretLines); err != nil {750			t.Fatal(err)751		}752		stored := scan[string](h, `select string_agg(data, '|' order by id) from action_events where kind='output'`)753		for _, leak := range []string{"abcdefghijklmnop", "hunter2", "pw123"} {754			if strings.Contains(stored, leak) {755				t.Errorf("server stored a secret: %q", stored)756			}757		}758		if !strings.Contains(stored, "fine line") || scan[int](h, `select count(*) from action_events`) != 4 {759			t.Errorf("stored output: %q", stored)760		}761		var flood []protocol.CommandOutput762		for i := 0; i < 2500; i++ {763			flood = append(flood, protocol.CommandOutput{Stream: "stdout", Data: fmt.Sprintf("line %d", i)})764		}765		for i := 0; i < len(flood); i += 500 {766			if err := b.cl.Events(ctx, id.String(), flood[i:i+500]); err != nil {767				t.Fatal(err)768			}769		}770		n := scan[int](h, `select count(*) from action_events where kind='output'`)771		last := scan[string](h, `select data from action_events order by id desc limit 1`)772		if n != 2001 || !strings.Contains(last, "truncated") {773			t.Errorf("output bound: %d rows, last %q", n, last)774		}775		if err := b.cl.Result(ctx, id.String(), protocol.CommandResult{Status: "weird"}); status(err) != 400 {776			t.Errorf("bad status: %v", err)777		}778		if err := b.cl.Result(ctx, id.String(), protocol.CommandResult{Status: "failed", Error: "TOKEN=abc123xyz failed", Result: map[string]any{"password": "p", "note": "ok"}}); err != nil {779			t.Fatal(err)780		}781		row := h.cmdRow(id)782		if row.State != "finished" || deref(row.ResultStatus) != "failed" || strings.Contains(fmt.Sprint(row.Result), "abc123xyz") || fmt.Sprint(row.Result["result"].(map[string]any)["password"]) != "[redacted]" {783			t.Errorf("result: %+v", row)784		}785		if err := b.cl.Result(ctx, id.String(), protocol.CommandResult{Status: "succeeded"}); status(err) != 409 {786			t.Errorf("result on a closed command: %v", err)787		}788		// An expired pending command is never delivered.789		late := h.command(b, "diagnostic.run", nil, cmdOpt{})790		h.exec(`update agent_commands set expires_at = now() - interval '1 second' where id=$1`, late)791		if resp, _ := b.cl.Commands(ctx, 0); len(resp.Commands) != 0 {792			t.Error("expired command delivered")793		}794	})795}796797func TestOfflineBufferingAndReplay(t *testing.T) {798	forModes(t, func(t *testing.T, mode string) {799		h := newHarness(t, mode, hopt{netProxy: true})800		e := h.enroll("n1")801		h.startAgent(e)802		seqs := func() int {803			return scan[int](h, `select count(*) from agent_report_seqs where agent_id=$1`, e.id.AgentID)804		}805		eventually(t, 15*time.Second, "live reports", func() bool { return seqs() >= 3 })806807		h.netProxy.cut()808		spooled := func() int {809			des, _ := os.ReadDir(agent.SpoolDir(e.dir))810			n := 0811			for _, d := range des {812				if strings.HasSuffix(d.Name(), ".rpt") {813					n++814				}815			}816			return n817		}818		eventually(t, 30*time.Second, "reports buffered on disk while offline", func() bool { return spooled() >= 5 })819		before := seqs()820		stateBefore := scan[time.Time](h, `select ts from node_state where node_id=$1`, e.id.NodeID)821		st, _ := e.dir.LoadStatus()822		if st == nil || st.Connection != "offline" || st.SpoolReports == 0 {823			t.Errorf("status while offline: %+v", st)824		}825		buffered := spooled()826827		h.netProxy.restore()828		eventually(t, 60*time.Second, "spool drained after reconnection", func() bool { return spooled() == 0 })829		eventually(t, 10*time.Second, "live reports resumed", func() bool {830			return scan[time.Time](h, `select ts from node_state where node_id=$1`, e.id.NodeID).After(stateBefore.Add(2 * time.Second))831		})832		if got := seqs(); got < before+buffered {833			t.Errorf("buffered reports lost: %d before, %d buffered, %d now", before, buffered, got)834		}835		// No duplicate: every (agent, seq) once, every sample key once (primary keys), and each stored836		// report timestamp carries its samples exactly once.837		if d := scan[int](h, `select count(*) from (select metric_id, label, ts from samples_raw where node_id=$1 group by 1,2,3 having count(*) > 1) x`, e.id.NodeID); d != 0 {838			t.Errorf("%d duplicated samples", d)839		}840		// Samples measured while offline are in the database with their original timestamps.841		if n := scan[int](h, `select count(distinct ts) from samples_raw where node_id=$1 and ts > $2 and ts < now() - interval '1 second'`, e.id.NodeID, stateBefore); n < buffered-1 {842			t.Errorf("only %d offline report timestamps stored for %d buffered reports", n, buffered)843		}844		// node_state is the newest report, not a replayed one.845		newest := scan[time.Time](h, `select max(ts) from agent_report_seqs where agent_id=$1`, e.id.AgentID)846		eventually(t, 5*time.Second, "node_state tracks the newest report", func() bool {847			ts := scan[time.Time](h, `select ts from node_state where node_id=$1`, e.id.NodeID)848			return !ts.Before(newest)849		})850	})851}852853func TestDatabaseOutage(t *testing.T) {854	forModes(t, func(t *testing.T, mode string) {855		h := newHarness(t, mode, hopt{dbProxy: true})856		e := h.enroll("n1")857		if _, err := h.send(e, report(1, time.Now(), sample("cpu.usage", 1))); err != nil {858			t.Fatal(err)859		}860		time.Sleep(300 * time.Millisecond) // let the authentication cache expire861		h.dbProxy.cut()862		_, err := h.send(e, report(2, time.Now(), sample("cpu.usage", 2)))863		if status(err) != 503 {864			t.Fatalf("database down: got %v, want 503", err)865		}866		if client.Permanent(err) || client.Revoked(err) {867			t.Error("an outage must look transient to the agent (buffer and retry), never like a revocation")868		}869		rec := httptest.NewRecorder()870		h.health.ServeHTTP(rec, httptest.NewRequest("GET", "/healthz", nil))871		if rec.Code != 503 || !strings.Contains(rec.Body.String(), `"ok":false`) {872			t.Errorf("healthz during outage: %d %s", rec.Code, rec.Body.String())873		}874		if _, err := e.cl.Commands(context.Background(), 0); status(err) != 503 {875			t.Errorf("commands during outage: %v", err)876		}877		h.dbProxy.restore()878		eventually(t, 20*time.Second, "service recovers by itself", func() bool {879			_, err := h.send(e, report(2, time.Now(), sample("cpu.usage", 2)))880			return err == nil881		})882		rec = httptest.NewRecorder()883		h.health.ServeHTTP(rec, httptest.NewRequest("GET", "/healthz", nil))884		if rec.Code != 200 {885			t.Errorf("healthz after recovery: %d", rec.Code)886		}887		if scan[int](h, `select count(*) from samples_raw`) != 2 {888			t.Error("report lost or duplicated across the outage")889		}890	})891}892893func mustUUID(s string) uuid.UUID { return uuid.MustParse(s) }894895func deref(s *string) string {896	if s == nil {897		return ""898	}899	return *s900}901902var _ = http.StatusOK903
Files main
Menu

Theme