Repository Public
1package ingest_test23import (4 "context"5 "crypto/ed25519"6 "crypto/rand"7 "encoding/json"8 "errors"9 "fmt"10 "log/slog"11 "math"12 "net/http"13 "net/http/httptest"14 "os"15 "strings"16 "sync"17 "sync/atomic"18 "testing"19 "time"2021 "github.com/google/uuid"2223 "clustr.spboucher.ai/mcc/internal/agent"24 "clustr.spboucher.ai/mcc/internal/agent/client"25 "clustr.spboucher.ai/mcc/internal/agent/state"26 "clustr.spboucher.ai/mcc/internal/pki"27 "clustr.spboucher.ai/mcc/internal/protocol"28)2930func TestEnrollment(t *testing.T) {31 forModes(t, func(t *testing.T, mode string) {32 h := newHarness(t, mode, hopt{})33 fp := h.fingerprint()3435 // Success: identity stored 0600 in a 0700 directory, node created with the real inventory, audit written.36 tok := h.token("alpha", 15*time.Minute)37 e, err := h.enrollErr("alpha", newHW(), tok, fp)38 if err != nil {39 t.Fatal(err)40 }41 for _, f := range []string{"agent.key", "agent.crt", "ca.crt", "identity.json", "command-keys.json"} {42 st, err := os.Stat(e.dir.File(f))43 if err != nil || st.Mode().Perm() != 0o600 {44 t.Errorf("%s: %v mode %v", f, err, st)45 }46 }47 if st, _ := os.Stat(e.dir.Path); st.Mode().Perm() != 0o700 {48 t.Errorf("state dir mode %o", st.Mode().Perm())49 }50 if got := scan[string](h, `select alias::text from nodes where id=$1 and hardware_uuid=$2 and registered_via='agent' and model='TestMac1,1' and jsonb_array_length(cpu_levels)=3`, e.id.NodeID, e.hw); got != "alpha" {51 t.Errorf("node alias %q", got)52 }53 if n := scan[int](h, `select count(*) from audit_log where action='agent.enrolled' and actor_kind='agent' and outcome='success' and target_id=$1`, e.id.NodeID); n != 1 {54 t.Errorf("%d audit entries", n)55 }56 keys, _ := e.dir.LoadCommandKeys()57 if len(keys) != 1 || keys[h.keyID] == nil {58 t.Errorf("command keys %v", keys)59 }60 cert, _, _ := e.dir.LoadCredentials()61 if id, err := pki.AgentIDFromCert(cert.Leaf); err != nil || id.String() != e.id.AgentID || len(cert.Leaf.ExtKeyUsage) != 1 || time.Until(cert.Leaf.NotAfter) > 31*24*time.Hour || time.Until(cert.Leaf.NotAfter) < 29*24*time.Hour {62 t.Errorf("certificate: %v %v", err, cert.Leaf.NotAfter)63 }6465 // Token reuse refused.66 if _, err := h.enrollErr("alpha", newHW(), tok, fp); status(err) != 401 || code(err) != "invalid_enrollment_token" {67 t.Errorf("reused token: %v", err)68 }69 // Expired token refused.70 expired := h.token("beta", time.Minute)71 h.exec(`update agent_enrollment_tokens set expires_at = now() - interval '1 second' where node_alias='beta'`)72 if _, err := h.enrollErr("beta", newHW(), expired, fp); status(err) != 401 {73 t.Errorf("expired token: %v", err)74 }75 // Revoked and malformed tokens refused.76 revoked := h.token("beta2", time.Minute)77 h.exec(`update agent_enrollment_tokens set revoked_at = now() where node_alias='beta2'`)78 if _, err := h.enrollErr("beta2", newHW(), revoked, fp); status(err) != 401 {79 t.Errorf("revoked token: %v", err)80 }81 if _, err := h.enrollErr("x", newHW(), "short", fp); status(err) != 401 {82 t.Errorf("malformed token: %v", err)83 }84 if n := scan[int](h, `select count(*) from nodes`); n != 1 {85 t.Errorf("refused enrollments created nodes: %d", n)86 }8788 // Concurrent double use: exactly one success.89 race := h.token("gamma", 15*time.Minute)90 var ok, refused atomic.Int3291 var wg sync.WaitGroup92 for i := 0; i < 8; i++ {93 wg.Add(1)94 go func() {95 defer wg.Done()96 if _, err := h.enrollErr("gamma", newHW(), race, fp); err == nil {97 ok.Add(1)98 } else if status(err) == 401 {99 refused.Add(1)100 }101 }()102 }103 wg.Wait()104 if ok.Load() != 1 || refused.Load() != 7 {105 t.Errorf("concurrent use: %d successes, %d refusals", ok.Load(), refused.Load())106 }107 if n := scan[int](h, `select count(*) from agents a join nodes n on n.id=a.node_id where n.alias='gamma'`); n != 1 {108 t.Errorf("%d agents for gamma", n)109 }110111 // Wrong CA fingerprint: refused, and the state directory holds no identity.112 bad := make([]byte, 32)113 wtok := h.token("delta", 15*time.Minute)114 we, err := h.enrollErr("delta", newHW(), wtok, bad)115 if err == nil {116 t.Fatal("wrong CA fingerprint accepted")117 }118 if _, lerr := we.dir.LoadIdentity(); !errors.Is(lerr, state.ErrNotEnrolled) {119 t.Errorf("identity written despite the pin failure: %v", lerr)120 }121 if _, _, lerr := we.dir.LoadCredentials(); !errors.Is(lerr, state.ErrNotEnrolled) {122 t.Errorf("credentials written despite the pin failure: %v", lerr)123 }124 if mode == "mtls" && !errors.Is(err, client.ErrPin) {125 t.Errorf("mtls: expected the handshake pin to fail, got %v", err)126 }127128 // A manually registered node (hardware_uuid NULL) is adopted.129 h.exec(`insert into nodes(id, alias, registered_via, site) values (gen_random_uuid(), 'manual-1', 'manual', 'lab')`)130 m := h.enroll("manual-1")131 if got := scan[string](h, `select registered_via || '/' || site from nodes where id=$1 and hardware_uuid=$2`, m.id.NodeID, m.hw); got != "manual/lab" {132 t.Errorf("adoption: %q", got)133 }134 // Same hardware, token for another alias: existing node and alias kept, reported in the audit; previous agent revoked.135 again, err := h.enrollErr("other-alias", m.hw, h.token("other-alias", time.Minute), fp)136 if err != nil {137 t.Fatal(err)138 }139 if again.id.NodeID != m.id.NodeID || scan[int](h, `select count(*) from nodes where alias='other-alias'`) != 0 {140 t.Error("hardware match must keep the existing node and alias")141 }142 if d := scan[string](h, `select detail from audit_log where action='agent.enrolled' order by seq desc limit 1`); !strings.Contains(d, "other-alias") || !strings.Contains(d, "manual-1") || !strings.Contains(d, "revoked") {143 t.Errorf("audit detail %q", d)144 }145 if r := scan[string](h, `select revoked_reason from agents where id=$1`, m.id.AgentID); r != "re-enrolled" {146 t.Errorf("previous agent: %q", r)147 }148 if _, err := h.send(m, report(1, time.Now())); status(err) != 401 && err == nil {149 t.Errorf("revoked previous identity still accepted: %v", err)150 }151 // An alias owned by other hardware is refused and the token is not burnt.152 tk := h.token("alpha", time.Minute)153 if _, err := h.enrollErr("alpha", newHW(), tk, fp); status(err) != 409 {154 t.Errorf("alias in use: %v", err)155 }156 if c := scan[bool](h, `select consumed_at is null from agent_enrollment_tokens where node_alias='alpha' order by created_at desc limit 1`); !c {157 t.Error("token consumed by a refused enrollment")158 }159 })160}161162func TestReportRoundtripAndIdempotence(t *testing.T) {163 forModes(t, func(t *testing.T, mode string) {164 h := newHarness(t, mode, hopt{})165 e := h.enroll("n1")166 now := time.Now().UTC().Truncate(time.Millisecond)167 rep := report(1, now, sample("cpu.usage", 12.5), sample("mem.used", 4e9),168 protocol.Sample{M: "net.rx", L: "en0", V: 100}, protocol.Sample{M: "net.rx", L: "utun1", V: 5})169 rep.Availability = map[string]string{"thermal.temp": "unsupported", "gpu.usage": "unavailable"}170 rep.Processes = []protocol.Process{{PID: 10, Name: "proc", User: "u", CPU: 3.5, RSS: 1 << 20}}171 rep.Capabilities = []string{"diagnostic.run"}172 rep.Volumes = []protocol.Volume{{Mount: "/", Device: "/dev/disk3s1", FS: "apfs", Kind: "internal", Total: 100, Used: 40, SMART: "verified"}}173 events0 := scan[int](h, `select count(*) from events_outbox where topic='node.state'`)174 ack, err := h.send(e, rep)175 if err != nil || ack.Seq != 1 || ack.IntervalS != 10 {176 t.Fatalf("%+v %v", ack, err)177 }178 count := func() int { return scan[int](h, `select count(*) from samples_raw where node_id=$1`, e.id.NodeID) }179 if count() != 4 {180 t.Fatalf("%d samples", count())181 }182 if v := scan[float64](h, `select ("values"->>'cpu.usage')::float8 from node_state where node_id=$1`, e.id.NodeID); v != 12.5 {183 t.Errorf("node_state cpu %v", v)184 }185 if v := scan[float64](h, `select ("values"->'net.rx'->>'utun1')::float8 from node_state where node_id=$1`, e.id.NodeID); v != 5 {186 t.Errorf("labeled value %v", v)187 }188 if a := scan[string](h, `select availability->>'thermal.temp' from node_state where node_id=$1`, e.id.NodeID); a != "unsupported" {189 t.Errorf("availability %q", a)190 }191 if n := scan[int](h, `select count(*) from events_outbox where topic='node.state'`) - events0; n != 1 {192 t.Errorf("%d node.state events for one report", n)193 }194 if got := scan[string](h, `select version || '/' || array_to_string(capabilities, ',') || '/' || last_seq from agents where id=$1`, e.id.AgentID); got != "test/diagnostic.run/1" {195 t.Errorf("agent row %q", got)196 }197 if scan[int](h, `select count(*) from volumes where node_id=$1 and mount='/' and smart_status='verified'`, e.id.NodeID) != 1 {198 t.Error("volume not stored")199 }200201 // Replay of the same seq: acknowledged, nothing re-inserted, no second event.202 ack, err = h.send(e, rep)203 if err != nil || ack.Seq != 1 {204 t.Fatalf("replay must be acknowledged: %v", err)205 }206 rep2 := *rep207 rep2.Samples = []protocol.Sample{sample("cpu.usage", 99), sample("load.1", 7)}208 if _, err := h.send(e, &rep2); err != nil { // same seq, different content: still a duplicate209 t.Fatal(err)210 }211 if count() != 4 || scan[int](h, `select count(*) from events_outbox where topic='node.state'`)-events0 != 1 {212 t.Errorf("duplicate seq was applied again: %d samples", count())213 }214215 // A newer report carrying a subset keeps the other values and drops the ones declared unavailable.216 rep3 := report(2, now.Add(10*time.Second), sample("cpu.usage", 20))217 rep3.Availability = map[string]string{"mem.used": "unavailable"}218 if _, err := h.send(e, rep3); err != nil {219 t.Fatal(err)220 }221 if got := scan[string](h, `select ("values"->>'cpu.usage') || '|' || coalesce("values"->>'mem.used','gone') || '|' || ("values"->'net.rx'->>'en0') from node_state where node_id=$1`, e.id.NodeID); got != "20|gone|100" {222 t.Errorf("merge: %q", got)223 }224225 // A buffered (older) report adds samples but never regresses live state.226 old := report(3, now.Add(-time.Hour), sample("cpu.usage", 77))227 old.Buffered = true228 old.Services = &protocol.ServiceSet{Managers: []string{"pm2"}, Items: []protocol.Service{{Manager: "pm2", Name: "stale", State: "online"}}}229 if _, err := h.send(e, old); err != nil {230 t.Fatal(err)231 }232 // Also an out-of-order report that is not flagged buffered.233 if _, err := h.send(e, report(4, now.Add(-30*time.Minute), sample("cpu.usage", 66))); err != nil {234 t.Fatal(err)235 }236 if v := scan[float64](h, `select ("values"->>'cpu.usage')::float8 from node_state where node_id=$1`, e.id.NodeID); v != 20 {237 t.Errorf("old report regressed node_state: cpu=%v", v)238 }239 if ts := scan[time.Time](h, `select ts from node_state where node_id=$1`, e.id.NodeID); !ts.Equal(now.Add(10 * time.Second)) {240 t.Errorf("node_state.ts regressed to %v", ts)241 }242 if scan[int](h, `select count(*) from services where node_id=$1`, e.id.NodeID) != 0 {243 t.Error("a buffered report changed service state")244 }245 if count() != 7 || scan[int64](h, `select last_seq from agents where id=$1`, e.id.AgentID) != 4 {246 t.Errorf("samples %d", count())247 }248 })249}250251func TestReportValidation(t *testing.T) {252 forModes(t, func(t *testing.T, mode string) {253 h := newHarness(t, mode, hopt{})254 e := h.enroll("n1")255 now := time.Now().UTC()256 ts := now.Format(time.RFC3339Nano)257 many := func(metric string, n int) *protocol.Report {258 r := report(50, now)259 for i := 0; i < n; i++ {260 r.Samples = append(r.Samples, protocol.Sample{M: metric, L: fmt.Sprintf("if%d", i), V: 1})261 }262 return r263 }264 type bad struct {265 name string266 rep *protocol.Report267 raw string268 want int269 }270 long := report(50, now, protocol.Sample{M: "net.rx", L: strings.Repeat("x", protocol.MaxLabelLen+1), V: 1})271 procs := report(50, now)272 for i := 0; i < protocol.MaxProcesses+1; i++ {273 procs.Processes = append(procs.Processes, protocol.Process{PID: int32(i), Name: "p"})274 }275 svc := report(50, now)276 svc.Services = &protocol.ServiceSet{Managers: []string{"pm2"}, Items: []protocol.Service{{Manager: "launchd", Name: "x", State: "online"}}}277 creds := report(50, now)278 creds.Volumes = []protocol.Volume{{Mount: "/Volumes/nas", Kind: "network", Remote: "//user:pw@192.0.2.9/share"}}279 cases := []bad{280 {name: "unknown metric", rep: report(50, now, sample("cpu.magic", 1)), want: 400},281 {name: "NaN", raw: `{"v":1,"seq":50,"ts":"` + ts + `","samples":[{"m":"cpu.usage","v":NaN}]}`, want: 400},282 {name: "infinite", raw: `{"v":1,"seq":50,"ts":"` + ts + `","samples":[{"m":"cpu.usage","v":1e999}]}`, want: 400},283 {name: "too many interface labels", rep: many("net.rx", protocol.MaxInterfaces+1), want: 400},284 {name: "too many volume labels", rep: many("disk.used", protocol.MaxVolumes+1), want: 400},285 {name: "label too long", rep: long, want: 400},286 {name: "label on unlabeled metric", rep: report(50, now, protocol.Sample{M: "cpu.usage", L: "x", V: 1}), want: 400},287 {name: "missing label", rep: report(50, now, sample("net.rx", 1)), want: 400},288 {name: "duplicate sample", rep: report(50, now, sample("cpu.usage", 1), sample("cpu.usage", 2)), want: 400},289 {name: "future timestamp", rep: report(50, now.Add(6*time.Minute), sample("cpu.usage", 1)), want: 400},290 {name: "older than retention", rep: report(50, now.Add(-8*24*time.Hour), sample("cpu.usage", 1)), want: 400},291 {name: "bad version", raw: `{"v":2,"seq":50,"ts":"` + ts + `","samples":[]}`, want: 400},292 {name: "seq zero", rep: report(0, now, sample("cpu.usage", 1)), want: 400},293 {name: "too many processes", rep: procs, want: 400},294 {name: "service of unlisted manager", rep: svc, want: 400},295 {name: "credentials in remote", rep: creds, want: 400},296 {name: "bad availability state", raw: `{"v":1,"seq":50,"ts":"` + ts + `","samples":[],"availability":{"cpu.usage":"broken"}}`, want: 400},297 {name: "not json", raw: `hello`, want: 400},298 }299 for _, c := range cases {300 var err error301 if c.rep != nil {302 _, err = h.send(e, c.rep)303 } else {304 err = h.sendRaw(e, c.raw)305 }306 if status(err) != c.want {307 t.Errorf("%s: got %v, want HTTP %d", c.name, err, c.want)308 }309 if !client.Permanent(err) {310 t.Errorf("%s: the agent must treat this as permanent (drop, do not retry)", c.name)311 }312 }313 // Oversize: 1.2 MiB of incompressible data.314 big := make([]byte, 1200<<10)315 rand.Read(big)316 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)317 _, err := e.cl.Report(ctx, big)318 cancel()319 if status(err) != 413 {320 t.Errorf("oversize: %v", err)321 }322 // A gzip bomb is cut at the decompressed limit.323 if err := h.sendRaw(e, `{"v":1,"seq":50,"ts":"`+ts+`","samples":[],"agent_version":"`+strings.Repeat("a", 5<<20)+`"}`); status(err) != 413 {324 t.Errorf("gzip bomb: %v", err)325 }326 if n := scan[int](h, `select (select count(*) from samples_raw) + (select count(*) from agent_report_seqs) + (select count(*) from node_state)`); n != 0 {327 t.Errorf("rejected reports wrote %d rows", n)328 }329 // Sanity: math.NaN cannot even be encoded by the agent.330 if _, err := json.Marshal(sample("cpu.usage", math.NaN())); err == nil {331 t.Error("NaN encodable")332 }333 // Accepted: maximum cardinality, unknown fields (forward compatibility).334 if _, err := h.send(e, many("net.rx", protocol.MaxInterfaces)); err != nil {335 t.Errorf("max interfaces refused: %v", err)336 }337 if err := h.sendRaw(e, `{"v":1,"seq":51,"ts":"`+ts+`","samples":[],"future_field":{"x":1}}`); err != nil {338 t.Errorf("unknown field must be ignored within v1: %v", err)339 }340 })341}342343func TestRevocationAndForeignCA(t *testing.T) {344 forModes(t, func(t *testing.T, mode string) {345 h := newHarness(t, mode, hopt{})346 e := h.enroll("n1")347 if _, err := h.send(e, report(1, time.Now(), sample("cpu.usage", 1))); err != nil {348 t.Fatal(err)349 }350 // A long-poll is open when the agent gets revoked.351 pollErr := make(chan error, 1)352 go func() {353 _, err := e.cl.Commands(context.Background(), 20*time.Second)354 pollErr <- err355 }()356 eventually(t, 3*time.Second, "long-poll open", func() bool { return h.srv.ActiveLongPolls() == 1 })357 revokedAt := time.Now()358 h.exec(`update agents set revoked_at = now(), revoked_reason = 'test' where id=$1`, e.id.AgentID)359 eventually(t, 5*time.Second, "revoked agent refused", func() bool {360 _, err := h.send(e, report(2, time.Now(), sample("cpu.usage", 1)))361 return status(err) == 401 && code(err) == "agent_revoked"362 })363 select {364 case err := <-pollErr:365 if status(err) != 401 {366 t.Errorf("long-poll of a revoked agent ended with %v", err)367 }368 case <-time.After(6 * time.Second):369 t.Error("long-poll of a revoked agent stayed open")370 }371 if d := time.Since(revokedAt); d > 6*time.Second {372 t.Errorf("revocation took %s", d)373 }374 if scan[int](h, `select count(*) from samples_raw`) != 1 {375 t.Error("a revoked agent wrote samples")376 }377378 // An identity issued by another CA (another Command Center) is refused.379 other := newHarness(t, mode, hopt{})380 foreign := other.enroll("stranger")381 // Point the foreign identity at our server.382 id, _ := foreign.dir.LoadIdentity()383 id.Endpoints = []string{h.endpoint}384 foreign.dir.SaveIdentity(id)385 if mode == "mtls" {386 // Trust our CA so the handshake reaches client authentication.387 os.WriteFile(foreign.dir.File("ca.crt"), h.ca.PEM, 0o600)388 }389 fcl, err := client.New(foreign.dir)390 if err != nil {391 t.Fatal(err)392 }393 defer fcl.Close()394 enc, _ := client.EncodeReport(report(1, time.Now(), sample("cpu.usage", 1)))395 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)396 _, err = fcl.Report(ctx, enc)397 cancel()398 switch mode {399 case "mtls":400 if err == nil || status(err) != 0 {401 t.Errorf("mtls: a certificate from another CA must fail the TLS handshake, got %v", err)402 }403 default:404 if status(err) != 401 || code(err) != "certificate_invalid" {405 t.Errorf("signed: %v", err)406 }407 }408 if scan[int](h, `select count(*) from samples_raw`) != 1 {409 t.Error("a foreign identity wrote samples")410 }411 // No certificate at all on the mTLS port.412 if mode == "mtls" {413 resp, err := insecureGet(h.mtlsAddr, "/agent/v1/commands")414 if err != nil || resp != 401 {415 t.Errorf("no client certificate: %d %v", resp, err)416 }417 }418 })419}420421func TestRenewal(t *testing.T) {422 forModes(t, func(t *testing.T, mode string) {423 h := newHarness(t, mode, hopt{})424 e := h.enroll("n1")425 before := scan[string](h, `select cert_serial from agents where id=$1`, e.id.AgentID)426 // A second client still holding the old certificate (as if the renewal response had been lost).427 oldCl, err := client.New(e.dir)428 if err != nil {429 t.Fatal(err)430 }431 defer oldCl.Close()432 ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)433 defer cancel()434 if err := e.cl.Renew(ctx); err != nil {435 t.Fatal(err)436 }437 after := scan[string](h, `select cert_serial from agents where id=$1`, e.id.AgentID)438 if after == before || scan[string](h, `select prev_cert_serial from agents where id=$1`, e.id.AgentID) != before {439 t.Fatalf("serial not rotated: %s -> %s", before, after)440 }441 if _, err := h.send(e, report(1, time.Now(), sample("cpu.usage", 1))); err != nil {442 t.Fatalf("new certificate refused: %v", err)443 }444 enc, _ := client.EncodeReport(report(2, time.Now(), sample("cpu.usage", 1)))445 if _, err := oldCl.Report(ctx, enc); err != nil {446 t.Errorf("previous certificate must stay valid during the overlap: %v", err)447 }448 h.exec(`update agents set prev_cert_valid_until = now() - interval '1 second' where id=$1`, e.id.AgentID)449 eventually(t, 5*time.Second, "old certificate refused after the overlap", func() bool {450 enc, _ := client.EncodeReport(report(3, time.Now(), sample("cpu.usage", 1)))451 _, err := oldCl.Report(ctx, enc)452 return status(err) == 401453 })454 if scan[int](h, `select count(*) from audit_log where action='agent.cert_renewed'`) != 1 {455 t.Error("renewal not audited")456 }457 })458}459460func TestServiceGoneSemanticsAndEvents(t *testing.T) {461 forModes(t, func(t *testing.T, mode string) {462 h := newHarness(t, mode, hopt{})463 e := h.enroll("n1")464 now := time.Now()465 seq := int64(0)466 push := func(managers []string, items ...protocol.Service) {467 seq++468 r := report(seq, now.Add(time.Duration(seq)*time.Second), sample("cpu.usage", 1))469 r.Services = &protocol.ServiceSet{Managers: managers, Items: items}470 if _, err := h.send(e, r); err != nil {471 t.Fatal(err)472 }473 }474 st := func(mgr, name string) string {475 return scan[string](h, `select coalesce((select state || case when gone_at is null then '' else '/gone' end from services where node_id=$1 and manager::text=$2 and name=$3), 'absent')`, e.id.NodeID, mgr, name)476 }477 ev := func() int { return scan[int](h, `select count(*) from events_outbox where topic='service.changed'`) }478479 push([]string{"pm2", "launchd"},480 protocol.Service{Manager: "pm2", Name: "api", State: "online", PID: 10, Ports: []int{8080}, RawState: "online"},481 protocol.Service{Manager: "pm2", Name: "worker", State: "stopped"},482 protocol.Service{Manager: "launchd", Name: "io.x", State: "online", Scope: "user"})483 if st("pm2", "api") != "online" || st("launchd", "io.x") != "online" || ev() != 3 {484 t.Fatalf("initial: %s %s events=%d", st("pm2", "api"), st("launchd", "io.x"), ev())485 }486 // Same observation again: no event.487 push([]string{"pm2", "launchd"},488 protocol.Service{Manager: "pm2", Name: "api", State: "online", PID: 10},489 protocol.Service{Manager: "pm2", Name: "worker", State: "stopped"},490 protocol.Service{Manager: "launchd", Name: "io.x", State: "online"})491 if ev() != 3 {492 t.Errorf("unchanged services produced events: %d", ev())493 }494 // pm2 observed with zero services; launchd NOT observed: its services are untouched.495 push([]string{"pm2"})496 if st("pm2", "api") != "online/gone" || st("pm2", "worker") != "stopped/gone" || st("launchd", "io.x") != "online" {497 t.Errorf("gone handling: api=%s worker=%s launchd=%s", st("pm2", "api"), st("pm2", "worker"), st("launchd", "io.x"))498 }499 if ev() != 5 {500 t.Errorf("events after gone: %d", ev())501 }502 // No manager observed at all (adapters failed): nothing changes.503 push([]string{})504 if st("launchd", "io.x") != "online" || ev() != 5 {505 t.Error("an empty observation changed services")506 }507 // The service comes back, in another state.508 push([]string{"pm2"}, protocol.Service{Manager: "pm2", Name: "api", State: "errored"})509 if st("pm2", "api") != "errored" || st("pm2", "worker") != "stopped/gone" || ev() != 6 {510 t.Errorf("return: %s events=%d", st("pm2", "api"), ev())511 }512 // State change only.513 push([]string{"pm2"}, protocol.Service{Manager: "pm2", Name: "api", State: "online"})514 if got := scan[string](h, `select payload->>'previous_state' || '>' || (payload->>'state') from events_outbox where topic='service.changed' order by seq desc limit 1`); got != "errored>online" || ev() != 7 {515 t.Errorf("state change event %q, %d events", got, ev())516 }517 // Inference and volumes follow the same complete-observation rule.518 seq++519 r := report(seq, now.Add(time.Minute), sample("cpu.usage", 1))520 r.Inference = &protocol.InferenceSet{Items: []protocol.InferenceServer{{Kind: "mlx_lm", Port: 8081, Model: "org/model", PID: 5, Health: "ok", HealthMS: 3}}}521 r.Volumes = []protocol.Volume{{Mount: "/", Kind: "internal", Total: 10, Used: 5}, {Mount: "/Volumes/ssd", Kind: "external", Total: 10, Used: 1}}522 h.send(e, r)523 seq++524 r = report(seq, now.Add(2*time.Minute), sample("cpu.usage", 1))525 r.Inference = &protocol.InferenceSet{}526 r.Volumes = []protocol.Volume{{Mount: "/", Kind: "internal", Total: 10, Used: 6}}527 h.send(e, r)528 if got := scan[string](h, `select (select count(*) from inference_servers where gone_at is not null)::text || '/' || (select count(*) from volumes where gone_at is not null and mount='/Volumes/ssd')::text || '/' || (select used_bytes from volumes where mount='/')::text`); got != "1/1/6" {529 t.Errorf("inference/volumes gone handling: %s", got)530 }531 // NAS: only registered targets are recorded; state aggregates observations.532 h.exec(`insert into nas_targets(id, name, host) values (gen_random_uuid(), 'nas-a', '192.0.2.50')`)533 seq++534 r = report(seq, now.Add(3*time.Minute), sample("cpu.usage", 1))535 r.NAS = []protocol.NASProbe{{Host: "192.0.2.50", Reachable: false}, {Host: "192.0.2.99", Reachable: true}}536 h.send(e, r)537 if got := scan[string](h, `select state || '/' || mounted_on from nas_targets where name='nas-a'`); got != "offline/0" {538 t.Errorf("nas: %s", got)539 }540 seq++541 r = report(seq, now.Add(4*time.Minute), sample("cpu.usage", 1))542 r.NAS = []protocol.NASProbe{{Host: "192.0.2.50", Reachable: true, Mounted: true, LatencyMS: 1.5}}543 h.send(e, r)544 if got := scan[string](h, `select state || '/' || mounted_on from nas_targets where name='nas-a'`); got != "online/1" {545 t.Errorf("nas: %s", got)546 }547 })548}549550// startAgent runs the real agent runtime against the harness.551func (h *harness) startAgent(e *enrolled) (stop func() int) {552 a, err := agent.New(agent.Options{Dir: e.dir, Version: "test", Interval: 400 * time.Millisecond, PollWait: 2 * time.Second,553 Log: slog.New(slog.DiscardHandler)})554 if err != nil {555 h.t.Fatal(err)556 }557 ctx, cancel := context.WithCancel(context.Background())558 done := make(chan int, 1)559 go func() { done <- a.Run(ctx) }()560 var once sync.Once561 code := 0562 stop = func() int {563 once.Do(func() {564 cancel()565 select {566 case code = <-done:567 case <-time.After(30 * time.Second):568 h.t.Error("agent did not stop")569 }570 })571 return code572 }573 h.t.Cleanup(func() { stop() })574 return stop575}576577func TestCommandsEndToEnd(t *testing.T) {578 forModes(t, func(t *testing.T, mode string) {579 h := newHarness(t, mode, hopt{})580 e := h.enroll("n1")581 stop := h.startAgent(e)582 eventually(t, 15*time.Second, "first report of the real agent", func() bool {583 return scan[int](h, `select count(*) from node_state where node_id=$1`, e.id.NodeID) == 1584 })585586 // Accepted and executed: a real read-only diagnostic on this machine.587 id := h.command(e, "diagnostic.run", map[string]any{"profile": "basic"}, cmdOpt{})588 r := h.waitCmd(id, "finished", 40*time.Second)589 if r.AckAccepted == nil || !*r.AckAccepted || r.ResultStatus == nil || *r.ResultStatus != "succeeded" {590 t.Fatalf("diagnostic: %+v", r)591 }592 lines := scan[int](h, `select count(*) from action_events e join agent_commands c on c.action_id=e.action_id where c.id=$1 and e.kind='output'`, id)593 if lines < 5 {594 t.Errorf("only %d output lines stored", lines)595 }596 if scan[int](h, `select count(*) from events_outbox where topic='action.output'`) == 0 {597 t.Error("no action.output event")598 }599600 // Compromised transport: none of these may execute.601 _, otherKey, _ := ed25519.GenerateKey(rand.Reader)602 outputBefore := scan[int](h, `select count(*) from action_events`)603 refused := map[string]struct {604 id string605 reason string606 }{}607 add := func(name, reason string, o cmdOpt, typ string, params any) {608 cid := h.command(e, typ, params, o)609 refused[name] = struct{ id, reason string }{cid.String(), reason}610 }611 add("forged signature (key the agent does not hold)", "bad signature", cmdOpt{key: otherKey, keyID: h.keyID}, "diagnostic.run", nil)612 add("unknown key id", "unknown signing key", cmdOpt{key: otherKey, keyID: "attacker"}, "diagnostic.run", nil)613 add("tampered payload", "bad signature", cmdOpt{tamper: true}, "diagnostic.run", nil)614 add("expired", "expired", cmdOpt{issued: time.Now().Add(-10 * time.Minute)}, "diagnostic.run", nil)615 add("issued in the future", "future", cmdOpt{issued: time.Now().Add(10 * time.Minute)}, "diagnostic.run", nil)616 add("addressed to another agent", "another agent", cmdOpt{agentID: "00000000-0000-7000-8000-000000000001"}, "diagnostic.run", nil)617 add("policy: service not allowed", "policy", cmdOpt{}, "service.restart", map[string]any{"manager": "pm2", "name": "api"})618 add("policy: reboot", "policy", cmdOpt{}, "system.reboot", nil)619 add("unknown type", "policy", cmdOpt{}, "shell.exec", map[string]any{"cmd": "id"})620 add("bad parameters", "unknown diagnostic profile", cmdOpt{}, "diagnostic.run", map[string]any{"profile": "x"})621 for name, c := range refused {622 var row cmdRow623 eventually(t, 30*time.Second, name+" refused", func() bool {624 row = h.cmdRow(mustUUID(c.id))625 return row.State == "finished"626 })627 if row.AckAccepted == nil || *row.AckAccepted || row.AckReason == nil || !strings.Contains(*row.AckReason, c.reason) || row.ResultStatus != nil {628 t.Errorf("%s: %+v (reason %v)", name, row, deref(row.AckReason))629 }630 }631 if n := scan[int](h, `select count(*) from action_events`); n != outputBefore {632 t.Errorf("a refused command produced output (%d -> %d)", outputBefore, n)633 }634635 // Replay: the transport hands the already executed envelope out again.636 h.exec(`update agent_commands set state='pending', delivered_at=null, acked_at=null, finished_at=null, ack_accepted=null, result_status=null where id=$1`, id)637 h.exec(`select pg_notify('mcc_commands', $1)`, e.id.AgentID)638 r = h.waitCmd(id, "finished", 30*time.Second)639 if r.AckAccepted == nil || *r.AckAccepted || !strings.Contains(deref(r.AckReason), "nonce already seen") {640 t.Errorf("replay: %+v %s", r, deref(r.AckReason))641 }642 if n := scan[int](h, `select count(*) from action_events`); n != outputBefore {643 t.Error("a replayed command was executed again")644 }645646 // Kill switch.647 os.WriteFile(e.dir.File(state.FreezeFile), nil, 0o600)648 fid := h.command(e, "diagnostic.run", nil, cmdOpt{})649 r = h.waitCmd(fid, "finished", 30*time.Second)650 if r.AckAccepted == nil || *r.AckAccepted || deref(r.AckReason) != "frozen" {651 t.Errorf("FREEZE: %+v %s", r, deref(r.AckReason))652 }653 eventually(t, 10*time.Second, "frozen agent declares no capability", func() bool {654 return scan[int](h, `select cardinality(capabilities) from agents where id=$1`, e.id.AgentID) == 0655 })656 os.Remove(e.dir.File(state.FreezeFile))657658 // A job allowed by the local policy runs, is cancelled cooperatively, and inventory refresh works.659 os.WriteFile(e.dir.File(state.PolicyFile), []byte(`{"allow":["job.run","inventory.refresh"],"jobs":{"nap":{"argv":["/bin/sleep","{s}"],"params":{"s":{"type":"int","min":1,"max":60}}},"say":{"argv":["/bin/echo","API_TOKEN=abcdef123456","{w}"],"params":{"w":{"type":"enum","values":["hi"]}}}}}`), 0o600)660 say := h.command(e, "job.run", map[string]any{"template": "say", "params": map[string]any{"w": "hi"}}, cmdOpt{})661 r = h.waitCmd(say, "finished", 30*time.Second)662 if deref(r.ResultStatus) != "succeeded" {663 t.Fatalf("job: %+v %s", r, deref(r.AckReason))664 }665 out := scan[string](h, `select string_agg(e.data, ' ') from action_events e join agent_commands c on c.action_id=e.action_id where c.id=$1`, say)666 if strings.Contains(out, "abcdef123456") || !strings.Contains(out, "hi") {667 t.Errorf("job output not redacted or missing: %q", out)668 }669 inj := h.command(e, "job.run", map[string]any{"template": "nap", "params": map[string]any{"s": "1; touch /tmp/x"}}, cmdOpt{})670 if r = h.waitCmd(inj, "finished", 30*time.Second); r.AckAccepted == nil || *r.AckAccepted {671 t.Errorf("injection attempt accepted: %+v", r)672 }673 nap := h.command(e, "job.run", map[string]any{"template": "nap", "params": map[string]any{"s": 30}}, cmdOpt{})674 h.waitCmd(nap, "acked", 30*time.Second)675 same := h.command(e, "job.run", map[string]any{"template": "nap", "params": map[string]any{"s": 1}}, cmdOpt{})676 if r = h.waitCmd(same, "finished", 30*time.Second); !strings.Contains(deref(r.AckReason), "busy") {677 t.Errorf("second command on a busy target: %+v %s", r, deref(r.AckReason))678 }679 h.exec(`update agent_commands set cancel_requested_at = now() where id=$1`, nap)680 h.exec(`select pg_notify('mcc_commands', $1)`, e.id.AgentID)681 if r = h.waitCmd(nap, "finished", 30*time.Second); deref(r.ResultStatus) != "cancelled" {682 t.Errorf("cancel: %+v", r)683 }684 if deref(h.waitCmd(h.command(e, "inventory.refresh", nil, cmdOpt{}), "finished", 40*time.Second).ResultStatus) != "succeeded" {685 t.Error("inventory.refresh failed")686 }687688 // agent.restart: result delivered, then the process exits with the restart code.689 os.WriteFile(e.dir.File(state.PolicyFile), []byte(`{"allow":["agent.restart"]}`), 0o600)690 rid := h.command(e, "agent.restart", nil, cmdOpt{})691 if deref(h.waitCmd(rid, "finished", 30*time.Second).ResultStatus) != "succeeded" {692 t.Error("agent.restart not reported")693 }694 time.Sleep(300 * time.Millisecond)695 if c := stop(); c != 75 {696 t.Errorf("exit code %d after agent.restart, want 75", c)697 }698 })699}700701func TestCommandIsolationBetweenAgents(t *testing.T) {702 forModes(t, func(t *testing.T, mode string) {703 h := newHarness(t, mode, hopt{})704 a, b := h.enroll("a"), h.enroll("b")705 id := h.command(b, "diagnostic.run", nil, cmdOpt{})706 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)707 defer cancel()708 // A's long-poll never receives B's command.709 if resp, err := a.cl.Commands(ctx, time.Second); err != nil || len(resp.Commands) != 0 {710 t.Fatalf("agent A received %d commands (%v)", len(resp.Commands), err)711 }712 resp, err := b.cl.Commands(ctx, time.Second)713 if err != nil || len(resp.Commands) != 1 {714 t.Fatalf("agent B: %v %v", resp, err)715 }716 // Delivered exactly once: no redelivery.717 if again, _ := b.cl.Commands(ctx, 0); len(again.Commands) != 0 {718 t.Error("command delivered twice")719 }720 for name, call := range map[string]func() error{721 "ack": func() error { return a.cl.Ack(ctx, id.String(), protocol.CommandAck{Accepted: true}) },722 "events": func() error {723 return a.cl.Events(ctx, id.String(), []protocol.CommandOutput{{Stream: "stdout", Data: "x"}})724 },725 "result": func() error { return a.cl.Result(ctx, id.String(), protocol.CommandResult{Status: "succeeded"}) },726 } {727 if err := call(); status(err) != 404 {728 t.Errorf("agent A %s on B's command: %v", name, err)729 }730 }731 if h.cmdRow(id).State != "delivered" || scan[int](h, `select count(*) from action_events`) != 0 {732 t.Error("agent A changed B's command")733 }734 // Lifecycle order is enforced for the owner too.735 if err := b.cl.Events(ctx, id.String(), []protocol.CommandOutput{{Stream: "stdout", Data: "x"}}); status(err) != 409 {736 t.Errorf("output before ack: %v", err)737 }738 if err := b.cl.Ack(ctx, id.String(), protocol.CommandAck{Accepted: true}); err != nil {739 t.Fatal(err)740 }741 if err := b.cl.Ack(ctx, id.String(), protocol.CommandAck{Accepted: true}); status(err) != 409 {742 t.Errorf("double ack: %v", err)743 }744 // Server-side second redaction pass and bounds.745 secretLines := []protocol.CommandOutput{746 {Stream: "stdout", Data: "Authorization: Bearer abcdefghijklmnop\nDB_PASSWORD=hunter2\nfine line"},747 {Stream: "stderr", Data: "https://user:pw123@example.org/x"},748 }749 if err := b.cl.Events(ctx, id.String(), secretLines); err != nil {750 t.Fatal(err)751 }752 stored := scan[string](h, `select string_agg(data, '|' order by id) from action_events where kind='output'`)753 for _, leak := range []string{"abcdefghijklmnop", "hunter2", "pw123"} {754 if strings.Contains(stored, leak) {755 t.Errorf("server stored a secret: %q", stored)756 }757 }758 if !strings.Contains(stored, "fine line") || scan[int](h, `select count(*) from action_events`) != 4 {759 t.Errorf("stored output: %q", stored)760 }761 var flood []protocol.CommandOutput762 for i := 0; i < 2500; i++ {763 flood = append(flood, protocol.CommandOutput{Stream: "stdout", Data: fmt.Sprintf("line %d", i)})764 }765 for i := 0; i < len(flood); i += 500 {766 if err := b.cl.Events(ctx, id.String(), flood[i:i+500]); err != nil {767 t.Fatal(err)768 }769 }770 n := scan[int](h, `select count(*) from action_events where kind='output'`)771 last := scan[string](h, `select data from action_events order by id desc limit 1`)772 if n != 2001 || !strings.Contains(last, "truncated") {773 t.Errorf("output bound: %d rows, last %q", n, last)774 }775 if err := b.cl.Result(ctx, id.String(), protocol.CommandResult{Status: "weird"}); status(err) != 400 {776 t.Errorf("bad status: %v", err)777 }778 if err := b.cl.Result(ctx, id.String(), protocol.CommandResult{Status: "failed", Error: "TOKEN=abc123xyz failed", Result: map[string]any{"password": "p", "note": "ok"}}); err != nil {779 t.Fatal(err)780 }781 row := h.cmdRow(id)782 if row.State != "finished" || deref(row.ResultStatus) != "failed" || strings.Contains(fmt.Sprint(row.Result), "abc123xyz") || fmt.Sprint(row.Result["result"].(map[string]any)["password"]) != "[redacted]" {783 t.Errorf("result: %+v", row)784 }785 if err := b.cl.Result(ctx, id.String(), protocol.CommandResult{Status: "succeeded"}); status(err) != 409 {786 t.Errorf("result on a closed command: %v", err)787 }788 // An expired pending command is never delivered.789 late := h.command(b, "diagnostic.run", nil, cmdOpt{})790 h.exec(`update agent_commands set expires_at = now() - interval '1 second' where id=$1`, late)791 if resp, _ := b.cl.Commands(ctx, 0); len(resp.Commands) != 0 {792 t.Error("expired command delivered")793 }794 })795}796797func TestOfflineBufferingAndReplay(t *testing.T) {798 forModes(t, func(t *testing.T, mode string) {799 h := newHarness(t, mode, hopt{netProxy: true})800 e := h.enroll("n1")801 h.startAgent(e)802 seqs := func() int {803 return scan[int](h, `select count(*) from agent_report_seqs where agent_id=$1`, e.id.AgentID)804 }805 eventually(t, 15*time.Second, "live reports", func() bool { return seqs() >= 3 })806807 h.netProxy.cut()808 spooled := func() int {809 des, _ := os.ReadDir(agent.SpoolDir(e.dir))810 n := 0811 for _, d := range des {812 if strings.HasSuffix(d.Name(), ".rpt") {813 n++814 }815 }816 return n817 }818 eventually(t, 30*time.Second, "reports buffered on disk while offline", func() bool { return spooled() >= 5 })819 before := seqs()820 stateBefore := scan[time.Time](h, `select ts from node_state where node_id=$1`, e.id.NodeID)821 st, _ := e.dir.LoadStatus()822 if st == nil || st.Connection != "offline" || st.SpoolReports == 0 {823 t.Errorf("status while offline: %+v", st)824 }825 buffered := spooled()826827 h.netProxy.restore()828 eventually(t, 60*time.Second, "spool drained after reconnection", func() bool { return spooled() == 0 })829 eventually(t, 10*time.Second, "live reports resumed", func() bool {830 return scan[time.Time](h, `select ts from node_state where node_id=$1`, e.id.NodeID).After(stateBefore.Add(2 * time.Second))831 })832 if got := seqs(); got < before+buffered {833 t.Errorf("buffered reports lost: %d before, %d buffered, %d now", before, buffered, got)834 }835 // No duplicate: every (agent, seq) once, every sample key once (primary keys), and each stored836 // report timestamp carries its samples exactly once.837 if d := scan[int](h, `select count(*) from (select metric_id, label, ts from samples_raw where node_id=$1 group by 1,2,3 having count(*) > 1) x`, e.id.NodeID); d != 0 {838 t.Errorf("%d duplicated samples", d)839 }840 // Samples measured while offline are in the database with their original timestamps.841 if n := scan[int](h, `select count(distinct ts) from samples_raw where node_id=$1 and ts > $2 and ts < now() - interval '1 second'`, e.id.NodeID, stateBefore); n < buffered-1 {842 t.Errorf("only %d offline report timestamps stored for %d buffered reports", n, buffered)843 }844 // node_state is the newest report, not a replayed one.845 newest := scan[time.Time](h, `select max(ts) from agent_report_seqs where agent_id=$1`, e.id.AgentID)846 eventually(t, 5*time.Second, "node_state tracks the newest report", func() bool {847 ts := scan[time.Time](h, `select ts from node_state where node_id=$1`, e.id.NodeID)848 return !ts.Before(newest)849 })850 })851}852853func TestDatabaseOutage(t *testing.T) {854 forModes(t, func(t *testing.T, mode string) {855 h := newHarness(t, mode, hopt{dbProxy: true})856 e := h.enroll("n1")857 if _, err := h.send(e, report(1, time.Now(), sample("cpu.usage", 1))); err != nil {858 t.Fatal(err)859 }860 time.Sleep(300 * time.Millisecond) // let the authentication cache expire861 h.dbProxy.cut()862 _, err := h.send(e, report(2, time.Now(), sample("cpu.usage", 2)))863 if status(err) != 503 {864 t.Fatalf("database down: got %v, want 503", err)865 }866 if client.Permanent(err) || client.Revoked(err) {867 t.Error("an outage must look transient to the agent (buffer and retry), never like a revocation")868 }869 rec := httptest.NewRecorder()870 h.health.ServeHTTP(rec, httptest.NewRequest("GET", "/healthz", nil))871 if rec.Code != 503 || !strings.Contains(rec.Body.String(), `"ok":false`) {872 t.Errorf("healthz during outage: %d %s", rec.Code, rec.Body.String())873 }874 if _, err := e.cl.Commands(context.Background(), 0); status(err) != 503 {875 t.Errorf("commands during outage: %v", err)876 }877 h.dbProxy.restore()878 eventually(t, 20*time.Second, "service recovers by itself", func() bool {879 _, err := h.send(e, report(2, time.Now(), sample("cpu.usage", 2)))880 return err == nil881 })882 rec = httptest.NewRecorder()883 h.health.ServeHTTP(rec, httptest.NewRequest("GET", "/healthz", nil))884 if rec.Code != 200 {885 t.Errorf("healthz after recovery: %d", rec.Code)886 }887 if scan[int](h, `select count(*) from samples_raw`) != 2 {888 t.Error("report lost or duplicated across the outage")889 }890 })891}892893func mustUUID(s string) uuid.UUID { return uuid.MustParse(s) }894895func deref(s *string) string {896 if s == nil {897 return ""898 }899 return *s900}901902var _ = http.StatusOK903