Repository Public
1package ingest_test23import (4 "bytes"5 "context"6 "crypto/ecdsa"7 "crypto/elliptic"8 "crypto/rand"9 "crypto/x509"10 "crypto/x509/pkix"11 "encoding/base64"12 "encoding/json"13 "encoding/pem"14 "io"15 "math/big"16 "net/http"17 "os"18 "path/filepath"19 "strconv"20 "testing"21 "time"2223 "github.com/google/uuid"2425 "clustr.spboucher.ai/mcc/internal/agent/client"26 "clustr.spboucher.ai/mcc/internal/agent/journal"27 "clustr.spboucher.ai/mcc/internal/ids"28 "clustr.spboucher.ai/mcc/internal/pki"29 "clustr.spboucher.ai/mcc/internal/protocol"30)3132// signedReq builds a signed request by hand so each element can be falsified.33type signedReq struct {34 method, path string35 body []byte36 certDER []byte37 key *ecdsa.PrivateKey38 ts int6439 nonce string40 mutate func(r *http.Request, body *[]byte)41}4243func (h *harness) doSigned(s signedReq) (int, string) {44 h.t.Helper()45 sig, err := protocol.SignRequest(s.key, s.method, s.path, s.ts, s.nonce, s.body)46 if err != nil {47 h.t.Fatal(err)48 }49 body := s.body50 req, _ := http.NewRequest(s.method, h.front.URL+s.path, nil)51 req.Header.Set(protocol.HeaderAgentCert, base64.StdEncoding.EncodeToString(s.certDER))52 req.Header.Set(protocol.HeaderAgentTS, strconv.FormatInt(s.ts, 10))53 req.Header.Set(protocol.HeaderAgentNonce, s.nonce)54 req.Header.Set(protocol.HeaderAgentSig, sig)55 req.Header.Set("Content-Type", "application/json")56 if s.mutate != nil {57 s.mutate(req, &body)58 }59 req.Body = io.NopCloser(bytes.NewReader(body))60 req.ContentLength = int64(len(body))61 resp, err := http.DefaultClient.Do(req)62 if err != nil {63 h.t.Fatal(err)64 }65 defer resp.Body.Close()66 var doc struct {67 Error struct{ Code string } `json:"error"`68 }69 json.NewDecoder(resp.Body).Decode(&doc)70 return resp.StatusCode, doc.Error.Code71}7273// caKey loads the CA private key of the harness (tests mint odd certificates with it).74func caKey(t *testing.T, dir string) *ecdsa.PrivateKey {75 b, err := os.ReadFile(filepath.Join(dir, "agent-ca.key"))76 if err != nil {77 t.Fatal(err)78 }79 blk, _ := pem.Decode(b)80 k, err := x509.ParseECPrivateKey(blk.Bytes)81 if err != nil {82 t.Fatal(err)83 }84 return k85}8687func TestSignedRequestRejections(t *testing.T) {88 h := newHarness(t, "signed", hopt{})89 e := h.enroll("n1")90 cert, _, _ := e.dir.LoadCredentials()91 key := cert.PrivateKey.(*ecdsa.PrivateKey)92 der := cert.Certificate[0]93 seq := int64(100)94 body := func() []byte {95 seq++96 b, _ := json.Marshal(report(seq, time.Now(), sample("cpu.usage", 1)))97 return b98 }99 base := func() signedReq {100 return signedReq{method: "POST", path: "/agent/v1/report", body: body(), certDER: der, key: key, ts: time.Now().UnixMilli(), nonce: protocol.NewRequestNonce()}101 }102 rows := func() int {103 return scan[int](h, `select (select count(*) from samples_raw) + (select count(*) from agent_report_seqs)`)104 }105106 // Reference: a valid signed report is accepted.107 ok := base()108 if st, c := h.doSigned(ok); st != 200 {109 t.Fatalf("valid signed report refused: %d %s", st, c)110 }111 written := rows()112 if written != 2 {113 t.Fatalf("%d rows after one report", written)114 }115116 otherKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)117 // Certificates minted with the real CA key but unfit for agent authentication.118 // The CA key file lives in the harness secrets dir; recover it through a second CA handle.119 type tc struct {120 name string121 req func() signedReq122 code string123 }124 other := newHarness(t, "signed", hopt{})125 foreign := other.enroll("stranger")126 fcert, _, _ := foreign.dir.LoadCredentials()127128 cases := []tc{129 {"wrong signature (another key)", func() signedReq { r := base(); r.key = otherKey; return r }, "signature_invalid"},130 {"body modified after signing", func() signedReq {131 r := base()132 r.mutate = func(_ *http.Request, b *[]byte) { *b = bytes.Replace(*b, []byte(`"v":1}`), []byte(`"v":9}`), 1) }133 return r134 }, "signature_invalid"},135 {"signature for another path", func() signedReq {136 r := base()137 r.mutate = func(req *http.Request, _ *[]byte) { req.URL.Path = "/agent/v1/renew" }138 return r139 }, "signature_invalid"},140 {"signature for another method/query", func() signedReq {141 r := base()142 r.mutate = func(req *http.Request, _ *[]byte) { req.URL.RawQuery = "x=1" }143 return r144 }, "signature_invalid"},145 {"timestamp 91 s in the past", func() signedReq { r := base(); r.ts = time.Now().Add(-91 * time.Second).UnixMilli(); return r }, "timestamp_out_of_window"},146 {"timestamp 91 s in the future", func() signedReq { r := base(); r.ts = time.Now().Add(91 * time.Second).UnixMilli(); return r }, "timestamp_out_of_window"},147 {"replayed nonce", func() signedReq { r := base(); r.nonce = ok.nonce; return r }, "nonce_replayed"},148 {"exact replay of an accepted request", func() signedReq { return ok }, "nonce_replayed"},149 {"certificate from another CA", func() signedReq {150 r := base()151 r.certDER, r.key = fcert.Certificate[0], fcert.PrivateKey.(*ecdsa.PrivateKey)152 return r153 }, "certificate_invalid"},154 {"missing certificate header", func() signedReq {155 r := base()156 r.mutate = func(req *http.Request, _ *[]byte) { req.Header.Del(protocol.HeaderAgentCert) }157 return r158 }, "signature_headers_missing"},159 {"missing signature header", func() signedReq {160 r := base()161 r.mutate = func(req *http.Request, _ *[]byte) { req.Header.Del(protocol.HeaderAgentSig) }162 return r163 }, "signature_headers_missing"},164 {"missing nonce", func() signedReq {165 r := base()166 r.mutate = func(req *http.Request, _ *[]byte) { req.Header.Del(protocol.HeaderAgentNonce) }167 return r168 }, "signature_headers_missing"},169 {"missing timestamp", func() signedReq {170 r := base()171 r.mutate = func(req *http.Request, _ *[]byte) { req.Header.Del(protocol.HeaderAgentTS) }172 return r173 }, "signature_headers_missing"},174 {"garbage certificate", func() signedReq {175 r := base()176 r.mutate = func(req *http.Request, _ *[]byte) { req.Header.Set(protocol.HeaderAgentCert, "AAAA") }177 return r178 }, "certificate_invalid"},179 }180 for _, c := range cases {181 st, code := h.doSigned(c.req())182 if st != 401 || code != c.code {183 t.Errorf("%s: got %d %q, want 401 %q", c.name, st, code, c.code)184 }185 }186 if rows() != written {187 t.Errorf("a rejected signed request wrote to the database (%d -> %d)", written, rows())188 }189 // No headers at all (a plain HTTP client hitting the public path).190 resp, err := http.Post(h.front.URL+"/agent/v1/report", "application/json", bytes.NewReader(body()))191 if err != nil || resp.StatusCode != 401 {192 t.Errorf("unsigned request: %v", err)193 }194 // Revoked agent.195 h.exec(`update agents set revoked_at = now() where id=$1`, e.id.AgentID)196 time.Sleep(300 * time.Millisecond)197 if st, code := h.doSigned(base()); st != 401 || code != "agent_revoked" {198 t.Errorf("revoked agent: %d %s", st, code)199 }200 if rows() != written {201 t.Error("revoked agent wrote")202 }203}204205// mint issues a certificate with the harness CA key and arbitrary properties.206func mint(t *testing.T, ca *pki.CA, cakey *ecdsa.PrivateKey, agentID uuid.UUID, notAfter time.Time, eku []x509.ExtKeyUsage) ([]byte, *ecdsa.PrivateKey) {207 key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)208 serial, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 120))209 tmpl := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: agentID.String()}, URIs: nil,210 NotBefore: time.Now().Add(-48 * time.Hour), NotAfter: notAfter, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: eku}211 tmpl.URIs = append(tmpl.URIs, pki.AgentURI(agentID))212 der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Cert, &key.PublicKey, cakey)213 if err != nil {214 t.Fatal(err)215 }216 return der, key217}218219func TestSignedRequestCertificateProperties(t *testing.T) {220 // A dedicated CA directory so the test can read the CA key.221 h := newHarness(t, "signed", hopt{})222 dir := t.TempDir()223 ca2, err := pki.LoadOrCreate(dir)224 if err != nil {225 t.Fatal(err)226 }227 key2 := caKey(t, dir)228 h2 := newHarnessWithCA(t, "signed", ca2)229 e := h2.enroll("n1")230 agentID := uuid.MustParse(e.id.AgentID)231 body, _ := json.Marshal(report(1, time.Now(), sample("cpu.usage", 1)))232 try := func(der []byte, key *ecdsa.PrivateKey) (int, string) {233 // Register the serial as the agent's current one: only the certificate property under test differs.234 c, _ := x509.ParseCertificate(der)235 h2.exec(`update agents set cert_serial=$2 where id=$1`, e.id.AgentID, pki.SerialString(c.SerialNumber))236 time.Sleep(250 * time.Millisecond)237 return h2.doSigned(signedReq{method: "POST", path: "/agent/v1/report", body: body, certDER: der, key: key, ts: time.Now().UnixMilli(), nonce: protocol.NewRequestNonce()})238 }239 clientAuth := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}240 der, key := mint(t, ca2, key2, agentID, time.Now().Add(time.Hour), clientAuth)241 if st, c := try(der, key); st != 200 {242 t.Fatalf("control certificate refused: %d %s", st, c)243 }244 der, key = mint(t, ca2, key2, agentID, time.Now().Add(-time.Hour), clientAuth)245 if st, c := try(der, key); st != 401 || c != "certificate_invalid" {246 t.Errorf("expired certificate: %d %s", st, c)247 }248 der, key = mint(t, ca2, key2, agentID, time.Now().Add(time.Hour), []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})249 if st, c := try(der, key); st != 401 || c != "certificate_invalid" {250 t.Errorf("certificate without clientAuth: %d %s", st, c)251 }252 // A valid certificate of agent A whose serial is registered for agent B must not pass as B.253 der, key = mint(t, ca2, key2, ids.New(), time.Now().Add(time.Hour), clientAuth)254 if st, c := try(der, key); st != 401 {255 t.Errorf("certificate naming another agent: %d %s", st, c)256 }257 if n := scan[int](h2, `select count(*) from agent_report_seqs`); n != 1 {258 t.Errorf("%d reports stored, want only the control one", n)259 }260 _ = h261}262263func TestLongPollThroughReverseProxy(t *testing.T) {264 h := newHarness(t, "signed", hopt{})265 e := h.enroll("n1")266 // 1. A command arriving during the poll is delivered through the proxy at once.267 type res struct {268 n int269 err error270 d time.Duration271 }272 ch := make(chan res, 1)273 start := time.Now()274 go func() {275 r, err := e.cl.Commands(context.Background(), 20*time.Second)276 n := 0277 if r != nil {278 n = len(r.Commands)279 }280 ch <- res{n, err, time.Since(start)}281 }()282 eventually(t, 5*time.Second, "long-poll held open behind the proxy", func() bool { return h.srv.ActiveLongPolls() == 1 })283 h.command(e, "diagnostic.run", nil, cmdOpt{})284 select {285 case r := <-ch:286 if r.err != nil || r.n != 1 || r.d > 5*time.Second {287 t.Fatalf("delivery through the proxy: %+v", r)288 }289 case <-time.After(10 * time.Second):290 t.Fatal("command not delivered through the proxy")291 }292 // 2. An empty poll returns at its deadline with an empty list.293 start = time.Now()294 r, err := e.cl.Commands(context.Background(), time.Second)295 if err != nil || len(r.Commands) != 0 || time.Since(start) < 900*time.Millisecond || time.Since(start) > 4*time.Second {296 t.Errorf("empty poll: %v %v after %s", r, err, time.Since(start))297 }298 // 3. The client goes away: the cancellation crosses the proxy and frees the server side.299 ctx, cancel := context.WithCancel(context.Background())300 go e.cl.Commands(ctx, 30*time.Second)301 eventually(t, 5*time.Second, "second long-poll open", func() bool { return h.srv.ActiveLongPolls() == 1 })302 cancel()303 eventually(t, 5*time.Second, "server released the abandoned long-poll", func() bool { return h.srv.ActiveLongPolls() == 0 })304}305306func TestUnknownOutcomeAfterAgentCrash(t *testing.T) {307 forModes(t, func(t *testing.T, mode string) {308 h := newHarness(t, mode, hopt{})309 e := h.enroll("n1")310 ctx := context.Background()311 // The agent takes a command, journals it, acknowledges it… and dies before finishing.312 id := h.command(e, "diagnostic.run", nil, cmdOpt{})313 if r, err := e.cl.Commands(ctx, time.Second); err != nil || len(r.Commands) != 1 {314 t.Fatal(err)315 }316 j, _, err := journal.Open(e.dir.File("commands.journal"))317 if err != nil {318 t.Fatal(err)319 }320 j.Started(id.String(), "diagnostic.run")321 j.Close()322 if err := e.cl.Ack(ctx, id.String(), protocol.CommandAck{Accepted: true}); err != nil {323 t.Fatal(err)324 }325 events := scan[int](h, `select count(*) from action_events`)326 // Restart.327 h.startAgent(e)328 r := h.waitCmd(id, "finished", 30*time.Second)329 if deref(r.ResultStatus) != "failed" || r.Result["result"].(map[string]any)["outcome_unknown"] != true {330 t.Fatalf("after a crash the command must be failed/outcome unknown: %+v", r)331 }332 time.Sleep(time.Second)333 if n := scan[int](h, `select count(*) from action_events`); n != events {334 t.Error("the interrupted command was executed again after the restart")335 }336 })337}338339var _ = client.IsURL340