Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/ingest/harness_test.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest_test4344import (45 "bytes"46 "compress/gzip"47 "context"48 "crypto/ed25519"49 "crypto/rand"50 "crypto/sha256"51 "crypto/tls"52 "encoding/hex"53 "encoding/json"54 "fmt"55 "io"56 "log/slog"57 "net"58 "net/http"59 "net/http/httptest"60 "net/http/httputil"61 "net/url"62 "strings"63 "sync"64 "testing"65 "time"6667 "github.com/google/uuid"68 "github.com/jackc/pgx/v5"69 "github.com/jackc/pgx/v5/pgxpool"7071 "clustr.spboucher.ai/mcc/internal/agent/client"72 "clustr.spboucher.ai/mcc/internal/agent/state"73 "clustr.spboucher.ai/mcc/internal/config"74 "clustr.spboucher.ai/mcc/internal/db"75 "clustr.spboucher.ai/mcc/internal/ids"76 "clustr.spboucher.ai/mcc/internal/ingest"77 "clustr.spboucher.ai/mcc/internal/pki"78 "clustr.spboucher.ai/mcc/internal/protocol"79 "clustr.spboucher.ai/mcc/internal/testdb"80)8182// Both agent transports run the same suite.83var modes = []string{"mtls", "signed"}8485func forModes(t *testing.T, fn func(t *testing.T, mode string)) {86 for _, m := range modes {87 t.Run(m, func(t *testing.T) { fn(t, m) })88 }89}9091// tcpProxy is a cuttable TCP relay, used to simulate a network or database outage.92type tcpProxy struct {93 ln net.Listener94 target string95 mu sync.Mutex96 down bool97 conns map[net.Conn]struct{}98}99100func newTCPProxy(t *testing.T, target string) *tcpProxy {101 ln, err := net.Listen("tcp", "127.0.0.1:0")102 if err != nil {103 t.Fatal(err)104 }105 p := &tcpProxy{ln: ln, target: target, conns: map[net.Conn]struct{}{}}106 go func() {107 for {108 c, err := ln.Accept()109 if err != nil {110 return111 }112 p.mu.Lock()113 down := p.down114 p.mu.Unlock()115 if down {116 c.Close()117 continue118 }119 up, err := net.DialTimeout("tcp", target, 2*time.Second)120 if err != nil {121 c.Close()122 continue123 }124 p.mu.Lock()125 p.conns[c], p.conns[up] = struct{}{}, struct{}{}126 p.mu.Unlock()127 go func() { io.Copy(up, c); up.Close(); c.Close() }()128 go func() { io.Copy(c, up); up.Close(); c.Close() }()129 }130 }()131 t.Cleanup(func() { ln.Close(); p.cut() })132 return p133}134135func (p *tcpProxy) addr() string { return p.ln.Addr().String() }136func (p *tcpProxy) cut() {137 p.mu.Lock()138 p.down = true139 for c := range p.conns {140 c.Close()141 }142 p.conns = map[net.Conn]struct{}{}143 p.mu.Unlock()144}145func (p *tcpProxy) restore() { p.mu.Lock(); p.down = false; p.mu.Unlock() }146147type harness struct {148 t *testing.T149 mode string150 pool *pgxpool.Pool // direct connection, for assertions151 srv *ingest.Server152 cfg *config.Config153 ca *pki.CA154 endpoint string // what agents are given155 mtlsAddr string156 signedLn string // loopback address of the signed listener (behind the proxy)157 front *httptest.Server158 dbProxy *tcpProxy159 netProxy *tcpProxy160 orchKey ed25519.PrivateKey161 keyID string162 health http.Handler163}164165type hopt struct {166 dbProxy bool167 netProxy bool168 retain time.Duration169 ca *pki.CA170}171172// newHarnessWithCA uses a CA the test created itself (to mint special certificates).173func newHarnessWithCA(t *testing.T, mode string, ca *pki.CA) *harness {174 return newHarness(t, mode, hopt{ca: ca})175}176177func newHarness(t *testing.T, mode string, o hopt) *harness {178 t.Helper()179 pool, dbURL := testdb.New(t)180 h := &harness{t: t, mode: mode, pool: pool}181 srvPool := pool182 if o.dbProxy {183 u, _ := url.Parse(dbURL)184 h.dbProxy = newTCPProxy(t, u.Host)185 u.Host = h.dbProxy.addr()186 q := u.Query()187 q.Set("connect_timeout", "2")188 u.RawQuery = q.Encode()189 var err error190 if srvPool, err = db.Open(context.Background(), u.String()); err != nil {191 t.Fatal(err)192 }193 t.Cleanup(srvPool.Close)194 }195 ca := o.ca196 if ca == nil {197 var err error198 if ca, err = pki.LoadOrCreate(t.TempDir()); err != nil {199 t.Fatal(err)200 }201 }202 h.ca = ca203 var err error204 if o.retain == 0 {205 o.retain = 7 * 24 * time.Hour206 }207 h.cfg = &config.Config{Env: config.Test, IngestServerName: "ingest.mcc.internal", RetainRaw: o.retain,208 Retain1m: 30 * 24 * time.Hour, Retain1h: 400 * 24 * time.Hour, FreshAfter: 30 * time.Second, OfflineAfter: 120 * time.Second}209210 mln, err := net.Listen("tcp", "127.0.0.1:0")211 if err != nil {212 t.Fatal(err)213 }214 sln, err := net.Listen("tcp", "127.0.0.1:0")215 if err != nil {216 t.Fatal(err)217 }218 h.mtlsAddr, h.signedLn = mln.Addr().String(), sln.Addr().String()219 // The front mimics control-api: a real reverse proxy before the signed listener.220 target, _ := url.Parse("http://" + h.signedLn)221 rp := httputil.NewSingleHostReverseProxy(target)222 rp.FlushInterval = -1223 rp.ErrorLog = slog.NewLogLogger(slog.DiscardHandler, slog.LevelError)224 h.front = httptest.NewServer(rp)225 t.Cleanup(h.front.Close)226227 h.endpoint = h.mtlsAddr228 if mode == "signed" {229 h.endpoint = h.front.URL230 }231 if o.netProxy {232 up := h.mtlsAddr233 if mode == "signed" {234 up = strings.TrimPrefix(h.front.URL, "http://")235 }236 h.netProxy = newTCPProxy(t, up)237 h.endpoint = h.netProxy.addr()238 if mode == "signed" {239 h.endpoint = "http://" + h.netProxy.addr()240 }241 }242 h.cfg.IngestAdvertise = []string{h.endpoint}243244 h.srv, err = ingest.New(ingest.Options{Cfg: h.cfg, Pool: srvPool, CA: ca, Version: "test", EnrollPerMinute: 6000, EnrollBurst: 1000,245 AuthCacheTTL: 150 * time.Millisecond, ReportIntervalS: 10, Log: slog.New(slog.DiscardHandler)})246 if err != nil {247 t.Fatal(err)248 }249 ctx, cancel := context.WithCancel(context.Background())250 if err := h.srv.SeedCatalog(ctx); err != nil {251 t.Fatal(err)252 }253 if err := h.srv.EnsurePartitions(ctx); err != nil {254 t.Fatal(err)255 }256 var wg sync.WaitGroup257 wg.Add(3)258 go func() { defer wg.Done(); h.srv.Serve(ctx, mln) }()259 go func() { defer wg.Done(); h.srv.ServeSigned(ctx, sln) }()260 go func() { defer wg.Done(); h.srv.ListenCommandsForTest(ctx) }()261 t.Cleanup(func() { cancel(); wg.Wait() })262 h.health = h.srv.HealthHandler()263264 // The orchestrator's signing key lives in the test only: ingest never sees it.265 pub, priv, _ := ed25519.GenerateKey(rand.Reader)266 h.orchKey, h.keyID = priv, "test-key-1"267 h.exec(`insert into command_keys(key_id, public_key) values ($1,$2)`, h.keyID, []byte(pub))268 return h269}270271func (h *harness) exec(sql string, args ...any) {272 h.t.Helper()273 if _, err := h.pool.Exec(context.Background(), sql, args...); err != nil {274 h.t.Fatalf("%s: %v", sql, err)275 }276}277278func scan[T any](h *harness, sql string, args ...any) T {279 h.t.Helper()280 var v T281 if err := h.pool.QueryRow(context.Background(), sql, args...).Scan(&v); err != nil {282 h.t.Fatalf("%s: %v", sql, err)283 }284 return v285}286287func (h *harness) token(alias string, ttl time.Duration) string {288 h.t.Helper()289 tok, err := ingest.CreateEnrollmentToken(context.Background(), h.pool, alias, "mac", ttl)290 if err != nil {291 h.t.Fatal(err)292 }293 return tok294}295296func (h *harness) fingerprint() []byte {297 fp, _ := pki.ParseFingerprint(h.ca.Fingerprint())298 return fp299}300301// testInventory is synthetic and lives only in the throwaway test database.302func testInventory(hw string) protocol.Inventory {303 return protocol.Inventory{HardwareUUID: hw, Hostname: "test-host", Kind: "mac", Model: "TestMac1,1", Chip: "Test Chip", Arch: "arm64",304 CPUCores: 8, CPULevels: []protocol.CPULevel{{Name: "Super", Cores: 2}, {Name: "Performance", Cores: 2}, {Name: "Efficiency", Cores: 4}},305 MemBytes: 16 << 30, StorageBytes: 500 << 30, OSName: "macOS", OSVersion: "27.0", OSBuild: "TEST",306 Interfaces: []protocol.Interface{{Name: "en0", Kind: "ethernet", Addrs: []string{"192.0.2.10/24"}, Up: true, Default: true}},307 Managers: []string{"launchd"}}308}309310func newHW() string { return "TEST-" + strings.ToUpper(uuid.NewString()) }311312type enrolled struct {313 dir *state.Dir314 cl *client.Client315 id *state.Identity316 hw string317}318319func (h *harness) enrollErr(alias, hw, token string, fp []byte) (*enrolled, error) {320 dir, err := state.Open(h.t.TempDir())321 if err != nil {322 h.t.Fatal(err)323 }324 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)325 defer cancel()326 id, err := client.Enroll(ctx, dir, client.EnrollParams{Endpoints: []string{h.endpoint}, Token: token, CAFingerprint: fp,327 Inventory: testInventory(hw), Version: "test", Attempts: 1})328 if err != nil {329 return &enrolled{dir: dir, hw: hw}, err330 }331 cl, err := client.New(dir)332 if err != nil {333 h.t.Fatal(err)334 }335 h.t.Cleanup(cl.Close)336 return &enrolled{dir: dir, cl: cl, id: id, hw: hw}, nil337}338339func (h *harness) enroll(alias string) *enrolled {340 h.t.Helper()341 e, err := h.enrollErr(alias, newHW(), h.token(alias, 15*time.Minute), h.fingerprint())342 if err != nil {343 h.t.Fatalf("enroll %s: %v", alias, err)344 }345 return e346}347348func sample(m string, v float64) protocol.Sample { return protocol.Sample{M: m, V: v} }349350func report(seq int64, ts time.Time, samples ...protocol.Sample) *protocol.Report {351 return &protocol.Report{V: protocol.Version, Seq: seq, TS: ts.UTC(), AgentVersion: "test", Samples: samples}352}353354func (h *harness) send(e *enrolled, rep *protocol.Report) (*protocol.ReportAck, error) {355 enc, err := client.EncodeReport(rep)356 if err != nil {357 h.t.Fatal(err)358 }359 ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)360 defer cancel()361 return e.cl.Report(ctx, enc)362}363364// sendRaw posts arbitrary bytes as a report body (to send what the Go encoder refuses to produce).365func (h *harness) sendRaw(e *enrolled, body string) error {366 var buf bytes.Buffer367 zw := gzip.NewWriter(&buf)368 zw.Write([]byte(body))369 zw.Close()370 ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)371 defer cancel()372 _, err := e.cl.Report(ctx, buf.Bytes())373 return err374}375376func status(err error) int {377 var ae *client.APIError378 if e, ok := err.(*client.APIError); ok {379 ae = e380 }381 if ae == nil {382 return 0383 }384 return ae.Status385}386387func code(err error) string {388 if e, ok := err.(*client.APIError); ok {389 return e.Code390 }391 return ""392}393394// command inserts an action and its signed command, as the orchestrator does.395type cmdOpt struct {396 key ed25519.PrivateKey397 keyID string398 issued time.Time399 lifetime time.Duration400 tamper bool401 agentID string // recipient written in the envelope (default: the row's agent)402}403404func (h *harness) command(e *enrolled, typ string, params any, o cmdOpt) uuid.UUID {405 h.t.Helper()406 if o.key == nil {407 o.key, o.keyID = h.orchKey, h.keyID408 }409 if o.issued.IsZero() {410 o.issued = time.Now()411 }412 if o.lifetime == 0 {413 o.lifetime = 60 * time.Second414 }415 if o.agentID == "" {416 o.agentID = e.id.AgentID417 }418 actionID, cmdID := ids.New(), ids.New()419 pj, _ := json.Marshal(params)420 if params == nil {421 pj = []byte("{}")422 }423 sum := sha256.Sum256(pj)424 h.exec(`insert into actions(id, type, criticality, node_id, target_kind, target_id, params, state, idempotency_key, request_hash, timeout_s, correlation_id)425 values ($1,$2,'low',$3::uuid,'node',$8,$4,'dispatched',$5,$6,30,$7)`, actionID, typ, e.id.NodeID, pj, cmdID.String(), sum[:], ids.New(), e.id.NodeID)426 sc, err := protocol.Sign(o.key, o.keyID, protocol.Command{V: protocol.Version, ID: cmdID.String(), ActionID: actionID.String(),427 AgentID: o.agentID, NodeID: e.id.NodeID, Type: typ, Params: pj, TimeoutS: 30,428 IssuedAt: o.issued.UTC(), ExpiresAt: o.issued.Add(o.lifetime).UTC(), Nonce: protocol.NewNonce()})429 if err != nil {430 h.t.Fatal(err)431 }432 if o.tamper {433 // A compromised transport rewrites the instruction after it was signed.434 sc.Payload = bytes.Replace(sc.Payload, []byte(`"`+typ+`"`), []byte(`"system.shutdown"`), 1)435 }436 // The row's own expiry is generous: expiry must be enforced by the agent on the signed envelope.437 h.exec(`insert into agent_commands(id, action_id, agent_id, envelope, signature, key_id, expires_at) values ($1,$2,$3,$4,$5,$6, now() + interval '5 minutes')`,438 cmdID, actionID, e.id.AgentID, sc.Payload, sc.Signature, sc.KeyID)439 h.exec(`select pg_notify('mcc_commands', $1)`, e.id.AgentID)440 return cmdID441}442443type cmdRow struct {444 State string445 AckAccepted *bool446 AckReason *string447 ResultStatus *string448 Result map[string]any449}450451func (h *harness) cmdRow(id uuid.UUID) cmdRow {452 h.t.Helper()453 var r cmdRow454 err := h.pool.QueryRow(context.Background(), `select state, ack_accepted, ack_reason, result_status, result from agent_commands where id=$1`, id).455 Scan(&r.State, &r.AckAccepted, &r.AckReason, &r.ResultStatus, &r.Result)456 if err != nil && err != pgx.ErrNoRows {457 h.t.Fatal(err)458 }459 return r460}461462func (h *harness) waitCmd(id uuid.UUID, state string, within time.Duration) cmdRow {463 h.t.Helper()464 deadline := time.Now().Add(within)465 for {466 r := h.cmdRow(id)467 if r.State == state {468 return r469 }470 if time.Now().After(deadline) {471 h.t.Fatalf("command %s: state %q after %s, want %q (%+v)", id, r.State, within, state, r)472 }473 time.Sleep(50 * time.Millisecond)474 }475}476477func eventually(t *testing.T, within time.Duration, what string, cond func() bool) {478 t.Helper()479 deadline := time.Now().Add(within)480 for !cond() {481 if time.Now().After(deadline) {482 t.Fatalf("timeout after %s waiting for: %s", within, what)483 }484 time.Sleep(50 * time.Millisecond)485 }486}487488func hexsum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }489490var _ = fmt.Sprintf491492func parseUUID(s string) ([16]byte, error) {493 id, err := uuid.Parse(s)494 return id, err495}496497func insecureGet(addr, path string) (int, error) {498 tr := &http.Transport{TLSClientConfig: insecureTLS()}499 defer tr.CloseIdleConnections()500 resp, err := (&http.Client{Transport: tr, Timeout: 5 * time.Second}).Get("https://" + addr + path)501 if err != nil {502 return 0, err503 }504 resp.Body.Close()505 return resp.StatusCode, nil506}507508func insecureTLS() *tls.Config { return &tls.Config{InsecureSkipVerify: true} } // test probe only: no client certificate509