SPB Git

Repository Public

harness_test.go 14.8 KB 509 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/ingest/harness_test.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest_test4344import (45	"bytes"46	"compress/gzip"47	"context"48	"crypto/ed25519"49	"crypto/rand"50	"crypto/sha256"51	"crypto/tls"52	"encoding/hex"53	"encoding/json"54	"fmt"55	"io"56	"log/slog"57	"net"58	"net/http"59	"net/http/httptest"60	"net/http/httputil"61	"net/url"62	"strings"63	"sync"64	"testing"65	"time"6667	"github.com/google/uuid"68	"github.com/jackc/pgx/v5"69	"github.com/jackc/pgx/v5/pgxpool"7071	"clustr.spboucher.ai/mcc/internal/agent/client"72	"clustr.spboucher.ai/mcc/internal/agent/state"73	"clustr.spboucher.ai/mcc/internal/config"74	"clustr.spboucher.ai/mcc/internal/db"75	"clustr.spboucher.ai/mcc/internal/ids"76	"clustr.spboucher.ai/mcc/internal/ingest"77	"clustr.spboucher.ai/mcc/internal/pki"78	"clustr.spboucher.ai/mcc/internal/protocol"79	"clustr.spboucher.ai/mcc/internal/testdb"80)8182// Both agent transports run the same suite.83var modes = []string{"mtls", "signed"}8485func forModes(t *testing.T, fn func(t *testing.T, mode string)) {86	for _, m := range modes {87		t.Run(m, func(t *testing.T) { fn(t, m) })88	}89}9091// tcpProxy is a cuttable TCP relay, used to simulate a network or database outage.92type tcpProxy struct {93	ln     net.Listener94	target string95	mu     sync.Mutex96	down   bool97	conns  map[net.Conn]struct{}98}99100func newTCPProxy(t *testing.T, target string) *tcpProxy {101	ln, err := net.Listen("tcp", "127.0.0.1:0")102	if err != nil {103		t.Fatal(err)104	}105	p := &tcpProxy{ln: ln, target: target, conns: map[net.Conn]struct{}{}}106	go func() {107		for {108			c, err := ln.Accept()109			if err != nil {110				return111			}112			p.mu.Lock()113			down := p.down114			p.mu.Unlock()115			if down {116				c.Close()117				continue118			}119			up, err := net.DialTimeout("tcp", target, 2*time.Second)120			if err != nil {121				c.Close()122				continue123			}124			p.mu.Lock()125			p.conns[c], p.conns[up] = struct{}{}, struct{}{}126			p.mu.Unlock()127			go func() { io.Copy(up, c); up.Close(); c.Close() }()128			go func() { io.Copy(c, up); up.Close(); c.Close() }()129		}130	}()131	t.Cleanup(func() { ln.Close(); p.cut() })132	return p133}134135func (p *tcpProxy) addr() string { return p.ln.Addr().String() }136func (p *tcpProxy) cut() {137	p.mu.Lock()138	p.down = true139	for c := range p.conns {140		c.Close()141	}142	p.conns = map[net.Conn]struct{}{}143	p.mu.Unlock()144}145func (p *tcpProxy) restore() { p.mu.Lock(); p.down = false; p.mu.Unlock() }146147type harness struct {148	t        *testing.T149	mode     string150	pool     *pgxpool.Pool // direct connection, for assertions151	srv      *ingest.Server152	cfg      *config.Config153	ca       *pki.CA154	endpoint string // what agents are given155	mtlsAddr string156	signedLn string // loopback address of the signed listener (behind the proxy)157	front    *httptest.Server158	dbProxy  *tcpProxy159	netProxy *tcpProxy160	orchKey  ed25519.PrivateKey161	keyID    string162	health   http.Handler163}164165type hopt struct {166	dbProxy  bool167	netProxy bool168	retain   time.Duration169	ca       *pki.CA170}171172// newHarnessWithCA uses a CA the test created itself (to mint special certificates).173func newHarnessWithCA(t *testing.T, mode string, ca *pki.CA) *harness {174	return newHarness(t, mode, hopt{ca: ca})175}176177func newHarness(t *testing.T, mode string, o hopt) *harness {178	t.Helper()179	pool, dbURL := testdb.New(t)180	h := &harness{t: t, mode: mode, pool: pool}181	srvPool := pool182	if o.dbProxy {183		u, _ := url.Parse(dbURL)184		h.dbProxy = newTCPProxy(t, u.Host)185		u.Host = h.dbProxy.addr()186		q := u.Query()187		q.Set("connect_timeout", "2")188		u.RawQuery = q.Encode()189		var err error190		if srvPool, err = db.Open(context.Background(), u.String()); err != nil {191			t.Fatal(err)192		}193		t.Cleanup(srvPool.Close)194	}195	ca := o.ca196	if ca == nil {197		var err error198		if ca, err = pki.LoadOrCreate(t.TempDir()); err != nil {199			t.Fatal(err)200		}201	}202	h.ca = ca203	var err error204	if o.retain == 0 {205		o.retain = 7 * 24 * time.Hour206	}207	h.cfg = &config.Config{Env: config.Test, IngestServerName: "ingest.mcc.internal", RetainRaw: o.retain,208		Retain1m: 30 * 24 * time.Hour, Retain1h: 400 * 24 * time.Hour, FreshAfter: 30 * time.Second, OfflineAfter: 120 * time.Second}209210	mln, err := net.Listen("tcp", "127.0.0.1:0")211	if err != nil {212		t.Fatal(err)213	}214	sln, err := net.Listen("tcp", "127.0.0.1:0")215	if err != nil {216		t.Fatal(err)217	}218	h.mtlsAddr, h.signedLn = mln.Addr().String(), sln.Addr().String()219	// The front mimics control-api: a real reverse proxy before the signed listener.220	target, _ := url.Parse("http://" + h.signedLn)221	rp := httputil.NewSingleHostReverseProxy(target)222	rp.FlushInterval = -1223	rp.ErrorLog = slog.NewLogLogger(slog.DiscardHandler, slog.LevelError)224	h.front = httptest.NewServer(rp)225	t.Cleanup(h.front.Close)226227	h.endpoint = h.mtlsAddr228	if mode == "signed" {229		h.endpoint = h.front.URL230	}231	if o.netProxy {232		up := h.mtlsAddr233		if mode == "signed" {234			up = strings.TrimPrefix(h.front.URL, "http://")235		}236		h.netProxy = newTCPProxy(t, up)237		h.endpoint = h.netProxy.addr()238		if mode == "signed" {239			h.endpoint = "http://" + h.netProxy.addr()240		}241	}242	h.cfg.IngestAdvertise = []string{h.endpoint}243244	h.srv, err = ingest.New(ingest.Options{Cfg: h.cfg, Pool: srvPool, CA: ca, Version: "test", EnrollPerMinute: 6000, EnrollBurst: 1000,245		AuthCacheTTL: 150 * time.Millisecond, ReportIntervalS: 10, Log: slog.New(slog.DiscardHandler)})246	if err != nil {247		t.Fatal(err)248	}249	ctx, cancel := context.WithCancel(context.Background())250	if err := h.srv.SeedCatalog(ctx); err != nil {251		t.Fatal(err)252	}253	if err := h.srv.EnsurePartitions(ctx); err != nil {254		t.Fatal(err)255	}256	var wg sync.WaitGroup257	wg.Add(3)258	go func() { defer wg.Done(); h.srv.Serve(ctx, mln) }()259	go func() { defer wg.Done(); h.srv.ServeSigned(ctx, sln) }()260	go func() { defer wg.Done(); h.srv.ListenCommandsForTest(ctx) }()261	t.Cleanup(func() { cancel(); wg.Wait() })262	h.health = h.srv.HealthHandler()263264	// The orchestrator's signing key lives in the test only: ingest never sees it.265	pub, priv, _ := ed25519.GenerateKey(rand.Reader)266	h.orchKey, h.keyID = priv, "test-key-1"267	h.exec(`insert into command_keys(key_id, public_key) values ($1,$2)`, h.keyID, []byte(pub))268	return h269}270271func (h *harness) exec(sql string, args ...any) {272	h.t.Helper()273	if _, err := h.pool.Exec(context.Background(), sql, args...); err != nil {274		h.t.Fatalf("%s: %v", sql, err)275	}276}277278func scan[T any](h *harness, sql string, args ...any) T {279	h.t.Helper()280	var v T281	if err := h.pool.QueryRow(context.Background(), sql, args...).Scan(&v); err != nil {282		h.t.Fatalf("%s: %v", sql, err)283	}284	return v285}286287func (h *harness) token(alias string, ttl time.Duration) string {288	h.t.Helper()289	tok, err := ingest.CreateEnrollmentToken(context.Background(), h.pool, alias, "mac", ttl)290	if err != nil {291		h.t.Fatal(err)292	}293	return tok294}295296func (h *harness) fingerprint() []byte {297	fp, _ := pki.ParseFingerprint(h.ca.Fingerprint())298	return fp299}300301// testInventory is synthetic and lives only in the throwaway test database.302func testInventory(hw string) protocol.Inventory {303	return protocol.Inventory{HardwareUUID: hw, Hostname: "test-host", Kind: "mac", Model: "TestMac1,1", Chip: "Test Chip", Arch: "arm64",304		CPUCores: 8, CPULevels: []protocol.CPULevel{{Name: "Super", Cores: 2}, {Name: "Performance", Cores: 2}, {Name: "Efficiency", Cores: 4}},305		MemBytes: 16 << 30, StorageBytes: 500 << 30, OSName: "macOS", OSVersion: "27.0", OSBuild: "TEST",306		Interfaces: []protocol.Interface{{Name: "en0", Kind: "ethernet", Addrs: []string{"192.0.2.10/24"}, Up: true, Default: true}},307		Managers:   []string{"launchd"}}308}309310func newHW() string { return "TEST-" + strings.ToUpper(uuid.NewString()) }311312type enrolled struct {313	dir *state.Dir314	cl  *client.Client315	id  *state.Identity316	hw  string317}318319func (h *harness) enrollErr(alias, hw, token string, fp []byte) (*enrolled, error) {320	dir, err := state.Open(h.t.TempDir())321	if err != nil {322		h.t.Fatal(err)323	}324	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)325	defer cancel()326	id, err := client.Enroll(ctx, dir, client.EnrollParams{Endpoints: []string{h.endpoint}, Token: token, CAFingerprint: fp,327		Inventory: testInventory(hw), Version: "test", Attempts: 1})328	if err != nil {329		return &enrolled{dir: dir, hw: hw}, err330	}331	cl, err := client.New(dir)332	if err != nil {333		h.t.Fatal(err)334	}335	h.t.Cleanup(cl.Close)336	return &enrolled{dir: dir, cl: cl, id: id, hw: hw}, nil337}338339func (h *harness) enroll(alias string) *enrolled {340	h.t.Helper()341	e, err := h.enrollErr(alias, newHW(), h.token(alias, 15*time.Minute), h.fingerprint())342	if err != nil {343		h.t.Fatalf("enroll %s: %v", alias, err)344	}345	return e346}347348func sample(m string, v float64) protocol.Sample { return protocol.Sample{M: m, V: v} }349350func report(seq int64, ts time.Time, samples ...protocol.Sample) *protocol.Report {351	return &protocol.Report{V: protocol.Version, Seq: seq, TS: ts.UTC(), AgentVersion: "test", Samples: samples}352}353354func (h *harness) send(e *enrolled, rep *protocol.Report) (*protocol.ReportAck, error) {355	enc, err := client.EncodeReport(rep)356	if err != nil {357		h.t.Fatal(err)358	}359	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)360	defer cancel()361	return e.cl.Report(ctx, enc)362}363364// sendRaw posts arbitrary bytes as a report body (to send what the Go encoder refuses to produce).365func (h *harness) sendRaw(e *enrolled, body string) error {366	var buf bytes.Buffer367	zw := gzip.NewWriter(&buf)368	zw.Write([]byte(body))369	zw.Close()370	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)371	defer cancel()372	_, err := e.cl.Report(ctx, buf.Bytes())373	return err374}375376func status(err error) int {377	var ae *client.APIError378	if e, ok := err.(*client.APIError); ok {379		ae = e380	}381	if ae == nil {382		return 0383	}384	return ae.Status385}386387func code(err error) string {388	if e, ok := err.(*client.APIError); ok {389		return e.Code390	}391	return ""392}393394// command inserts an action and its signed command, as the orchestrator does.395type cmdOpt struct {396	key      ed25519.PrivateKey397	keyID    string398	issued   time.Time399	lifetime time.Duration400	tamper   bool401	agentID  string // recipient written in the envelope (default: the row's agent)402}403404func (h *harness) command(e *enrolled, typ string, params any, o cmdOpt) uuid.UUID {405	h.t.Helper()406	if o.key == nil {407		o.key, o.keyID = h.orchKey, h.keyID408	}409	if o.issued.IsZero() {410		o.issued = time.Now()411	}412	if o.lifetime == 0 {413		o.lifetime = 60 * time.Second414	}415	if o.agentID == "" {416		o.agentID = e.id.AgentID417	}418	actionID, cmdID := ids.New(), ids.New()419	pj, _ := json.Marshal(params)420	if params == nil {421		pj = []byte("{}")422	}423	sum := sha256.Sum256(pj)424	h.exec(`insert into actions(id, type, criticality, node_id, target_kind, target_id, params, state, idempotency_key, request_hash, timeout_s, correlation_id)425		values ($1,$2,'low',$3::uuid,'node',$8,$4,'dispatched',$5,$6,30,$7)`, actionID, typ, e.id.NodeID, pj, cmdID.String(), sum[:], ids.New(), e.id.NodeID)426	sc, err := protocol.Sign(o.key, o.keyID, protocol.Command{V: protocol.Version, ID: cmdID.String(), ActionID: actionID.String(),427		AgentID: o.agentID, NodeID: e.id.NodeID, Type: typ, Params: pj, TimeoutS: 30,428		IssuedAt: o.issued.UTC(), ExpiresAt: o.issued.Add(o.lifetime).UTC(), Nonce: protocol.NewNonce()})429	if err != nil {430		h.t.Fatal(err)431	}432	if o.tamper {433		// A compromised transport rewrites the instruction after it was signed.434		sc.Payload = bytes.Replace(sc.Payload, []byte(`"`+typ+`"`), []byte(`"system.shutdown"`), 1)435	}436	// The row's own expiry is generous: expiry must be enforced by the agent on the signed envelope.437	h.exec(`insert into agent_commands(id, action_id, agent_id, envelope, signature, key_id, expires_at) values ($1,$2,$3,$4,$5,$6, now() + interval '5 minutes')`,438		cmdID, actionID, e.id.AgentID, sc.Payload, sc.Signature, sc.KeyID)439	h.exec(`select pg_notify('mcc_commands', $1)`, e.id.AgentID)440	return cmdID441}442443type cmdRow struct {444	State        string445	AckAccepted  *bool446	AckReason    *string447	ResultStatus *string448	Result       map[string]any449}450451func (h *harness) cmdRow(id uuid.UUID) cmdRow {452	h.t.Helper()453	var r cmdRow454	err := h.pool.QueryRow(context.Background(), `select state, ack_accepted, ack_reason, result_status, result from agent_commands where id=$1`, id).455		Scan(&r.State, &r.AckAccepted, &r.AckReason, &r.ResultStatus, &r.Result)456	if err != nil && err != pgx.ErrNoRows {457		h.t.Fatal(err)458	}459	return r460}461462func (h *harness) waitCmd(id uuid.UUID, state string, within time.Duration) cmdRow {463	h.t.Helper()464	deadline := time.Now().Add(within)465	for {466		r := h.cmdRow(id)467		if r.State == state {468			return r469		}470		if time.Now().After(deadline) {471			h.t.Fatalf("command %s: state %q after %s, want %q (%+v)", id, r.State, within, state, r)472		}473		time.Sleep(50 * time.Millisecond)474	}475}476477func eventually(t *testing.T, within time.Duration, what string, cond func() bool) {478	t.Helper()479	deadline := time.Now().Add(within)480	for !cond() {481		if time.Now().After(deadline) {482			t.Fatalf("timeout after %s waiting for: %s", within, what)483		}484		time.Sleep(50 * time.Millisecond)485	}486}487488func hexsum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }489490var _ = fmt.Sprintf491492func parseUUID(s string) ([16]byte, error) {493	id, err := uuid.Parse(s)494	return id, err495}496497func insecureGet(addr, path string) (int, error) {498	tr := &http.Transport{TLSClientConfig: insecureTLS()}499	defer tr.CloseIdleConnections()500	resp, err := (&http.Client{Transport: tr, Timeout: 5 * time.Second}).Get("https://" + addr + path)501	if err != nil {502		return 0, err503	}504	resp.Body.Close()505	return resp.StatusCode, nil506}507508func insecureTLS() *tls.Config { return &tls.Config{InsecureSkipVerify: true} } // test probe only: no client certificate509
Files main
Menu

Theme