SPB Git

Repository Public

orchestrator_test.go 22.1 KB 677 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/orchestrator/orchestrator_test.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package orchestrator_test4344import (45	"context"46	"errors"47	"os"48	"path/filepath"49	"sync"50	"testing"51	"time"5253	"github.com/google/uuid"54	"github.com/jackc/pgx/v5"55	"github.com/jackc/pgx/v5/pgxpool"5657	"clustr.spboucher.ai/mcc/internal/actions"58	"clustr.spboucher.ai/mcc/internal/orchestrator"59	"clustr.spboucher.ai/mcc/internal/orchestrator/fakeagent"60	"clustr.spboucher.ai/mcc/internal/protocol"61	"clustr.spboucher.ai/mcc/internal/testdb"62)6364var bg = context.Background()6566type env struct {67	t       *testing.T68	pool    *pgxpool.Pool69	clock   *testdb.Clock70	keyPath string71	orch    *orchestrator.Orchestrator72	node    uuid.UUID73	agentID uuid.UUID74	agent   *fakeagent.Agent75	user    uuid.UUID76	svc     uuid.UUID77}7879var allCaps = []string{"inventory.refresh", "diagnostic.run", "service.restart", "service.logs.tail", "system.reboot", "cache.purge"}8081func newEnv(t *testing.T) *env {82	t.Helper()83	pool, _ := testdb.New(t)84	e := &env{t: t, pool: pool, clock: testdb.NewClock(), keyPath: filepath.Join(t.TempDir(), "secrets", "command-signing.key")}85	e.node = testdb.Node(t, pool, "T-NODE-1")86	e.agentID = testdb.Agent(t, pool, e.node, e.clock.Now(), allCaps...)87	e.svc = testdb.Service(t, pool, e.node, "web", true, "online")88	e.user = testdb.User(t, pool, "op", "owner")89	e.orch = e.newOrch()90	e.agent = fakeagent.New(pool, e.agentID)91	e.agent.Now = e.clock.Now92	return e93}9495// newOrch builds a brand-new orchestrator instance, as after a process restart.96func (e *env) newOrch() *orchestrator.Orchestrator {97	e.t.Helper()98	s, err := orchestrator.LoadSigner(bg, e.pool, e.keyPath)99	if err != nil {100		e.t.Fatal(err)101	}102	return orchestrator.New(e.pool, s, orchestrator.Options{Now: e.clock.Now, SweepEvery: 30 * time.Millisecond})103}104105func (e *env) enqueue(typ, kind string, target uuid.UUID, params map[string]any, by *uuid.UUID) uuid.UUID {106	e.t.Helper()107	t, _ := actions.Lookup(typ)108	var id uuid.UUID109	err := pgx.BeginFunc(bg, e.pool, func(tx pgx.Tx) error {110		tg, err := actions.ResolveTarget(bg, tx, kind, target)111		if err != nil {112			return err113		}114		id, _, err = actions.Insert(bg, tx, actions.Request{Type: t, Target: tg, Params: params, RequestedBy: by, Via: "api",115			IdempotencyKey: uuid.NewString(), RequestHash: []byte("h"), CorrelationID: uuid.New(), Now: e.clock.Now()})116		return err117	})118	if err != nil {119		e.t.Fatal(err)120	}121	return id122}123124func (e *env) step(o *orchestrator.Orchestrator) {125	e.t.Helper()126	if err := o.Step(bg); err != nil {127		e.t.Fatal(err)128	}129}130131type row struct {132	State, Code, Msg string133	Unknown          bool134	Result           map[string]any135	Started          *time.Time136}137138func (e *env) action(id uuid.UUID) row {139	e.t.Helper()140	var r row141	if err := e.pool.QueryRow(bg, `select state::text, coalesce(error_code,''), coalesce(error_message,''),142		coalesce((result->>'outcome_unknown')::boolean,false), result, started_at from actions where id=$1`, id).143		Scan(&r.State, &r.Code, &r.Msg, &r.Unknown, &r.Result, &r.Started); err != nil {144		e.t.Fatal(err)145	}146	return r147}148149func (e *env) count(sql string, args ...any) int {150	e.t.Helper()151	var n int152	if err := e.pool.QueryRow(bg, sql, args...).Scan(&n); err != nil {153		e.t.Fatal(err)154	}155	return n156}157158func (e *env) want(id uuid.UUID, state string) row {159	e.t.Helper()160	r := e.action(id)161	if r.State != state {162		e.t.Fatalf("action state = %s (%s: %s), want %s", r.State, r.Code, r.Msg, state)163	}164	return r165}166167func (e *env) locks() int { return e.count(`select count(*) from action_locks`) }168169func TestSignerKeyFileAndPublicKeyOnly(t *testing.T) {170	e := newEnv(t)171	st, err := os.Stat(e.keyPath)172	if err != nil || st.Mode().Perm() != 0o600 {173		t.Fatalf("key file: %v mode %v", err, st.Mode())174	}175	s1, _ := orchestrator.LoadSigner(bg, e.pool, e.keyPath)176	s2, _ := orchestrator.LoadSigner(bg, e.pool, e.keyPath)177	if s1.KeyID != s2.KeyID || s1.KeyID != orchestrator.KeyID(s1.Public()) {178		t.Fatal("key id not stable")179	}180	var n, size int181	e.pool.QueryRow(bg, `select count(*), max(length(public_key)) from command_keys`).Scan(&n, &size)182	if n != 1 || size != 32 {183		t.Fatalf("command_keys: %d rows, %d bytes", n, size)184	}185	os.Chmod(e.keyPath, 0o644)186	if _, err := orchestrator.LoadSigner(bg, e.pool, e.keyPath); err == nil {187		t.Fatal("a group/world readable key must be refused")188	}189}190191func TestHappyPathWithOutput(t *testing.T) {192	e := newEnv(t)193	id := e.enqueue("diagnostic.run", "node", e.node, map[string]any{"profile": "disk"}, &e.user)194	if e.locks() != 1 {195		t.Fatal("lock not taken")196	}197	e.step(e.orch)198	e.want(id, "dispatched")199	var seen *protocol.Command200	e.agent.Handle = func(c *protocol.Command) fakeagent.Behavior {201		seen = c202		return fakeagent.Behavior{Output: []protocol.CommandOutput{{Stream: "stdout", Data: "line 1"}, {Stream: "stderr", Data: "warn"}},203			Result: &protocol.CommandResult{Status: "succeeded", Result: map[string]any{"checked": 3}, Duration: 0.4}}204	}205	if n, err := e.agent.Run(bg); err != nil || n != 1 {206		t.Fatalf("agent: %d %v", n, err)207	}208	if seen == nil || seen.Type != "diagnostic.run" || seen.ActionID != id.String() || seen.NodeID != e.node.String() ||209		seen.ExpiresAt.Sub(seen.IssuedAt) > protocol.MaxCommandLifetime || seen.TimeoutS != 120 {210		t.Fatalf("command: %+v", seen)211	}212	if string(seen.Params) != `{"profile":"disk"}` { // exactly what the agent's strict decoder accepts213		t.Fatalf("params: %s", seen.Params)214	}215	e.step(e.orch)216	r := e.want(id, "succeeded")217	if r.Unknown || r.Result["checked"] != float64(3) || r.Started == nil {218		t.Fatalf("result: %+v", r)219	}220	if e.locks() != 0 {221		t.Fatal("lock not released")222	}223	if n := e.count(`select count(*) from action_events where action_id=$1 and kind='output'`, id); n != 2 {224		t.Fatalf("output events: %d", n)225	}226	// queued, dispatched, succeeded (acknowledgement and result were observed in the same pass)227	if n := e.count(`select count(*) from action_events where action_id=$1 and kind='state'`, id); n != 3 {228		t.Fatalf("state events: %d", n)229	}230	if n := e.count(`select count(*) from events_outbox where topic='action.changed' and payload->>'action_id'=$1`, id.String()); n != 3 {231		t.Fatalf("action.changed events: %d", n)232	}233}234235func TestAgentRefusesByPolicy(t *testing.T) {236	e := newEnv(t)237	id := e.enqueue("cache.purge", "node", e.node, map[string]any{"cache": "pip"}, nil)238	e.step(e.orch)239	e.agent.Handle = func(*protocol.Command) fakeagent.Behavior {240		return fakeagent.Behavior{Refuse: "cache 'pip' is not in the local allowlist"}241	}242	e.agent.Run(bg)243	e.step(e.orch)244	r := e.want(id, "failed")245	if r.Code != "agent_refused" || r.Unknown || r.Msg != "The agent refused the command: cache 'pip' is not in the local allowlist" {246		t.Fatalf("%+v", r)247	}248	if len(e.agent.Executed()) != 0 || e.locks() != 0 {249		t.Fatal("refused command executed or lock kept")250	}251}252253func TestUndeliveredCommandIsAgentUnreachable(t *testing.T) {254	e := newEnv(t)255	id := e.enqueue("inventory.refresh", "node", e.node, nil, nil)256	e.step(e.orch)257	e.clock.Advance(89 * time.Second)258	e.step(e.orch)259	e.want(id, "dispatched")260	e.clock.Advance(2 * time.Second)261	e.step(e.orch)262	r := e.want(id, "failed")263	if r.Code != "agent_unreachable" || r.Unknown {264		t.Fatalf("%+v", r)265	}266	if e.count(`select count(*) from agent_commands where action_id=$1 and state='expired'`, id) != 1 || e.locks() != 0 {267		t.Fatal("command not expired or lock kept")268	}269	// The transport must not deliver it afterwards.270	if n, _ := e.agent.Run(bg); n != 0 {271		t.Fatal("expired command delivered")272	}273}274275func TestDeliveredWithoutResultIsOutcomeUnknown(t *testing.T) {276	for _, ack := range []bool{true, false} {277		e := newEnv(t)278		id := e.enqueue("inventory.refresh", "node", e.node, nil, nil)279		e.step(e.orch)280		ds, err := e.agent.Deliver(bg, nil)281		if err != nil || len(ds) != 1 || ds[0].Err != nil {282			t.Fatalf("deliver: %v %+v", err, ds)283		}284		if ack {285			e.agent.Ack(bg, ds[0], true, "")286			e.step(e.orch)287			e.want(id, "running")288		}289		// expires_at (90 s) + timeout (60 s) + grace (30 s) = 180 s290		e.clock.Advance(179 * time.Second)291		e.step(e.orch)292		if s := e.action(id).State; s != "running" && s != "dispatched" {293			t.Fatalf("finished too early: %s", s)294		}295		e.clock.Advance(2 * time.Second)296		e.step(e.orch)297		r := e.want(id, "timed_out")298		if !r.Unknown || r.Code != "no_result" || e.locks() != 0 {299			t.Fatalf("ack=%v: %+v locks=%d", ack, r, e.locks())300		}301		if e.count(`select count(*) from agent_commands where action_id=$1`, id) != 1 {302			t.Fatal("a command was re-issued")303		}304		// Many more passes and a restart never send anything again.305		for i := 0; i < 3; i++ {306			e.step(e.newOrch())307		}308		if e.count(`select count(*) from agent_commands`) != 1 {309			t.Fatal("a command was re-issued after the timeout")310		}311	}312}313314func TestLateResultIsRecordedNotRewritten(t *testing.T) {315	e := newEnv(t)316	id := e.enqueue("inventory.refresh", "node", e.node, nil, nil)317	e.step(e.orch)318	ds, _ := e.agent.Deliver(bg, nil)319	e.agent.Ack(bg, ds[0], true, "")320	e.clock.Advance(181 * time.Second)321	e.step(e.orch)322	e.want(id, "timed_out")323	// The transport records a result after all.324	if _, err := e.pool.Exec(bg, `update agent_commands set result_status='succeeded', result='{}' where id=$1`, ds[0].ID); err != nil {325		t.Fatal(err)326	}327	e.step(e.orch)328	r := e.want(id, "timed_out")329	if r.Result["late_result"] != "succeeded" || !r.Unknown {330		t.Fatalf("%+v", r)331	}332}333334func TestCrashRestartNeverIssuesASecondCommand(t *testing.T) {335	e := newEnv(t)336	id := e.enqueue("inventory.refresh", "node", e.node, nil, nil)337338	// Crash right after the action was queued: a fresh process picks it up.339	o2 := e.newOrch()340	e.step(o2)341	e.want(id, "dispatched")342343	// Crash right after the command was inserted: fresh processes reconcile, they do not re-issue.344	for i := 0; i < 3; i++ {345		e.step(e.newOrch())346	}347	if n := e.count(`select count(*) from agent_commands where action_id=$1`, id); n != 1 {348		t.Fatalf("%d commands after restart", n)349	}350351	// Crash after delivery and acknowledgement.352	ds, _ := e.agent.Deliver(bg, nil)353	if len(ds) != 1 {354		t.Fatalf("delivered %d", len(ds))355	}356	e.step(e.newOrch())357	e.agent.Ack(bg, ds[0], true, "")358	e.agent.MarkExecuted(ds[0])359	o3 := e.newOrch()360	e.step(o3)361	e.want(id, "running")362	e.step(e.newOrch())363	e.agent.Result(bg, ds[0], protocol.CommandResult{Status: "succeeded"})364	o4 := e.newOrch()365	e.step(o4)366	e.step(o4)367	e.want(id, "succeeded")368	if n := e.count(`select count(*) from agent_commands`); n != 1 || len(e.agent.Executed()) != 1 {369		t.Fatalf("commands=%d executed=%d", n, len(e.agent.Executed()))370	}371	// The schema itself forbids a second command for the action.372	_, err := e.pool.Exec(bg, `insert into agent_commands(id, action_id, agent_id, envelope, signature, key_id, expires_at)373		values ($1,$2,$3,'x','y','k',now())`, uuid.New(), id, e.agentID)374	if err == nil {375		t.Fatal("the database accepted a second command for one action")376	}377}378379func TestTwoInstancesNoDuplicates(t *testing.T) {380	e := newEnv(t)381	const n = 40382	var all []uuid.UUID383	for i := 0; i < n; i++ { // service.logs.tail takes no lock: many actions on one target384		all = append(all, e.enqueue("service.logs.tail", "service", e.svc, map[string]any{"lines": 10}, nil))385	}386	a, b := e.newOrch(), e.newOrch()387	ctx, cancel := context.WithCancel(bg)388	defer cancel()389	var wg sync.WaitGroup390	for _, o := range []*orchestrator.Orchestrator{a, b} {391		wg.Add(1)392		go func() { defer wg.Done(); o.Run(ctx) }()393	}394	go e.agent.Loop(ctx, 10*time.Millisecond)395	deadline := time.Now().Add(20 * time.Second)396	for e.count(`select count(*) from actions where state='succeeded'`) != n {397		if time.Now().After(deadline) {398			t.Fatalf("only %d succeeded", e.count(`select count(*) from actions where state='succeeded'`))399		}400		time.Sleep(20 * time.Millisecond)401	}402	cancel()403	wg.Wait()404	if c := e.count(`select count(*) from agent_commands`); c != n {405		t.Fatalf("%d commands for %d actions", c, n)406	}407	if c := e.count(`select count(distinct action_id) from agent_commands`); c != n {408		t.Fatalf("%d distinct actions", c)409	}410	ex := e.agent.Executed()411	uniq := map[string]bool{}412	for _, x := range ex {413		uniq[x] = true414	}415	if len(ex) != n || len(uniq) != n {416		t.Fatalf("executed %d (%d distinct), want %d", len(ex), len(uniq), n)417	}418	_ = all419}420421func TestFreezeHoldsQueuedActions(t *testing.T) {422	e := newEnv(t)423	e.pool.Exec(bg, `insert into settings(key,value) values ('orchestrator.frozen','true')`)424	id := e.enqueue("inventory.refresh", "node", e.node, nil, nil)425	for i := 0; i < 3; i++ {426		e.step(e.orch)427	}428	e.want(id, "queued")429	if e.count(`select count(*) from agent_commands`) != 0 {430		t.Fatal("a command was signed while frozen")431	}432	if e.orch.Status()["frozen"] != true {433		t.Fatal("frozen state not visible")434	}435	e.pool.Exec(bg, `update settings set value='false' where key='orchestrator.frozen'`)436	e.step(e.orch)437	e.want(id, "dispatched")438}439440func TestPrechecksAreVerifiedAgainAtDispatch(t *testing.T) {441	cases := []struct {442		name, sql, code string443	}{444		{"agent revoked", `update agents set revoked_at=now()`, "agent_offline"},445		{"agent went silent", `update agents set last_seen_at=now()-interval '1 hour'`, "agent_offline"},446		{"capability withdrawn", `update agents set capabilities='{}'`, "capability_missing"},447		{"node entered maintenance", `update nodes set maintenance=true`, "node_in_maintenance"},448		{"node retired", `update nodes set retired_at=now()`, "not_found"},449	}450	for _, c := range cases {451		t.Run(c.name, func(t *testing.T) {452			e := newEnv(t)453			id := e.enqueue("inventory.refresh", "node", e.node, nil, nil)454			if _, err := e.pool.Exec(bg, c.sql); err != nil {455				t.Fatal(err)456			}457			e.step(e.orch)458			r := e.want(id, "failed")459			if r.Code != c.code || r.Unknown || e.count(`select count(*) from agent_commands`) != 0 || e.locks() != 0 {460				t.Fatalf("%+v", r)461			}462		})463	}464}465466func TestCooperativeCancel(t *testing.T) {467	e := newEnv(t)468	cancelIt := func(id uuid.UUID) string {469		var st string470		err := pgx.BeginFunc(bg, e.pool, func(tx pgx.Tx) (err error) { st, err = actions.RequestCancel(bg, tx, id, e.clock.Now()); return })471		if err != nil {472			t.Fatal(err)473		}474		return st475	}476	// Before delivery: the command is withdrawn; cancellation is certain.477	id := e.enqueue("inventory.refresh", "node", e.node, nil, nil)478	e.step(e.orch)479	if st := cancelIt(id); st != "cancelled" {480		t.Fatalf("state %s", st)481	}482	e.want(id, "cancelled")483	if n, _ := e.agent.Run(bg); n != 0 || e.locks() != 0 {484		t.Fatal("withdrawn command was delivered, or lock kept")485	}486	// While running: only a request; the agent's result decides.487	id = e.enqueue("inventory.refresh", "node", e.node, nil, nil)488	e.step(e.orch)489	ds, _ := e.agent.Deliver(bg, nil)490	e.agent.Ack(bg, ds[0], true, "")491	e.step(e.orch)492	if st := cancelIt(id); st != "running" {493		t.Fatalf("state %s", st)494	}495	e.step(e.orch)496	e.want(id, "running")497	if asked, _ := e.agent.CancelRequested(bg, ds[0]); !asked {498		t.Fatal("agent not told to cancel")499	}500	e.agent.Result(bg, ds[0], protocol.CommandResult{Status: "cancelled"})501	e.step(e.orch)502	e.want(id, "cancelled")503	if e.locks() != 0 {504		t.Fatal("lock kept")505	}506	// A finished action cannot be cancelled.507	err := pgx.BeginFunc(bg, e.pool, func(tx pgx.Tx) error { _, err := actions.RequestCancel(bg, tx, id, e.clock.Now()); return err })508	if !errors.Is(err, actions.ErrFinished) {509		t.Fatalf("err = %v", err)510	}511	// The agent may also finish normally despite the request.512	id = e.enqueue("inventory.refresh", "node", e.node, nil, nil)513	e.step(e.orch)514	ds, _ = e.agent.Deliver(bg, nil)515	e.agent.Ack(bg, ds[0], true, "")516	cancelIt(id)517	e.agent.Result(bg, ds[0], protocol.CommandResult{Status: "succeeded"})518	e.step(e.orch)519	e.want(id, "succeeded")520}521522func TestTamperedCommandIsDetectedByTheAgent(t *testing.T) {523	e := newEnv(t)524	id := e.enqueue("inventory.refresh", "node", e.node, nil, nil)525	e.step(e.orch)526	ds, _ := e.agent.Deliver(bg, func(sc *protocol.SignedCommand) {527		sc.Payload = []byte(string(sc.Payload[:len(sc.Payload)-1]) + ` }`) // any byte change528	})529	if len(ds) != 1 || !errors.Is(ds[0].Err, protocol.ErrBadSignature) || ds[0].Command != nil {530		t.Fatalf("tampering not detected: %+v", ds)531	}532	e.agent.Ack(bg, ds[0], false, "verification failed: "+ds[0].Err.Error())533	e.step(e.orch)534	r := e.want(id, "failed")535	if r.Code != "agent_refused" || len(e.agent.Executed()) != 0 {536		t.Fatalf("%+v", r)537	}538539	// A command for another agent, and a replay, are refused too.540	other := fakeagent.New(e.pool, uuid.New())541	other.Now = e.clock.Now542	id2 := e.enqueue("inventory.refresh", "node", e.node, nil, nil)543	e.step(e.orch)544	e.pool.Exec(bg, `update agent_commands set agent_id=agent_id where action_id=$1`, id2)545	ds, _ = e.agent.Deliver(bg, nil)546	if len(ds) != 1 || ds[0].Err != nil {547		t.Fatalf("%+v", ds)548	}549	e.pool.Exec(bg, `update agent_commands set state='pending' where id=$1`, ds[0].ID) // a faulty transport hands it over twice550	again, _ := e.agent.Deliver(bg, nil)551	if len(again) != 1 || !errors.Is(again[0].Err, protocol.ErrReplay) {552		t.Fatalf("replay not detected: %+v", again)553	}554}555556func TestApprovalExpiresAndReleasesTheLock(t *testing.T) {557	e := newEnv(t)558	id := e.enqueue("system.reboot", "node", e.node, nil, &e.user)559	e.want(id, "pending_approval")560	e.clock.Advance(29 * time.Minute)561	e.step(e.orch)562	e.want(id, "pending_approval")563	if e.locks() != 1 {564		t.Fatal("pending approval must hold the lock")565	}566	e.clock.Advance(2 * time.Minute)567	e.step(e.orch)568	r := e.want(id, "expired")569	if r.Code != "approval_expired" || e.locks() != 0 || e.count(`select count(*) from agent_commands`) != 0 {570		t.Fatalf("%+v", r)571	}572}573574func TestServerSideMaintenanceSet(t *testing.T) {575	e := newEnv(t)576	e.pool.Exec(bg, `update agents set revoked_at=now()`) // no agent needed at all577	id := e.enqueue("node.maintenance.set", "node", e.node, map[string]any{"enabled": true, "reason": "fan swap"}, nil)578	e.step(e.orch)579	r := e.want(id, "succeeded")580	if r.Result["maintenance"] != true || r.Result["previous"] != false {581		t.Fatalf("%+v", r)582	}583	var m bool584	var reason string585	e.pool.QueryRow(bg, `select maintenance, maintenance_reason from nodes where id=$1`, e.node).Scan(&m, &reason)586	if !m || reason != "fan swap" || e.count(`select count(*) from agent_commands`) != 0 || e.locks() != 0 {587		t.Fatalf("maintenance=%v reason=%q", m, reason)588	}589	// Allowed again while in maintenance (maintenance-related type).590	id = e.enqueue("node.maintenance.set", "node", e.node, map[string]any{"enabled": false}, nil)591	e.step(e.orch)592	e.want(id, "succeeded")593	e.pool.QueryRow(bg, `select maintenance, maintenance_reason from nodes where id=$1`, e.node).Scan(&m, &reason)594	if m || reason != "" {595		t.Fatal("maintenance not lifted")596	}597}598599func TestSurvivesDatabaseConnectionLoss(t *testing.T) {600	e := newEnv(t)601	ctx, cancel := context.WithCancel(bg)602	defer cancel()603	done := make(chan struct{})604	go func() { e.orch.Run(ctx); close(done) }()605	go e.agent.Loop(ctx, 15*time.Millisecond)606	time.Sleep(100 * time.Millisecond)607	// Kill every connection of this database, including the LISTEN connection, several times.608	admin, err := pgx.Connect(bg, "postgres://127.0.0.1/postgres?sslmode=disable")609	if err != nil {610		t.Skip(err)611	}612	defer admin.Close(bg)613	var dbname string614	e.pool.QueryRow(bg, `select current_database()`).Scan(&dbname)615	for i := 0; i < 3; i++ {616		if _, err := admin.Exec(bg, `select pg_terminate_backend(pid) from pg_stat_activity where datname=$1 and pid<>pg_backend_pid()`, dbname); err != nil {617			t.Fatal(err)618		}619		time.Sleep(40 * time.Millisecond)620	}621	var id uuid.UUID622	for i := 0; ; i++ { // the pool itself needs a retry after its connections were killed623		func() {624			defer func() { recover() }()625			id = e.enqueue("inventory.refresh", "node", e.node, nil, nil)626		}()627		if id != uuid.Nil || i > 20 {628			break629		}630		time.Sleep(50 * time.Millisecond)631	}632	deadline := time.Now().Add(25 * time.Second)633	for {634		var st string635		if err := e.pool.QueryRow(bg, `select state::text from actions where id=$1`, id).Scan(&st); err == nil && st == "succeeded" {636			break637		}638		if time.Now().After(deadline) {639			t.Fatalf("orchestrator did not recover (status %v)", e.orch.Status())640		}641		time.Sleep(50 * time.Millisecond)642	}643	if e.count(`select count(*) from agent_commands`) != 1 {644		t.Fatal("duplicate command after recovery")645	}646	cancel()647	select {648	case <-done:649	case <-time.After(5 * time.Second):650		t.Fatal("Run did not stop")651	}652}653654func TestLeaderRunsOnce(t *testing.T) {655	e := newEnv(t)656	inside := make(chan struct{})657	release := make(chan struct{})658	go orchestrator.Leader(bg, e.pool, 4242, func(context.Context) error { close(inside); <-release; return nil })659	<-inside660	ran, err := orchestrator.Leader(bg, e.pool, 4242, func(context.Context) error { return nil })661	if ran || err != nil {662		t.Fatalf("second leader ran=%v err=%v", ran, err)663	}664	close(release)665	deadline := time.Now().Add(3 * time.Second)666	for {667		ran, _ = orchestrator.Leader(bg, e.pool, 4242, func(context.Context) error { return nil })668		if ran {669			break670		}671		if time.Now().After(deadline) {672			t.Fatal("lock never released")673		}674		time.Sleep(10 * time.Millisecond)675	}676}677
Files main
Menu

Theme