SPB Git

Repository Public

runner_test.go 18.0 KB 525 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/jobs/runner_test.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package jobs_test4344import (45	"context"46	"path/filepath"47	"strings"48	"testing"49	"time"5051	"github.com/google/uuid"52	"github.com/jackc/pgx/v5"53	"github.com/jackc/pgx/v5/pgxpool"5455	"clustr.spboucher.ai/mcc/internal/actions"56	"clustr.spboucher.ai/mcc/internal/ids"57	"clustr.spboucher.ai/mcc/internal/jobs"58	"clustr.spboucher.ai/mcc/internal/orchestrator"59	"clustr.spboucher.ai/mcc/internal/orchestrator/fakeagent"60	"clustr.spboucher.ai/mcc/internal/protocol"61	"clustr.spboucher.ai/mcc/internal/testdb"62)6364var bg = context.Background()6566type env struct {67	t      *testing.T68	pool   *pgxpool.Pool69	clock  *testdb.Clock70	orch   *orchestrator.Orchestrator71	runner *jobs.Runner72	user   uuid.UUID73	nodes  map[string]uuid.UUID74	agents map[string]*fakeagent.Agent75}7677func newEnv(t *testing.T) *env {78	t.Helper()79	pool, _ := testdb.New(t)80	e := &env{t: t, pool: pool, clock: testdb.NewClock(), nodes: map[string]uuid.UUID{}, agents: map[string]*fakeagent.Agent{}}81	s, err := orchestrator.LoadSigner(bg, pool, filepath.Join(t.TempDir(), "k"))82	if err != nil {83		t.Fatal(err)84	}85	e.orch = orchestrator.New(pool, s, orchestrator.Options{Now: e.clock.Now})86	e.runner = &jobs.Runner{Pool: pool, Now: e.clock.Now, FreshAfter: 30 * time.Second, OfflineAfter: 120 * time.Second}87	e.user = testdb.User(t, pool, "op", "operator")88	return e89}9091// node adds a node whose agent declares the "report" template and reported `age` ago with the given load.92func (e *env) node(alias string, load any, age time.Duration, caps ...string) uuid.UUID {93	e.t.Helper()94	if caps == nil {95		caps = []string{"job.run:report"}96	}97	id := testdb.Node(e.t, e.pool, alias)98	aid := testdb.Agent(e.t, e.pool, id, e.clock.Now().Add(-age), caps...)99	vals := map[string]any{}100	if load != nil {101		vals["load.1"] = load102	}103	testdb.State(e.t, e.pool, id, e.clock.Now().Add(-age), vals)104	e.nodes[alias] = id105	a := fakeagent.New(e.pool, aid)106	a.Now = e.clock.Now107	e.agents[alias] = a108	return id109}110111func (e *env) job(template string, placement map[string]any, maxAttempts int, deps ...uuid.UUID) uuid.UUID {112	e.t.Helper()113	id := ids.New()114	if placement == nil {115		placement = map[string]any{}116	}117	if deps == nil {118		deps = []uuid.UUID{}119	}120	if _, err := e.pool.Exec(bg, `insert into jobs(id, template, owner_id, params, placement, max_attempts, depends_on, created_at) values ($1,$2,$3,'{"x":1}',$4,$5,$6,$7)`,121		id, template, e.user, placement, maxAttempts, deps, e.clock.Now()); err != nil {122		e.t.Fatal(err)123	}124	return id125}126127func (e *env) tick() {128	e.t.Helper()129	if err := e.runner.Tick(bg); err != nil {130		e.t.Fatal(err)131	}132}133134func (e *env) step() {135	e.t.Helper()136	if err := e.orch.Step(bg); err != nil {137		e.t.Fatal(err)138	}139}140141// cycle = runner pass, orchestrator pass, every agent, orchestrator pass, runner pass.142func (e *env) cycle() {143	e.t.Helper()144	e.tick()145	e.step()146	for _, a := range e.agents {147		if _, err := a.Run(bg); err != nil {148			e.t.Fatal(err)149		}150	}151	e.step()152	e.tick()153}154155type jrow struct {156	State   string157	Node    *uuid.UUID158	Attempt int159	Err     string160	Next    *time.Time161	Result  map[string]any162}163164func (e *env) get(id uuid.UUID) jrow {165	e.t.Helper()166	var j jrow167	if err := e.pool.QueryRow(bg, `select state::text, node_id, attempt, coalesce(error,''), next_attempt_at, result from jobs where id=$1`, id).168		Scan(&j.State, &j.Node, &j.Attempt, &j.Err, &j.Next, &j.Result); err != nil {169		e.t.Fatal(err)170	}171	return j172}173174func (e *env) count(sql string, args ...any) int {175	e.t.Helper()176	var n int177	if err := e.pool.QueryRow(bg, sql, args...).Scan(&n); err != nil {178		e.t.Fatal(err)179	}180	return n181}182183func TestPlacementUsesRealNodeState(t *testing.T) {184	e := newEnv(t)185	e.node("BUSY", 8.0, 0)                       // load 0.8 per core186	e.node("CALM", 1.0, 0)                       // load 0.1 per core  <- expected187	e.node("STALE", 0.0, 5*time.Minute)          // best figures, but old: not eligible188	e.node("NOCAP", 0.0, 0, "inventory.refresh") // does not declare the template189	e.node("NODATA", nil, 0)                     // fresh but no load figure: only after nodes with data190	maint := e.node("MAINT", 0.0, 0)             // in maintenance191	e.pool.Exec(bg, `update nodes set maintenance=true where id=$1`, maint)192193	id := e.job("report", nil, 1)194	e.tick()195	j := e.get(id)196	if j.State != "running" || j.Node == nil || *j.Node != e.nodes["CALM"] || j.Attempt != 1 {197		t.Fatalf("placement: %+v (CALM=%s)", j, e.nodes["CALM"])198	}199	// The run is an action of type job.run: audit, output and lock come with it.200	var typ, kind, target string201	var node uuid.UUID202	e.pool.QueryRow(bg, `select type, target_kind, target_id, node_id from actions where job_id=$1`, id).Scan(&typ, &kind, &target, &node)203	if typ != "job.run" || kind != "job" || target != id.String() || node != e.nodes["CALM"] || e.count(`select count(*) from action_locks where target_key=$1`, "job:"+id.String()) != 1 {204		t.Fatalf("action: %s %s %s", typ, kind, target)205	}206	// Explicit node.207	id2 := e.job("report", map[string]any{"node_id": e.nodes["BUSY"]}, 1)208	e.tick()209	if j := e.get(id2); j.Node == nil || *j.Node != e.nodes["BUSY"] {210		t.Fatalf("explicit placement: %+v", j)211	}212	// Explicit node that is offline: the job waits and says why; it is not an attempt.213	id3 := e.job("report", map[string]any{"node_id": e.nodes["STALE"]}, 1)214	e.tick()215	if j := e.get(id3); j.State != "queued" || j.Attempt != 0 || !strings.Contains(j.Err, "offline") {216		t.Fatalf("offline explicit node: %+v", j)217	}218	// A template nobody fresh declares: waits.219	id4 := e.job("unknown-template", nil, 1)220	e.tick()221	if j := e.get(id4); j.State != "queued" || !strings.Contains(j.Err, "no eligible node") {222		t.Fatalf("no eligible node: %+v", j)223	}224	// Constraints.225	e.pool.Exec(bg, `update nodes set site='lab', tags='{gpu}' where id=$1`, e.nodes["BUSY"])226	id5 := e.job("report", map[string]any{"site": "lab", "tags": []string{"gpu"}}, 1)227	e.tick()228	if j := e.get(id5); j.Node == nil || *j.Node != e.nodes["BUSY"] {229		t.Fatalf("constraint placement: %+v", j)230	}231	// Only a node without load data left: it is used (eligible), after those with data.232	e.pool.Exec(bg, `update nodes set maintenance=true where alias in ('BUSY','CALM')`)233	id6 := e.job("report", nil, 1)234	e.tick()235	if j := e.get(id6); j.Node == nil || *j.Node != e.nodes["NODATA"] {236		t.Fatalf("no-data placement: %+v", j)237	}238}239240func TestJobRunsToSuccess(t *testing.T) {241	e := newEnv(t)242	e.node("N1", 1.0, 0)243	var seen *protocol.Command244	e.agents["N1"].Handle = func(c *protocol.Command) fakeagent.Behavior {245		seen = c246		return fakeagent.Behavior{Output: []protocol.CommandOutput{{Stream: "stdout", Data: "42 files"}},247			Result: &protocol.CommandResult{Status: "succeeded", Result: map[string]any{"files": 42, "usage": map[string]any{"cpu_s": 1.5}}}}248	}249	id := e.job("report", nil, 1)250	e.cycle()251	j := e.get(id)252	if j.State != "succeeded" || j.Result["files"] != float64(42) || j.Err != "" {253		t.Fatalf("%+v", j)254	}255	if seen == nil || seen.Type != "job.run" || string(seen.Params) != `{"params":{"x":1},"template":"report"}` {256		t.Fatalf("command: %+v %s", seen, seen.Params)257	}258	if e.count(`select count(*) from jobs where id=$1 and usage->>'cpu_s'='1.5' and progress=1 and finished_at is not null`, id) != 1 {259		t.Fatal("usage or progress not recorded")260	}261	if e.count(`select count(*) from events_outbox where topic='job.changed' and payload->>'job_id'=$1`, id.String()) < 2 {262		t.Fatal("job.changed events missing")263	}264	if e.count(`select count(*) from action_locks`) != 0 {265		t.Fatal("lock kept")266	}267}268269func TestRetriesAreBoundedWithBackoff(t *testing.T) {270	e := newEnv(t)271	e.node("N1", 1.0, 0)272	code := 3273	e.agents["N1"].Handle = func(*protocol.Command) fakeagent.Behavior {274		return fakeagent.Behavior{Result: &protocol.CommandResult{Status: "failed", ExitCode: &code, Error: "exit status 3"}}275	}276	id := e.job("report", nil, 3)277	e.cycle()278	j := e.get(id)279	if j.State != "queued" || j.Attempt != 1 || j.Next == nil || !j.Next.Equal(e.clock.Now().Add(30*time.Second)) || !strings.Contains(j.Err, "exit status 3") {280		t.Fatalf("after attempt 1: %+v", j)281	}282	e.clock.Advance(29 * time.Second)283	e.cycle()284	if e.get(id).Attempt != 1 {285		t.Fatal("retried before the backoff elapsed")286	}287	e.clock.Advance(2 * time.Second)288	testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})289	e.cycle()290	j = e.get(id)291	if j.State != "queued" || j.Attempt != 2 || !j.Next.Equal(e.clock.Now().Add(60*time.Second)) {292		t.Fatalf("after attempt 2: %+v", j)293	}294	e.clock.Advance(61 * time.Second)295	testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})296	e.cycle()297	j = e.get(id)298	if j.State != "failed" || j.Attempt != 3 || j.Next != nil {299		t.Fatalf("after attempt 3: %+v", j)300	}301	e.clock.Advance(time.Hour)302	e.cycle()303	if n := e.count(`select count(*) from actions where job_id=$1`, id); n != 3 {304		t.Fatalf("%d runs for max_attempts=3", n)305	}306	if n := e.count(`select count(*) from agent_commands`); n != 3 || len(e.agents["N1"].Executed()) != 3 {307		t.Fatalf("%d commands", n)308	}309	if jobs.Backoff(1) != 30*time.Second || jobs.Backoff(4) != 4*time.Minute || jobs.Backoff(30) != 10*time.Minute {310		t.Fatal("backoff schedule")311	}312}313314func TestNoRetryAfterUnknownOutcomeOrRefusal(t *testing.T) {315	e := newEnv(t)316	e.node("N1", 1.0, 0)317	// Delivered and acknowledged, then silence.318	id := e.job("report", nil, 5)319	e.tick()320	e.step()321	ds, _ := e.agents["N1"].Deliver(bg, nil)322	e.agents["N1"].Ack(bg, ds[0], true, "")323	e.agents["N1"].MarkExecuted(ds[0])324	e.step()325	e.clock.Advance(2 * time.Hour) // past expires_at + timeout + grace326	e.step()327	e.tick()328	j := e.get(id)329	if j.State != "timed_out" || j.Attempt != 1 || j.Result["outcome_unknown"] != true || !strings.Contains(j.Err, "Outcome unknown") {330		t.Fatalf("%+v", j)331	}332	testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})333	for i := 0; i < 3; i++ {334		e.clock.Advance(15 * time.Minute)335		testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})336		e.cycle()337	}338	if n := e.count(`select count(*) from actions where job_id=$1`, id); n != 1 || e.count(`select count(*) from agent_commands`) != 1 {339		t.Fatalf("a job with an unknown outcome was run again (%d runs)", n)340	}341342	// Refused by the agent's local policy: deterministic, not retried.343	e.agents["N1"].Handle = func(*protocol.Command) fakeagent.Behavior {344		return fakeagent.Behavior{Refuse: "template disabled locally"}345	}346	id2 := e.job("report", nil, 5)347	e.cycle()348	if j := e.get(id2); j.State != "failed" || j.Attempt != 1 || !strings.Contains(j.Err, "template disabled locally") {349		t.Fatalf("%+v", j)350	}351352	// Never delivered: certain that nothing ran, so a retry is safe.353	e.agents["N1"].Handle = nil354	id3 := e.job("report", nil, 2)355	e.tick()356	e.step()357	e.clock.Advance(91 * time.Second) // command expired, never fetched358	e.step()359	testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})360	e.tick()361	if j := e.get(id3); j.State != "queued" || j.Attempt != 1 || !strings.Contains(j.Err, "agent_unreachable") {362		t.Fatalf("%+v", j)363	}364	e.clock.Advance(31 * time.Second)365	testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})366	e.cycle()367	if j := e.get(id3); j.State != "succeeded" || j.Attempt != 2 {368		t.Fatalf("%+v", j)369	}370	for _, c := range []struct {371		code    string372		unknown bool373		want    bool374	}{{"command_failed", false, true}, {"agent_unreachable", false, true}, {"no_result", true, false}, {"command_failed", true, false},375		{"agent_refused", false, false}, {"agent_timeout", false, false}, {"", false, false}} {376		if jobs.Retryable(c.code, c.unknown) != c.want {377			t.Errorf("Retryable(%q,%v) != %v", c.code, c.unknown, c.want)378		}379	}380}381382func TestDependencies(t *testing.T) {383	e := newEnv(t)384	e.node("N1", 1.0, 0)385	a := e.job("report", nil, 1)386	b := e.job("report", nil, 1, a)387	e.tick()388	if e.get(a).State != "running" || e.get(b).State != "queued" || e.get(b).Err != "waiting for dependencies" {389		t.Fatalf("a=%+v b=%+v", e.get(a), e.get(b))390	}391	e.step()392	e.agents["N1"].Run(bg)393	e.step()394	e.tick() // a settles, b starts in the same pass395	if e.get(a).State != "succeeded" || e.get(b).State != "running" {396		t.Fatalf("a=%+v b=%+v", e.get(a), e.get(b))397	}398	e.cycle()399	if e.get(b).State != "succeeded" {400		t.Fatalf("b=%+v", e.get(b))401	}402	// A dependency that fails, or does not exist, fails the dependent job without running it.403	e.agents["N1"].Handle = func(*protocol.Command) fakeagent.Behavior {404		return fakeagent.Behavior{Result: &protocol.CommandResult{Status: "failed", Error: "boom"}}405	}406	c := e.job("report", nil, 1)407	d := e.job("report", nil, 1, c)408	x := e.job("report", nil, 1, uuid.New())409	e.cycle()410	e.cycle()411	if e.get(c).State != "failed" || e.get(d).State != "failed" || e.get(d).Attempt != 0 || !strings.Contains(e.get(d).Err, "did not succeed") || e.get(x).State != "failed" {412		t.Fatalf("c=%+v d=%+v x=%+v", e.get(c), e.get(d), e.get(x))413	}414	if e.count(`select count(*) from actions where job_id in ($1,$2)`, d, x) != 0 {415		t.Fatal("a job with a failed dependency was run")416	}417}418419func TestScheduleSingleCatchUp(t *testing.T) {420	e := newEnv(t)421	e.node("N1", 1.0, 0)422	sid := ids.New()423	base := e.clock.Now().Add(-3 * time.Hour) // the scheduler was down for three hours424	if _, err := e.pool.Exec(bg, `insert into job_schedules(id, name, template, every_seconds, next_run_at) values ($1,'nightly','report',600,$2)`, sid, base); err != nil {425		t.Fatal(err)426	}427	e.tick()428	if n := e.count(`select count(*) from jobs where schedule_id=$1`, sid); n != 1 {429		t.Fatalf("%d jobs created for 18 missed runs, want exactly 1 catch-up", n)430	}431	var next time.Time432	e.pool.QueryRow(bg, `select next_run_at from job_schedules where id=$1`, sid).Scan(&next)433	if !next.After(e.clock.Now()) || next.Sub(e.clock.Now()) > 10*time.Minute || next.Sub(base)%(10*time.Minute) != 0 {434		t.Fatalf("next_run_at %s not on the cadence after now %s", next, e.clock.Now())435	}436	for i := 0; i < 3; i++ {437		e.tick()438	}439	if n := e.count(`select count(*) from jobs where schedule_id=$1`, sid); n != 1 {440		t.Fatalf("%d jobs after repeated ticks", n)441	}442	e.clock.Set(next)443	e.tick()444	if n := e.count(`select count(*) from jobs where schedule_id=$1`, sid); n != 2 {445		t.Fatalf("%d jobs at the next slot", n)446	}447	// Disabled schedules never fire.448	e.pool.Exec(bg, `update job_schedules set enabled=false`)449	e.clock.Advance(time.Hour)450	e.tick()451	if n := e.count(`select count(*) from jobs where schedule_id=$1`, sid); n != 2 {452		t.Fatalf("disabled schedule fired (%d)", n)453	}454	// Scheduled jobs run as system actions.455	if e.count(`select count(*) from actions where requested_by is null and requested_via='scheduler' and type='job.run'`) == 0 {456		t.Fatal("scheduled run not created as a system action")457	}458}459460func TestCancelAndFreeze(t *testing.T) {461	e := newEnv(t)462	e.node("N1", 1.0, 0)463	cancel := func(id uuid.UUID) (string, error) {464		var st string465		err := pgx.BeginFunc(bg, e.pool, func(tx pgx.Tx) (err error) { st, err = actions.CancelJob(bg, tx, id, e.clock.Now()); return })466		return st, err467	}468	// Frozen: jobs wait.469	e.pool.Exec(bg, `insert into settings(key,value) values ('orchestrator.frozen','true')`)470	q := e.job("report", nil, 1)471	e.tick()472	if j := e.get(q); j.State != "queued" || !strings.Contains(j.Err, "emergency stop") {473		t.Fatalf("%+v", j)474	}475	// Queued: cancelled at once.476	if st, err := cancel(q); err != nil || st != "cancelled" || e.get(q).State != "cancelled" {477		t.Fatalf("%s %v", st, err)478	}479	e.pool.Exec(bg, `delete from settings`)480	e.cycle()481	if e.get(q).State != "cancelled" || e.count(`select count(*) from actions where job_id=$1`, q) != 0 {482		t.Fatal("a cancelled job was started")483	}484	// Running: cooperative.485	r := e.job("report", nil, 3)486	e.tick()487	e.step()488	ds, _ := e.agents["N1"].Deliver(bg, nil)489	e.agents["N1"].Ack(bg, ds[0], true, "")490	e.step()491	if st, err := cancel(r); err != nil || st != "running" {492		t.Fatalf("%s %v", st, err)493	}494	e.tick()495	if e.get(r).State != "running" {496		t.Fatal("a running job must wait for the agent's answer")497	}498	if asked, _ := e.agents["N1"].CancelRequested(bg, ds[0]); !asked {499		t.Fatal("agent not asked to stop")500	}501	e.agents["N1"].Result(bg, ds[0], protocol.CommandResult{Status: "cancelled"})502	e.step()503	e.tick()504	if j := e.get(r); j.State != "cancelled" || j.Attempt != 1 {505		t.Fatalf("%+v", j)506	}507	if _, err := cancel(r); err == nil {508		t.Fatal("cancelling a finished job must be refused")509	}510	// A cancelled job whose run fails is never retried.511	f := e.job("report", nil, 3)512	e.tick()513	e.step()514	ds, _ = e.agents["N1"].Deliver(bg, nil)515	e.agents["N1"].Ack(bg, ds[0], true, "")516	e.step()517	cancel(f)518	e.agents["N1"].Result(bg, ds[0], protocol.CommandResult{Status: "failed", Error: "killed"})519	e.step()520	e.tick()521	if j := e.get(f); j.State != "failed" || j.Attempt != 1 {522		t.Fatalf("%+v", j)523	}524}525
Files main
Menu

Theme