Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/jobs/runner_test.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package jobs_test4344import (45 "context"46 "path/filepath"47 "strings"48 "testing"49 "time"5051 "github.com/google/uuid"52 "github.com/jackc/pgx/v5"53 "github.com/jackc/pgx/v5/pgxpool"5455 "clustr.spboucher.ai/mcc/internal/actions"56 "clustr.spboucher.ai/mcc/internal/ids"57 "clustr.spboucher.ai/mcc/internal/jobs"58 "clustr.spboucher.ai/mcc/internal/orchestrator"59 "clustr.spboucher.ai/mcc/internal/orchestrator/fakeagent"60 "clustr.spboucher.ai/mcc/internal/protocol"61 "clustr.spboucher.ai/mcc/internal/testdb"62)6364var bg = context.Background()6566type env struct {67 t *testing.T68 pool *pgxpool.Pool69 clock *testdb.Clock70 orch *orchestrator.Orchestrator71 runner *jobs.Runner72 user uuid.UUID73 nodes map[string]uuid.UUID74 agents map[string]*fakeagent.Agent75}7677func newEnv(t *testing.T) *env {78 t.Helper()79 pool, _ := testdb.New(t)80 e := &env{t: t, pool: pool, clock: testdb.NewClock(), nodes: map[string]uuid.UUID{}, agents: map[string]*fakeagent.Agent{}}81 s, err := orchestrator.LoadSigner(bg, pool, filepath.Join(t.TempDir(), "k"))82 if err != nil {83 t.Fatal(err)84 }85 e.orch = orchestrator.New(pool, s, orchestrator.Options{Now: e.clock.Now})86 e.runner = &jobs.Runner{Pool: pool, Now: e.clock.Now, FreshAfter: 30 * time.Second, OfflineAfter: 120 * time.Second}87 e.user = testdb.User(t, pool, "op", "operator")88 return e89}9091// node adds a node whose agent declares the "report" template and reported `age` ago with the given load.92func (e *env) node(alias string, load any, age time.Duration, caps ...string) uuid.UUID {93 e.t.Helper()94 if caps == nil {95 caps = []string{"job.run:report"}96 }97 id := testdb.Node(e.t, e.pool, alias)98 aid := testdb.Agent(e.t, e.pool, id, e.clock.Now().Add(-age), caps...)99 vals := map[string]any{}100 if load != nil {101 vals["load.1"] = load102 }103 testdb.State(e.t, e.pool, id, e.clock.Now().Add(-age), vals)104 e.nodes[alias] = id105 a := fakeagent.New(e.pool, aid)106 a.Now = e.clock.Now107 e.agents[alias] = a108 return id109}110111func (e *env) job(template string, placement map[string]any, maxAttempts int, deps ...uuid.UUID) uuid.UUID {112 e.t.Helper()113 id := ids.New()114 if placement == nil {115 placement = map[string]any{}116 }117 if deps == nil {118 deps = []uuid.UUID{}119 }120 if _, err := e.pool.Exec(bg, `insert into jobs(id, template, owner_id, params, placement, max_attempts, depends_on, created_at) values ($1,$2,$3,'{"x":1}',$4,$5,$6,$7)`,121 id, template, e.user, placement, maxAttempts, deps, e.clock.Now()); err != nil {122 e.t.Fatal(err)123 }124 return id125}126127func (e *env) tick() {128 e.t.Helper()129 if err := e.runner.Tick(bg); err != nil {130 e.t.Fatal(err)131 }132}133134func (e *env) step() {135 e.t.Helper()136 if err := e.orch.Step(bg); err != nil {137 e.t.Fatal(err)138 }139}140141// cycle = runner pass, orchestrator pass, every agent, orchestrator pass, runner pass.142func (e *env) cycle() {143 e.t.Helper()144 e.tick()145 e.step()146 for _, a := range e.agents {147 if _, err := a.Run(bg); err != nil {148 e.t.Fatal(err)149 }150 }151 e.step()152 e.tick()153}154155type jrow struct {156 State string157 Node *uuid.UUID158 Attempt int159 Err string160 Next *time.Time161 Result map[string]any162}163164func (e *env) get(id uuid.UUID) jrow {165 e.t.Helper()166 var j jrow167 if err := e.pool.QueryRow(bg, `select state::text, node_id, attempt, coalesce(error,''), next_attempt_at, result from jobs where id=$1`, id).168 Scan(&j.State, &j.Node, &j.Attempt, &j.Err, &j.Next, &j.Result); err != nil {169 e.t.Fatal(err)170 }171 return j172}173174func (e *env) count(sql string, args ...any) int {175 e.t.Helper()176 var n int177 if err := e.pool.QueryRow(bg, sql, args...).Scan(&n); err != nil {178 e.t.Fatal(err)179 }180 return n181}182183func TestPlacementUsesRealNodeState(t *testing.T) {184 e := newEnv(t)185 e.node("BUSY", 8.0, 0) // load 0.8 per core186 e.node("CALM", 1.0, 0) // load 0.1 per core <- expected187 e.node("STALE", 0.0, 5*time.Minute) // best figures, but old: not eligible188 e.node("NOCAP", 0.0, 0, "inventory.refresh") // does not declare the template189 e.node("NODATA", nil, 0) // fresh but no load figure: only after nodes with data190 maint := e.node("MAINT", 0.0, 0) // in maintenance191 e.pool.Exec(bg, `update nodes set maintenance=true where id=$1`, maint)192193 id := e.job("report", nil, 1)194 e.tick()195 j := e.get(id)196 if j.State != "running" || j.Node == nil || *j.Node != e.nodes["CALM"] || j.Attempt != 1 {197 t.Fatalf("placement: %+v (CALM=%s)", j, e.nodes["CALM"])198 }199 // The run is an action of type job.run: audit, output and lock come with it.200 var typ, kind, target string201 var node uuid.UUID202 e.pool.QueryRow(bg, `select type, target_kind, target_id, node_id from actions where job_id=$1`, id).Scan(&typ, &kind, &target, &node)203 if typ != "job.run" || kind != "job" || target != id.String() || node != e.nodes["CALM"] || e.count(`select count(*) from action_locks where target_key=$1`, "job:"+id.String()) != 1 {204 t.Fatalf("action: %s %s %s", typ, kind, target)205 }206 // Explicit node.207 id2 := e.job("report", map[string]any{"node_id": e.nodes["BUSY"]}, 1)208 e.tick()209 if j := e.get(id2); j.Node == nil || *j.Node != e.nodes["BUSY"] {210 t.Fatalf("explicit placement: %+v", j)211 }212 // Explicit node that is offline: the job waits and says why; it is not an attempt.213 id3 := e.job("report", map[string]any{"node_id": e.nodes["STALE"]}, 1)214 e.tick()215 if j := e.get(id3); j.State != "queued" || j.Attempt != 0 || !strings.Contains(j.Err, "offline") {216 t.Fatalf("offline explicit node: %+v", j)217 }218 // A template nobody fresh declares: waits.219 id4 := e.job("unknown-template", nil, 1)220 e.tick()221 if j := e.get(id4); j.State != "queued" || !strings.Contains(j.Err, "no eligible node") {222 t.Fatalf("no eligible node: %+v", j)223 }224 // Constraints.225 e.pool.Exec(bg, `update nodes set site='lab', tags='{gpu}' where id=$1`, e.nodes["BUSY"])226 id5 := e.job("report", map[string]any{"site": "lab", "tags": []string{"gpu"}}, 1)227 e.tick()228 if j := e.get(id5); j.Node == nil || *j.Node != e.nodes["BUSY"] {229 t.Fatalf("constraint placement: %+v", j)230 }231 // Only a node without load data left: it is used (eligible), after those with data.232 e.pool.Exec(bg, `update nodes set maintenance=true where alias in ('BUSY','CALM')`)233 id6 := e.job("report", nil, 1)234 e.tick()235 if j := e.get(id6); j.Node == nil || *j.Node != e.nodes["NODATA"] {236 t.Fatalf("no-data placement: %+v", j)237 }238}239240func TestJobRunsToSuccess(t *testing.T) {241 e := newEnv(t)242 e.node("N1", 1.0, 0)243 var seen *protocol.Command244 e.agents["N1"].Handle = func(c *protocol.Command) fakeagent.Behavior {245 seen = c246 return fakeagent.Behavior{Output: []protocol.CommandOutput{{Stream: "stdout", Data: "42 files"}},247 Result: &protocol.CommandResult{Status: "succeeded", Result: map[string]any{"files": 42, "usage": map[string]any{"cpu_s": 1.5}}}}248 }249 id := e.job("report", nil, 1)250 e.cycle()251 j := e.get(id)252 if j.State != "succeeded" || j.Result["files"] != float64(42) || j.Err != "" {253 t.Fatalf("%+v", j)254 }255 if seen == nil || seen.Type != "job.run" || string(seen.Params) != `{"params":{"x":1},"template":"report"}` {256 t.Fatalf("command: %+v %s", seen, seen.Params)257 }258 if e.count(`select count(*) from jobs where id=$1 and usage->>'cpu_s'='1.5' and progress=1 and finished_at is not null`, id) != 1 {259 t.Fatal("usage or progress not recorded")260 }261 if e.count(`select count(*) from events_outbox where topic='job.changed' and payload->>'job_id'=$1`, id.String()) < 2 {262 t.Fatal("job.changed events missing")263 }264 if e.count(`select count(*) from action_locks`) != 0 {265 t.Fatal("lock kept")266 }267}268269func TestRetriesAreBoundedWithBackoff(t *testing.T) {270 e := newEnv(t)271 e.node("N1", 1.0, 0)272 code := 3273 e.agents["N1"].Handle = func(*protocol.Command) fakeagent.Behavior {274 return fakeagent.Behavior{Result: &protocol.CommandResult{Status: "failed", ExitCode: &code, Error: "exit status 3"}}275 }276 id := e.job("report", nil, 3)277 e.cycle()278 j := e.get(id)279 if j.State != "queued" || j.Attempt != 1 || j.Next == nil || !j.Next.Equal(e.clock.Now().Add(30*time.Second)) || !strings.Contains(j.Err, "exit status 3") {280 t.Fatalf("after attempt 1: %+v", j)281 }282 e.clock.Advance(29 * time.Second)283 e.cycle()284 if e.get(id).Attempt != 1 {285 t.Fatal("retried before the backoff elapsed")286 }287 e.clock.Advance(2 * time.Second)288 testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})289 e.cycle()290 j = e.get(id)291 if j.State != "queued" || j.Attempt != 2 || !j.Next.Equal(e.clock.Now().Add(60*time.Second)) {292 t.Fatalf("after attempt 2: %+v", j)293 }294 e.clock.Advance(61 * time.Second)295 testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})296 e.cycle()297 j = e.get(id)298 if j.State != "failed" || j.Attempt != 3 || j.Next != nil {299 t.Fatalf("after attempt 3: %+v", j)300 }301 e.clock.Advance(time.Hour)302 e.cycle()303 if n := e.count(`select count(*) from actions where job_id=$1`, id); n != 3 {304 t.Fatalf("%d runs for max_attempts=3", n)305 }306 if n := e.count(`select count(*) from agent_commands`); n != 3 || len(e.agents["N1"].Executed()) != 3 {307 t.Fatalf("%d commands", n)308 }309 if jobs.Backoff(1) != 30*time.Second || jobs.Backoff(4) != 4*time.Minute || jobs.Backoff(30) != 10*time.Minute {310 t.Fatal("backoff schedule")311 }312}313314func TestNoRetryAfterUnknownOutcomeOrRefusal(t *testing.T) {315 e := newEnv(t)316 e.node("N1", 1.0, 0)317 // Delivered and acknowledged, then silence.318 id := e.job("report", nil, 5)319 e.tick()320 e.step()321 ds, _ := e.agents["N1"].Deliver(bg, nil)322 e.agents["N1"].Ack(bg, ds[0], true, "")323 e.agents["N1"].MarkExecuted(ds[0])324 e.step()325 e.clock.Advance(2 * time.Hour) // past expires_at + timeout + grace326 e.step()327 e.tick()328 j := e.get(id)329 if j.State != "timed_out" || j.Attempt != 1 || j.Result["outcome_unknown"] != true || !strings.Contains(j.Err, "Outcome unknown") {330 t.Fatalf("%+v", j)331 }332 testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})333 for i := 0; i < 3; i++ {334 e.clock.Advance(15 * time.Minute)335 testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})336 e.cycle()337 }338 if n := e.count(`select count(*) from actions where job_id=$1`, id); n != 1 || e.count(`select count(*) from agent_commands`) != 1 {339 t.Fatalf("a job with an unknown outcome was run again (%d runs)", n)340 }341342 // Refused by the agent's local policy: deterministic, not retried.343 e.agents["N1"].Handle = func(*protocol.Command) fakeagent.Behavior {344 return fakeagent.Behavior{Refuse: "template disabled locally"}345 }346 id2 := e.job("report", nil, 5)347 e.cycle()348 if j := e.get(id2); j.State != "failed" || j.Attempt != 1 || !strings.Contains(j.Err, "template disabled locally") {349 t.Fatalf("%+v", j)350 }351352 // Never delivered: certain that nothing ran, so a retry is safe.353 e.agents["N1"].Handle = nil354 id3 := e.job("report", nil, 2)355 e.tick()356 e.step()357 e.clock.Advance(91 * time.Second) // command expired, never fetched358 e.step()359 testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})360 e.tick()361 if j := e.get(id3); j.State != "queued" || j.Attempt != 1 || !strings.Contains(j.Err, "agent_unreachable") {362 t.Fatalf("%+v", j)363 }364 e.clock.Advance(31 * time.Second)365 testdb.State(t, e.pool, e.nodes["N1"], e.clock.Now(), map[string]any{"load.1": 1.0})366 e.cycle()367 if j := e.get(id3); j.State != "succeeded" || j.Attempt != 2 {368 t.Fatalf("%+v", j)369 }370 for _, c := range []struct {371 code string372 unknown bool373 want bool374 }{{"command_failed", false, true}, {"agent_unreachable", false, true}, {"no_result", true, false}, {"command_failed", true, false},375 {"agent_refused", false, false}, {"agent_timeout", false, false}, {"", false, false}} {376 if jobs.Retryable(c.code, c.unknown) != c.want {377 t.Errorf("Retryable(%q,%v) != %v", c.code, c.unknown, c.want)378 }379 }380}381382func TestDependencies(t *testing.T) {383 e := newEnv(t)384 e.node("N1", 1.0, 0)385 a := e.job("report", nil, 1)386 b := e.job("report", nil, 1, a)387 e.tick()388 if e.get(a).State != "running" || e.get(b).State != "queued" || e.get(b).Err != "waiting for dependencies" {389 t.Fatalf("a=%+v b=%+v", e.get(a), e.get(b))390 }391 e.step()392 e.agents["N1"].Run(bg)393 e.step()394 e.tick() // a settles, b starts in the same pass395 if e.get(a).State != "succeeded" || e.get(b).State != "running" {396 t.Fatalf("a=%+v b=%+v", e.get(a), e.get(b))397 }398 e.cycle()399 if e.get(b).State != "succeeded" {400 t.Fatalf("b=%+v", e.get(b))401 }402 // A dependency that fails, or does not exist, fails the dependent job without running it.403 e.agents["N1"].Handle = func(*protocol.Command) fakeagent.Behavior {404 return fakeagent.Behavior{Result: &protocol.CommandResult{Status: "failed", Error: "boom"}}405 }406 c := e.job("report", nil, 1)407 d := e.job("report", nil, 1, c)408 x := e.job("report", nil, 1, uuid.New())409 e.cycle()410 e.cycle()411 if e.get(c).State != "failed" || e.get(d).State != "failed" || e.get(d).Attempt != 0 || !strings.Contains(e.get(d).Err, "did not succeed") || e.get(x).State != "failed" {412 t.Fatalf("c=%+v d=%+v x=%+v", e.get(c), e.get(d), e.get(x))413 }414 if e.count(`select count(*) from actions where job_id in ($1,$2)`, d, x) != 0 {415 t.Fatal("a job with a failed dependency was run")416 }417}418419func TestScheduleSingleCatchUp(t *testing.T) {420 e := newEnv(t)421 e.node("N1", 1.0, 0)422 sid := ids.New()423 base := e.clock.Now().Add(-3 * time.Hour) // the scheduler was down for three hours424 if _, err := e.pool.Exec(bg, `insert into job_schedules(id, name, template, every_seconds, next_run_at) values ($1,'nightly','report',600,$2)`, sid, base); err != nil {425 t.Fatal(err)426 }427 e.tick()428 if n := e.count(`select count(*) from jobs where schedule_id=$1`, sid); n != 1 {429 t.Fatalf("%d jobs created for 18 missed runs, want exactly 1 catch-up", n)430 }431 var next time.Time432 e.pool.QueryRow(bg, `select next_run_at from job_schedules where id=$1`, sid).Scan(&next)433 if !next.After(e.clock.Now()) || next.Sub(e.clock.Now()) > 10*time.Minute || next.Sub(base)%(10*time.Minute) != 0 {434 t.Fatalf("next_run_at %s not on the cadence after now %s", next, e.clock.Now())435 }436 for i := 0; i < 3; i++ {437 e.tick()438 }439 if n := e.count(`select count(*) from jobs where schedule_id=$1`, sid); n != 1 {440 t.Fatalf("%d jobs after repeated ticks", n)441 }442 e.clock.Set(next)443 e.tick()444 if n := e.count(`select count(*) from jobs where schedule_id=$1`, sid); n != 2 {445 t.Fatalf("%d jobs at the next slot", n)446 }447 // Disabled schedules never fire.448 e.pool.Exec(bg, `update job_schedules set enabled=false`)449 e.clock.Advance(time.Hour)450 e.tick()451 if n := e.count(`select count(*) from jobs where schedule_id=$1`, sid); n != 2 {452 t.Fatalf("disabled schedule fired (%d)", n)453 }454 // Scheduled jobs run as system actions.455 if e.count(`select count(*) from actions where requested_by is null and requested_via='scheduler' and type='job.run'`) == 0 {456 t.Fatal("scheduled run not created as a system action")457 }458}459460func TestCancelAndFreeze(t *testing.T) {461 e := newEnv(t)462 e.node("N1", 1.0, 0)463 cancel := func(id uuid.UUID) (string, error) {464 var st string465 err := pgx.BeginFunc(bg, e.pool, func(tx pgx.Tx) (err error) { st, err = actions.CancelJob(bg, tx, id, e.clock.Now()); return })466 return st, err467 }468 // Frozen: jobs wait.469 e.pool.Exec(bg, `insert into settings(key,value) values ('orchestrator.frozen','true')`)470 q := e.job("report", nil, 1)471 e.tick()472 if j := e.get(q); j.State != "queued" || !strings.Contains(j.Err, "emergency stop") {473 t.Fatalf("%+v", j)474 }475 // Queued: cancelled at once.476 if st, err := cancel(q); err != nil || st != "cancelled" || e.get(q).State != "cancelled" {477 t.Fatalf("%s %v", st, err)478 }479 e.pool.Exec(bg, `delete from settings`)480 e.cycle()481 if e.get(q).State != "cancelled" || e.count(`select count(*) from actions where job_id=$1`, q) != 0 {482 t.Fatal("a cancelled job was started")483 }484 // Running: cooperative.485 r := e.job("report", nil, 3)486 e.tick()487 e.step()488 ds, _ := e.agents["N1"].Deliver(bg, nil)489 e.agents["N1"].Ack(bg, ds[0], true, "")490 e.step()491 if st, err := cancel(r); err != nil || st != "running" {492 t.Fatalf("%s %v", st, err)493 }494 e.tick()495 if e.get(r).State != "running" {496 t.Fatal("a running job must wait for the agent's answer")497 }498 if asked, _ := e.agents["N1"].CancelRequested(bg, ds[0]); !asked {499 t.Fatal("agent not asked to stop")500 }501 e.agents["N1"].Result(bg, ds[0], protocol.CommandResult{Status: "cancelled"})502 e.step()503 e.tick()504 if j := e.get(r); j.State != "cancelled" || j.Attempt != 1 {505 t.Fatalf("%+v", j)506 }507 if _, err := cancel(r); err == nil {508 t.Fatal("cancelling a finished job must be refused")509 }510 // A cancelled job whose run fails is never retried.511 f := e.job("report", nil, 3)512 e.tick()513 e.step()514 ds, _ = e.agents["N1"].Deliver(bg, nil)515 e.agents["N1"].Ack(bg, ds[0], true, "")516 e.step()517 cancel(f)518 e.agents["N1"].Result(bg, ds[0], protocol.CommandResult{Status: "failed", Error: "killed"})519 e.step()520 e.tick()521 if j := e.get(f); j.State != "failed" || j.Attempt != 1 {522 t.Fatalf("%+v", j)523 }524}525