SPB Git

Repository Public

ops_jobs_test.go 8.0 KB 200 lines go

1package api_test23import (4	"context"5	"sync"6	"testing"7	"time"89	"github.com/google/uuid"1011	"clustr.spboucher.ai/mcc/internal/jobs"12)1314type jobBody struct {15	ID       string         `json:"id"`16	State    string         `json:"state"`17	Template string         `json:"template"`18	NodeID   string         `json:"node_id"`19	ActionID string         `json:"action_id"`20	Attempt  int            `json:"attempt"`21	Owner    string         `json:"owner_display_name"`22	Result   map[string]any `json:"result"`23}2425func (e *opsEnv) runner() *jobs.Runner {26	return &jobs.Runner{Pool: e.Pool, FreshAfter: 30 * time.Second, OfflineAfter: 120 * time.Second}27}2829func TestJobsAPI(t *testing.T) {30	e := newOpsEnv(t)31	ctx := context.Background()32	// Templates are what the agents declare.33	var tpl struct {34		Items []struct {35			Name      string36			Available bool37			NodeIDs   []string `json:"node_ids"`38		}39	}40	e.viewer.must("GET", "/job-templates", nil, 200).json(t, &tpl)41	if len(tpl.Items) != 2 || tpl.Items[0].Name != "disk.report" || !tpl.Items[0].Available || len(tpl.Items[0].NodeIDs) != 2 {42		t.Fatalf("templates: %+v", tpl)43	}44	submit := func(c *client, key string, body map[string]any) resp {45		return c.do("POST", "/jobs", body, "Idempotency-Key", key)46	}47	// Refusals.48	if r := submit(e.viewer, uuid.NewString(), map[string]any{"template": "disk.report"}); r.Status != 403 {49		t.Fatalf("viewer submit: %d", r.Status)50	}51	if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "rm.everything"}); r.Status != 422 || r.code() != "unprocessable" {52		t.Fatalf("undeclared template: %d %s", r.Status, r.Raw)53	}54	if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "../x"}); r.Status != 400 {55		t.Fatalf("bad template name: %d", r.Status)56	}57	if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "max_attempts": 9}); r.Status != 400 {58		t.Fatalf("max_attempts: %d", r.Status)59	}60	if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "depends_on": []string{uuid.NewString()}}); r.Status != 422 {61		t.Fatalf("unknown dependency: %d", r.Status)62	}63	if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "placement": map[string]any{"node_id": uuid.NewString()}}); r.Status != 404 {64		t.Fatalf("unknown node: %d", r.Status)65	}66	if r := e.oper.do("POST", "/jobs", map[string]any{"template": "disk.report"}); r.Status != 400 {67		t.Fatalf("missing Idempotency-Key: %d", r.Status)68	}69	if e.count(`select count(*) from jobs`) != 0 {70		t.Fatal("refused submissions left rows")71	}72	// Idempotency: replay, conflict, 20 concurrent identical submissions.73	key := uuid.NewString()74	body := map[string]any{"template": "disk.report", "params": map[string]any{"path_alias": "home"}, "max_attempts": 2}75	var j1, j2 jobBody76	r1 := submit(e.oper, key, body)77	r2 := submit(e.oper, key, body)78	r1.json(t, &j1)79	r2.json(t, &j2)80	if r1.Status != 201 || r2.Status != 200 || j1.ID != j2.ID || j1.State != "queued" || j1.Owner != "User oscar" {81		t.Fatalf("replay: %d %d %+v", r1.Status, r2.Status, j1)82	}83	if r := submit(e.oper, key, map[string]any{"template": "sleep"}); r.Status != 409 || r.code() != "idempotency_conflict" {84		t.Fatalf("conflict: %d %s", r.Status, r.Raw)85	}86	key = uuid.NewString()87	var wg sync.WaitGroup88	var mu sync.Mutex89	codes := map[int]int{}90	for i := 0; i < 20; i++ {91		wg.Add(1)92		go func() {93			defer wg.Done()94			r := submit(e.admin, key, map[string]any{"template": "sleep", "placement": map[string]any{"node_id": e.node}})95			mu.Lock()96			codes[r.Status]++97			mu.Unlock()98		}()99	}100	wg.Wait()101	if codes[201] != 1 || codes[200] != 19 || e.count(`select count(*) from jobs where template='sleep'`) != 1 {102		t.Fatalf("concurrent submissions: %v", codes)103	}104	// Run: placement picks the least loaded node (T-ALPHA, load 1.0 vs 5.0).105	rn := e.runner()106	if err := rn.Tick(ctx); err != nil {107		t.Fatal(err)108	}109	e.step()110	e.agent.Run(ctx)111	e.step()112	rn.Tick(ctx)113	var got jobBody114	e.viewer.must("GET", "/jobs/"+j1.ID, nil, 200).json(t, &got)115	if got.State != "succeeded" || got.NodeID != e.node.String() || got.ActionID == "" || got.Attempt != 1 {116		t.Fatalf("%+v", got)117	}118	// The run is visible as an action, with its audit trail.119	var det struct{ Action actionBody }120	e.viewer.must("GET", "/actions/"+got.ActionID, nil, 200).json(t, &det)121	if det.Action.Type != "job.run" || det.Action.State != "succeeded" {122		t.Fatalf("%+v", det)123	}124	var list struct{ Items []jobBody }125	e.viewer.must("GET", "/jobs?state=succeeded&template=disk.report", nil, 200).json(t, &list)126	if len(list.Items) != 1 {127		t.Fatalf("list: %d", len(list.Items))128	}129	// Cancel.130	var c jobBody131	submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "placement": map[string]any{"node_id": e.node2}}).json(t, &c)132	if r := e.viewer.do("POST", "/jobs/"+c.ID+"/cancel", nil); r.Status != 403 {133		t.Fatalf("viewer cancel: %d", r.Status)134	}135	e.oper.must("POST", "/jobs/"+c.ID+"/cancel", nil, 200).json(t, &c)136	if c.State != "cancelled" {137		t.Fatalf("%+v", c)138	}139	if r := e.oper.do("POST", "/jobs/"+c.ID+"/cancel", nil); r.Status != 409 {140		t.Fatalf("cancel twice: %d", r.Status)141	}142	// job.cancel is also an action type (server-side).143	var d jobBody144	submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "depends_on": []string{c.ID}}).json(t, &d)145	a := e.oper.create(act("job.cancel", "job", uuid.MustParse(d.ID)))146	if e.run(a.ID) != "succeeded" || e.count(`select count(*) from jobs where id=$1 and state='cancelled'`, d.ID) != 1 {147		t.Fatal("job.cancel action did not cancel the job")148	}149	if e.count(`select count(*) from audit_log where action in ('job.create','job.cancel') and outcome='success'`) != 5 {150		t.Fatalf("audit entries: %d", e.count(`select count(*) from audit_log where action in ('job.create','job.cancel') and outcome='success'`))151	}152	// Frozen.153	e.Pool.Exec(ctx, `insert into settings(key,value) values ('orchestrator.frozen','true')`)154	if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report"}); r.Status != 503 || r.code() != "orchestrator_frozen" {155		t.Fatalf("frozen: %d %s", r.Status, r.Raw)156	}157}158159func TestJobSchedulesAPI(t *testing.T) {160	e := newOpsEnv(t)161	var s struct {162		ID      string    `json:"id"`163		Enabled bool      `json:"enabled"`164		Every   int       `json:"every_seconds"`165		Next    time.Time `json:"next_run_at"`166	}167	if r := e.oper.do("POST", "/job-schedules", map[string]any{"name": "x", "template": "disk.report", "every_seconds": 10}); r.Status != 400 {168		t.Fatalf("every_seconds < 60: %d", r.Status)169	}170	if r := e.oper.do("POST", "/job-schedules", map[string]any{"name": "x", "template": "nope", "every_seconds": 600}); r.Status != 422 {171		t.Fatalf("undeclared template: %d", r.Status)172	}173	if r := e.viewer.do("POST", "/job-schedules", map[string]any{"name": "x", "template": "disk.report", "every_seconds": 600}); r.Status != 403 {174		t.Fatalf("viewer: %d", r.Status)175	}176	e.oper.must("POST", "/job-schedules", map[string]any{"name": "disk hourly", "template": "disk.report", "every_seconds": 3600}, 201).json(t, &s)177	if !s.Enabled || s.Every != 3600 || time.Until(s.Next) < 59*time.Minute {178		t.Fatalf("%+v", s)179	}180	if r := e.oper.do("POST", "/job-schedules", map[string]any{"name": "disk hourly", "template": "disk.report", "every_seconds": 3600}); r.Status != 409 {181		t.Fatalf("duplicate name: %d", r.Status)182	}183	e.oper.must("PATCH", "/job-schedules/"+s.ID, map[string]any{"enabled": false, "every_seconds": 600}, 200).json(t, &s)184	if s.Enabled || s.Every != 600 || time.Until(s.Next) > 11*time.Minute {185		t.Fatalf("%+v", s)186	}187	var list struct{ Items []struct{ Name string } }188	e.viewer.must("GET", "/job-schedules", nil, 200).json(t, &list)189	if len(list.Items) != 1 || list.Items[0].Name != "disk hourly" {190		t.Fatalf("%+v", list)191	}192	e.oper.must("DELETE", "/job-schedules/"+s.ID, nil, 204)193	if r := e.oper.do("DELETE", "/job-schedules/"+s.ID, nil); r.Status != 404 {194		t.Fatalf("delete twice: %d", r.Status)195	}196	if e.count(`select count(*) from audit_log where action like 'job_schedule.%' and outcome='success'`) != 3 {197		t.Fatal("schedule changes not audited")198	}199}200
Files main
Menu

Theme