Repository Public
1package api_test23import (4 "context"5 "sync"6 "testing"7 "time"89 "github.com/google/uuid"1011 "clustr.spboucher.ai/mcc/internal/jobs"12)1314type jobBody struct {15 ID string `json:"id"`16 State string `json:"state"`17 Template string `json:"template"`18 NodeID string `json:"node_id"`19 ActionID string `json:"action_id"`20 Attempt int `json:"attempt"`21 Owner string `json:"owner_display_name"`22 Result map[string]any `json:"result"`23}2425func (e *opsEnv) runner() *jobs.Runner {26 return &jobs.Runner{Pool: e.Pool, FreshAfter: 30 * time.Second, OfflineAfter: 120 * time.Second}27}2829func TestJobsAPI(t *testing.T) {30 e := newOpsEnv(t)31 ctx := context.Background()32 // Templates are what the agents declare.33 var tpl struct {34 Items []struct {35 Name string36 Available bool37 NodeIDs []string `json:"node_ids"`38 }39 }40 e.viewer.must("GET", "/job-templates", nil, 200).json(t, &tpl)41 if len(tpl.Items) != 2 || tpl.Items[0].Name != "disk.report" || !tpl.Items[0].Available || len(tpl.Items[0].NodeIDs) != 2 {42 t.Fatalf("templates: %+v", tpl)43 }44 submit := func(c *client, key string, body map[string]any) resp {45 return c.do("POST", "/jobs", body, "Idempotency-Key", key)46 }47 // Refusals.48 if r := submit(e.viewer, uuid.NewString(), map[string]any{"template": "disk.report"}); r.Status != 403 {49 t.Fatalf("viewer submit: %d", r.Status)50 }51 if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "rm.everything"}); r.Status != 422 || r.code() != "unprocessable" {52 t.Fatalf("undeclared template: %d %s", r.Status, r.Raw)53 }54 if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "../x"}); r.Status != 400 {55 t.Fatalf("bad template name: %d", r.Status)56 }57 if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "max_attempts": 9}); r.Status != 400 {58 t.Fatalf("max_attempts: %d", r.Status)59 }60 if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "depends_on": []string{uuid.NewString()}}); r.Status != 422 {61 t.Fatalf("unknown dependency: %d", r.Status)62 }63 if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "placement": map[string]any{"node_id": uuid.NewString()}}); r.Status != 404 {64 t.Fatalf("unknown node: %d", r.Status)65 }66 if r := e.oper.do("POST", "/jobs", map[string]any{"template": "disk.report"}); r.Status != 400 {67 t.Fatalf("missing Idempotency-Key: %d", r.Status)68 }69 if e.count(`select count(*) from jobs`) != 0 {70 t.Fatal("refused submissions left rows")71 }72 // Idempotency: replay, conflict, 20 concurrent identical submissions.73 key := uuid.NewString()74 body := map[string]any{"template": "disk.report", "params": map[string]any{"path_alias": "home"}, "max_attempts": 2}75 var j1, j2 jobBody76 r1 := submit(e.oper, key, body)77 r2 := submit(e.oper, key, body)78 r1.json(t, &j1)79 r2.json(t, &j2)80 if r1.Status != 201 || r2.Status != 200 || j1.ID != j2.ID || j1.State != "queued" || j1.Owner != "User oscar" {81 t.Fatalf("replay: %d %d %+v", r1.Status, r2.Status, j1)82 }83 if r := submit(e.oper, key, map[string]any{"template": "sleep"}); r.Status != 409 || r.code() != "idempotency_conflict" {84 t.Fatalf("conflict: %d %s", r.Status, r.Raw)85 }86 key = uuid.NewString()87 var wg sync.WaitGroup88 var mu sync.Mutex89 codes := map[int]int{}90 for i := 0; i < 20; i++ {91 wg.Add(1)92 go func() {93 defer wg.Done()94 r := submit(e.admin, key, map[string]any{"template": "sleep", "placement": map[string]any{"node_id": e.node}})95 mu.Lock()96 codes[r.Status]++97 mu.Unlock()98 }()99 }100 wg.Wait()101 if codes[201] != 1 || codes[200] != 19 || e.count(`select count(*) from jobs where template='sleep'`) != 1 {102 t.Fatalf("concurrent submissions: %v", codes)103 }104 // Run: placement picks the least loaded node (T-ALPHA, load 1.0 vs 5.0).105 rn := e.runner()106 if err := rn.Tick(ctx); err != nil {107 t.Fatal(err)108 }109 e.step()110 e.agent.Run(ctx)111 e.step()112 rn.Tick(ctx)113 var got jobBody114 e.viewer.must("GET", "/jobs/"+j1.ID, nil, 200).json(t, &got)115 if got.State != "succeeded" || got.NodeID != e.node.String() || got.ActionID == "" || got.Attempt != 1 {116 t.Fatalf("%+v", got)117 }118 // The run is visible as an action, with its audit trail.119 var det struct{ Action actionBody }120 e.viewer.must("GET", "/actions/"+got.ActionID, nil, 200).json(t, &det)121 if det.Action.Type != "job.run" || det.Action.State != "succeeded" {122 t.Fatalf("%+v", det)123 }124 var list struct{ Items []jobBody }125 e.viewer.must("GET", "/jobs?state=succeeded&template=disk.report", nil, 200).json(t, &list)126 if len(list.Items) != 1 {127 t.Fatalf("list: %d", len(list.Items))128 }129 // Cancel.130 var c jobBody131 submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "placement": map[string]any{"node_id": e.node2}}).json(t, &c)132 if r := e.viewer.do("POST", "/jobs/"+c.ID+"/cancel", nil); r.Status != 403 {133 t.Fatalf("viewer cancel: %d", r.Status)134 }135 e.oper.must("POST", "/jobs/"+c.ID+"/cancel", nil, 200).json(t, &c)136 if c.State != "cancelled" {137 t.Fatalf("%+v", c)138 }139 if r := e.oper.do("POST", "/jobs/"+c.ID+"/cancel", nil); r.Status != 409 {140 t.Fatalf("cancel twice: %d", r.Status)141 }142 // job.cancel is also an action type (server-side).143 var d jobBody144 submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report", "depends_on": []string{c.ID}}).json(t, &d)145 a := e.oper.create(act("job.cancel", "job", uuid.MustParse(d.ID)))146 if e.run(a.ID) != "succeeded" || e.count(`select count(*) from jobs where id=$1 and state='cancelled'`, d.ID) != 1 {147 t.Fatal("job.cancel action did not cancel the job")148 }149 if e.count(`select count(*) from audit_log where action in ('job.create','job.cancel') and outcome='success'`) != 5 {150 t.Fatalf("audit entries: %d", e.count(`select count(*) from audit_log where action in ('job.create','job.cancel') and outcome='success'`))151 }152 // Frozen.153 e.Pool.Exec(ctx, `insert into settings(key,value) values ('orchestrator.frozen','true')`)154 if r := submit(e.oper, uuid.NewString(), map[string]any{"template": "disk.report"}); r.Status != 503 || r.code() != "orchestrator_frozen" {155 t.Fatalf("frozen: %d %s", r.Status, r.Raw)156 }157}158159func TestJobSchedulesAPI(t *testing.T) {160 e := newOpsEnv(t)161 var s struct {162 ID string `json:"id"`163 Enabled bool `json:"enabled"`164 Every int `json:"every_seconds"`165 Next time.Time `json:"next_run_at"`166 }167 if r := e.oper.do("POST", "/job-schedules", map[string]any{"name": "x", "template": "disk.report", "every_seconds": 10}); r.Status != 400 {168 t.Fatalf("every_seconds < 60: %d", r.Status)169 }170 if r := e.oper.do("POST", "/job-schedules", map[string]any{"name": "x", "template": "nope", "every_seconds": 600}); r.Status != 422 {171 t.Fatalf("undeclared template: %d", r.Status)172 }173 if r := e.viewer.do("POST", "/job-schedules", map[string]any{"name": "x", "template": "disk.report", "every_seconds": 600}); r.Status != 403 {174 t.Fatalf("viewer: %d", r.Status)175 }176 e.oper.must("POST", "/job-schedules", map[string]any{"name": "disk hourly", "template": "disk.report", "every_seconds": 3600}, 201).json(t, &s)177 if !s.Enabled || s.Every != 3600 || time.Until(s.Next) < 59*time.Minute {178 t.Fatalf("%+v", s)179 }180 if r := e.oper.do("POST", "/job-schedules", map[string]any{"name": "disk hourly", "template": "disk.report", "every_seconds": 3600}); r.Status != 409 {181 t.Fatalf("duplicate name: %d", r.Status)182 }183 e.oper.must("PATCH", "/job-schedules/"+s.ID, map[string]any{"enabled": false, "every_seconds": 600}, 200).json(t, &s)184 if s.Enabled || s.Every != 600 || time.Until(s.Next) > 11*time.Minute {185 t.Fatalf("%+v", s)186 }187 var list struct{ Items []struct{ Name string } }188 e.viewer.must("GET", "/job-schedules", nil, 200).json(t, &list)189 if len(list.Items) != 1 || list.Items[0].Name != "disk hourly" {190 t.Fatalf("%+v", list)191 }192 e.oper.must("DELETE", "/job-schedules/"+s.ID, nil, 204)193 if r := e.oper.do("DELETE", "/job-schedules/"+s.ID, nil); r.Status != 404 {194 t.Fatalf("delete twice: %d", r.Status)195 }196 if e.count(`select count(*) from audit_log where action like 'job_schedule.%' and outcome='success'`) != 3 {197 t.Fatal("schedule changes not audited")198 }199}200