Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/api/ops_jobs.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package api4344import (45 "context"46 "encoding/json"47 "errors"48 "strings"49 "time"5051 "github.com/google/uuid"52 "github.com/jackc/pgx/v5"53 "github.com/jackc/pgx/v5/pgconn"5455 "clustr.spboucher.ai/mcc/internal/actions"56 "clustr.spboucher.ai/mcc/internal/audit"57 "clustr.spboucher.ai/mcc/internal/authz"58 "clustr.spboucher.ai/mcc/internal/httpx"59 "clustr.spboucher.ai/mcc/internal/ids"60)6162// Jobs API. A job is a request to run an agent-side template; each attempt is63// carried by a job.run action created by the job runner (internal/jobs).6465const jobSelect = `select j.id, j.template, j.owner_id, u.display_name, j.params, j.state::text, j.node_id, n.alias::text, j.placement, j.progress,66 j.attempt, j.max_attempts, j.schedule_id, j.depends_on, j.created_at, j.started_at, j.finished_at, j.next_attempt_at, j.result, j.error, j.usage,67 (select a.id from actions a where a.job_id=j.id and a.type='job.run' order by a.created_at desc, a.id desc limit 1)68 from jobs j left join users u on u.id=j.owner_id left join nodes n on n.id=j.node_id`6970func scanJob(r pgx.Row) (Job, error) {71 var j Job72 var state string73 var result, usage map[string]any74 err := r.Scan(&j.Id, &j.Template, &j.OwnerId, &j.OwnerDisplayName, &j.Params, &state, &j.NodeId, &j.NodeAlias, &j.Placement, &j.Progress,75 &j.Attempt, &j.MaxAttempts, &j.ScheduleId, &j.DependsOn, &j.CreatedAt, &j.StartedAt, &j.FinishedAt, &j.NextAttemptAt, &result, &j.Error, &usage, &j.ActionId)76 j.State = JobState(state)77 if j.Params == nil {78 j.Params = map[string]any{}79 }80 if j.DependsOn == nil {81 j.DependsOn = []uuid.UUID{}82 }83 if result != nil {84 j.Result = &result85 }86 if usage != nil {87 j.Usage = &usage88 }89 return j, err90}9192func (h *opsHandlers) loadJob(ctx context.Context, id uuid.UUID) (Job, error) {93 j, err := scanJob(h.Pool.QueryRow(ctx, jobSelect+` where j.id=$1`, id))94 return j, notFound(err)95}9697// ListJobTemplates exposes exactly what the agents declare ("job.run:<template>"98// capabilities). The server knows a template by its name only: parameters are99// validated by the agent, so no schema is invented here.100func (h *opsHandlers) ListJobTemplates(ctx context.Context, _ ListJobTemplatesRequestObject) (ListJobTemplatesResponseObject, error) {101 rows, err := h.Pool.Query(ctx, `select c, array_agg(a.node_id order by n.alias),102 bool_or(greatest(a.last_seen_at, ns.received_at) > $1 and not n.maintenance)103 from agents a join nodes n on n.id=a.node_id and n.retired_at is null left join node_state ns on ns.node_id=a.node_id,104 unnest(a.capabilities) c105 where a.revoked_at is null and c like 'job.run:%' group by c order by c`, nowUTC().Add(-h.Cfg.OfflineAfter))106 if err != nil {107 return nil, err108 }109 defer rows.Close()110 jr, _ := actions.Lookup("job.run")111 out := JobTemplateList{Items: []JobTemplate{}}112 for rows.Next() {113 var c string114 var nodes []uuid.UUID115 var online *bool116 if err := rows.Scan(&c, &nodes, &online); err != nil {117 return nil, err118 }119 name, ok := actions.TemplateOf(c)120 if !ok {121 continue // malformed declaration: never offered122 }123 out.Items = append(out.Items, JobTemplate{Name: name, Title: name, Description: "Declared by the agent allowlist; parameters are validated by the agent.",124 ParamsSchema: map[string]any{"type": "object", "additionalProperties": true}, TimeoutS: jr.TimeoutS, MaxAttempts: 5,125 Available: deref(online), NodeIds: &nodes})126 }127 return ListJobTemplates200JSONResponse{Body: out}, rows.Err()128}129130// templateDeclared reports whether a template exists, optionally on a given node.131func (h *opsHandlers) templateDeclared(ctx context.Context, template string, node *uuid.UUID) (bool, error) {132 var ok bool133 err := h.Pool.QueryRow(ctx, `select exists(select 1 from agents a join nodes n on n.id=a.node_id and n.retired_at is null134 where a.revoked_at is null and $1 = any(a.capabilities) and ($2::uuid is null or a.node_id=$2))`, actions.JobCapability(template), node).Scan(&ok)135 return ok, err136}137138func placementJSON(p *JobPlacement) map[string]any {139 out := map[string]any{}140 if p == nil {141 return out142 }143 if p.NodeId != nil {144 out["node_id"] = p.NodeId.String()145 }146 if p.Site != nil && *p.Site != "" {147 out["site"] = *p.Site148 }149 if p.Role != nil && *p.Role != "" {150 out["role"] = *p.Role151 }152 if p.Tags != nil && len(*p.Tags) > 0 {153 out["tags"] = *p.Tags154 }155 return out156}157158// checkJobSpec validates what jobs and schedules have in common.159func (h *opsHandlers) checkJobSpec(ctx context.Context, template string, params *map[string]any, p *JobPlacement) error {160 if !actions.TemplateName.MatchString(template) {161 return httpx.Validation("Invalid template name.")162 }163 if params != nil {164 if b, _ := json.Marshal(*params); len(b) > 16<<10 {165 return httpx.Validation("params must be at most 16 KiB of JSON.")166 }167 }168 if p != nil && p.Tags != nil && len(*p.Tags) > 16 {169 return httpx.Validation("At most 16 placement tags.")170 }171 var node *uuid.UUID172 if p != nil && p.NodeId != nil {173 node = p.NodeId174 var one int175 if err := h.Pool.QueryRow(ctx, `select 1 from nodes where id=$1 and retired_at is null`, *node).Scan(&one); err != nil {176 return notFound(err)177 }178 }179 // A job is one job.run action per attempt: the caller must be allowed to run it there.180 res := authz.Resource{}181 if node != nil {182 res = authz.Resource{Kind: "node", ID: node.String()}183 }184 if err := h.authorize(ctx, authz.ActionsRunMedium, res); err != nil {185 return err186 }187 if ok, err := h.templateDeclared(ctx, template, nil); err != nil {188 return err189 } else if !ok {190 return actions.Refuse("unprocessable", "No agent declares this job template.")191 }192 if node != nil {193 if ok, err := h.templateDeclared(ctx, template, node); err != nil {194 return err195 } else if !ok {196 return actions.Refuse("capability_missing", "The agent of this node does not declare this job template.")197 }198 }199 return nil200}201202func (h *opsHandlers) CreateJob(ctx context.Context, req CreateJobRequestObject) (CreateJobResponseObject, error) {203 if req.Body == nil {204 return nil, httpx.Validation("Body required.")205 }206 b := req.Body207 me := h.userID(ctx)208 ap := map[string]any{"template": b.Template}209 refuse := func(err error) (CreateJobResponseObject, error) {210 return nil, h.refused(ctx, "job.create", "job", "", ap, err)211 }212 key := strings.TrimSpace(req.Params.IdempotencyKey)213 if len(key) < 8 || len(key) > 128 {214 return refuse(httpx.Validation("Idempotency-Key must be 8 to 128 characters."))215 }216 maxAttempts := 1217 if b.MaxAttempts != nil {218 maxAttempts = *b.MaxAttempts219 }220 if maxAttempts < 1 || maxAttempts > 5 {221 return refuse(httpx.Validation("max_attempts must be between 1 and 5."))222 }223 deps := deref(b.DependsOn)224 if len(deps) > 16 {225 return refuse(httpx.Validation("At most 16 dependencies."))226 }227 if deps == nil {228 deps = []uuid.UUID{}229 }230 if err := h.checkJobSpec(ctx, b.Template, b.Params, b.Placement); err != nil {231 return refuse(err)232 }233 params := deref(b.Params)234 if params == nil {235 params = map[string]any{}236 }237 placement := placementJSON(b.Placement)238 hash := actions.HashRequest(map[string]any{"template": b.Template, "params": params, "placement": placement, "max_attempts": maxAttempts, "depends_on": deps})239 dedupe := "user:" + me.String() + ":" + key240241 replay := func(q actions.Querier) (CreateJobResponseObject, bool, error) {242 var id uuid.UUID243 var have []byte244 err := q.QueryRow(ctx, `select id, request_hash from jobs where dedupe_key=$1`, dedupe).Scan(&id, &have)245 if errors.Is(err, pgx.ErrNoRows) {246 return nil, false, nil247 }248 if err != nil {249 return nil, false, err250 }251 if string(have) != string(hash) {252 return nil, true, actions.ErrIdempotencyConflict253 }254 j, err := h.loadJob(ctx, id)255 return CreateJob200JSONResponse{Body: j}, true, err256 }257 if r, found, err := replay(h.Pool); found || err != nil {258 if err != nil {259 return refuse(err)260 }261 return r, nil262 }263 if frozen, err := actions.Frozen(ctx, h.Pool); err != nil {264 return nil, err265 } else if frozen {266 return refuse(actions.ErrFrozen)267 }268 if len(deps) > 0 {269 var n int270 if err := h.Pool.QueryRow(ctx, `select count(*) from jobs where id = any($1)`, deps).Scan(&n); err != nil {271 return nil, err272 }273 if n != len(deps) {274 return refuse(actions.Refuse("unprocessable", "depends_on names a job that does not exist (or repeats one)."))275 }276 }277 id := ids.New()278 created := false279 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {280 tag, err := tx.Exec(ctx, `insert into jobs(id, template, owner_id, params, placement, max_attempts, dedupe_key, depends_on, request_hash, created_at)281 values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) on conflict (dedupe_key) do nothing`, id, b.Template, me, params, placement, maxAttempts, dedupe, deps, hash, nowUTC())282 if err != nil || tag.RowsAffected() == 0 {283 return err284 }285 created = true286 if err := actions.JobChanged(ctx, tx, id); err != nil {287 return err288 }289 ap["job_id"], ap["max_attempts"] = id.String(), maxAttempts290 return h.audit(ctx, tx, "job.create", "job", id.String(), audit.Success, "", ap)291 })292 if err != nil {293 return nil, err294 }295 if !created { // a concurrent identical request won296 r, _, err := replay(h.Pool)297 if err != nil {298 return refuse(err)299 }300 return r, nil301 }302 j, err := h.loadJob(ctx, id)303 if err != nil {304 return nil, err305 }306 return CreateJob201JSONResponse{Body: j}, nil307}308309func (h *opsHandlers) GetJob(ctx context.Context, req GetJobRequestObject) (GetJobResponseObject, error) {310 j, err := h.loadJob(ctx, req.Id)311 if err != nil {312 return nil, err313 }314 return GetJob200JSONResponse{Body: j}, nil315}316317func (h *opsHandlers) ListJobs(ctx context.Context, req ListJobsRequestObject) (ListJobsResponseObject, error) {318 p := req.Params319 var w where320 if p.State != nil {321 w.add("j.state = ?::job_state", string(*p.State))322 }323 if p.Template != nil {324 w.add("j.template = ?", *p.Template)325 }326 if p.NodeId != nil {327 w.add("j.node_id = ?", *p.NodeId)328 }329 if p.OwnerId != nil {330 w.add("j.owner_id = ?", *p.OwnerId)331 }332 if p.ScheduleId != nil {333 w.add("j.schedule_id = ?", *p.ScheduleId)334 }335 cur, err := uuidCursor(p.Cursor)336 if err != nil {337 return nil, err338 }339 if cur != nil {340 w.add("j.id < ?", *cur)341 }342 limit := pageLimit(p.Limit)343 rows, err := h.Pool.Query(ctx, jobSelect+w.sql()+" order by j.id desc limit "+w.arg(limit+1), w.args...)344 if err != nil {345 return nil, err346 }347 items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Job, error) { return scanJob(r) })348 if err != nil {349 return nil, err350 }351 out := JobList{Items: items}352 if len(items) > limit {353 out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())354 }355 if out.Items == nil {356 out.Items = []Job{}357 }358 return ListJobs200JSONResponse{Body: out}, nil359}360361func (h *opsHandlers) CancelJob(ctx context.Context, req CancelJobRequestObject) (CancelJobResponseObject, error) {362 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {363 state, err := actions.CancelJob(ctx, tx, req.Id, nowUTC())364 if err != nil {365 return err366 }367 detail := "cancelled"368 if state == "running" {369 detail = "cancellation requested from the agent"370 }371 return h.audit(ctx, tx, "job.cancel", "job", req.Id.String(), audit.Success, detail, nil)372 })373 if err != nil {374 return nil, h.refused(ctx, "job.cancel", "job", req.Id.String(), nil, err)375 }376 j, err := h.loadJob(ctx, req.Id)377 if err != nil {378 return nil, err379 }380 return CancelJob200JSONResponse{Body: j}, nil381}382383// ── Schedules ──────────────────────────────────────────────────────────────384385const scheduleSelect = `select id, name, template, params, placement, every_seconds, enabled, next_run_at, last_run_at, created_by, created_at from job_schedules`386387func scanSchedule(r pgx.Row) (JobSchedule, error) {388 var s JobSchedule389 err := r.Scan(&s.Id, &s.Name, &s.Template, &s.Params, &s.Placement, &s.EverySeconds, &s.Enabled, &s.NextRunAt, &s.LastRunAt, &s.CreatedBy, &s.CreatedAt)390 if s.Params == nil {391 s.Params = map[string]any{}392 }393 return s, err394}395396func uniqueViolation(err error) bool {397 var pe *pgconn.PgError398 return errors.As(err, &pe) && pe.Code == "23505"399}400401func (h *opsHandlers) ListJobSchedules(ctx context.Context, req ListJobSchedulesRequestObject) (ListJobSchedulesResponseObject, error) {402 var w where403 cur, err := uuidCursor(req.Params.Cursor)404 if err != nil {405 return nil, err406 }407 if cur != nil {408 w.add("id < ?", *cur)409 }410 limit := pageLimit(req.Params.Limit)411 rows, err := h.Pool.Query(ctx, scheduleSelect+w.sql()+" order by id desc limit "+w.arg(limit+1), w.args...)412 if err != nil {413 return nil, err414 }415 items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (JobSchedule, error) { return scanSchedule(r) })416 if err != nil {417 return nil, err418 }419 out := JobScheduleList{Items: items}420 if len(items) > limit {421 out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())422 }423 if out.Items == nil {424 out.Items = []JobSchedule{}425 }426 return ListJobSchedules200JSONResponse{Body: out}, nil427}428429func (h *opsHandlers) CreateJobSchedule(ctx context.Context, req CreateJobScheduleRequestObject) (CreateJobScheduleResponseObject, error) {430 if req.Body == nil {431 return nil, httpx.Validation("Body required.")432 }433 b := req.Body434 name := strings.TrimSpace(b.Name)435 if name == "" || len(name) > 80 || b.EverySeconds < 60 || b.EverySeconds > 366*86400 {436 return nil, httpx.Validation("name (1-80 characters) and every_seconds (>= 60) are required.")437 }438 if err := h.checkJobSpec(ctx, b.Template, b.Params, b.Placement); err != nil {439 return nil, h.refused(ctx, "job_schedule.create", "job_schedule", "", map[string]any{"template": b.Template}, err)440 }441 now := nowUTC()442 first := now.Add(time.Duration(b.EverySeconds) * time.Second)443 if b.FirstRunAt != nil {444 first = *b.FirstRunAt445 }446 params := deref(b.Params)447 if params == nil {448 params = map[string]any{}449 }450 enabled := b.Enabled == nil || *b.Enabled451 id := ids.New()452 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {453 if _, err := tx.Exec(ctx, `insert into job_schedules(id, name, template, params, placement, every_seconds, enabled, next_run_at, created_by, created_at)454 values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, id, name, b.Template, params, placementJSON(b.Placement), b.EverySeconds, enabled, first, h.userID(ctx), now); err != nil {455 return err456 }457 return h.audit(ctx, tx, "job_schedule.create", "job_schedule", id.String(), audit.Success, "", map[string]any{"name": name, "template": b.Template, "every_seconds": b.EverySeconds})458 })459 if uniqueViolation(err) {460 return nil, httpx.Conflict("conflict", "A schedule with this name already exists.")461 }462 if err != nil {463 return nil, err464 }465 s, err := scanSchedule(h.Pool.QueryRow(ctx, scheduleSelect+` where id=$1`, id))466 if err != nil {467 return nil, err468 }469 return CreateJobSchedule201JSONResponse{Body: s}, nil470}471472func (h *opsHandlers) UpdateJobSchedule(ctx context.Context, req UpdateJobScheduleRequestObject) (UpdateJobScheduleResponseObject, error) {473 if req.Body == nil {474 return nil, httpx.Validation("Body required.")475 }476 b := req.Body477 cur, err := scanSchedule(h.Pool.QueryRow(ctx, scheduleSelect+` where id=$1`, req.Id))478 if err != nil {479 return nil, notFound(err)480 }481 if b.Name != nil {482 if n := strings.TrimSpace(*b.Name); n == "" || len(n) > 80 {483 return nil, httpx.Validation("name must be 1 to 80 characters.")484 }485 }486 if b.EverySeconds != nil && (*b.EverySeconds < 60 || *b.EverySeconds > 366*86400) {487 return nil, httpx.Validation("every_seconds must be >= 60.")488 }489 if b.Params != nil || b.Placement != nil {490 if err := h.checkJobSpec(ctx, cur.Template, b.Params, b.Placement); err != nil {491 return nil, err492 }493 }494 now := nowUTC()495 err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {496 var placement any497 if b.Placement != nil {498 placement = placementJSON(b.Placement)499 }500 var params any501 if b.Params != nil {502 params = *b.Params503 }504 var name *string505 if b.Name != nil {506 name = ptr(strings.TrimSpace(*b.Name))507 }508 // Re-enabling a schedule, or changing its period, restarts the cadence from now:509 // runs missed while it was disabled are not replayed.510 tag, err := tx.Exec(ctx, `update job_schedules set name=coalesce($2,name), params=coalesce($3,params), placement=coalesce($4,placement),511 every_seconds=coalesce($5,every_seconds), enabled=coalesce($6,enabled),512 next_run_at = case when ($6::boolean is true and not enabled) or ($5::int is not null and $5 <> every_seconds)513 then $7::timestamptz + make_interval(secs => coalesce($5,every_seconds)) else next_run_at end514 where id=$1`, req.Id, name, params, placement, b.EverySeconds, b.Enabled, now)515 if err != nil {516 return err517 }518 if tag.RowsAffected() == 0 {519 return httpx.ErrNotFound520 }521 return h.audit(ctx, tx, "job_schedule.update", "job_schedule", req.Id.String(), audit.Success, "", map[string]any{"enabled": b.Enabled, "every_seconds": b.EverySeconds})522 })523 if uniqueViolation(err) {524 return nil, httpx.Conflict("conflict", "A schedule with this name already exists.")525 }526 if err != nil {527 return nil, err528 }529 s, err := scanSchedule(h.Pool.QueryRow(ctx, scheduleSelect+` where id=$1`, req.Id))530 if err != nil {531 return nil, err532 }533 return UpdateJobSchedule200JSONResponse{Body: s}, nil534}535536func (h *opsHandlers) DeleteJobSchedule(ctx context.Context, req DeleteJobScheduleRequestObject) (DeleteJobScheduleResponseObject, error) {537 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {538 tag, err := tx.Exec(ctx, `delete from job_schedules where id=$1`, req.Id)539 if err != nil {540 return err541 }542 if tag.RowsAffected() == 0 {543 return httpx.ErrNotFound544 }545 return h.audit(ctx, tx, "job_schedule.delete", "job_schedule", req.Id.String(), audit.Success, "", nil)546 })547 if err != nil {548 return nil, err549 }550 return DeleteJobSchedule204Response{}, nil551}552