SPB Git

Repository Public

ops_jobs.go 19.0 KB 552 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/api/ops_jobs.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package api4344import (45	"context"46	"encoding/json"47	"errors"48	"strings"49	"time"5051	"github.com/google/uuid"52	"github.com/jackc/pgx/v5"53	"github.com/jackc/pgx/v5/pgconn"5455	"clustr.spboucher.ai/mcc/internal/actions"56	"clustr.spboucher.ai/mcc/internal/audit"57	"clustr.spboucher.ai/mcc/internal/authz"58	"clustr.spboucher.ai/mcc/internal/httpx"59	"clustr.spboucher.ai/mcc/internal/ids"60)6162// Jobs API. A job is a request to run an agent-side template; each attempt is63// carried by a job.run action created by the job runner (internal/jobs).6465const jobSelect = `select j.id, j.template, j.owner_id, u.display_name, j.params, j.state::text, j.node_id, n.alias::text, j.placement, j.progress,66	j.attempt, j.max_attempts, j.schedule_id, j.depends_on, j.created_at, j.started_at, j.finished_at, j.next_attempt_at, j.result, j.error, j.usage,67	(select a.id from actions a where a.job_id=j.id and a.type='job.run' order by a.created_at desc, a.id desc limit 1)68	from jobs j left join users u on u.id=j.owner_id left join nodes n on n.id=j.node_id`6970func scanJob(r pgx.Row) (Job, error) {71	var j Job72	var state string73	var result, usage map[string]any74	err := r.Scan(&j.Id, &j.Template, &j.OwnerId, &j.OwnerDisplayName, &j.Params, &state, &j.NodeId, &j.NodeAlias, &j.Placement, &j.Progress,75		&j.Attempt, &j.MaxAttempts, &j.ScheduleId, &j.DependsOn, &j.CreatedAt, &j.StartedAt, &j.FinishedAt, &j.NextAttemptAt, &result, &j.Error, &usage, &j.ActionId)76	j.State = JobState(state)77	if j.Params == nil {78		j.Params = map[string]any{}79	}80	if j.DependsOn == nil {81		j.DependsOn = []uuid.UUID{}82	}83	if result != nil {84		j.Result = &result85	}86	if usage != nil {87		j.Usage = &usage88	}89	return j, err90}9192func (h *opsHandlers) loadJob(ctx context.Context, id uuid.UUID) (Job, error) {93	j, err := scanJob(h.Pool.QueryRow(ctx, jobSelect+` where j.id=$1`, id))94	return j, notFound(err)95}9697// ListJobTemplates exposes exactly what the agents declare ("job.run:<template>"98// capabilities). The server knows a template by its name only: parameters are99// validated by the agent, so no schema is invented here.100func (h *opsHandlers) ListJobTemplates(ctx context.Context, _ ListJobTemplatesRequestObject) (ListJobTemplatesResponseObject, error) {101	rows, err := h.Pool.Query(ctx, `select c, array_agg(a.node_id order by n.alias),102			bool_or(greatest(a.last_seen_at, ns.received_at) > $1 and not n.maintenance)103		from agents a join nodes n on n.id=a.node_id and n.retired_at is null left join node_state ns on ns.node_id=a.node_id,104			unnest(a.capabilities) c105		where a.revoked_at is null and c like 'job.run:%' group by c order by c`, nowUTC().Add(-h.Cfg.OfflineAfter))106	if err != nil {107		return nil, err108	}109	defer rows.Close()110	jr, _ := actions.Lookup("job.run")111	out := JobTemplateList{Items: []JobTemplate{}}112	for rows.Next() {113		var c string114		var nodes []uuid.UUID115		var online *bool116		if err := rows.Scan(&c, &nodes, &online); err != nil {117			return nil, err118		}119		name, ok := actions.TemplateOf(c)120		if !ok {121			continue // malformed declaration: never offered122		}123		out.Items = append(out.Items, JobTemplate{Name: name, Title: name, Description: "Declared by the agent allowlist; parameters are validated by the agent.",124			ParamsSchema: map[string]any{"type": "object", "additionalProperties": true}, TimeoutS: jr.TimeoutS, MaxAttempts: 5,125			Available: deref(online), NodeIds: &nodes})126	}127	return ListJobTemplates200JSONResponse{Body: out}, rows.Err()128}129130// templateDeclared reports whether a template exists, optionally on a given node.131func (h *opsHandlers) templateDeclared(ctx context.Context, template string, node *uuid.UUID) (bool, error) {132	var ok bool133	err := h.Pool.QueryRow(ctx, `select exists(select 1 from agents a join nodes n on n.id=a.node_id and n.retired_at is null134		where a.revoked_at is null and $1 = any(a.capabilities) and ($2::uuid is null or a.node_id=$2))`, actions.JobCapability(template), node).Scan(&ok)135	return ok, err136}137138func placementJSON(p *JobPlacement) map[string]any {139	out := map[string]any{}140	if p == nil {141		return out142	}143	if p.NodeId != nil {144		out["node_id"] = p.NodeId.String()145	}146	if p.Site != nil && *p.Site != "" {147		out["site"] = *p.Site148	}149	if p.Role != nil && *p.Role != "" {150		out["role"] = *p.Role151	}152	if p.Tags != nil && len(*p.Tags) > 0 {153		out["tags"] = *p.Tags154	}155	return out156}157158// checkJobSpec validates what jobs and schedules have in common.159func (h *opsHandlers) checkJobSpec(ctx context.Context, template string, params *map[string]any, p *JobPlacement) error {160	if !actions.TemplateName.MatchString(template) {161		return httpx.Validation("Invalid template name.")162	}163	if params != nil {164		if b, _ := json.Marshal(*params); len(b) > 16<<10 {165			return httpx.Validation("params must be at most 16 KiB of JSON.")166		}167	}168	if p != nil && p.Tags != nil && len(*p.Tags) > 16 {169		return httpx.Validation("At most 16 placement tags.")170	}171	var node *uuid.UUID172	if p != nil && p.NodeId != nil {173		node = p.NodeId174		var one int175		if err := h.Pool.QueryRow(ctx, `select 1 from nodes where id=$1 and retired_at is null`, *node).Scan(&one); err != nil {176			return notFound(err)177		}178	}179	// A job is one job.run action per attempt: the caller must be allowed to run it there.180	res := authz.Resource{}181	if node != nil {182		res = authz.Resource{Kind: "node", ID: node.String()}183	}184	if err := h.authorize(ctx, authz.ActionsRunMedium, res); err != nil {185		return err186	}187	if ok, err := h.templateDeclared(ctx, template, nil); err != nil {188		return err189	} else if !ok {190		return actions.Refuse("unprocessable", "No agent declares this job template.")191	}192	if node != nil {193		if ok, err := h.templateDeclared(ctx, template, node); err != nil {194			return err195		} else if !ok {196			return actions.Refuse("capability_missing", "The agent of this node does not declare this job template.")197		}198	}199	return nil200}201202func (h *opsHandlers) CreateJob(ctx context.Context, req CreateJobRequestObject) (CreateJobResponseObject, error) {203	if req.Body == nil {204		return nil, httpx.Validation("Body required.")205	}206	b := req.Body207	me := h.userID(ctx)208	ap := map[string]any{"template": b.Template}209	refuse := func(err error) (CreateJobResponseObject, error) {210		return nil, h.refused(ctx, "job.create", "job", "", ap, err)211	}212	key := strings.TrimSpace(req.Params.IdempotencyKey)213	if len(key) < 8 || len(key) > 128 {214		return refuse(httpx.Validation("Idempotency-Key must be 8 to 128 characters."))215	}216	maxAttempts := 1217	if b.MaxAttempts != nil {218		maxAttempts = *b.MaxAttempts219	}220	if maxAttempts < 1 || maxAttempts > 5 {221		return refuse(httpx.Validation("max_attempts must be between 1 and 5."))222	}223	deps := deref(b.DependsOn)224	if len(deps) > 16 {225		return refuse(httpx.Validation("At most 16 dependencies."))226	}227	if deps == nil {228		deps = []uuid.UUID{}229	}230	if err := h.checkJobSpec(ctx, b.Template, b.Params, b.Placement); err != nil {231		return refuse(err)232	}233	params := deref(b.Params)234	if params == nil {235		params = map[string]any{}236	}237	placement := placementJSON(b.Placement)238	hash := actions.HashRequest(map[string]any{"template": b.Template, "params": params, "placement": placement, "max_attempts": maxAttempts, "depends_on": deps})239	dedupe := "user:" + me.String() + ":" + key240241	replay := func(q actions.Querier) (CreateJobResponseObject, bool, error) {242		var id uuid.UUID243		var have []byte244		err := q.QueryRow(ctx, `select id, request_hash from jobs where dedupe_key=$1`, dedupe).Scan(&id, &have)245		if errors.Is(err, pgx.ErrNoRows) {246			return nil, false, nil247		}248		if err != nil {249			return nil, false, err250		}251		if string(have) != string(hash) {252			return nil, true, actions.ErrIdempotencyConflict253		}254		j, err := h.loadJob(ctx, id)255		return CreateJob200JSONResponse{Body: j}, true, err256	}257	if r, found, err := replay(h.Pool); found || err != nil {258		if err != nil {259			return refuse(err)260		}261		return r, nil262	}263	if frozen, err := actions.Frozen(ctx, h.Pool); err != nil {264		return nil, err265	} else if frozen {266		return refuse(actions.ErrFrozen)267	}268	if len(deps) > 0 {269		var n int270		if err := h.Pool.QueryRow(ctx, `select count(*) from jobs where id = any($1)`, deps).Scan(&n); err != nil {271			return nil, err272		}273		if n != len(deps) {274			return refuse(actions.Refuse("unprocessable", "depends_on names a job that does not exist (or repeats one)."))275		}276	}277	id := ids.New()278	created := false279	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {280		tag, err := tx.Exec(ctx, `insert into jobs(id, template, owner_id, params, placement, max_attempts, dedupe_key, depends_on, request_hash, created_at)281			values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) on conflict (dedupe_key) do nothing`, id, b.Template, me, params, placement, maxAttempts, dedupe, deps, hash, nowUTC())282		if err != nil || tag.RowsAffected() == 0 {283			return err284		}285		created = true286		if err := actions.JobChanged(ctx, tx, id); err != nil {287			return err288		}289		ap["job_id"], ap["max_attempts"] = id.String(), maxAttempts290		return h.audit(ctx, tx, "job.create", "job", id.String(), audit.Success, "", ap)291	})292	if err != nil {293		return nil, err294	}295	if !created { // a concurrent identical request won296		r, _, err := replay(h.Pool)297		if err != nil {298			return refuse(err)299		}300		return r, nil301	}302	j, err := h.loadJob(ctx, id)303	if err != nil {304		return nil, err305	}306	return CreateJob201JSONResponse{Body: j}, nil307}308309func (h *opsHandlers) GetJob(ctx context.Context, req GetJobRequestObject) (GetJobResponseObject, error) {310	j, err := h.loadJob(ctx, req.Id)311	if err != nil {312		return nil, err313	}314	return GetJob200JSONResponse{Body: j}, nil315}316317func (h *opsHandlers) ListJobs(ctx context.Context, req ListJobsRequestObject) (ListJobsResponseObject, error) {318	p := req.Params319	var w where320	if p.State != nil {321		w.add("j.state = ?::job_state", string(*p.State))322	}323	if p.Template != nil {324		w.add("j.template = ?", *p.Template)325	}326	if p.NodeId != nil {327		w.add("j.node_id = ?", *p.NodeId)328	}329	if p.OwnerId != nil {330		w.add("j.owner_id = ?", *p.OwnerId)331	}332	if p.ScheduleId != nil {333		w.add("j.schedule_id = ?", *p.ScheduleId)334	}335	cur, err := uuidCursor(p.Cursor)336	if err != nil {337		return nil, err338	}339	if cur != nil {340		w.add("j.id < ?", *cur)341	}342	limit := pageLimit(p.Limit)343	rows, err := h.Pool.Query(ctx, jobSelect+w.sql()+" order by j.id desc limit "+w.arg(limit+1), w.args...)344	if err != nil {345		return nil, err346	}347	items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Job, error) { return scanJob(r) })348	if err != nil {349		return nil, err350	}351	out := JobList{Items: items}352	if len(items) > limit {353		out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())354	}355	if out.Items == nil {356		out.Items = []Job{}357	}358	return ListJobs200JSONResponse{Body: out}, nil359}360361func (h *opsHandlers) CancelJob(ctx context.Context, req CancelJobRequestObject) (CancelJobResponseObject, error) {362	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {363		state, err := actions.CancelJob(ctx, tx, req.Id, nowUTC())364		if err != nil {365			return err366		}367		detail := "cancelled"368		if state == "running" {369			detail = "cancellation requested from the agent"370		}371		return h.audit(ctx, tx, "job.cancel", "job", req.Id.String(), audit.Success, detail, nil)372	})373	if err != nil {374		return nil, h.refused(ctx, "job.cancel", "job", req.Id.String(), nil, err)375	}376	j, err := h.loadJob(ctx, req.Id)377	if err != nil {378		return nil, err379	}380	return CancelJob200JSONResponse{Body: j}, nil381}382383// ── Schedules ──────────────────────────────────────────────────────────────384385const scheduleSelect = `select id, name, template, params, placement, every_seconds, enabled, next_run_at, last_run_at, created_by, created_at from job_schedules`386387func scanSchedule(r pgx.Row) (JobSchedule, error) {388	var s JobSchedule389	err := r.Scan(&s.Id, &s.Name, &s.Template, &s.Params, &s.Placement, &s.EverySeconds, &s.Enabled, &s.NextRunAt, &s.LastRunAt, &s.CreatedBy, &s.CreatedAt)390	if s.Params == nil {391		s.Params = map[string]any{}392	}393	return s, err394}395396func uniqueViolation(err error) bool {397	var pe *pgconn.PgError398	return errors.As(err, &pe) && pe.Code == "23505"399}400401func (h *opsHandlers) ListJobSchedules(ctx context.Context, req ListJobSchedulesRequestObject) (ListJobSchedulesResponseObject, error) {402	var w where403	cur, err := uuidCursor(req.Params.Cursor)404	if err != nil {405		return nil, err406	}407	if cur != nil {408		w.add("id < ?", *cur)409	}410	limit := pageLimit(req.Params.Limit)411	rows, err := h.Pool.Query(ctx, scheduleSelect+w.sql()+" order by id desc limit "+w.arg(limit+1), w.args...)412	if err != nil {413		return nil, err414	}415	items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (JobSchedule, error) { return scanSchedule(r) })416	if err != nil {417		return nil, err418	}419	out := JobScheduleList{Items: items}420	if len(items) > limit {421		out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())422	}423	if out.Items == nil {424		out.Items = []JobSchedule{}425	}426	return ListJobSchedules200JSONResponse{Body: out}, nil427}428429func (h *opsHandlers) CreateJobSchedule(ctx context.Context, req CreateJobScheduleRequestObject) (CreateJobScheduleResponseObject, error) {430	if req.Body == nil {431		return nil, httpx.Validation("Body required.")432	}433	b := req.Body434	name := strings.TrimSpace(b.Name)435	if name == "" || len(name) > 80 || b.EverySeconds < 60 || b.EverySeconds > 366*86400 {436		return nil, httpx.Validation("name (1-80 characters) and every_seconds (>= 60) are required.")437	}438	if err := h.checkJobSpec(ctx, b.Template, b.Params, b.Placement); err != nil {439		return nil, h.refused(ctx, "job_schedule.create", "job_schedule", "", map[string]any{"template": b.Template}, err)440	}441	now := nowUTC()442	first := now.Add(time.Duration(b.EverySeconds) * time.Second)443	if b.FirstRunAt != nil {444		first = *b.FirstRunAt445	}446	params := deref(b.Params)447	if params == nil {448		params = map[string]any{}449	}450	enabled := b.Enabled == nil || *b.Enabled451	id := ids.New()452	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {453		if _, err := tx.Exec(ctx, `insert into job_schedules(id, name, template, params, placement, every_seconds, enabled, next_run_at, created_by, created_at)454			values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, id, name, b.Template, params, placementJSON(b.Placement), b.EverySeconds, enabled, first, h.userID(ctx), now); err != nil {455			return err456		}457		return h.audit(ctx, tx, "job_schedule.create", "job_schedule", id.String(), audit.Success, "", map[string]any{"name": name, "template": b.Template, "every_seconds": b.EverySeconds})458	})459	if uniqueViolation(err) {460		return nil, httpx.Conflict("conflict", "A schedule with this name already exists.")461	}462	if err != nil {463		return nil, err464	}465	s, err := scanSchedule(h.Pool.QueryRow(ctx, scheduleSelect+` where id=$1`, id))466	if err != nil {467		return nil, err468	}469	return CreateJobSchedule201JSONResponse{Body: s}, nil470}471472func (h *opsHandlers) UpdateJobSchedule(ctx context.Context, req UpdateJobScheduleRequestObject) (UpdateJobScheduleResponseObject, error) {473	if req.Body == nil {474		return nil, httpx.Validation("Body required.")475	}476	b := req.Body477	cur, err := scanSchedule(h.Pool.QueryRow(ctx, scheduleSelect+` where id=$1`, req.Id))478	if err != nil {479		return nil, notFound(err)480	}481	if b.Name != nil {482		if n := strings.TrimSpace(*b.Name); n == "" || len(n) > 80 {483			return nil, httpx.Validation("name must be 1 to 80 characters.")484		}485	}486	if b.EverySeconds != nil && (*b.EverySeconds < 60 || *b.EverySeconds > 366*86400) {487		return nil, httpx.Validation("every_seconds must be >= 60.")488	}489	if b.Params != nil || b.Placement != nil {490		if err := h.checkJobSpec(ctx, cur.Template, b.Params, b.Placement); err != nil {491			return nil, err492		}493	}494	now := nowUTC()495	err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {496		var placement any497		if b.Placement != nil {498			placement = placementJSON(b.Placement)499		}500		var params any501		if b.Params != nil {502			params = *b.Params503		}504		var name *string505		if b.Name != nil {506			name = ptr(strings.TrimSpace(*b.Name))507		}508		// Re-enabling a schedule, or changing its period, restarts the cadence from now:509		// runs missed while it was disabled are not replayed.510		tag, err := tx.Exec(ctx, `update job_schedules set name=coalesce($2,name), params=coalesce($3,params), placement=coalesce($4,placement),511				every_seconds=coalesce($5,every_seconds), enabled=coalesce($6,enabled),512				next_run_at = case when ($6::boolean is true and not enabled) or ($5::int is not null and $5 <> every_seconds)513					then $7::timestamptz + make_interval(secs => coalesce($5,every_seconds)) else next_run_at end514			where id=$1`, req.Id, name, params, placement, b.EverySeconds, b.Enabled, now)515		if err != nil {516			return err517		}518		if tag.RowsAffected() == 0 {519			return httpx.ErrNotFound520		}521		return h.audit(ctx, tx, "job_schedule.update", "job_schedule", req.Id.String(), audit.Success, "", map[string]any{"enabled": b.Enabled, "every_seconds": b.EverySeconds})522	})523	if uniqueViolation(err) {524		return nil, httpx.Conflict("conflict", "A schedule with this name already exists.")525	}526	if err != nil {527		return nil, err528	}529	s, err := scanSchedule(h.Pool.QueryRow(ctx, scheduleSelect+` where id=$1`, req.Id))530	if err != nil {531		return nil, err532	}533	return UpdateJobSchedule200JSONResponse{Body: s}, nil534}535536func (h *opsHandlers) DeleteJobSchedule(ctx context.Context, req DeleteJobScheduleRequestObject) (DeleteJobScheduleResponseObject, error) {537	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {538		tag, err := tx.Exec(ctx, `delete from job_schedules where id=$1`, req.Id)539		if err != nil {540			return err541		}542		if tag.RowsAffected() == 0 {543			return httpx.ErrNotFound544		}545		return h.audit(ctx, tx, "job_schedule.delete", "job_schedule", req.Id.String(), audit.Success, "", nil)546	})547	if err != nil {548		return nil, err549	}550	return DeleteJobSchedule204Response{}, nil551}552
Files main
Menu

Theme