Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/jobs/runner.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Package jobs runs jobs: placement, one job.run action per attempt (so a run43// inherits the audit trail, output, locks and "never re-issue" rules of44// actions), bounded retries, dependencies and schedules.45//46// Templates are the agent-side job templates. agents.capabilities holds plain47// strings, so a template is declared as the capability "job.run:<template>";48// a job is runnable on a node only if that node's agent declares it.49//50// Retry policy (bounded by max_attempts, exponential backoff 30 s → 10 min):51// an attempt is retried only when its outcome is KNOWN and nothing is left52// running — the command never reached the agent (agent_unreachable), the53// node was not usable at dispatch (agent_offline, node_in_maintenance), or the54// agent ran the job and reported a failure (command_failed). It is never55// retried after an unknown outcome (delivered, no result), after an agent-side56// timeout, after a policy refusal, or after a cancellation request.57package jobs5859import (60 "context"61 "errors"62 "fmt"63 "time"6465 "github.com/google/uuid"66 "github.com/jackc/pgx/v5"67 "github.com/jackc/pgx/v5/pgxpool"6869 "clustr.spboucher.ai/mcc/internal/actions"70 "clustr.spboucher.ai/mcc/internal/httpx"71 "clustr.spboucher.ai/mcc/internal/ids"72)7374type Runner struct {75 Pool *pgxpool.Pool76 Now func() time.Time77 FreshAfter time.Duration // automatic placement only considers nodes that reported within this delay78 OfflineAfter time.Duration79}8081func (r *Runner) now() time.Time {82 if r.Now != nil {83 return r.Now()84 }85 return time.Now().UTC()86}8788// Retryable reports whether a failed job.run action may be attempted again.89func Retryable(errorCode string, outcomeUnknown bool) bool {90 if outcomeUnknown {91 return false92 }93 switch errorCode {94 case "agent_unreachable", "agent_offline", "node_in_maintenance", "command_failed":95 return true96 }97 return false98}99100// Backoff before attempt n+1 (n = attempts already made).101func Backoff(attempt int) time.Duration {102 d := 30 * time.Second103 for i := 1; i < attempt && d < 10*time.Minute; i++ {104 d *= 2105 }106 return min(d, 10*time.Minute)107}108109// Tick is one pass: fire due schedules, settle running jobs, start queued ones.110func (r *Runner) Tick(ctx context.Context) error {111 if err := r.schedules(ctx); err != nil {112 return err113 }114 if err := r.settle(ctx); err != nil {115 return err116 }117 return r.start(ctx)118}119120// schedules creates at most ONE job per due schedule, however long the121// scheduler was down: missed runs are not replayed in bulk.122func (r *Runner) schedules(ctx context.Context) error {123 now := r.now()124 return pgx.BeginFunc(ctx, r.Pool, func(tx pgx.Tx) error {125 rows, err := tx.Query(ctx, `select id, template, params, placement, every_seconds, next_run_at, created_by from job_schedules126 where enabled and next_run_at <= $1 order by next_run_at for update skip locked`, now)127 if err != nil {128 return err129 }130 type sched struct {131 id uuid.UUID132 template string133 params map[string]any134 placement map[string]any135 every int136 next time.Time137 by *uuid.UUID138 }139 due, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (sched, error) {140 var s sched141 return s, row.Scan(&s.id, &s.template, &s.params, &s.placement, &s.every, &s.next, &s.by)142 })143 if err != nil {144 return err145 }146 for _, s := range due {147 jobID := ids.New()148 tag, err := tx.Exec(ctx, `insert into jobs(id, template, owner_id, params, placement, schedule_id, dedupe_key, created_at, max_attempts)149 values ($1,$2,$3,$4,$5,$6,$7,$8,1) on conflict (dedupe_key) do nothing`,150 jobID, s.template, s.by, s.params, s.placement, s.id, fmt.Sprintf("sched:%s:%d", s.id, s.next.Unix()), now)151 if err != nil {152 return err153 }154 if tag.RowsAffected() == 1 {155 if err := actions.JobChanged(ctx, tx, jobID); err != nil {156 return err157 }158 }159 every := time.Duration(s.every) * time.Second160 next := s.next.Add((now.Sub(s.next)/every + 1) * every) // first slot strictly after now161 if _, err := tx.Exec(ctx, `update job_schedules set last_run_at=$2, next_run_at=$3 where id=$1`, s.id, now, next); err != nil {162 return err163 }164 }165 return nil166 })167}168169// settle follows the job.run action of every running job.170func (r *Runner) settle(ctx context.Context) error {171 now := r.now()172 rows, err := r.Pool.Query(ctx, `select id from jobs where state='running' order by created_at`)173 if err != nil {174 return err175 }176 list, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])177 if err != nil {178 return err179 }180 for _, id := range list {181 err := pgx.BeginFunc(ctx, r.Pool, func(tx pgx.Tx) error {182 var attempt, maxAttempts int183 var cancelAt *time.Time184 var pinned bool185 err := tx.QueryRow(ctx, `select attempt, max_attempts, cancel_requested_at, placement ? 'node_id' from jobs where id=$1 and state='running' for update skip locked`, id).186 Scan(&attempt, &maxAttempts, &cancelAt, &pinned)187 if errors.Is(err, pgx.ErrNoRows) {188 return nil189 }190 if err != nil {191 return err192 }193 var astate string194 var code, msg *string195 var result map[string]any196 err = tx.QueryRow(ctx, `select state::text, error_code, error_message, result from actions where job_id=$1 and type='job.run' order by created_at desc, id desc limit 1`, id).197 Scan(&astate, &code, &msg, &result)198 if errors.Is(err, pgx.ErrNoRows) {199 astate, code, msg = "failed", ptr("internal"), ptr("No run was recorded for this attempt.")200 } else if err != nil {201 return err202 }203 if !actions.IsTerminal(astate) {204 return nil205 }206 unknown, _ := result["outcome_unknown"].(bool)207 final, errText := astate, deref(msg)208 switch astate {209 case "succeeded":210 errText = ""211 case "failed":212 if Retryable(deref(code), unknown) && attempt < maxAttempts && cancelAt == nil {213 // Safe to try again: a new attempt is a new action (and, if it gets that far, a new command).214 next := now.Add(Backoff(attempt))215 if _, err := tx.Exec(ctx, `update jobs set state='queued', next_attempt_at=$2, error=$3,216 node_id = case when $4 then node_id else null end where id=$1`, id, next, "attempt "+fmt.Sprint(attempt)+" failed ("+deref(code)+"): "+errText, pinned); err != nil {217 return err218 }219 return actions.JobChanged(ctx, tx, id)220 }221 if cancelAt != nil && deref(code) != "command_failed" {222 final = "cancelled"223 }224 case "timed_out":225 if unknown {226 errText = "Outcome unknown: the run reached the agent but no result came back. Not retried. " + errText227 }228 case "cancelled":229 default: // rejected, expired: cannot happen for job.run, but never leave a job running forever230 final = "failed"231 }232 var usage any233 if u, ok := result["usage"].(map[string]any); ok {234 usage = u235 }236 if _, err := tx.Exec(ctx, `update jobs set state=$2::job_state, finished_at=$3, result=$4, error=nullif($5,''), usage=$6, next_attempt_at=null,237 progress = case when $2='succeeded' then 1 else progress end where id=$1`, id, final, now, result, errText, usage); err != nil {238 return err239 }240 return actions.JobChanged(ctx, tx, id)241 })242 if err != nil {243 return err244 }245 }246 return nil247}248249// start places and launches queued jobs whose dependencies have succeeded.250func (r *Runner) start(ctx context.Context) error {251 now := r.now()252 rows, err := r.Pool.Query(ctx, `select id from jobs where state='queued' and (next_attempt_at is null or next_attempt_at <= $1) order by created_at, id limit 200`, now)253 if err != nil {254 return err255 }256 list, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])257 if err != nil {258 return err259 }260 for _, id := range list {261 waiting, err := r.startOne(ctx, id, now)262 if err != nil {263 return err264 }265 // Say why a job is still queued; this is information, not a failure.266 if _, err := r.Pool.Exec(ctx, `update jobs set error=nullif($2,'') where id=$1 and state='queued' and coalesce(error,'') not like 'attempt %'267 and coalesce(error,'') is distinct from $2`, id, waiting); err != nil {268 return err269 }270 }271 return nil272}273274func (r *Runner) startOne(ctx context.Context, id uuid.UUID, now time.Time) (waiting string, err error) {275 tx, err := r.Pool.Begin(ctx)276 if err != nil {277 return "", err278 }279 defer tx.Rollback(ctx)280 var (281 template string282 owner *uuid.UUID283 params map[string]any284 placement struct {285 NodeID *uuid.UUID `json:"node_id"`286 Site *string `json:"site"`287 Role *string `json:"role"`288 Tags []string `json:"tags"`289 }290 attempt int291 deps []uuid.UUID292 timeout *int293 )294 err = tx.QueryRow(ctx, `select template, owner_id, params, placement, attempt, depends_on, timeout_s from jobs295 where id=$1 and state='queued' for update skip locked`, id).Scan(&template, &owner, ¶ms, &placement, &attempt, &deps, &timeout)296 if errors.Is(err, pgx.ErrNoRows) {297 return "", nil298 }299 if err != nil {300 return "", err301 }302 fail := func(msg string) (string, error) {303 if _, err := tx.Exec(ctx, `update jobs set state='failed', finished_at=$2, error=$3, next_attempt_at=null where id=$1`, id, now, msg); err != nil {304 return "", err305 }306 if err := actions.JobChanged(ctx, tx, id); err != nil {307 return "", err308 }309 return "", tx.Commit(ctx)310 }311 // Dependencies: every one must have succeeded.312 if len(deps) > 0 {313 rows, err := tx.Query(ctx, `select d.id, j.state::text from unnest($1::uuid[]) d(id) left join jobs j on j.id=d.id`, deps)314 if err != nil {315 return "", err316 }317 type dep struct {318 id uuid.UUID319 state *string320 }321 ds, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (dep, error) {322 var d dep323 return d, row.Scan(&d.id, &d.state)324 })325 if err != nil {326 return "", err327 }328 pending := 0329 for _, d := range ds {330 switch deref(d.state) {331 case "succeeded":332 case "queued", "running":333 pending++334 default:335 return fail(fmt.Sprintf("dependency %s did not succeed (%s)", d.id, orStr(deref(d.state), "missing")))336 }337 }338 if pending > 0 {339 return "waiting for dependencies", nil340 }341 }342 if frozen, err := actions.Frozen(ctx, tx); err != nil {343 return "", err344 } else if frozen {345 return "waiting: the emergency stop is engaged", nil346 }347 // Placement, from real data only.348 node := placement.NodeID349 if node == nil {350 var picked uuid.UUID351 var tags any352 if len(placement.Tags) > 0 {353 tags = placement.Tags354 }355 err := tx.QueryRow(ctx, `select n.id from nodes n356 join agents a on a.node_id=n.id and a.revoked_at is null357 left join node_state ns on ns.node_id=n.id358 where n.retired_at is null and not n.maintenance and $1 = any(a.capabilities)359 and greatest(a.last_seen_at, ns.received_at) > $2360 and ($3::text is null or n.site=$3) and ($4::text is null or n.role=$4) and ($5::text[] is null or n.tags @> $5)361 order by (ns.values ? 'load.1' and coalesce(n.cpu_cores,0) > 0) desc,362 (ns.values->>'load.1')::float8 / nullif(n.cpu_cores,0), n.alias363 limit 1`, actions.JobCapability(template), now.Add(-r.FreshAfter), placement.Site, placement.Role, tags).Scan(&picked)364 if errors.Is(err, pgx.ErrNoRows) {365 return "waiting: no eligible node (fresh, not in maintenance, declaring the template)", nil366 }367 if err != nil {368 return "", err369 }370 node = &picked371 }372 if _, err := tx.Exec(ctx, `update jobs set node_id=$2, attempt=attempt+1, state='running', started_at=coalesce(started_at,$3), next_attempt_at=null, error=null where id=$1`,373 id, *node, now); err != nil {374 return "", err375 }376 t, _ := actions.Lookup("job.run")377 tg, err := actions.ResolveTarget(ctx, tx, "job", id)378 if err != nil {379 return "", err380 }381 ap := map[string]any{"template": template}382 if len(params) > 0 {383 ap["params"] = params384 }385 if _, err := actions.Precheck(ctx, tx, t, tg, ap, actions.PrecheckOpts{Now: now, OfflineAfter: r.OfflineAfter}); err != nil {386 var he *httpx.Error387 if errors.As(err, &he) && he.Status < 500 {388 return "waiting: " + he.Message, nil // rolled back: not an attempt389 }390 return "", err391 }392 via := "api"393 if owner == nil {394 via = "scheduler"395 }396 if _, _, err := actions.Insert(ctx, tx, actions.Request{Type: t, Target: tg, Params: ap, RequestedBy: owner, Via: via,397 IdempotencyKey: fmt.Sprintf("job:%s:%d", id, attempt+1), RequestHash: actions.HashRequest(ap), CorrelationID: uuid.New(),398 JobID: &id, TimeoutS: deref(timeout), Now: now}); err != nil {399 var he *httpx.Error400 if errors.As(err, &he) && he.Status < 500 {401 return "waiting: " + he.Message, nil402 }403 return "", err404 }405 if err := actions.JobChanged(ctx, tx, id); err != nil {406 return "", err407 }408 return "", tx.Commit(ctx)409}410411func ptr[T any](v T) *T { return &v }412413func deref[T any](p *T) (v T) {414 if p != nil {415 v = *p416 }417 return v418}419420func orStr(s, def string) string {421 if s == "" {422 return def423 }424 return s425}426