SPB Git

Repository Public

runner.go 14.0 KB 426 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/jobs/runner.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Package jobs runs jobs: placement, one job.run action per attempt (so a run43// inherits the audit trail, output, locks and "never re-issue" rules of44// actions), bounded retries, dependencies and schedules.45//46// Templates are the agent-side job templates. agents.capabilities holds plain47// strings, so a template is declared as the capability "job.run:<template>";48// a job is runnable on a node only if that node's agent declares it.49//50// Retry policy (bounded by max_attempts, exponential backoff 30 s → 10 min):51// an attempt is retried only when its outcome is KNOWN and nothing is left52// running — the command never reached the agent (agent_unreachable), the53// node was not usable at dispatch (agent_offline, node_in_maintenance), or the54// agent ran the job and reported a failure (command_failed). It is never55// retried after an unknown outcome (delivered, no result), after an agent-side56// timeout, after a policy refusal, or after a cancellation request.57package jobs5859import (60	"context"61	"errors"62	"fmt"63	"time"6465	"github.com/google/uuid"66	"github.com/jackc/pgx/v5"67	"github.com/jackc/pgx/v5/pgxpool"6869	"clustr.spboucher.ai/mcc/internal/actions"70	"clustr.spboucher.ai/mcc/internal/httpx"71	"clustr.spboucher.ai/mcc/internal/ids"72)7374type Runner struct {75	Pool         *pgxpool.Pool76	Now          func() time.Time77	FreshAfter   time.Duration // automatic placement only considers nodes that reported within this delay78	OfflineAfter time.Duration79}8081func (r *Runner) now() time.Time {82	if r.Now != nil {83		return r.Now()84	}85	return time.Now().UTC()86}8788// Retryable reports whether a failed job.run action may be attempted again.89func Retryable(errorCode string, outcomeUnknown bool) bool {90	if outcomeUnknown {91		return false92	}93	switch errorCode {94	case "agent_unreachable", "agent_offline", "node_in_maintenance", "command_failed":95		return true96	}97	return false98}99100// Backoff before attempt n+1 (n = attempts already made).101func Backoff(attempt int) time.Duration {102	d := 30 * time.Second103	for i := 1; i < attempt && d < 10*time.Minute; i++ {104		d *= 2105	}106	return min(d, 10*time.Minute)107}108109// Tick is one pass: fire due schedules, settle running jobs, start queued ones.110func (r *Runner) Tick(ctx context.Context) error {111	if err := r.schedules(ctx); err != nil {112		return err113	}114	if err := r.settle(ctx); err != nil {115		return err116	}117	return r.start(ctx)118}119120// schedules creates at most ONE job per due schedule, however long the121// scheduler was down: missed runs are not replayed in bulk.122func (r *Runner) schedules(ctx context.Context) error {123	now := r.now()124	return pgx.BeginFunc(ctx, r.Pool, func(tx pgx.Tx) error {125		rows, err := tx.Query(ctx, `select id, template, params, placement, every_seconds, next_run_at, created_by from job_schedules126			where enabled and next_run_at <= $1 order by next_run_at for update skip locked`, now)127		if err != nil {128			return err129		}130		type sched struct {131			id        uuid.UUID132			template  string133			params    map[string]any134			placement map[string]any135			every     int136			next      time.Time137			by        *uuid.UUID138		}139		due, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (sched, error) {140			var s sched141			return s, row.Scan(&s.id, &s.template, &s.params, &s.placement, &s.every, &s.next, &s.by)142		})143		if err != nil {144			return err145		}146		for _, s := range due {147			jobID := ids.New()148			tag, err := tx.Exec(ctx, `insert into jobs(id, template, owner_id, params, placement, schedule_id, dedupe_key, created_at, max_attempts)149				values ($1,$2,$3,$4,$5,$6,$7,$8,1) on conflict (dedupe_key) do nothing`,150				jobID, s.template, s.by, s.params, s.placement, s.id, fmt.Sprintf("sched:%s:%d", s.id, s.next.Unix()), now)151			if err != nil {152				return err153			}154			if tag.RowsAffected() == 1 {155				if err := actions.JobChanged(ctx, tx, jobID); err != nil {156					return err157				}158			}159			every := time.Duration(s.every) * time.Second160			next := s.next.Add((now.Sub(s.next)/every + 1) * every) // first slot strictly after now161			if _, err := tx.Exec(ctx, `update job_schedules set last_run_at=$2, next_run_at=$3 where id=$1`, s.id, now, next); err != nil {162				return err163			}164		}165		return nil166	})167}168169// settle follows the job.run action of every running job.170func (r *Runner) settle(ctx context.Context) error {171	now := r.now()172	rows, err := r.Pool.Query(ctx, `select id from jobs where state='running' order by created_at`)173	if err != nil {174		return err175	}176	list, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])177	if err != nil {178		return err179	}180	for _, id := range list {181		err := pgx.BeginFunc(ctx, r.Pool, func(tx pgx.Tx) error {182			var attempt, maxAttempts int183			var cancelAt *time.Time184			var pinned bool185			err := tx.QueryRow(ctx, `select attempt, max_attempts, cancel_requested_at, placement ? 'node_id' from jobs where id=$1 and state='running' for update skip locked`, id).186				Scan(&attempt, &maxAttempts, &cancelAt, &pinned)187			if errors.Is(err, pgx.ErrNoRows) {188				return nil189			}190			if err != nil {191				return err192			}193			var astate string194			var code, msg *string195			var result map[string]any196			err = tx.QueryRow(ctx, `select state::text, error_code, error_message, result from actions where job_id=$1 and type='job.run' order by created_at desc, id desc limit 1`, id).197				Scan(&astate, &code, &msg, &result)198			if errors.Is(err, pgx.ErrNoRows) {199				astate, code, msg = "failed", ptr("internal"), ptr("No run was recorded for this attempt.")200			} else if err != nil {201				return err202			}203			if !actions.IsTerminal(astate) {204				return nil205			}206			unknown, _ := result["outcome_unknown"].(bool)207			final, errText := astate, deref(msg)208			switch astate {209			case "succeeded":210				errText = ""211			case "failed":212				if Retryable(deref(code), unknown) && attempt < maxAttempts && cancelAt == nil {213					// Safe to try again: a new attempt is a new action (and, if it gets that far, a new command).214					next := now.Add(Backoff(attempt))215					if _, err := tx.Exec(ctx, `update jobs set state='queued', next_attempt_at=$2, error=$3,216						node_id = case when $4 then node_id else null end where id=$1`, id, next, "attempt "+fmt.Sprint(attempt)+" failed ("+deref(code)+"): "+errText, pinned); err != nil {217						return err218					}219					return actions.JobChanged(ctx, tx, id)220				}221				if cancelAt != nil && deref(code) != "command_failed" {222					final = "cancelled"223				}224			case "timed_out":225				if unknown {226					errText = "Outcome unknown: the run reached the agent but no result came back. Not retried. " + errText227				}228			case "cancelled":229			default: // rejected, expired: cannot happen for job.run, but never leave a job running forever230				final = "failed"231			}232			var usage any233			if u, ok := result["usage"].(map[string]any); ok {234				usage = u235			}236			if _, err := tx.Exec(ctx, `update jobs set state=$2::job_state, finished_at=$3, result=$4, error=nullif($5,''), usage=$6, next_attempt_at=null,237				progress = case when $2='succeeded' then 1 else progress end where id=$1`, id, final, now, result, errText, usage); err != nil {238				return err239			}240			return actions.JobChanged(ctx, tx, id)241		})242		if err != nil {243			return err244		}245	}246	return nil247}248249// start places and launches queued jobs whose dependencies have succeeded.250func (r *Runner) start(ctx context.Context) error {251	now := r.now()252	rows, err := r.Pool.Query(ctx, `select id from jobs where state='queued' and (next_attempt_at is null or next_attempt_at <= $1) order by created_at, id limit 200`, now)253	if err != nil {254		return err255	}256	list, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])257	if err != nil {258		return err259	}260	for _, id := range list {261		waiting, err := r.startOne(ctx, id, now)262		if err != nil {263			return err264		}265		// Say why a job is still queued; this is information, not a failure.266		if _, err := r.Pool.Exec(ctx, `update jobs set error=nullif($2,'') where id=$1 and state='queued' and coalesce(error,'') not like 'attempt %'267			and coalesce(error,'') is distinct from $2`, id, waiting); err != nil {268			return err269		}270	}271	return nil272}273274func (r *Runner) startOne(ctx context.Context, id uuid.UUID, now time.Time) (waiting string, err error) {275	tx, err := r.Pool.Begin(ctx)276	if err != nil {277		return "", err278	}279	defer tx.Rollback(ctx)280	var (281		template  string282		owner     *uuid.UUID283		params    map[string]any284		placement struct {285			NodeID *uuid.UUID `json:"node_id"`286			Site   *string    `json:"site"`287			Role   *string    `json:"role"`288			Tags   []string   `json:"tags"`289		}290		attempt int291		deps    []uuid.UUID292		timeout *int293	)294	err = tx.QueryRow(ctx, `select template, owner_id, params, placement, attempt, depends_on, timeout_s from jobs295		where id=$1 and state='queued' for update skip locked`, id).Scan(&template, &owner, &params, &placement, &attempt, &deps, &timeout)296	if errors.Is(err, pgx.ErrNoRows) {297		return "", nil298	}299	if err != nil {300		return "", err301	}302	fail := func(msg string) (string, error) {303		if _, err := tx.Exec(ctx, `update jobs set state='failed', finished_at=$2, error=$3, next_attempt_at=null where id=$1`, id, now, msg); err != nil {304			return "", err305		}306		if err := actions.JobChanged(ctx, tx, id); err != nil {307			return "", err308		}309		return "", tx.Commit(ctx)310	}311	// Dependencies: every one must have succeeded.312	if len(deps) > 0 {313		rows, err := tx.Query(ctx, `select d.id, j.state::text from unnest($1::uuid[]) d(id) left join jobs j on j.id=d.id`, deps)314		if err != nil {315			return "", err316		}317		type dep struct {318			id    uuid.UUID319			state *string320		}321		ds, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (dep, error) {322			var d dep323			return d, row.Scan(&d.id, &d.state)324		})325		if err != nil {326			return "", err327		}328		pending := 0329		for _, d := range ds {330			switch deref(d.state) {331			case "succeeded":332			case "queued", "running":333				pending++334			default:335				return fail(fmt.Sprintf("dependency %s did not succeed (%s)", d.id, orStr(deref(d.state), "missing")))336			}337		}338		if pending > 0 {339			return "waiting for dependencies", nil340		}341	}342	if frozen, err := actions.Frozen(ctx, tx); err != nil {343		return "", err344	} else if frozen {345		return "waiting: the emergency stop is engaged", nil346	}347	// Placement, from real data only.348	node := placement.NodeID349	if node == nil {350		var picked uuid.UUID351		var tags any352		if len(placement.Tags) > 0 {353			tags = placement.Tags354		}355		err := tx.QueryRow(ctx, `select n.id from nodes n356			join agents a on a.node_id=n.id and a.revoked_at is null357			left join node_state ns on ns.node_id=n.id358			where n.retired_at is null and not n.maintenance and $1 = any(a.capabilities)359			  and greatest(a.last_seen_at, ns.received_at) > $2360			  and ($3::text is null or n.site=$3) and ($4::text is null or n.role=$4) and ($5::text[] is null or n.tags @> $5)361			order by (ns.values ? 'load.1' and coalesce(n.cpu_cores,0) > 0) desc,362			         (ns.values->>'load.1')::float8 / nullif(n.cpu_cores,0), n.alias363			limit 1`, actions.JobCapability(template), now.Add(-r.FreshAfter), placement.Site, placement.Role, tags).Scan(&picked)364		if errors.Is(err, pgx.ErrNoRows) {365			return "waiting: no eligible node (fresh, not in maintenance, declaring the template)", nil366		}367		if err != nil {368			return "", err369		}370		node = &picked371	}372	if _, err := tx.Exec(ctx, `update jobs set node_id=$2, attempt=attempt+1, state='running', started_at=coalesce(started_at,$3), next_attempt_at=null, error=null where id=$1`,373		id, *node, now); err != nil {374		return "", err375	}376	t, _ := actions.Lookup("job.run")377	tg, err := actions.ResolveTarget(ctx, tx, "job", id)378	if err != nil {379		return "", err380	}381	ap := map[string]any{"template": template}382	if len(params) > 0 {383		ap["params"] = params384	}385	if _, err := actions.Precheck(ctx, tx, t, tg, ap, actions.PrecheckOpts{Now: now, OfflineAfter: r.OfflineAfter}); err != nil {386		var he *httpx.Error387		if errors.As(err, &he) && he.Status < 500 {388			return "waiting: " + he.Message, nil // rolled back: not an attempt389		}390		return "", err391	}392	via := "api"393	if owner == nil {394		via = "scheduler"395	}396	if _, _, err := actions.Insert(ctx, tx, actions.Request{Type: t, Target: tg, Params: ap, RequestedBy: owner, Via: via,397		IdempotencyKey: fmt.Sprintf("job:%s:%d", id, attempt+1), RequestHash: actions.HashRequest(ap), CorrelationID: uuid.New(),398		JobID: &id, TimeoutS: deref(timeout), Now: now}); err != nil {399		var he *httpx.Error400		if errors.As(err, &he) && he.Status < 500 {401			return "waiting: " + he.Message, nil402		}403		return "", err404	}405	if err := actions.JobChanged(ctx, tx, id); err != nil {406		return "", err407	}408	return "", tx.Commit(ctx)409}410411func ptr[T any](v T) *T { return &v }412413func deref[T any](p *T) (v T) {414	if p != nil {415		v = *p416	}417	return v418}419420func orStr(s, def string) string {421	if s == "" {422		return def423	}424	return s425}426
Files main
Menu

Theme