SPB Git

Repository Public

orchestrator.go 20.0 KB 612 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/orchestrator/orchestrator.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package orchestrator4344import (45	"context"46	"encoding/json"47	"errors"48	"log/slog"49	"sync"50	"time"5152	"github.com/google/uuid"53	"github.com/jackc/pgx/v5"54	"github.com/jackc/pgx/v5/pgxpool"5556	"clustr.spboucher.ai/mcc/internal/actions"57	"clustr.spboucher.ai/mcc/internal/httpx"58	"clustr.spboucher.ai/mcc/internal/ids"59	"clustr.spboucher.ai/mcc/internal/protocol"60)6162// Options tunes the worker. Zero values are production defaults.63type Options struct {64	CommandTTL   time.Duration    // validity of a signed command (<= protocol.MaxCommandLifetime); default 90 s65	Grace        time.Duration    // added to expires_at + timeout before declaring the outcome unknown; default 30 s66	SweepEvery   time.Duration    // safety-net pass when no notification arrives; default 2 s67	OfflineAfter time.Duration    // agent considered offline after this silence; default 120 s68	Now          func() time.Time // clock (tests)69}7071// Orchestrator moves actions through their life cycle.72//73// Hard rules, enforced here and by the schema:74//   - one action issues at most ONE command (same transaction as queued →75//     dispatched; unique index agent_commands(action_id)); nothing ever76//     re-issues it, neither a restart nor a second instance;77//   - a command never delivered before expires_at ends the action `failed`78//     with error_code agent_unreachable (it certainly did not run);79//   - a command delivered without a result before the deadline ends the action80//     `timed_out` with result.outcome_unknown=true: it may or may not have run;81//   - while settings['orchestrator.frozen'] is true nothing is dispatched.82type Orchestrator struct {83	pool   *pgxpool.Pool84	signer *Signer85	opt    Options86	wake   chan struct{}8788	mu       sync.Mutex89	lastStep time.Time90	lastErr  string91	frozen   bool92	counts   struct{ dispatched, finished, expired int64 }93}9495func New(pool *pgxpool.Pool, signer *Signer, opt Options) *Orchestrator {96	if opt.CommandTTL <= 0 || opt.CommandTTL > protocol.MaxCommandLifetime {97		opt.CommandTTL = 90 * time.Second98	}99	if opt.Grace <= 0 {100		opt.Grace = 30 * time.Second101	}102	if opt.SweepEvery <= 0 {103		opt.SweepEvery = 2 * time.Second104	}105	if opt.OfflineAfter <= 0 {106		opt.OfflineAfter = 120 * time.Second107	}108	if opt.Now == nil {109		opt.Now = func() time.Time { return time.Now().UTC() }110	}111	return &Orchestrator{pool: pool, signer: signer, opt: opt, wake: make(chan struct{}, 1)}112}113114// Status is published in the service heartbeat.115func (o *Orchestrator) Status() map[string]any {116	o.mu.Lock()117	defer o.mu.Unlock()118	s := map[string]any{"frozen": o.frozen, "key_id": o.signer.KeyID, "dispatched": o.counts.dispatched, "finished": o.counts.finished,119		"expired": o.counts.expired}120	if !o.lastStep.IsZero() {121		s["last_pass_at"] = o.lastStep122	}123	if o.lastErr != "" {124		s["last_error"] = o.lastErr125	}126	return s127}128129// Run works until ctx ends. A database outage is survived: passes fail, are130// retried with backoff, and the first successful pass reconciles everything131// from the tables.132func (o *Orchestrator) Run(ctx context.Context) {133	go o.listen(ctx)134	backoff := time.Duration(0)135	for ctx.Err() == nil {136		err := o.Step(ctx)137		wait := o.opt.SweepEvery138		o.mu.Lock()139		if err != nil && ctx.Err() == nil {140			if backoff == 0 {141				backoff = time.Second142			} else if backoff < 15*time.Second {143				backoff *= 2144			}145			wait, o.lastErr = backoff, err.Error()146			slog.Warn("orchestrator: pass failed", "err", err, "retry_in", backoff)147		} else {148			backoff, o.lastErr, o.lastStep = 0, "", time.Now().UTC()149		}150		o.mu.Unlock()151		t := time.NewTimer(wait)152		select {153		case <-ctx.Done():154			t.Stop()155			return156		case <-o.wake:157			if err != nil { // do not spin on notifications while the database is failing158				select {159				case <-ctx.Done():160				case <-t.C:161				}162			}163		case <-t.C:164		}165		t.Stop()166	}167}168169// listen wakes the loop on command updates and new actions. Notifications are170// only an accelerator: the periodic pass finds everything without them.171func (o *Orchestrator) listen(ctx context.Context) {172	backoff := time.Second173	for ctx.Err() == nil {174		err := func() error {175			conn, err := o.pool.Acquire(ctx)176			if err != nil {177				return err178			}179			defer conn.Release()180			defer func() {181				// A connection that still listens must not go back to the pool as is.182				c, cancel := context.WithTimeout(context.Background(), 2*time.Second)183				defer cancel()184				if _, err := conn.Exec(c, "unlisten *"); err != nil {185					conn.Conn().Close(c)186				}187			}()188			for _, ch := range []string{actions.ChanCommandUpdates, actions.ChanActions} {189				if _, err := conn.Exec(ctx, "listen "+ch); err != nil {190					return err191				}192			}193			backoff = time.Second194			for {195				if _, err := conn.Conn().WaitForNotification(ctx); err != nil {196					return err197				}198				o.Wake()199			}200		}()201		if ctx.Err() != nil {202			return203		}204		slog.Warn("orchestrator: listener disconnected", "err", err, "retry_in", backoff)205		select {206		case <-ctx.Done():207			return208		case <-time.After(backoff):209		}210		if backoff < 15*time.Second {211			backoff *= 2212		}213	}214}215216// Wake requests an immediate pass.217func (o *Orchestrator) Wake() {218	select {219	case o.wake <- struct{}{}:220	default:221	}222}223224// Step is one complete pass: expire approvals, dispatch what is queued,225// reconcile everything in flight. It is idempotent and safe to run from226// several instances at once.227func (o *Orchestrator) Step(ctx context.Context) error {228	if err := o.expireApprovals(ctx); err != nil {229		return err230	}231	for i := 0; i < 200; i++ {232		more, err := o.dispatchOne(ctx)233		if err != nil {234			return err235		}236		if !more {237			break238		}239	}240	if err := o.reconcile(ctx); err != nil {241		return err242	}243	return o.lateResults(ctx)244}245246func (o *Orchestrator) expireApprovals(ctx context.Context) error {247	now := o.opt.Now()248	return pgx.BeginFunc(ctx, o.pool, func(tx pgx.Tx) error {249		rows, err := tx.Query(ctx, `select id from actions where state='pending_approval' and created_at <= $1 for update skip locked`, now.Add(-actions.ApprovalTTL))250		if err != nil {251			return err252		}253		idsToExpire, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])254		if err != nil {255			return err256		}257		for _, id := range idsToExpire {258			if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "expired", ErrorCode: "approval_expired",259				ErrorMessage: "No approval was given within 30 minutes.", At: now}); err != nil {260				return err261			}262		}263		return nil264	})265}266267// dispatchOne claims one queued action and either executes it (server-side268// types) or signs and records its command. Claim, command and state change269// commit together or not at all.270func (o *Orchestrator) dispatchOne(ctx context.Context) (more bool, err error) {271	now := o.opt.Now()272	tx, err := o.pool.Begin(ctx)273	if err != nil {274		return false, err275	}276	defer tx.Rollback(ctx)277278	frozen, err := actions.Frozen(ctx, tx)279	if err != nil {280		return false, err281	}282	o.mu.Lock()283	o.frozen = frozen284	o.mu.Unlock()285	if frozen {286		return false, nil // queued actions wait; their state says so287	}288	var (289		id       uuid.UUID290		typ, tk  string291		targetID string292		params   map[string]any293		timeout  int294		override bool295	)296	err = tx.QueryRow(ctx, `select id, type, target_kind, target_id, params, timeout_s, override_maintenance from actions297		where state='queued' order by queued_at nulls first, id limit 1 for update skip locked`).298		Scan(&id, &typ, &tk, &targetID, &params, &timeout, &override)299	if errors.Is(err, pgx.ErrNoRows) {300		return false, nil301	}302	if err != nil {303		return false, err304	}305	fail := func(code, msg string) (bool, error) {306		if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "failed", ErrorCode: code, ErrorMessage: msg, At: now}); err != nil {307			return false, err308		}309		return true, tx.Commit(ctx)310	}311	refused := func(err error) (bool, error) {312		var he *httpx.Error313		if errors.As(err, &he) && he.Status < 500 {314			return fail(he.Code, he.Message)315		}316		return false, err317	}318	t, ok := actions.Lookup(typ)319	if !ok {320		return fail("unknown_type", "This action type is not in the catalog.")321	}322	tid, perr := uuid.Parse(targetID)323	if perr != nil {324		return fail("precheck_failed", "Invalid target.")325	}326	tg, err := actions.ResolveTarget(ctx, tx, tk, tid)327	if err != nil {328		return refused(err)329	}330	// Pre-checks are verified again now: the world may have changed since the request.331	agentID, err := actions.Precheck(ctx, tx, t, tg, params, actions.PrecheckOpts{OverrideMaintenance: override, Now: now, OfflineAfter: o.opt.OfflineAfter})332	if err != nil {333		return refused(err)334	}335336	if t.ServerSide {337		if _, err := actions.Transition(ctx, tx, id, "queued", "running", now); err != nil {338			return false, err339		}340		res, err := actions.ExecuteServerSide(ctx, tx, typ, tg, params, now)341		if err != nil {342			var he *httpx.Error343			if errors.As(err, &he) && he.Status < 500 {344				if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "failed", ErrorCode: he.Code, ErrorMessage: he.Message, At: now}); err != nil {345					return false, err346				}347				return true, tx.Commit(ctx)348			}349			return false, err350		}351		if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "succeeded", Result: res, At: now}); err != nil {352			return false, err353		}354		o.count(func() { o.counts.finished++ })355		return true, tx.Commit(ctx)356	}357358	// Parameters as the agent's executors decode them (strictly: unknown fields359	// are refused). A service target is addressed by manager and name; node360	// targets need nothing more than the validated user parameters.361	wire := map[string]any{}362	for k, v := range params {363		wire[k] = v364	}365	if tg.Wire["kind"] == "service" {366		wire["manager"], wire["name"] = tg.Wire["manager"], tg.Wire["name"]367	}368	raw, err := json.Marshal(wire)369	if err != nil {370		return false, err371	}372	cmd := protocol.Command{V: protocol.Version, ID: ids.New().String(), ActionID: id.String(), AgentID: agentID.String(), NodeID: tg.NodeID.String(),373		Type: typ, Params: raw, TimeoutS: timeout, IssuedAt: now, ExpiresAt: now.Add(o.opt.CommandTTL), Nonce: protocol.NewNonce()}374	signed, err := protocol.Sign(o.signer.Private(), o.signer.KeyID, cmd)375	if err != nil {376		return false, err377	}378	deadline := cmd.ExpiresAt.Add(time.Duration(timeout)*time.Second + o.opt.Grace)379	// The unique index on agent_commands(action_id) makes a second command for380	// this action impossible, whatever the code does.381	if _, err := tx.Exec(ctx, `insert into agent_commands(id, action_id, agent_id, envelope, signature, key_id, issued_at, expires_at, state)382		values ($1,$2,$3,$4,$5,$6,$7,$8,'pending')`, cmd.ID, id, agentID, signed.Payload, signed.Signature, signed.KeyID, cmd.IssuedAt, cmd.ExpiresAt); err != nil {383		return false, err384	}385	if _, err := tx.Exec(ctx, `update actions set deadline_at=$2 where id=$1`, id, deadline); err != nil {386		return false, err387	}388	if _, err := actions.Transition(ctx, tx, id, "queued", "dispatched", now); err != nil {389		return false, err390	}391	if _, err := tx.Exec(ctx, `select pg_notify($1,$2)`, actions.ChanCommands, agentID.String()); err != nil {392		return false, err393	}394	if err := tx.Commit(ctx); err != nil {395		return false, err396	}397	o.count(func() { o.counts.dispatched++ })398	return true, nil399}400401func (o *Orchestrator) count(f func()) {402	o.mu.Lock()403	f()404	o.mu.Unlock()405}406407// reconcile brings every in-flight action in line with its command row.408func (o *Orchestrator) reconcile(ctx context.Context) error {409	rows, err := o.pool.Query(ctx, `select id from actions where state in ('dispatched','running') order by created_at`)410	if err != nil {411		return err412	}413	list, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])414	if err != nil {415		return err416	}417	for _, id := range list {418		if err := o.reconcileOne(ctx, id); err != nil {419			return err420		}421	}422	return nil423}424425func (o *Orchestrator) reconcileOne(ctx context.Context, id uuid.UUID) error {426	now := o.opt.Now()427	return pgx.BeginFunc(ctx, o.pool, func(tx pgx.Tx) error {428		var state string429		var deadline *time.Time430		err := tx.QueryRow(ctx, `select state::text, deadline_at from actions where id=$1 and state in ('dispatched','running') for update skip locked`, id).Scan(&state, &deadline)431		if errors.Is(err, pgx.ErrNoRows) {432			return nil // finished meanwhile, or another instance is on it433		}434		if err != nil {435			return err436		}437		var (438			cmdID     uuid.UUID439			cstate    string440			expires   time.Time441			delivered *time.Time442			acked     *time.Time443			accepted  *bool444			reason    *string445			rstatus   *string446			result    map[string]any447		)448		err = tx.QueryRow(ctx, `select id, state, expires_at, delivered_at, acked_at, ack_accepted, ack_reason, result_status, result449			from agent_commands where action_id=$1 for update`, id).Scan(&cmdID, &cstate, &expires, &delivered, &acked, &accepted, &reason, &rstatus, &result)450		finish := func(out actions.Outcome) error {451			out.At = now452			_, err := actions.Finish(ctx, tx, id, out)453			o.count(func() { o.counts.finished++ })454			return err455		}456		if errors.Is(err, pgx.ErrNoRows) {457			return finish(actions.Outcome{State: "failed", ErrorCode: "internal", ErrorMessage: "No command is recorded for this action; nothing was sent."})458		}459		if err != nil {460			return err461		}462		if acked != nil { // record when execution really started, even if the result is already there463			if _, err := tx.Exec(ctx, `update actions set started_at=coalesce(started_at,$2) where id=$1`, id, *acked); err != nil {464				return err465			}466		}467		switch cstate {468		case "finished":469			if accepted != nil && !*accepted {470				msg := "The agent refused the command."471				if reason != nil && *reason != "" {472					msg = "The agent refused the command: " + *reason473				}474				return finish(actions.Outcome{State: "failed", ErrorCode: "agent_refused", ErrorMessage: msg})475			}476			if rstatus == nil {477				return finish(actions.Outcome{State: "failed", ErrorCode: "protocol_error", ErrorMessage: "The command finished without a result status."})478			}479			out := actions.Outcome{State: *rstatus, Result: result}480			if s, ok := result["error"].(string); ok {481				out.ErrorMessage = s482			}483			switch *rstatus {484			case "failed":485				out.ErrorCode = "command_failed"486			case "timed_out":487				out.ErrorCode = "agent_timeout" // the agent stopped it at its time limit: outcome known488			}489			return finish(out)490		case "cancelled": // withdrawn before delivery491			return finish(actions.Outcome{State: "cancelled"})492		case "expired": // defensive: an expired command always ends its action in the same transaction493			return finish(actions.Outcome{State: "failed", ErrorCode: "agent_unreachable", ErrorMessage: "The command expired."})494		case "pending":495			if !now.Before(expires) {496				if _, err := tx.Exec(ctx, `update agent_commands set state='expired', finished_at=$2 where id=$1`, cmdID, now); err != nil {497					return err498				}499				o.count(func() { o.counts.expired++ })500				return finish(actions.Outcome{State: "failed", ErrorCode: "agent_unreachable",501					ErrorMessage: "The command was never delivered to the agent before it expired. It did not run."})502			}503		case "delivered", "acked":504			if cstate == "acked" && state == "dispatched" {505				at := now506				if acked != nil {507					at = *acked508				}509				if _, err := actions.Transition(ctx, tx, id, "dispatched", "running", at); err != nil {510					return err511				}512			}513			if deadline != nil && !now.Before(*deadline) {514				if _, err := tx.Exec(ctx, `update agent_commands set state='expired', finished_at=$2 where id=$1`, cmdID, now); err != nil {515					return err516				}517				o.count(func() { o.counts.expired++ })518				return finish(actions.Outcome{State: "timed_out", ErrorCode: "no_result", OutcomeUnknown: true,519					ErrorMessage: "The command reached the agent but no result came back before the deadline. It may or may not have run; it will not be sent again."})520			}521		}522		return nil523	})524}525526// lateResults records a result that arrived after its action was closed as527// "outcome unknown". The final state is not rewritten (history stays honest);528// the late result is added next to it.529func (o *Orchestrator) lateResults(ctx context.Context) error {530	now := o.opt.Now()531	return pgx.BeginFunc(ctx, o.pool, func(tx pgx.Tx) error {532		rows, err := tx.Query(ctx, `select a.id, c.result_status from actions a join agent_commands c on c.action_id=a.id533			where a.state='timed_out' and (a.result->>'outcome_unknown')::boolean and not (a.result ? 'late_result')534			  and c.result_status is not null and a.finished_at > $1 for update of a skip locked`, now.Add(-24*time.Hour))535		if err != nil {536			return err537		}538		type late struct {539			id     uuid.UUID540			status string541		}542		list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (late, error) {543			var l late544			return l, r.Scan(&l.id, &l.status)545		})546		if err != nil {547			return err548		}549		for _, l := range list {550			if _, err := tx.Exec(ctx, `update actions set result = result || jsonb_build_object('late_result', $2::text) where id=$1`, l.id, l.status); err != nil {551				return err552			}553			if err := actions.Event(ctx, tx, l.id, "note", "", "late result received after the deadline: "+l.status, now); err != nil {554				return err555			}556			if err := actions.Changed(ctx, tx, l.id, false); err != nil {557				return err558			}559		}560		return nil561	})562}563564// Leader runs fn only if this process obtains the advisory lock `key`, so a565// periodic task never runs twice at once across instances. The lock is tied566// to a connection: it disappears with the process.567func Leader(ctx context.Context, pool *pgxpool.Pool, key int64, fn func(context.Context) error) (ran bool, err error) {568	conn, err := pool.Acquire(ctx)569	if err != nil {570		return false, err571	}572	defer conn.Release()573	var got bool574	if err := conn.QueryRow(ctx, `select pg_try_advisory_lock($1)`, key).Scan(&got); err != nil || !got {575		return false, err576	}577	defer func() {578		c, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)579		defer cancel()580		if _, uerr := conn.Exec(c, `select pg_advisory_unlock($1)`, key); uerr != nil {581			conn.Conn().Close(c) // never return a connection that may still hold the lock582		}583	}()584	return true, fn(ctx)585}586587// Periodic runs fn every interval under Leader until ctx ends. Errors are588// logged and retried at the next tick: a database outage never stops the loop.589func Periodic(ctx context.Context, pool *pgxpool.Pool, name string, every time.Duration, key int64, fn func(context.Context) error) {590	t := time.NewTicker(every)591	defer t.Stop()592	for {593		if _, err := Leader(ctx, pool, key, fn); err != nil && ctx.Err() == nil {594			slog.Warn("periodic task failed", "task", name, "err", err)595		}596		select {597		case <-ctx.Done():598			return599		case <-t.C:600		}601	}602}603604// Advisory lock keys of the periodic tasks hosted by the orchestrator.605const (606	LockJobs        int64 = 0x6d6363_10607	LockAlerts      int64 = 0x6d6363_11608	LockMaintenance int64 = 0x6d6363_12609	LockNotify      int64 = 0x6d6363_13610	LockHousekeep   int64 = 0x6d6363_14611)612
Files main
Menu

Theme