Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/orchestrator/orchestrator.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package orchestrator4344import (45 "context"46 "encoding/json"47 "errors"48 "log/slog"49 "sync"50 "time"5152 "github.com/google/uuid"53 "github.com/jackc/pgx/v5"54 "github.com/jackc/pgx/v5/pgxpool"5556 "clustr.spboucher.ai/mcc/internal/actions"57 "clustr.spboucher.ai/mcc/internal/httpx"58 "clustr.spboucher.ai/mcc/internal/ids"59 "clustr.spboucher.ai/mcc/internal/protocol"60)6162// Options tunes the worker. Zero values are production defaults.63type Options struct {64 CommandTTL time.Duration // validity of a signed command (<= protocol.MaxCommandLifetime); default 90 s65 Grace time.Duration // added to expires_at + timeout before declaring the outcome unknown; default 30 s66 SweepEvery time.Duration // safety-net pass when no notification arrives; default 2 s67 OfflineAfter time.Duration // agent considered offline after this silence; default 120 s68 Now func() time.Time // clock (tests)69}7071// Orchestrator moves actions through their life cycle.72//73// Hard rules, enforced here and by the schema:74// - one action issues at most ONE command (same transaction as queued →75// dispatched; unique index agent_commands(action_id)); nothing ever76// re-issues it, neither a restart nor a second instance;77// - a command never delivered before expires_at ends the action `failed`78// with error_code agent_unreachable (it certainly did not run);79// - a command delivered without a result before the deadline ends the action80// `timed_out` with result.outcome_unknown=true: it may or may not have run;81// - while settings['orchestrator.frozen'] is true nothing is dispatched.82type Orchestrator struct {83 pool *pgxpool.Pool84 signer *Signer85 opt Options86 wake chan struct{}8788 mu sync.Mutex89 lastStep time.Time90 lastErr string91 frozen bool92 counts struct{ dispatched, finished, expired int64 }93}9495func New(pool *pgxpool.Pool, signer *Signer, opt Options) *Orchestrator {96 if opt.CommandTTL <= 0 || opt.CommandTTL > protocol.MaxCommandLifetime {97 opt.CommandTTL = 90 * time.Second98 }99 if opt.Grace <= 0 {100 opt.Grace = 30 * time.Second101 }102 if opt.SweepEvery <= 0 {103 opt.SweepEvery = 2 * time.Second104 }105 if opt.OfflineAfter <= 0 {106 opt.OfflineAfter = 120 * time.Second107 }108 if opt.Now == nil {109 opt.Now = func() time.Time { return time.Now().UTC() }110 }111 return &Orchestrator{pool: pool, signer: signer, opt: opt, wake: make(chan struct{}, 1)}112}113114// Status is published in the service heartbeat.115func (o *Orchestrator) Status() map[string]any {116 o.mu.Lock()117 defer o.mu.Unlock()118 s := map[string]any{"frozen": o.frozen, "key_id": o.signer.KeyID, "dispatched": o.counts.dispatched, "finished": o.counts.finished,119 "expired": o.counts.expired}120 if !o.lastStep.IsZero() {121 s["last_pass_at"] = o.lastStep122 }123 if o.lastErr != "" {124 s["last_error"] = o.lastErr125 }126 return s127}128129// Run works until ctx ends. A database outage is survived: passes fail, are130// retried with backoff, and the first successful pass reconciles everything131// from the tables.132func (o *Orchestrator) Run(ctx context.Context) {133 go o.listen(ctx)134 backoff := time.Duration(0)135 for ctx.Err() == nil {136 err := o.Step(ctx)137 wait := o.opt.SweepEvery138 o.mu.Lock()139 if err != nil && ctx.Err() == nil {140 if backoff == 0 {141 backoff = time.Second142 } else if backoff < 15*time.Second {143 backoff *= 2144 }145 wait, o.lastErr = backoff, err.Error()146 slog.Warn("orchestrator: pass failed", "err", err, "retry_in", backoff)147 } else {148 backoff, o.lastErr, o.lastStep = 0, "", time.Now().UTC()149 }150 o.mu.Unlock()151 t := time.NewTimer(wait)152 select {153 case <-ctx.Done():154 t.Stop()155 return156 case <-o.wake:157 if err != nil { // do not spin on notifications while the database is failing158 select {159 case <-ctx.Done():160 case <-t.C:161 }162 }163 case <-t.C:164 }165 t.Stop()166 }167}168169// listen wakes the loop on command updates and new actions. Notifications are170// only an accelerator: the periodic pass finds everything without them.171func (o *Orchestrator) listen(ctx context.Context) {172 backoff := time.Second173 for ctx.Err() == nil {174 err := func() error {175 conn, err := o.pool.Acquire(ctx)176 if err != nil {177 return err178 }179 defer conn.Release()180 defer func() {181 // A connection that still listens must not go back to the pool as is.182 c, cancel := context.WithTimeout(context.Background(), 2*time.Second)183 defer cancel()184 if _, err := conn.Exec(c, "unlisten *"); err != nil {185 conn.Conn().Close(c)186 }187 }()188 for _, ch := range []string{actions.ChanCommandUpdates, actions.ChanActions} {189 if _, err := conn.Exec(ctx, "listen "+ch); err != nil {190 return err191 }192 }193 backoff = time.Second194 for {195 if _, err := conn.Conn().WaitForNotification(ctx); err != nil {196 return err197 }198 o.Wake()199 }200 }()201 if ctx.Err() != nil {202 return203 }204 slog.Warn("orchestrator: listener disconnected", "err", err, "retry_in", backoff)205 select {206 case <-ctx.Done():207 return208 case <-time.After(backoff):209 }210 if backoff < 15*time.Second {211 backoff *= 2212 }213 }214}215216// Wake requests an immediate pass.217func (o *Orchestrator) Wake() {218 select {219 case o.wake <- struct{}{}:220 default:221 }222}223224// Step is one complete pass: expire approvals, dispatch what is queued,225// reconcile everything in flight. It is idempotent and safe to run from226// several instances at once.227func (o *Orchestrator) Step(ctx context.Context) error {228 if err := o.expireApprovals(ctx); err != nil {229 return err230 }231 for i := 0; i < 200; i++ {232 more, err := o.dispatchOne(ctx)233 if err != nil {234 return err235 }236 if !more {237 break238 }239 }240 if err := o.reconcile(ctx); err != nil {241 return err242 }243 return o.lateResults(ctx)244}245246func (o *Orchestrator) expireApprovals(ctx context.Context) error {247 now := o.opt.Now()248 return pgx.BeginFunc(ctx, o.pool, func(tx pgx.Tx) error {249 rows, err := tx.Query(ctx, `select id from actions where state='pending_approval' and created_at <= $1 for update skip locked`, now.Add(-actions.ApprovalTTL))250 if err != nil {251 return err252 }253 idsToExpire, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])254 if err != nil {255 return err256 }257 for _, id := range idsToExpire {258 if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "expired", ErrorCode: "approval_expired",259 ErrorMessage: "No approval was given within 30 minutes.", At: now}); err != nil {260 return err261 }262 }263 return nil264 })265}266267// dispatchOne claims one queued action and either executes it (server-side268// types) or signs and records its command. Claim, command and state change269// commit together or not at all.270func (o *Orchestrator) dispatchOne(ctx context.Context) (more bool, err error) {271 now := o.opt.Now()272 tx, err := o.pool.Begin(ctx)273 if err != nil {274 return false, err275 }276 defer tx.Rollback(ctx)277278 frozen, err := actions.Frozen(ctx, tx)279 if err != nil {280 return false, err281 }282 o.mu.Lock()283 o.frozen = frozen284 o.mu.Unlock()285 if frozen {286 return false, nil // queued actions wait; their state says so287 }288 var (289 id uuid.UUID290 typ, tk string291 targetID string292 params map[string]any293 timeout int294 override bool295 )296 err = tx.QueryRow(ctx, `select id, type, target_kind, target_id, params, timeout_s, override_maintenance from actions297 where state='queued' order by queued_at nulls first, id limit 1 for update skip locked`).298 Scan(&id, &typ, &tk, &targetID, ¶ms, &timeout, &override)299 if errors.Is(err, pgx.ErrNoRows) {300 return false, nil301 }302 if err != nil {303 return false, err304 }305 fail := func(code, msg string) (bool, error) {306 if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "failed", ErrorCode: code, ErrorMessage: msg, At: now}); err != nil {307 return false, err308 }309 return true, tx.Commit(ctx)310 }311 refused := func(err error) (bool, error) {312 var he *httpx.Error313 if errors.As(err, &he) && he.Status < 500 {314 return fail(he.Code, he.Message)315 }316 return false, err317 }318 t, ok := actions.Lookup(typ)319 if !ok {320 return fail("unknown_type", "This action type is not in the catalog.")321 }322 tid, perr := uuid.Parse(targetID)323 if perr != nil {324 return fail("precheck_failed", "Invalid target.")325 }326 tg, err := actions.ResolveTarget(ctx, tx, tk, tid)327 if err != nil {328 return refused(err)329 }330 // Pre-checks are verified again now: the world may have changed since the request.331 agentID, err := actions.Precheck(ctx, tx, t, tg, params, actions.PrecheckOpts{OverrideMaintenance: override, Now: now, OfflineAfter: o.opt.OfflineAfter})332 if err != nil {333 return refused(err)334 }335336 if t.ServerSide {337 if _, err := actions.Transition(ctx, tx, id, "queued", "running", now); err != nil {338 return false, err339 }340 res, err := actions.ExecuteServerSide(ctx, tx, typ, tg, params, now)341 if err != nil {342 var he *httpx.Error343 if errors.As(err, &he) && he.Status < 500 {344 if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "failed", ErrorCode: he.Code, ErrorMessage: he.Message, At: now}); err != nil {345 return false, err346 }347 return true, tx.Commit(ctx)348 }349 return false, err350 }351 if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "succeeded", Result: res, At: now}); err != nil {352 return false, err353 }354 o.count(func() { o.counts.finished++ })355 return true, tx.Commit(ctx)356 }357358 // Parameters as the agent's executors decode them (strictly: unknown fields359 // are refused). A service target is addressed by manager and name; node360 // targets need nothing more than the validated user parameters.361 wire := map[string]any{}362 for k, v := range params {363 wire[k] = v364 }365 if tg.Wire["kind"] == "service" {366 wire["manager"], wire["name"] = tg.Wire["manager"], tg.Wire["name"]367 }368 raw, err := json.Marshal(wire)369 if err != nil {370 return false, err371 }372 cmd := protocol.Command{V: protocol.Version, ID: ids.New().String(), ActionID: id.String(), AgentID: agentID.String(), NodeID: tg.NodeID.String(),373 Type: typ, Params: raw, TimeoutS: timeout, IssuedAt: now, ExpiresAt: now.Add(o.opt.CommandTTL), Nonce: protocol.NewNonce()}374 signed, err := protocol.Sign(o.signer.Private(), o.signer.KeyID, cmd)375 if err != nil {376 return false, err377 }378 deadline := cmd.ExpiresAt.Add(time.Duration(timeout)*time.Second + o.opt.Grace)379 // The unique index on agent_commands(action_id) makes a second command for380 // this action impossible, whatever the code does.381 if _, err := tx.Exec(ctx, `insert into agent_commands(id, action_id, agent_id, envelope, signature, key_id, issued_at, expires_at, state)382 values ($1,$2,$3,$4,$5,$6,$7,$8,'pending')`, cmd.ID, id, agentID, signed.Payload, signed.Signature, signed.KeyID, cmd.IssuedAt, cmd.ExpiresAt); err != nil {383 return false, err384 }385 if _, err := tx.Exec(ctx, `update actions set deadline_at=$2 where id=$1`, id, deadline); err != nil {386 return false, err387 }388 if _, err := actions.Transition(ctx, tx, id, "queued", "dispatched", now); err != nil {389 return false, err390 }391 if _, err := tx.Exec(ctx, `select pg_notify($1,$2)`, actions.ChanCommands, agentID.String()); err != nil {392 return false, err393 }394 if err := tx.Commit(ctx); err != nil {395 return false, err396 }397 o.count(func() { o.counts.dispatched++ })398 return true, nil399}400401func (o *Orchestrator) count(f func()) {402 o.mu.Lock()403 f()404 o.mu.Unlock()405}406407// reconcile brings every in-flight action in line with its command row.408func (o *Orchestrator) reconcile(ctx context.Context) error {409 rows, err := o.pool.Query(ctx, `select id from actions where state in ('dispatched','running') order by created_at`)410 if err != nil {411 return err412 }413 list, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])414 if err != nil {415 return err416 }417 for _, id := range list {418 if err := o.reconcileOne(ctx, id); err != nil {419 return err420 }421 }422 return nil423}424425func (o *Orchestrator) reconcileOne(ctx context.Context, id uuid.UUID) error {426 now := o.opt.Now()427 return pgx.BeginFunc(ctx, o.pool, func(tx pgx.Tx) error {428 var state string429 var deadline *time.Time430 err := tx.QueryRow(ctx, `select state::text, deadline_at from actions where id=$1 and state in ('dispatched','running') for update skip locked`, id).Scan(&state, &deadline)431 if errors.Is(err, pgx.ErrNoRows) {432 return nil // finished meanwhile, or another instance is on it433 }434 if err != nil {435 return err436 }437 var (438 cmdID uuid.UUID439 cstate string440 expires time.Time441 delivered *time.Time442 acked *time.Time443 accepted *bool444 reason *string445 rstatus *string446 result map[string]any447 )448 err = tx.QueryRow(ctx, `select id, state, expires_at, delivered_at, acked_at, ack_accepted, ack_reason, result_status, result449 from agent_commands where action_id=$1 for update`, id).Scan(&cmdID, &cstate, &expires, &delivered, &acked, &accepted, &reason, &rstatus, &result)450 finish := func(out actions.Outcome) error {451 out.At = now452 _, err := actions.Finish(ctx, tx, id, out)453 o.count(func() { o.counts.finished++ })454 return err455 }456 if errors.Is(err, pgx.ErrNoRows) {457 return finish(actions.Outcome{State: "failed", ErrorCode: "internal", ErrorMessage: "No command is recorded for this action; nothing was sent."})458 }459 if err != nil {460 return err461 }462 if acked != nil { // record when execution really started, even if the result is already there463 if _, err := tx.Exec(ctx, `update actions set started_at=coalesce(started_at,$2) where id=$1`, id, *acked); err != nil {464 return err465 }466 }467 switch cstate {468 case "finished":469 if accepted != nil && !*accepted {470 msg := "The agent refused the command."471 if reason != nil && *reason != "" {472 msg = "The agent refused the command: " + *reason473 }474 return finish(actions.Outcome{State: "failed", ErrorCode: "agent_refused", ErrorMessage: msg})475 }476 if rstatus == nil {477 return finish(actions.Outcome{State: "failed", ErrorCode: "protocol_error", ErrorMessage: "The command finished without a result status."})478 }479 out := actions.Outcome{State: *rstatus, Result: result}480 if s, ok := result["error"].(string); ok {481 out.ErrorMessage = s482 }483 switch *rstatus {484 case "failed":485 out.ErrorCode = "command_failed"486 case "timed_out":487 out.ErrorCode = "agent_timeout" // the agent stopped it at its time limit: outcome known488 }489 return finish(out)490 case "cancelled": // withdrawn before delivery491 return finish(actions.Outcome{State: "cancelled"})492 case "expired": // defensive: an expired command always ends its action in the same transaction493 return finish(actions.Outcome{State: "failed", ErrorCode: "agent_unreachable", ErrorMessage: "The command expired."})494 case "pending":495 if !now.Before(expires) {496 if _, err := tx.Exec(ctx, `update agent_commands set state='expired', finished_at=$2 where id=$1`, cmdID, now); err != nil {497 return err498 }499 o.count(func() { o.counts.expired++ })500 return finish(actions.Outcome{State: "failed", ErrorCode: "agent_unreachable",501 ErrorMessage: "The command was never delivered to the agent before it expired. It did not run."})502 }503 case "delivered", "acked":504 if cstate == "acked" && state == "dispatched" {505 at := now506 if acked != nil {507 at = *acked508 }509 if _, err := actions.Transition(ctx, tx, id, "dispatched", "running", at); err != nil {510 return err511 }512 }513 if deadline != nil && !now.Before(*deadline) {514 if _, err := tx.Exec(ctx, `update agent_commands set state='expired', finished_at=$2 where id=$1`, cmdID, now); err != nil {515 return err516 }517 o.count(func() { o.counts.expired++ })518 return finish(actions.Outcome{State: "timed_out", ErrorCode: "no_result", OutcomeUnknown: true,519 ErrorMessage: "The command reached the agent but no result came back before the deadline. It may or may not have run; it will not be sent again."})520 }521 }522 return nil523 })524}525526// lateResults records a result that arrived after its action was closed as527// "outcome unknown". The final state is not rewritten (history stays honest);528// the late result is added next to it.529func (o *Orchestrator) lateResults(ctx context.Context) error {530 now := o.opt.Now()531 return pgx.BeginFunc(ctx, o.pool, func(tx pgx.Tx) error {532 rows, err := tx.Query(ctx, `select a.id, c.result_status from actions a join agent_commands c on c.action_id=a.id533 where a.state='timed_out' and (a.result->>'outcome_unknown')::boolean and not (a.result ? 'late_result')534 and c.result_status is not null and a.finished_at > $1 for update of a skip locked`, now.Add(-24*time.Hour))535 if err != nil {536 return err537 }538 type late struct {539 id uuid.UUID540 status string541 }542 list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (late, error) {543 var l late544 return l, r.Scan(&l.id, &l.status)545 })546 if err != nil {547 return err548 }549 for _, l := range list {550 if _, err := tx.Exec(ctx, `update actions set result = result || jsonb_build_object('late_result', $2::text) where id=$1`, l.id, l.status); err != nil {551 return err552 }553 if err := actions.Event(ctx, tx, l.id, "note", "", "late result received after the deadline: "+l.status, now); err != nil {554 return err555 }556 if err := actions.Changed(ctx, tx, l.id, false); err != nil {557 return err558 }559 }560 return nil561 })562}563564// Leader runs fn only if this process obtains the advisory lock `key`, so a565// periodic task never runs twice at once across instances. The lock is tied566// to a connection: it disappears with the process.567func Leader(ctx context.Context, pool *pgxpool.Pool, key int64, fn func(context.Context) error) (ran bool, err error) {568 conn, err := pool.Acquire(ctx)569 if err != nil {570 return false, err571 }572 defer conn.Release()573 var got bool574 if err := conn.QueryRow(ctx, `select pg_try_advisory_lock($1)`, key).Scan(&got); err != nil || !got {575 return false, err576 }577 defer func() {578 c, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)579 defer cancel()580 if _, uerr := conn.Exec(c, `select pg_advisory_unlock($1)`, key); uerr != nil {581 conn.Conn().Close(c) // never return a connection that may still hold the lock582 }583 }()584 return true, fn(ctx)585}586587// Periodic runs fn every interval under Leader until ctx ends. Errors are588// logged and retried at the next tick: a database outage never stops the loop.589func Periodic(ctx context.Context, pool *pgxpool.Pool, name string, every time.Duration, key int64, fn func(context.Context) error) {590 t := time.NewTicker(every)591 defer t.Stop()592 for {593 if _, err := Leader(ctx, pool, key, fn); err != nil && ctx.Err() == nil {594 slog.Warn("periodic task failed", "task", name, "err", err)595 }596 select {597 case <-ctx.Done():598 return599 case <-t.C:600 }601 }602}603604// Advisory lock keys of the periodic tasks hosted by the orchestrator.605const (606 LockJobs int64 = 0x6d6363_10607 LockAlerts int64 = 0x6d6363_11608 LockMaintenance int64 = 0x6d6363_12609 LockNotify int64 = 0x6d6363_13610 LockHousekeep int64 = 0x6d6363_14611)612