Repository Public
1// action-orchestrator is the durable worker of MacLustr Command Center: it2// dispatches actions as signed agent commands and follows them to their end,3// and hosts the periodic engines (jobs, alerts, notifications, maintenance4// calendar, housekeeping). It keeps no state of its own: after a crash or a5// restart everything is rebuilt from PostgreSQL, and no command is ever6// issued twice. It listens only on a local health port.7package main89import (10 "context"11 "encoding/json"12 "log/slog"13 "net"14 "net/http"15 "os"16 "sync"17 "time"1819 "github.com/jackc/pgx/v5/pgxpool"2021 "clustr.spboucher.ai/mcc/internal/alerts"22 "clustr.spboucher.ai/mcc/internal/config"23 "clustr.spboucher.ai/mcc/internal/db"24 "clustr.spboucher.ai/mcc/internal/jobs"25 "clustr.spboucher.ai/mcc/internal/maintenance"26 "clustr.spboucher.ai/mcc/internal/notify"27 "clustr.spboucher.ai/mcc/internal/orchestrator"28 "clustr.spboucher.ai/mcc/internal/svc"29)3031// Version is set by the linker.32var Version = "dev"3334func main() {35 if err := run(); err != nil {36 slog.Error("fatal", "err", err)37 os.Exit(1)38 }39}4041// connect waits for the database instead of exiting: under a process manager42// an exit would become a crash loop for the whole outage.43func connect(ctx context.Context, url string) (*pgxpool.Pool, error) {44 backoff := time.Second45 for {46 pool, err := db.Open(ctx, url)47 if err == nil {48 if _, err = db.Migrate(ctx, pool); err == nil {49 return pool, nil50 }51 pool.Close()52 }53 if ctx.Err() != nil {54 return nil, ctx.Err()55 }56 slog.Warn("database not ready", "err", err, "retry_in", backoff)57 select {58 case <-ctx.Done():59 return nil, ctx.Err()60 case <-time.After(backoff):61 }62 if backoff < 30*time.Second {63 backoff *= 264 }65 }66}6768// loops records the last run of each periodic task for the status report.69type loops struct {70 mu sync.Mutex71 last map[string]map[string]any72}7374func (l *loops) wrap(name string, fn func(context.Context) error) func(context.Context) error {75 return func(ctx context.Context) error {76 start := time.Now()77 err := fn(ctx)78 st := map[string]any{"last_run_at": start.UTC(), "ms": time.Since(start).Milliseconds()}79 if err != nil {80 st["last_error"] = err.Error()81 }82 l.mu.Lock()83 l.last[name] = st84 l.mu.Unlock()85 return err86 }87}8889func (l *loops) snapshot() map[string]any {90 l.mu.Lock()91 defer l.mu.Unlock()92 out := map[string]any{}93 for k, v := range l.last {94 out[k] = v95 }96 return out97}9899func run() error {100 ctx, stop := svc.Init("action-orchestrator")101 defer stop()102 cfg, err := config.Load()103 if err != nil {104 return err105 }106 pool, err := connect(ctx, cfg.DatabaseURL)107 if err != nil {108 return err109 }110 defer pool.Close()111112 signer, err := orchestrator.LoadSigner(ctx, pool, cfg.SecretPath("command-signing.key"))113 if err != nil {114 return err115 }116 if err := alerts.SeedBuiltins(ctx, pool); err != nil {117 return err118 }119 orch := orchestrator.New(pool, signer, orchestrator.Options{OfflineAfter: cfg.OfflineAfter})120 apns := notify.NewAPNs(notify.APNsConfig{KeyFile: cfg.APNsKeyFile, KeyID: cfg.APNsKeyID, TeamID: cfg.APNsTeamID, TopicIOS: cfg.APNsTopicIOS, TopicMacOS: cfg.APNsTopicMacOS})121 if st := apns.Status(); st.State == "inactive" {122 slog.Warn("push notifications inactive", "reason", st.Reason)123 }124 runner := &jobs.Runner{Pool: pool, FreshAfter: cfg.FreshAfter, OfflineAfter: cfg.OfflineAfter}125 engine := &alerts.Engine{Pool: pool, FreshAfter: cfg.FreshAfter}126 dispatcher := ¬ify.Dispatcher{Pool: pool, APNs: apns}127 sched := &maintenance.Scheduler{Pool: pool}128 lp := &loops{last: map[string]map[string]any{}}129130 status := func() map[string]any {131 return map[string]any{"orchestrator": orch.Status(), "apns": apns.Status(), "tasks": lp.snapshot()}132 }133 var wg sync.WaitGroup134 start := func(f func()) {135 wg.Add(1)136 go func() { defer wg.Done(); f() }()137 }138 start(func() { orch.Run(ctx) })139 start(func() { svc.Heartbeat(ctx, pool, "action-orchestrator", Version, status) })140 periodic := func(name string, every time.Duration, key int64, fn func(context.Context) error) {141 start(func() { orchestrator.Periodic(ctx, pool, name, every, key, lp.wrap(name, fn)) })142 }143 periodic("jobs", 2*time.Second, orchestrator.LockJobs, runner.Tick)144 periodic("alerts", 15*time.Second, orchestrator.LockAlerts, engine.Evaluate)145 periodic("notifications", 5*time.Second, orchestrator.LockNotify, func(ctx context.Context) error { _, err := dispatcher.Drain(ctx); return err })146 periodic("maintenance", 10*time.Second, orchestrator.LockMaintenance, sched.Tick)147 periodic("housekeeping", time.Hour, orchestrator.LockHousekeep, func(ctx context.Context) error {148 ev, nt, err := orchestrator.Housekeep(ctx, pool, time.Now().UTC())149 if ev+nt > 0 {150 slog.Info("housekeeping", "action_events", ev, "notifications", nt)151 }152 return err153 })154155 mux := http.NewServeMux()156 mux.HandleFunc("GET /health/live", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok\n")) })157 mux.HandleFunc("GET /health/ready", func(w http.ResponseWriter, r *http.Request) {158 c, cancel := context.WithTimeout(r.Context(), 2*time.Second)159 defer cancel()160 w.Header().Set("Content-Type", "application/json")161 if err := pool.Ping(c); err != nil {162 w.WriteHeader(http.StatusServiceUnavailable)163 json.NewEncoder(w).Encode(map[string]any{"status": "unavailable", "database": "down"})164 return165 }166 json.NewEncoder(w).Encode(map[string]any{"status": "ok", "version": Version, "detail": status()})167 })168 ln, err := net.Listen("tcp", cfg.OrchestratorListen)169 if err != nil {170 return err171 }172 slog.Info("listening (health only)", "addr", cfg.OrchestratorListen, "env", cfg.Env, "version", Version, "command_key", signer.KeyID)173 err = svc.ServeHTTP(ctx, &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second}, ln)174 stop()175 // Graceful stop: every loop finishes its current transaction and returns.176 done := make(chan struct{})177 go func() { wg.Wait(); close(done) }()178 select {179 case <-done:180 case <-time.After(15 * time.Second):181 slog.Warn("shutdown: some tasks did not stop in time")182 }183 return err184}185