SPB Git

Repository Public

main.go 6.0 KB 185 lines go

1// action-orchestrator is the durable worker of MacLustr Command Center: it2// dispatches actions as signed agent commands and follows them to their end,3// and hosts the periodic engines (jobs, alerts, notifications, maintenance4// calendar, housekeeping). It keeps no state of its own: after a crash or a5// restart everything is rebuilt from PostgreSQL, and no command is ever6// issued twice. It listens only on a local health port.7package main89import (10	"context"11	"encoding/json"12	"log/slog"13	"net"14	"net/http"15	"os"16	"sync"17	"time"1819	"github.com/jackc/pgx/v5/pgxpool"2021	"clustr.spboucher.ai/mcc/internal/alerts"22	"clustr.spboucher.ai/mcc/internal/config"23	"clustr.spboucher.ai/mcc/internal/db"24	"clustr.spboucher.ai/mcc/internal/jobs"25	"clustr.spboucher.ai/mcc/internal/maintenance"26	"clustr.spboucher.ai/mcc/internal/notify"27	"clustr.spboucher.ai/mcc/internal/orchestrator"28	"clustr.spboucher.ai/mcc/internal/svc"29)3031// Version is set by the linker.32var Version = "dev"3334func main() {35	if err := run(); err != nil {36		slog.Error("fatal", "err", err)37		os.Exit(1)38	}39}4041// connect waits for the database instead of exiting: under a process manager42// an exit would become a crash loop for the whole outage.43func connect(ctx context.Context, url string) (*pgxpool.Pool, error) {44	backoff := time.Second45	for {46		pool, err := db.Open(ctx, url)47		if err == nil {48			if _, err = db.Migrate(ctx, pool); err == nil {49				return pool, nil50			}51			pool.Close()52		}53		if ctx.Err() != nil {54			return nil, ctx.Err()55		}56		slog.Warn("database not ready", "err", err, "retry_in", backoff)57		select {58		case <-ctx.Done():59			return nil, ctx.Err()60		case <-time.After(backoff):61		}62		if backoff < 30*time.Second {63			backoff *= 264		}65	}66}6768// loops records the last run of each periodic task for the status report.69type loops struct {70	mu   sync.Mutex71	last map[string]map[string]any72}7374func (l *loops) wrap(name string, fn func(context.Context) error) func(context.Context) error {75	return func(ctx context.Context) error {76		start := time.Now()77		err := fn(ctx)78		st := map[string]any{"last_run_at": start.UTC(), "ms": time.Since(start).Milliseconds()}79		if err != nil {80			st["last_error"] = err.Error()81		}82		l.mu.Lock()83		l.last[name] = st84		l.mu.Unlock()85		return err86	}87}8889func (l *loops) snapshot() map[string]any {90	l.mu.Lock()91	defer l.mu.Unlock()92	out := map[string]any{}93	for k, v := range l.last {94		out[k] = v95	}96	return out97}9899func run() error {100	ctx, stop := svc.Init("action-orchestrator")101	defer stop()102	cfg, err := config.Load()103	if err != nil {104		return err105	}106	pool, err := connect(ctx, cfg.DatabaseURL)107	if err != nil {108		return err109	}110	defer pool.Close()111112	signer, err := orchestrator.LoadSigner(ctx, pool, cfg.SecretPath("command-signing.key"))113	if err != nil {114		return err115	}116	if err := alerts.SeedBuiltins(ctx, pool); err != nil {117		return err118	}119	orch := orchestrator.New(pool, signer, orchestrator.Options{OfflineAfter: cfg.OfflineAfter})120	apns := notify.NewAPNs(notify.APNsConfig{KeyFile: cfg.APNsKeyFile, KeyID: cfg.APNsKeyID, TeamID: cfg.APNsTeamID, TopicIOS: cfg.APNsTopicIOS, TopicMacOS: cfg.APNsTopicMacOS})121	if st := apns.Status(); st.State == "inactive" {122		slog.Warn("push notifications inactive", "reason", st.Reason)123	}124	runner := &jobs.Runner{Pool: pool, FreshAfter: cfg.FreshAfter, OfflineAfter: cfg.OfflineAfter}125	engine := &alerts.Engine{Pool: pool, FreshAfter: cfg.FreshAfter}126	dispatcher := &notify.Dispatcher{Pool: pool, APNs: apns}127	sched := &maintenance.Scheduler{Pool: pool}128	lp := &loops{last: map[string]map[string]any{}}129130	status := func() map[string]any {131		return map[string]any{"orchestrator": orch.Status(), "apns": apns.Status(), "tasks": lp.snapshot()}132	}133	var wg sync.WaitGroup134	start := func(f func()) {135		wg.Add(1)136		go func() { defer wg.Done(); f() }()137	}138	start(func() { orch.Run(ctx) })139	start(func() { svc.Heartbeat(ctx, pool, "action-orchestrator", Version, status) })140	periodic := func(name string, every time.Duration, key int64, fn func(context.Context) error) {141		start(func() { orchestrator.Periodic(ctx, pool, name, every, key, lp.wrap(name, fn)) })142	}143	periodic("jobs", 2*time.Second, orchestrator.LockJobs, runner.Tick)144	periodic("alerts", 15*time.Second, orchestrator.LockAlerts, engine.Evaluate)145	periodic("notifications", 5*time.Second, orchestrator.LockNotify, func(ctx context.Context) error { _, err := dispatcher.Drain(ctx); return err })146	periodic("maintenance", 10*time.Second, orchestrator.LockMaintenance, sched.Tick)147	periodic("housekeeping", time.Hour, orchestrator.LockHousekeep, func(ctx context.Context) error {148		ev, nt, err := orchestrator.Housekeep(ctx, pool, time.Now().UTC())149		if ev+nt > 0 {150			slog.Info("housekeeping", "action_events", ev, "notifications", nt)151		}152		return err153	})154155	mux := http.NewServeMux()156	mux.HandleFunc("GET /health/live", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok\n")) })157	mux.HandleFunc("GET /health/ready", func(w http.ResponseWriter, r *http.Request) {158		c, cancel := context.WithTimeout(r.Context(), 2*time.Second)159		defer cancel()160		w.Header().Set("Content-Type", "application/json")161		if err := pool.Ping(c); err != nil {162			w.WriteHeader(http.StatusServiceUnavailable)163			json.NewEncoder(w).Encode(map[string]any{"status": "unavailable", "database": "down"})164			return165		}166		json.NewEncoder(w).Encode(map[string]any{"status": "ok", "version": Version, "detail": status()})167	})168	ln, err := net.Listen("tcp", cfg.OrchestratorListen)169	if err != nil {170		return err171	}172	slog.Info("listening (health only)", "addr", cfg.OrchestratorListen, "env", cfg.Env, "version", Version, "command_key", signer.KeyID)173	err = svc.ServeHTTP(ctx, &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second}, ln)174	stop()175	// Graceful stop: every loop finishes its current transaction and returns.176	done := make(chan struct{})177	go func() { wg.Wait(); close(done) }()178	select {179	case <-done:180	case <-time.After(15 * time.Second):181		slog.Warn("shutdown: some tasks did not stop in time")182	}183	return err184}185
Files main
Menu

Theme