Repository Public
1package api23import (4 "bytes"5 "context"6 "encoding/json"7 "errors"8 "math"9 "strconv"10 "time"1112 "github.com/google/uuid"13 "github.com/jackc/pgx/v5"1415 "clustr.spboucher.ai/mcc/internal/audit"16 "clustr.spboucher.ai/mcc/internal/authz"17 "clustr.spboucher.ai/mcc/internal/httpx"18)1920// ── Settings ───────────────────────────────────────────────────────────────21//22// The settings table also holds internal state of other services (the ingest23// CA under `pki.ca`, the event stream watermark…). Only the keys declared24// below are readable and writable through the API, each with its own25// validation. Settings never hold secrets: a credential lives in a file of26// the secrets directory and nowhere else.2728const admSettingFrozen = "orchestrator.frozen"2930type admSetting struct {31 key string32 description string33 def func(c *core) any34 // check validates a decoded JSON value and returns the canonical value to store.35 check func(v any) (any, error)36 ownerOnly bool37}3839var errAdmSettingValue = errors.New("invalid value")4041// admInt accepts a JSON integer within [min, max].42func admInt(min, max int64) func(any) (any, error) {43 return func(v any) (any, error) {44 n, ok := v.(json.Number)45 if !ok {46 return nil, errAdmSettingValue47 }48 i, err := strconv.ParseInt(n.String(), 10, 64)49 if err != nil || i < min || i > max {50 return nil, errAdmSettingValue51 }52 return i, nil53 }54}5556func admBool(v any) (any, error) {57 b, ok := v.(bool)58 if !ok {59 return nil, errAdmSettingValue60 }61 return b, nil62}6364func admEnum(values ...string) func(any) (any, error) {65 return func(v any) (any, error) {66 s, ok := v.(string)67 if !ok {68 return nil, errAdmSettingValue69 }70 for _, x := range values {71 if s == x {72 return s, nil73 }74 }75 return nil, errAdmSettingValue76 }77}7879func admDays(d time.Duration) int64 { return int64(math.Round(d.Hours() / 24)) }8081// admSettings is the allowlist, in display order.82var admSettings = []admSetting{83 {key: "retention.raw_days", description: "Days of raw (10 s) samples kept. Integer, 1 to 30.",84 def: func(c *core) any { return admDays(c.Cfg.RetainRaw) }, check: admInt(1, 30)},85 {key: "retention.1m_days", description: "Days of 1-minute aggregates kept. Integer, 7 to 180.",86 def: func(c *core) any { return admDays(c.Cfg.Retain1m) }, check: admInt(7, 180)},87 {key: "retention.1h_days", description: "Days of 1-hour aggregates kept. Integer, 30 to 1825.",88 def: func(c *core) any { return admDays(c.Cfg.Retain1h) }, check: admInt(30, 1825)},89 {key: "alerts.default_for_seconds", description: "Default time a condition must hold before a new alert rule fires. Integer, 0 to 86400.",90 def: func(*core) any { return int64(300) }, check: admInt(0, 86400)},91 {key: "alerts.default_clear_seconds", description: "Default time a condition must be clear before an alert resolves. Integer, 0 to 86400.",92 def: func(*core) any { return int64(120) }, check: admInt(0, 86400)},93 {key: "alerts.default_severity", description: "Default severity of a new alert rule: info, warning or critical.",94 def: func(*core) any { return "warning" }, check: admEnum("info", "warning", "critical")},95 {key: admSettingFrozen, description: "Emergency stop: when true the orchestrator issues no command to any agent. Boolean; owner only.",96 def: func(*core) any { return false }, check: admBool, ownerOnly: true},97}9899func admSettingByKey(key string) (admSetting, bool) {100 for _, s := range admSettings {101 if s.key == key {102 return s, true103 }104 }105 return admSetting{}, false106}107108type admStored struct {109 value json.RawMessage110 updatedAt time.Time111 updatedBy *uuid.UUID112}113114// toSetting renders a setting. A key that was never written reports its115// default with the Unix epoch as update time and says so in the description.116func (h *adminHandlers) toSetting(s admSetting, st *admStored) Setting {117 out := Setting{Key: s.key, Description: ptr(s.description)}118 if st == nil {119 out.Value, out.UpdatedAt = s.def(h.core), time.Unix(0, 0).UTC()120 out.Description = ptr(s.description + " Never changed: this is the default.")121 return out122 }123 var v any124 if json.Unmarshal(st.value, &v) != nil {125 v = s.def(h.core)126 }127 out.Value, out.UpdatedAt, out.UpdatedBy = v, st.updatedAt, st.updatedBy128 return out129}130131func (h *adminHandlers) ListSettings(ctx context.Context, _ ListSettingsRequestObject) (ListSettingsResponseObject, error) {132 keys := make([]string, len(admSettings))133 for i, s := range admSettings {134 keys[i] = s.key135 }136 rows, err := h.Pool.Query(ctx, `select key, value, updated_at, updated_by from settings where key = any($1)`, keys)137 if err != nil {138 return nil, err139 }140 stored := map[string]*admStored{}141 var key string142 var st admStored143 if _, err := pgx.ForEachRow(rows, []any{&key, &st.value, &st.updatedAt, &st.updatedBy}, func() error {144 c := st145 c.value = append(json.RawMessage(nil), st.value...)146 stored[key] = &c147 return nil148 }); err != nil {149 return nil, err150 }151 out := SettingList{Items: make([]Setting, 0, len(admSettings))}152 for _, s := range admSettings {153 out.Items = append(out.Items, h.toSetting(s, stored[s.key]))154 }155 return ListSettings200JSONResponse{Body: out}, nil156}157158func (h *adminHandlers) PutSetting(ctx context.Context, req PutSettingRequestObject) (PutSettingResponseObject, error) {159 s, ok := admSettingByKey(req.Key)160 if !ok {161 return nil, httpx.ErrNotFound // unknown keys, and internal ones, do not exist for the API162 }163 if req.Body == nil {164 return nil, httpx.Validation("Body required.")165 }166 // Re-decode the value with exact numbers (the generated type decodes into float64).167 raw, err := json.Marshal(req.Body.Value)168 if err != nil {169 return nil, clInvalid("value", "is not valid JSON")170 }171 var decoded any172 dec := json.NewDecoder(bytes.NewReader(raw))173 dec.UseNumber()174 if err := dec.Decode(&decoded); err != nil {175 return nil, clInvalid("value", "is not valid JSON")176 }177 value, err := s.check(decoded)178 if err != nil {179 return nil, clInvalid("value", "is not valid for "+s.key+": "+s.description)180 }181 actor := principalFrom(ctx).Session182 if s.ownerOnly && actor.User.Role != authz.Owner {183 h.audit(ctx, h.Pool, "setting.update", "setting", s.key, audit.Denied, "owner only", nil)184 return nil, httpx.ErrForbidden.With("reason", "owner_only")185 }186 tx, err := h.Pool.Begin(ctx)187 if err != nil {188 return nil, err189 }190 defer tx.Rollback(ctx)191 var before json.RawMessage192 err = tx.QueryRow(ctx, `select value from settings where key = $1 for update`, s.key).Scan(&before)193 if err != nil && !errors.Is(err, pgx.ErrNoRows) {194 return nil, err195 }196 stored, err := json.Marshal(value) // a Go string would otherwise be sent as raw JSON text197 if err != nil {198 return nil, err199 }200 st := admStored{updatedBy: &actor.User.ID}201 if err := tx.QueryRow(ctx, `insert into settings(key, value, updated_at, updated_by) values ($1, $2, now(), $3)202 on conflict (key) do update set value = excluded.value, updated_at = excluded.updated_at, updated_by = excluded.updated_by203 returning value, updated_at`, s.key, stored, actor.User.ID).Scan(&st.value, &st.updatedAt); err != nil {204 return nil, err205 }206 params := map[string]any{"to": value}207 if before != nil {208 var old any209 if json.Unmarshal(before, &old) == nil {210 params["from"] = old211 }212 }213 if err := h.audit(ctx, tx, "setting.update", "setting", s.key, audit.Success, "", params); err != nil {214 return nil, err215 }216 if err := tx.Commit(ctx); err != nil {217 return nil, err218 }219 return PutSetting200JSONResponse{Body: h.toSetting(s, &st)}, nil220}221222// frozen reports the emergency stop. Only the JSON value `true` engages it.223func (c *core) admFrozen(ctx context.Context) (bool, error) {224 var raw []byte225 err := c.Pool.QueryRow(ctx, `select value from settings where key = $1`, admSettingFrozen).Scan(&raw)226 if errors.Is(err, pgx.ErrNoRows) {227 return false, nil228 }229 return string(raw) == "true", err230}231232// ── Backend status ─────────────────────────────────────────────────────────233234const (235 admBeatOK = 30 * time.Second // heartbeats are written every 10 s236 admBeatDown = 120 * time.Second // beyond this the component is considered down237)238239// admComponent turns a heartbeat into a status. No heartbeat at all is240// "unknown" (the component may never have been started), not "down".241func admComponent(beatAt *time.Time, version string, now time.Time) ComponentStatus {242 if beatAt == nil {243 return ComponentStatus{Status: ComponentStatusStatusUnknown, Detail: ptr("No heartbeat was ever recorded for this component.")}244 }245 age := now.Sub(*beatAt)246 out := ComponentStatus{HeartbeatAt: beatAt}247 detail := "version " + version + ", last heartbeat " + strconv.Itoa(int(age.Seconds())) + " s ago"248 switch {249 case age <= admBeatOK:250 out.Status = ComponentStatusStatusOk251 case age <= admBeatDown:252 out.Status = ComponentStatusStatusDegraded253 default:254 out.Status = ComponentStatusStatusDown255 }256 out.Detail = &detail257 return out258}259260func (h *adminHandlers) GetAdminStatus(ctx context.Context, _ GetAdminStatusRequestObject) (GetAdminStatusResponseObject, error) {261 now := time.Now().UTC()262 out := AdminStatus{Version: Version, GeneratedAt: now, Backup: BackupStatus{Status: BackupStatusStatusUnknown}}263 if Commit != "" {264 out.Commit = &Commit265 }266267 // Database: this very request needs it, so measure it rather than assume it.268 pctx, cancel := context.WithTimeout(ctx, 3*time.Second)269 start := time.Now()270 err := h.Pool.Ping(pctx)271 cancel()272 if err != nil {273 out.Database = ComponentStatus{Status: ComponentStatusStatusDown, Detail: ptr("The database did not answer.")}274 out.Ingest = ComponentStatus{Status: ComponentStatusStatusUnknown, Detail: ptr("Unknown: the database is unreachable.")}275 out.Orchestrator = out.Ingest276 out.Backup.Detail = out.Ingest.Detail277 return GetAdminStatus200JSONResponse{Body: out}, nil278 }279 ms := float32(time.Since(start).Microseconds()) / 1000280 out.Database = ComponentStatus{Status: ComponentStatusStatusOk, LatencyMs: &ms}281282 beats := map[string]ComponentStatus{}283 rows, err := h.Pool.Query(ctx, `select service, version, beat_at from service_heartbeats where service = any($1)`,284 []string{"telemetry-ingest", "action-orchestrator"})285 if err != nil {286 return nil, err287 }288 var svcName, version string289 var beatAt time.Time290 if _, err := pgx.ForEachRow(rows, []any{&svcName, &version, &beatAt}, func() error {291 at := beatAt292 beats[svcName] = admComponent(&at, version, now)293 return nil294 }); err != nil {295 return nil, err296 }297 component := func(name string) ComponentStatus {298 if c, ok := beats[name]; ok {299 return c300 }301 return admComponent(nil, "", now)302 }303 out.Ingest, out.Orchestrator = component("telemetry-ingest"), component("action-orchestrator")304305 if out.Frozen, err = h.admFrozen(ctx); err != nil {306 return nil, err307 }308309 // Agents: connected = a non-revoked agent heard from within the offline threshold.310 var connected, total int311 if err := h.Pool.QueryRow(ctx, `select count(*) filter (where a.last_seen_at > $1), count(*) from agents a join nodes n on n.id = a.node_id312 where a.revoked_at is null and n.retired_at is null`, now.Add(-h.Cfg.OfflineAfter)).Scan(&connected, &total); err != nil {313 return nil, err314 }315 out.AgentsConnected = &connected316 if out.Ingest.Detail != nil {317 d := *out.Ingest.Detail + "; " + strconv.Itoa(connected) + " of " + strconv.Itoa(total) + " enrolled agents connected"318 out.Ingest.Detail = &d319 }320 if err := h.Pool.QueryRow(ctx, `select max(received_at) from node_state`).Scan(&out.LastReportAt); err != nil {321 return nil, err322 }323 out.SseClients = ptr(h.Hub.Subscribers())324325 // Backups: the Command Center's own backups (kind "mcc") when any is observed, else every observed backup.326 // "ok" needs evidence: an observed successful run, and no observed failure.327 var n, failed, okCount int328 if err := h.Pool.QueryRow(ctx, `with b as (select * from backups where kind = 'mcc' or not exists (select 1 from backups where kind = 'mcc'))329 select count(*), count(*) filter (where last_status = 'failed'), count(*) filter (where last_status = 'ok' and last_success_at is not null),330 max(last_run_at), max(last_success_at) from b`).Scan(&n, &failed, &okCount, &out.Backup.LastRunAt, &out.Backup.LastSuccessAt); err != nil {331 return nil, err332 }333 switch {334 case n == 0:335 out.Backup.Detail = ptr("No backup has been observed.")336 case failed > 0:337 out.Backup.Status = BackupStatusStatusFailed338 out.Backup.Detail = ptr(strconv.Itoa(failed) + " of " + strconv.Itoa(n) + " observed backups failed on their last run.")339 case okCount == n:340 out.Backup.Status = BackupStatusStatusOk341 out.Backup.Detail = ptr(strconv.Itoa(n) + " observed backups succeeded on their last run.")342 default:343 out.Backup.Detail = ptr(strconv.Itoa(n-okCount) + " of " + strconv.Itoa(n) + " observed backups have no known result.")344 }345 return GetAdminStatus200JSONResponse{Body: out}, nil346}347