SPB Git

Repository Public

admin_settings.go 12.7 KB 347 lines go

1package api23import (4	"bytes"5	"context"6	"encoding/json"7	"errors"8	"math"9	"strconv"10	"time"1112	"github.com/google/uuid"13	"github.com/jackc/pgx/v5"1415	"clustr.spboucher.ai/mcc/internal/audit"16	"clustr.spboucher.ai/mcc/internal/authz"17	"clustr.spboucher.ai/mcc/internal/httpx"18)1920// ── Settings ───────────────────────────────────────────────────────────────21//22// The settings table also holds internal state of other services (the ingest23// CA under `pki.ca`, the event stream watermark…). Only the keys declared24// below are readable and writable through the API, each with its own25// validation. Settings never hold secrets: a credential lives in a file of26// the secrets directory and nowhere else.2728const admSettingFrozen = "orchestrator.frozen"2930type admSetting struct {31	key         string32	description string33	def         func(c *core) any34	// check validates a decoded JSON value and returns the canonical value to store.35	check     func(v any) (any, error)36	ownerOnly bool37}3839var errAdmSettingValue = errors.New("invalid value")4041// admInt accepts a JSON integer within [min, max].42func admInt(min, max int64) func(any) (any, error) {43	return func(v any) (any, error) {44		n, ok := v.(json.Number)45		if !ok {46			return nil, errAdmSettingValue47		}48		i, err := strconv.ParseInt(n.String(), 10, 64)49		if err != nil || i < min || i > max {50			return nil, errAdmSettingValue51		}52		return i, nil53	}54}5556func admBool(v any) (any, error) {57	b, ok := v.(bool)58	if !ok {59		return nil, errAdmSettingValue60	}61	return b, nil62}6364func admEnum(values ...string) func(any) (any, error) {65	return func(v any) (any, error) {66		s, ok := v.(string)67		if !ok {68			return nil, errAdmSettingValue69		}70		for _, x := range values {71			if s == x {72				return s, nil73			}74		}75		return nil, errAdmSettingValue76	}77}7879func admDays(d time.Duration) int64 { return int64(math.Round(d.Hours() / 24)) }8081// admSettings is the allowlist, in display order.82var admSettings = []admSetting{83	{key: "retention.raw_days", description: "Days of raw (10 s) samples kept. Integer, 1 to 30.",84		def: func(c *core) any { return admDays(c.Cfg.RetainRaw) }, check: admInt(1, 30)},85	{key: "retention.1m_days", description: "Days of 1-minute aggregates kept. Integer, 7 to 180.",86		def: func(c *core) any { return admDays(c.Cfg.Retain1m) }, check: admInt(7, 180)},87	{key: "retention.1h_days", description: "Days of 1-hour aggregates kept. Integer, 30 to 1825.",88		def: func(c *core) any { return admDays(c.Cfg.Retain1h) }, check: admInt(30, 1825)},89	{key: "alerts.default_for_seconds", description: "Default time a condition must hold before a new alert rule fires. Integer, 0 to 86400.",90		def: func(*core) any { return int64(300) }, check: admInt(0, 86400)},91	{key: "alerts.default_clear_seconds", description: "Default time a condition must be clear before an alert resolves. Integer, 0 to 86400.",92		def: func(*core) any { return int64(120) }, check: admInt(0, 86400)},93	{key: "alerts.default_severity", description: "Default severity of a new alert rule: info, warning or critical.",94		def: func(*core) any { return "warning" }, check: admEnum("info", "warning", "critical")},95	{key: admSettingFrozen, description: "Emergency stop: when true the orchestrator issues no command to any agent. Boolean; owner only.",96		def: func(*core) any { return false }, check: admBool, ownerOnly: true},97}9899func admSettingByKey(key string) (admSetting, bool) {100	for _, s := range admSettings {101		if s.key == key {102			return s, true103		}104	}105	return admSetting{}, false106}107108type admStored struct {109	value     json.RawMessage110	updatedAt time.Time111	updatedBy *uuid.UUID112}113114// toSetting renders a setting. A key that was never written reports its115// default with the Unix epoch as update time and says so in the description.116func (h *adminHandlers) toSetting(s admSetting, st *admStored) Setting {117	out := Setting{Key: s.key, Description: ptr(s.description)}118	if st == nil {119		out.Value, out.UpdatedAt = s.def(h.core), time.Unix(0, 0).UTC()120		out.Description = ptr(s.description + " Never changed: this is the default.")121		return out122	}123	var v any124	if json.Unmarshal(st.value, &v) != nil {125		v = s.def(h.core)126	}127	out.Value, out.UpdatedAt, out.UpdatedBy = v, st.updatedAt, st.updatedBy128	return out129}130131func (h *adminHandlers) ListSettings(ctx context.Context, _ ListSettingsRequestObject) (ListSettingsResponseObject, error) {132	keys := make([]string, len(admSettings))133	for i, s := range admSettings {134		keys[i] = s.key135	}136	rows, err := h.Pool.Query(ctx, `select key, value, updated_at, updated_by from settings where key = any($1)`, keys)137	if err != nil {138		return nil, err139	}140	stored := map[string]*admStored{}141	var key string142	var st admStored143	if _, err := pgx.ForEachRow(rows, []any{&key, &st.value, &st.updatedAt, &st.updatedBy}, func() error {144		c := st145		c.value = append(json.RawMessage(nil), st.value...)146		stored[key] = &c147		return nil148	}); err != nil {149		return nil, err150	}151	out := SettingList{Items: make([]Setting, 0, len(admSettings))}152	for _, s := range admSettings {153		out.Items = append(out.Items, h.toSetting(s, stored[s.key]))154	}155	return ListSettings200JSONResponse{Body: out}, nil156}157158func (h *adminHandlers) PutSetting(ctx context.Context, req PutSettingRequestObject) (PutSettingResponseObject, error) {159	s, ok := admSettingByKey(req.Key)160	if !ok {161		return nil, httpx.ErrNotFound // unknown keys, and internal ones, do not exist for the API162	}163	if req.Body == nil {164		return nil, httpx.Validation("Body required.")165	}166	// Re-decode the value with exact numbers (the generated type decodes into float64).167	raw, err := json.Marshal(req.Body.Value)168	if err != nil {169		return nil, clInvalid("value", "is not valid JSON")170	}171	var decoded any172	dec := json.NewDecoder(bytes.NewReader(raw))173	dec.UseNumber()174	if err := dec.Decode(&decoded); err != nil {175		return nil, clInvalid("value", "is not valid JSON")176	}177	value, err := s.check(decoded)178	if err != nil {179		return nil, clInvalid("value", "is not valid for "+s.key+": "+s.description)180	}181	actor := principalFrom(ctx).Session182	if s.ownerOnly && actor.User.Role != authz.Owner {183		h.audit(ctx, h.Pool, "setting.update", "setting", s.key, audit.Denied, "owner only", nil)184		return nil, httpx.ErrForbidden.With("reason", "owner_only")185	}186	tx, err := h.Pool.Begin(ctx)187	if err != nil {188		return nil, err189	}190	defer tx.Rollback(ctx)191	var before json.RawMessage192	err = tx.QueryRow(ctx, `select value from settings where key = $1 for update`, s.key).Scan(&before)193	if err != nil && !errors.Is(err, pgx.ErrNoRows) {194		return nil, err195	}196	stored, err := json.Marshal(value) // a Go string would otherwise be sent as raw JSON text197	if err != nil {198		return nil, err199	}200	st := admStored{updatedBy: &actor.User.ID}201	if err := tx.QueryRow(ctx, `insert into settings(key, value, updated_at, updated_by) values ($1, $2, now(), $3)202		on conflict (key) do update set value = excluded.value, updated_at = excluded.updated_at, updated_by = excluded.updated_by203		returning value, updated_at`, s.key, stored, actor.User.ID).Scan(&st.value, &st.updatedAt); err != nil {204		return nil, err205	}206	params := map[string]any{"to": value}207	if before != nil {208		var old any209		if json.Unmarshal(before, &old) == nil {210			params["from"] = old211		}212	}213	if err := h.audit(ctx, tx, "setting.update", "setting", s.key, audit.Success, "", params); err != nil {214		return nil, err215	}216	if err := tx.Commit(ctx); err != nil {217		return nil, err218	}219	return PutSetting200JSONResponse{Body: h.toSetting(s, &st)}, nil220}221222// frozen reports the emergency stop. Only the JSON value `true` engages it.223func (c *core) admFrozen(ctx context.Context) (bool, error) {224	var raw []byte225	err := c.Pool.QueryRow(ctx, `select value from settings where key = $1`, admSettingFrozen).Scan(&raw)226	if errors.Is(err, pgx.ErrNoRows) {227		return false, nil228	}229	return string(raw) == "true", err230}231232// ── Backend status ─────────────────────────────────────────────────────────233234const (235	admBeatOK   = 30 * time.Second  // heartbeats are written every 10 s236	admBeatDown = 120 * time.Second // beyond this the component is considered down237)238239// admComponent turns a heartbeat into a status. No heartbeat at all is240// "unknown" (the component may never have been started), not "down".241func admComponent(beatAt *time.Time, version string, now time.Time) ComponentStatus {242	if beatAt == nil {243		return ComponentStatus{Status: ComponentStatusStatusUnknown, Detail: ptr("No heartbeat was ever recorded for this component.")}244	}245	age := now.Sub(*beatAt)246	out := ComponentStatus{HeartbeatAt: beatAt}247	detail := "version " + version + ", last heartbeat " + strconv.Itoa(int(age.Seconds())) + " s ago"248	switch {249	case age <= admBeatOK:250		out.Status = ComponentStatusStatusOk251	case age <= admBeatDown:252		out.Status = ComponentStatusStatusDegraded253	default:254		out.Status = ComponentStatusStatusDown255	}256	out.Detail = &detail257	return out258}259260func (h *adminHandlers) GetAdminStatus(ctx context.Context, _ GetAdminStatusRequestObject) (GetAdminStatusResponseObject, error) {261	now := time.Now().UTC()262	out := AdminStatus{Version: Version, GeneratedAt: now, Backup: BackupStatus{Status: BackupStatusStatusUnknown}}263	if Commit != "" {264		out.Commit = &Commit265	}266267	// Database: this very request needs it, so measure it rather than assume it.268	pctx, cancel := context.WithTimeout(ctx, 3*time.Second)269	start := time.Now()270	err := h.Pool.Ping(pctx)271	cancel()272	if err != nil {273		out.Database = ComponentStatus{Status: ComponentStatusStatusDown, Detail: ptr("The database did not answer.")}274		out.Ingest = ComponentStatus{Status: ComponentStatusStatusUnknown, Detail: ptr("Unknown: the database is unreachable.")}275		out.Orchestrator = out.Ingest276		out.Backup.Detail = out.Ingest.Detail277		return GetAdminStatus200JSONResponse{Body: out}, nil278	}279	ms := float32(time.Since(start).Microseconds()) / 1000280	out.Database = ComponentStatus{Status: ComponentStatusStatusOk, LatencyMs: &ms}281282	beats := map[string]ComponentStatus{}283	rows, err := h.Pool.Query(ctx, `select service, version, beat_at from service_heartbeats where service = any($1)`,284		[]string{"telemetry-ingest", "action-orchestrator"})285	if err != nil {286		return nil, err287	}288	var svcName, version string289	var beatAt time.Time290	if _, err := pgx.ForEachRow(rows, []any{&svcName, &version, &beatAt}, func() error {291		at := beatAt292		beats[svcName] = admComponent(&at, version, now)293		return nil294	}); err != nil {295		return nil, err296	}297	component := func(name string) ComponentStatus {298		if c, ok := beats[name]; ok {299			return c300		}301		return admComponent(nil, "", now)302	}303	out.Ingest, out.Orchestrator = component("telemetry-ingest"), component("action-orchestrator")304305	if out.Frozen, err = h.admFrozen(ctx); err != nil {306		return nil, err307	}308309	// Agents: connected = a non-revoked agent heard from within the offline threshold.310	var connected, total int311	if err := h.Pool.QueryRow(ctx, `select count(*) filter (where a.last_seen_at > $1), count(*) from agents a join nodes n on n.id = a.node_id312		where a.revoked_at is null and n.retired_at is null`, now.Add(-h.Cfg.OfflineAfter)).Scan(&connected, &total); err != nil {313		return nil, err314	}315	out.AgentsConnected = &connected316	if out.Ingest.Detail != nil {317		d := *out.Ingest.Detail + "; " + strconv.Itoa(connected) + " of " + strconv.Itoa(total) + " enrolled agents connected"318		out.Ingest.Detail = &d319	}320	if err := h.Pool.QueryRow(ctx, `select max(received_at) from node_state`).Scan(&out.LastReportAt); err != nil {321		return nil, err322	}323	out.SseClients = ptr(h.Hub.Subscribers())324325	// Backups: the Command Center's own backups (kind "mcc") when any is observed, else every observed backup.326	// "ok" needs evidence: an observed successful run, and no observed failure.327	var n, failed, okCount int328	if err := h.Pool.QueryRow(ctx, `with b as (select * from backups where kind = 'mcc' or not exists (select 1 from backups where kind = 'mcc'))329		select count(*), count(*) filter (where last_status = 'failed'), count(*) filter (where last_status = 'ok' and last_success_at is not null),330			max(last_run_at), max(last_success_at) from b`).Scan(&n, &failed, &okCount, &out.Backup.LastRunAt, &out.Backup.LastSuccessAt); err != nil {331		return nil, err332	}333	switch {334	case n == 0:335		out.Backup.Detail = ptr("No backup has been observed.")336	case failed > 0:337		out.Backup.Status = BackupStatusStatusFailed338		out.Backup.Detail = ptr(strconv.Itoa(failed) + " of " + strconv.Itoa(n) + " observed backups failed on their last run.")339	case okCount == n:340		out.Backup.Status = BackupStatusStatusOk341		out.Backup.Detail = ptr(strconv.Itoa(n) + " observed backups succeeded on their last run.")342	default:343		out.Backup.Detail = ptr(strconv.Itoa(n-okCount) + " of " + strconv.Itoa(n) + " observed backups have no known result.")344	}345	return GetAdminStatus200JSONResponse{Body: out}, nil346}347
Files main
Menu

Theme