Repository Public
1package api23import (4 "context"5 "regexp"6 "time"78 "github.com/google/uuid"9 "github.com/jackc/pgx/v5"1011 "clustr.spboucher.ai/mcc/internal/authz"12)1314const clMaxOverviewNodes = 50001516// clVolUse is what one internal volume reports.17type clVolUse struct {18 Device, FS string19 Mount string20 Total *int6421 Used *int6422}2324var clAPFSContainer = regexp.MustCompile(`^(?:/dev/)?(disk\d+)s\d+`)2526// clStorageUse totals the internal storage of one node from its volumes.27// APFS volumes of one container all report the container's size, so they are28// grouped by container (size counted once, usage added up); other file29// systems are one group per device. Volumes whose size or usage was not30// observed do not contribute. ok is false when nothing contributed.31func clStorageUse(vols []clVolUse) (total, used int64, ok bool) {32 type grp struct{ total, used int64 }33 groups := map[string]*grp{}34 for _, v := range vols {35 if v.Total == nil || v.Used == nil || *v.Total <= 0 || *v.Used < 0 {36 continue37 }38 key := v.Device39 if key == "" {40 key = "mount:" + v.Mount41 } else if v.FS == "apfs" {42 if m := clAPFSContainer.FindStringSubmatch(v.Device); m != nil {43 key = "apfs:" + m[1]44 }45 }46 g := groups[key]47 if g == nil {48 g = &grp{}49 groups[key] = g50 }51 g.total = max(g.total, *v.Total)52 g.used += *v.Used53 }54 for _, g := range groups {55 total += g.total56 used += min(g.used, g.total)57 ok = true58 }59 return total, used, ok60}6162// GetClusterOverview aggregates the current state. Capacity and usage only63// sum what FRESH nodes report: the last values of a stale or offline node are64// not current, so they are left out and `coverage` says how many nodes each65// total stands for (compare with nodes.total). Sections the caller may not66// read are omitted.67func (h *clusterHandlers) GetClusterOverview(ctx context.Context, _ GetClusterOverviewRequestObject) (GetClusterOverviewResponseObject, error) {68 acc, err := h.clAccessFor(ctx)69 if err != nil {70 return nil, err71 }72 now := time.Now().UTC()73 out := ClusterOverview{GeneratedAt: now}7475 rows, err := h.Pool.Query(ctx, `select `+clNodeCols+`, s.values`+clNodeFrom+` where n.retired_at is null order by n.id limit `+itoaInt(clMaxOverviewNodes),76 h.clFreshArgs(now)...)77 if err != nil {78 return nil, err79 }80 type member struct {81 n clNode82 values map[string]any83 }84 members, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (member, error) {85 var x member86 var raw []byte87 var err error88 x.n, err = clScanNode(r, &raw)89 x.values = clValues(raw)90 return x, err91 })92 if err != nil {93 return nil, err94 }95 freshOf := map[uuid.UUID]Freshness{}96 filter := acc.scoped(authz.NodesRead)97 visible := members[:0]98 for _, x := range members {99 if filter && !acc.can(authz.NodesRead, x.n.facets()...) {100 continue101 }102 visible = append(visible, x)103 freshOf[x.n.ID] = h.clFresh(now, x.n.Seen)104 }105106 // Internal volumes of the visible nodes, for storage capacity.107 volsOf := map[uuid.UUID][]clVolUse{}108 vrows, err := h.Pool.Query(ctx, `select v.node_id, coalesce(v.device,''), coalesce(v.fs,''), v.mount, v.total_bytes, v.used_bytes109 from volumes v join nodes n on n.id = v.node_id where v.gone_at is null and v.kind = 'internal' and n.retired_at is null limit 100000`)110 if err != nil {111 return nil, err112 }113 var vid uuid.UUID114 var vu clVolUse115 if _, err := pgx.ForEachRow(vrows, []any{&vid, &vu.Device, &vu.FS, &vu.Mount, &vu.Total, &vu.Used}, func() error {116 volsOf[vid] = append(volsOf[vid], vu)117 return nil118 }); err != nil {119 return nil, err120 }121122 var memUsed, stoUsed int64123 for _, x := range visible {124 out.Nodes.Total++125 if x.n.Maintenance {126 out.Nodes.Maintenance++127 }128 switch freshOf[x.n.ID] {129 case FreshnessFresh:130 out.Nodes.Fresh++131 case FreshnessStale:132 out.Nodes.Stale++133 continue134 default:135 out.Nodes.Offline++136 continue137 }138 // Fresh nodes only from here on.139 if x.n.InventoryAt != nil && x.n.CPUCores != nil && *x.n.CPUCores > 0 {140 out.Capacity.CpuCores.Total += int64(*x.n.CPUCores)141 out.Capacity.CpuCores.Coverage++142 }143 total, hasTotal := clNumber(x.values, "mem.total")144 if !hasTotal && x.n.InventoryAt != nil && x.n.MemBytes != nil {145 total, hasTotal = float64(*x.n.MemBytes), true146 }147 if used, ok := clNumber(x.values, "mem.used"); ok && hasTotal && total > 0 {148 out.Capacity.Memory.Total += int64(total)149 out.Capacity.Memory.Coverage++150 memUsed += int64(used)151 }152 if t, u, ok := clStorageUse(volsOf[x.n.ID]); ok {153 out.Capacity.Storage.Total += t154 out.Capacity.Storage.Coverage++155 stoUsed += u156 }157 }158 if out.Capacity.Memory.Coverage > 0 {159 out.Capacity.Memory.Used = &memUsed160 }161 if out.Capacity.Storage.Coverage > 0 {162 out.Capacity.Storage.Used = &stoUsed163 }164165 if acc.can(authz.AlertsRead) {166 sev := SeverityCounts{}167 rows, err := h.Pool.Query(ctx, `select severity::text, count(*) from alerts where state in ('firing','acknowledged') group by 1`)168 if err != nil {169 return nil, err170 }171 var name string172 var n int173 if _, err := pgx.ForEachRow(rows, []any{&name, &n}, func() error {174 switch name {175 case "info":176 sev.Info = n177 case "warning":178 sev.Warning = n179 case "critical":180 sev.Critical = n181 }182 return nil183 }); err != nil {184 return nil, err185 }186 out.Alerts = &sev187 var open int188 if err := h.Pool.QueryRow(ctx, `select count(*) from incidents where state <> 'resolved'`).Scan(&open); err != nil {189 return nil, err190 }191 out.OpenIncidents = &open192 }193194 if acc.can(authz.JobsRead) {195 jobs := JobCounts{}196 if err := h.Pool.QueryRow(ctx, `select count(*) filter (where state = 'queued'), count(*) filter (where state = 'running') from jobs197 where state in ('queued','running')`).Scan(&jobs.Queued, &jobs.Running); err != nil {198 return nil, err199 }200 out.Jobs = &jobs201 }202203 if acc.can(authz.ServicesRead) {204 // The state of a service on a node that is not reporting is not known now.205 sc := ServiceCounts{}206 rows, err := h.Pool.Query(ctx, `select sv.node_id, sv.state, count(*) from services sv join nodes n on n.id = sv.node_id207 where sv.gone_at is null and n.retired_at is null group by 1, 2`)208 if err != nil {209 return nil, err210 }211 var id uuid.UUID212 var state string213 var n int214 if _, err := pgx.ForEachRow(rows, []any{&id, &state, &n}, func() error {215 f, ok := freshOf[id]216 switch {217 case !ok: // hidden node218 case f == FreshnessOffline:219 sc.Unknown += n220 case state == "online":221 sc.Online += n222 case state == "stopped":223 sc.Stopped += n224 case state == "errored":225 sc.Errored += n226 default:227 sc.Unknown += n228 }229 return nil230 }); err != nil {231 return nil, err232 }233 out.Services = &sc234 }235236 if acc.can(authz.ModelsRead) {237 mc := ModelCounts{}238 rows, err := h.Pool.Query(ctx, `select i.node_id, i.health, count(*) from inference_servers i join nodes n on n.id = i.node_id239 where i.gone_at is null and n.retired_at is null group by 1, 2`)240 if err != nil {241 return nil, err242 }243 var id uuid.UUID244 var health string245 var n int246 if _, err := pgx.ForEachRow(rows, []any{&id, &health, &n}, func() error {247 if f, ok := freshOf[id]; ok {248 mc.Total += n249 if health == "ok" && f == FreshnessFresh {250 mc.Online += n251 }252 }253 return nil254 }); err != nil {255 return nil, err256 }257 out.Models = &mc258 }259260 {261 ns := NasSummary{}262 rows, err := h.Pool.Query(ctx, `select state, state_at from nas_targets limit 500`)263 if err != nil {264 return nil, err265 }266 var state string267 var at *time.Time268 if _, err := pgx.ForEachRow(rows, []any{&state, &at}, func() error {269 ns.Total++270 switch clNasState(state, at, now) {271 case NasTargetStateOnline:272 ns.Online++273 case NasTargetStateOffline:274 ns.Offline++275 default:276 ns.Unknown++277 }278 return nil279 }); err != nil {280 return nil, err281 }282 out.Nas = &ns283284 bs := BackupSummary{}285 rows, err = h.Pool.Query(ctx, `select last_status, last_success_at from backups limit 500`)286 if err != nil {287 return nil, err288 }289 var status string290 var okAt *time.Time291 if _, err := pgx.ForEachRow(rows, []any{&status, &okAt}, func() error {292 bs.Total++293 switch clBackupStatus(status) {294 case BackupLastStatusOk:295 bs.Ok++296 case BackupLastStatusFailed:297 bs.Failed++298 default:299 bs.Unknown++300 }301 if okAt != nil && (bs.LastSuccessAt == nil || okAt.After(*bs.LastSuccessAt)) {302 t := *okAt303 bs.LastSuccessAt = &t304 }305 return nil306 }); err != nil {307 return nil, err308 }309 out.Backups = &bs310 }311312 if acc.can(authz.ActionsRead) {313 actions, err := h.recentActions(ctx, 10)314 if err != nil {315 return nil, err316 }317 out.RecentActions = &actions318 }319 return GetClusterOverview200JSONResponse{Body: out}, nil320}321322// recentActions renders the latest actions for the overview. outcome_unknown323// follows DESIGN-BRIEF §4: the action ended failed or timed out after its324// command had been delivered to the agent without a result coming back.325func (h *clusterHandlers) recentActions(ctx context.Context, limit int) ([]Action, error) {326 rows, err := h.Pool.Query(ctx, `select a.id, a.type, a.criticality, a.node_id, n.alias::text, a.target_kind, a.target_id, a.params, a.state::text,327 a.requested_by, ru.display_name, a.requested_via, a.justification, a.approval_required, a.approved_by, au.display_name, a.approved_at,328 a.rejection_reason, a.created_at, a.queued_at, a.started_at, a.finished_at, a.deadline_at, a.cancel_requested_at, a.timeout_s,329 a.result, a.error_code, a.error_message, a.correlation_id, a.job_id,330 a.state in ('failed','timed_out') and exists (select 1 from agent_commands c331 where c.action_id = a.id and c.delivered_at is not null and c.finished_at is null)332 from actions a left join nodes n on n.id = a.node_id left join users ru on ru.id = a.requested_by left join users au on au.id = a.approved_by333 order by a.created_at desc, a.id desc limit $1`, limit)334 if err != nil {335 return nil, err336 }337 defer rows.Close()338 out := []Action{}339 for rows.Next() {340 var (341 a Action342 crit, tKind, tID string343 state string344 params, result []byte345 errCode, errMsg *string346 rejection, approve *string347 )348 if err := rows.Scan(&a.Id, &a.Type, &crit, &a.NodeId, &a.NodeAlias, &tKind, &tID, ¶ms, &state, &a.RequestedBy, &a.RequestedByDisplayName,349 &a.RequestedVia, &a.Justification, &a.ApprovalRequired, &a.ApprovedBy, &approve, &a.ApprovedAt, &rejection, &a.CreatedAt, &a.QueuedAt,350 &a.StartedAt, &a.FinishedAt, &a.DeadlineAt, &a.CancelRequestedAt, &a.TimeoutS, &result, &errCode, &errMsg, &a.CorrelationId, &a.JobId,351 &a.OutcomeUnknown); err != nil {352 return nil, err353 }354 tid, err := uuid.Parse(tID)355 if err != nil {356 continue // a target that is not a resource id cannot be expressed in the contract357 }358 a.Criticality, a.State, a.Target = Criticality(crit), ActionState(state), ActionTarget{Kind: ActionTargetKind(tKind), Id: tid}359 a.Params = clObject(params)360 if len(result) > 0 && string(result) != "null" {361 r := clObject(result)362 a.Result = &r363 }364 a.ApprovedByDisplayName, a.RejectionReason, a.ErrorCode, a.ErrorMessage = approve, clNonEmpty(rejection), clNonEmpty(errCode), clNonEmpty(errMsg)365 if tKind == "node" && a.NodeAlias != nil {366 a.TargetLabel = a.NodeAlias367 }368 out = append(out, a)369 }370 return out, rows.Err()371}372