SPB Git

Repository Public

cluster_overview.go 11.0 KB 372 lines go

1package api23import (4	"context"5	"regexp"6	"time"78	"github.com/google/uuid"9	"github.com/jackc/pgx/v5"1011	"clustr.spboucher.ai/mcc/internal/authz"12)1314const clMaxOverviewNodes = 50001516// clVolUse is what one internal volume reports.17type clVolUse struct {18	Device, FS string19	Mount      string20	Total      *int6421	Used       *int6422}2324var clAPFSContainer = regexp.MustCompile(`^(?:/dev/)?(disk\d+)s\d+`)2526// clStorageUse totals the internal storage of one node from its volumes.27// APFS volumes of one container all report the container's size, so they are28// grouped by container (size counted once, usage added up); other file29// systems are one group per device. Volumes whose size or usage was not30// observed do not contribute. ok is false when nothing contributed.31func clStorageUse(vols []clVolUse) (total, used int64, ok bool) {32	type grp struct{ total, used int64 }33	groups := map[string]*grp{}34	for _, v := range vols {35		if v.Total == nil || v.Used == nil || *v.Total <= 0 || *v.Used < 0 {36			continue37		}38		key := v.Device39		if key == "" {40			key = "mount:" + v.Mount41		} else if v.FS == "apfs" {42			if m := clAPFSContainer.FindStringSubmatch(v.Device); m != nil {43				key = "apfs:" + m[1]44			}45		}46		g := groups[key]47		if g == nil {48			g = &grp{}49			groups[key] = g50		}51		g.total = max(g.total, *v.Total)52		g.used += *v.Used53	}54	for _, g := range groups {55		total += g.total56		used += min(g.used, g.total)57		ok = true58	}59	return total, used, ok60}6162// GetClusterOverview aggregates the current state. Capacity and usage only63// sum what FRESH nodes report: the last values of a stale or offline node are64// not current, so they are left out and `coverage` says how many nodes each65// total stands for (compare with nodes.total). Sections the caller may not66// read are omitted.67func (h *clusterHandlers) GetClusterOverview(ctx context.Context, _ GetClusterOverviewRequestObject) (GetClusterOverviewResponseObject, error) {68	acc, err := h.clAccessFor(ctx)69	if err != nil {70		return nil, err71	}72	now := time.Now().UTC()73	out := ClusterOverview{GeneratedAt: now}7475	rows, err := h.Pool.Query(ctx, `select `+clNodeCols+`, s.values`+clNodeFrom+` where n.retired_at is null order by n.id limit `+itoaInt(clMaxOverviewNodes),76		h.clFreshArgs(now)...)77	if err != nil {78		return nil, err79	}80	type member struct {81		n      clNode82		values map[string]any83	}84	members, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (member, error) {85		var x member86		var raw []byte87		var err error88		x.n, err = clScanNode(r, &raw)89		x.values = clValues(raw)90		return x, err91	})92	if err != nil {93		return nil, err94	}95	freshOf := map[uuid.UUID]Freshness{}96	filter := acc.scoped(authz.NodesRead)97	visible := members[:0]98	for _, x := range members {99		if filter && !acc.can(authz.NodesRead, x.n.facets()...) {100			continue101		}102		visible = append(visible, x)103		freshOf[x.n.ID] = h.clFresh(now, x.n.Seen)104	}105106	// Internal volumes of the visible nodes, for storage capacity.107	volsOf := map[uuid.UUID][]clVolUse{}108	vrows, err := h.Pool.Query(ctx, `select v.node_id, coalesce(v.device,''), coalesce(v.fs,''), v.mount, v.total_bytes, v.used_bytes109		from volumes v join nodes n on n.id = v.node_id where v.gone_at is null and v.kind = 'internal' and n.retired_at is null limit 100000`)110	if err != nil {111		return nil, err112	}113	var vid uuid.UUID114	var vu clVolUse115	if _, err := pgx.ForEachRow(vrows, []any{&vid, &vu.Device, &vu.FS, &vu.Mount, &vu.Total, &vu.Used}, func() error {116		volsOf[vid] = append(volsOf[vid], vu)117		return nil118	}); err != nil {119		return nil, err120	}121122	var memUsed, stoUsed int64123	for _, x := range visible {124		out.Nodes.Total++125		if x.n.Maintenance {126			out.Nodes.Maintenance++127		}128		switch freshOf[x.n.ID] {129		case FreshnessFresh:130			out.Nodes.Fresh++131		case FreshnessStale:132			out.Nodes.Stale++133			continue134		default:135			out.Nodes.Offline++136			continue137		}138		// Fresh nodes only from here on.139		if x.n.InventoryAt != nil && x.n.CPUCores != nil && *x.n.CPUCores > 0 {140			out.Capacity.CpuCores.Total += int64(*x.n.CPUCores)141			out.Capacity.CpuCores.Coverage++142		}143		total, hasTotal := clNumber(x.values, "mem.total")144		if !hasTotal && x.n.InventoryAt != nil && x.n.MemBytes != nil {145			total, hasTotal = float64(*x.n.MemBytes), true146		}147		if used, ok := clNumber(x.values, "mem.used"); ok && hasTotal && total > 0 {148			out.Capacity.Memory.Total += int64(total)149			out.Capacity.Memory.Coverage++150			memUsed += int64(used)151		}152		if t, u, ok := clStorageUse(volsOf[x.n.ID]); ok {153			out.Capacity.Storage.Total += t154			out.Capacity.Storage.Coverage++155			stoUsed += u156		}157	}158	if out.Capacity.Memory.Coverage > 0 {159		out.Capacity.Memory.Used = &memUsed160	}161	if out.Capacity.Storage.Coverage > 0 {162		out.Capacity.Storage.Used = &stoUsed163	}164165	if acc.can(authz.AlertsRead) {166		sev := SeverityCounts{}167		rows, err := h.Pool.Query(ctx, `select severity::text, count(*) from alerts where state in ('firing','acknowledged') group by 1`)168		if err != nil {169			return nil, err170		}171		var name string172		var n int173		if _, err := pgx.ForEachRow(rows, []any{&name, &n}, func() error {174			switch name {175			case "info":176				sev.Info = n177			case "warning":178				sev.Warning = n179			case "critical":180				sev.Critical = n181			}182			return nil183		}); err != nil {184			return nil, err185		}186		out.Alerts = &sev187		var open int188		if err := h.Pool.QueryRow(ctx, `select count(*) from incidents where state <> 'resolved'`).Scan(&open); err != nil {189			return nil, err190		}191		out.OpenIncidents = &open192	}193194	if acc.can(authz.JobsRead) {195		jobs := JobCounts{}196		if err := h.Pool.QueryRow(ctx, `select count(*) filter (where state = 'queued'), count(*) filter (where state = 'running') from jobs197			where state in ('queued','running')`).Scan(&jobs.Queued, &jobs.Running); err != nil {198			return nil, err199		}200		out.Jobs = &jobs201	}202203	if acc.can(authz.ServicesRead) {204		// The state of a service on a node that is not reporting is not known now.205		sc := ServiceCounts{}206		rows, err := h.Pool.Query(ctx, `select sv.node_id, sv.state, count(*) from services sv join nodes n on n.id = sv.node_id207			where sv.gone_at is null and n.retired_at is null group by 1, 2`)208		if err != nil {209			return nil, err210		}211		var id uuid.UUID212		var state string213		var n int214		if _, err := pgx.ForEachRow(rows, []any{&id, &state, &n}, func() error {215			f, ok := freshOf[id]216			switch {217			case !ok: // hidden node218			case f == FreshnessOffline:219				sc.Unknown += n220			case state == "online":221				sc.Online += n222			case state == "stopped":223				sc.Stopped += n224			case state == "errored":225				sc.Errored += n226			default:227				sc.Unknown += n228			}229			return nil230		}); err != nil {231			return nil, err232		}233		out.Services = &sc234	}235236	if acc.can(authz.ModelsRead) {237		mc := ModelCounts{}238		rows, err := h.Pool.Query(ctx, `select i.node_id, i.health, count(*) from inference_servers i join nodes n on n.id = i.node_id239			where i.gone_at is null and n.retired_at is null group by 1, 2`)240		if err != nil {241			return nil, err242		}243		var id uuid.UUID244		var health string245		var n int246		if _, err := pgx.ForEachRow(rows, []any{&id, &health, &n}, func() error {247			if f, ok := freshOf[id]; ok {248				mc.Total += n249				if health == "ok" && f == FreshnessFresh {250					mc.Online += n251				}252			}253			return nil254		}); err != nil {255			return nil, err256		}257		out.Models = &mc258	}259260	{261		ns := NasSummary{}262		rows, err := h.Pool.Query(ctx, `select state, state_at from nas_targets limit 500`)263		if err != nil {264			return nil, err265		}266		var state string267		var at *time.Time268		if _, err := pgx.ForEachRow(rows, []any{&state, &at}, func() error {269			ns.Total++270			switch clNasState(state, at, now) {271			case NasTargetStateOnline:272				ns.Online++273			case NasTargetStateOffline:274				ns.Offline++275			default:276				ns.Unknown++277			}278			return nil279		}); err != nil {280			return nil, err281		}282		out.Nas = &ns283284		bs := BackupSummary{}285		rows, err = h.Pool.Query(ctx, `select last_status, last_success_at from backups limit 500`)286		if err != nil {287			return nil, err288		}289		var status string290		var okAt *time.Time291		if _, err := pgx.ForEachRow(rows, []any{&status, &okAt}, func() error {292			bs.Total++293			switch clBackupStatus(status) {294			case BackupLastStatusOk:295				bs.Ok++296			case BackupLastStatusFailed:297				bs.Failed++298			default:299				bs.Unknown++300			}301			if okAt != nil && (bs.LastSuccessAt == nil || okAt.After(*bs.LastSuccessAt)) {302				t := *okAt303				bs.LastSuccessAt = &t304			}305			return nil306		}); err != nil {307			return nil, err308		}309		out.Backups = &bs310	}311312	if acc.can(authz.ActionsRead) {313		actions, err := h.recentActions(ctx, 10)314		if err != nil {315			return nil, err316		}317		out.RecentActions = &actions318	}319	return GetClusterOverview200JSONResponse{Body: out}, nil320}321322// recentActions renders the latest actions for the overview. outcome_unknown323// follows DESIGN-BRIEF §4: the action ended failed or timed out after its324// command had been delivered to the agent without a result coming back.325func (h *clusterHandlers) recentActions(ctx context.Context, limit int) ([]Action, error) {326	rows, err := h.Pool.Query(ctx, `select a.id, a.type, a.criticality, a.node_id, n.alias::text, a.target_kind, a.target_id, a.params, a.state::text,327			a.requested_by, ru.display_name, a.requested_via, a.justification, a.approval_required, a.approved_by, au.display_name, a.approved_at,328			a.rejection_reason, a.created_at, a.queued_at, a.started_at, a.finished_at, a.deadline_at, a.cancel_requested_at, a.timeout_s,329			a.result, a.error_code, a.error_message, a.correlation_id, a.job_id,330			a.state in ('failed','timed_out') and exists (select 1 from agent_commands c331				where c.action_id = a.id and c.delivered_at is not null and c.finished_at is null)332		from actions a left join nodes n on n.id = a.node_id left join users ru on ru.id = a.requested_by left join users au on au.id = a.approved_by333		order by a.created_at desc, a.id desc limit $1`, limit)334	if err != nil {335		return nil, err336	}337	defer rows.Close()338	out := []Action{}339	for rows.Next() {340		var (341			a                  Action342			crit, tKind, tID   string343			state              string344			params, result     []byte345			errCode, errMsg    *string346			rejection, approve *string347		)348		if err := rows.Scan(&a.Id, &a.Type, &crit, &a.NodeId, &a.NodeAlias, &tKind, &tID, &params, &state, &a.RequestedBy, &a.RequestedByDisplayName,349			&a.RequestedVia, &a.Justification, &a.ApprovalRequired, &a.ApprovedBy, &approve, &a.ApprovedAt, &rejection, &a.CreatedAt, &a.QueuedAt,350			&a.StartedAt, &a.FinishedAt, &a.DeadlineAt, &a.CancelRequestedAt, &a.TimeoutS, &result, &errCode, &errMsg, &a.CorrelationId, &a.JobId,351			&a.OutcomeUnknown); err != nil {352			return nil, err353		}354		tid, err := uuid.Parse(tID)355		if err != nil {356			continue // a target that is not a resource id cannot be expressed in the contract357		}358		a.Criticality, a.State, a.Target = Criticality(crit), ActionState(state), ActionTarget{Kind: ActionTargetKind(tKind), Id: tid}359		a.Params = clObject(params)360		if len(result) > 0 && string(result) != "null" {361			r := clObject(result)362			a.Result = &r363		}364		a.ApprovedByDisplayName, a.RejectionReason, a.ErrorCode, a.ErrorMessage = approve, clNonEmpty(rejection), clNonEmpty(errCode), clNonEmpty(errMsg)365		if tKind == "node" && a.NodeAlias != nil {366			a.TargetLabel = a.NodeAlias367		}368		out = append(out, a)369	}370	return out, rows.Err()371}372
Files main
Menu

Theme