SPB Git

Repository Public

cluster_services.go 20.5 KB 592 lines go

1package api23import (4	"context"5	"encoding/json"6	"errors"7	"net/url"8	"slices"9	"strconv"10	"strings"11	"time"1213	"github.com/google/uuid"14	"github.com/jackc/pgx/v5"1516	"clustr.spboucher.ai/mcc/internal/audit"17	"clustr.spboucher.ai/mcc/internal/authz"18	"clustr.spboucher.ai/mcc/internal/events"19	"clustr.spboucher.ai/mcc/internal/httpx"20)2122// ── Services ───────────────────────────────────────────────────────────────2324const clServiceCols = `sv.id, sv.node_id, n.alias::text, n.site, n.tags, sv.manager::text, sv.name, sv.state, sv.pid, sv.restarts, sv.mem_bytes,25	sv.cpu_pct::float8, sv.uptime_s, sv.version, sv.ports, sv.app, sv.public_url, sv.health, sv.managed, sv.meta, sv.first_seen_at, sv.last_seen_at,26	sv.gone_at, greatest(s.received_at, n.last_seen_at)`2728const clServiceFrom = ` from services sv join nodes n on n.id = sv.node_id left join node_state s on s.node_id = n.id `2930type clService struct {31	ID, NodeID  uuid.UUID32	NodeAlias   string33	Site        string34	Tags        []string35	Manager     string36	Name        string37	State       string38	PID         *int39	Restarts    *int40	MemBytes    *int6441	CPU         *float6442	UptimeS     *int6443	Version     *string44	Ports       []int3245	App         *string46	PublicURL   *string47	Health      []byte48	Managed     bool49	Meta        []byte50	FirstSeenAt time.Time51	LastSeenAt  time.Time52	GoneAt      *time.Time53	NodeSeen    *time.Time54}5556func clScanService(row pgx.Row, extra ...any) (clService, error) {57	var s clService58	dst := append([]any{&s.ID, &s.NodeID, &s.NodeAlias, &s.Site, &s.Tags, &s.Manager, &s.Name, &s.State, &s.PID, &s.Restarts, &s.MemBytes,59		&s.CPU, &s.UptimeS, &s.Version, &s.Ports, &s.App, &s.PublicURL, &s.Health, &s.Managed, &s.Meta, &s.FirstSeenAt, &s.LastSeenAt,60		&s.GoneAt, &s.NodeSeen}, extra...)61	return s, row.Scan(dst...)62}6364func (s *clService) facets() []authz.Resource { return clServiceFacets(s.ID, s.NodeID, s.Site, s.Tags) }6566// clServiceState maps the stored word to the contract vocabulary; anything67// else is unknown (the raw manager state stays in meta).68func clServiceState(s string) ServiceState {69	switch ServiceState(s) {70	case ServiceStateOnline, ServiceStateStopped, ServiceStateErrored:71		return ServiceState(s)72	}73	return ServiceStateUnknown74}7576// toService renders a service. The state is the last one observed: gone_at77// (no longer listed by its manager) and freshness (age of the node's last78// report) tell how much it can be trusted.79func (h *clusterHandlers) toService(s *clService, now time.Time, withMeta bool) Service {80	out := Service{Id: s.ID, NodeId: s.NodeID, NodeAlias: s.NodeAlias, Manager: ServiceManager(s.Manager), Name: s.Name, State: clServiceState(s.State),81		Pid: s.PID, Restarts: s.Restarts, MemBytes: s.MemBytes, CpuPct: clF32(s.CPU), UptimeS: s.UptimeS, Version: clNonEmpty(s.Version),82		Ports: make([]int, 0, len(s.Ports)), App: clNonEmpty(s.App), PublicUrl: clNonEmpty(s.PublicURL), Managed: s.Managed,83		FirstSeenAt: s.FirstSeenAt, LastSeenAt: s.LastSeenAt, GoneAt: s.GoneAt, ObservedAt: &s.LastSeenAt, Freshness: h.clFresh(now, s.NodeSeen)}84	for _, p := range s.Ports {85		out.Ports = append(out.Ports, int(p))86	}87	if len(s.Health) > 0 && string(s.Health) != "null" {88		var hv struct {89			Status    string     `json:"status"`90			CheckedAt *time.Time `json:"checked_at"`91			LatencyMS *float32   `json:"latency_ms"`92			Detail    string     `json:"detail"`93		}94		if json.Unmarshal(s.Health, &hv) == nil {95			sh := ServiceHealth{Status: ServiceHealthStatusUnknown, CheckedAt: hv.CheckedAt, LatencyMs: hv.LatencyMS, Detail: clNonEmpty(&hv.Detail)}96			if hv.Status == "ok" || hv.Status == "failing" {97				sh.Status = ServiceHealthStatus(hv.Status)98			}99			out.Health = &sh100		}101	}102	if withMeta {103		meta := audit.Redact(clObject(s.Meta)) // second line of defence: the agent already redacts104		out.Meta = &meta105	}106	return out107}108109func (h *clusterHandlers) listServices(ctx context.Context, acc *clAccess, p ListServicesParams, cursorKind string) (ServiceList, error) {110	var out ServiceList111	limit, err := clLimit(p.Limit)112	if err != nil {113		return out, err114	}115	q, err := clQuery(p.Q)116	if err != nil {117		return out, err118	}119	b := &clSQL{}120	b.and("n.retired_at is null")121	if q != "" {122		a := b.arg(clLike(q))123		b.and(`(sv.name ilike ` + a + ` escape '\' or coalesce(sv.app,'') ilike ` + a + ` escape '\')`)124	}125	if p.NodeId != nil {126		b.and("sv.node_id = " + b.arg(*p.NodeId))127	}128	if p.Manager != nil {129		if !slices.Contains([]ServiceManager{ServiceManagerPm2, ServiceManagerLaunchd, ServiceManagerSystemd, ServiceManagerDocker}, *p.Manager) {130			return out, clInvalid("manager", "must be pm2, launchd, systemd or docker")131		}132		b.and("sv.manager = " + b.arg(string(*p.Manager)) + "::service_manager")133	}134	if p.State != nil {135		switch *p.State {136		case ServiceStateOnline, ServiceStateStopped, ServiceStateErrored:137			// A service that is gone is not in any current state.138			b.and("sv.gone_at is null and sv.state = " + b.arg(string(*p.State)))139		case ServiceStateUnknown:140			b.and("sv.gone_at is null and sv.state not in ('online','stopped','errored')")141		default:142			return out, clInvalid("state", "must be online, stopped, errored or unknown")143		}144	}145	if p.App != nil {146		if len(*p.App) > 120 {147			return out, clInvalid("app", "must be at most 120 characters")148		}149		b.and("lower(sv.app) = lower(" + b.arg(*p.App) + ")")150	}151	if p.Managed != nil {152		b.and("sv.managed = " + b.arg(*p.Managed))153	}154	cur, err := clDecodeCursor(p.Cursor, cursorKind, 3)155	if err != nil {156		return out, err157	}158	orderBy, err := b.clKeyset("lower(sv.name)", "text", "sv.id", false, cur)159	if err != nil {160		return out, err161	}162	// The sort key is read back from the database: the cursor must compare exactly as the ORDER BY does.163	rows, err := h.Pool.Query(ctx, `select `+clServiceCols+`, lower(sv.name)`+clServiceFrom+b.where()+orderBy+` limit `+b.arg(limit+1), b.args...)164	if err != nil {165		return out, err166	}167	var keys []string168	list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (clService, error) {169		var key string170		s, err := clScanService(r, &key)171		keys = append(keys, key)172		return s, err173	})174	if err != nil {175		return out, err176	}177	out.Items = make([]Service, 0, len(list))178	if len(list) > limit {179		list = list[:limit]180		out.NextCursor = clEncodeCursor(cursorKind, "0", keys[limit-1], list[limit-1].ID.String())181	}182	now := time.Now().UTC()183	filter := acc.scoped(authz.ServicesRead)184	for i := range list {185		if filter && !acc.can(authz.ServicesRead, list[i].facets()...) {186			continue187		}188		out.Items = append(out.Items, h.toService(&list[i], now, false))189	}190	return out, nil191}192193func (h *clusterHandlers) ListServices(ctx context.Context, req ListServicesRequestObject) (ListServicesResponseObject, error) {194	acc, err := h.clAccessFor(ctx)195	if err != nil {196		return nil, err197	}198	list, err := h.listServices(ctx, acc, req.Params, "services")199	if err != nil {200		return nil, err201	}202	return ListServices200JSONResponse{Body: list}, nil203}204205func (h *clusterHandlers) serviceByID(ctx context.Context, q clQuerier, id uuid.UUID, lock string) (clService, error) {206	s, err := clScanService(q.QueryRow(ctx, `select `+clServiceCols+clServiceFrom+` where sv.id = $1`+lock, id))207	if errors.Is(err, pgx.ErrNoRows) {208		return s, httpx.ErrNotFound209	}210	return s, err211}212213func (h *clusterHandlers) GetService(ctx context.Context, req GetServiceRequestObject) (GetServiceResponseObject, error) {214	acc, err := h.clAccessFor(ctx)215	if err != nil {216		return nil, err217	}218	s, err := h.serviceByID(ctx, h.Pool, req.Id, "")219	if err != nil {220		return nil, err221	}222	if !acc.can(authz.ServicesRead, s.facets()...) {223		return nil, httpx.ErrNotFound224	}225	return GetService200JSONResponse{Body: h.toService(&s, time.Now().UTC(), true)}, nil226}227228// clPublicURL validates the public URL of a service: an absolute http(s) URL229// without credentials. The empty string clears it.230func clPublicURL(v string) (*string, error) {231	v = strings.TrimSpace(v)232	if v == "" {233		return nil, nil234	}235	u, err := url.Parse(v)236	switch {237	case len(v) > 300:238		return nil, clInvalid("public_url", "must be at most 300 characters")239	case err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || !clPrintable(v):240		return nil, clInvalid("public_url", "must be an absolute http or https URL")241	case u.User != nil:242		return nil, clInvalid("public_url", "must not contain credentials")243	}244	return &v, nil245}246247func (h *clusterHandlers) UpdateService(ctx context.Context, req UpdateServiceRequestObject) (UpdateServiceResponseObject, error) {248	if req.Body == nil {249		return nil, httpx.Validation("Body required.")250	}251	if req.Body.Managed == nil && req.Body.PublicUrl == nil {252		return nil, httpx.Validation("Nothing to update: give managed or public_url.")253	}254	var newURL *string255	if req.Body.PublicUrl != nil {256		u, err := clPublicURL(*req.Body.PublicUrl)257		if err != nil {258			return nil, err259		}260		newURL = u261	}262	acc, err := h.clAccessFor(ctx)263	if err != nil {264		return nil, err265	}266	tx, err := h.Pool.Begin(ctx)267	if err != nil {268		return nil, err269	}270	defer tx.Rollback(ctx)271	s, err := h.serviceByID(ctx, tx, req.Id, " for update of sv")272	if err != nil {273		return nil, err274	}275	if !acc.can(authz.ServicesRead, s.facets()...) {276		return nil, httpx.ErrNotFound277	}278	if err := h.clAuthorizeWrite(ctx, acc, authz.SettingsManage, authz.Resource{Kind: "service", ID: s.ID.String()}, nil); err != nil {279		return nil, err280	}281	changes := map[string]any{}282	managed := s.Managed283	if req.Body.Managed != nil && *req.Body.Managed != s.Managed {284		managed = *req.Body.Managed285		changes["managed"] = map[string]any{"from": s.Managed, "to": managed}286	}287	pub := s.PublicURL288	if req.Body.PublicUrl != nil {289		old, next := "", ""290		if s.PublicURL != nil {291			old = *s.PublicURL292		}293		if newURL != nil {294			next = *newURL295		}296		if old != next {297			pub = newURL298			changes["public_url"] = map[string]any{"from": old, "to": next}299		}300	}301	if len(changes) > 0 {302		if _, err := tx.Exec(ctx, `update services set managed = $2, public_url = $3 where id = $1`, s.ID, managed, pub); err != nil {303			return nil, err304		}305		if err := h.audit(ctx, tx, "service.update", "service", s.ID.String(), audit.Success, "",306			map[string]any{"node_id": s.NodeID.String(), "manager": s.Manager, "name": s.Name, "changes": changes}); err != nil {307			return nil, err308		}309		ev := ServiceChangedEvent{ServiceId: s.ID, NodeId: s.NodeID, Manager: ServiceManager(s.Manager), Name: s.Name, State: clServiceState(s.State)}310		if s.GoneAt != nil {311			ev.Gone = ptr(true)312		}313		if err := events.Publish(ctx, tx, "service.changed", ev); err != nil {314			return nil, err315		}316		if err := tx.Commit(ctx); err != nil {317			return nil, err318		}319		s.Managed, s.PublicURL = managed, pub320	}321	return UpdateService200JSONResponse{Body: h.toService(&s, time.Now().UTC(), true)}, nil322}323324// GetServiceHistory is the timeline of a service, built only from recorded325// facts: the changes observed by ingestion (service_events, written by a326// trigger on the services table) and the actions that targeted the service.327// Action entries are included only for callers who may read actions.328func (h *clusterHandlers) GetServiceHistory(ctx context.Context, req GetServiceHistoryRequestObject) (GetServiceHistoryResponseObject, error) {329	limit, err := clLimit(req.Params.Limit)330	if err != nil {331		return nil, err332	}333	acc, err := h.clAccessFor(ctx)334	if err != nil {335		return nil, err336	}337	s, err := h.serviceByID(ctx, h.Pool, req.Id, "")338	if err != nil {339		return nil, err340	}341	if !acc.can(authz.ServicesRead, s.facets()...) {342		return nil, httpx.ErrNotFound343	}344	kind := "service-history:" + s.ID.String()345	cur, err := clDecodeCursor(req.Params.Cursor, kind, 2)346	if err != nil {347		return nil, err348	}349	var afterTS *time.Time350	afterKey := ""351	if cur != nil {352		t, err := clCursorTime(cur[0])353		if err != nil {354			return nil, err355		}356		afterTS, afterKey = &t, cur[1]357	}358	rows, err := h.Pool.Query(ctx, `select h.ts, h.kind, h.from_state, h.to_state, h.restarts_from, h.restarts_to, h.version_from, h.version_to,359			h.action_id, h.action_type, h.action_state, h.actor, h.key360		from (361			select e.ts, e.kind, e.from_state, e.to_state, e.restarts_from, e.restarts_to, e.version_from, e.version_to,362				null::uuid as action_id, null::text as action_type, null::text as action_state, null::text as actor,363				'e' || lpad(e.id::text, 19, '0') as key364			from service_events e where e.service_id = $1365			union all366			select a.created_at, 'action', null, null, null, null, null, null, a.id, a.type, a.state::text, u.display_name, 'a' || a.id::text367			from actions a left join users u on u.id = a.requested_by368			where $3::boolean and a.target_kind = 'service' and a.target_id = $2369		) h370		where ($4::timestamptz is null or (h.ts, h.key) < ($4::timestamptz, $5::text))371		order by h.ts desc, h.key desc limit $6`,372		s.ID, s.ID.String(), acc.can(authz.ActionsRead, s.facets()...), afterTS, afterKey, limit+1)373	if err != nil {374		return nil, err375	}376	type entry struct {377		e   ServiceHistoryEntry378		key string379	}380	list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (entry, error) {381		var (382			x                          entry383			kind                       string384			from, to, vFrom, vTo       *string385			rFrom, rTo                 *int386			actionType, actionState, a *string387		)388		if err := r.Scan(&x.e.Ts, &kind, &from, &to, &rFrom, &rTo, &vFrom, &vTo, &x.e.ActionId, &actionType, &actionState, &a, &x.key); err != nil {389			return x, err390		}391		state := func(p *string) *ServiceState {392			if p == nil {393				return nil394			}395			return ptr(clServiceState(*p))396		}397		x.e.Kind = ServiceHistoryEntryKind(kind)398		switch kind {399		case "appeared":400			x.e.Summary, x.e.ToState, x.e.Version = "Service first listed by "+s.Manager+".", state(to), clNonEmpty(vTo)401		case "disappeared":402			x.e.Summary, x.e.FromState = "Service no longer listed by "+s.Manager+".", state(from)403		case "state_change":404			x.e.FromState, x.e.ToState = state(from), state(to)405			x.e.Summary = "State changed from " + clDeref(from, "unknown") + " to " + clDeref(to, "unknown") + "."406		case "restart":407			x.e.Summary = "Restart counter went from " + clItoaPtr(rFrom) + " to " + clItoaPtr(rTo) + "."408		case "deploy":409			x.e.Summary, x.e.Version = "Version changed from "+clDeref(vFrom, "?")+" to "+clDeref(vTo, "?")+".", clNonEmpty(vTo)410		case "action":411			x.e.Summary, x.e.ActorDisplayName = "Action "+clDeref(actionType, "")+": "+clDeref(actionState, "")+".", clNonEmpty(a)412		default:413			x.e.Kind, x.e.Summary = ServiceHistoryEntryKindStateChange, "Recorded change."414		}415		return x, nil416	})417	if err != nil {418		return nil, err419	}420	out := ServiceHistory{Items: make([]ServiceHistoryEntry, 0, len(list))}421	if len(list) > limit {422		list = list[:limit]423		out.NextCursor = clEncodeCursor(kind, clTimeKey(list[limit-1].e.Ts), list[limit-1].key)424	}425	for _, x := range list {426		out.Items = append(out.Items, x.e)427	}428	return GetServiceHistory200JSONResponse{Body: out}, nil429}430431func clDeref(p *string, def string) string {432	if p == nil || *p == "" {433		return def434	}435	return *p436}437438func clItoaPtr(p *int) string {439	if p == nil {440		return "?"441	}442	return strconv.Itoa(*p)443}444445// ── Inference servers ──────────────────────────────────────────────────────446447const clModelCols = `i.id, i.node_id, n.alias::text, n.site, n.tags, i.kind, i.port, i.model, i.quantization, i.pid, i.rss_bytes, i.uptime_s,448	i.disk_bytes, i.health, i.health_at, i.health_latency_ms::float8, i.listed_models, i.last_seen_at, i.gone_at, greatest(s.received_at, n.last_seen_at)`449450const clModelFrom = ` from inference_servers i join nodes n on n.id = i.node_id left join node_state s on s.node_id = n.id `451452const clModelKey = `(lower(n.alias::text) || ' ' || lpad(i.port::text, 5, '0'))`453454type clModel struct {455	ID, NodeID  uuid.UUID456	NodeAlias   string457	Site        string458	Tags        []string459	Kind        string460	Port        int461	Model       *string462	Quant       *string463	PID         *int464	RSS         *int64465	UptimeS     *int64466	DiskBytes   *int64467	Health      string468	HealthAt    *time.Time469	HealthMS    *float64470	Listed      []byte471	LastSeenAt  time.Time472	GoneAt      *time.Time473	NodeSeen    *time.Time474	sortKeyText string475}476477func clScanModel(row pgx.Row) (clModel, error) {478	var m clModel479	err := row.Scan(&m.ID, &m.NodeID, &m.NodeAlias, &m.Site, &m.Tags, &m.Kind, &m.Port, &m.Model, &m.Quant, &m.PID, &m.RSS, &m.UptimeS,480		&m.DiskBytes, &m.Health, &m.HealthAt, &m.HealthMS, &m.Listed, &m.LastSeenAt, &m.GoneAt, &m.NodeSeen, &m.sortKeyText)481	return m, err482}483484// clMetricsSupport states what request-level metrics exist for a kind of485// inference server. The agent protocol carries none of them, so none is ever486// available; the reason says whether the server itself lacks the metric or487// whether it is simply not collected.488func clMetricsSupport(kind string) InferenceMetricsSupport {489	un := func(reason string) MetricSupport {490		return MetricSupport{Availability: AvailabilityUnsupported, Reason: &reason}491	}492	if kind == "mlx_lm" {493		return InferenceMetricsSupport{494			Requests:   un("mlx_lm.server exposes no request counter"),495			TokensPerS: un("mlx_lm.server exposes no throughput metric"),496			Ttft:       un("mlx_lm.server exposes no latency metric"),497			Latency:    un("mlx_lm.server exposes no latency metric"),498			Queue:      un("mlx_lm.server has no observable queue"),499		}500	}501	r := "not collected by the agent for " + kind + " servers"502	return InferenceMetricsSupport{Requests: un(r), TokensPerS: un(r), Ttft: un(r), Latency: un(r), Queue: un(r)}503}504505func (h *clusterHandlers) toModel(m *clModel, now time.Time) InferenceServer {506	out := InferenceServer{Id: m.ID, NodeId: m.NodeID, NodeAlias: m.NodeAlias, Kind: m.Kind, Port: m.Port, Model: clNonEmpty(m.Model),507		Quantization: clNonEmpty(m.Quant), Pid: m.PID, RssBytes: m.RSS, UptimeS: m.UptimeS, DiskBytes: m.DiskBytes, Health: InferenceHealthUnknown,508		HealthAt: m.HealthAt, HealthLatencyMs: clF32(m.HealthMS), ListedModels: []string{}, MetricsSupport: clMetricsSupport(m.Kind),509		LastSeenAt: m.LastSeenAt, GoneAt: m.GoneAt, ObservedAt: &m.LastSeenAt, Freshness: h.clFresh(now, m.NodeSeen)}510	if m.Health == "ok" || m.Health == "failing" {511		out.Health = InferenceHealth(m.Health)512	}513	if len(m.Listed) > 0 {514		var l []string515		if json.Unmarshal(m.Listed, &l) == nil && l != nil {516			out.ListedModels = l517		}518	}519	return out520}521522func (h *clusterHandlers) ListInferenceServers(ctx context.Context, req ListInferenceServersRequestObject) (ListInferenceServersResponseObject, error) {523	p := req.Params524	limit, err := clLimit(p.Limit)525	if err != nil {526		return nil, err527	}528	acc, err := h.clAccessFor(ctx)529	if err != nil {530		return nil, err531	}532	b := &clSQL{}533	b.and("n.retired_at is null")534	if p.NodeId != nil {535		b.and("i.node_id = " + b.arg(*p.NodeId))536	}537	if p.Health != nil {538		switch *p.Health {539		case InferenceHealthOk, InferenceHealthFailing:540			b.and("i.gone_at is null and i.health = " + b.arg(string(*p.Health)))541		case InferenceHealthUnknown:542			b.and("i.gone_at is null and i.health not in ('ok','failing')")543		default:544			return nil, clInvalid("health", "must be ok, failing or unknown")545		}546	}547	cur, err := clDecodeCursor(p.Cursor, "models", 3)548	if err != nil {549		return nil, err550	}551	orderBy, err := b.clKeyset(clModelKey, "text", "i.id", false, cur)552	if err != nil {553		return nil, err554	}555	rows, err := h.Pool.Query(ctx, `select `+clModelCols+`, `+clModelKey+clModelFrom+b.where()+orderBy+` limit `+b.arg(limit+1), b.args...)556	if err != nil {557		return nil, err558	}559	list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (clModel, error) { return clScanModel(r) })560	if err != nil {561		return nil, err562	}563	out := InferenceServerList{Items: make([]InferenceServer, 0, len(list))}564	if len(list) > limit {565		list = list[:limit]566		out.NextCursor = clEncodeCursor("models", "0", list[limit-1].sortKeyText, list[limit-1].ID.String())567	}568	now := time.Now().UTC()569	filter := acc.scoped(authz.ModelsRead)570	for i := range list {571		if filter && !acc.can(authz.ModelsRead, clNodeFacets(list[i].NodeID, list[i].Site, list[i].Tags)...) {572			continue573		}574		out.Items = append(out.Items, h.toModel(&list[i], now))575	}576	return ListInferenceServers200JSONResponse{Body: out}, nil577}578579func (h *clusterHandlers) GetInferenceServer(ctx context.Context, req GetInferenceServerRequestObject) (GetInferenceServerResponseObject, error) {580	acc, err := h.clAccessFor(ctx)581	if err != nil {582		return nil, err583	}584	m, err := clScanModel(h.Pool.QueryRow(ctx, `select `+clModelCols+`, `+clModelKey+clModelFrom+` where i.id = $1`, req.Id))585	if errors.Is(err, pgx.ErrNoRows) || (err == nil && !acc.can(authz.ModelsRead, clNodeFacets(m.NodeID, m.Site, m.Tags)...)) {586		return nil, httpx.ErrNotFound587	} else if err != nil {588		return nil, err589	}590	return GetInferenceServer200JSONResponse{Body: h.toModel(&m, time.Now().UTC())}, nil591}592
Files main
Menu

Theme