Repository Public
1package api23import (4 "context"5 "encoding/json"6 "errors"7 "net/url"8 "slices"9 "strconv"10 "strings"11 "time"1213 "github.com/google/uuid"14 "github.com/jackc/pgx/v5"1516 "clustr.spboucher.ai/mcc/internal/audit"17 "clustr.spboucher.ai/mcc/internal/authz"18 "clustr.spboucher.ai/mcc/internal/events"19 "clustr.spboucher.ai/mcc/internal/httpx"20)2122// ── Services ───────────────────────────────────────────────────────────────2324const clServiceCols = `sv.id, sv.node_id, n.alias::text, n.site, n.tags, sv.manager::text, sv.name, sv.state, sv.pid, sv.restarts, sv.mem_bytes,25 sv.cpu_pct::float8, sv.uptime_s, sv.version, sv.ports, sv.app, sv.public_url, sv.health, sv.managed, sv.meta, sv.first_seen_at, sv.last_seen_at,26 sv.gone_at, greatest(s.received_at, n.last_seen_at)`2728const clServiceFrom = ` from services sv join nodes n on n.id = sv.node_id left join node_state s on s.node_id = n.id `2930type clService struct {31 ID, NodeID uuid.UUID32 NodeAlias string33 Site string34 Tags []string35 Manager string36 Name string37 State string38 PID *int39 Restarts *int40 MemBytes *int6441 CPU *float6442 UptimeS *int6443 Version *string44 Ports []int3245 App *string46 PublicURL *string47 Health []byte48 Managed bool49 Meta []byte50 FirstSeenAt time.Time51 LastSeenAt time.Time52 GoneAt *time.Time53 NodeSeen *time.Time54}5556func clScanService(row pgx.Row, extra ...any) (clService, error) {57 var s clService58 dst := append([]any{&s.ID, &s.NodeID, &s.NodeAlias, &s.Site, &s.Tags, &s.Manager, &s.Name, &s.State, &s.PID, &s.Restarts, &s.MemBytes,59 &s.CPU, &s.UptimeS, &s.Version, &s.Ports, &s.App, &s.PublicURL, &s.Health, &s.Managed, &s.Meta, &s.FirstSeenAt, &s.LastSeenAt,60 &s.GoneAt, &s.NodeSeen}, extra...)61 return s, row.Scan(dst...)62}6364func (s *clService) facets() []authz.Resource { return clServiceFacets(s.ID, s.NodeID, s.Site, s.Tags) }6566// clServiceState maps the stored word to the contract vocabulary; anything67// else is unknown (the raw manager state stays in meta).68func clServiceState(s string) ServiceState {69 switch ServiceState(s) {70 case ServiceStateOnline, ServiceStateStopped, ServiceStateErrored:71 return ServiceState(s)72 }73 return ServiceStateUnknown74}7576// toService renders a service. The state is the last one observed: gone_at77// (no longer listed by its manager) and freshness (age of the node's last78// report) tell how much it can be trusted.79func (h *clusterHandlers) toService(s *clService, now time.Time, withMeta bool) Service {80 out := Service{Id: s.ID, NodeId: s.NodeID, NodeAlias: s.NodeAlias, Manager: ServiceManager(s.Manager), Name: s.Name, State: clServiceState(s.State),81 Pid: s.PID, Restarts: s.Restarts, MemBytes: s.MemBytes, CpuPct: clF32(s.CPU), UptimeS: s.UptimeS, Version: clNonEmpty(s.Version),82 Ports: make([]int, 0, len(s.Ports)), App: clNonEmpty(s.App), PublicUrl: clNonEmpty(s.PublicURL), Managed: s.Managed,83 FirstSeenAt: s.FirstSeenAt, LastSeenAt: s.LastSeenAt, GoneAt: s.GoneAt, ObservedAt: &s.LastSeenAt, Freshness: h.clFresh(now, s.NodeSeen)}84 for _, p := range s.Ports {85 out.Ports = append(out.Ports, int(p))86 }87 if len(s.Health) > 0 && string(s.Health) != "null" {88 var hv struct {89 Status string `json:"status"`90 CheckedAt *time.Time `json:"checked_at"`91 LatencyMS *float32 `json:"latency_ms"`92 Detail string `json:"detail"`93 }94 if json.Unmarshal(s.Health, &hv) == nil {95 sh := ServiceHealth{Status: ServiceHealthStatusUnknown, CheckedAt: hv.CheckedAt, LatencyMs: hv.LatencyMS, Detail: clNonEmpty(&hv.Detail)}96 if hv.Status == "ok" || hv.Status == "failing" {97 sh.Status = ServiceHealthStatus(hv.Status)98 }99 out.Health = &sh100 }101 }102 if withMeta {103 meta := audit.Redact(clObject(s.Meta)) // second line of defence: the agent already redacts104 out.Meta = &meta105 }106 return out107}108109func (h *clusterHandlers) listServices(ctx context.Context, acc *clAccess, p ListServicesParams, cursorKind string) (ServiceList, error) {110 var out ServiceList111 limit, err := clLimit(p.Limit)112 if err != nil {113 return out, err114 }115 q, err := clQuery(p.Q)116 if err != nil {117 return out, err118 }119 b := &clSQL{}120 b.and("n.retired_at is null")121 if q != "" {122 a := b.arg(clLike(q))123 b.and(`(sv.name ilike ` + a + ` escape '\' or coalesce(sv.app,'') ilike ` + a + ` escape '\')`)124 }125 if p.NodeId != nil {126 b.and("sv.node_id = " + b.arg(*p.NodeId))127 }128 if p.Manager != nil {129 if !slices.Contains([]ServiceManager{ServiceManagerPm2, ServiceManagerLaunchd, ServiceManagerSystemd, ServiceManagerDocker}, *p.Manager) {130 return out, clInvalid("manager", "must be pm2, launchd, systemd or docker")131 }132 b.and("sv.manager = " + b.arg(string(*p.Manager)) + "::service_manager")133 }134 if p.State != nil {135 switch *p.State {136 case ServiceStateOnline, ServiceStateStopped, ServiceStateErrored:137 // A service that is gone is not in any current state.138 b.and("sv.gone_at is null and sv.state = " + b.arg(string(*p.State)))139 case ServiceStateUnknown:140 b.and("sv.gone_at is null and sv.state not in ('online','stopped','errored')")141 default:142 return out, clInvalid("state", "must be online, stopped, errored or unknown")143 }144 }145 if p.App != nil {146 if len(*p.App) > 120 {147 return out, clInvalid("app", "must be at most 120 characters")148 }149 b.and("lower(sv.app) = lower(" + b.arg(*p.App) + ")")150 }151 if p.Managed != nil {152 b.and("sv.managed = " + b.arg(*p.Managed))153 }154 cur, err := clDecodeCursor(p.Cursor, cursorKind, 3)155 if err != nil {156 return out, err157 }158 orderBy, err := b.clKeyset("lower(sv.name)", "text", "sv.id", false, cur)159 if err != nil {160 return out, err161 }162 // The sort key is read back from the database: the cursor must compare exactly as the ORDER BY does.163 rows, err := h.Pool.Query(ctx, `select `+clServiceCols+`, lower(sv.name)`+clServiceFrom+b.where()+orderBy+` limit `+b.arg(limit+1), b.args...)164 if err != nil {165 return out, err166 }167 var keys []string168 list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (clService, error) {169 var key string170 s, err := clScanService(r, &key)171 keys = append(keys, key)172 return s, err173 })174 if err != nil {175 return out, err176 }177 out.Items = make([]Service, 0, len(list))178 if len(list) > limit {179 list = list[:limit]180 out.NextCursor = clEncodeCursor(cursorKind, "0", keys[limit-1], list[limit-1].ID.String())181 }182 now := time.Now().UTC()183 filter := acc.scoped(authz.ServicesRead)184 for i := range list {185 if filter && !acc.can(authz.ServicesRead, list[i].facets()...) {186 continue187 }188 out.Items = append(out.Items, h.toService(&list[i], now, false))189 }190 return out, nil191}192193func (h *clusterHandlers) ListServices(ctx context.Context, req ListServicesRequestObject) (ListServicesResponseObject, error) {194 acc, err := h.clAccessFor(ctx)195 if err != nil {196 return nil, err197 }198 list, err := h.listServices(ctx, acc, req.Params, "services")199 if err != nil {200 return nil, err201 }202 return ListServices200JSONResponse{Body: list}, nil203}204205func (h *clusterHandlers) serviceByID(ctx context.Context, q clQuerier, id uuid.UUID, lock string) (clService, error) {206 s, err := clScanService(q.QueryRow(ctx, `select `+clServiceCols+clServiceFrom+` where sv.id = $1`+lock, id))207 if errors.Is(err, pgx.ErrNoRows) {208 return s, httpx.ErrNotFound209 }210 return s, err211}212213func (h *clusterHandlers) GetService(ctx context.Context, req GetServiceRequestObject) (GetServiceResponseObject, error) {214 acc, err := h.clAccessFor(ctx)215 if err != nil {216 return nil, err217 }218 s, err := h.serviceByID(ctx, h.Pool, req.Id, "")219 if err != nil {220 return nil, err221 }222 if !acc.can(authz.ServicesRead, s.facets()...) {223 return nil, httpx.ErrNotFound224 }225 return GetService200JSONResponse{Body: h.toService(&s, time.Now().UTC(), true)}, nil226}227228// clPublicURL validates the public URL of a service: an absolute http(s) URL229// without credentials. The empty string clears it.230func clPublicURL(v string) (*string, error) {231 v = strings.TrimSpace(v)232 if v == "" {233 return nil, nil234 }235 u, err := url.Parse(v)236 switch {237 case len(v) > 300:238 return nil, clInvalid("public_url", "must be at most 300 characters")239 case err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || !clPrintable(v):240 return nil, clInvalid("public_url", "must be an absolute http or https URL")241 case u.User != nil:242 return nil, clInvalid("public_url", "must not contain credentials")243 }244 return &v, nil245}246247func (h *clusterHandlers) UpdateService(ctx context.Context, req UpdateServiceRequestObject) (UpdateServiceResponseObject, error) {248 if req.Body == nil {249 return nil, httpx.Validation("Body required.")250 }251 if req.Body.Managed == nil && req.Body.PublicUrl == nil {252 return nil, httpx.Validation("Nothing to update: give managed or public_url.")253 }254 var newURL *string255 if req.Body.PublicUrl != nil {256 u, err := clPublicURL(*req.Body.PublicUrl)257 if err != nil {258 return nil, err259 }260 newURL = u261 }262 acc, err := h.clAccessFor(ctx)263 if err != nil {264 return nil, err265 }266 tx, err := h.Pool.Begin(ctx)267 if err != nil {268 return nil, err269 }270 defer tx.Rollback(ctx)271 s, err := h.serviceByID(ctx, tx, req.Id, " for update of sv")272 if err != nil {273 return nil, err274 }275 if !acc.can(authz.ServicesRead, s.facets()...) {276 return nil, httpx.ErrNotFound277 }278 if err := h.clAuthorizeWrite(ctx, acc, authz.SettingsManage, authz.Resource{Kind: "service", ID: s.ID.String()}, nil); err != nil {279 return nil, err280 }281 changes := map[string]any{}282 managed := s.Managed283 if req.Body.Managed != nil && *req.Body.Managed != s.Managed {284 managed = *req.Body.Managed285 changes["managed"] = map[string]any{"from": s.Managed, "to": managed}286 }287 pub := s.PublicURL288 if req.Body.PublicUrl != nil {289 old, next := "", ""290 if s.PublicURL != nil {291 old = *s.PublicURL292 }293 if newURL != nil {294 next = *newURL295 }296 if old != next {297 pub = newURL298 changes["public_url"] = map[string]any{"from": old, "to": next}299 }300 }301 if len(changes) > 0 {302 if _, err := tx.Exec(ctx, `update services set managed = $2, public_url = $3 where id = $1`, s.ID, managed, pub); err != nil {303 return nil, err304 }305 if err := h.audit(ctx, tx, "service.update", "service", s.ID.String(), audit.Success, "",306 map[string]any{"node_id": s.NodeID.String(), "manager": s.Manager, "name": s.Name, "changes": changes}); err != nil {307 return nil, err308 }309 ev := ServiceChangedEvent{ServiceId: s.ID, NodeId: s.NodeID, Manager: ServiceManager(s.Manager), Name: s.Name, State: clServiceState(s.State)}310 if s.GoneAt != nil {311 ev.Gone = ptr(true)312 }313 if err := events.Publish(ctx, tx, "service.changed", ev); err != nil {314 return nil, err315 }316 if err := tx.Commit(ctx); err != nil {317 return nil, err318 }319 s.Managed, s.PublicURL = managed, pub320 }321 return UpdateService200JSONResponse{Body: h.toService(&s, time.Now().UTC(), true)}, nil322}323324// GetServiceHistory is the timeline of a service, built only from recorded325// facts: the changes observed by ingestion (service_events, written by a326// trigger on the services table) and the actions that targeted the service.327// Action entries are included only for callers who may read actions.328func (h *clusterHandlers) GetServiceHistory(ctx context.Context, req GetServiceHistoryRequestObject) (GetServiceHistoryResponseObject, error) {329 limit, err := clLimit(req.Params.Limit)330 if err != nil {331 return nil, err332 }333 acc, err := h.clAccessFor(ctx)334 if err != nil {335 return nil, err336 }337 s, err := h.serviceByID(ctx, h.Pool, req.Id, "")338 if err != nil {339 return nil, err340 }341 if !acc.can(authz.ServicesRead, s.facets()...) {342 return nil, httpx.ErrNotFound343 }344 kind := "service-history:" + s.ID.String()345 cur, err := clDecodeCursor(req.Params.Cursor, kind, 2)346 if err != nil {347 return nil, err348 }349 var afterTS *time.Time350 afterKey := ""351 if cur != nil {352 t, err := clCursorTime(cur[0])353 if err != nil {354 return nil, err355 }356 afterTS, afterKey = &t, cur[1]357 }358 rows, err := h.Pool.Query(ctx, `select h.ts, h.kind, h.from_state, h.to_state, h.restarts_from, h.restarts_to, h.version_from, h.version_to,359 h.action_id, h.action_type, h.action_state, h.actor, h.key360 from (361 select e.ts, e.kind, e.from_state, e.to_state, e.restarts_from, e.restarts_to, e.version_from, e.version_to,362 null::uuid as action_id, null::text as action_type, null::text as action_state, null::text as actor,363 'e' || lpad(e.id::text, 19, '0') as key364 from service_events e where e.service_id = $1365 union all366 select a.created_at, 'action', null, null, null, null, null, null, a.id, a.type, a.state::text, u.display_name, 'a' || a.id::text367 from actions a left join users u on u.id = a.requested_by368 where $3::boolean and a.target_kind = 'service' and a.target_id = $2369 ) h370 where ($4::timestamptz is null or (h.ts, h.key) < ($4::timestamptz, $5::text))371 order by h.ts desc, h.key desc limit $6`,372 s.ID, s.ID.String(), acc.can(authz.ActionsRead, s.facets()...), afterTS, afterKey, limit+1)373 if err != nil {374 return nil, err375 }376 type entry struct {377 e ServiceHistoryEntry378 key string379 }380 list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (entry, error) {381 var (382 x entry383 kind string384 from, to, vFrom, vTo *string385 rFrom, rTo *int386 actionType, actionState, a *string387 )388 if err := r.Scan(&x.e.Ts, &kind, &from, &to, &rFrom, &rTo, &vFrom, &vTo, &x.e.ActionId, &actionType, &actionState, &a, &x.key); err != nil {389 return x, err390 }391 state := func(p *string) *ServiceState {392 if p == nil {393 return nil394 }395 return ptr(clServiceState(*p))396 }397 x.e.Kind = ServiceHistoryEntryKind(kind)398 switch kind {399 case "appeared":400 x.e.Summary, x.e.ToState, x.e.Version = "Service first listed by "+s.Manager+".", state(to), clNonEmpty(vTo)401 case "disappeared":402 x.e.Summary, x.e.FromState = "Service no longer listed by "+s.Manager+".", state(from)403 case "state_change":404 x.e.FromState, x.e.ToState = state(from), state(to)405 x.e.Summary = "State changed from " + clDeref(from, "unknown") + " to " + clDeref(to, "unknown") + "."406 case "restart":407 x.e.Summary = "Restart counter went from " + clItoaPtr(rFrom) + " to " + clItoaPtr(rTo) + "."408 case "deploy":409 x.e.Summary, x.e.Version = "Version changed from "+clDeref(vFrom, "?")+" to "+clDeref(vTo, "?")+".", clNonEmpty(vTo)410 case "action":411 x.e.Summary, x.e.ActorDisplayName = "Action "+clDeref(actionType, "")+": "+clDeref(actionState, "")+".", clNonEmpty(a)412 default:413 x.e.Kind, x.e.Summary = ServiceHistoryEntryKindStateChange, "Recorded change."414 }415 return x, nil416 })417 if err != nil {418 return nil, err419 }420 out := ServiceHistory{Items: make([]ServiceHistoryEntry, 0, len(list))}421 if len(list) > limit {422 list = list[:limit]423 out.NextCursor = clEncodeCursor(kind, clTimeKey(list[limit-1].e.Ts), list[limit-1].key)424 }425 for _, x := range list {426 out.Items = append(out.Items, x.e)427 }428 return GetServiceHistory200JSONResponse{Body: out}, nil429}430431func clDeref(p *string, def string) string {432 if p == nil || *p == "" {433 return def434 }435 return *p436}437438func clItoaPtr(p *int) string {439 if p == nil {440 return "?"441 }442 return strconv.Itoa(*p)443}444445// ── Inference servers ──────────────────────────────────────────────────────446447const clModelCols = `i.id, i.node_id, n.alias::text, n.site, n.tags, i.kind, i.port, i.model, i.quantization, i.pid, i.rss_bytes, i.uptime_s,448 i.disk_bytes, i.health, i.health_at, i.health_latency_ms::float8, i.listed_models, i.last_seen_at, i.gone_at, greatest(s.received_at, n.last_seen_at)`449450const clModelFrom = ` from inference_servers i join nodes n on n.id = i.node_id left join node_state s on s.node_id = n.id `451452const clModelKey = `(lower(n.alias::text) || ' ' || lpad(i.port::text, 5, '0'))`453454type clModel struct {455 ID, NodeID uuid.UUID456 NodeAlias string457 Site string458 Tags []string459 Kind string460 Port int461 Model *string462 Quant *string463 PID *int464 RSS *int64465 UptimeS *int64466 DiskBytes *int64467 Health string468 HealthAt *time.Time469 HealthMS *float64470 Listed []byte471 LastSeenAt time.Time472 GoneAt *time.Time473 NodeSeen *time.Time474 sortKeyText string475}476477func clScanModel(row pgx.Row) (clModel, error) {478 var m clModel479 err := row.Scan(&m.ID, &m.NodeID, &m.NodeAlias, &m.Site, &m.Tags, &m.Kind, &m.Port, &m.Model, &m.Quant, &m.PID, &m.RSS, &m.UptimeS,480 &m.DiskBytes, &m.Health, &m.HealthAt, &m.HealthMS, &m.Listed, &m.LastSeenAt, &m.GoneAt, &m.NodeSeen, &m.sortKeyText)481 return m, err482}483484// clMetricsSupport states what request-level metrics exist for a kind of485// inference server. The agent protocol carries none of them, so none is ever486// available; the reason says whether the server itself lacks the metric or487// whether it is simply not collected.488func clMetricsSupport(kind string) InferenceMetricsSupport {489 un := func(reason string) MetricSupport {490 return MetricSupport{Availability: AvailabilityUnsupported, Reason: &reason}491 }492 if kind == "mlx_lm" {493 return InferenceMetricsSupport{494 Requests: un("mlx_lm.server exposes no request counter"),495 TokensPerS: un("mlx_lm.server exposes no throughput metric"),496 Ttft: un("mlx_lm.server exposes no latency metric"),497 Latency: un("mlx_lm.server exposes no latency metric"),498 Queue: un("mlx_lm.server has no observable queue"),499 }500 }501 r := "not collected by the agent for " + kind + " servers"502 return InferenceMetricsSupport{Requests: un(r), TokensPerS: un(r), Ttft: un(r), Latency: un(r), Queue: un(r)}503}504505func (h *clusterHandlers) toModel(m *clModel, now time.Time) InferenceServer {506 out := InferenceServer{Id: m.ID, NodeId: m.NodeID, NodeAlias: m.NodeAlias, Kind: m.Kind, Port: m.Port, Model: clNonEmpty(m.Model),507 Quantization: clNonEmpty(m.Quant), Pid: m.PID, RssBytes: m.RSS, UptimeS: m.UptimeS, DiskBytes: m.DiskBytes, Health: InferenceHealthUnknown,508 HealthAt: m.HealthAt, HealthLatencyMs: clF32(m.HealthMS), ListedModels: []string{}, MetricsSupport: clMetricsSupport(m.Kind),509 LastSeenAt: m.LastSeenAt, GoneAt: m.GoneAt, ObservedAt: &m.LastSeenAt, Freshness: h.clFresh(now, m.NodeSeen)}510 if m.Health == "ok" || m.Health == "failing" {511 out.Health = InferenceHealth(m.Health)512 }513 if len(m.Listed) > 0 {514 var l []string515 if json.Unmarshal(m.Listed, &l) == nil && l != nil {516 out.ListedModels = l517 }518 }519 return out520}521522func (h *clusterHandlers) ListInferenceServers(ctx context.Context, req ListInferenceServersRequestObject) (ListInferenceServersResponseObject, error) {523 p := req.Params524 limit, err := clLimit(p.Limit)525 if err != nil {526 return nil, err527 }528 acc, err := h.clAccessFor(ctx)529 if err != nil {530 return nil, err531 }532 b := &clSQL{}533 b.and("n.retired_at is null")534 if p.NodeId != nil {535 b.and("i.node_id = " + b.arg(*p.NodeId))536 }537 if p.Health != nil {538 switch *p.Health {539 case InferenceHealthOk, InferenceHealthFailing:540 b.and("i.gone_at is null and i.health = " + b.arg(string(*p.Health)))541 case InferenceHealthUnknown:542 b.and("i.gone_at is null and i.health not in ('ok','failing')")543 default:544 return nil, clInvalid("health", "must be ok, failing or unknown")545 }546 }547 cur, err := clDecodeCursor(p.Cursor, "models", 3)548 if err != nil {549 return nil, err550 }551 orderBy, err := b.clKeyset(clModelKey, "text", "i.id", false, cur)552 if err != nil {553 return nil, err554 }555 rows, err := h.Pool.Query(ctx, `select `+clModelCols+`, `+clModelKey+clModelFrom+b.where()+orderBy+` limit `+b.arg(limit+1), b.args...)556 if err != nil {557 return nil, err558 }559 list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (clModel, error) { return clScanModel(r) })560 if err != nil {561 return nil, err562 }563 out := InferenceServerList{Items: make([]InferenceServer, 0, len(list))}564 if len(list) > limit {565 list = list[:limit]566 out.NextCursor = clEncodeCursor("models", "0", list[limit-1].sortKeyText, list[limit-1].ID.String())567 }568 now := time.Now().UTC()569 filter := acc.scoped(authz.ModelsRead)570 for i := range list {571 if filter && !acc.can(authz.ModelsRead, clNodeFacets(list[i].NodeID, list[i].Site, list[i].Tags)...) {572 continue573 }574 out.Items = append(out.Items, h.toModel(&list[i], now))575 }576 return ListInferenceServers200JSONResponse{Body: out}, nil577}578579func (h *clusterHandlers) GetInferenceServer(ctx context.Context, req GetInferenceServerRequestObject) (GetInferenceServerResponseObject, error) {580 acc, err := h.clAccessFor(ctx)581 if err != nil {582 return nil, err583 }584 m, err := clScanModel(h.Pool.QueryRow(ctx, `select `+clModelCols+`, `+clModelKey+clModelFrom+` where i.id = $1`, req.Id))585 if errors.Is(err, pgx.ErrNoRows) || (err == nil && !acc.can(authz.ModelsRead, clNodeFacets(m.NodeID, m.Site, m.Tags)...)) {586 return nil, httpx.ErrNotFound587 } else if err != nil {588 return nil, err589 }590 return GetInferenceServer200JSONResponse{Body: h.toModel(&m, time.Now().UTC())}, nil591}592