Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/maintenance/scheduler.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package maintenance4344import (45 "context"46 "errors"47 "time"4849 "github.com/google/uuid"50 "github.com/jackc/pgx/v5"51 "github.com/jackc/pgx/v5/pgxpool"5253 "clustr.spboucher.ai/mcc/internal/audit"54 "clustr.spboucher.ai/mcc/internal/events"55 "clustr.spboucher.ai/mcc/internal/ids"56)5758// Database part of the package: loading windows and applying occurrences.59//60// What an active occurrence changes, and how it is undone:61// - set_maintenance_mode: nodes in scope that are NOT already in maintenance62// are switched on, with a reason naming the window; each such node is63// recorded in maintenance_window_nodes. At the end only recorded nodes are64// switched off, and only if their reason is still the one the window65// wrote. A node put in maintenance by hand (before, or again during the66// window with another reason) therefore stays in maintenance. If another67// window in progress also covers the node, the record is handed over to68// that window instead, so the node leaves maintenance when the LAST69// covering window ends.70// - silence_alerts: one silence linked to the window for the occurrence; at71// the end its ends_at is brought back to now.72//73// State survives restarts: status, activated_occurrence and74// maintenance_window_nodes say exactly what is applied.7576// Columns is the column list read by Scan.77const Columns = `id, title, description, category, starts_at, ends_at, timezone, recurrence, recurrence_until, coalesce(scope->'node_ids','[]'::jsonb),78 set_maintenance_mode, silence_alerts, status, activated_occurrence, completed_occurrence, created_by, created_at, updated_at`7980// Scan reads Columns, then any extra destinations.81func Scan(row pgx.Row, extra ...any) (Window, error) {82 var w Window83 err := row.Scan(append([]any{&w.ID, &w.Title, &w.Description, &w.Category, &w.StartsAt, &w.EndsAt, &w.Timezone, &w.Recurrence, &w.RecurrenceUntil, &w.NodeIDs,84 &w.SetMaintenanceMode, &w.SilenceAlerts, &w.Status, &w.ActivatedOccurrence, &w.CompletedOccurrence, &w.CreatedBy, &w.CreatedAt, &w.UpdatedAt}, extra...)...)85 if w.NodeIDs == nil {86 w.NodeIDs = []uuid.UUID{}87 }88 return w, err89}9091// Reason is what the scheduler writes in nodes.maintenance_reason.92func Reason(w Window) string {93 t := w.Title94 if len(t) > 120 {95 t = t[:120]96 }97 return "Maintenance window: " + t + " [" + w.ID.String()[:8] + "]"98}99100func changed(ctx context.Context, tx pgx.Tx, change string, w Window, occ *time.Time) error {101 p := map[string]any{"change": change, "window_id": w.ID, "node_ids": w.NodeIDs}102 if occ != nil {103 p["occurrence"] = *occ104 }105 return events.Publish(ctx, tx, "maintenance.changed", p)106}107108// Changed publishes maintenance.changed for a window.109func Changed(ctx context.Context, tx pgx.Tx, change string, w Window, occ *time.Time) error {110 return changed(ctx, tx, change, w, occ)111}112113// Activate applies an occurrence: maintenance mode, silence, status.114func Activate(ctx context.Context, tx pgx.Tx, w Window, occ Occurrence, now time.Time) error {115 reason := Reason(w)116 if w.SetMaintenanceMode {117 for _, n := range w.NodeIDs {118 tag, err := tx.Exec(ctx, `update nodes set maintenance=true, maintenance_reason=$2 where id=$1 and not maintenance and retired_at is null`, n, reason)119 if err != nil {120 return err121 }122 if tag.RowsAffected() == 0 {123 continue // already in maintenance (by hand or by another window): not ours to undo124 }125 if _, err := tx.Exec(ctx, `insert into maintenance_window_nodes(window_id, node_id, reason, set_at) values ($1,$2,$3,$4)126 on conflict (window_id, node_id) do update set reason=excluded.reason, set_at=excluded.set_at`, w.ID, n, reason, now); err != nil {127 return err128 }129 if err := events.Publish(ctx, tx, "node.updated", map[string]any{"node_id": n, "change": "updated"}); err != nil {130 return err131 }132 }133 }134 if w.SilenceAlerts && len(w.NodeIDs) > 0 {135 if _, err := tx.Exec(ctx, `insert into silences(id, reason, scope, starts_at, ends_at, created_by, maintenance_window_id, created_at)136 values ($1,$2,jsonb_build_object('node_ids',$3::uuid[]),$4,$5,$6,$7,$8)`, ids.New(), reason, w.NodeIDs, occ.Start, occ.End, w.CreatedBy, w.ID, now); err != nil {137 return err138 }139 }140 if _, err := tx.Exec(ctx, `update maintenance_windows set status='in_progress', activated_occurrence=$2, updated_at=$3 where id=$1`, w.ID, occ.Start, now); err != nil {141 return err142 }143 return changed(ctx, tx, "occurrence_started", w, &occ.Start)144}145146// Deactivate undoes what the active occurrence applied and sets the new147// status. It reverts ONLY what this window set (see the package comment).148func Deactivate(ctx context.Context, tx pgx.Tx, w Window, status string, completed *time.Time, now time.Time) error {149 rows, err := tx.Query(ctx, `select node_id, reason from maintenance_window_nodes where window_id=$1`, w.ID)150 if err != nil {151 return err152 }153 type applied struct {154 node uuid.UUID155 reason string156 }157 list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (applied, error) {158 var a applied159 return a, r.Scan(&a.node, &a.reason)160 })161 if err != nil {162 return err163 }164 for _, a := range list {165 var heir uuid.UUID166 err := tx.QueryRow(ctx, `select id from maintenance_windows where id <> $1 and status='in_progress' and set_maintenance_mode167 and scope->'node_ids' ? $2 order by id limit 1`, w.ID, a.node.String()).Scan(&heir)168 switch {169 case err == nil:170 // Another window in progress covers this node: it inherits the duty to switch it off.171 if _, err := tx.Exec(ctx, `insert into maintenance_window_nodes(window_id, node_id, reason, set_at) values ($1,$2,$3,$4)172 on conflict (window_id, node_id) do nothing`, heir, a.node, a.reason, now); err != nil {173 return err174 }175 case errors.Is(err, pgx.ErrNoRows):176 tag, err := tx.Exec(ctx, `update nodes set maintenance=false, maintenance_reason='' where id=$1 and maintenance and maintenance_reason=$2`, a.node, a.reason)177 if err != nil {178 return err179 }180 if tag.RowsAffected() == 1 {181 if err := events.Publish(ctx, tx, "node.updated", map[string]any{"node_id": a.node, "change": "updated"}); err != nil {182 return err183 }184 }185 default:186 return err187 }188 }189 if _, err := tx.Exec(ctx, `delete from maintenance_window_nodes where window_id=$1`, w.ID); err != nil {190 return err191 }192 if _, err := tx.Exec(ctx, `update silences set ends_at=greatest(starts_at, $2) where maintenance_window_id=$1 and ends_at > $2`, w.ID, now); err != nil {193 return err194 }195 _, err = tx.Exec(ctx, `update maintenance_windows set status=$2, activated_occurrence=null, completed_occurrence=coalesce($3, completed_occurrence), updated_at=$4 where id=$1`,196 w.ID, status, completed, now)197 return err198}199200// StatusAfter is the status of a window once the occurrence starting at201// `occ` is over: back to scheduled when another occurrence follows, else completed.202func StatusAfter(w Window, occ time.Time) string {203 if _, more := Next(w, occ); more {204 return StatusScheduled205 }206 return StatusCompleted207}208209// Scheduler starts and ends occurrences.210type Scheduler struct {211 Pool *pgxpool.Pool212 Now func() time.Time213}214215func (s *Scheduler) now() time.Time {216 if s.Now != nil {217 return s.Now()218 }219 return time.Now().UTC()220}221222// Tick brings every window in line with the clock. Idempotent.223func (s *Scheduler) Tick(ctx context.Context) error {224 now := s.now()225 rows, err := s.Pool.Query(ctx, `select id from maintenance_windows where status in ('scheduled','in_progress') and starts_at <= $1 order by starts_at, id`, now)226 if err != nil {227 return err228 }229 list, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])230 if err != nil {231 return err232 }233 // Ends first, then starts: a node handed from one window to the next never flickers out of maintenance... and234 // a window that ends while another starts sees the other as not yet in progress, which is the conservative order.235 for _, phase := range []string{StatusInProgress, StatusScheduled} {236 for _, id := range list {237 if err := s.advance(ctx, id, phase, now); err != nil {238 return err239 }240 }241 }242 return nil243}244245func (s *Scheduler) advance(ctx context.Context, id uuid.UUID, phase string, now time.Time) error {246 return pgx.BeginFunc(ctx, s.Pool, func(tx pgx.Tx) error {247 w, err := Scan(tx.QueryRow(ctx, `select `+Columns+` from maintenance_windows where id=$1 and status=$2 for update skip locked`, id, phase))248 if errors.Is(err, pgx.ErrNoRows) {249 return nil250 }251 if err != nil {252 return err253 }254 sys := func(action, detail string, occ time.Time) error {255 _, err := audit.Append(ctx, tx, audit.Entry{ActorKind: "system", ActorLabel: "maintenance-scheduler", Action: action, TargetKind: "maintenance_window",256 TargetID: w.ID.String(), Outcome: audit.Success, Detail: detail, Params: map[string]any{"occurrence": occ, "nodes": len(w.NodeIDs)}})257 return err258 }259 switch w.Status {260 case StatusInProgress:261 if w.ActivatedOccurrence == nil { // inconsistent row: repair by ending it262 return Deactivate(ctx, tx, w, StatusAfter(w, now), nil, now)263 }264 occ, ok := OccurrenceStarting(w, *w.ActivatedOccurrence)265 if ok && now.Before(occ.End) {266 return nil267 }268 if err := Deactivate(ctx, tx, w, StatusAfter(w, *w.ActivatedOccurrence), nil, now); err != nil {269 return err270 }271 if err := changed(ctx, tx, "occurrence_ended", w, w.ActivatedOccurrence); err != nil {272 return err273 }274 return sys("maintenance.occurrence_end", "", *w.ActivatedOccurrence)275 case StatusScheduled:276 occ, ok := OccurrenceAt(w, now)277 if ok && (w.CompletedOccurrence == nil || !w.CompletedOccurrence.Equal(occ.Start)) {278 if err := Activate(ctx, tx, w, occ, now); err != nil {279 return err280 }281 return sys("maintenance.occurrence_start", "", occ.Start)282 }283 if _, more := Next(w, now); !ok && !more {284 // Nothing in progress and nothing to come: the series is over (possibly missed while the scheduler was down).285 if _, err := tx.Exec(ctx, `update maintenance_windows set status='completed', updated_at=$2 where id=$1`, w.ID, now); err != nil {286 return err287 }288 if err := changed(ctx, tx, "window_completed", w, nil); err != nil {289 return err290 }291 return sys("maintenance.window_end", "series over without an occurrence in progress", now)292 }293 }294 return nil295 })296}297