Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/maintenance/scheduler_test.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package maintenance_test4344import (45 "context"46 "testing"47 "time"4849 "github.com/google/uuid"50 "github.com/jackc/pgx/v5/pgxpool"5152 "clustr.spboucher.ai/mcc/internal/ids"53 mt "clustr.spboucher.ai/mcc/internal/maintenance"54 "clustr.spboucher.ai/mcc/internal/testdb"55)5657var bg = context.Background()5859type senv struct {60 t *testing.T61 pool *pgxpool.Pool62 clock *testdb.Clock63}6465func newSenv(t *testing.T) *senv {66 t.Helper()67 pool, _ := testdb.New(t)68 e := &senv{t: t, pool: pool, clock: testdb.NewClock()}69 e.clock.Set(time.Date(2026, 11, 20, 15, 0, 0, 0, time.UTC))70 return e71}7273// tick runs a brand-new scheduler each time: nothing may depend on process memory.74func (e *senv) tick() {75 e.t.Helper()76 if err := (&mt.Scheduler{Pool: e.pool, Now: e.clock.Now}).Tick(bg); err != nil {77 e.t.Fatal(err)78 }79}8081type wopt func(*mt.Window)8283func (e *senv) window(title string, start time.Time, dur time.Duration, nodes []uuid.UUID, opts ...wopt) uuid.UUID {84 e.t.Helper()85 w := mt.Window{ID: ids.New(), Title: title, Category: "planned", StartsAt: start, EndsAt: start.Add(dur), Timezone: "America/Toronto", Recurrence: "none",86 NodeIDs: nodes, SetMaintenanceMode: true, SilenceAlerts: true}87 for _, o := range opts {88 o(&w)89 }90 if err := mt.Validate(w); err != nil {91 e.t.Fatal(err)92 }93 if _, err := e.pool.Exec(bg, `insert into maintenance_windows(id, title, category, starts_at, ends_at, timezone, recurrence, recurrence_until, scope, set_maintenance_mode, silence_alerts)94 values ($1,$2,$3,$4,$5,$6,$7,$8,jsonb_build_object('node_ids',$9::uuid[]),$10,$11)`, w.ID, w.Title, w.Category, w.StartsAt, w.EndsAt, w.Timezone, w.Recurrence,95 w.RecurrenceUntil, w.NodeIDs, w.SetMaintenanceMode, w.SilenceAlerts); err != nil {96 e.t.Fatal(err)97 }98 return w.ID99}100101func (e *senv) maint(node uuid.UUID) (bool, string) {102 e.t.Helper()103 var m bool104 var r string105 if err := e.pool.QueryRow(bg, `select maintenance, maintenance_reason from nodes where id=$1`, node).Scan(&m, &r); err != nil {106 e.t.Fatal(err)107 }108 return m, r109}110111func (e *senv) status(w uuid.UUID) (string, *time.Time) {112 e.t.Helper()113 var s string114 var occ *time.Time115 if err := e.pool.QueryRow(bg, `select status, activated_occurrence from maintenance_windows where id=$1`, w).Scan(&s, &occ); err != nil {116 e.t.Fatal(err)117 }118 return s, occ119}120121func (e *senv) count(sql string, args ...any) int {122 e.t.Helper()123 var n int124 if err := e.pool.QueryRow(bg, sql, args...).Scan(&n); err != nil {125 e.t.Fatal(err)126 }127 return n128}129130func (e *senv) activeSilences(w uuid.UUID) int {131 return e.count(`select count(*) from silences where maintenance_window_id=$1 and starts_at <= $2 and ends_at > $2`, w, e.clock.Now())132}133134func TestSchedulerActivatesAndRevertsOnlyWhatItSet(t *testing.T) {135 e := newSenv(t)136 a, b, manual, retagged := testdb.Node(t, e.pool, "T-A"), testdb.Node(t, e.pool, "T-B"), testdb.Node(t, e.pool, "T-MANUAL"), testdb.Node(t, e.pool, "T-RETAGGED")137 outside := testdb.Node(t, e.pool, "T-OUTSIDE")138 e.pool.Exec(bg, `update nodes set maintenance=true, maintenance_reason='bad RAM, waiting for parts' where id=$1`, manual)139 start := e.clock.Now().Add(10 * time.Minute)140 w := e.window("Firmware updates", start, time.Hour, []uuid.UUID{a, b, manual, retagged})141142 e.tick()143 if m, _ := e.maint(a); m {144 t.Fatal("activated before the start")145 }146 if s, _ := e.status(w); s != "scheduled" {147 t.Fatalf("status %s", s)148 }149 e.clock.Set(start)150 e.tick()151 s, occ := e.status(w)152 if s != "in_progress" || occ == nil || !occ.Equal(start) {153 t.Fatalf("status %s occ %v", s, occ)154 }155 for _, n := range []uuid.UUID{a, b, retagged} {156 if m, r := e.maint(n); !m || r == "" {157 t.Fatalf("node not in maintenance: %v %q", m, r)158 }159 }160 if m, _ := e.maint(outside); m {161 t.Fatal("a node outside the scope was touched")162 }163 if _, r := e.maint(manual); r != "bad RAM, waiting for parts" {164 t.Fatalf("manual reason overwritten: %q", r)165 }166 if e.activeSilences(w) != 1 || e.count(`select count(*) from maintenance_window_nodes where window_id=$1`, w) != 3 {167 t.Fatal("silence or applied-node records missing")168 }169 var scope string170 var ends time.Time171 e.pool.QueryRow(bg, `select scope::text, ends_at from silences where maintenance_window_id=$1`, w).Scan(&scope, &ends)172 if !ends.Equal(start.Add(time.Hour)) || len(scope) < 40 {173 t.Fatalf("silence: %s until %s", scope, ends)174 }175 // Idempotent, also across "restarts" in the middle of the window.176 for i := 0; i < 3; i++ {177 e.clock.Advance(10 * time.Minute)178 e.tick()179 }180 if e.count(`select count(*) from silences where maintenance_window_id=$1`, w) != 1 || e.count(`select count(*) from audit_log where action='maintenance.occurrence_start'`) != 1 {181 t.Fatal("occurrence applied more than once")182 }183 // During the window somebody takes over one node by hand, with their own reason.184 e.pool.Exec(bg, `update nodes set maintenance_reason='keep offline: disk replacement tomorrow' where id=$1`, retagged)185186 e.clock.Set(start.Add(time.Hour))187 e.tick()188 if s, occ := e.status(w); s != "completed" || occ != nil {189 t.Fatalf("status %s occ %v", s, occ)190 }191 for _, n := range []uuid.UUID{a, b} {192 if m, r := e.maint(n); m || r != "" {193 t.Fatalf("node left in maintenance: %v %q", m, r)194 }195 }196 if m, r := e.maint(manual); !m || r != "bad RAM, waiting for parts" {197 t.Fatal("a node put in maintenance by hand before the window was taken out of maintenance")198 }199 if m, r := e.maint(retagged); !m || r != "keep offline: disk replacement tomorrow" {200 t.Fatal("a node re-flagged by hand during the window was taken out of maintenance")201 }202 if e.activeSilences(w) != 0 || e.count(`select count(*) from maintenance_window_nodes`) != 0 {203 t.Fatal("silence still active or records left")204 }205 if e.count(`select count(*) from audit_log where action in ('maintenance.occurrence_start','maintenance.occurrence_end') and actor_kind='system'`) != 2 {206 t.Fatal("scheduler actions not audited")207 }208 if e.count(`select count(*) from events_outbox where topic='maintenance.changed' and payload->>'change' in ('occurrence_started','occurrence_ended')`) != 2 {209 t.Fatal("maintenance.changed events missing")210 }211 e.clock.Advance(24 * time.Hour)212 e.tick()213 if m, _ := e.maint(a); m {214 t.Fatal("completed window re-applied")215 }216}217218func TestSchedulerOverlappingWindows(t *testing.T) {219 e := newSenv(t)220 a, b := testdb.Node(t, e.pool, "T-A"), testdb.Node(t, e.pool, "T-B")221 t0 := e.clock.Now()222 w1 := e.window("W1", t0, time.Hour, []uuid.UUID{a, b}) // 15:00–16:00223 w2 := e.window("W2", t0.Add(30*time.Minute), 90*time.Minute, []uuid.UUID{a}) // 15:30–17:00224 e.tick()225 e.clock.Set(t0.Add(30 * time.Minute))226 e.tick()227 if s, _ := e.status(w2); s != "in_progress" {228 t.Fatalf("W2 %s", s)229 }230 e.clock.Set(t0.Add(time.Hour)) // W1 ends, W2 still covers A231 e.tick()232 if s, _ := e.status(w1); s != "completed" {233 t.Fatalf("W1 %s", s)234 }235 if m, _ := e.maint(a); !m {236 t.Fatal("node left maintenance although another window in progress still covers it")237 }238 if m, _ := e.maint(b); m {239 t.Fatal("node covered by the ended window only is still in maintenance")240 }241 if e.activeSilences(w1) != 0 || e.activeSilences(w2) != 1 {242 t.Fatal("silences not matching the windows in progress")243 }244 e.clock.Set(t0.Add(2 * time.Hour)) // W2 ends: the last covering window switches the node off245 e.tick()246 if m, r := e.maint(a); m || r != "" {247 t.Fatalf("node still in maintenance after the last window: %q", r)248 }249 if e.count(`select count(*) from maintenance_window_nodes`) != 0 {250 t.Fatal("records left")251 }252}253254func TestSchedulerRestartsAndMissedWindows(t *testing.T) {255 e := newSenv(t)256 a, b := testdb.Node(t, e.pool, "T-A"), testdb.Node(t, e.pool, "T-B")257 t0 := e.clock.Now()258 // The scheduler was down when W1 started: it applies it as soon as it runs, mid-window.259 w1 := e.window("W1", t0.Add(-20*time.Minute), time.Hour, []uuid.UUID{a})260 // It was down for the whole of W2: nothing is applied afterwards.261 w2 := e.window("W2", t0.Add(-3*time.Hour), time.Hour, []uuid.UUID{b})262 e.tick()263 if s, occ := e.status(w1); s != "in_progress" || !occ.Equal(t0.Add(-20*time.Minute)) {264 t.Fatalf("W1 %s", s)265 }266 if m, _ := e.maint(a); !m {267 t.Fatal("W1 not applied after the restart")268 }269 if s, _ := e.status(w2); s != "completed" {270 t.Fatalf("W2 %s", s)271 }272 if m, _ := e.maint(b); m || e.count(`select count(*) from silences where maintenance_window_id=$1`, w2) != 0 {273 t.Fatal("a window that is over was applied")274 }275 // The scheduler is down when W1 ends: the end is applied at the next run.276 e.clock.Set(t0.Add(5 * time.Hour))277 e.tick()278 if m, _ := e.maint(a); m {279 t.Fatal("W1 not ended after the restart")280 }281 if s, _ := e.status(w1); s != "completed" {282 t.Fatalf("W1 %s", s)283 }284}285286func TestSchedulerRecurringAndFlags(t *testing.T) {287 e := newSenv(t)288 a, b := testdb.Node(t, e.pool, "T-A"), testdb.Node(t, e.pool, "T-B")289 // Daily 03:00–04:00 Toronto across the 2026-11-01 fall-back, until 2026-11-02.290 loc, _ := time.LoadLocation("America/Toronto")291 first := time.Date(2026, 10, 31, 3, 0, 0, 0, loc)292 until := time.Date(2026, 11, 2, 3, 0, 0, 0, loc)293 w := e.window("Nightly", first.UTC(), time.Hour, []uuid.UUID{a}, func(w *mt.Window) { w.Recurrence = "daily"; w.RecurrenceUntil = &until })294 // Effects can be switched off individually.295 onlySilence := e.window("Silence only", first.UTC(), time.Hour, []uuid.UUID{b}, func(w *mt.Window) { w.SetMaintenanceMode = false })296 for day, wantUTC := range []int{7, 8, 8} { // 03:00 EDT = 07:00Z, 03:00 EST = 08:00Z297 start := time.Date(2026, 10, 31+day, wantUTC, 0, 0, 0, time.UTC)298 e.clock.Set(start.Add(-time.Minute))299 e.tick()300 if m, _ := e.maint(a); m {301 t.Fatalf("day %d: active one minute before local 03:00", day)302 }303 e.clock.Set(start)304 e.tick()305 s, occ := e.status(w)306 if m, _ := e.maint(a); !m || s != "in_progress" || !occ.Equal(start) {307 t.Fatalf("day %d: not active at local 03:00 (%s %v)", day, s, occ)308 }309 e.clock.Set(start.Add(time.Hour))310 e.tick()311 s, _ = e.status(w)312 want := "scheduled"313 if day == 2 {314 want = "completed" // recurrence_until reached315 }316 if m, _ := e.maint(a); m || s != want {317 t.Fatalf("day %d: after the end: maintenance=%v status=%s want %s", day, m, s, want)318 }319 }320 if n := e.count(`select count(*) from silences where maintenance_window_id=$1`, w); n != 3 {321 t.Fatalf("%d silences for 3 occurrences", n)322 }323 if m, _ := e.maint(b); m {324 t.Fatal("set_maintenance_mode=false but the node was put in maintenance")325 }326 if e.count(`select count(*) from silences where maintenance_window_id=$1`, onlySilence) != 1 {327 t.Fatal("silence-only window created no silence")328 }329 // silence_alerts=false and an informational window without nodes.330 e.clock.Set(time.Date(2027, 1, 10, 12, 0, 0, 0, time.UTC))331 noSilence := e.window("No silence", e.clock.Now(), time.Hour, []uuid.UUID{b}, func(w *mt.Window) { w.SilenceAlerts = false })332 info := e.window("Electrician on site", e.clock.Now(), time.Hour, nil)333 e.tick()334 if m, _ := e.maint(b); !m || e.count(`select count(*) from silences where maintenance_window_id in ($1,$2)`, noSilence, info) != 0 {335 t.Fatal("flags not respected")336 }337 if s, _ := e.status(info); s != "in_progress" {338 t.Fatalf("informational window %s", s)339 }340 // A manually completed occurrence is not applied again by the scheduler.341 e.pool.Exec(bg, `update maintenance_windows set status='scheduled', activated_occurrence=null, completed_occurrence=starts_at where id=$1`, info)342 e.tick()343 if s, _ := e.status(info); s != "scheduled" {344 t.Fatalf("completed occurrence re-activated: %s", s)345 }346}347