SPB Git

Repository Public

scheduler_test.go 12.9 KB 347 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/maintenance/scheduler_test.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package maintenance_test4344import (45	"context"46	"testing"47	"time"4849	"github.com/google/uuid"50	"github.com/jackc/pgx/v5/pgxpool"5152	"clustr.spboucher.ai/mcc/internal/ids"53	mt "clustr.spboucher.ai/mcc/internal/maintenance"54	"clustr.spboucher.ai/mcc/internal/testdb"55)5657var bg = context.Background()5859type senv struct {60	t     *testing.T61	pool  *pgxpool.Pool62	clock *testdb.Clock63}6465func newSenv(t *testing.T) *senv {66	t.Helper()67	pool, _ := testdb.New(t)68	e := &senv{t: t, pool: pool, clock: testdb.NewClock()}69	e.clock.Set(time.Date(2026, 11, 20, 15, 0, 0, 0, time.UTC))70	return e71}7273// tick runs a brand-new scheduler each time: nothing may depend on process memory.74func (e *senv) tick() {75	e.t.Helper()76	if err := (&mt.Scheduler{Pool: e.pool, Now: e.clock.Now}).Tick(bg); err != nil {77		e.t.Fatal(err)78	}79}8081type wopt func(*mt.Window)8283func (e *senv) window(title string, start time.Time, dur time.Duration, nodes []uuid.UUID, opts ...wopt) uuid.UUID {84	e.t.Helper()85	w := mt.Window{ID: ids.New(), Title: title, Category: "planned", StartsAt: start, EndsAt: start.Add(dur), Timezone: "America/Toronto", Recurrence: "none",86		NodeIDs: nodes, SetMaintenanceMode: true, SilenceAlerts: true}87	for _, o := range opts {88		o(&w)89	}90	if err := mt.Validate(w); err != nil {91		e.t.Fatal(err)92	}93	if _, err := e.pool.Exec(bg, `insert into maintenance_windows(id, title, category, starts_at, ends_at, timezone, recurrence, recurrence_until, scope, set_maintenance_mode, silence_alerts)94		values ($1,$2,$3,$4,$5,$6,$7,$8,jsonb_build_object('node_ids',$9::uuid[]),$10,$11)`, w.ID, w.Title, w.Category, w.StartsAt, w.EndsAt, w.Timezone, w.Recurrence,95		w.RecurrenceUntil, w.NodeIDs, w.SetMaintenanceMode, w.SilenceAlerts); err != nil {96		e.t.Fatal(err)97	}98	return w.ID99}100101func (e *senv) maint(node uuid.UUID) (bool, string) {102	e.t.Helper()103	var m bool104	var r string105	if err := e.pool.QueryRow(bg, `select maintenance, maintenance_reason from nodes where id=$1`, node).Scan(&m, &r); err != nil {106		e.t.Fatal(err)107	}108	return m, r109}110111func (e *senv) status(w uuid.UUID) (string, *time.Time) {112	e.t.Helper()113	var s string114	var occ *time.Time115	if err := e.pool.QueryRow(bg, `select status, activated_occurrence from maintenance_windows where id=$1`, w).Scan(&s, &occ); err != nil {116		e.t.Fatal(err)117	}118	return s, occ119}120121func (e *senv) count(sql string, args ...any) int {122	e.t.Helper()123	var n int124	if err := e.pool.QueryRow(bg, sql, args...).Scan(&n); err != nil {125		e.t.Fatal(err)126	}127	return n128}129130func (e *senv) activeSilences(w uuid.UUID) int {131	return e.count(`select count(*) from silences where maintenance_window_id=$1 and starts_at <= $2 and ends_at > $2`, w, e.clock.Now())132}133134func TestSchedulerActivatesAndRevertsOnlyWhatItSet(t *testing.T) {135	e := newSenv(t)136	a, b, manual, retagged := testdb.Node(t, e.pool, "T-A"), testdb.Node(t, e.pool, "T-B"), testdb.Node(t, e.pool, "T-MANUAL"), testdb.Node(t, e.pool, "T-RETAGGED")137	outside := testdb.Node(t, e.pool, "T-OUTSIDE")138	e.pool.Exec(bg, `update nodes set maintenance=true, maintenance_reason='bad RAM, waiting for parts' where id=$1`, manual)139	start := e.clock.Now().Add(10 * time.Minute)140	w := e.window("Firmware updates", start, time.Hour, []uuid.UUID{a, b, manual, retagged})141142	e.tick()143	if m, _ := e.maint(a); m {144		t.Fatal("activated before the start")145	}146	if s, _ := e.status(w); s != "scheduled" {147		t.Fatalf("status %s", s)148	}149	e.clock.Set(start)150	e.tick()151	s, occ := e.status(w)152	if s != "in_progress" || occ == nil || !occ.Equal(start) {153		t.Fatalf("status %s occ %v", s, occ)154	}155	for _, n := range []uuid.UUID{a, b, retagged} {156		if m, r := e.maint(n); !m || r == "" {157			t.Fatalf("node not in maintenance: %v %q", m, r)158		}159	}160	if m, _ := e.maint(outside); m {161		t.Fatal("a node outside the scope was touched")162	}163	if _, r := e.maint(manual); r != "bad RAM, waiting for parts" {164		t.Fatalf("manual reason overwritten: %q", r)165	}166	if e.activeSilences(w) != 1 || e.count(`select count(*) from maintenance_window_nodes where window_id=$1`, w) != 3 {167		t.Fatal("silence or applied-node records missing")168	}169	var scope string170	var ends time.Time171	e.pool.QueryRow(bg, `select scope::text, ends_at from silences where maintenance_window_id=$1`, w).Scan(&scope, &ends)172	if !ends.Equal(start.Add(time.Hour)) || len(scope) < 40 {173		t.Fatalf("silence: %s until %s", scope, ends)174	}175	// Idempotent, also across "restarts" in the middle of the window.176	for i := 0; i < 3; i++ {177		e.clock.Advance(10 * time.Minute)178		e.tick()179	}180	if e.count(`select count(*) from silences where maintenance_window_id=$1`, w) != 1 || e.count(`select count(*) from audit_log where action='maintenance.occurrence_start'`) != 1 {181		t.Fatal("occurrence applied more than once")182	}183	// During the window somebody takes over one node by hand, with their own reason.184	e.pool.Exec(bg, `update nodes set maintenance_reason='keep offline: disk replacement tomorrow' where id=$1`, retagged)185186	e.clock.Set(start.Add(time.Hour))187	e.tick()188	if s, occ := e.status(w); s != "completed" || occ != nil {189		t.Fatalf("status %s occ %v", s, occ)190	}191	for _, n := range []uuid.UUID{a, b} {192		if m, r := e.maint(n); m || r != "" {193			t.Fatalf("node left in maintenance: %v %q", m, r)194		}195	}196	if m, r := e.maint(manual); !m || r != "bad RAM, waiting for parts" {197		t.Fatal("a node put in maintenance by hand before the window was taken out of maintenance")198	}199	if m, r := e.maint(retagged); !m || r != "keep offline: disk replacement tomorrow" {200		t.Fatal("a node re-flagged by hand during the window was taken out of maintenance")201	}202	if e.activeSilences(w) != 0 || e.count(`select count(*) from maintenance_window_nodes`) != 0 {203		t.Fatal("silence still active or records left")204	}205	if e.count(`select count(*) from audit_log where action in ('maintenance.occurrence_start','maintenance.occurrence_end') and actor_kind='system'`) != 2 {206		t.Fatal("scheduler actions not audited")207	}208	if e.count(`select count(*) from events_outbox where topic='maintenance.changed' and payload->>'change' in ('occurrence_started','occurrence_ended')`) != 2 {209		t.Fatal("maintenance.changed events missing")210	}211	e.clock.Advance(24 * time.Hour)212	e.tick()213	if m, _ := e.maint(a); m {214		t.Fatal("completed window re-applied")215	}216}217218func TestSchedulerOverlappingWindows(t *testing.T) {219	e := newSenv(t)220	a, b := testdb.Node(t, e.pool, "T-A"), testdb.Node(t, e.pool, "T-B")221	t0 := e.clock.Now()222	w1 := e.window("W1", t0, time.Hour, []uuid.UUID{a, b})                       // 15:00–16:00223	w2 := e.window("W2", t0.Add(30*time.Minute), 90*time.Minute, []uuid.UUID{a}) // 15:30–17:00224	e.tick()225	e.clock.Set(t0.Add(30 * time.Minute))226	e.tick()227	if s, _ := e.status(w2); s != "in_progress" {228		t.Fatalf("W2 %s", s)229	}230	e.clock.Set(t0.Add(time.Hour)) // W1 ends, W2 still covers A231	e.tick()232	if s, _ := e.status(w1); s != "completed" {233		t.Fatalf("W1 %s", s)234	}235	if m, _ := e.maint(a); !m {236		t.Fatal("node left maintenance although another window in progress still covers it")237	}238	if m, _ := e.maint(b); m {239		t.Fatal("node covered by the ended window only is still in maintenance")240	}241	if e.activeSilences(w1) != 0 || e.activeSilences(w2) != 1 {242		t.Fatal("silences not matching the windows in progress")243	}244	e.clock.Set(t0.Add(2 * time.Hour)) // W2 ends: the last covering window switches the node off245	e.tick()246	if m, r := e.maint(a); m || r != "" {247		t.Fatalf("node still in maintenance after the last window: %q", r)248	}249	if e.count(`select count(*) from maintenance_window_nodes`) != 0 {250		t.Fatal("records left")251	}252}253254func TestSchedulerRestartsAndMissedWindows(t *testing.T) {255	e := newSenv(t)256	a, b := testdb.Node(t, e.pool, "T-A"), testdb.Node(t, e.pool, "T-B")257	t0 := e.clock.Now()258	// The scheduler was down when W1 started: it applies it as soon as it runs, mid-window.259	w1 := e.window("W1", t0.Add(-20*time.Minute), time.Hour, []uuid.UUID{a})260	// It was down for the whole of W2: nothing is applied afterwards.261	w2 := e.window("W2", t0.Add(-3*time.Hour), time.Hour, []uuid.UUID{b})262	e.tick()263	if s, occ := e.status(w1); s != "in_progress" || !occ.Equal(t0.Add(-20*time.Minute)) {264		t.Fatalf("W1 %s", s)265	}266	if m, _ := e.maint(a); !m {267		t.Fatal("W1 not applied after the restart")268	}269	if s, _ := e.status(w2); s != "completed" {270		t.Fatalf("W2 %s", s)271	}272	if m, _ := e.maint(b); m || e.count(`select count(*) from silences where maintenance_window_id=$1`, w2) != 0 {273		t.Fatal("a window that is over was applied")274	}275	// The scheduler is down when W1 ends: the end is applied at the next run.276	e.clock.Set(t0.Add(5 * time.Hour))277	e.tick()278	if m, _ := e.maint(a); m {279		t.Fatal("W1 not ended after the restart")280	}281	if s, _ := e.status(w1); s != "completed" {282		t.Fatalf("W1 %s", s)283	}284}285286func TestSchedulerRecurringAndFlags(t *testing.T) {287	e := newSenv(t)288	a, b := testdb.Node(t, e.pool, "T-A"), testdb.Node(t, e.pool, "T-B")289	// Daily 03:00–04:00 Toronto across the 2026-11-01 fall-back, until 2026-11-02.290	loc, _ := time.LoadLocation("America/Toronto")291	first := time.Date(2026, 10, 31, 3, 0, 0, 0, loc)292	until := time.Date(2026, 11, 2, 3, 0, 0, 0, loc)293	w := e.window("Nightly", first.UTC(), time.Hour, []uuid.UUID{a}, func(w *mt.Window) { w.Recurrence = "daily"; w.RecurrenceUntil = &until })294	// Effects can be switched off individually.295	onlySilence := e.window("Silence only", first.UTC(), time.Hour, []uuid.UUID{b}, func(w *mt.Window) { w.SetMaintenanceMode = false })296	for day, wantUTC := range []int{7, 8, 8} { // 03:00 EDT = 07:00Z, 03:00 EST = 08:00Z297		start := time.Date(2026, 10, 31+day, wantUTC, 0, 0, 0, time.UTC)298		e.clock.Set(start.Add(-time.Minute))299		e.tick()300		if m, _ := e.maint(a); m {301			t.Fatalf("day %d: active one minute before local 03:00", day)302		}303		e.clock.Set(start)304		e.tick()305		s, occ := e.status(w)306		if m, _ := e.maint(a); !m || s != "in_progress" || !occ.Equal(start) {307			t.Fatalf("day %d: not active at local 03:00 (%s %v)", day, s, occ)308		}309		e.clock.Set(start.Add(time.Hour))310		e.tick()311		s, _ = e.status(w)312		want := "scheduled"313		if day == 2 {314			want = "completed" // recurrence_until reached315		}316		if m, _ := e.maint(a); m || s != want {317			t.Fatalf("day %d: after the end: maintenance=%v status=%s want %s", day, m, s, want)318		}319	}320	if n := e.count(`select count(*) from silences where maintenance_window_id=$1`, w); n != 3 {321		t.Fatalf("%d silences for 3 occurrences", n)322	}323	if m, _ := e.maint(b); m {324		t.Fatal("set_maintenance_mode=false but the node was put in maintenance")325	}326	if e.count(`select count(*) from silences where maintenance_window_id=$1`, onlySilence) != 1 {327		t.Fatal("silence-only window created no silence")328	}329	// silence_alerts=false and an informational window without nodes.330	e.clock.Set(time.Date(2027, 1, 10, 12, 0, 0, 0, time.UTC))331	noSilence := e.window("No silence", e.clock.Now(), time.Hour, []uuid.UUID{b}, func(w *mt.Window) { w.SilenceAlerts = false })332	info := e.window("Electrician on site", e.clock.Now(), time.Hour, nil)333	e.tick()334	if m, _ := e.maint(b); !m || e.count(`select count(*) from silences where maintenance_window_id in ($1,$2)`, noSilence, info) != 0 {335		t.Fatal("flags not respected")336	}337	if s, _ := e.status(info); s != "in_progress" {338		t.Fatalf("informational window %s", s)339	}340	// A manually completed occurrence is not applied again by the scheduler.341	e.pool.Exec(bg, `update maintenance_windows set status='scheduled', activated_occurrence=null, completed_occurrence=starts_at where id=$1`, info)342	e.tick()343	if s, _ := e.status(info); s != "scheduled" {344		t.Fatalf("completed occurrence re-activated: %s", s)345	}346}347
Files main
Menu

Theme