SPB Git

Repository Public

scheduler.go 11.6 KB 297 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/maintenance/scheduler.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package maintenance4344import (45	"context"46	"errors"47	"time"4849	"github.com/google/uuid"50	"github.com/jackc/pgx/v5"51	"github.com/jackc/pgx/v5/pgxpool"5253	"clustr.spboucher.ai/mcc/internal/audit"54	"clustr.spboucher.ai/mcc/internal/events"55	"clustr.spboucher.ai/mcc/internal/ids"56)5758// Database part of the package: loading windows and applying occurrences.59//60// What an active occurrence changes, and how it is undone:61//   - set_maintenance_mode: nodes in scope that are NOT already in maintenance62//     are switched on, with a reason naming the window; each such node is63//     recorded in maintenance_window_nodes. At the end only recorded nodes are64//     switched off, and only if their reason is still the one the window65//     wrote. A node put in maintenance by hand (before, or again during the66//     window with another reason) therefore stays in maintenance. If another67//     window in progress also covers the node, the record is handed over to68//     that window instead, so the node leaves maintenance when the LAST69//     covering window ends.70//   - silence_alerts: one silence linked to the window for the occurrence; at71//     the end its ends_at is brought back to now.72//73// State survives restarts: status, activated_occurrence and74// maintenance_window_nodes say exactly what is applied.7576// Columns is the column list read by Scan.77const Columns = `id, title, description, category, starts_at, ends_at, timezone, recurrence, recurrence_until, coalesce(scope->'node_ids','[]'::jsonb),78	set_maintenance_mode, silence_alerts, status, activated_occurrence, completed_occurrence, created_by, created_at, updated_at`7980// Scan reads Columns, then any extra destinations.81func Scan(row pgx.Row, extra ...any) (Window, error) {82	var w Window83	err := row.Scan(append([]any{&w.ID, &w.Title, &w.Description, &w.Category, &w.StartsAt, &w.EndsAt, &w.Timezone, &w.Recurrence, &w.RecurrenceUntil, &w.NodeIDs,84		&w.SetMaintenanceMode, &w.SilenceAlerts, &w.Status, &w.ActivatedOccurrence, &w.CompletedOccurrence, &w.CreatedBy, &w.CreatedAt, &w.UpdatedAt}, extra...)...)85	if w.NodeIDs == nil {86		w.NodeIDs = []uuid.UUID{}87	}88	return w, err89}9091// Reason is what the scheduler writes in nodes.maintenance_reason.92func Reason(w Window) string {93	t := w.Title94	if len(t) > 120 {95		t = t[:120]96	}97	return "Maintenance window: " + t + " [" + w.ID.String()[:8] + "]"98}99100func changed(ctx context.Context, tx pgx.Tx, change string, w Window, occ *time.Time) error {101	p := map[string]any{"change": change, "window_id": w.ID, "node_ids": w.NodeIDs}102	if occ != nil {103		p["occurrence"] = *occ104	}105	return events.Publish(ctx, tx, "maintenance.changed", p)106}107108// Changed publishes maintenance.changed for a window.109func Changed(ctx context.Context, tx pgx.Tx, change string, w Window, occ *time.Time) error {110	return changed(ctx, tx, change, w, occ)111}112113// Activate applies an occurrence: maintenance mode, silence, status.114func Activate(ctx context.Context, tx pgx.Tx, w Window, occ Occurrence, now time.Time) error {115	reason := Reason(w)116	if w.SetMaintenanceMode {117		for _, n := range w.NodeIDs {118			tag, err := tx.Exec(ctx, `update nodes set maintenance=true, maintenance_reason=$2 where id=$1 and not maintenance and retired_at is null`, n, reason)119			if err != nil {120				return err121			}122			if tag.RowsAffected() == 0 {123				continue // already in maintenance (by hand or by another window): not ours to undo124			}125			if _, err := tx.Exec(ctx, `insert into maintenance_window_nodes(window_id, node_id, reason, set_at) values ($1,$2,$3,$4)126				on conflict (window_id, node_id) do update set reason=excluded.reason, set_at=excluded.set_at`, w.ID, n, reason, now); err != nil {127				return err128			}129			if err := events.Publish(ctx, tx, "node.updated", map[string]any{"node_id": n, "change": "updated"}); err != nil {130				return err131			}132		}133	}134	if w.SilenceAlerts && len(w.NodeIDs) > 0 {135		if _, err := tx.Exec(ctx, `insert into silences(id, reason, scope, starts_at, ends_at, created_by, maintenance_window_id, created_at)136			values ($1,$2,jsonb_build_object('node_ids',$3::uuid[]),$4,$5,$6,$7,$8)`, ids.New(), reason, w.NodeIDs, occ.Start, occ.End, w.CreatedBy, w.ID, now); err != nil {137			return err138		}139	}140	if _, err := tx.Exec(ctx, `update maintenance_windows set status='in_progress', activated_occurrence=$2, updated_at=$3 where id=$1`, w.ID, occ.Start, now); err != nil {141		return err142	}143	return changed(ctx, tx, "occurrence_started", w, &occ.Start)144}145146// Deactivate undoes what the active occurrence applied and sets the new147// status. It reverts ONLY what this window set (see the package comment).148func Deactivate(ctx context.Context, tx pgx.Tx, w Window, status string, completed *time.Time, now time.Time) error {149	rows, err := tx.Query(ctx, `select node_id, reason from maintenance_window_nodes where window_id=$1`, w.ID)150	if err != nil {151		return err152	}153	type applied struct {154		node   uuid.UUID155		reason string156	}157	list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (applied, error) {158		var a applied159		return a, r.Scan(&a.node, &a.reason)160	})161	if err != nil {162		return err163	}164	for _, a := range list {165		var heir uuid.UUID166		err := tx.QueryRow(ctx, `select id from maintenance_windows where id <> $1 and status='in_progress' and set_maintenance_mode167			and scope->'node_ids' ? $2 order by id limit 1`, w.ID, a.node.String()).Scan(&heir)168		switch {169		case err == nil:170			// Another window in progress covers this node: it inherits the duty to switch it off.171			if _, err := tx.Exec(ctx, `insert into maintenance_window_nodes(window_id, node_id, reason, set_at) values ($1,$2,$3,$4)172				on conflict (window_id, node_id) do nothing`, heir, a.node, a.reason, now); err != nil {173				return err174			}175		case errors.Is(err, pgx.ErrNoRows):176			tag, err := tx.Exec(ctx, `update nodes set maintenance=false, maintenance_reason='' where id=$1 and maintenance and maintenance_reason=$2`, a.node, a.reason)177			if err != nil {178				return err179			}180			if tag.RowsAffected() == 1 {181				if err := events.Publish(ctx, tx, "node.updated", map[string]any{"node_id": a.node, "change": "updated"}); err != nil {182					return err183				}184			}185		default:186			return err187		}188	}189	if _, err := tx.Exec(ctx, `delete from maintenance_window_nodes where window_id=$1`, w.ID); err != nil {190		return err191	}192	if _, err := tx.Exec(ctx, `update silences set ends_at=greatest(starts_at, $2) where maintenance_window_id=$1 and ends_at > $2`, w.ID, now); err != nil {193		return err194	}195	_, err = tx.Exec(ctx, `update maintenance_windows set status=$2, activated_occurrence=null, completed_occurrence=coalesce($3, completed_occurrence), updated_at=$4 where id=$1`,196		w.ID, status, completed, now)197	return err198}199200// StatusAfter is the status of a window once the occurrence starting at201// `occ` is over: back to scheduled when another occurrence follows, else completed.202func StatusAfter(w Window, occ time.Time) string {203	if _, more := Next(w, occ); more {204		return StatusScheduled205	}206	return StatusCompleted207}208209// Scheduler starts and ends occurrences.210type Scheduler struct {211	Pool *pgxpool.Pool212	Now  func() time.Time213}214215func (s *Scheduler) now() time.Time {216	if s.Now != nil {217		return s.Now()218	}219	return time.Now().UTC()220}221222// Tick brings every window in line with the clock. Idempotent.223func (s *Scheduler) Tick(ctx context.Context) error {224	now := s.now()225	rows, err := s.Pool.Query(ctx, `select id from maintenance_windows where status in ('scheduled','in_progress') and starts_at <= $1 order by starts_at, id`, now)226	if err != nil {227		return err228	}229	list, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])230	if err != nil {231		return err232	}233	// Ends first, then starts: a node handed from one window to the next never flickers out of maintenance... and234	// a window that ends while another starts sees the other as not yet in progress, which is the conservative order.235	for _, phase := range []string{StatusInProgress, StatusScheduled} {236		for _, id := range list {237			if err := s.advance(ctx, id, phase, now); err != nil {238				return err239			}240		}241	}242	return nil243}244245func (s *Scheduler) advance(ctx context.Context, id uuid.UUID, phase string, now time.Time) error {246	return pgx.BeginFunc(ctx, s.Pool, func(tx pgx.Tx) error {247		w, err := Scan(tx.QueryRow(ctx, `select `+Columns+` from maintenance_windows where id=$1 and status=$2 for update skip locked`, id, phase))248		if errors.Is(err, pgx.ErrNoRows) {249			return nil250		}251		if err != nil {252			return err253		}254		sys := func(action, detail string, occ time.Time) error {255			_, err := audit.Append(ctx, tx, audit.Entry{ActorKind: "system", ActorLabel: "maintenance-scheduler", Action: action, TargetKind: "maintenance_window",256				TargetID: w.ID.String(), Outcome: audit.Success, Detail: detail, Params: map[string]any{"occurrence": occ, "nodes": len(w.NodeIDs)}})257			return err258		}259		switch w.Status {260		case StatusInProgress:261			if w.ActivatedOccurrence == nil { // inconsistent row: repair by ending it262				return Deactivate(ctx, tx, w, StatusAfter(w, now), nil, now)263			}264			occ, ok := OccurrenceStarting(w, *w.ActivatedOccurrence)265			if ok && now.Before(occ.End) {266				return nil267			}268			if err := Deactivate(ctx, tx, w, StatusAfter(w, *w.ActivatedOccurrence), nil, now); err != nil {269				return err270			}271			if err := changed(ctx, tx, "occurrence_ended", w, w.ActivatedOccurrence); err != nil {272				return err273			}274			return sys("maintenance.occurrence_end", "", *w.ActivatedOccurrence)275		case StatusScheduled:276			occ, ok := OccurrenceAt(w, now)277			if ok && (w.CompletedOccurrence == nil || !w.CompletedOccurrence.Equal(occ.Start)) {278				if err := Activate(ctx, tx, w, occ, now); err != nil {279					return err280				}281				return sys("maintenance.occurrence_start", "", occ.Start)282			}283			if _, more := Next(w, now); !ok && !more {284				// Nothing in progress and nothing to come: the series is over (possibly missed while the scheduler was down).285				if _, err := tx.Exec(ctx, `update maintenance_windows set status='completed', updated_at=$2 where id=$1`, w.ID, now); err != nil {286					return err287				}288				if err := changed(ctx, tx, "window_completed", w, nil); err != nil {289					return err290				}291				return sys("maintenance.window_end", "series over without an occurrence in progress", now)292			}293		}294		return nil295	})296}297
Files main
Menu

Theme