SPB Git

Repository Public

events.go 7.2 KB 243 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/events/events.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Package events is the durable event stream behind GET /api/v1/events.43//44// Producers call Publish inside the transaction that changes state: the row45// in events_outbox gets a monotonically increasing seq and a NOTIFY wakes the46// streamers. Consumers read by seq, so a reconnecting client resumes exactly47// where it stopped, or is told to resync if its position was pruned.48package events4950import (51	"context"52	"encoding/json"53	"errors"54	"log/slog"55	"sync"56	"time"5758	"github.com/jackc/pgx/v5"59	"github.com/jackc/pgx/v5/pgxpool"60)6162const channel = "mcc_events"6364// Retention of the outbox; clients further behind must resync.65const Retention = time.Hour6667type Event struct {68	Seq     int64           `json:"seq"`69	TS      time.Time       `json:"ts"`70	Topic   string          `json:"topic"`71	Payload json.RawMessage `json:"payload"`72}7374// Publish records an event. q is a pool or a transaction.75func Publish(ctx context.Context, q interface {76	QueryRow(ctx context.Context, sql string, args ...any) pgx.Row77}, topic string, payload any) error {78	b, err := json.Marshal(payload)79	if err != nil {80		return err81	}82	var seq int6483	// NOTIFY is delivered at commit, so listeners never see uncommitted rows.84	return q.QueryRow(ctx, `with e as (insert into events_outbox(topic, payload) values ($1,$2) returning seq)85		select seq from e, pg_notify($3, '') `, topic, b, channel).Scan(&seq)86}8788// ErrResync means the requested position is older than the retained stream.89var ErrResync = errors.New("events: position no longer retained")9091// Hub fans committed events out to subscribers. One Hub per process.92type Hub struct {93	pool *pgxpool.Pool94	mu   sync.Mutex95	subs map[chan struct{}]struct{}96}9798func NewHub(pool *pgxpool.Pool) *Hub {99	return &Hub{pool: pool, subs: map[chan struct{}]struct{}{}}100}101102// Run listens for notifications until ctx ends, reconnecting with backoff.103// While disconnected, subscribers are woken periodically so they poll: a104// database blip delays events but never loses them.105func (h *Hub) Run(ctx context.Context) {106	tick := time.NewTicker(5 * time.Second)107	defer tick.Stop()108	go func() {109		for {110			select {111			case <-ctx.Done():112				return113			case <-tick.C:114				h.wake() // safety net for missed notifications115			}116		}117	}()118	backoff := time.Second119	for ctx.Err() == nil {120		err := h.listen(ctx)121		if ctx.Err() != nil {122			return123		}124		slog.Warn("events: listener disconnected", "err", err, "retry_in", backoff)125		select {126		case <-ctx.Done():127			return128		case <-time.After(backoff):129		}130		if backoff < 15*time.Second {131			backoff *= 2132		}133	}134}135136func (h *Hub) listen(ctx context.Context) error {137	conn, err := h.pool.Acquire(ctx)138	if err != nil {139		return err140	}141	defer conn.Release()142	if _, err := conn.Exec(ctx, "listen "+channel); err != nil {143		return err144	}145	h.wake()146	for {147		if _, err := conn.Conn().WaitForNotification(ctx); err != nil {148			return err149		}150		h.wake()151	}152}153154func (h *Hub) wake() {155	h.mu.Lock()156	defer h.mu.Unlock()157	for ch := range h.subs {158		select {159		case ch <- struct{}{}:160		default:161		}162	}163}164165// Subscribe returns a wake-up channel and an unsubscribe function.166func (h *Hub) Subscribe() (<-chan struct{}, func()) {167	ch := make(chan struct{}, 1)168	h.mu.Lock()169	h.subs[ch] = struct{}{}170	h.mu.Unlock()171	return ch, func() {172		h.mu.Lock()173		delete(h.subs, ch)174		h.mu.Unlock()175	}176}177178// Subscribers returns the number of streams currently attached to this process.179func (h *Hub) Subscribers() int {180	h.mu.Lock()181	defer h.mu.Unlock()182	return len(h.subs)183}184185// Head returns the latest sequence number (0 when the stream is empty).186func (h *Hub) Head(ctx context.Context) (int64, error) {187	var seq int64188	err := h.pool.QueryRow(ctx, `select coalesce(max(seq),0) from events_outbox`).Scan(&seq)189	return seq, err190}191192// Since returns up to limit events after seq. It returns ErrResync when193// events between seq and the oldest retained one have been pruned.194func (h *Hub) Since(ctx context.Context, seq int64, limit int) ([]Event, error) {195	var oldest, head int64196	if err := h.pool.QueryRow(ctx, `select coalesce(min(seq),0), coalesce(max(seq),0) from events_outbox`).Scan(&oldest, &head); err != nil {197		return nil, err198	}199	if seq > head {200		return nil, ErrResync // position from another database generation201	}202	if oldest > seq+1 {203		var pruned bool204		// Rows below `oldest` existed and were pruned unless the stream simply started later.205		if err := h.pool.QueryRow(ctx, `select coalesce((value->>'pruned_through')::bigint,0) > $1 from settings where key='events.pruned'`, seq).Scan(&pruned); err != nil && err != pgx.ErrNoRows {206			return nil, err207		}208		if pruned {209			return nil, ErrResync210		}211	}212	rows, err := h.pool.Query(ctx, `select seq, ts, topic, payload from events_outbox where seq > $1 order by seq limit $2`, seq, limit)213	if err != nil {214		return nil, err215	}216	return pgx.CollectRows(rows, func(r pgx.CollectableRow) (Event, error) {217		var e Event218		err := r.Scan(&e.Seq, &e.TS, &e.Topic, &e.Payload)219		return e, err220	})221}222223// Prune deletes events older than Retention and records how far it pruned.224func Prune(ctx context.Context, pool *pgxpool.Pool) error {225	tx, err := pool.Begin(ctx)226	if err != nil {227		return err228	}229	defer tx.Rollback(ctx)230	var through *int64231	if err := tx.QueryRow(ctx, `with d as (delete from events_outbox where ts < now() - $1::interval returning seq) select max(seq) from d`,232		Retention.String()).Scan(&through); err != nil {233		return err234	}235	if through != nil {236		if _, err := tx.Exec(ctx, `insert into settings(key, value) values ('events.pruned', jsonb_build_object('pruned_through', $1::bigint))237			on conflict (key) do update set value = excluded.value, updated_at = now()`, *through); err != nil {238			return err239		}240	}241	return tx.Commit(ctx)242}243
Files main
Menu

Theme