import { createHash } from "node:crypto"; import type { FastifyInstance } from "fastify"; import { z } from "zod"; import { assertUrlAllowed, newId, UrlPolicyError } from "@websensor/core"; import { getConnector } from "@websensor/connectors"; import { db, sql, textArray } from "@websensor/db"; import { config } from "./config"; import { EVENT_SELECT, eventConditions, listEvents } from "./queries"; /** * Owner-scoped routes (anonymous owner token, phase 1): watchlists, alert rules (+ webhook * channel), notifications, bookmarks, saved views and custom URL monitors (spec §42–45, §81, §100). */ export function ownerToken(headers: Record): string | null { const t = headers["x-websensor-owner"]; return typeof t === "string" && /^[A-Za-z0-9_-]{16,80}$/.test(t) ? t : null; } const WATCH_KINDS = ["entity", "source", "keyword", "category", "url", "event_type", "country", "group"] as const; export async function registerUserRoutes(app: FastifyInstance): Promise { const requireOwner = (req: { headers: Record }, reply: { status: (n: number) => { send: (b: unknown) => unknown } }): string | null => { const owner = ownerToken(req.headers); if (!owner) reply.status(401).send({ error: "owner_token_required", detail: "Send an X-WebSensor-Owner header (16–80 URL-safe characters)." }); return owner; }; // ---- Watchlists ----------------------------------------------------------------------------- app.get("/api/v1/watchlists", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const rows = await db.execute>(sql`select w.id, w.name, w.created_at, coalesce((select json_agg(json_build_object('kind', i.kind, 'value', i.value, 'added_at', i.added_at) order by i.added_at) from watchlist_items i where i.watchlist_id = w.id), '[]'::json) as items from watchlists w where owner_token = ${owner} order by created_at`); return { items: rows.rows }; }); const wlBody = z.object({ name: z.string().min(1).max(80).default("My watchlist"), items: z.array(z.object({ kind: z.enum(WATCH_KINDS), value: z.string().min(1).max(300) })).max(300).default([]) }); app.post("/api/v1/watchlists", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const body = wlBody.parse(req.body ?? {}); const count = (await db.execute<{ n: string }>(sql`select count(*)::text as n from watchlists where owner_token = ${owner}`)).rows[0]?.n; if (Number(count) >= 20) return reply.status(429).send({ error: "too_many_watchlists" }); const id = newId("wl"); await db.execute(sql`insert into watchlists (id, owner_token, name) values (${id}, ${owner}, ${body.name})`); for (const it of body.items) await db.execute(sql`insert into watchlist_items (watchlist_id, kind, value) values (${id}, ${it.kind}, ${it.value}) on conflict do nothing`); return { id, name: body.name, items: body.items }; }); app.put<{ Params: { id: string } }>("/api/v1/watchlists/:id", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const w = (await db.execute<{ id: string }>(sql`select id from watchlists where id = ${req.params.id} and owner_token = ${owner}`)).rows[0]; if (!w) return reply.status(404).send({ error: "not_found" }); const body = wlBody.partial().parse(req.body ?? {}); if (body.name) await db.execute(sql`update watchlists set name = ${body.name} where id = ${w.id}`); if (body.items) { await db.execute(sql`delete from watchlist_items where watchlist_id = ${w.id}`); for (const it of body.items) await db.execute(sql`insert into watchlist_items (watchlist_id, kind, value) values (${w.id}, ${it.kind}, ${it.value}) on conflict do nothing`); } return { ok: true }; }); app.delete<{ Params: { id: string } }>("/api/v1/watchlists/:id", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; await db.execute(sql`delete from watchlists where id = ${req.params.id} and owner_token = ${owner}`); return { ok: true }; }); app.get<{ Params: { id: string }; Querystring: { limit?: string } }>("/api/v1/watchlists/:id/events", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const items = (await db.execute<{ kind: string; value: string }>(sql`select i.kind, i.value from watchlist_items i join watchlists w on w.id = i.watchlist_id where w.id = ${req.params.id} and w.owner_token = ${owner}`)).rows; if (!items.length) return reply.send({ items: [] }); const by = (k: string): string[] => items.filter((i) => i.kind === k).map((i) => i.value); const conds = [] as ReturnType[]; const ents = by("entity"); const srcs = by("source"); const cats = by("category"); const types = by("event_type"); const countries = by("country"); const urls = by("url"); if (ents.length) conds.push(sql`exists (select 1 from event_entities x where x.event_id = e.id and x.entity_id = any(${textArray(ents)}))`); if (srcs.length) conds.push(sql`e.source_id = any(${textArray(srcs)})`); if (cats.length) conds.push(sql`e.categories && ${textArray(cats)}`); if (types.length) conds.push(sql`e.event_type = any(${textArray(types)})`); if (countries.length) conds.push(sql`e.country = any(${textArray(countries.map((c) => c.toUpperCase()))})`); for (const u of urls) conds.push(sql`(e.url = ${u} or e.url like ${u.replace(/\/$/, "") + "/%"})`); for (const k of by("keyword")) conds.push(sql`(e.title ilike ${"%" + k + "%"} or e.summary ilike ${"%" + k + "%"})`); if (!conds.length) return { items: [] }; const rows = await db.execute>(sql`select ${EVENT_SELECT} from events e join sources s on s.id = e.source_id join sensors sen on sen.id = e.sensor_id where (s.kind = 'registry' or s.owner_token = ${owner}) and (${sql.join(conds, sql` or `)}) order by e.detected_at desc limit ${Math.min(200, Number(req.query.limit ?? 50))}`); return { items: rows.rows }; }); // ---- Alerts ----------------------------------------------------------------------------------- const ruleSchema = z.object({ importance_min: z.number().min(0).max(100).optional(), signal_min: z.number().min(0).max(100).optional(), event_types: z.array(z.string().max(60)).max(60).optional(), groups: z.array(z.string().max(30)).max(12).optional(), entities: z.array(z.string().max(120)).max(100).optional(), sources: z.array(z.string().max(120)).max(100).optional(), keywords: z.array(z.string().max(80)).max(50).optional(), categories: z.array(z.string().max(40)).max(40).optional(), countries: z.array(z.string().max(3)).max(60).optional(), silent_only: z.boolean().optional(), first_party_only: z.boolean().optional(), confirmed_only: z.boolean().optional(), }); const alertBody = z.object({ name: z.string().min(1).max(80), rule: ruleSchema, channel: z.enum(["web", "webhook"]).default("web"), channel_config: z.object({ url: z.string().url().max(500).optional(), secret: z.string().max(200).optional() }).default({}) }); app.get("/api/v1/alerts", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const rows = await db.execute>(sql`select a.id, a.name, a.rule, a.channel, a.channel_config - 'secret' as channel_config, a.enabled, a.created_at, a.last_fired_at, a.fired_count, (select count(*) from notifications n where n.alert_id = a.id and n.created_at >= now() - interval '24 hours')::int as fired_24h from alerts a where owner_token = ${owner} order by created_at`); return { items: rows.rows, channels: { web: "active", webhook: "active", email: "planned", slack: "planned", discord: "planned", telegram: "planned", push: "planned" } }; }); app.post("/api/v1/alerts", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const body = alertBody.parse(req.body ?? {}); if (body.channel === "webhook") { if (!body.channel_config.url) return reply.status(400).send({ error: "webhook_url_required" }); try { const u = await assertUrlAllowed(body.channel_config.url); if (u.url.protocol !== "https:") return reply.status(400).send({ error: "webhook_https_required" }); } catch (e) { return reply.status(400).send({ error: "webhook_url_rejected", detail: e instanceof UrlPolicyError ? e.message : String(e) }); } } const count = (await db.execute<{ n: string }>(sql`select count(*)::text as n from alerts where owner_token = ${owner}`)).rows[0]?.n; if (Number(count) >= 50) return reply.status(429).send({ error: "too_many_alerts" }); const id = newId("alr"); await db.execute(sql`insert into alerts (id, owner_token, name, rule, channel, channel_config) values (${id}, ${owner}, ${body.name}, ${JSON.stringify(body.rule)}::jsonb, ${body.channel}, ${JSON.stringify(body.channel_config)}::jsonb)`); return { id, name: body.name, rule: body.rule, channel: body.channel }; }); app.patch<{ Params: { id: string } }>("/api/v1/alerts/:id", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const body = z.object({ enabled: z.boolean().optional(), name: z.string().min(1).max(80).optional() }).parse(req.body ?? {}); if (body.enabled !== undefined) await db.execute(sql`update alerts set enabled = ${body.enabled} where id = ${req.params.id} and owner_token = ${owner}`); if (body.name) await db.execute(sql`update alerts set name = ${body.name} where id = ${req.params.id} and owner_token = ${owner}`); return { ok: true }; }); app.delete<{ Params: { id: string } }>("/api/v1/alerts/:id", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; await db.execute(sql`delete from alerts where id = ${req.params.id} and owner_token = ${owner}`); return { ok: true }; }); app.get<{ Querystring: { limit?: string; unread?: string } }>("/api/v1/notifications", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const rows = await db.execute>(sql` select n.id, n.alert_id, a.name as alert_name, n.event_id, n.channel, n.status, n.created_at, n.read_at, n.delivered_at, n.error, json_build_object('id', e.id, 'slug', e.slug, 'title', e.title, 'importance', e.importance, 'signal_score', e.signal_score, 'event_type', e.event_type, 'silent_change', e.silent_change, 'detected_at', e.detected_at, 'source', json_build_object('id', s.id, 'name', s.name)) as event from notifications n join alerts a on a.id = n.alert_id join events e on e.id = n.event_id join sources s on s.id = e.source_id where a.owner_token = ${owner} ${req.query.unread === "1" ? sql`and n.read_at is null` : sql``} order by n.created_at desc limit ${Math.min(200, Number(req.query.limit ?? 50))}`); const unread = (await db.execute<{ n: string }>(sql`select count(*)::text as n from notifications n join alerts a on a.id = n.alert_id where a.owner_token = ${owner} and n.read_at is null`)).rows[0]?.n; return { items: rows.rows, unread: Number(unread ?? 0) }; }); app.post("/api/v1/notifications/read", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const body = z.object({ ids: z.array(z.number().int()).max(500).optional() }).parse(req.body ?? {}); if (body.ids?.length) await db.execute(sql`update notifications n set read_at = now() from alerts a where a.id = n.alert_id and a.owner_token = ${owner} and n.id = any(${sql.raw("array[" + body.ids.map((i) => Number(i)).join(",") + "]::bigint[]")})`); else await db.execute(sql`update notifications n set read_at = now() from alerts a where a.id = n.alert_id and a.owner_token = ${owner} and n.read_at is null`); return { ok: true }; }); // ---- Bookmarks --------------------------------------------------------------------------------- app.get<{ Querystring: { limit?: string } }>("/api/v1/bookmarks", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const rows = await db.execute>(sql`select b.created_at as bookmarked_at, b.note, x.* from bookmarks b join lateral (select ${EVENT_SELECT} from events e join sources s on s.id = e.source_id join sensors sen on sen.id = e.sensor_id where e.id = b.event_id) x on true where b.owner_token = ${owner} order by b.created_at desc limit ${Math.min(500, Number(req.query.limit ?? 100))}`); return { items: rows.rows }; }); app.post("/api/v1/bookmarks", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const body = z.object({ event_id: z.string().min(4).max(80), note: z.string().max(500).optional() }).parse(req.body ?? {}); const ev = (await db.execute<{ id: string }>(sql`select id from events where id = ${body.event_id} or slug = ${body.event_id}`)).rows[0]; if (!ev) return reply.status(404).send({ error: "event_not_found" }); const count = (await db.execute<{ n: string }>(sql`select count(*)::text as n from bookmarks where owner_token = ${owner}`)).rows[0]?.n; if (Number(count) >= 2000) return reply.status(429).send({ error: "too_many_bookmarks" }); await db.execute(sql`insert into bookmarks (owner_token, event_id, note) values (${owner}, ${ev.id}, ${body.note ?? null}) on conflict (owner_token, event_id) do update set note = coalesce(excluded.note, bookmarks.note)`); return { ok: true, event_id: ev.id }; }); app.delete<{ Params: { id: string } }>("/api/v1/bookmarks/:id", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; await db.execute(sql`delete from bookmarks where owner_token = ${owner} and (event_id = ${req.params.id} or event_id = (select id from events where slug = ${req.params.id}))`); return { ok: true }; }); app.get("/api/v1/bookmarks/ids", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const rows = await db.execute<{ event_id: string }>(sql`select event_id from bookmarks where owner_token = ${owner}`); return { ids: rows.rows.map((r) => r.event_id) }; }); // ---- Saved views ------------------------------------------------------------------------------- app.get("/api/v1/views", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; return { items: (await db.execute>(sql`select id, name, query, created_at from saved_views where owner_token = ${owner} order by created_at`)).rows }; }); app.post("/api/v1/views", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const body = z.object({ name: z.string().min(1).max(60), query: z.string().min(1).max(600) }).parse(req.body ?? {}); const count = (await db.execute<{ n: string }>(sql`select count(*)::text as n from saved_views where owner_token = ${owner}`)).rows[0]?.n; if (Number(count) >= 50) return reply.status(429).send({ error: "too_many_views" }); const id = newId("view"); await db.execute(sql`insert into saved_views (id, owner_token, name, query) values (${id}, ${owner}, ${body.name}, ${body.query})`); return { id, ...body }; }); app.delete<{ Params: { id: string } }>("/api/v1/views/:id", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; await db.execute(sql`delete from saved_views where id = ${req.params.id} and owner_token = ${owner}`); return { ok: true }; }); // ---- Custom URL monitors (spec §45, §87) ----------------------------------------------------------- const monitorBody = z.object({ url: z.string().url().max(2000), name: z.string().min(1).max(80).optional(), /** hourly | daily (maps to tier C / D) */ frequency: z.enum(["hourly", "daily"]).default("hourly"), /** low = only big changes, normal, high = every meaningful change */ sensitivity: z.enum(["low", "normal", "high"]).default("normal"), selector: z.string().max(200).optional(), keywords: z.array(z.string().min(1).max(60)).max(20).optional(), }); const ownerSourceId = (owner: string): string => `custom-${createHash("sha256").update(owner).digest("hex").slice(0, 12)}`; app.get("/api/v1/monitors", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const sid = ownerSourceId(owner); const rows = await db.execute>(sql`select id, name, url, tier, health, status, enabled, config, next_check_at, last_check_at, last_change_at, last_event_at, last_status, last_error, total_runs, raw_changes, meaningful_changes, created_at from sensors where source_id = ${sid} order by created_at`); return { items: rows.rows, limit: config.monitorsPerOwner, source_id: sid }; }); app.post("/api/v1/monitors", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const body = monitorBody.parse(req.body ?? {}); let host: string; try { const u = await assertUrlAllowed(body.url); if (!["http:", "https:"].includes(u.url.protocol)) return reply.status(400).send({ error: "scheme_not_allowed" }); host = u.url.hostname; } catch (e) { return reply.status(400).send({ error: "url_rejected", detail: e instanceof UrlPolicyError ? e.message : "URL not allowed" }); } const sid = ownerSourceId(owner); const n = Number((await db.execute<{ n: string }>(sql`select count(*)::text as n from sensors where source_id = ${sid} and enabled`)).rows[0]?.n ?? 0); if (n >= config.monitorsPerOwner) return reply.status(429).send({ error: "monitor_limit_reached", limit: config.monitorsPerOwner }); // Sensor auto-test (spec §72): fetch + normalize before activation; baseline is taken by the engine. const sensorId = `${sid}_${newId("m").slice(2)}`; const cfg: Record = { custom: true, sensitivity: body.sensitivity, ...(body.selector ? { selector: body.selector } : {}), ...(body.keywords?.length ? { keywords: body.keywords } : {}) }; const tier = body.frequency === "hourly" ? "C" : "D"; const connector = getConnector("http"); const endpoint = { id: sensorId, sourceId: sid, name: body.name ?? host, url: body.url, type: "HTML" as const, tier: tier as "C" | "D", connector: "http", config: cfg, etag: null, lastModified: null, state: null }; const obs = await connector.fetch(endpoint); if (obs.error) return reply.status(422).send({ error: "fetch_failed", detail: `${obs.error.code}: ${obs.error.message}` }); if (obs.meta.status >= 400) return reply.status(422).send({ error: "http_error", status: obs.meta.status }); let norm; try { norm = await connector.normalize(endpoint, obs); } catch (e) { return reply.status(422).send({ error: "unparseable", detail: (e as Error).message }); } if ((norm.text ?? "").length < 40 && norm.mode === "text") return reply.status(422).send({ error: "thin_content", detail: "The page has almost no server-rendered text (client-side app?)." }); await db.execute(sql`insert into sources (id, name, domain, homepage, description, categories, tier, importance_weight, enabled, llm_enabled, kind, owner_token, first_party, notes) values (${sid}, ${"Custom monitors"}, ${"custom.websensor.io"}, ${null}, ${"Private URL monitors"}, ${"{custom}"}::text[], 'C', 0.5, true, false, 'custom', ${owner}, true, 'owner-scoped custom monitors') on conflict (id) do nothing`); await db.execute(sql`insert into sensors (id, source_id, name, url, type, connector, tier, importance_weight, config, priority, status, validated_at, enabled, next_check_at) values (${sensorId}, ${sid}, ${body.name ?? host}, ${body.url}, 'HTML', 'http', ${tier}, ${body.sensitivity === "high" ? 1.3 : body.sensitivity === "low" ? 0.7 : 1}, ${JSON.stringify(cfg)}::jsonb, 3, 'VALIDATED', now(), true, now())`); return { id: sensorId, source_id: sid, url: body.url, tier, status: "VALIDATED", test: { http_status: obs.meta.status, content_type: obs.meta.contentType, bytes: obs.meta.contentLength, title: norm.title ?? null, mode: norm.mode, extraction_confidence: norm.extractionConfidence } }; }); app.delete<{ Params: { id: string } }>("/api/v1/monitors/:id", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const sid = ownerSourceId(owner); await db.execute(sql`delete from sensors where id = ${req.params.id} and source_id = ${sid}`); return { ok: true }; }); app.get<{ Params: { id: string }; Querystring: { limit?: string } }>("/api/v1/monitors/:id/events", async (req, reply) => { const owner = requireOwner(req, reply); if (!owner) return; const sid = ownerSourceId(owner); const sensor = (await db.execute<{ id: string }>(sql`select id from sensors where id = ${req.params.id} and source_id = ${sid}`)).rows[0]; if (!sensor) return reply.status(404).send({ error: "not_found" }); const [events, changes] = await Promise.all([ listEvents({ sensor: sensor.id, limit: Math.min(100, Number(req.query.limit ?? 30)), includeCustom: true }), db.execute>(sql`select id, detected_at, kind, signal, change_class, field_changes, meaningful, event_id, old_snapshot_id, new_snapshot_id from changes where sensor_id = ${sensor.id} order by detected_at desc limit 30`).then((r) => r.rows), ]); return { events: events.items, changes }; }); // keep eventConditions referenced for owner-scoped searches later void eventConditions; }