spb/datacenterindex
Public
HTML 53.9%
TypeScript 44.5%
JavaScript 0.6%
SQL 0.5%
1"use client";23import Link from "next/link";4import { useMemo, useState } from "react";5import { Badge, Button, EmptyState, RelativeTime, SearchInput, SkeletonRows, Table, TBody, Td, Th, THead, Tr } from "@/components/ui";6import { cn } from "@/components/ui/cn";7import { DASH, fmtInt } from "@/lib/format";8import { SOURCE_KIND_LABEL } from "@/lib/labels";9import type { ConnectorHealthDTO } from "@/lib/admin-api";10import { PauseResumeButton, RunForm, ScheduleEditor } from "./connector-actions";11import { ErrorNote, fmtMs, fmtPct01, HEALTH, HealthBadge, HealthDot, Mono, RefreshButton, Segmented, Select } from "./primitives";12import { AdminPageHeader } from "./shell";13import { useToast } from "./toast";14import { useAdminMutation, useAdminQuery } from "./use-admin-query";15import { useUrlState } from "./use-url-state";1617/* ------------------------------------------------------------------------------------------18 Connector HEALTH CENTER (§11–12): one row per connector with the crawl / extraction / publication19 counters of the last window, a spec status badge, quarantine and reprocess controls.20 ------------------------------------------------------------------------------------------ */2122type SortKey =23 | "id"24 | "kind"25 | "health"26 | "lastRunAt"27 | "lastSuccessAt"28 | "lastFailureAt"29 | "nextRunAt"30 | "urlsDiscovered"31 | "urlsFetched"32 | "newDocs"33 | "changedDocs"34 | "extracted"35 | "recordsCreated"36 | "recordsModified"37 | "rejectedClaims"38 | "httpErrors"39 | "antiBotEscalations"40 | "scrapflyRequests"41 | "firecrawlRequests"42 | "failed"43 | "extractionSuccess"44 | "avgResponseMs"45 | "credits"46 | "parserVersion";4748const HEALTH_ORDER: Record<string, number> = { blocked: 0, failing: 1, schema_change: 2, degraded: 3, quarantine: 4, no_new_content: 5, ok: 6, never_run: 7, paused: 8 };49const HEALTH_FILTERS = ["all", "ok", "degraded", "blocked", "schema_change", "no_new_content", "failing", "quarantine", "paused", "never_run", "enabled", "disabled"] as const;50type HealthFilter = (typeof HEALTH_FILTERS)[number];51const COLUMN_SETS = ["compact", "full"] as const;52type ColumnSet = (typeof COLUMN_SETS)[number];5354function num(v: unknown): number | null {55 return typeof v === "number" && Number.isFinite(v) ? v : null;56}5758function sortValue(c: ConnectorHealthDTO, k: SortKey): number | string {59 switch (k) {60 case "credits":61 return c.cost.credits;62 case "health":63 return HEALTH_ORDER[c.health] ?? 9;64 case "lastRunAt":65 case "nextRunAt":66 case "lastSuccessAt":67 case "lastFailureAt":68 return c[k] ? new Date(c[k] as string).getTime() : -1;69 case "id":70 case "kind":71 case "parserVersion":72 return c[k];73 default: {74 const v = num(c[k]);75 return v ?? -1;76 }77 }78}7980/** A counter cell: "—" when the API does not carry the field (older API), highlighted when non-zero and `hot`. */81function Count({ v, hot, className }: { v: number | null | undefined; hot?: "accent" | "warn" | "danger"; className?: string }) {82 if (v === null || v === undefined) return <span className="text-ink-4">{DASH}</span>;83 return <span className={cn(v > 0 && hot === "accent" && "text-accent-ink", v > 0 && hot === "warn" && "text-warn", v > 0 && hot === "danger" && "text-danger", className)}>{fmtInt(v)}</span>;84}8586export function ConnectorsTable() {87 const q = useAdminQuery<ConnectorHealthDTO[]>("connectors", { refreshMs: 60_000, timeoutMs: 30_000 });88 const [params, setParams] = useUrlState();89 const kind = params.kind ?? "";90 const health = (HEALTH_FILTERS.includes(params.health as HealthFilter) ? params.health : "all") as HealthFilter;91 const columns = (COLUMN_SETS.includes(params.cols as ColumnSet) ? params.cols : "compact") as ColumnSet;92 const full = columns === "full";93 const search = params.q ?? "";94 const [sort, setSort] = useState<{ key: SortKey; dir: "asc" | "desc" }>({ key: "health", dir: "asc" });95 const [open, setOpen] = useState<{ id: string; mode: "run" | "schedule" } | null>(null);96 const { busy, run } = useAdminMutation();97 const toast = useToast();98 const [bulk, setBulk] = useState<{ done: number; total: number } | null>(null);99100 const list = useMemo(() => q.data ?? [], [q.data]);101 const counts = useMemo(() => {102 const c: Record<string, number> = { all: list.length, enabled: 0, disabled: 0 };103 for (const x of list) {104 c[x.health] = (c[x.health] ?? 0) + 1;105 if (x.enabled) c.enabled!++;106 else c.disabled!++;107 }108 return c;109 }, [list]);110 const kinds = useMemo(() => [...new Set(list.map((c) => c.kind))].sort(), [list]);111112 const rows = useMemo(() => {113 const s = search.trim().toLowerCase();114 const out = list.filter((c) => (!kind || c.kind === kind) && (health === "all" || (health === "enabled" ? c.enabled : health === "disabled" ? !c.enabled : c.health === health)) && (!s || c.id.includes(s) || c.sourceName.toLowerCase().includes(s) || c.domain.toLowerCase().includes(s)));115 out.sort((a, b) => {116 const va = sortValue(a, sort.key);117 const vb = sortValue(b, sort.key);118 const cmp = typeof va === "number" && typeof vb === "number" ? va - vb : String(va).localeCompare(String(vb));119 return (sort.dir === "asc" ? cmp : -cmp) || a.id.localeCompare(b.id);120 });121 return out;122 }, [list, kind, health, search, sort]);123124 const totals = useMemo(() => {125 const t = { urlsFetched: 0, newDocs: 0, changedDocs: 0, recordsCreated: 0, rejectedClaims: 0, httpErrors: 0, antiBotEscalations: 0, scrapfly: 0, firecrawl: 0 };126 for (const c of rows) {127 t.urlsFetched += c.urlsFetched ?? c.fetched ?? 0;128 t.newDocs += c.newDocs ?? 0;129 t.changedDocs += c.changedDocs ?? c.changed ?? 0;130 t.recordsCreated += c.recordsCreated ?? 0;131 t.rejectedClaims += c.rejectedClaims ?? 0;132 t.httpErrors += c.httpErrors ?? 0;133 t.antiBotEscalations += c.antiBotEscalations ?? 0;134 t.scrapfly += c.scrapflyRequests ?? c.cost.scrapfly;135 t.firecrawl += c.firecrawlRequests ?? c.cost.firecrawl;136 }137 return t;138 }, [rows]);139140 const th = (key: SortKey, label: string, numeric?: boolean, title?: string) => (141 <Th numeric={numeric} sort={sort.key === key ? sort.dir : null} className="cursor-pointer select-none" title={title} onClick={() => setSort((s) => ({ key, dir: s.key === key && s.dir === "asc" ? "desc" : s.key === key ? "asc" : numeric ? "desc" : "asc" }))}>142 {label}143 </Th>144 );145146 /** Sequential POST run {task: reprocess} for every enabled, non-quarantined connector in the current view. */147 const reprocessAll = async () => {148 const targets = rows.filter((c) => c.enabled && !c.quarantine && c.health !== "paused");149 if (!targets.length) return toast({ tone: "warn", title: "Nothing to reprocess", description: "No enabled connector in the current view." });150 if (!window.confirm(`Enqueue a REPROCESS run (re-extract stored bodies, no network fetch) for ${targets.length} enabled connector${targets.length === 1 ? "" : "s"} in the current view?`)) return;151 setBulk({ done: 0, total: targets.length });152 let ok = 0;153 let failed = 0;154 for (const c of targets) {155 const r = await run<{ enqueued: boolean }>(`reprocess-all`, `connectors/${encodeURIComponent(c.id)}/run`, { body: { task: "reprocess" } });156 if (r.ok) ok++;157 else failed++;158 setBulk((b) => (b ? { ...b, done: b.done + 1 } : b));159 }160 setBulk(null);161 toast({ tone: failed ? "warn" : "ok", title: `Reprocess enqueued for ${ok} connector${ok === 1 ? "" : "s"}`, description: failed ? `${failed} failed to enqueue` : undefined });162 void q.refresh();163 };164165 return (166 <div>167 <AdminPageHeader168 title="Connector health center"169 description={q.data ? `${fmtInt(list.length)} connectors · ${fmtInt(counts.enabled ?? 0)} enabled · counters over the last window (documents registry + connector runs), status from the last run` : "Connector control center"}170 aside={171 <>172 <Button size="xs" variant="secondary" onClick={() => void reprocessAll()} disabled={busy !== null || bulk !== null || rows.length === 0} title="POST run {task: reprocess} for every enabled connector in the current view, one after the other">173 {bulk ? `Reprocessing ${bulk.done}/${bulk.total}…` : "Reprocess stale (all enabled)"}174 </Button>175 <RefreshButton onClick={() => void q.refresh()} busy={q.refreshing} updatedAt={q.updatedAt} />176 </>177 }178 />179 <ErrorNote error={q.error && !q.data ? q.error : null} onRetry={() => void q.refresh()} className="mb-3" />180181 <div className="mb-3 flex flex-col gap-2 lg:flex-row lg:items-center">182 <Segmented label="Health" value={health} onChange={(v) => setParams({ health: v === "all" ? null : v })} options={HEALTH_FILTERS.map((h) => ({ value: h, label: h === "all" ? "All" : h === "enabled" || h === "disabled" ? h[0]!.toUpperCase() + h.slice(1) : (HEALTH[h]?.label ?? h).toLowerCase().replace(/^./, (m) => m.toUpperCase()), count: counts[h] ?? 0 }))} />183 <div className="flex flex-1 flex-wrap gap-2">184 <Select value={kind} onChange={(e) => setParams({ kind: e.target.value || null })} className="w-[170px] shrink-0" aria-label="Kind">185 <option value="">All kinds</option>186 {kinds.map((k) => (187 <option key={k} value={k}>188 {SOURCE_KIND_LABEL[k] ?? k}189 </option>190 ))}191 </Select>192 <SearchInput size="sm" value={search} onChange={(v) => setParams({ q: v || null })} placeholder="Search id, name, domain" wrapperClassName="min-w-[180px] flex-1" />193 <Segmented label="Columns" value={columns} onChange={(v) => setParams({ cols: v === "compact" ? null : v })} options={[{ value: "compact", label: "Compact" }, { value: "full", label: "All columns" }]} />194 </div>195 </div>196197 {q.data && rows.length > 0 && (198 <p className="mb-2 text-[11.5px] text-ink-3">199 In view: <span className="figure text-ink-2">{fmtInt(totals.urlsFetched)}</span> URLs fetched · <span className="figure text-ink-2">{fmtInt(totals.newDocs)}</span> new docs · <span className="figure text-ink-2">{fmtInt(totals.changedDocs)}</span> changed · <span className="figure text-ink-2">{fmtInt(totals.recordsCreated)}</span> records created · <span className={cn("figure", totals.rejectedClaims > 0 ? "text-warn" : "text-ink-2")}>{fmtInt(totals.rejectedClaims)}</span> rejected claims · <span className={cn("figure", totals.httpErrors > 0 ? "text-danger" : "text-ink-2")}>{fmtInt(totals.httpErrors)}</span> HTTP errors · <span className="figure text-ink-2">{fmtInt(totals.antiBotEscalations)}</span> anti-bot escalations · Scrapfly <span className="figure text-ink-2">{fmtInt(totals.scrapfly)}</span> / Firecrawl <span className="figure text-ink-2">{fmtInt(totals.firecrawl)}</span> requests.200 </p>201 )}202203 {q.loading && !q.data ? (204 <SkeletonRows rows={10} />205 ) : rows.length === 0 ? (206 <EmptyState compact title="No connectors match" description="Adjust the health / kind filters or the search." />207 ) : (208 <div className="max-w-full overflow-x-auto rounded-sm border border-hair bg-surface">209 <Table className={cn("text-[12.5px] sm:[&_td]:py-1.5", full ? "min-w-[2400px]" : "min-w-[1360px]")} responsive={false}>210 <THead>211 <Tr>212 <Th className="pl-3">{""}</Th>213 {th("id", "Source")}214 {full && <Th>Domain</Th>}215 {th("kind", "Kind")}216 {th("health", "Status")}217 {th("lastRunAt", "Last crawl")}218 {full && th("lastSuccessAt", "Last success")}219 {th("lastFailureAt", "Last failure")}220 {full && th("nextRunAt", "Next")}221 {th("urlsDiscovered", "URLs disc.", true, "URLs discovered")}222 {th("urlsFetched", "URLs fetched", true)}223 {th("newDocs", "New docs", true)}224 {th("changedDocs", "Changed", true, "Changed documents")}225 {th("extracted", "Extract.", true, "Extractions")}226 {full && th("extractionSuccess", "Extract %", true, "Extraction success rate")}227 {th("recordsCreated", "Rec. created", true, "Records created")}228 {full && th("recordsModified", "Rec. modified", true, "Records modified")}229 {th("rejectedClaims", "Rej. claims", true, "Rejected claims")}230 {th("httpErrors", "HTTP err.", true, "HTTP errors")}231 {full && th("antiBotEscalations", "Anti-bot", true, "Anti-bot escalations (L2+)")}232 {full && th("scrapflyRequests", "Scrapfly", true, "Scrapfly requests")}233 {full && th("firecrawlRequests", "Firecrawl", true, "Firecrawl requests")}234 {th("avgResponseMs", "Avg latency", true)}235 {full && th("credits", "Credits", true, "Premium credits (7 d)")}236 {full && th("parserVersion", "Parser")}237 <Th className="pr-3">Actions</Th>238 </Tr>239 </THead>240 <TBody>241 {rows.map((c) => {242 const isOpen = open?.id === c.id;243 return (244 <ConnectorRow key={c.id} c={c} full={full} open={isOpen ? open.mode : null} onToggle={(mode) => setOpen((o) => (o?.id === c.id && o.mode === mode ? null : { id: c.id, mode }))} onChanged={() => void q.refresh()} mutate={q.mutate} />245 );246 })}247 </TBody>248 </Table>249 </div>250 )}251 <p className="mt-2 text-[11px] text-ink-4">Status: HEALTHY · DEGRADED (partial failures / slow) · BLOCKED (anti-bot or robots) · SCHEMA CHANGE (parser yields nothing on changed pages) · NO NEW CONTENT · FAILED · QUARANTINE (extracts and previews, publishes nothing) · PAUSED · NEVER RUN. Counters that the API does not report yet show “—”.</p>252 </div>253 );254}255256function ConnectorRow({ c, full, open, onToggle, onChanged, mutate }: { c: ConnectorHealthDTO; full: boolean; open: "run" | "schedule" | null; onToggle: (m: "run" | "schedule") => void; onChanged: () => void; mutate: (fn: (prev: ConnectorHealthDTO[] | null) => ConnectorHealthDTO[] | null) => void }) {257 const cost = c.cost;258 const { busy, run } = useAdminMutation();259 const toast = useToast();260 const quarantined = Boolean(c.quarantine) || c.health === "quarantine";261 const colSpan = full ? 27 : 18;262263 const toggleQuarantine = async () => {264 const on = !quarantined;265 if (on && !window.confirm(`Quarantine ${c.id}? It will keep extracting and previewing but publish nothing until released.`)) return;266 const r = await run<{ id: string; quarantine: boolean; health: string }>("quarantine", `connectors/${encodeURIComponent(c.id)}/quarantine`, { body: { on } });267 if (r.ok) {268 toast({ tone: "ok", title: on ? `${c.id} quarantined` : `${c.id} released from quarantine` });269 mutate((prev) => prev?.map((x) => (x.id === c.id ? { ...x, quarantine: r.data.quarantine, health: r.data.health as ConnectorHealthDTO["health"] } : x)) ?? prev);270 } else toast({ tone: "danger", title: "Quarantine change failed", description: r.error });271 };272 const reprocess = async () => {273 const r = await run<{ enqueued: boolean; job?: { id?: string } }>("reprocess", `connectors/${encodeURIComponent(c.id)}/run`, { body: { task: "reprocess" } });274 if (r.ok) toast({ tone: "ok", title: `Reprocess enqueued · ${c.id}`, description: r.data.job?.id ? `job ${r.data.job.id}` : "re-extracts stored bodies, no network fetch" });275 else toast({ tone: "danger", title: "Reprocess failed to enqueue", description: r.error });276 };277278 return (279 <>280 <Tr className={cn(!c.enabled && "opacity-70", open && "bg-surface-2/60")}>281 <Td className="pl-3">282 <HealthDot health={c.lastStatus === "running" ? "running" : quarantined ? "quarantine" : c.health} />283 </Td>284 <Td primary>285 <Link href={`/admin/connectors/${encodeURIComponent(c.id)}`} className="font-mono text-[12.5px] font-medium text-ink hover:text-accent hover:underline">286 {c.id}287 </Link>288 <div className="max-w-[220px] truncate text-[11.5px] text-ink-3" title={`${c.sourceName} · ${c.domain}`}>289 {c.sourceName}290 {!full && <span className="text-ink-4"> · {c.domain}</span>}291 </div>292 </Td>293 {full && (294 <Td muted>295 <Mono className="text-[11.5px]">{c.domain}</Mono>296 </Td>297 )}298 <Td muted>{SOURCE_KIND_LABEL[c.kind] ?? c.kind}</Td>299 <Td>300 <div className="flex flex-col items-start gap-1">301 <HealthBadge health={c.lastStatus === "running" ? "running" : quarantined ? "quarantine" : c.health} />302 <span className="flex flex-wrap gap-1">303 {!c.enabled && <Badge size="xs">disabled</Badge>}304 <Badge size="xs" mono tone="outline">305 {c.mode}306 </Badge>307 {(c.consecutiveFailures ?? 0) > 0 && (308 <Badge size="xs" tone="danger" title="Consecutive failed runs">309 ×{c.consecutiveFailures}310 </Badge>311 )}312 {c.blockedSince && (313 <Badge size="xs" tone="danger" title={`Blocked since ${c.blockedSince}`}>314 blocked <RelativeTime iso={c.blockedSince} />315 </Badge>316 )}317 </span>318 </div>319 </Td>320 <Td muted>{c.lastRunAt ? <RelativeTime iso={c.lastRunAt} /> : DASH}</Td>321 {full && <Td muted>{c.lastSuccessAt ? <RelativeTime iso={c.lastSuccessAt} /> : DASH}</Td>}322 <Td className={cn(c.lastFailureAt && c.lastSuccessAt && c.lastFailureAt > c.lastSuccessAt ? "text-danger" : "text-ink-3")}>{c.lastFailureAt ? <RelativeTime iso={c.lastFailureAt} /> : DASH}</Td>323 {full && <Td muted>{c.nextRunAt ? <RelativeTime iso={c.nextRunAt} /> : DASH}</Td>}324 <Td numeric>325 <Count v={c.urlsDiscovered ?? c.discovered} />326 </Td>327 <Td numeric>328 <Count v={c.urlsFetched ?? c.fetched} />329 </Td>330 <Td numeric>331 <Count v={c.newDocs} hot="accent" />332 </Td>333 <Td numeric>334 <Count v={c.changedDocs ?? c.changed} hot="accent" />335 </Td>336 <Td numeric>337 <Count v={c.extracted} />338 </Td>339 {full && <Td numeric className={cn(c.extractionSuccess != null && c.extractionSuccess < 0.7 && "text-warn")}>{fmtPct01(c.extractionSuccess)}</Td>}340 <Td numeric>341 <Count v={c.recordsCreated} hot="accent" />342 </Td>343 {full && (344 <Td numeric>345 <Count v={c.recordsModified} />346 </Td>347 )}348 <Td numeric>349 <Count v={c.rejectedClaims} hot="warn" />350 </Td>351 <Td numeric>352 <Count v={c.httpErrors ?? c.failed} hot="danger" />353 </Td>354 {full && (355 <Td numeric>356 <Count v={c.antiBotEscalations} hot="warn" />357 </Td>358 )}359 {full && (360 <Td numeric>361 <Count v={c.scrapflyRequests ?? cost.scrapfly} />362 </Td>363 )}364 {full && (365 <Td numeric>366 <Count v={c.firecrawlRequests ?? cost.firecrawl} />367 </Td>368 )}369 <Td numeric>{c.avgResponseMs != null ? fmtMs(c.avgResponseMs) : DASH}</Td>370 {full && (371 <Td numeric title={`direct ${cost.direct} · firecrawl ${cost.firecrawl} · scrapfly ${cost.scrapfly} · ${cost.credits} credits (7 d)`}>372 {cost.credits > 0 ? <span className="text-accent">{fmtInt(cost.credits)}</span> : <span className="text-ink-4">0</span>}373 </Td>374 )}375 {full && (376 <Td>377 <Mono>{c.parserVersion}</Mono>378 </Td>379 )}380 <Td className="pr-3">381 <div className="flex items-center gap-1">382 <Button size="xs" variant={open === "run" ? "primary" : "secondary"} onClick={() => onToggle("run")}>383 Run384 </Button>385 <Button size="xs" variant="ghost" onClick={() => void reprocess()} disabled={busy !== null} title="Re-extract stored bodies with the current parser (no network fetch)">386 Reprocess387 </Button>388 <PauseResumeButton connector={c} onChanged={(paused) => mutate((prev) => prev?.map((x) => (x.id === c.id ? { ...x, health: paused ? "paused" : x.lastRunAt ? "ok" : "never_run", enabled: !paused && x.enabled } : x)) ?? prev)} />389 <Button size="xs" variant={quarantined ? "secondary" : "ghost"} onClick={() => void toggleQuarantine()} disabled={busy !== null} title={quarantined ? "Release: the connector publishes again" : "Quarantine: extract and preview, publish nothing"}>390 {quarantined ? "Release" : "Quarantine"}391 </Button>392 <Button size="xs" variant={open === "schedule" ? "primary" : "ghost"} onClick={() => onToggle("schedule")}>393 Schedule394 </Button>395 </div>396 </Td>397 </Tr>398 {open && (399 <tr className="bg-surface-2/60">400 <td colSpan={colSpan} className="px-3 pb-3 pt-1">401 {open === "run" ? <RunForm connector={c} compact onDone={onChanged} /> : <ScheduleEditor connector={c} onSaved={(schedule) => mutate((prev) => prev?.map((x) => (x.id === c.id ? { ...x, schedule } : x)) ?? prev)} />}402 </td>403 </tr>404 )}405 </>406 );407}408