HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1import type { Metadata } from 'next';2import Link from 'next/link';3import { DataStrip } from '@/components/layout/terminal';4import { BreadcrumbLd } from '@/components/meta/breadcrumb-ld';5import { TierBadge } from '@/components/ui/badges';6import { DataTable, EmptyRow, Td, Th } from '@/components/ui/data-table';7import { Hint } from '@/components/ui/hint';8import { Container, Note, PageHeader } from '@/components/ui/section';9import { Unavailable } from '@/components/ui/unavailable';10import { adminApi } from '@/lib/admin/admin-api';11import { getAdminToken } from '@/lib/admin/session';12import type { AdminConnector } from '@/lib/admin/types';13import { apiD3, safe } from '@/lib/api';14import { cn } from '@/lib/cn';15import { fmtAgo, fmtDateTime, fmtDuration, fmtInt, fmtPct, num, titleCase } from '@/lib/format';16import { routes, SITE_NAME } from '@/lib/site';17import type { SourceRow11 } from '@/lib/types';1819export const metadata: Metadata = { title: 'Sources — health center: every site AI Atlas reads, tier, freshness and connector status', description: 'Transparency page: the sources AI Atlas crawls, their tier, last successful observation, documents and connector health. Operators see request-level health when signed in.', alternates: { canonical: '/sources' } };20/** Reads the admin cookie to decide whether to show operator columns → dynamic. API fetches stay ISR-cached. */21export const dynamic = 'force-dynamic';2223const HEALTH: Record<string, string> = { ok: 'text-positive', healthy: 'text-positive', degraded: 'text-warning', failing: 'text-danger', broken: 'text-danger', disabled: 'text-ink-3', unknown: 'text-ink-3' };24const DOT: Record<string, string> = { ok: 'bg-positive', healthy: 'bg-positive', degraded: 'bg-warning', disabled: 'bg-ink-3', unknown: 'bg-ink-3' };2526function statusOf(s: SourceRow11): string {27 if (!s.enabled) return 'disabled';28 const hs = (s.connectors ?? []).map((c) => c.health);29 if (!hs.length) return 'no connector';30 if (hs.some((h) => h === 'failing' || h === 'broken')) return 'failing';31 if (hs.some((h) => h === 'degraded')) return 'degraded';32 if (hs.every((h) => h === 'ok' || h === 'healthy')) return 'ok';33 return hs[0] ?? 'unknown';34}35function lastSuccess(s: SourceRow11): string | null {36 const ts = (s.connectors ?? []).map((c) => c.last_success_at).filter((x): x is string => !!x);37 return ts.length ? ts.sort().at(-1)! : s.last_crawled_at;38}3940export default async function SourcesPage() {41 const token = await getAdminToken();42 const [res, admin] = await Promise.all([43 safe(apiD3.sources()),44 token45 ? adminApi.connectors().catch(() => null)46 : Promise.resolve(null),47 ]);48 const items = ((res?.items ?? []) as SourceRow11[]).slice().sort((a, b) => a.tier - b.tier || (num(b.documents) ?? 0) - (num(a.documents) ?? 0));49 const byConnector = new Map<string, AdminConnector>((admin?.items ?? []).map((c) => [c.name, c]));50 const isAdmin = !!admin;51 const connectors = items.flatMap((s) => s.connectors ?? []);52 const count = (pred: (h: string) => boolean) => connectors.filter((c) => pred(c.health)).length;53 const docs = items.reduce((n, s) => n + (num(s.documents) ?? 0), 0);54 const snaps = items.reduce((n, s) => n + (num(s.snapshots) ?? 0), 0);55 const claims = items.reduce((n, s) => n + (num(s.claims) ?? 0), 0);56 const stale = items.filter((s) => {57 const t = lastSuccess(s);58 return s.enabled && t && Date.now() - new Date(t).getTime() > 3 * 86400000;59 }).length;6061 return (62 <Container wide>63 <BreadcrumbLd items={[{ name: SITE_NAME, href: '/' }, { name: 'Sources', href: '/sources' }]} />64 <PageHeader eyebrow="Sources · health center" title="What AI Atlas reads" lede="Every source is crawled directly and archived. Tier grades reliability (1 official → 4 unverified); status is the live health of the connectors reading it. Signed-in operators see request-level health." aside={res ? <p className="tnum text-sm text-ink-3">{fmtInt(res.total ?? items.length)} sources · {fmtInt(connectors.length)} connectors</p> : undefined} />65 <div className="pb-16">66 {!res ? (67 <Unavailable what="Sources" />68 ) : (69 <>70 <DataStrip71 dense72 items={[73 { label: 'Sources', value: fmtInt(res.total ?? items.length), definition: 'Registered sources (registry/sources.yaml), enabled or not.' },74 { label: 'Documents', value: fmtInt(docs), definition: 'Distinct documents discovered across all sources.' },75 { label: 'Snapshots', value: snaps ? fmtInt(snaps) : '—', definition: 'Archived fetches (raw + text) across all sources.' },76 { label: 'Claims', value: claims ? fmtInt(claims) : '—', definition: 'Claims whose source is this registry entry.' },77 { label: 'Connectors OK', value: fmtInt(count((h) => h === 'ok' || h === 'healthy')), definition: 'Connectors whose last run succeeded within their interval.' },78 { label: 'Degraded', value: fmtInt(count((h) => h === 'degraded')), definition: 'Connectors with recent failures or a breakage suspicion.', delta: count((h) => h === 'degraded') ? { value: 'attention', tone: 'neutral' } : undefined },79 { label: 'Failing', value: fmtInt(count((h) => h === 'failing' || h === 'broken')), definition: 'Connectors whose circuit breaker is open or whose last runs failed.' },80 { label: 'Stale > 3 d', value: fmtInt(stale), definition: 'Enabled sources with no successful observation in the last 3 days.' },81 ]}82 />83 <div className="mt-4 flex flex-wrap items-center gap-x-4 gap-y-1 text-xs text-ink-3">84 <span>85 Tiers:{' '}86 {Object.entries(res.tiers ?? {}).map(([t, l], i) => (87 <span key={t}>88 {i > 0 && ' · '}89 <TierBadge tier={Number(t)} /> {l}90 </span>91 ))}92 </span>93 {isAdmin && <span className="rounded-[3px] bg-accent-soft px-1.5 text-[10px] uppercase tracking-wide text-accent">operator columns on</span>}94 </div>95 <DataTable caption="Sources" scroll compact className="mt-4">96 <thead>97 <tr>98 <Th>Source</Th>99 <Th>Category</Th>100 <Th>Tier</Th>101 <Th>Last successful observation</Th>102 <Th num>Documents</Th>103 <Th>Status</Th>104 <Th>Connectors</Th>105 {isAdmin && (106 <>107 <Th>Last request</Th>108 <Th>109 HTTP <Hint text="Not exposed per connector by /admin/connectors (documents carry last_status individually) — shown as —." />110 </Th>111 <Th num>112 Unchanged share <Hint text="docs_unchanged / docs_fetched of the last run — a proxy for the conditional-request hit rate, which the API does not report directly." />113 </Th>114 <Th num>Docs changed</Th>115 <Th num>Claims</Th>116 <Th num>Errors 7 d</Th>117 <Th>Parser</Th>118 <Th>119 LLM <Hint text="needs_llm flag from the connector registry; the LLM fallback *rate* is not exposed by the API." />120 </Th>121 <Th num>Latency</Th>122 </>123 )}124 </tr>125 </thead>126 <tbody>127 {items.length === 0 && <EmptyRow cols={isAdmin ? 16 : 7}>No sources registered.</EmptyRow>}128 {items.map((s) => {129 const st = statusOf(s);130 const ls = lastSuccess(s);131 const conns = (s.connectors ?? []).map((c) => byConnector.get(c.name)).filter((c): c is AdminConnector => !!c);132 const run = conns.map((c) => c.last_run).find((r) => r) ?? null;133 const fetched = num(run?.docs_fetched);134 const unchanged = num(run?.docs_unchanged);135 return (136 <tr key={s.key} className={!s.enabled ? 'opacity-60' : undefined}>137 <Td primary>138 <a href={s.base_url ?? `https://${s.domain}`} target="_blank" rel="noopener noreferrer" className="hover:text-accent">139 {s.name}140 </a>141 <span className="mono block text-[11px] text-ink-3">142 {s.domain}143 {s.organization ? ` · ${s.organization.name}` : ''}144 </span>145 </Td>146 <Td label="Category" className="text-ink-2">{titleCase(s.category)}<span className="block text-[11px] text-ink-3">{titleCase(s.kind)}</span></Td>147 <Td label="Tier">148 <TierBadge tier={s.tier} />149 </Td>150 <Td label="Last observation" className="text-ink-2" title={ls ? fmtDateTime(ls) : undefined}>{ls ? fmtAgo(ls) : <span className="text-ink-3">never</span>}</Td>151 <Td num label="Documents" className="tnum">{fmtInt(s.documents)}</Td>152 <Td label="Status">153 <span className={cn('inline-flex items-center gap-1.5 text-xs', HEALTH[st] ?? 'text-ink-3')}>154 <span className={cn('inline-block size-1.5 rounded-full', DOT[st] ?? (st === 'no connector' ? 'bg-ink-3' : 'bg-danger'))} aria-hidden />155 {st}156 </span>157 </Td>158 <Td label="Connectors">159 {s.connectors?.length ? (160 <ul className="space-y-0.5 text-xs">161 {s.connectors.map((c) => (162 <li key={c.name} className="flex flex-wrap items-center gap-x-2">163 <span className="mono text-ink-2">{c.name}</span>164 <span className={HEALTH[c.health] ?? 'text-ink-3'}>{c.health}</span>165 <span className="text-ink-3">every {fmtDuration(c.interval_seconds)}</span>166 </li>167 ))}168 </ul>169 ) : (170 <span className="text-ink-3">—</span>171 )}172 </Td>173 {isAdmin && (174 <>175 <Td label="Last request" className="text-xs text-ink-2">{conns[0]?.last_attempt_at ? fmtAgo(conns[0].last_attempt_at) : '—'}</Td>176 <Td label="HTTP" className="text-xs text-ink-3">—</Td>177 <Td num label="Unchanged share" className="tnum text-xs">{fetched && unchanged !== null ? fmtPct((unchanged / fetched) * 100, 0) : '—'}</Td>178 <Td num label="Docs changed" className="tnum text-xs">{run ? fmtInt(run.docs_changed) : '—'}</Td>179 <Td num label="Claims" className="tnum text-xs">{run ? fmtInt(run.claims_written) : '—'}</Td>180 <Td num label="Errors 7 d" className={cn('tnum text-xs', conns.some((c) => (num((c as unknown as { errors_7d?: unknown }).errors_7d) ?? 0) > 0) && 'text-danger')}>{conns.length ? fmtInt(conns.reduce((n, c) => n + (num((c as unknown as { errors_7d?: unknown }).errors_7d) ?? 0), 0)) : '—'}</Td>181 <Td label="Parser" className="mono text-xs text-ink-2">{conns[0]?.parser_version ?? '—'}</Td>182 <Td label="LLM" className="text-xs text-ink-2">{conns.length ? (conns.some((c) => (c.meta as { needs_llm?: boolean } | undefined)?.needs_llm) ? 'fallback on' : 'deterministic') : '—'}</Td>183 <Td num label="Latency" className="tnum text-xs">{num(run?.duration_ms) === null ? '—' : fmtDuration(Math.round((num(run?.duration_ms) ?? 0) / 1000))}</Td>184 </>185 )}186 </tr>187 );188 })}189 </tbody>190 </DataTable>191 <Note className="mt-4">192 Crawling policy: robots.txt honoured, per-domain rate limits, conditional requests, identified user agent (<Link href={routes.bot()} className="link mono">AIAtlasBot</Link>). Tiers and confidence are explained in the <Link href={routes.methodology()} className="link">methodology</Link>.193 {isAdmin ? <> Operator columns come from <span className="mono">/admin/connectors</span> (last run); HTTP status, conditional-hit rate and LLM fallback rate are not exposed per connector by the API and show “—”. Full console: <Link href={routes.admin('connectors')} className="link">Connectors →</Link></> : token ? ' Operator columns unavailable (admin API did not answer).' : null}194 </Note>195 </>196 )}197 </div>198 </Container>199 );200}201