SPB Git
spboucher.ai

spb/cancerindex

Public

Pulse (/pulse: approvals, new recruiting Phase III, registration momentum, ranking moves, publications, dataset refreshes) and /data-updates log; lighter home (no map SVG) and graph defaults

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 11, 2026) parent 96eddff Browse files

7 changed files +746 −7

7 changed files
Unified Split
modified apps/web/qa/smoke.mjs +2 −0
@@ -50,6 +50,8 @@ const ROUTES = [
50 { path: '/approvals', expect: ['approvals', 'FDA'], maxKb: 900 }, 50 { path: '/approvals', expect: ['approvals', 'FDA'], maxKb: 900 },
51 { path: '/pipeline', expect: ['pipeline', 'Phase'], maxKb: 900 }, 51 { path: '/pipeline', expect: ['pipeline', 'Phase'], maxKb: 900 },
52 { path: '/methodology/trial-map', expect: ['ISO'] }, 52 { path: '/methodology/trial-map', expect: ['ISO'] },
53+ { path: '/pulse', expect: ['What changed in cancer', 'Phase III'] },
54+ { path: '/data-updates', expect: ['Data update log', 'ING-'] },
53 { path: '/api/v1/research-gap', expect: ['data'], kind: 'json', optionalLocal: true }, 55 { path: '/api/v1/research-gap', expect: ['data'], kind: 'json', optionalLocal: true },
54 { path: '/api/v1/trials/intelligence?limit=2', expect: ['data'], kind: 'json', optionalLocal: true }, 56 { path: '/api/v1/trials/intelligence?limit=2', expect: ['data'], kind: 'json', optionalLocal: true },
55 { path: '/api/v1/epidemiology/metrics', expect: ['data'], kind: 'json', optionalLocal: true }, 57 { path: '/api/v1/epidemiology/metrics', expect: ['data'], kind: 'json', optionalLocal: true },
added apps/web/src/app/data-updates/page.tsx +141 −0
@@ -0,0 +1,141 @@
1+import type { Metadata } from 'next';
2+import Link from 'next/link';
3+import { PageHeader, Section, Note } from '@/components/ui/section';
4+import { EmptyState } from '@/components/ui/empty-state';
5+import { Badge, StatusBadge } from '@/components/ui/badge';
6+import { connectorStates, recentIngests } from '@/lib/queries/pulse';
7+import { fmtDate, fmtDateTime, fmtDuration, fmtInt, humanize, relativeTime } from '@/lib/format';
8+
9+export const metadata: Metadata = { title: 'Data update log', description: 'Every connector run of the last 60 days with records created, updated and rejected, dataset versions, and the health and last successful import of each source.' };
10+export const revalidate = 600;
11+
12+/** Stale = an active scheduled connector without a success in the last 14 days (the worker uses 2× its interval; 14 days is the display rule here). */
13+const STALE_DAYS = 14;
14+
15+export default async function DataUpdatesPage() {
16+ const [runs, states] = await Promise.all([recentIngests(60, 400), connectorStates()]);
17+ const byDay = new Map<string, typeof runs>();
18+ for (const r of runs) {
19+ const d = String(r.started_at).slice(0, 10);
20+ byDay.set(d, [...(byDay.get(d) ?? []), r]);
21+ }
22+ const now = Date.now();
23+ return (
24+ <div className="pb-8">
25+ <PageHeader kicker="Data" title="Data update log" lede="Transparency on freshness (SPEC §61-62): each source is versioned and every ingestion is a run with counters; historical values are never silently replaced. Runs marked partial stopped on their time budget and resume from a cursor." />
26+
27+ <Section id="sources" kicker="Sources" title="Connector state" description="Health from the hourly probe, last successful import, schedule (UTC cron) and the dataset version the source reported last. Sources under license review or awaiting credentials never run.">
28+ <div className="ci-table-wrap">
29+ <table className="ci-table">
30+ <thead>
31+ <tr>
32+ <th>Source</th>
33+ <th>Category</th>
34+ <th>Status</th>
35+ <th>Health</th>
36+ <th>Last successful import</th>
37+ <th>Schedule</th>
38+ <th>Dataset version</th>
39+ <th className="num">Source records</th>
40+ </tr>
41+ </thead>
42+ <tbody>
43+ {states.map((s) => {
44+ const stale = s.status === 'active' && !!s.schedule && (!s.last_success_at || now - new Date(s.last_success_at).getTime() > STALE_DAYS * 86_400_000);
45+ return (
46+ <tr key={s.connector_id}>
47+ <td>
48+ <Link href={`/source/${s.connector_id}`} className="ci-link">
49+ {s.source_name ?? s.connector_id}
50+ </Link>
51+ </td>
52+ <td className="text-[12.5px]">{humanize(s.category)}</td>
53+ <td>
54+ <StatusBadge status={s.status} />
55+ {s.paused ? <Badge tone="warn" className="ml-1">paused</Badge> : null}
56+ </td>
57+ <td>
58+ <StatusBadge status={s.health} />
59+ {stale ? (
60+ <Badge tone="warn" className="ml-1" title={`No successful run in ${STALE_DAYS} days`}>
61+ stale
62+ </Badge>
63+ ) : null}
64+ </td>
65+ <td className="whitespace-nowrap text-[12.5px]" title={s.last_success_at ? fmtDateTime(s.last_success_at) : ''}>
66+ {s.last_success_at ? `${fmtDate(s.last_success_at)} (${relativeTime(s.last_success_at)})` : <span className="text-ink-4">never</span>}
67+ </td>
68+ <td className="ci-mono text-[11.5px]">{s.schedule ?? '—'}</td>
69+ <td className="ci-mono text-[11.5px]">{s.last_dataset_version ?? '—'}</td>
70+ <td className="num">{fmtInt(s.record_count)}</td>
71+ </tr>
72+ );
73+ })}
74+ </tbody>
75+ </table>
76+ </div>
77+ </Section>
78+
79+ <Section id="runs" kicker="Runs" title={`Ingestion runs, last 60 days (${fmtInt(runs.length)})`} description="Most recent first, grouped by day. Created / updated / unchanged / rejected are source records; the anomaly column shows refused destructive updates.">
80+ {runs.length ? (
81+ <div className="space-y-6">
82+ {[...byDay.entries()].map(([day, rows]) => (
83+ <div key={day}>
84+ <h3 className="mb-1 text-[15px]">{fmtDate(day)}</h3>
85+ <div className="ci-table-wrap">
86+ <table className="ci-table">
87+ <thead>
88+ <tr>
89+ <th>Started</th>
90+ <th>Source</th>
91+ <th>Mode</th>
92+ <th>Status</th>
93+ <th className="num">Fetched</th>
94+ <th className="num">Created</th>
95+ <th className="num">Updated</th>
96+ <th className="num">Unchanged</th>
97+ <th className="num">Rejected</th>
98+ <th>Duration</th>
99+ <th>Dataset version</th>
100+ <th>Anomaly</th>
101+ </tr>
102+ </thead>
103+ <tbody>
104+ {rows.map((r) => (
105+ <tr key={r.id}>
106+ <td className="whitespace-nowrap text-[12px]">{fmtDateTime(r.started_at)}</td>
107+ <td>
108+ <Link href={`/source/${r.connector_id}`} className="ci-link">
109+ {r.source_name ?? r.connector_id}
110+ </Link>
111+ </td>
112+ <td className="text-[12px]">{r.mode}</td>
113+ <td>
114+ <StatusBadge status={r.status} />
115+ </td>
116+ <td className="num">{fmtInt(r.records_fetched)}</td>
117+ <td className="num">{fmtInt(r.records_created)}</td>
118+ <td className="num">{fmtInt(r.records_updated)}</td>
119+ <td className="num text-ink-3">{fmtInt(r.records_unchanged)}</td>
120+ <td className={`num ${r.records_rejected > 0 ? 'text-warn' : ''}`}>{fmtInt(r.records_rejected)}</td>
121+ <td className="whitespace-nowrap text-[12px]">{fmtDuration(r.duration_ms)}</td>
122+ <td className="ci-mono text-[11.5px]">{r.dataset_version ?? '—'}</td>
123+ <td className="text-[12px] text-danger">{r.anomaly ?? ''}</td>
124+ </tr>
125+ ))}
126+ </tbody>
127+ </table>
128+ </div>
129+ </div>
130+ ))}
131+ </div>
132+ ) : (
133+ <EmptyState title="No ingestion run in the last 60 days" />
134+ )}
135+ </Section>
136+ <div className="mt-6">
137+ <Note>Run identifiers are `ING-&lt;CONNECTOR&gt;-YYYYMMDD-nnnnnn`; operators can open a run on the admin console. Derived layers (counters, intelligence, rankings) are recomputed after every run that changed records, then daily at 06:00, 06:15 and 06:30 UTC.</Note>
138+ </div>
139+ </div>
140+ );
141+}
added apps/web/src/app/pulse/page.tsx +347 −0
@@ -0,0 +1,347 @@
1+import type { Metadata } from 'next';
2+import Link from 'next/link';
3+import { PageHeader, Section, Note } from '@/components/ui/section';
4+import { EmptyState } from '@/components/ui/empty-state';
5+import { Freshness } from '@/components/ui/freshness';
6+import { Badge, ClaimBadge, StatusBadge } from '@/components/ui/badge';
7+import { SourceBadge } from '@/components/ui/source-badge';
8+import { recentApprovals, newPhase3Recruiting, trialPulseByPhase, newTrialsByCancer, rankingMoves, recentPublications, recentIngests, recentEvents } from '@/lib/queries/pulse';
9+import { fmtDate, fmtInt, fmtValue, humanize, phaseLabel, scopeLabel, truncate } from '@/lib/format';
10+
11+export const metadata: Metadata = { title: 'Pulse — what changed in cancer', description: 'New oncology approvals, newly recruiting Phase III trials, registration momentum, ranking moves, recent publications and dataset updates — every item is a dated source record.' };
12+export const revalidate = 900;
13+
14+const APPROVAL_DAYS = 90;
15+const TRIAL_DAYS = 30;
16+const PUB_DAYS = 60;
17+
18+function Delta({ current, previous }: { current: number; previous: number }) {
19+ if (previous === 0) return <span className="text-ink-4" title="No studies in the previous window">—</span>;
20+ const pct = ((current - previous) / previous) * 100;
21+ const cls = pct > 0 ? 'text-ok' : pct < 0 ? 'text-danger' : 'text-ink-3';
22+ return (
23+ <span className={`ci-num ${cls}`} title={`${fmtInt(current)} vs ${fmtInt(previous)} in the previous ${TRIAL_DAYS} days`}>
24+ {pct > 0 ? '+' : ''}
25+ {pct.toFixed(0)} %
26+ </span>
27+ );
28+}
29+
30+export default async function PulsePage() {
31+ const [approvals, phase3, pulse, byCancer, moves, pubs, ingests, events] = await Promise.all([recentApprovals(APPROVAL_DAYS, 40), newPhase3Recruiting(TRIAL_DAYS, 25), trialPulseByPhase(TRIAL_DAYS), newTrialsByCancer(TRIAL_DAYS, 10), rankingMoves(3, 20), recentPublications(PUB_DAYS, 15), recentIngests(14, 60), recentEvents(30, 30)]);
32+ const all = pulse.rows.find((r) => r.phase === 'ALL');
33+ const byAuthority = new Map<string, number>();
34+ for (const a of approvals) byAuthority.set(`${a.authority} · ${a.jurisdiction}`, (byAuthority.get(`${a.authority} · ${a.jurisdiction}`) ?? 0) + 1);
35+ const lastIngest = ingests.find((r) => r.status === 'succeeded' || r.status === 'partial');
36+
37+ return (
38+ <div className="pb-8">
39+ <PageHeader kicker="Pulse" title="What changed in cancer" lede="A dated feed assembled from records already in the index: regulatory decisions, newly registered Phase III studies, registration momentum, ranking moves, recent publications and dataset refreshes. Nothing here is written by a model; every line links to its source record.">
40+ <p className="mt-3 flex flex-wrap items-center gap-2 text-[12.5px] text-ink-3">
41+ <ClaimBadge kind="observed" /> <ClaimBadge kind="regulatory" /> <ClaimBadge kind="computed" />
42+ <span>
43+ Windows: approvals {APPROVAL_DAYS} days · trials {TRIAL_DAYS} days · publications {PUB_DAYS} days · datasets 14 days.
44+ </span>
45+ <Link href="/data-updates" className="ci-link">
46+ Data update log →
47+ </Link>
48+ </p>
49+ </PageHeader>
50+
51+ <div className="grid gap-10 lg:grid-cols-[1.35fr_1fr]">
52+ <div className="space-y-10">
53+ <Section id="approvals" kicker="Regulatory" title={`New oncology approval records (${fmtInt(approvals.length)})`} description={`Approval and marketing records dated within the last ${APPROVAL_DAYS} days, all authorities ingested. One line is one authority's decision for one application or product identifier; the cancer is shown only when the record names exactly one.`} actions={<Link href="/approvals" className="ci-link">All approvals →</Link>}>
54+ {approvals.length ? (
55+ <>
56+ <p className="mb-2 flex flex-wrap gap-1.5 text-[12px]">
57+ {[...byAuthority.entries()].map(([k, n]) => (
58+ <Badge key={k} tone="outline">
59+ {k}: {fmtInt(n)}
60+ </Badge>
61+ ))}
62+ </p>
63+ <ul className="divide-y divide-rule text-[13.5px]">
64+ {approvals.map((a) => (
65+ <li key={a.id} className="grid gap-x-3 gap-y-0.5 py-2 sm:grid-cols-[92px_1fr]">
66+ <span className="ci-mono text-[12px] text-ink-3">{fmtDate(a.approval_date)}</span>
67+ <span className="min-w-0">
68+ <Link href={`/drug/${a.drug_slug}`} className="ci-link font-medium">
69+ {a.drug_name}
70+ </Link>{' '}
71+ <Badge tone="outline">
72+ {a.authority} · {a.jurisdiction}
73+ </Badge>{' '}
74+ <StatusBadge status={a.status} />{' '}
75+ {a.cancer_slug ? (
76+ <Link href={`/cancer/${a.cancer_slug}`} className="ci-link">
77+ {a.cancer_name}
78+ </Link>
79+ ) : a.tumor_agnostic ? (
80+ <Badge tone="accent">tumor-agnostic</Badge>
81+ ) : (
82+ <span className="text-ink-3">cancer not stated in this record</span>
83+ )}
84+ <span className="mt-0.5 block text-[12.5px] text-ink-2" title={a.indication}>
85+ {truncate(a.indication, 180)}
86+ </span>
87+ <span className="mt-0.5 flex items-center gap-1.5 text-[11.5px] text-ink-3">
88+ {a.approval_type ? <span>{a.approval_type}</span> : null}
89+ <SourceBadge p={{ sourceSlug: a.source_slug }} compact />
90+ </span>
91+ </span>
92+ </li>
93+ ))}
94+ </ul>
95+ </>
96+ ) : (
97+ <EmptyState compact title="No approval record dated in this window">Approval feeds are refreshed weekly (openFDA, Health Canada DPD); records appear here as soon as they carry a date within the window.</EmptyState>
98+ )}
99+ </Section>
100+
101+ <Section id="phase3" kicker="Clinical research" title={`Newly registered Phase III studies now recruiting (${fmtInt(phase3.length)})`} description={`Interventional studies with Phase III among their phases, first posted on ClinicalTrials.gov in the last ${TRIAL_DAYS} days and recruiting. Cancer = first reconciled condition.`} actions={<Link href="/trials?status=RECRUITING&phase=PHASE3" className="ci-link">All recruiting Phase III →</Link>}>
102+ {phase3.length ? (
103+ <div className="ci-table-wrap">
104+ <table className="ci-table">
105+ <thead>
106+ <tr>
107+ <th>First posted</th>
108+ <th>NCT</th>
109+ <th>Title</th>
110+ <th>Cancer</th>
111+ <th>Sponsor</th>
112+ <th className="num">Enrollment (n)</th>
113+ </tr>
114+ </thead>
115+ <tbody>
116+ {phase3.map((t) => (
117+ <tr key={t.id}>
118+ <td className="whitespace-nowrap text-[12.5px]">{fmtDate(t.first_posted_date)}</td>
119+ <td>
120+ <Link href={`/trial/${t.nct_id}`} className="ci-mono ci-link">
121+ {t.nct_id}
122+ </Link>
123+ </td>
124+ <td className="min-w-[260px] max-w-[460px]">
125+ <Link href={`/trial/${t.nct_id}`} className="text-ink no-underline hover:text-accent">
126+ {t.brief_title}
127+ </Link>
128+ {t.acronym ? <span className="ml-1 text-[12px] text-ink-3">({t.acronym})</span> : null}
129+ </td>
130+ <td>{t.cancer_slug ? <Link href={`/cancer/${t.cancer_slug}`} className="ci-link">{t.cancer_name}</Link> : <span className="text-ink-4">unmapped</span>}</td>
131+ <td className="max-w-[200px] truncate text-[12.5px]" title={t.lead_sponsor ?? ''}>
132+ {t.lead_sponsor ?? '—'}
133+ {t.lead_sponsor_class === 'INDUSTRY' ? <span className="ml-1 text-[11px] text-ink-3">industry</span> : null}
134+ </td>
135+ <td className="num">{t.enrollment_count == null ? '—' : fmtInt(t.enrollment_count)}</td>
136+ </tr>
137+ ))}
138+ </tbody>
139+ </table>
140+ </div>
141+ ) : (
142+ <EmptyState compact title="No Phase III study first posted in this window">The ClinicalTrials.gov connector runs daily; newly posted studies appear after the next run.</EmptyState>
143+ )}
144+ <Freshness dataUpdatedAt={phase3[0]?.updated_at ?? null} extra="source: clinicaltrials · first_posted_date as posted by the registrant" />
145+ </Section>
146+
147+ <Section id="moves" kicker="Rankings" title="Largest ranking moves" description="Top-level cancers whose rank changed by three places or more between the current snapshot and the previous one of the same metric and scope. A move usually follows a dataset refresh (new registry year, new trials) or a formula change; the ranking page explains each rank.">
148+ {moves.length ? (
149+ <div className="ci-table-wrap">
150+ <table className="ci-table">
151+ <thead>
152+ <tr>
153+ <th>Metric</th>
154+ <th>Scope</th>
155+ <th>Cancer</th>
156+ <th className="num">Rank</th>
157+ <th className="num">Previous</th>
158+ <th className="num">Value</th>
159+ <th>Snapshot</th>
160+ </tr>
161+ </thead>
162+ <tbody>
163+ {moves.map((m) => (
164+ <tr key={`${m.metric_slug}-${m.scope_key}-${m.cancer_id}`}>
165+ <td>
166+ <Link href={`/rankings/${m.metric_slug}?scope=${encodeURIComponent(m.scope_key)}`} className="ci-link">
167+ {m.metric_name}
168+ </Link>
169+ </td>
170+ <td className="text-[12px] text-ink-3">{scopeLabel(m.scope_key)}</td>
171+ <td>
172+ <Link href={`/cancer/${m.slug}/rankings`} className="ci-link">
173+ {m.canonical_name}
174+ </Link>
175+ </td>
176+ <td className="num font-medium">{m.rank}</td>
177+ <td className="num">
178+ {m.previous_rank} <span className={m.previous_rank > m.rank ? 'text-ok' : 'text-danger'}>{m.previous_rank > m.rank ? '▲' : '▼'}</span>
179+ </td>
180+ <td className="num">{fmtValue(m.value, m.unit)}</td>
181+ <td className="whitespace-nowrap text-[12px] text-ink-3">{fmtDate(m.generated_at)}</td>
182+ </tr>
183+ ))}
184+ </tbody>
185+ </table>
186+ </div>
187+ ) : (
188+ <EmptyState compact title="No ranking moved by three places or more">Ranks are compared with the previous snapshot of the same metric and scope; the first snapshot has no previous rank.</EmptyState>
189+ )}
190+ </Section>
191+
192+ <Section id="publications" kicker="Literature" title={`Recently published records indexed (${fmtInt(pubs.length)})`} description={`PubMed records with a publication date in the last ${PUB_DAYS} days already linked to an index entity; randomized trials, systematic reviews and meta-analyses first. Publication dates are as recorded by PubMed.`}>
193+ {pubs.length ? (
194+ <ul className="divide-y divide-rule text-[13.5px]">
195+ {pubs.map((p) => (
196+ <li key={p.id} className="grid gap-x-3 py-2 sm:grid-cols-[92px_1fr]">
197+ <span className="ci-mono text-[12px] text-ink-3">{fmtDate(p.pub_date)}</span>
198+ <span className="min-w-0">
199+ <Link href={p.pmid ? `/publication/${p.pmid}` : '#'} className="ci-link">
200+ {p.title}
201+ </Link>
202+ <span className="mt-0.5 flex flex-wrap items-center gap-1.5 text-[11.5px] text-ink-3">
203+ {p.journal ? <span>{p.journal}</span> : null}
204+ {p.publication_types.filter((t) => /trial|review|meta/i.test(t)).slice(0, 2).map((t) => (
205+ <Badge key={t} tone="outline">
206+ {t}
207+ </Badge>
208+ ))}
209+ {p.pmid ? <span className="ci-mono">PMID {p.pmid}</span> : null}
210+ </span>
211+ </span>
212+ </li>
213+ ))}
214+ </ul>
215+ ) : (
216+ <EmptyState compact title="No publication dated in this window">Literature is linked through entity queries (PubMed) and connector references; recent records appear after the nightly PubMed run.</EmptyState>
217+ )}
218+ </Section>
219+ </div>
220+
221+ <aside className="space-y-10">
222+ <Section id="registrations" kicker="Momentum" title="Trial registrations, last 30 days" description="New oncology studies first posted on ClinicalTrials.gov per phase, compared with the preceding 30 days (all study types; a study with two phases counts in both).">
223+ {pulse.rows.length ? (
224+ <>
225+ {all ? (
226+ <p className="mb-2 text-[13.5px]">
227+ <span className="ci-num font-display text-2xl text-ink">{fmtInt(all.current)}</span> <span className="text-ink-2">new studies</span> <Delta current={all.current} previous={all.previous} />
228+ </p>
229+ ) : null}
230+ <div className="ci-table-wrap">
231+ <table className="ci-table">
232+ <thead>
233+ <tr>
234+ <th>Phase</th>
235+ <th className="num">Last 30 d</th>
236+ <th className="num">Previous 30 d</th>
237+ <th className="num">Change</th>
238+ </tr>
239+ </thead>
240+ <tbody>
241+ {pulse.rows
242+ .filter((r) => r.phase !== 'ALL')
243+ .map((r) => (
244+ <tr key={r.phase}>
245+ <td>{r.phase === 'NA' ? 'Not applicable' : phaseLabel(r.phase)}</td>
246+ <td className="num">{fmtInt(r.current)}</td>
247+ <td className="num">{fmtInt(r.previous)}</td>
248+ <td className="num">
249+ <Delta current={r.current} previous={r.previous} />
250+ </td>
251+ </tr>
252+ ))}
253+ </tbody>
254+ </table>
255+ </div>
256+ <Freshness dataUpdatedAt={pulse.asOf} extra="trial_pulse · ci-trial-pulse-v1 · source: clinicaltrials" />
257+ </>
258+ ) : (
259+ <EmptyState compact title="Registration pulse not yet computed" />
260+ )}
261+ </Section>
262+
263+ <Section id="by-cancer" kicker="Momentum" title="Most new studies by cancer" description="Top-level cancers by studies first posted in the last 30 days (conditions reconciled to the taxonomy, descendants included), with the preceding 30 days.">
264+ {byCancer.length ? (
265+ <div className="ci-table-wrap">
266+ <table className="ci-table">
267+ <thead>
268+ <tr>
269+ <th>Cancer</th>
270+ <th className="num">Last 30 d</th>
271+ <th className="num">Previous</th>
272+ </tr>
273+ </thead>
274+ <tbody>
275+ {byCancer.map((c) => (
276+ <tr key={c.id}>
277+ <td>
278+ <Link href={`/cancer/${c.slug}/trials`} className="ci-link">
279+ {c.canonical_name}
280+ </Link>
281+ </td>
282+ <td className="num">{fmtInt(c.current)}</td>
283+ <td className="num text-ink-3">{fmtInt(c.previous)}</td>
284+ </tr>
285+ ))}
286+ </tbody>
287+ </table>
288+ </div>
289+ ) : (
290+ <EmptyState compact title="Data not yet available" />
291+ )}
292+ </Section>
293+
294+ <Section id="datasets" kicker="Data" title="Dataset refreshes, last 14 days" description="Connector runs (excluding dry runs and probes) with the number of source records created or updated and the dataset version reported by the source." actions={<Link href="/data-updates" className="ci-link">Full log →</Link>}>
295+ {ingests.length ? (
296+ <ul className="divide-y divide-rule text-[13px]">
297+ {ingests.slice(0, 12).map((r) => (
298+ <li key={r.id} className="flex flex-wrap items-baseline justify-between gap-x-3 py-1.5">
299+ <span className="min-w-0">
300+ <Link href={`/source/${r.connector_id}`} className="ci-link">
301+ {r.source_name ?? r.connector_id}
302+ </Link>{' '}
303+ <StatusBadge status={r.status} />
304+ {r.dataset_version ? <span className="ci-mono ml-1 text-[11px] text-ink-3">{r.dataset_version}</span> : null}
305+ </span>
306+ <span className="ci-num text-[12px] text-ink-3">
307+ {fmtDate(r.started_at)} · +{fmtInt(r.records_created)} / ~{fmtInt(r.records_updated)}
308+ </span>
309+ </li>
310+ ))}
311+ </ul>
312+ ) : (
313+ <EmptyState compact title="No connector run in the last 14 days" />
314+ )}
315+ <Freshness dataUpdatedAt={lastIngest?.finished_at ?? null} extra="ingest_runs" />
316+ </Section>
317+
318+ <Section id="events" kicker="Changes" title="Entity change events" description="Non-bulk changes recorded by connectors in the last 30 days (approvals added, aliases, merges, deprecations).">
319+ {events.length ? (
320+ <ul className="divide-y divide-rule text-[13px]">
321+ {events.slice(0, 20).map((e) => (
322+ <li key={e.id} className="py-1.5">
323+ <span className="ci-mono mr-2 text-[11.5px] text-ink-3">{fmtDate(e.created_at)}</span>
324+ <Badge tone="outline">{humanize(e.kind)}</Badge>{' '}
325+ {e.entity_href ? (
326+ <Link href={e.entity_href} className="ci-link">
327+ {e.entity_name ?? e.entity_id}
328+ </Link>
329+ ) : (
330+ <span>{e.entity_name ?? e.entity_id}</span>
331+ )}
332+ <span className="block text-[12px] text-ink-2">{truncate(e.summary, 160)}</span>
333+ </li>
334+ ))}
335+ </ul>
336+ ) : (
337+ <EmptyState compact title="No non-bulk change event in the last 30 days" />
338+ )}
339+ </Section>
340+ </aside>
341+ </div>
342+ <div className="mt-8">
343+ <Note>Guideline changes, safety communications and trial-result events are not yet connected (no guideline or pharmacovigilance connector is live); this page will list them as event types once a licensed source is ingested. Nothing is inferred from press releases.</Note>
344+ </div>
345+ </div>
346+ );
347+}
modified apps/web/src/components/home/trial-map-module.tsx +3 −6
@@ -3,7 +3,7 @@ import { Section } from '@/components/ui/section';
3 import { EmptyState } from '@/components/ui/empty-state'; 3 import { EmptyState } from '@/components/ui/empty-state';
4 import { Freshness } from '@/components/ui/freshness'; 4 import { Freshness } from '@/components/ui/freshness';
5 import { ClaimBadge } from '@/components/ui/badge'; 5 import { ClaimBadge } from '@/components/ui/badge';
6−import { WorldMap, type MapCountryDatum } from '@/components/charts/world-map'; 6+import type { MapCountryDatum } from '@/components/charts/world-map';
7 import { countryCounts } from '@/lib/queries/trial-sites'; 7 import { countryCounts } from '@/lib/queries/trial-sites';
8 import { fmtInt } from '@/lib/format'; 8 import { fmtInt } from '@/lib/format';
9 9
@@ -24,7 +24,7 @@ export async function TrialMapModule({ topN = 8 }: { topN?: number }) {
24 description="Study locations with status RECRUITING (or in a recruiting study when the site has no status), all cancers, any phase. A site is one registrant-entered location; multi-site studies weigh by their number of sites." 24 description="Study locations with status RECRUITING (or in a recruiting study when the site has no status), all cancers, any phase. A site is one registrant-entered location; multi-site studies weigh by their number of sites."
25 actions={ 25 actions={
26 <Link href="/trials/map?recruiting=1" className="ci-link"> 26 <Link href="/trials/map?recruiting=1" className="ci-link">
27− Full map and filters → 27+ Full choropleth map and filters →
28 </Link> 28 </Link>
29 } 29 }
30 > 30 >
@@ -33,10 +33,7 @@ export async function TrialMapModule({ topN = 8 }: { topN?: number }) {
33 Country aggregates appear after <code className="ci-mono">pnpm cix intel</code> has run on ingested ClinicalTrials.gov locations. 33 Country aggregates appear after <code className="ci-mono">pnpm cix intel</code> has run on ingested ClinicalTrials.gov locations.
34 </EmptyState> 34 </EmptyState>
35 ) : ( 35 ) : (
36− <div className="grid gap-4 lg:grid-cols-[3fr_2fr]"> 36+ <div>
37− <div className="min-w-0">
38− <WorldMap data={data} metric="sites" compact ariaLabel={`World map of recruiting clinical trial sites per country, all cancers: ${fmtInt(totalSites)} sites in ${fmtInt(rows.length)} countries. The list beside the map gives the leading countries; the full table is on the trial map page.`} describedBy="trial-map-top" />
39− </div>
40 <div className="min-w-0"> 37 <div className="min-w-0">
41 <div className="ci-table-wrap"> 38 <div className="ci-table-wrap">
42 <table className="ci-table" id="trial-map-top"> 39 <table className="ci-table" id="trial-map-top">
modified apps/web/src/lib/queries/graph.ts +1 −1
@@ -18,7 +18,7 @@ import { type CancerContext, type EdgeGroup, type FocusRef, type GraphEdge, type
18 * and depends on the web `@/lib/db` helpers, so it cannot be imported from the Fastify app. 18 * and depends on the web `@/lib/db` helpers, so it cannot be imported from the Fastify app.
19 */ 19 */
20 20
21−export const DEFAULT_GROUP_LIMIT = 25; 21+export const DEFAULT_GROUP_LIMIT = 15;
22 export const EXPANDED_GROUP_LIMIT = 200; 22 export const EXPANDED_GROUP_LIMIT = 200;
23 export const TRIAL_GROUP_LIMIT = 10; 23 export const TRIAL_GROUP_LIMIT = 10;
24 /** Cohort thresholds for the derived gene ↔ cancer link (frequency and cases affected). */ 24 /** Cohort thresholds for the derived gene ↔ cancer link (frequency and cases affected). */
added apps/web/src/lib/queries/pulse.ts +248 −0
@@ -0,0 +1,248 @@
1+import 'server-only';
2+import { run, sql, safe } from '@/lib/db';
3+
4+/**
5+ * "What changed in cancer" (SPEC §42-43, §62): every item is a dated record already in the database
6+ * (an approval row, a registered study, a ranking row, an ingest run). Nothing is generated; the
7+ * page only orders and groups existing evidence by recency.
8+ */
9+
10+export interface PulseApproval {
11+ id: number;
12+ approval_date: string;
13+ authority: string;
14+ jurisdiction: string;
15+ status: string;
16+ approval_type: string | null;
17+ indication: string;
18+ drug_id: string;
19+ drug_slug: string;
20+ drug_name: string;
21+ cancer_id: string | null;
22+ cancer_slug: string | null;
23+ cancer_name: string | null;
24+ tumor_agnostic: boolean;
25+ source_slug: string;
26+ provenance_id: number;
27+}
28+export async function recentApprovals(days = 90, limit = 40): Promise<PulseApproval[]> {
29+ return safe(
30+ () =>
31+ run<PulseApproval>(sql`
32+ SELECT a.id, a.approval_date, a.authority, a.jurisdiction, a.status, a.approval_type, a.indication, a.tumor_agnostic, a.provenance_id,
33+ d.id AS drug_id, d.slug AS drug_slug, d.name AS drug_name, c.id AS cancer_id, c.slug AS cancer_slug, c.canonical_name AS cancer_name, s.slug AS source_slug
34+ FROM drug_approvals a JOIN drugs d ON d.id = a.drug_id LEFT JOIN cancers c ON c.id = a.cancer_id JOIN sources s ON s.id = a.source_id
35+ WHERE a.approval_date ~ '^\\d{4}-\\d{2}-\\d{2}$' AND a.approval_date::date >= current_date - ${days}::int AND a.approval_date::date <= current_date
36+ ORDER BY a.approval_date DESC, a.id DESC LIMIT ${limit}`),
37+ [] as PulseApproval[],
38+ );
39+}
40+
41+export interface PulseTrial {
42+ id: string;
43+ nct_id: string;
44+ brief_title: string;
45+ acronym: string | null;
46+ phases: string[];
47+ overall_status: string | null;
48+ first_posted_date: string | null;
49+ enrollment_count: number | null;
50+ lead_sponsor: string | null;
51+ lead_sponsor_class: string | null;
52+ countries: string[];
53+ cancer_slug: string | null;
54+ cancer_name: string | null;
55+ updated_at: Date | string;
56+}
57+/** Interventional Phase III studies first posted in the window and recruiting now, with their first mapped cancer. */
58+export async function newPhase3Recruiting(days = 30, limit = 25): Promise<PulseTrial[]> {
59+ return safe(
60+ () =>
61+ run<PulseTrial>(sql`
62+ SELECT t.id, t.nct_id, t.brief_title, t.acronym, t.phases, t.overall_status, t.first_posted_date, t.enrollment_count, t.lead_sponsor, t.lead_sponsor_class, t.countries, t.updated_at,
63+ m.slug AS cancer_slug, m.canonical_name AS cancer_name
64+ FROM clinical_trials t
65+ LEFT JOIN LATERAL (
66+ SELECT c.slug, c.canonical_name FROM trial_conditions tc JOIN cancers c ON c.id = tc.cancer_id
67+ WHERE tc.trial_id = t.id ORDER BY (tc.match_type = 'PROBABILISTIC'), c.depth, c.canonical_name LIMIT 1) m ON true
68+ WHERE t.study_type = 'INTERVENTIONAL' AND 'PHASE3' = ANY(t.phases) AND t.overall_status = 'RECRUITING'
69+ AND t.first_posted_date ~ '^\\d{4}-\\d{2}-\\d{2}$' AND t.first_posted_date::date >= current_date - ${days}::int
70+ ORDER BY t.first_posted_date DESC, t.nct_id LIMIT ${limit}`),
71+ [] as PulseTrial[],
72+ );
73+}
74+
75+export interface PhasePulse {
76+ phase: string;
77+ current: number;
78+ previous: number;
79+}
80+/** New oncology studies per phase: last `days` vs the preceding `days` (from trial_pulse, cancer_id NULL = all studies). */
81+export async function trialPulseByPhase(days = 30): Promise<{ rows: PhasePulse[]; asOf: Date | string | null }> {
82+ const rows = await safe(
83+ () =>
84+ run<{ phase: string; current: string; previous: string; as_of: Date | string | null }>(sql`
85+ SELECT phase,
86+ sum(new_trials) FILTER (WHERE day > current_date - ${days}::int) AS current,
87+ sum(new_trials) FILTER (WHERE day <= current_date - ${days}::int AND day > current_date - ${days * 2}::int) AS previous,
88+ max(updated_at) AS as_of
89+ FROM trial_pulse WHERE cancer_id IS NULL AND day > current_date - ${days * 2}::int
90+ GROUP BY phase ORDER BY CASE phase WHEN 'ALL' THEN 0 WHEN 'EARLY_PHASE1' THEN 1 WHEN 'PHASE1' THEN 2 WHEN 'PHASE2' THEN 3 WHEN 'PHASE3' THEN 4 WHEN 'PHASE4' THEN 5 ELSE 9 END`),
91+ [] as Array<{ phase: string; current: string; previous: string; as_of: Date | string | null }>,
92+ );
93+ return { rows: rows.map((r) => ({ phase: r.phase, current: Number(r.current ?? 0), previous: Number(r.previous ?? 0) })), asOf: rows[0]?.as_of ?? null };
94+}
95+
96+export interface CancerPulse {
97+ id: string;
98+ slug: string;
99+ canonical_name: string;
100+ current: number;
101+ previous: number;
102+}
103+/** Top-level cancers with the most newly registered studies in the window (any phase), with the previous window for comparison. */
104+export async function newTrialsByCancer(days = 30, limit = 10): Promise<CancerPulse[]> {
105+ return safe(
106+ () =>
107+ run<CancerPulse>(sql`
108+ SELECT c.id, c.slug, c.canonical_name,
109+ coalesce(sum(p.new_trials) FILTER (WHERE p.day > current_date - ${days}::int), 0)::int AS current,
110+ coalesce(sum(p.new_trials) FILTER (WHERE p.day <= current_date - ${days}::int), 0)::int AS previous
111+ FROM trial_pulse p JOIN cancers c ON c.id = p.cancer_id
112+ WHERE p.phase = 'ALL' AND c.top_level AND c.status = 'active' AND p.day > current_date - ${days * 2}::int
113+ GROUP BY c.id, c.slug, c.canonical_name ORDER BY current DESC, c.canonical_name LIMIT ${limit}`),
114+ [] as CancerPulse[],
115+ );
116+}
117+
118+export interface RankingMove {
119+ metric_slug: string;
120+ metric_name: string;
121+ unit: string;
122+ scope_key: string;
123+ cancer_id: string;
124+ slug: string;
125+ canonical_name: string;
126+ rank: number;
127+ previous_rank: number;
128+ value: number;
129+ eligible_entities: number;
130+ generated_at: Date | string;
131+}
132+/** Largest rank changes between the current snapshot and the previous one for the same metric and scope (§131). */
133+export async function rankingMoves(minDelta = 3, limit = 20): Promise<RankingMove[]> {
134+ return safe(
135+ () =>
136+ run<RankingMove>(sql`
137+ SELECT r.metric_slug, m.name AS metric_name, r.unit, r.scope_key, r.cancer_id, c.slug, c.canonical_name, r.rank, r.previous_rank, r.value, r.eligible_entities, s.generated_at
138+ FROM rankings r JOIN ranking_snapshots s ON s.id = r.snapshot_id JOIN cancers c ON c.id = r.cancer_id JOIN metric_definitions m ON m.slug = r.metric_slug
139+ WHERE s.is_current AND r.previous_rank IS NOT NULL AND abs(r.previous_rank - r.rank) >= ${minDelta} AND s.entity_level = 'top'
140+ ORDER BY abs(r.previous_rank - r.rank) DESC, s.generated_at DESC, r.metric_slug, r.rank LIMIT ${limit}`),
141+ [] as RankingMove[],
142+ );
143+}
144+
145+export interface PulsePublication {
146+ id: string;
147+ pmid: string | null;
148+ doi: string | null;
149+ title: string;
150+ journal: string | null;
151+ pub_date: string | null;
152+ publication_types: string[];
153+ retracted: boolean;
154+}
155+/** Most recently published records indexed (PubMed), favouring trials, reviews and meta-analyses. */
156+export async function recentPublications(days = 60, limit = 15): Promise<PulsePublication[]> {
157+ return safe(
158+ () =>
159+ run<PulsePublication>(sql`
160+ SELECT id, pmid, doi, title, journal, pub_date, publication_types, retracted FROM publications
161+ WHERE pub_date ~ '^\\d{4}-\\d{2}-\\d{2}$' AND pub_date::date >= current_date - ${days}::int AND pub_date::date <= current_date AND NOT retracted
162+ ORDER BY (publication_types && ARRAY['Randomized Controlled Trial','Clinical Trial, Phase III','Meta-Analysis','Systematic Review','Clinical Trial']::text[]) DESC, pub_date DESC LIMIT ${limit}`),
163+ [] as PulsePublication[],
164+ );
165+}
166+
167+export interface IngestRow {
168+ id: string;
169+ connector_id: string;
170+ source_name: string | null;
171+ mode: string;
172+ status: string;
173+ started_at: Date | string;
174+ finished_at: Date | string | null;
175+ duration_ms: number | null;
176+ records_fetched: number;
177+ records_created: number;
178+ records_updated: number;
179+ records_unchanged: number;
180+ records_rejected: number;
181+ dataset_version: string | null;
182+ anomaly: string | null;
183+}
184+/** Ingest runs of the last `days` days (all statuses), most recent first (§62 data update log). */
185+export async function recentIngests(days = 30, limit = 200): Promise<IngestRow[]> {
186+ return safe(
187+ () =>
188+ run<IngestRow>(sql`
189+ SELECT r.id, r.connector_id, s.name AS source_name, r.mode, r.status, r.started_at, r.finished_at, r.duration_ms, r.records_fetched, r.records_created, r.records_updated, r.records_unchanged, r.records_rejected, r.dataset_version, r.anomaly
190+ FROM ingest_runs r LEFT JOIN sources s ON s.slug = r.connector_id
191+ WHERE r.started_at >= now() - (${days}::text || ' days')::interval AND r.mode NOT IN ('dry_run','probe')
192+ ORDER BY r.started_at DESC LIMIT ${limit}`),
193+ [] as IngestRow[],
194+ );
195+}
196+
197+export interface ConnectorState {
198+ connector_id: string;
199+ source_name: string | null;
200+ category: string | null;
201+ status: string | null;
202+ health: string;
203+ last_success_at: Date | string | null;
204+ last_attempt_at: Date | string | null;
205+ paused: boolean;
206+ schedule: string | null;
207+ last_dataset_version: string | null;
208+ record_count: number;
209+}
210+/** One line per connector: health, last success, schedule, latest dataset version, source records held. */
211+export async function connectorStates(): Promise<ConnectorState[]> {
212+ return safe(
213+ () =>
214+ run<ConnectorState>(sql`
215+ SELECT s.slug AS connector_id, s.name AS source_name, s.category, s.status, coalesce(cc.health, 'unknown') AS health, cc.last_success_at, cc.last_attempt_at, coalesce(cc.paused, false) AS paused,
216+ s.manifest->>'schedule' AS schedule,
217+ (SELECT r.dataset_version FROM ingest_runs r WHERE r.connector_id = s.slug AND r.status IN ('succeeded','partial') ORDER BY r.started_at DESC LIMIT 1) AS last_dataset_version,
218+ (SELECT count(*) FROM source_records sr WHERE sr.source_id = s.id)::int AS record_count
219+ FROM sources s LEFT JOIN connector_cursors cc ON cc.connector_id = s.slug
220+ ORDER BY s.status = 'active' DESC, s.tier, s.name`),
221+ [] as ConnectorState[],
222+ );
223+}
224+
225+/** Recent change events other than bulk creations (approval_added, alias_added, deprecated, merged…). */
226+export interface PulseEvent {
227+ id: number;
228+ entity_type: string;
229+ entity_id: string;
230+ kind: string;
231+ summary: string;
232+ created_at: Date | string;
233+ entity_name: string | null;
234+ entity_href: string | null;
235+}
236+export async function recentEvents(days = 30, limit = 40): Promise<PulseEvent[]> {
237+ return safe(
238+ () =>
239+ run<PulseEvent>(sql`
240+ SELECT e.id, e.entity_type, e.entity_id, e.kind, e.summary, e.created_at,
241+ CASE e.entity_type WHEN 'cancer' THEN (SELECT canonical_name FROM cancers WHERE id = e.entity_id) WHEN 'gene' THEN (SELECT symbol FROM genes WHERE id = e.entity_id) WHEN 'drug' THEN (SELECT name FROM drugs WHERE id = e.entity_id) WHEN 'trial' THEN (SELECT nct_id FROM clinical_trials WHERE id = e.entity_id) END AS entity_name,
242+ CASE e.entity_type WHEN 'cancer' THEN (SELECT '/cancer/' || slug FROM cancers WHERE id = e.entity_id) WHEN 'gene' THEN (SELECT '/gene/' || symbol FROM genes WHERE id = e.entity_id) WHEN 'drug' THEN (SELECT '/drug/' || slug FROM drugs WHERE id = e.entity_id) WHEN 'trial' THEN (SELECT '/trial/' || nct_id FROM clinical_trials WHERE id = e.entity_id) END AS entity_href
243+ FROM change_events e
244+ WHERE e.created_at >= now() - (${days}::text || ' days')::interval AND e.kind <> 'created'
245+ ORDER BY e.created_at DESC, e.id DESC LIMIT ${limit}`),
246+ [] as PulseEvent[],
247+ );
248+}
modified apps/web/src/lib/site.ts +4 −0
@@ -19,6 +19,8 @@ export const NAV = [
19 19
20 /** Secondary navigation (mobile menu + footer): everything not in the primary bar. */ 20 /** Secondary navigation (mobile menu + footer): everything not in the primary bar. */
21 export const MORE_NAV = [ 21 export const MORE_NAV = [
22+ { href: '/pulse', label: 'Pulse' },
23+ { href: '/data-updates', label: 'Data updates' },
22 { href: '/taxonomy', label: 'Taxonomy' }, 24 { href: '/taxonomy', label: 'Taxonomy' },
23 { href: '/countries', label: 'Countries' }, 25 { href: '/countries', label: 'Countries' },
24 { href: '/compare', label: 'Compare' }, 26 { href: '/compare', label: 'Compare' },
@@ -34,6 +36,8 @@ export const FOOTER_NAV = [
34 { href: '/trust', label: 'Trust & policies' }, 36 { href: '/trust', label: 'Trust & policies' },
35 { href: '/explore', label: 'Data explorer' }, 37 { href: '/explore', label: 'Data explorer' },
36 { href: '/data', label: 'Data downloads' }, 38 { href: '/data', label: 'Data downloads' },
39+ { href: '/pulse', label: 'Pulse' },
40+ { href: '/data-updates', label: 'Data update log' },
37 { href: '/developers', label: 'Developers (API)' }, 41 { href: '/developers', label: 'Developers (API)' },
38 { href: '/sources', label: 'Sources' }, 42 { href: '/sources', label: 'Sources' },
39 { href: '/methodology', label: 'Methodology' }, 43 { href: '/methodology', label: 'Methodology' },
40 44
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere