SPB Git forge
spboucher.ai ↗

spb/cancerindex

Public
37commits 1branches 0releases
2.9 MBsize
maindefault branch
10 days agolast push
TypeScript 97.2% SQL 1.5% CSS 0.6% JavaScript 0.5%

Pulse (/pulse: approvals, new recruiting Phase III, registration momentum, ranking moves, publications, dataset refreshes) and /data-updates log; lighter home (no map SVG) and graph defaults

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 13 days ago (Sep 11, 2026) parent 96eddff

7 changed files +746 −7

modified apps/web/qa/smoke.mjs +2 −0
@@ -50,6 +50,8 @@ const ROUTES = [
50 50 { path: '/approvals', expect: ['approvals', 'FDA'], maxKb: 900 },
51 51 { path: '/pipeline', expect: ['pipeline', 'Phase'], maxKb: 900 },
52 52 { path: '/methodology/trial-map', expect: ['ISO'] },
53 + { path: '/pulse', expect: ['What changed in cancer', 'Phase III'] },
54 + { path: '/data-updates', expect: ['Data update log', 'ING-'] },
53 55 { path: '/api/v1/research-gap', expect: ['data'], kind: 'json', optionalLocal: true },
54 56 { path: '/api/v1/trials/intelligence?limit=2', expect: ['data'], kind: 'json', optionalLocal: true },
55 57 { path: '/api/v1/epidemiology/metrics', expect: ['data'], kind: 'json', optionalLocal: true },
added apps/web/src/app/data-updates/page.tsx +141 −0
@@ -0,0 +1,141 @@
1 +import type { Metadata } from 'next';
2 +import Link from 'next/link';
3 +import { PageHeader, Section, Note } from '@/components/ui/section';
4 +import { EmptyState } from '@/components/ui/empty-state';
5 +import { Badge, StatusBadge } from '@/components/ui/badge';
6 +import { connectorStates, recentIngests } from '@/lib/queries/pulse';
7 +import { fmtDate, fmtDateTime, fmtDuration, fmtInt, humanize, relativeTime } from '@/lib/format';
8 +
9 +export const metadata: Metadata = { title: 'Data update log', description: 'Every connector run of the last 60 days with records created, updated and rejected, dataset versions, and the health and last successful import of each source.' };
10 +export const revalidate = 600;
11 +
12 +/** Stale = an active scheduled connector without a success in the last 14 days (the worker uses 2× its interval; 14 days is the display rule here). */
13 +const STALE_DAYS = 14;
14 +
15 +export default async function DataUpdatesPage() {
16 + const [runs, states] = await Promise.all([recentIngests(60, 400), connectorStates()]);
17 + const byDay = new Map<string, typeof runs>();
18 + for (const r of runs) {
19 + const d = String(r.started_at).slice(0, 10);
20 + byDay.set(d, [...(byDay.get(d) ?? []), r]);
21 + }
22 + const now = Date.now();
23 + return (
24 + <div className="pb-8">
25 + <PageHeader kicker="Data" title="Data update log" lede="Transparency on freshness (SPEC §61-62): each source is versioned and every ingestion is a run with counters; historical values are never silently replaced. Runs marked partial stopped on their time budget and resume from a cursor." />
26 +
27 + <Section id="sources" kicker="Sources" title="Connector state" description="Health from the hourly probe, last successful import, schedule (UTC cron) and the dataset version the source reported last. Sources under license review or awaiting credentials never run.">
28 + <div className="ci-table-wrap">
29 + <table className="ci-table">
30 + <thead>
31 + <tr>
32 + <th>Source</th>
33 + <th>Category</th>
34 + <th>Status</th>
35 + <th>Health</th>
36 + <th>Last successful import</th>
37 + <th>Schedule</th>
38 + <th>Dataset version</th>
39 + <th className="num">Source records</th>
40 + </tr>
41 + </thead>
42 + <tbody>
43 + {states.map((s) => {
44 + const stale = s.status === 'active' && !!s.schedule && (!s.last_success_at || now - new Date(s.last_success_at).getTime() > STALE_DAYS * 86_400_000);
45 + return (
46 + <tr key={s.connector_id}>
47 + <td>
48 + <Link href={`/source/${s.connector_id}`} className="ci-link">
49 + {s.source_name ?? s.connector_id}
50 + </Link>
51 + </td>
52 + <td className="text-[12.5px]">{humanize(s.category)}</td>
53 + <td>
54 + <StatusBadge status={s.status} />
55 + {s.paused ? <Badge tone="warn" className="ml-1">paused</Badge> : null}
56 + </td>
57 + <td>
58 + <StatusBadge status={s.health} />
59 + {stale ? (
60 + <Badge tone="warn" className="ml-1" title={`No successful run in ${STALE_DAYS} days`}>
61 + stale
62 + </Badge>
63 + ) : null}
64 + </td>
65 + <td className="whitespace-nowrap text-[12.5px]" title={s.last_success_at ? fmtDateTime(s.last_success_at) : ''}>
66 + {s.last_success_at ? `${fmtDate(s.last_success_at)} (${relativeTime(s.last_success_at)})` : <span className="text-ink-4">never</span>}
67 + </td>
68 + <td className="ci-mono text-[11.5px]">{s.schedule ?? '—'}</td>
69 + <td className="ci-mono text-[11.5px]">{s.last_dataset_version ?? '—'}</td>
70 + <td className="num">{fmtInt(s.record_count)}</td>
71 + </tr>
72 + );
73 + })}
74 + </tbody>
75 + </table>
76 + </div>
77 + </Section>
78 +
79 + <Section id="runs" kicker="Runs" title={`Ingestion runs, last 60 days (${fmtInt(runs.length)})`} description="Most recent first, grouped by day. Created / updated / unchanged / rejected are source records; the anomaly column shows refused destructive updates.">
80 + {runs.length ? (
81 + <div className="space-y-6">
82 + {[...byDay.entries()].map(([day, rows]) => (
83 + <div key={day}>
84 + <h3 className="mb-1 text-[15px]">{fmtDate(day)}</h3>
85 + <div className="ci-table-wrap">
86 + <table className="ci-table">
87 + <thead>
88 + <tr>
89 + <th>Started</th>
90 + <th>Source</th>
91 + <th>Mode</th>
92 + <th>Status</th>
93 + <th className="num">Fetched</th>
94 + <th className="num">Created</th>
95 + <th className="num">Updated</th>
96 + <th className="num">Unchanged</th>
97 + <th className="num">Rejected</th>
98 + <th>Duration</th>
99 + <th>Dataset version</th>
100 + <th>Anomaly</th>
101 + </tr>
102 + </thead>
103 + <tbody>
104 + {rows.map((r) => (
105 + <tr key={r.id}>
106 + <td className="whitespace-nowrap text-[12px]">{fmtDateTime(r.started_at)}</td>
107 + <td>
108 + <Link href={`/source/${r.connector_id}`} className="ci-link">
109 + {r.source_name ?? r.connector_id}
110 + </Link>
111 + </td>
112 + <td className="text-[12px]">{r.mode}</td>
113 + <td>
114 + <StatusBadge status={r.status} />
115 + </td>
116 + <td className="num">{fmtInt(r.records_fetched)}</td>
117 + <td className="num">{fmtInt(r.records_created)}</td>
118 + <td className="num">{fmtInt(r.records_updated)}</td>
119 + <td className="num text-ink-3">{fmtInt(r.records_unchanged)}</td>
120 + <td className={`num ${r.records_rejected > 0 ? 'text-warn' : ''}`}>{fmtInt(r.records_rejected)}</td>
121 + <td className="whitespace-nowrap text-[12px]">{fmtDuration(r.duration_ms)}</td>
122 + <td className="ci-mono text-[11.5px]">{r.dataset_version ?? '—'}</td>
123 + <td className="text-[12px] text-danger">{r.anomaly ?? ''}</td>
124 + </tr>
125 + ))}
126 + </tbody>
127 + </table>
128 + </div>
129 + </div>
130 + ))}
131 + </div>
132 + ) : (
133 + <EmptyState title="No ingestion run in the last 60 days" />
134 + )}
135 + </Section>
136 + <div className="mt-6">
137 + <Note>Run identifiers are `ING-&lt;CONNECTOR&gt;-YYYYMMDD-nnnnnn`; operators can open a run on the admin console. Derived layers (counters, intelligence, rankings) are recomputed after every run that changed records, then daily at 06:00, 06:15 and 06:30 UTC.</Note>
138 + </div>
139 + </div>
140 + );
141 +}
added apps/web/src/app/pulse/page.tsx +347 −0
@@ -0,0 +1,347 @@
1 +import type { Metadata } from 'next';
2 +import Link from 'next/link';
3 +import { PageHeader, Section, Note } from '@/components/ui/section';
4 +import { EmptyState } from '@/components/ui/empty-state';
5 +import { Freshness } from '@/components/ui/freshness';
6 +import { Badge, ClaimBadge, StatusBadge } from '@/components/ui/badge';
7 +import { SourceBadge } from '@/components/ui/source-badge';
8 +import { recentApprovals, newPhase3Recruiting, trialPulseByPhase, newTrialsByCancer, rankingMoves, recentPublications, recentIngests, recentEvents } from '@/lib/queries/pulse';
9 +import { fmtDate, fmtInt, fmtValue, humanize, phaseLabel, scopeLabel, truncate } from '@/lib/format';
10 +
11 +export const metadata: Metadata = { title: 'Pulse — what changed in cancer', description: 'New oncology approvals, newly recruiting Phase III trials, registration momentum, ranking moves, recent publications and dataset updates — every item is a dated source record.' };
12 +export const revalidate = 900;
13 +
14 +const APPROVAL_DAYS = 90;
15 +const TRIAL_DAYS = 30;
16 +const PUB_DAYS = 60;
17 +
18 +function Delta({ current, previous }: { current: number; previous: number }) {
19 + if (previous === 0) return <span className="text-ink-4" title="No studies in the previous window">—</span>;
20 + const pct = ((current - previous) / previous) * 100;
21 + const cls = pct > 0 ? 'text-ok' : pct < 0 ? 'text-danger' : 'text-ink-3';
22 + return (
23 + <span className={`ci-num ${cls}`} title={`${fmtInt(current)} vs ${fmtInt(previous)} in the previous ${TRIAL_DAYS} days`}>
24 + {pct > 0 ? '+' : ''}
25 + {pct.toFixed(0)} %
26 + </span>
27 + );
28 +}
29 +
30 +export default async function PulsePage() {
31 + const [approvals, phase3, pulse, byCancer, moves, pubs, ingests, events] = await Promise.all([recentApprovals(APPROVAL_DAYS, 40), newPhase3Recruiting(TRIAL_DAYS, 25), trialPulseByPhase(TRIAL_DAYS), newTrialsByCancer(TRIAL_DAYS, 10), rankingMoves(3, 20), recentPublications(PUB_DAYS, 15), recentIngests(14, 60), recentEvents(30, 30)]);
32 + const all = pulse.rows.find((r) => r.phase === 'ALL');
33 + const byAuthority = new Map<string, number>();
34 + for (const a of approvals) byAuthority.set(`${a.authority} · ${a.jurisdiction}`, (byAuthority.get(`${a.authority} · ${a.jurisdiction}`) ?? 0) + 1);
35 + const lastIngest = ingests.find((r) => r.status === 'succeeded' || r.status === 'partial');
36 +
37 + return (
38 + <div className="pb-8">
39 + <PageHeader kicker="Pulse" title="What changed in cancer" lede="A dated feed assembled from records already in the index: regulatory decisions, newly registered Phase III studies, registration momentum, ranking moves, recent publications and dataset refreshes. Nothing here is written by a model; every line links to its source record.">
40 + <p className="mt-3 flex flex-wrap items-center gap-2 text-[12.5px] text-ink-3">
41 + <ClaimBadge kind="observed" /> <ClaimBadge kind="regulatory" /> <ClaimBadge kind="computed" />
42 + <span>
43 + Windows: approvals {APPROVAL_DAYS} days · trials {TRIAL_DAYS} days · publications {PUB_DAYS} days · datasets 14 days.
44 + </span>
45 + <Link href="/data-updates" className="ci-link">
46 + Data update log →
47 + </Link>
48 + </p>
49 + </PageHeader>
50 +
51 + <div className="grid gap-10 lg:grid-cols-[1.35fr_1fr]">
52 + <div className="space-y-10">
53 + <Section id="approvals" kicker="Regulatory" title={`New oncology approval records (${fmtInt(approvals.length)})`} description={`Approval and marketing records dated within the last ${APPROVAL_DAYS} days, all authorities ingested. One line is one authority's decision for one application or product identifier; the cancer is shown only when the record names exactly one.`} actions={<Link href="/approvals" className="ci-link">All approvals →</Link>}>
54 + {approvals.length ? (
55 + <>
56 + <p className="mb-2 flex flex-wrap gap-1.5 text-[12px]">
57 + {[...byAuthority.entries()].map(([k, n]) => (
58 + <Badge key={k} tone="outline">
59 + {k}: {fmtInt(n)}
60 + </Badge>
61 + ))}
62 + </p>
63 + <ul className="divide-y divide-rule text-[13.5px]">
64 + {approvals.map((a) => (
65 + <li key={a.id} className="grid gap-x-3 gap-y-0.5 py-2 sm:grid-cols-[92px_1fr]">
66 + <span className="ci-mono text-[12px] text-ink-3">{fmtDate(a.approval_date)}</span>
67 + <span className="min-w-0">
68 + <Link href={`/drug/${a.drug_slug}`} className="ci-link font-medium">
69 + {a.drug_name}
70 + </Link>{' '}
71 + <Badge tone="outline">
72 + {a.authority} · {a.jurisdiction}
73 + </Badge>{' '}
74 + <StatusBadge status={a.status} />{' '}
75 + {a.cancer_slug ? (
76 + <Link href={`/cancer/${a.cancer_slug}`} className="ci-link">
77 + {a.cancer_name}
78 + </Link>
79 + ) : a.tumor_agnostic ? (
80 + <Badge tone="accent">tumor-agnostic</Badge>
81 + ) : (
82 + <span className="text-ink-3">cancer not stated in this record</span>
83 + )}
84 + <span className="mt-0.5 block text-[12.5px] text-ink-2" title={a.indication}>
85 + {truncate(a.indication, 180)}
86 + </span>
87 + <span className="mt-0.5 flex items-center gap-1.5 text-[11.5px] text-ink-3">
88 + {a.approval_type ? <span>{a.approval_type}</span> : null}
89 + <SourceBadge p={{ sourceSlug: a.source_slug }} compact />
90 + </span>
91 + </span>
92 + </li>
93 + ))}
94 + </ul>
95 + </>
96 + ) : (
97 + <EmptyState compact title="No approval record dated in this window">Approval feeds are refreshed weekly (openFDA, Health Canada DPD); records appear here as soon as they carry a date within the window.</EmptyState>
98 + )}
99 + </Section>
100 +
101 + <Section id="phase3" kicker="Clinical research" title={`Newly registered Phase III studies now recruiting (${fmtInt(phase3.length)})`} description={`Interventional studies with Phase III among their phases, first posted on ClinicalTrials.gov in the last ${TRIAL_DAYS} days and recruiting. Cancer = first reconciled condition.`} actions={<Link href="/trials?status=RECRUITING&phase=PHASE3" className="ci-link">All recruiting Phase III →</Link>}>
102 + {phase3.length ? (
103 + <div className="ci-table-wrap">
104 + <table className="ci-table">
105 + <thead>
106 + <tr>
107 + <th>First posted</th>
108 + <th>NCT</th>
109 + <th>Title</th>
110 + <th>Cancer</th>
111 + <th>Sponsor</th>
112 + <th className="num">Enrollment (n)</th>
113 + </tr>
114 + </thead>
115 + <tbody>
116 + {phase3.map((t) => (
117 + <tr key={t.id}>
118 + <td className="whitespace-nowrap text-[12.5px]">{fmtDate(t.first_posted_date)}</td>
119 + <td>
120 + <Link href={`/trial/${t.nct_id}`} className="ci-mono ci-link">
121 + {t.nct_id}
122 + </Link>
123 + </td>
124 + <td className="min-w-[260px] max-w-[460px]">
125 + <Link href={`/trial/${t.nct_id}`} className="text-ink no-underline hover:text-accent">
126 + {t.brief_title}
127 + </Link>
128 + {t.acronym ? <span className="ml-1 text-[12px] text-ink-3">({t.acronym})</span> : null}
129 + </td>
130 + <td>{t.cancer_slug ? <Link href={`/cancer/${t.cancer_slug}`} className="ci-link">{t.cancer_name}</Link> : <span className="text-ink-4">unmapped</span>}</td>
131 + <td className="max-w-[200px] truncate text-[12.5px]" title={t.lead_sponsor ?? ''}>
132 + {t.lead_sponsor ?? '—'}
133 + {t.lead_sponsor_class === 'INDUSTRY' ? <span className="ml-1 text-[11px] text-ink-3">industry</span> : null}
134 + </td>
135 + <td className="num">{t.enrollment_count == null ? '—' : fmtInt(t.enrollment_count)}</td>
136 + </tr>
137 + ))}
138 + </tbody>
139 + </table>
140 + </div>
141 + ) : (
142 + <EmptyState compact title="No Phase III study first posted in this window">The ClinicalTrials.gov connector runs daily; newly posted studies appear after the next run.</EmptyState>
143 + )}
144 + <Freshness dataUpdatedAt={phase3[0]?.updated_at ?? null} extra="source: clinicaltrials · first_posted_date as posted by the registrant" />
145 + </Section>
146 +
147 + <Section id="moves" kicker="Rankings" title="Largest ranking moves" description="Top-level cancers whose rank changed by three places or more between the current snapshot and the previous one of the same metric and scope. A move usually follows a dataset refresh (new registry year, new trials) or a formula change; the ranking page explains each rank.">
148 + {moves.length ? (
149 + <div className="ci-table-wrap">
150 + <table className="ci-table">
151 + <thead>
152 + <tr>
153 + <th>Metric</th>
154 + <th>Scope</th>
155 + <th>Cancer</th>
156 + <th className="num">Rank</th>
157 + <th className="num">Previous</th>
158 + <th className="num">Value</th>
159 + <th>Snapshot</th>
160 + </tr>
161 + </thead>
162 + <tbody>
163 + {moves.map((m) => (
164 + <tr key={`${m.metric_slug}-${m.scope_key}-${m.cancer_id}`}>
165 + <td>
166 + <Link href={`/rankings/${m.metric_slug}?scope=${encodeURIComponent(m.scope_key)}`} className="ci-link">
167 + {m.metric_name}
168 + </Link>
169 + </td>
170 + <td className="text-[12px] text-ink-3">{scopeLabel(m.scope_key)}</td>
171 + <td>
172 + <Link href={`/cancer/${m.slug}/rankings`} className="ci-link">
173 + {m.canonical_name}
174 + </Link>
175 + </td>
176 + <td className="num font-medium">{m.rank}</td>
177 + <td className="num">
178 + {m.previous_rank} <span className={m.previous_rank > m.rank ? 'text-ok' : 'text-danger'}>{m.previous_rank > m.rank ? '▲' : '▼'}</span>
179 + </td>
180 + <td className="num">{fmtValue(m.value, m.unit)}</td>
181 + <td className="whitespace-nowrap text-[12px] text-ink-3">{fmtDate(m.generated_at)}</td>
182 + </tr>
183 + ))}
184 + </tbody>
185 + </table>
186 + </div>
187 + ) : (
188 + <EmptyState compact title="No ranking moved by three places or more">Ranks are compared with the previous snapshot of the same metric and scope; the first snapshot has no previous rank.</EmptyState>
189 + )}
190 + </Section>
191 +
192 + <Section id="publications" kicker="Literature" title={`Recently published records indexed (${fmtInt(pubs.length)})`} description={`PubMed records with a publication date in the last ${PUB_DAYS} days already linked to an index entity; randomized trials, systematic reviews and meta-analyses first. Publication dates are as recorded by PubMed.`}>
193 + {pubs.length ? (
194 + <ul className="divide-y divide-rule text-[13.5px]">
195 + {pubs.map((p) => (
196 + <li key={p.id} className="grid gap-x-3 py-2 sm:grid-cols-[92px_1fr]">
197 + <span className="ci-mono text-[12px] text-ink-3">{fmtDate(p.pub_date)}</span>
198 + <span className="min-w-0">
199 + <Link href={p.pmid ? `/publication/${p.pmid}` : '#'} className="ci-link">
200 + {p.title}
201 + </Link>
202 + <span className="mt-0.5 flex flex-wrap items-center gap-1.5 text-[11.5px] text-ink-3">
203 + {p.journal ? <span>{p.journal}</span> : null}
204 + {p.publication_types.filter((t) => /trial|review|meta/i.test(t)).slice(0, 2).map((t) => (
205 + <Badge key={t} tone="outline">
206 + {t}
207 + </Badge>
208 + ))}
209 + {p.pmid ? <span className="ci-mono">PMID {p.pmid}</span> : null}
210 + </span>
211 + </span>
212 + </li>
213 + ))}
214 + </ul>
215 + ) : (
216 + <EmptyState compact title="No publication dated in this window">Literature is linked through entity queries (PubMed) and connector references; recent records appear after the nightly PubMed run.</EmptyState>
217 + )}
218 + </Section>
219 + </div>
220 +
221 + <aside className="space-y-10">
222 + <Section id="registrations" kicker="Momentum" title="Trial registrations, last 30 days" description="New oncology studies first posted on ClinicalTrials.gov per phase, compared with the preceding 30 days (all study types; a study with two phases counts in both).">
223 + {pulse.rows.length ? (
224 + <>
225 + {all ? (
226 + <p className="mb-2 text-[13.5px]">
227 + <span className="ci-num font-display text-2xl text-ink">{fmtInt(all.current)}</span> <span className="text-ink-2">new studies</span> <Delta current={all.current} previous={all.previous} />
228 + </p>
229 + ) : null}
230 + <div className="ci-table-wrap">
231 + <table className="ci-table">
232 + <thead>
233 + <tr>
234 + <th>Phase</th>
235 + <th className="num">Last 30 d</th>
236 + <th className="num">Previous 30 d</th>
237 + <th className="num">Change</th>
238 + </tr>
239 + </thead>
240 + <tbody>
241 + {pulse.rows
242 + .filter((r) => r.phase !== 'ALL')
243 + .map((r) => (
244 + <tr key={r.phase}>
245 + <td>{r.phase === 'NA' ? 'Not applicable' : phaseLabel(r.phase)}</td>
246 + <td className="num">{fmtInt(r.current)}</td>
247 + <td className="num">{fmtInt(r.previous)}</td>
248 + <td className="num">
249 + <Delta current={r.current} previous={r.previous} />
250 + </td>
251 + </tr>
252 + ))}
253 + </tbody>
254 + </table>
255 + </div>
256 + <Freshness dataUpdatedAt={pulse.asOf} extra="trial_pulse · ci-trial-pulse-v1 · source: clinicaltrials" />
257 + </>
258 + ) : (
259 + <EmptyState compact title="Registration pulse not yet computed" />
260 + )}
261 + </Section>
262 +
263 + <Section id="by-cancer" kicker="Momentum" title="Most new studies by cancer" description="Top-level cancers by studies first posted in the last 30 days (conditions reconciled to the taxonomy, descendants included), with the preceding 30 days.">
264 + {byCancer.length ? (
265 + <div className="ci-table-wrap">
266 + <table className="ci-table">
267 + <thead>
268 + <tr>
269 + <th>Cancer</th>
270 + <th className="num">Last 30 d</th>
271 + <th className="num">Previous</th>
272 + </tr>
273 + </thead>
274 + <tbody>
275 + {byCancer.map((c) => (
276 + <tr key={c.id}>
277 + <td>
278 + <Link href={`/cancer/${c.slug}/trials`} className="ci-link">
279 + {c.canonical_name}
280 + </Link>
281 + </td>
282 + <td className="num">{fmtInt(c.current)}</td>
283 + <td className="num text-ink-3">{fmtInt(c.previous)}</td>
284 + </tr>
285 + ))}
286 + </tbody>
287 + </table>
288 + </div>
289 + ) : (
290 + <EmptyState compact title="Data not yet available" />
291 + )}
292 + </Section>
293 +
294 + <Section id="datasets" kicker="Data" title="Dataset refreshes, last 14 days" description="Connector runs (excluding dry runs and probes) with the number of source records created or updated and the dataset version reported by the source." actions={<Link href="/data-updates" className="ci-link">Full log →</Link>}>
295 + {ingests.length ? (
296 + <ul className="divide-y divide-rule text-[13px]">
297 + {ingests.slice(0, 12).map((r) => (
298 + <li key={r.id} className="flex flex-wrap items-baseline justify-between gap-x-3 py-1.5">
299 + <span className="min-w-0">
300 + <Link href={`/source/${r.connector_id}`} className="ci-link">
301 + {r.source_name ?? r.connector_id}
302 + </Link>{' '}
303 + <StatusBadge status={r.status} />
304 + {r.dataset_version ? <span className="ci-mono ml-1 text-[11px] text-ink-3">{r.dataset_version}</span> : null}
305 + </span>
306 + <span className="ci-num text-[12px] text-ink-3">
307 + {fmtDate(r.started_at)} · +{fmtInt(r.records_created)} / ~{fmtInt(r.records_updated)}
308 + </span>
309 + </li>
310 + ))}
311 + </ul>
312 + ) : (
313 + <EmptyState compact title="No connector run in the last 14 days" />
314 + )}
315 + <Freshness dataUpdatedAt={lastIngest?.finished_at ?? null} extra="ingest_runs" />
316 + </Section>
317 +
318 + <Section id="events" kicker="Changes" title="Entity change events" description="Non-bulk changes recorded by connectors in the last 30 days (approvals added, aliases, merges, deprecations).">
319 + {events.length ? (
320 + <ul className="divide-y divide-rule text-[13px]">
321 + {events.slice(0, 20).map((e) => (
322 + <li key={e.id} className="py-1.5">
323 + <span className="ci-mono mr-2 text-[11.5px] text-ink-3">{fmtDate(e.created_at)}</span>
324 + <Badge tone="outline">{humanize(e.kind)}</Badge>{' '}
325 + {e.entity_href ? (
326 + <Link href={e.entity_href} className="ci-link">
327 + {e.entity_name ?? e.entity_id}
328 + </Link>
329 + ) : (
330 + <span>{e.entity_name ?? e.entity_id}</span>
331 + )}
332 + <span className="block text-[12px] text-ink-2">{truncate(e.summary, 160)}</span>
333 + </li>
334 + ))}
335 + </ul>
336 + ) : (
337 + <EmptyState compact title="No non-bulk change event in the last 30 days" />
338 + )}
339 + </Section>
340 + </aside>
341 + </div>
342 + <div className="mt-8">
343 + <Note>Guideline changes, safety communications and trial-result events are not yet connected (no guideline or pharmacovigilance connector is live); this page will list them as event types once a licensed source is ingested. Nothing is inferred from press releases.</Note>
344 + </div>
345 + </div>
346 + );
347 +}
modified apps/web/src/components/home/trial-map-module.tsx +3 −6
@@ -3,7 +3,7 @@ import { Section } from '@/components/ui/section';
3 3 import { EmptyState } from '@/components/ui/empty-state';
4 4 import { Freshness } from '@/components/ui/freshness';
5 5 import { ClaimBadge } from '@/components/ui/badge';
6 −import { WorldMap, type MapCountryDatum } from '@/components/charts/world-map';
6 +import type { MapCountryDatum } from '@/components/charts/world-map';
7 7 import { countryCounts } from '@/lib/queries/trial-sites';
8 8 import { fmtInt } from '@/lib/format';
9 9
@@ -24,7 +24,7 @@ export async function TrialMapModule({ topN = 8 }: { topN?: number }) {
24 24 description="Study locations with status RECRUITING (or in a recruiting study when the site has no status), all cancers, any phase. A site is one registrant-entered location; multi-site studies weigh by their number of sites."
25 25 actions={
26 26 <Link href="/trials/map?recruiting=1" className="ci-link">
27 − Full map and filters →
27 + Full choropleth map and filters →
28 28 </Link>
29 29 }
30 30 >
@@ -33,10 +33,7 @@ export async function TrialMapModule({ topN = 8 }: { topN?: number }) {
33 33 Country aggregates appear after <code className="ci-mono">pnpm cix intel</code> has run on ingested ClinicalTrials.gov locations.
34 34 </EmptyState>
35 35 ) : (
36 − <div className="grid gap-4 lg:grid-cols-[3fr_2fr]">
37 − <div className="min-w-0">
38 − <WorldMap data={data} metric="sites" compact ariaLabel={`World map of recruiting clinical trial sites per country, all cancers: ${fmtInt(totalSites)} sites in ${fmtInt(rows.length)} countries. The list beside the map gives the leading countries; the full table is on the trial map page.`} describedBy="trial-map-top" />
39 − </div>
36 + <div>
40 37 <div className="min-w-0">
41 38 <div className="ci-table-wrap">
42 39 <table className="ci-table" id="trial-map-top">
modified apps/web/src/lib/queries/graph.ts +1 −1
@@ -18,7 +18,7 @@ import { type CancerContext, type EdgeGroup, type FocusRef, type GraphEdge, type
18 18 * and depends on the web `@/lib/db` helpers, so it cannot be imported from the Fastify app.
19 19 */
20 20
21 −export const DEFAULT_GROUP_LIMIT = 25;
21 +export const DEFAULT_GROUP_LIMIT = 15;
22 22 export const EXPANDED_GROUP_LIMIT = 200;
23 23 export const TRIAL_GROUP_LIMIT = 10;
24 24 /** Cohort thresholds for the derived gene ↔ cancer link (frequency and cases affected). */
added apps/web/src/lib/queries/pulse.ts +248 −0
@@ -0,0 +1,248 @@
1 +import 'server-only';
2 +import { run, sql, safe } from '@/lib/db';
3 +
4 +/**
5 + * "What changed in cancer" (SPEC §42-43, §62): every item is a dated record already in the database
6 + * (an approval row, a registered study, a ranking row, an ingest run). Nothing is generated; the
7 + * page only orders and groups existing evidence by recency.
8 + */
9 +
10 +export interface PulseApproval {
11 + id: number;
12 + approval_date: string;
13 + authority: string;
14 + jurisdiction: string;
15 + status: string;
16 + approval_type: string | null;
17 + indication: string;
18 + drug_id: string;
19 + drug_slug: string;
20 + drug_name: string;
21 + cancer_id: string | null;
22 + cancer_slug: string | null;
23 + cancer_name: string | null;
24 + tumor_agnostic: boolean;
25 + source_slug: string;
26 + provenance_id: number;
27 +}
28 +export async function recentApprovals(days = 90, limit = 40): Promise<PulseApproval[]> {
29 + return safe(
30 + () =>
31 + run<PulseApproval>(sql`
32 + SELECT a.id, a.approval_date, a.authority, a.jurisdiction, a.status, a.approval_type, a.indication, a.tumor_agnostic, a.provenance_id,
33 + d.id AS drug_id, d.slug AS drug_slug, d.name AS drug_name, c.id AS cancer_id, c.slug AS cancer_slug, c.canonical_name AS cancer_name, s.slug AS source_slug
34 + FROM drug_approvals a JOIN drugs d ON d.id = a.drug_id LEFT JOIN cancers c ON c.id = a.cancer_id JOIN sources s ON s.id = a.source_id
35 + WHERE a.approval_date ~ '^\\d{4}-\\d{2}-\\d{2}$' AND a.approval_date::date >= current_date - ${days}::int AND a.approval_date::date <= current_date
36 + ORDER BY a.approval_date DESC, a.id DESC LIMIT ${limit}`),
37 + [] as PulseApproval[],
38 + );
39 +}
40 +
41 +export interface PulseTrial {
42 + id: string;
43 + nct_id: string;
44 + brief_title: string;
45 + acronym: string | null;
46 + phases: string[];
47 + overall_status: string | null;
48 + first_posted_date: string | null;
49 + enrollment_count: number | null;
50 + lead_sponsor: string | null;
51 + lead_sponsor_class: string | null;
52 + countries: string[];
53 + cancer_slug: string | null;
54 + cancer_name: string | null;
55 + updated_at: Date | string;
56 +}
57 +/** Interventional Phase III studies first posted in the window and recruiting now, with their first mapped cancer. */
58 +export async function newPhase3Recruiting(days = 30, limit = 25): Promise<PulseTrial[]> {
59 + return safe(
60 + () =>
61 + run<PulseTrial>(sql`
62 + SELECT t.id, t.nct_id, t.brief_title, t.acronym, t.phases, t.overall_status, t.first_posted_date, t.enrollment_count, t.lead_sponsor, t.lead_sponsor_class, t.countries, t.updated_at,
63 + m.slug AS cancer_slug, m.canonical_name AS cancer_name
64 + FROM clinical_trials t
65 + LEFT JOIN LATERAL (
66 + SELECT c.slug, c.canonical_name FROM trial_conditions tc JOIN cancers c ON c.id = tc.cancer_id
67 + WHERE tc.trial_id = t.id ORDER BY (tc.match_type = 'PROBABILISTIC'), c.depth, c.canonical_name LIMIT 1) m ON true
68 + WHERE t.study_type = 'INTERVENTIONAL' AND 'PHASE3' = ANY(t.phases) AND t.overall_status = 'RECRUITING'
69 + AND t.first_posted_date ~ '^\\d{4}-\\d{2}-\\d{2}$' AND t.first_posted_date::date >= current_date - ${days}::int
70 + ORDER BY t.first_posted_date DESC, t.nct_id LIMIT ${limit}`),
71 + [] as PulseTrial[],
72 + );
73 +}
74 +
75 +export interface PhasePulse {
76 + phase: string;
77 + current: number;
78 + previous: number;
79 +}
80 +/** New oncology studies per phase: last `days` vs the preceding `days` (from trial_pulse, cancer_id NULL = all studies). */
81 +export async function trialPulseByPhase(days = 30): Promise<{ rows: PhasePulse[]; asOf: Date | string | null }> {
82 + const rows = await safe(
83 + () =>
84 + run<{ phase: string; current: string; previous: string; as_of: Date | string | null }>(sql`
85 + SELECT phase,
86 + sum(new_trials) FILTER (WHERE day > current_date - ${days}::int) AS current,
87 + sum(new_trials) FILTER (WHERE day <= current_date - ${days}::int AND day > current_date - ${days * 2}::int) AS previous,
88 + max(updated_at) AS as_of
89 + FROM trial_pulse WHERE cancer_id IS NULL AND day > current_date - ${days * 2}::int
90 + GROUP BY phase ORDER BY CASE phase WHEN 'ALL' THEN 0 WHEN 'EARLY_PHASE1' THEN 1 WHEN 'PHASE1' THEN 2 WHEN 'PHASE2' THEN 3 WHEN 'PHASE3' THEN 4 WHEN 'PHASE4' THEN 5 ELSE 9 END`),
91 + [] as Array<{ phase: string; current: string; previous: string; as_of: Date | string | null }>,
92 + );
93 + return { rows: rows.map((r) => ({ phase: r.phase, current: Number(r.current ?? 0), previous: Number(r.previous ?? 0) })), asOf: rows[0]?.as_of ?? null };
94 +}
95 +
96 +export interface CancerPulse {
97 + id: string;
98 + slug: string;
99 + canonical_name: string;
100 + current: number;
101 + previous: number;
102 +}
103 +/** Top-level cancers with the most newly registered studies in the window (any phase), with the previous window for comparison. */
104 +export async function newTrialsByCancer(days = 30, limit = 10): Promise<CancerPulse[]> {
105 + return safe(
106 + () =>
107 + run<CancerPulse>(sql`
108 + SELECT c.id, c.slug, c.canonical_name,
109 + coalesce(sum(p.new_trials) FILTER (WHERE p.day > current_date - ${days}::int), 0)::int AS current,
110 + coalesce(sum(p.new_trials) FILTER (WHERE p.day <= current_date - ${days}::int), 0)::int AS previous
111 + FROM trial_pulse p JOIN cancers c ON c.id = p.cancer_id
112 + WHERE p.phase = 'ALL' AND c.top_level AND c.status = 'active' AND p.day > current_date - ${days * 2}::int
113 + GROUP BY c.id, c.slug, c.canonical_name ORDER BY current DESC, c.canonical_name LIMIT ${limit}`),
114 + [] as CancerPulse[],
115 + );
116 +}
117 +
118 +export interface RankingMove {
119 + metric_slug: string;
120 + metric_name: string;
121 + unit: string;
122 + scope_key: string;
123 + cancer_id: string;
124 + slug: string;
125 + canonical_name: string;
126 + rank: number;
127 + previous_rank: number;
128 + value: number;
129 + eligible_entities: number;
130 + generated_at: Date | string;
131 +}
132 +/** Largest rank changes between the current snapshot and the previous one for the same metric and scope (§131). */
133 +export async function rankingMoves(minDelta = 3, limit = 20): Promise<RankingMove[]> {
134 + return safe(
135 + () =>
136 + run<RankingMove>(sql`
137 + SELECT r.metric_slug, m.name AS metric_name, r.unit, r.scope_key, r.cancer_id, c.slug, c.canonical_name, r.rank, r.previous_rank, r.value, r.eligible_entities, s.generated_at
138 + FROM rankings r JOIN ranking_snapshots s ON s.id = r.snapshot_id JOIN cancers c ON c.id = r.cancer_id JOIN metric_definitions m ON m.slug = r.metric_slug
139 + WHERE s.is_current AND r.previous_rank IS NOT NULL AND abs(r.previous_rank - r.rank) >= ${minDelta} AND s.entity_level = 'top'
140 + ORDER BY abs(r.previous_rank - r.rank) DESC, s.generated_at DESC, r.metric_slug, r.rank LIMIT ${limit}`),
141 + [] as RankingMove[],
142 + );
143 +}
144 +
145 +export interface PulsePublication {
146 + id: string;
147 + pmid: string | null;
148 + doi: string | null;
149 + title: string;
150 + journal: string | null;
151 + pub_date: string | null;
152 + publication_types: string[];
153 + retracted: boolean;
154 +}
155 +/** Most recently published records indexed (PubMed), favouring trials, reviews and meta-analyses. */
156 +export async function recentPublications(days = 60, limit = 15): Promise<PulsePublication[]> {
157 + return safe(
158 + () =>
159 + run<PulsePublication>(sql`
160 + SELECT id, pmid, doi, title, journal, pub_date, publication_types, retracted FROM publications
161 + WHERE pub_date ~ '^\\d{4}-\\d{2}-\\d{2}$' AND pub_date::date >= current_date - ${days}::int AND pub_date::date <= current_date AND NOT retracted
162 + ORDER BY (publication_types && ARRAY['Randomized Controlled Trial','Clinical Trial, Phase III','Meta-Analysis','Systematic Review','Clinical Trial']::text[]) DESC, pub_date DESC LIMIT ${limit}`),
163 + [] as PulsePublication[],
164 + );
165 +}
166 +
167 +export interface IngestRow {
168 + id: string;
169 + connector_id: string;
170 + source_name: string | null;
171 + mode: string;
172 + status: string;
173 + started_at: Date | string;
174 + finished_at: Date | string | null;
175 + duration_ms: number | null;
176 + records_fetched: number;
177 + records_created: number;
178 + records_updated: number;
179 + records_unchanged: number;
180 + records_rejected: number;
181 + dataset_version: string | null;
182 + anomaly: string | null;
183 +}
184 +/** Ingest runs of the last `days` days (all statuses), most recent first (§62 data update log). */
185 +export async function recentIngests(days = 30, limit = 200): Promise<IngestRow[]> {
186 + return safe(
187 + () =>
188 + run<IngestRow>(sql`
189 + SELECT r.id, r.connector_id, s.name AS source_name, r.mode, r.status, r.started_at, r.finished_at, r.duration_ms, r.records_fetched, r.records_created, r.records_updated, r.records_unchanged, r.records_rejected, r.dataset_version, r.anomaly
190 + FROM ingest_runs r LEFT JOIN sources s ON s.slug = r.connector_id
191 + WHERE r.started_at >= now() - (${days}::text || ' days')::interval AND r.mode NOT IN ('dry_run','probe')
192 + ORDER BY r.started_at DESC LIMIT ${limit}`),
193 + [] as IngestRow[],
194 + );
195 +}
196 +
197 +export interface ConnectorState {
198 + connector_id: string;
199 + source_name: string | null;
200 + category: string | null;
201 + status: string | null;
202 + health: string;
203 + last_success_at: Date | string | null;
204 + last_attempt_at: Date | string | null;
205 + paused: boolean;
206 + schedule: string | null;
207 + last_dataset_version: string | null;
208 + record_count: number;
209 +}
210 +/** One line per connector: health, last success, schedule, latest dataset version, source records held. */
211 +export async function connectorStates(): Promise<ConnectorState[]> {
212 + return safe(
213 + () =>
214 + run<ConnectorState>(sql`
215 + SELECT s.slug AS connector_id, s.name AS source_name, s.category, s.status, coalesce(cc.health, 'unknown') AS health, cc.last_success_at, cc.last_attempt_at, coalesce(cc.paused, false) AS paused,
216 + s.manifest->>'schedule' AS schedule,
217 + (SELECT r.dataset_version FROM ingest_runs r WHERE r.connector_id = s.slug AND r.status IN ('succeeded','partial') ORDER BY r.started_at DESC LIMIT 1) AS last_dataset_version,
218 + (SELECT count(*) FROM source_records sr WHERE sr.source_id = s.id)::int AS record_count
219 + FROM sources s LEFT JOIN connector_cursors cc ON cc.connector_id = s.slug
220 + ORDER BY s.status = 'active' DESC, s.tier, s.name`),
221 + [] as ConnectorState[],
222 + );
223 +}
224 +
225 +/** Recent change events other than bulk creations (approval_added, alias_added, deprecated, merged…). */
226 +export interface PulseEvent {
227 + id: number;
228 + entity_type: string;
229 + entity_id: string;
230 + kind: string;
231 + summary: string;
232 + created_at: Date | string;
233 + entity_name: string | null;
234 + entity_href: string | null;
235 +}
236 +export async function recentEvents(days = 30, limit = 40): Promise<PulseEvent[]> {
237 + return safe(
238 + () =>
239 + run<PulseEvent>(sql`
240 + SELECT e.id, e.entity_type, e.entity_id, e.kind, e.summary, e.created_at,
241 + CASE e.entity_type WHEN 'cancer' THEN (SELECT canonical_name FROM cancers WHERE id = e.entity_id) WHEN 'gene' THEN (SELECT symbol FROM genes WHERE id = e.entity_id) WHEN 'drug' THEN (SELECT name FROM drugs WHERE id = e.entity_id) WHEN 'trial' THEN (SELECT nct_id FROM clinical_trials WHERE id = e.entity_id) END AS entity_name,
242 + CASE e.entity_type WHEN 'cancer' THEN (SELECT '/cancer/' || slug FROM cancers WHERE id = e.entity_id) WHEN 'gene' THEN (SELECT '/gene/' || symbol FROM genes WHERE id = e.entity_id) WHEN 'drug' THEN (SELECT '/drug/' || slug FROM drugs WHERE id = e.entity_id) WHEN 'trial' THEN (SELECT '/trial/' || nct_id FROM clinical_trials WHERE id = e.entity_id) END AS entity_href
243 + FROM change_events e
244 + WHERE e.created_at >= now() - (${days}::text || ' days')::interval AND e.kind <> 'created'
245 + ORDER BY e.created_at DESC, e.id DESC LIMIT ${limit}`),
246 + [] as PulseEvent[],
247 + );
248 +}
modified apps/web/src/lib/site.ts +4 −0
@@ -19,6 +19,8 @@ export const NAV = [
19 19
20 20 /** Secondary navigation (mobile menu + footer): everything not in the primary bar. */
21 21 export const MORE_NAV = [
22 + { href: '/pulse', label: 'Pulse' },
23 + { href: '/data-updates', label: 'Data updates' },
22 24 { href: '/taxonomy', label: 'Taxonomy' },
23 25 { href: '/countries', label: 'Countries' },
24 26 { href: '/compare', label: 'Compare' },
@@ -34,6 +36,8 @@ export const FOOTER_NAV = [
34 36 { href: '/trust', label: 'Trust & policies' },
35 37 { href: '/explore', label: 'Data explorer' },
36 38 { href: '/data', label: 'Data downloads' },
39 + { href: '/pulse', label: 'Pulse' },
40 + { href: '/data-updates', label: 'Data update log' },
37 41 { href: '/developers', label: 'Developers (API)' },
38 42 { href: '/sources', label: 'Sources' },
39 43 { href: '/methodology', label: 'Methodology' },
40 44