SPB Git forge
spboucher.ai ↗

spb/cancerindex

Public
37commits 1branches 0releases
2.9 MBsize
maindefault branch
10 days agolast push
TypeScript 97.2% SQL 1.5% CSS 0.6% JavaScript 0.5%

Wave 2: SDK checkpoints/abort/anomaly guard/alerts, cix doctor, backups, perf indexes, migration 0001; MeSH, ChEMBL, openFDA, cBioPortal connectors; web pagination/QA/tests/caching, countries, compare, key figures; USCS sex-specific rate fix

Simon-Pierre Boucher committed 16 days ago (Sep 8, 2026) parent 4354e64

159 changed files +18,136 −784

modified apps/api/src/routes/admin.ts +8 −0
@@ -151,6 +151,14 @@ export const adminRoutes: FastifyPluginAsyncZod = async (app) => {
151 151 });
152 152 }
153 153
154 + app.get('/alerts', { schema: { tags: ['admin'], summary: 'Internal alerts (§170): connector failures, anomalies, schema drift, stale/failing sources', security: [{ adminToken: [] }], querystring: z.object({ status: z.enum(['open', 'acknowledged', 'resolved', 'active']).default('active'), connectorId: z.string().optional(), limit: z.coerce.number().int().min(1).max(500).default(100) }), response: ok(AnyList) } }, async (req) => {
155 + const q = req.query;
156 + const conds = [q.status === 'active' ? sql`status IN ('open','acknowledged')` : sql`status = ${q.status}`];
157 + if (q.connectorId) conds.push(sql`connector_id = ${q.connectorId}`);
158 + const rows = await app.db.execute<Record<string, unknown>>(sql`SELECT * FROM system_alerts WHERE ${sql.join(conds, sql` AND `)} ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warn' THEN 1 ELSE 2 END, last_seen_at DESC LIMIT ${q.limit}`);
159 + return respond(app, camelRows(rows), []);
160 + });
161 +
154 162 app.get('/audit', { schema: { tags: ['admin'], summary: 'Recent audit log entries', security: [{ adminToken: [] }], querystring: z.object({ limit: z.coerce.number().int().min(1).max(500).default(100) }), response: ok(AnyList) } }, async (req) => {
155 163 const rows = await app.db.execute<Record<string, unknown>>(sql`SELECT * FROM audit_log ORDER BY created_at DESC, id DESC LIMIT ${req.query.limit}`);
156 164 return respond(app, camelRows(rows), []);
modified apps/web/next.config.ts +9 −0
@@ -37,6 +37,11 @@ const nextConfig: NextConfig = {
37 37 },
38 38 outputFileTracingRoot: path.resolve(__dirname, '../..'),
39 39 async headers() {
40 + // Static-ish public routes: let ngrok/CDN/browser caches share a rendered page for 10 min and serve
41 + // stale for an hour while revalidating. Never for /admin, /search, /api (user- or query-specific).
42 + const cacheable = ['/cancer/:path*', '/gene/:path*', '/variant/:path*', '/drug/:path*', '/trial/:path*', '/publication/:path*', '/source/:path*', '/rankings', '/rankings/:path*'];
43 + const PUBLIC_CACHE = { key: 'Cache-Control', value: 'public, s-maxage=600, stale-while-revalidate=3600' };
44 + const NO_STORE = { key: 'Cache-Control', value: 'private, no-store' };
40 45 return [
41 46 {
42 47 source: '/(.*)',
@@ -47,6 +52,10 @@ const nextConfig: NextConfig = {
47 52 { key: 'Permissions-Policy', value: 'camera=(), microphone=(), geolocation=()' },
48 53 ],
49 54 },
55 + ...cacheable.map((source) => ({ source, headers: [PUBLIC_CACHE] })),
56 + { source: '/admin/:path*', headers: [NO_STORE] },
57 + { source: '/search', headers: [NO_STORE] },
58 + { source: '/api/:path*', headers: [NO_STORE] },
50 59 ];
51 60 },
52 61 };
modified apps/web/package.json +3 −1
@@ -7,7 +7,9 @@
7 7 "build": "next build --webpack",
8 8 "start": "next start -p 8250",
9 9 "typecheck": "tsc -p tsconfig.json --noEmit",
10 − "test": "vitest run --passWithNoTests"
10 + "test": "vitest run --passWithNoTests",
11 + "qa": "node qa/smoke.mjs",
12 + "qa:prod": "BASE_URL=https://www.cancerindex.io node qa/smoke.mjs"
11 13 },
12 14 "dependencies": {
13 15 "@cancerindex/database": "workspace:*",
added apps/web/qa/smoke.mjs +239 −0
@@ -0,0 +1,239 @@
1 +#!/usr/bin/env node
2 +/**
3 + * CancerIndex web smoke suite — read-only.
4 + *
5 + * BASE_URL=http://127.0.0.1:8253 node qa/smoke.mjs (default)
6 + * BASE_URL=https://www.cancerindex.io node qa/smoke.mjs (production, read-only)
7 + *
8 + * Checks, for every route in ROUTES: HTTP 200, key text present, HTML weight under MAX_KB, no
9 + * "Data not yet available" on pages that must have data, and — when Playwright is available — no
10 + * horizontal overflow at 390 px plus a console-error scan. Playwright is resolved from a sibling
11 + * project's node_modules (never installed globally); without it the suite degrades to HTTP-only.
12 + *
13 + * Exit code 1 when any hard check fails. Prints a table.
14 + */
15 +import { createRequire } from 'node:module';
16 +import { existsSync } from 'node:fs';
17 +import os from 'node:os';
18 +import path from 'node:path';
19 +
20 +const BASE_URL = (process.env.BASE_URL ?? 'http://127.0.0.1:8253').replace(/\/$/, '');
21 +const MAX_KB = Number(process.env.MAX_KB ?? 600);
22 +const TIMEOUT_MS = Number(process.env.TIMEOUT_MS ?? 30000);
23 +const MOBILE_WIDTH = 390;
24 +const ONLY_HTTP = process.env.HTTP_ONLY === '1';
25 +
26 +/** @type {Array<{path: string, expect: string[], mustHaveData?: boolean, optionalLocal?: boolean, resolve?: (ctx: Record<string,string>) => string | null, kind?: 'html'|'xml'|'json'}>} */
27 +const ROUTES = [
28 + { path: '/', expect: ['CancerIndex'] },
29 + { path: '/cancers', expect: ['Cancers'] },
30 + { path: '/cancer/malignant-pancreatic-neoplasm', expect: ['Malignant Pancreatic Neoplasm', 'CI-CAN-'] },
31 + // "observations" only appears in the data branch (Freshness "N observations"); the empty branch is caught by mustHaveData.
32 + { path: '/cancer/malignant-pancreatic-neoplasm/statistics', expect: ['Population statistics', 'observations'], mustHaveData: true },
33 + { path: '/cancer/malignant-pancreatic-neoplasm/trials', expect: ['Registered studies', 'NCT'] },
34 + { path: '/cancer/malignant-pancreatic-neoplasm/evidence', expect: ['Curated clinical evidence', 'civic'] },
35 + { path: '/cancer/malignant-pancreatic-neoplasm/rankings', expect: ['Rankings'] },
36 + { path: '/gene/TP53', expect: ['TP53', 'Clinical evidence'] },
37 + { path: '/variant/braf-v600e', expect: ['V600E', 'Evidence by cancer'] },
38 + { path: '/drug/osimertinib', expect: ['Osimertinib', 'Approvals'], resolve: (ctx) => ctx.drug ?? '/drug/osimertinib' },
39 + { path: '/trial/<first NCT>', expect: ['NCT', 'Conditions'], resolve: (ctx) => ctx.trial ?? null },
40 + { path: '/rankings', expect: ['Rankings', 'metric'] },
41 + { path: '/rankings/mortality_count', expect: ['mortality', 'rank'] },
42 + { path: '/taxonomy', expect: ['Taxonomy'] },
43 + { path: '/sources', expect: ['Sources', 'license'] },
44 + { path: '/source/ncit-evs', expect: ['NCI', 'ncit-evs'] },
45 + { path: '/methodology', expect: ['Methodology'] },
46 + { path: '/search?q=glio', expect: ['Results for', 'glio'] },
47 + { path: '/sitemap.xml', expect: ['<sitemapindex', '<loc>'], kind: 'xml' },
48 + { path: '/api/v1/stats', expect: ['data'], kind: 'json', optionalLocal: true },
49 +];
50 +
51 +const c = { green: (s) => `\x1b[32m${s}\x1b[0m`, red: (s) => `\x1b[31m${s}\x1b[0m`, yellow: (s) => `\x1b[33m${s}\x1b[0m`, dim: (s) => `\x1b[2m${s}\x1b[0m` };
52 +const isLocal = /^(https?:\/\/)?(127\.0\.0\.1|localhost|0\.0\.0\.0)(:|\/|$)/.test(BASE_URL);
53 +
54 +async function fetchText(url) {
55 + const ctl = new AbortController();
56 + const t = setTimeout(() => ctl.abort(), TIMEOUT_MS);
57 + const started = performance.now();
58 + try {
59 + const res = await fetch(url, { signal: ctl.signal, redirect: 'follow', headers: { 'user-agent': 'cancerindex-smoke/1.0', accept: 'text/html,application/xml,application/json;q=0.9,*/*;q=0.8' } });
60 + const buf = Buffer.from(await res.arrayBuffer());
61 + return { status: res.status, bytes: buf.byteLength, text: buf.toString('utf8'), ms: Math.round(performance.now() - started), headers: res.headers, finalUrl: res.url };
62 + } finally {
63 + clearTimeout(t);
64 + }
65 +}
66 +
67 +/** Discover a real drug slug and NCT id from the running site (no DB access from the QA suite). */
68 +async function discover() {
69 + const ctx = {};
70 + try {
71 + const d = await fetchText(`${BASE_URL}/drug/osimertinib`);
72 + if (d.status === 200) ctx.drug = '/drug/osimertinib';
73 + else {
74 + const list = await fetchText(`${BASE_URL}/drugs`);
75 + const m = /href="(\/drug\/[a-z0-9-]+)"/.exec(list.text);
76 + if (m) ctx.drug = m[1];
77 + }
78 + } catch {}
79 + try {
80 + const list = await fetchText(`${BASE_URL}/trials`);
81 + const m = /href="(\/trial\/NCT\d{8})"/.exec(list.text);
82 + if (m) ctx.trial = m[1];
83 + } catch {}
84 + return ctx;
85 +}
86 +
87 +async function loadPlaywright() {
88 + if (ONLY_HTTP) return null;
89 + const candidates = [
90 + process.env.PLAYWRIGHT_DIR,
91 + path.join(os.homedir(), 'Desktop/Projets/apps-web/fetcha/qa'),
92 + path.join(os.homedir(), 'Desktop/Projets/apps-web/rareindex'),
93 + path.join(os.homedir(), 'Desktop/Projets/apps-web/fetcha'),
94 + process.cwd(),
95 + ].filter(Boolean);
96 + for (const dir of candidates) {
97 + const pkg = path.join(dir, 'node_modules', 'playwright', 'package.json');
98 + if (!existsSync(pkg)) continue;
99 + try {
100 + const req = createRequire(path.join(dir, 'package.json'));
101 + const pw = req('playwright');
102 + const browser = await pw.chromium.launch({ headless: true });
103 + return { pw, browser, from: dir };
104 + } catch (err) {
105 + console.error(c.dim(`playwright at ${dir} unusable: ${err.message.split('\n')[0]}`));
106 + }
107 + }
108 + return null;
109 +}
110 +
111 +function pad(s, n, right = false) {
112 + s = String(s);
113 + if (s.length >= n) return s.slice(0, n);
114 + return right ? ' '.repeat(n - s.length) + s : s + ' '.repeat(n - s.length);
115 +}
116 +
117 +async function main() {
118 + console.log(`CancerIndex smoke · ${BASE_URL} · max ${MAX_KB} KB · viewport ${MOBILE_WIDTH}px`);
119 + const ctx = await discover();
120 + const pwr = await loadPlaywright();
121 + if (pwr) console.log(c.dim(`Playwright from ${pwr.from}`));
122 + else console.log(c.yellow('Playwright unavailable — HTTP-only mode (no overflow / console checks)'));
123 +
124 + const rows = [];
125 + let failures = 0;
126 + let warnings = 0;
127 + const page = pwr ? await pwr.browser.newPage({ viewport: { width: MOBILE_WIDTH, height: 844 }, deviceScaleFactor: 2, isMobile: true, hasTouch: true }) : null;
128 +
129 + for (const r of ROUTES) {
130 + const p = r.resolve ? r.resolve(ctx) : r.path;
131 + if (!p) {
132 + rows.push({ path: r.path, status: '—', kb: '—', ms: '—', overflow: '—', result: c.yellow('SKIP'), note: 'could not discover an instance' });
133 + warnings++;
134 + continue;
135 + }
136 + const url = `${BASE_URL}${p}`;
137 + const problems = [];
138 + const notes = [];
139 + let res;
140 + try {
141 + res = await fetchText(url);
142 + } catch (err) {
143 + rows.push({ path: p, status: 'ERR', kb: '—', ms: '—', overflow: '—', result: c.red('FAIL'), note: err.message });
144 + failures++;
145 + continue;
146 + }
147 + const kb = Math.round(res.bytes / 1024);
148 + const optional = r.optionalLocal && isLocal;
149 + if (res.status !== 200) {
150 + if (optional && (res.status === 500 || res.status === 502 || res.status === 503)) notes.push(`API proxy target down locally (${res.status}) — optional`);
151 + else problems.push(`status ${res.status}`);
152 + } else {
153 + const lower = res.text.toLowerCase();
154 + for (const key of r.expect) if (!lower.includes(key.toLowerCase())) problems.push(`missing text "${key}"`);
155 + if (r.kind !== 'json' && r.kind !== 'xml') {
156 + if (kb > MAX_KB) problems.push(`weight ${kb} KB > ${MAX_KB} KB`);
157 + if (r.mustHaveData && lower.includes('data not yet available')) problems.push('"Data not yet available" on a page that must have data');
158 + if (/application error|internal server error|something went wrong rendering/i.test(res.text)) problems.push('error boundary rendered');
159 + }
160 + if (r.kind === 'json') {
161 + try {
162 + const j = JSON.parse(res.text);
163 + if (!j || typeof j !== 'object' || !('data' in j)) problems.push('JSON envelope missing "data"');
164 + } catch {
165 + problems.push('invalid JSON');
166 + }
167 + }
168 + const cc = res.headers.get('cache-control') ?? '';
169 + if (/^\/(cancer|gene|drug|trial|source)\//.test(p) || /^\/rankings/.test(p)) {
170 + if (!/s-maxage/.test(cc)) notes.push(`no s-maxage (cache-control: ${cc || 'none'})`);
171 + }
172 + }
173 +
174 + let overflow = '—';
175 + if (page && res.status === 200 && r.kind !== 'json' && r.kind !== 'xml') {
176 + const consoleErrors = [];
177 + const onConsole = (m) => {
178 + if (m.type() === 'error') consoleErrors.push(m.text());
179 + };
180 + page.on('console', onConsole);
181 + try {
182 + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: TIMEOUT_MS });
183 + await page.waitForTimeout(250);
184 + const m = await page.evaluate(() => {
185 + const doc = document.documentElement;
186 + const bodyOverflow = Math.max(doc.scrollWidth, document.body.scrollWidth) - doc.clientWidth;
187 + // Elements wider than the viewport that are not inside a horizontally scrollable wrapper.
188 + const offenders = [];
189 + const vw = doc.clientWidth;
190 + for (const el of document.querySelectorAll('body *')) {
191 + const rect = el.getBoundingClientRect();
192 + if (rect.right > vw + 1 && rect.width > 0) {
193 + let scrollable = false;
194 + for (let a = el.parentElement; a; a = a.parentElement) {
195 + const ov = getComputedStyle(a).overflowX;
196 + if (ov === 'auto' || ov === 'scroll' || ov === 'hidden' || ov === 'clip') {
197 + scrollable = true;
198 + break;
199 + }
200 + }
201 + if (!scrollable && el.tagName !== 'HTML' && el.tagName !== 'BODY') offenders.push(`${el.tagName.toLowerCase()}${el.className && typeof el.className === 'string' ? '.' + el.className.split(' ').slice(0, 2).join('.') : ''} ${Math.round(rect.right - vw)}px`);
202 + if (offenders.length >= 3) break;
203 + }
204 + }
205 + return { bodyOverflow, offenders };
206 + });
207 + overflow = m.bodyOverflow > 1 ? `${m.bodyOverflow}px` : 'ok';
208 + if (m.bodyOverflow > 1) problems.push(`horizontal overflow ${m.bodyOverflow}px at ${MOBILE_WIDTH}px${m.offenders.length ? ` (${m.offenders.join('; ')})` : ''}`);
209 + const realErrors = consoleErrors.filter((t) => !/favicon|net::ERR_|Failed to load resource|hydrat/i.test(t));
210 + if (realErrors.length) notes.push(`${realErrors.length} console error(s): ${realErrors[0].slice(0, 90)}`);
211 + } catch (err) {
212 + notes.push(`browser: ${err.message.split('\n')[0].slice(0, 80)}`);
213 + } finally {
214 + page.off('console', onConsole);
215 + }
216 + }
217 +
218 + const ok = problems.length === 0;
219 + if (!ok) failures++;
220 + if (notes.length) warnings++;
221 + rows.push({ path: p, status: res.status, kb, ms: res.ms, overflow, result: ok ? c.green('PASS') : c.red('FAIL'), note: [...problems, ...notes].join(' · ') });
222 + }
223 +
224 + if (pwr) await pwr.browser.close();
225 +
226 + console.log('');
227 + console.log(`${pad('Route', 52)} ${pad('Status', 6, true)} ${pad('KB', 6, true)} ${pad('ms', 6, true)} ${pad('390px', 7)} Result Notes`);
228 + console.log('-'.repeat(120));
229 + for (const r of rows) console.log(`${pad(r.path, 52)} ${pad(r.status, 6, true)} ${pad(r.kb, 6, true)} ${pad(r.ms, 6, true)} ${pad(r.overflow, 7)} ${r.result} ${r.note ? c.dim(r.note) : ''}`);
230 + console.log('-'.repeat(120));
231 + const passed = rows.filter((r) => r.result.includes('PASS')).length;
232 + console.log(`${passed}/${rows.length} routes passed · ${failures} failure(s) · ${warnings} warning(s) · ${BASE_URL}`);
233 + process.exit(failures ? 1 : 0);
234 +}
235 +
236 +main().catch((err) => {
237 + console.error(c.red(`smoke crashed: ${err.stack ?? err}`));
238 + process.exit(2);
239 +});
added apps/web/src/app/cancer/[slug]/[[...tab]]/loading.tsx +6 −0
@@ -0,0 +1,6 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +/** Streams immediately while the cancer bundle + tab data load (also guarantees a 200 shell on DB errors → error.tsx). */
4 +export default function Loading() {
5 + return <PageSkeleton title="Loading cancer entity" tabs={10} />;
6 +}
modified apps/web/src/app/cancer/[slug]/[[...tab]]/page.tsx +9 −5
@@ -62,9 +62,11 @@ export default async function CancerPage({ params, searchParams }: { params: Pro
62 62 body = <GenomicsTab b={b} cohort={str(sp, 'cohort') || null} />;
63 63 break;
64 64 }
65 − case 'evidence':
66 − body = <EvidenceTab b={b} />;
65 + case 'evidence': {
66 + const sp = await searchParams;
67 + body = <EvidenceTab b={b} evPage={int(sp, 'evPage', 1, 1, 100_000)} />;
67 68 break;
69 + }
68 70 case 'drugs': {
69 71 const sp = await searchParams;
70 72 body = <DrugsTab b={b} jurisdiction={str(sp, 'jurisdiction') || null} />;
@@ -72,12 +74,14 @@ export default async function CancerPage({ params, searchParams }: { params: Pro
72 74 }
73 75 case 'trials': {
74 76 const sp = await searchParams;
75 − body = <TrialsTab b={b} status={str(sp, 'status')} phase={str(sp, 'phase')} page={int(sp, 'page', 1, 1, 1000)} />;
77 + body = <TrialsTab b={b} status={str(sp, 'status')} phase={str(sp, 'phase')} page={int(sp, 'page', 1, 1, 100_000)} />;
76 78 break;
77 79 }
78 − case 'research':
79 − body = <ResearchTab b={b} />;
80 + case 'research': {
81 + const sp = await searchParams;
82 + body = <ResearchTab b={b} pPage={int(sp, 'pPage', 1, 1, 100_000)} />;
80 83 break;
84 + }
81 85 case 'rankings':
82 86 body = <RankingsTab b={b} />;
83 87 break;
added apps/web/src/app/cancers/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading cancers" />;
5 +}
modified apps/web/src/app/cancers/page.tsx +2 −1
@@ -10,7 +10,8 @@ import { fmtInt, humanize } from '@/lib/format';
10 10 import { str, int, oneOf, bool, withParams, type SP } from '@/lib/search-params';
11 11
12 12 export const metadata: Metadata = { title: 'Cancers', description: 'Explore every indexed cancer entity with filters for type, anatomical site, hematologic and pediatric relevance.' };
13 −export const dynamic = 'force-dynamic';
13 +// Filtered list: rendered per request (searchParams); 600 s is the CDN/ISR hint shared by all list pages.
14 +export const revalidate = 600;
14 15
15 16 const PAGE_SIZE = 50;
16 17
added apps/web/src/app/compare/page.tsx +260 −0
@@ -0,0 +1,260 @@
1 +import type { Metadata } from 'next';
2 +import Link from 'next/link';
3 +import type { ReactNode } from 'react';
4 +import { PageHeader, Section, Note } from '@/components/ui/section';
5 +import { EmptyState } from '@/components/ui/empty-state';
6 +import { Freshness } from '@/components/ui/freshness';
7 +import { Badge, ClaimBadge, ConfidenceBadge } from '@/components/ui/badge';
8 +import { SourceBadge } from '@/components/ui/source-badge';
9 +import { CompareBars, type CompareBarDatum } from '@/components/charts/compare-bars';
10 +import { loadCompare, parseCompareIds, COMPARE_MIN, COMPARE_MAX, type CompareEntity } from '@/lib/queries/compare';
11 +import { loadProvenance, toInfo } from '@/lib/queries/provenance';
12 +import { bestRankFor } from '@/lib/queries/rankings';
13 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
14 +import { str, type SP } from '@/lib/search-params';
15 +import { fmtInt, fmtValue, humanize, scopeLabel, toDate, unitLabel } from '@/lib/format';
16 +import { ComparePicker } from './picker';
17 +
18 +export const dynamic = 'force-dynamic';
19 +
20 +export async function generateMetadata({ searchParams }: { searchParams: Promise<SP> }): Promise<Metadata> {
21 + const ids = parseCompareIds(str(await searchParams, 'ids'));
22 + const ents = ids.length ? await loadCompare(ids) : [];
23 + const names = ents.map((e) => e.cancer.canonical_name);
24 + return {
25 + title: names.length >= 2 ? `Compare: ${names.join(' vs ')}` : 'Compare cancers',
26 + description: names.length >= 2 ? `Side-by-side facts for ${names.join(', ')}: taxonomy, US registry figures, trials, literature, curated evidence, cohorts and current ranks — every value with its source and period.` : 'Compare two to four cancers side by side: taxonomy, US registry figures, clinical research, literature, molecular evidence and current ranks, with a source on every value.',
27 + alternates: { canonical: ids.length ? `/compare?ids=${ids.join(',')}` : '/compare' },
28 + robots: ids.length ? { index: false, follow: true } : undefined,
29 + };
30 +}
31 +
32 +const EPI_ROWS: Array<[string, string]> = [
33 + ['mortality_count', 'US annual deaths'],
34 + ['incidence_count', 'US annual new cases'],
35 + ['as_mortality_rate', 'US age-standardized mortality'],
36 + ['as_incidence_rate', 'US age-standardized incidence'],
37 +];
38 +
39 +const RANK_METRICS = ['mortality_count', 'incidence_count', 'as_mortality_rate', 'as_incidence_rate', 'mortality_incidence_ratio', 'trial_gap', 'research_gap', 'active_trials', 'recruiting_trials', 'phase3_trials', 'publications_5y', 'publications_12m', 'publication_growth', 'curated_evidence_items', 'associated_genes', 'genomic_cohorts'];
40 +
41 +/**
42 + * /compare?ids=a,b,c (§100): side-by-side table of facts that exist in the database. Registry figures below the
43 + * top level fall back to the nearest top-level ancestor with an explicit label (methodology#compare).
44 + */
45 +export default async function ComparePage({ searchParams }: { searchParams: Promise<SP> }) {
46 + const sp = await searchParams;
47 + const ids = parseCompareIds(str(sp, 'ids'));
48 + const ents = ids.length ? await loadCompare(ids) : [];
49 + const missing = ids.filter((s) => !ents.some((e) => e.cancer.slug === s));
50 + const enough = ents.length >= COMPARE_MIN;
51 + const provIds = ents.flatMap((e) => [...e.figures.values()].map((f) => f.provenance_id));
52 + const prov = await loadProvenance(provIds);
53 +
54 + const registryNote = (e: CompareEntity) => (e.registry && e.registry.depth > 0 ? `registry level: ${e.registry.canonical_name}` : null);
55 + const freshest = ents
56 + .flatMap((e) => [...[...e.figures.values()].map((f) => toDate(f.updated_at)), e.counters ? toDate(e.counters.updated_at) : null, ...e.ranks.map((r) => toDate(r.generated_at))])
57 + .filter((d): d is Date => !!d)
58 + .sort((a, b) => b.getTime() - a.getTime())[0];
59 +
60 + return (
61 + <div>
62 + <PageHeader kicker="Comparison engine" title="Compare cancers" lede="Two to four cancers side by side. Only facts present in the database are shown; every figure carries its source, period and unit. Entities at different hierarchy depths are labelled so counts are not read as equivalent." />
63 + <ComparePicker selected={ents.map((e) => ({ slug: e.cancer.slug, name: e.cancer.canonical_name }))} max={COMPARE_MAX} min={COMPARE_MIN} />
64 + {missing.length ? (
65 + <p className="mt-2 text-[12.5px] text-warn">
66 + Unknown or merged slug{missing.length > 1 ? 's' : ''}: <span className="ci-mono">{missing.join(', ')}</span> — ignored.
67 + </p>
68 + ) : null}
69 +
70 + {!enough ? (
71 + <div className="mt-6">
72 + <EmptyState title={ents.length === 1 ? 'Add at least one more cancer' : 'Pick cancers to compare'} knows={[{ label: 'Pancreatic vs lung vs glioblastoma', href: '/compare?ids=malignant-pancreatic-neoplasm,malignant-lung-neoplasm,glioblastoma' }, { label: 'Breast vs prostate', href: '/compare?ids=malignant-breast-neoplasm,malignant-prostate-neoplasm' }, { label: 'Cancer explorer', href: '/cancers' }]}>
73 + Use the picker above or open <span className="ci-mono">/compare?ids=slug-1,slug-2</span> with cancer slugs from their pages.
74 + </EmptyState>
75 + </div>
76 + ) : (
77 + <div className="mt-6 space-y-8">
78 + <CompareTable ents={ents} title="Taxonomy" id="taxonomy" rows={[
79 + { k: 'CancerIndex ID', cells: ents.map((e) => <span className="ci-mono text-[12px]">{e.cancer.id}</span>) },
80 + { k: 'Entity type', cells: ents.map((e) => humanize(e.cancer.entity_type)) },
81 + { k: 'Hierarchy depth', cells: ents.map((e) => <span className="ci-num">{e.cancer.depth}</span>) },
82 + { k: 'Top-level registry site', cells: ents.map((e) => (e.cancer.top_level ? 'Yes' : e.registry ? <span>No — nearest: <Link className="ci-link" href={`/cancer/${e.registry.slug}`}>{e.registry.canonical_name}</Link> <span className="text-ink-3">({e.registry.depth} level{e.registry.depth === 1 ? '' : 's'} up)</span></span> : <span className="text-ink-3">No top-level ancestor</span>)) },
83 + { k: 'Parents', cells: ents.map((e) => (e.parents.length ? <ul className="space-y-0.5">{e.parents.map((p) => <li key={`${p.hierarchy_type}-${p.slug}`}><Link className="ci-link" href={`/cancer/${p.slug}`}>{p.canonical_name}</Link> <span className="text-[11px] text-ink-3">{p.hierarchy_type}</span></li>)}</ul> : <span className="text-ink-3">root</span>)) },
84 + { k: 'NCIt / OncoTree', cells: ents.map((e) => <span className="ci-mono text-[12px]">{[e.cancer.primary_ncit_code, e.cancer.primary_oncotree_code].filter(Boolean).join(' · ') || '—'}</span>) },
85 + { k: 'Badges', cells: ents.map((e) => <span className="flex flex-wrap gap-1">{e.cancer.rare_cancer === true ? <Badge tone="accent">Rare</Badge> : null}{e.cancer.pediatric_relevant ? <Badge>Pediatric</Badge> : null}{e.cancer.hematologic ? <Badge>Hematologic</Badge> : null}{e.cancer.solid_tumor && !e.cancer.hematologic ? <Badge tone="outline">Solid tumor</Badge> : null}{!e.cancer.malignant ? <Badge tone="outline">Non-malignant</Badge> : null}{e.cancer.top_level ? <Badge tone="outline">Top-level</Badge> : null}{e.cancer.rare_cancer == null ? <span className="text-[11.5px] text-ink-3">rarity unknown</span> : null}</span>) },
86 + ]} caption={<>Taxonomy from NCIt / OncoTree (<ClaimBadge kind="curated" />). Badges are rule-derived from stored attributes.</>} />
87 +
88 + <CompareTable
89 + ents={ents}
90 + title="US registry figures (latest year available per metric)"
91 + id="registry"
92 + headNote={ents.map((e) => registryNote(e))}
93 + rows={EPI_ROWS.map(([metric, label]) => ({
94 + k: label,
95 + cells: ents.map((e) => {
96 + const f = e.figures.get(metric);
97 + if (!f) return e.registry ? <span className="text-[12.5px] text-ink-3">{e.registry.depth > 0 ? 'no registry observation at this level; none for the ancestor either' : 'no observation'}</span> : <span className="text-[12.5px] text-ink-3">no registry observation at this level (no top-level ancestor)</span>;
98 + const p = toInfo(prov.get(f.provenance_id), 'normalized') ?? { sourceSlug: f.source_slug, sourceName: f.source_name };
99 + return (
100 + <span className="flex flex-col gap-0.5">
101 + <span className="flex flex-wrap items-baseline gap-x-1.5">
102 + <span className="ci-num text-[16px] font-medium">{fmtValue(f.value, f.unit)}</span>
103 + <span className="text-[11px] text-ink-3">{unitLabel(f.unit)}</span>
104 + {f.estimate_type !== 'observed' ? <span className="text-[11px] italic text-warn">{f.estimate_type}</span> : null}
105 + </span>
106 + <span className="text-[11.5px] text-ink-3">
107 + {f.year_end && f.year_end !== f.year ? `${f.year}–${f.year_end}` : f.year} · both sexes · all ages{f.standard_population ? ` · ${f.standard_population.replace(/\s*\(.*\)$/, '')}` : ''}
108 + </span>
109 + <span className="flex flex-wrap items-center gap-1">
110 + <SourceBadge p={p} />
111 + <ClaimBadge kind="observed" />
112 + </span>
113 + {e.registry && e.registry.depth > 0 ? <span className="text-[11px] italic text-warn">figures for {e.registry.canonical_name}</span> : null}
114 + </span>
115 + );
116 + }),
117 + }))}
118 + caption={<>Observations as published (site definitions on each entity's Statistics tab). No global figures: IARC / GLOBOCAN is under license review and SEER awaits credentials. For entities below the top level the nearest top-level ancestor's registry figures are shown and labelled; they describe the whole site group, not the subtype.</>}
119 + />
120 +
121 + <CompareTable
122 + ents={ents}
123 + title="Counters (entity + descendants)"
124 + id="counters"
125 + rows={[
126 + ['Active trials', 'active_trial_count', 'clinicaltrials', '/trials'],
127 + ['Recruiting trials', 'recruiting_trial_count', 'clinicaltrials', '/trials?status=RECRUITING'],
128 + ['Active Phase III trials', 'phase3_trial_count', 'clinicaltrials', '/trials?phase=PHASE3'],
129 + ['Publications, last 5 years', 'publication_count_5y', 'pubmed', '/research'],
130 + ['Publications, last 12 months', 'publication_count_12m', 'pubmed', '/research'],
131 + ['Curated evidence items', 'evidence_count', 'civic', '/evidence'],
132 + ['Genes with evidence', 'gene_count', 'civic', '/genomics'],
133 + ['Genomic cohorts', 'cohort_count', 'gdc', '/genomics'],
134 + ['Drugs (any / approved)', 'drug_count', 'civic', '/drugs'],
135 + ].map(([label, field, src, path]) => ({
136 + k: label as string,
137 + cells: ents.map((e) => {
138 + const c = e.counters;
139 + if (!c) return <span className="text-[12.5px] text-ink-3">counters not computed</span>;
140 + const v = c[field as keyof typeof c] as number;
141 + return (
142 + <span className="flex flex-col gap-0.5">
143 + <Link href={`/cancer/${e.cancer.slug}${path}`} className="ci-num text-[16px] font-medium no-underline hover:text-accent">
144 + {fmtInt(v)}
145 + {field === 'drug_count' ? <span className="text-[13px] text-ink-3"> / {fmtInt(c.approved_drug_count)}</span> : null}
146 + </Link>
147 + <span className="flex flex-wrap items-center gap-1 text-[11px] text-ink-3">
148 + <SourceBadge p={{ sourceSlug: src as string, layer: 'derived', note: 'Counter computed by CancerIndex over the entity and its descendants.' }} compact />
149 + <ClaimBadge kind="computed" />
150 + </span>
151 + </span>
152 + );
153 + }),
154 + }))}
155 + caption={<>Counts aggregate over descendants across hierarchy types, so a broad family counts more than one of its subtypes; publication counts are query-based per entity. Counters refreshed {freshest ? <span title={freshest.toISOString()}>{freshest.toISOString().slice(0, 10)}</span> : 'unknown'}.</>}
156 + />
157 +
158 + <CompareTable
159 + ents={ents}
160 + title="Current ranks"
161 + id="ranks"
162 + rows={RANK_METRICS.filter((m) => ents.some((e) => e.ranks.some((r) => r.metric_slug === m))).map((m) => ({
163 + k: ents.flatMap((e) => e.ranks).find((r) => r.metric_slug === m)?.metric_name ?? humanize(m),
164 + cells: ents.map((e) => {
165 + const own = bestRankFor(
166 + e.ranks.filter((r) => r.cancer_id === e.cancer.id),
167 + m,
168 + { geo: 'USA', level: e.cancer.top_level ? 'top' : 'all' },
169 + );
170 + const viaAncestor = !own && e.registry && e.registry.depth > 0 ? bestRankFor(e.ranks.filter((r) => r.cancer_id === e.registry!.id), m, { geo: 'USA', level: 'top' }) : null;
171 + const r = own ?? viaAncestor;
172 + if (!r) return <span className="text-[12px] text-ink-3">not ranked</span>;
173 + return (
174 + <span className="flex flex-col gap-0.5 text-[12.5px]">
175 + <Link className="ci-link" href={`/rankings/${r.metric_slug}?scope=${encodeURIComponent(r.scope_key)}`}>
176 + <span className="ci-num font-medium">#{r.rank}</span> of {fmtInt(r.eligible_entities)}
177 + </Link>
178 + <span className="text-[11px] text-ink-3">
179 + {fmtValue(r.value, r.unit)} {unitLabel(r.unit)} · {scopeLabel(r.scope_key)}
180 + </span>
181 + <span className="flex flex-wrap gap-1">
182 + <ConfidenceBadge level={r.confidence} />
183 + {viaAncestor ? <span className="text-[11px] italic text-warn">rank of {e.registry!.canonical_name}</span> : null}
184 + </span>
185 + </span>
186 + );
187 + }),
188 + }))}
189 + caption={<>Ranks come from current snapshots (metric × scope × formula version); "of n" is the number of eligible entities in that scope. Top-level entities are ranked among the 36 mutually exclusive site groups, others among all malignant entities — the two are not comparable. <Link className="ci-link" href="/methodology#versioning">Versioning</Link>.</>}
190 + emptyText="No current ranking snapshot covers these entities."
191 + />
192 +
193 + <Section id="charts" kicker="Charts" title="Side by side" description="Bars are proportional within each chart; missing values are shown as a dash, never as zero.">
194 + <div className="grid gap-6 sm:grid-cols-2 lg:grid-cols-4">
195 + <CompareBars title="US annual deaths" unit="count" data={barsFor(ents, (e) => e.figures.get('mortality_count'))} caption={<>Registry observation · latest year · <ClaimBadge kind="observed" /></>} />
196 + <CompareBars title="US age-standardized mortality" unit="per_100k" data={barsFor(ents, (e) => e.figures.get('as_mortality_rate'))} caption="Per 100,000 · standard population declared by the source" />
197 + <CompareBars title="Active trials" unit="count" data={ents.map((e) => ({ label: e.cancer.canonical_name, value: e.counters ? e.counters.active_trial_count : null, note: 'ClinicalTrials.gov · entity + descendants', href: `/cancer/${e.cancer.slug}/trials` }))} caption={<>Counter · <ClaimBadge kind="computed" /></>} />
198 + <CompareBars title="Publications, last 5 years" unit="count" data={ents.map((e) => ({ label: e.cancer.canonical_name, value: e.counters ? e.counters.publication_count_5y : null, note: 'PubMed · stored query per entity', href: `/cancer/${e.cancer.slug}/research` }))} caption={<>Counter · <ClaimBadge kind="computed" /></>} />
199 + </div>
200 + </Section>
201 +
202 + <Note>Comparisons juxtapose facts of different natures (registry counts, registrations, literature volume). They are not a judgment of severity or priority, and population figures never describe an individual.</Note>
203 + <Freshness dataUpdatedAt={freshest ?? null} extra="each cell states its own period and source" />
204 + </div>
205 + )}
206 + </div>
207 + );
208 +}
209 +
210 +function barsFor(ents: CompareEntity[], pick: (e: CompareEntity) => { value: number; year: number; estimate_type: string } | undefined): CompareBarDatum[] {
211 + return ents.map((e) => {
212 + const f = pick(e);
213 + const anc = e.registry && e.registry.depth > 0 ? e.registry.canonical_name : null;
214 + return { label: e.cancer.canonical_name, value: f ? Number(f.value) : null, note: f ? `${f.year} · ${f.estimate_type}${anc ? ` · registry level: ${anc}` : ''}` : 'no registry observation', href: `/cancer/${e.cancer.slug}/statistics`, muted: !!anc };
215 + });
216 +}
217 +
218 +function CompareTable({ ents, title, id, rows, caption, headNote, emptyText }: { ents: CompareEntity[]; title: string; id: string; rows: Array<{ k: string; cells: ReactNode[] }>; caption?: ReactNode; headNote?: Array<string | null>; emptyText?: string }) {
219 + const nonEmpty = rows;
220 + return (
221 + <Section id={id} kicker="Facts" title={title}>
222 + {nonEmpty.length === 0 ? (
223 + <EmptyState compact>{emptyText ?? 'Nothing to compare in this block.'}</EmptyState>
224 + ) : (
225 + <div className="ci-table-wrap">
226 + <table className="ci-table" style={{ minWidth: `${180 + ents.length * 200}px` }}>
227 + <thead>
228 + <tr>
229 + <th className="sticky-col">Fact</th>
230 + {ents.map((e, i) => (
231 + <th key={e.cancer.id}>
232 + <Link className="ci-link normal-case tracking-normal" href={`/cancer/${e.cancer.slug}`} style={{ fontSize: 13 }}>
233 + {e.cancer.canonical_name}
234 + </Link>
235 + {headNote?.[i] ? <span className="block text-[10.5px] font-normal normal-case tracking-normal text-warn">{headNote[i]}</span> : null}
236 + </th>
237 + ))}
238 + </tr>
239 + </thead>
240 + <tbody>
241 + {nonEmpty.map((r) => (
242 + <tr key={r.k}>
243 + <th scope="row" className="sticky-col text-left text-[12.5px] font-medium text-ink-2" style={{ verticalAlign: 'top', padding: '7px 10px' }}>
244 + {r.k}
245 + </th>
246 + {r.cells.map((c, i) => (
247 + <td key={ents[i]?.cancer.id ?? i} className="text-[13px]">
248 + {c}
249 + </td>
250 + ))}
251 + </tr>
252 + ))}
253 + </tbody>
254 + </table>
255 + </div>
256 + )}
257 + {caption ? <p className="mt-2 flex flex-wrap items-center gap-1 text-[12px] text-ink-3">{caption}</p> : null}
258 + </Section>
259 + );
260 +}
added apps/web/src/app/compare/picker.tsx +137 −0
@@ -0,0 +1,137 @@
1 +'use client';
2 +
3 +import { useEffect, useRef, useState } from 'react';
4 +import { useRouter } from 'next/navigation';
5 +import { Search, X, Plus } from 'lucide-react';
6 +
7 +interface Hit {
8 + type: string;
9 + id: string;
10 + title: string;
11 + subtitle?: string | null;
12 + href: string;
13 + match: string;
14 +}
15 +
16 +/**
17 + * Compare picker: adds/removes cancers by slug; the selection lives in the URL (?ids=a,b,c) so a comparison
18 + * is shareable. Uses the existing /api/search route (database-backed) and keeps only cancer hits.
19 + */
20 +export function ComparePicker({ selected, max, min }: { selected: Array<{ slug: string; name: string }>; max: number; min: number }) {
21 + const router = useRouter();
22 + const [q, setQ] = useState('');
23 + const [hits, setHits] = useState<Hit[]>([]);
24 + const [loading, setLoading] = useState(false);
25 + const [open, setOpen] = useState(false);
26 + const box = useRef<HTMLDivElement>(null);
27 +
28 + useEffect(() => {
29 + const term = q.trim();
30 + if (term.length < 2) {
31 + setHits([]);
32 + return;
33 + }
34 + const ctrl = new AbortController();
35 + const t = setTimeout(async () => {
36 + setLoading(true);
37 + try {
38 + const r = await fetch(`/api/search?q=${encodeURIComponent(term)}&limit=20`, { signal: ctrl.signal });
39 + const j = (await r.json()) as { data: Hit[] };
40 + const chosen = new Set(selected.map((s) => s.slug));
41 + setHits((j.data ?? []).filter((h) => h.type === 'cancer' && !chosen.has(h.href.replace(/^\/cancer\//, ''))).slice(0, 8));
42 + setOpen(true);
43 + } catch {
44 + /* aborted */
45 + } finally {
46 + setLoading(false);
47 + }
48 + }, 150);
49 + return () => {
50 + clearTimeout(t);
51 + ctrl.abort();
52 + };
53 + }, [q, selected]);
54 +
55 + useEffect(() => {
56 + const onDoc = (e: MouseEvent) => {
57 + if (box.current && !box.current.contains(e.target as Node)) setOpen(false);
58 + };
59 + document.addEventListener('mousedown', onDoc);
60 + return () => document.removeEventListener('mousedown', onDoc);
61 + }, []);
62 +
63 + const navigate = (slugs: string[]) => {
64 + router.push(slugs.length ? `/compare?ids=${slugs.map(encodeURIComponent).join(',')}` : '/compare');
65 + };
66 + const add = (href: string) => {
67 + const slug = href.replace(/^\/cancer\//, '');
68 + if (!slug || selected.some((s) => s.slug === slug) || selected.length >= max) return;
69 + setQ('');
70 + setHits([]);
71 + setOpen(false);
72 + navigate([...selected.map((s) => s.slug), slug]);
73 + };
74 + const remove = (slug: string) => navigate(selected.filter((s) => s.slug !== slug).map((s) => s.slug));
75 + const full = selected.length >= max;
76 +
77 + return (
78 + <div className="border border-rule bg-paper-2 p-3">
79 + <div className="flex flex-wrap items-center gap-2">
80 + {selected.map((s) => (
81 + <span key={s.slug} className="inline-flex items-center gap-1 border border-rule-strong bg-paper px-2 py-1 text-[13px]">
82 + {s.name}
83 + <button type="button" onClick={() => remove(s.slug)} aria-label={`Remove ${s.name} from the comparison`} className="ml-0.5 text-ink-3 hover:text-danger">
84 + <X className="h-3.5 w-3.5" aria-hidden />
85 + </button>
86 + </span>
87 + ))}
88 + {selected.length === 0 ? <span className="text-[13px] text-ink-3">No cancer selected yet.</span> : null}
89 + </div>
90 + <div ref={box} className="relative mt-2">
91 + <label className="sr-only" htmlFor="compare-picker-input">
92 + Add a cancer to the comparison
93 + </label>
94 + <div className={`flex items-center gap-2 border bg-paper px-2.5 ${full ? 'border-rule text-ink-4' : 'border-rule-strong focus-within:border-accent'}`}>
95 + {full ? <Plus className="h-4 w-4 shrink-0" aria-hidden /> : <Search className="h-4 w-4 shrink-0 text-ink-3" aria-hidden />}
96 + <input
97 + id="compare-picker-input"
98 + value={q}
99 + onChange={(e) => setQ(e.target.value)}
100 + onFocus={() => hits.length && setOpen(true)}
101 + onKeyDown={(e) => {
102 + if (e.key === 'Enter' && hits[0]) {
103 + e.preventDefault();
104 + add(hits[0].href);
105 + } else if (e.key === 'Escape') setOpen(false);
106 + }}
107 + disabled={full}
108 + placeholder={full ? `Maximum ${max} cancers — remove one to add another` : `Add a cancer (${selected.length}/${max}) — e.g. glioblastoma, pancreatic, KRAS…`}
109 + className="w-full bg-transparent py-2 text-[14px] outline-none placeholder:text-ink-4 disabled:cursor-not-allowed"
110 + autoComplete="off"
111 + spellCheck={false}
112 + role="combobox"
113 + aria-expanded={open && hits.length > 0}
114 + aria-controls="compare-picker-results"
115 + />
116 + </div>
117 + {open && q.trim().length >= 2 ? (
118 + <ul id="compare-picker-results" role="listbox" className="absolute left-0 right-0 z-20 mt-1 max-h-72 overflow-y-auto border border-rule-strong bg-paper shadow-lg">
119 + {hits.map((h) => (
120 + <li key={h.id} role="option" aria-selected={false}>
121 + <button type="button" onClick={() => add(h.href)} className="flex w-full items-baseline justify-between gap-2 px-3 py-2 text-left text-[13.5px] hover:bg-accent-soft">
122 + <span className="min-w-0 truncate">{h.title}</span>
123 + {h.subtitle ? <span className="shrink-0 text-[11.5px] text-ink-3">{h.subtitle}</span> : null}
124 + </button>
125 + </li>
126 + ))}
127 + {!loading && hits.length === 0 ? <li className="px-3 py-2 text-[13px] text-ink-3">No matching cancer entity.</li> : null}
128 + {loading ? <li className="px-3 py-2 text-[13px] text-ink-3">Searching…</li> : null}
129 + </ul>
130 + ) : null}
131 + </div>
132 + <p className="mt-1.5 text-[11.5px] text-ink-3">
133 + Compare {min} to {max} cancers. The selection is in the URL — copy it to share this comparison.
134 + </p>
135 + </div>
136 + );
137 +}
added apps/web/src/app/countries/page.tsx +163 −0
@@ -0,0 +1,163 @@
1 +import type { Metadata } from 'next';
2 +import Link from 'next/link';
3 +import { PageHeader, Section, Note } from '@/components/ui/section';
4 +import { EmptyState } from '@/components/ui/empty-state';
5 +import { Freshness } from '@/components/ui/freshness';
6 +import { Badge } from '@/components/ui/badge';
7 +import { SourceBadge } from '@/components/ui/source-badge';
8 +import { listGeographiesWithObservations, topCancersFor, allSitesObservations, WHO_REGION_LABEL } from '@/lib/queries/geography';
9 +import { listSources } from '@/lib/queries/sources';
10 +import { fmtInt, fmtValue, humanize, toDate, unitLabel } from '@/lib/format';
11 +
12 +export const metadata: Metadata = {
13 + title: 'Countries',
14 + description: 'Geographies with cancer epidemiology observations in CancerIndex: years covered, sources, and the latest-year figures per country.',
15 + alternates: { canonical: '/countries' },
16 +};
17 +export const revalidate = 3600;
18 +
19 +/**
20 + * /countries (§47): every geography that carries at least one epidemiology observation, with its year span,
21 + * the sources behind it and either the all-sites totals (when the source publishes one) or the top cancer by
22 + * deaths / new cases for the latest year. Geographies without data are not listed — nothing is invented.
23 + */
24 +export default async function CountriesPage() {
25 + const [geos, sources] = await Promise.all([listGeographiesWithObservations(), listSources()]);
26 + const epiSources = sources.filter((s) => ['cdc-uscs', 'cdc-wonder', 'seer', 'seer-explorer', 'iarc-globocan'].includes(s.slug));
27 +
28 + const details = await Promise.all(
29 + geos.map(async (g) => {
30 + const [allSites, deaths, cases] = await Promise.all([allSitesObservations(g.id, g.max_year, 'all'), topCancersFor(g, 'mortality_count', g.max_year, 'all', 1), topCancersFor(g, 'incidence_count', g.max_year, 'all', 1)]);
31 + return { g, allSites, deaths, cases };
32 + }),
33 + );
34 + const freshest = geos.map((g) => toDate(g.last_updated)).filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? null;
35 +
36 + return (
37 + <div>
38 + <PageHeader kicker="Geographies" title="Countries with epidemiology data" lede="Population statistics are attached per geography, year and sex, exactly as published by a registry or statistical agency. This page lists only geographies for which at least one observation has been ingested.">
39 + <p className="mt-3 flex flex-wrap items-center gap-1.5 text-[12.5px] text-ink-3">
40 + Epidemiology sources:
41 + {epiSources.map((s) => (
42 + <span key={s.slug} className="inline-flex items-center gap-1">
43 + <SourceBadge p={{ sourceSlug: s.slug, sourceName: s.name, license: s.license, layer: 'raw' }} compact />
44 + <Badge tone={s.status === 'active' ? 'ok' : s.status === 'review' || s.status === 'awaiting_credentials' ? 'warn' : 'neutral'}>{humanize(s.status)}</Badge>
45 + </span>
46 + ))}
47 + </p>
48 + </PageHeader>
49 +
50 + <Section id="list" kicker="Coverage" title={`${fmtInt(geos.length)} ${geos.length === 1 ? 'geography' : 'geographies'} with observations`} description="Years covered, distinct cancer entities with at least one observation, and the latest-year figures. Totals across all cancer sites are shown only when the source itself publishes an all-sites row; sites are never summed by CancerIndex.">
51 + {geos.length === 0 ? (
52 + <EmptyState title="No geography has observations yet">
53 + Country pages appear as soon as a licensed registry connector ingests observations. IARC / GLOBOCAN stays under license review and SEER awaits credentials, so global coverage is not available yet.
54 + </EmptyState>
55 + ) : (
56 + <>
57 + <div className="ci-table-wrap">
58 + <table className="ci-table">
59 + <thead>
60 + <tr>
61 + <th>Geography</th>
62 + <th>ISO3</th>
63 + <th>WHO region</th>
64 + <th className="num">Years</th>
65 + <th className="num">Cancers</th>
66 + <th className="num">Observations</th>
67 + <th>Sources</th>
68 + <th>Latest year</th>
69 + <th>Highest annual deaths</th>
70 + <th>Highest annual new cases</th>
71 + </tr>
72 + </thead>
73 + <tbody>
74 + {details.map(({ g, allSites, deaths, cases }) => {
75 + const total = (metric: string) => allSites.find((a) => a.metric === metric);
76 + const td = total('mortality_count');
77 + const tc = total('incidence_count');
78 + const d0 = deaths.rows[0];
79 + const c0 = cases.rows[0];
80 + return (
81 + <tr key={g.id}>
82 + <td>
83 + <Link className="ci-link font-medium" href={`/country/${g.slug}`}>
84 + {g.name}
85 + </Link>
86 + <span className="ml-1.5 text-[11.5px] text-ink-3">{humanize(g.kind)}</span>
87 + </td>
88 + <td className="ci-mono">{g.iso3 ?? '—'}</td>
89 + <td className="text-[12.5px] text-ink-2">{g.who_region ? WHO_REGION_LABEL[g.who_region] ?? g.who_region : '—'}</td>
90 + <td className="num">
91 + {g.min_year}–{g.max_year}
92 + </td>
93 + <td className="num">{fmtInt(g.n_cancers)}</td>
94 + <td className="num">{fmtInt(g.n_obs)}</td>
95 + <td>
96 + <span className="inline-flex flex-wrap gap-1">
97 + {(g.sources ?? []).map((s) => (
98 + <SourceBadge key={s.slug} p={{ sourceSlug: s.slug, sourceName: s.name }} compact />
99 + ))}
100 + </span>
101 + </td>
102 + <td className="ci-num">{g.max_year}</td>
103 + <td className="text-[12.5px]">
104 + {td ? (
105 + <span>
106 + All sites: <span className="ci-num font-medium">{fmtValue(td.value, td.unit)}</span> {unitLabel(td.unit)} ({td.year})
107 + </span>
108 + ) : d0 ? (
109 + <span>
110 + <Link className="ci-link" href={`/cancer/${d0.slug}`}>
111 + {d0.canonical_name}
112 + </Link>{' '}
113 + <span className="ci-num font-medium">{fmtValue(d0.value, d0.unit)}</span> <span className="text-ink-3">({deaths.year})</span>
114 + </span>
115 + ) : (
116 + <span className="text-ink-4">—</span>
117 + )}
118 + </td>
119 + <td className="text-[12.5px]">
120 + {tc ? (
121 + <span>
122 + All sites: <span className="ci-num font-medium">{fmtValue(tc.value, tc.unit)}</span> {unitLabel(tc.unit)} ({tc.year})
123 + </span>
124 + ) : c0 ? (
125 + <span>
126 + <Link className="ci-link" href={`/cancer/${c0.slug}`}>
127 + {c0.canonical_name}
128 + </Link>{' '}
129 + <span className="ci-num font-medium">{fmtValue(c0.value, c0.unit)}</span> <span className="text-ink-3">({cases.year})</span>
130 + </span>
131 + ) : (
132 + <span className="text-ink-4">—</span>
133 + )}
134 + </td>
135 + </tr>
136 + );
137 + })}
138 + </tbody>
139 + </table>
140 + </div>
141 + <p className="mt-2 text-[12px] text-ink-3">"Highest annual deaths / new cases" is the top single site group for the latest year of that metric, both sexes, all ages — not a national total. Where a source publishes an all-sites aggregate it is shown instead and labelled "All sites".</p>
142 + <Freshness dataUpdatedAt={freshest} extra={`${fmtInt(geos.reduce((n, g) => n + g.n_obs, 0))} observations in total`} />
143 + </>
144 + )}
145 + </Section>
146 +
147 + <Section id="status" kicker="Coverage status" title="Why only these geographies?" level={3}>
148 + <Note>
149 + CancerIndex publishes population statistics only from sources whose license has been reviewed (CLAUDE.md §10.4). U.S. Cancer Statistics (CDC, NPCR/SEER incidence and NVSS mortality) is approved and ingested. The IARC Global Cancer Observatory (GLOBOCAN), which would provide estimates for 185 countries, is under license review; the SEER API awaits credentials. Until then no country outside the United States has observations, and no global ranking of countries is shown. Estimates are never substituted for registry counts.
150 + </Note>
151 + <p className="mt-2 text-[13px]">
152 + <Link className="ci-link" href="/sources">
153 + Source registry and license status →
154 + </Link>{' '}
155 + ·{' '}
156 + <Link className="ci-link" href="/methodology#country-scopes">
157 + Methodology: country scopes →
158 + </Link>
159 + </p>
160 + </Section>
161 + </div>
162 + );
163 +}
added apps/web/src/app/country/[slug]/page.tsx +405 −0
@@ -0,0 +1,405 @@
1 +import type { Metadata } from 'next';
2 +import Link from 'next/link';
3 +import { notFound } from 'next/navigation';
4 +import { PageHeader, Section, KV, Note } from '@/components/ui/section';
5 +import { EmptyState } from '@/components/ui/empty-state';
6 +import { Freshness } from '@/components/ui/freshness';
7 +import { Badge, ClaimBadge, ConfidenceBadge } from '@/components/ui/badge';
8 +import { SourceBadge } from '@/components/ui/source-badge';
9 +import { Breadcrumbs } from '@/components/layout/breadcrumbs';
10 +import { TrendChart, type TrendSeries } from '@/components/charts/trend-chart';
11 +import { getGeographyBySlug, coverageFor, yearsFor, allSitesObservations, topCancersFor, trendFor, geographyScopeCode, WHO_REGION_LABEL, SEXES, BURDEN_METRICS, type Sex, type TopCancersResult } from '@/lib/queries/geography';
12 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
13 +import { loadProvenance, toInfo } from '@/lib/queries/provenance';
14 +import { listSources } from '@/lib/queries/sources';
15 +import { jsonLd } from '@/lib/seo';
16 +import { SITE_URL, SITE_NAME } from '@/lib/site';
17 +import { str, int, oneOf, type SP } from '@/lib/search-params';
18 +import { fmtInt, fmtValue, humanize, toDate, unitLabel } from '@/lib/format';
19 +
20 +export const revalidate = 3600;
21 +
22 +type Params = { slug: string };
23 +
24 +export async function generateMetadata({ params }: { params: Promise<Params> }): Promise<Metadata> {
25 + const geo = await getGeographyBySlug((await params).slug);
26 + if (!geo) return { title: 'Not found' };
27 + return {
28 + title: `${geo.name} — cancer statistics`,
29 + description: `Cancer incidence and mortality observations for ${geo.name}: top cancers by annual deaths, new cases and age-standardized rates per year and sex, with trends and sources.`,
30 + alternates: { canonical: `/country/${geo.slug}` },
31 + };
32 +}
33 +
34 +/**
35 + * /country/[slug] (§47): header, latest-year summary, top-cancer tables per metric with sex/year selectors,
36 + * a multi-line trend of the top 8 cancers' age-standardized mortality, and a sources/freshness block.
37 + * Every value comes from epidemiology_observations; ranks from the matching ranking snapshot when it exists.
38 + */
39 +export default async function CountryPage({ params, searchParams }: { params: Promise<Params>; searchParams: Promise<SP> }) {
40 + const { slug } = await params;
41 + const sp = await searchParams;
42 + const geo = await getGeographyBySlug(slug);
43 + if (!geo) notFound();
44 + const [coverage, years, sources] = await Promise.all([coverageFor(geo.id), yearsFor(geo.id), listSources()]);
45 + const latestYear = years[0];
46 + const sex = oneOf<Sex>(sp, 'sex', SEXES, 'all');
47 + const year = latestYear ? int(sp, 'year', latestYear, years[years.length - 1]!, latestYear) : null;
48 + const scopeCode = geographyScopeCode(geo);
49 +
50 + const crumbs = [{ label: 'Countries', href: '/countries' }, ...(geo.parent_slug && geo.parent_name && geo.kind === 'subdivision' ? [{ label: geo.parent_name, href: `/country/${geo.parent_slug}` }] : []), { label: geo.name }];
51 +
52 + if (!latestYear || year == null) {
53 + return (
54 + <div>
55 + <Breadcrumbs items={crumbs} className="pt-5" />
56 + <PageHeader kicker={humanize(geo.kind)} title={geo.name} lede={headerLede(geo)} />
57 + <EmptyState title="No epidemiology observation for this geography yet">
58 + Country statistics appear once a licensed registry connector has ingested observations for {geo.name}. IARC / GLOBOCAN (185 countries) is under license review and the SEER API awaits credentials.
59 + <div className="mt-1">
60 + <Link className="ci-link" href="/countries">
61 + Geographies with data
62 + </Link>
63 + </div>
64 + </EmptyState>
65 + </div>
66 + );
67 + }
68 +
69 + const [allSites, ...tops] = await Promise.all([allSitesObservations(geo.id, year, sex), ...BURDEN_METRICS.map((m) => topCancersFor(geo, m, year, sex, 40))]);
70 + const byMetric = new Map(tops.map((t) => [t.metric, t]));
71 + const asmr = byMetric.get('as_mortality_rate');
72 + const trendIds = (asmr?.rows.length ? asmr : byMetric.get('mortality_count'))?.rows.slice(0, 8).map((r) => r.cancer_id) ?? [];
73 + const trendMetric = asmr?.rows.length ? 'as_mortality_rate' : 'mortality_count';
74 + const trend = await trendFor(geo.id, trendMetric, sex, trendIds);
75 + const provIds = tops.flatMap((t) => t.rows.slice(0, 1).map((r) => r.provenance_id));
76 + const prov = await loadProvenance([...provIds, ...allSites.map((a) => a.provenance_id)]);
77 +
78 + const usedSources = [...new Set(coverage.map((c) => c.source_slug))].map((s) => sources.find((x) => x.slug === s)).filter((s): s is NonNullable<typeof s> => !!s);
79 + const freshest = coverage.map((c) => toDate(c.last_updated)).filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? null;
80 + const nCancers = Math.max(...coverage.map((c) => c.n_cancers), 0);
81 + const estimateTypes = [...new Set(coverage.flatMap((c) => c.estimate_types))];
82 + const standardPop = coverage.find((c) => c.standard_population)?.standard_population ?? null;
83 +
84 + const ld = {
85 + '@context': 'https://schema.org',
86 + '@type': 'Dataset',
87 + name: `Cancer incidence and mortality observations — ${geo.name}`,
88 + description: `Per-site cancer observations for ${geo.name} (${years[years.length - 1]}–${latestYear}): annual deaths, new cases and age-standardized rates by sex, as published by ${usedSources.map((s) => s.name).join('; ') || 'the source registry'}. Normalized by CancerIndex without changing values.`,
89 + url: `${SITE_URL}/country/${geo.slug}`,
90 + creator: { '@type': 'Organization', name: SITE_NAME, url: SITE_URL },
91 + isBasedOn: usedSources.map((s) => s.homepage ?? `${SITE_URL}/source/${s.slug}`),
92 + spatialCoverage: { '@type': 'Place', name: geo.name, ...(geo.iso3 ? { identifier: geo.iso3 } : {}) },
93 + temporalCoverage: `${years[years.length - 1]}/${latestYear}`,
94 + variableMeasured: [...new Set(coverage.map((c) => EPI_METRIC_LABEL[c.metric] ?? c.metric))],
95 + license: usedSources.map((s) => s.license).filter(Boolean).join('; ') || undefined,
96 + ...(freshest ? { dateModified: freshest.toISOString() } : {}),
97 + };
98 +
99 + const q = (over: Partial<{ sex: string; year: number }>) => {
100 + const p = new URLSearchParams();
101 + const s = over.sex ?? sex;
102 + const y = over.year ?? year;
103 + if (s !== 'all') p.set('sex', s);
104 + if (y !== latestYear) p.set('year', String(y));
105 + const qs = p.toString();
106 + return `/country/${geo.slug}${qs ? `?${qs}` : ''}`;
107 + };
108 +
109 + return (
110 + <article>
111 + <script type="application/ld+json" dangerouslySetInnerHTML={{ __html: jsonLd(ld) }} />
112 + <Breadcrumbs items={crumbs} className="pt-5" />
113 + <PageHeader kicker={humanize(geo.kind)} title={geo.name} lede={headerLede(geo)}>
114 + <KV
115 + className="mt-3 max-w-xl"
116 + items={[
117 + { k: 'ISO 3166-1', v: geo.iso3 ? <span className="ci-mono">{geo.iso3}{geo.iso2 ? ` · ${geo.iso2}` : ''}</span> : null },
118 + { k: 'WHO region', v: geo.who_region ? WHO_REGION_LABEL[geo.who_region] ?? geo.who_region : null },
119 + { k: 'Population', v: geo.population ? <span className="ci-num">{fmtInt(geo.population)}{geo.population_year ? ` (${geo.population_year})` : ''}</span> : <span className="text-ink-3">not seeded — CancerIndex shows only rates and counts published by the source</span> },
120 + { k: 'Years covered', v: <span className="ci-num">{years[years.length - 1]}–{latestYear}</span> },
121 + { k: 'Cancer entities', v: <span className="ci-num">{fmtInt(nCancers)}</span> },
122 + { k: 'Estimate types', v: estimateTypes.map((t) => <Badge key={t} tone={t === 'observed' ? 'ok' : 'warn'} className="mr-1">{t}</Badge>) },
123 + ]}
124 + />
125 + </PageHeader>
126 +
127 + {/* Selectors: sex + year (server-side URL state) */}
128 + <nav aria-label="Scope" className="flex flex-wrap items-center gap-x-6 gap-y-2 border-y border-rule py-2 text-[13px]">
129 + <div className="flex items-center gap-1.5">
130 + <span className="ci-kicker mr-1">Sex</span>
131 + {SEXES.map((s) => (
132 + <Link key={s} href={q({ sex: s })} aria-current={s === sex ? 'page' : undefined} className={`border px-2 py-0.5 no-underline ${s === sex ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
133 + {s === 'all' ? 'Both sexes' : humanize(s)}
134 + </Link>
135 + ))}
136 + </div>
137 + <div className="flex flex-wrap items-center gap-1.5">
138 + <span className="ci-kicker mr-1">Year</span>
139 + {years.slice(0, 8).map((y) => (
140 + <Link key={y} href={q({ year: y })} aria-current={y === year ? 'page' : undefined} className={`ci-num border px-2 py-0.5 no-underline ${y === year ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
141 + {y}
142 + </Link>
143 + ))}
144 + {years.length > 8 ? (
145 + <details className="relative">
146 + <summary className="cursor-pointer border border-rule px-2 py-0.5 text-ink-2 hover:border-accent">earlier…</summary>
147 + <div className="absolute left-0 z-10 mt-1 flex max-w-[320px] flex-wrap gap-1 border border-rule-strong bg-paper p-2 shadow-lg">
148 + {years.slice(8).map((y) => (
149 + <Link key={y} href={q({ year: y })} className={`ci-num border px-2 py-0.5 no-underline ${y === year ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
150 + {y}
151 + </Link>
152 + ))}
153 + </div>
154 + </details>
155 + ) : null}
156 + </div>
157 + </nav>
158 +
159 + {/* Summary cards */}
160 + <Section id="summary" kicker="Summary" title={`${year} · ${sex === 'all' ? 'both sexes' : humanize(sex)} · all ages`} description="Totals across all cancer sites are shown only when the source publishes an all-sites row. Otherwise the per-site figures below stand on their own; CancerIndex never sums site groups (definitions overlap between sources).">
161 + {allSites.length ? (
162 + <ul className="grid grid-cols-1 gap-px bg-rule sm:grid-cols-2 lg:grid-cols-4">
163 + {allSites.map((a) => (
164 + <li key={a.metric} className="bg-paper px-3 py-2.5">
165 + <span className="text-[11.5px] font-medium uppercase tracking-wide text-ink-3">{EPI_METRIC_LABEL[a.metric] ?? a.metric} · all sites</span>
166 + <span className="mt-0.5 flex items-baseline gap-1.5">
167 + <span className="ci-num font-display text-2xl">{fmtValue(a.value, a.unit)}</span>
168 + <span className="text-[11.5px] text-ink-3">{unitLabel(a.unit)}</span>
169 + </span>
170 + <span className="block text-[11.5px] text-ink-3">
171 + {a.year} · {a.site_definition}
172 + </span>
173 + <span className="mt-1 flex items-center gap-1.5">
174 + <SourceBadge p={toInfo(prov.get(a.provenance_id)) ?? { sourceSlug: a.source_slug, sourceName: a.source_name }} />
175 + <ClaimBadge kind="observed" />
176 + {a.estimate_type !== 'observed' ? <Badge tone="warn">{a.estimate_type}</Badge> : null}
177 + </span>
178 + </li>
179 + ))}
180 + </ul>
181 + ) : (
182 + <ul className="grid grid-cols-1 gap-px bg-rule sm:grid-cols-2 lg:grid-cols-4">
183 + {BURDEN_METRICS.map((m) => {
184 + const t = byMetric.get(m);
185 + const top = t?.rows[0];
186 + return (
187 + <li key={m} className="bg-paper px-3 py-2.5">
188 + <span className="text-[11.5px] font-medium uppercase tracking-wide text-ink-3">{EPI_METRIC_LABEL[m] ?? m}</span>
189 + {top && t ? (
190 + <>
191 + <span className="mt-0.5 block text-[12px] text-ink-3">
192 + highest site group{t.year !== year ? ` (${t.year}, latest available)` : ''}:
193 + </span>
194 + <Link href={`/cancer/${top.slug}`} className="ci-link block truncate text-[14px]">
195 + {top.canonical_name}
196 + </Link>
197 + <span className="flex items-baseline gap-1.5">
198 + <span className="ci-num font-display text-2xl">{fmtValue(top.value, top.unit)}</span>
199 + <span className="text-[11.5px] text-ink-3">{unitLabel(top.unit)}</span>
200 + </span>
201 + <span className="mt-1 flex flex-wrap items-center gap-1.5">
202 + <SourceBadge p={toInfo(prov.get(top.provenance_id)) ?? { sourceSlug: top.source_slug, sourceName: top.source_name }} />
203 + <ClaimBadge kind="observed" />
204 + <span className="text-[11px] text-ink-3">{fmtInt(t.rows.length)} site groups</span>
205 + </span>
206 + </>
207 + ) : (
208 + <span className="mt-1 block text-[12.5px] text-ink-3">no observation for {year}</span>
209 + )}
210 + </li>
211 + );
212 + })}
213 + </ul>
214 + )}
215 + {!allSites.length ? <p className="mt-2 text-[12px] text-ink-3">No all-sites total in source for this scope — per-site figures below; no national total is computed.</p> : null}
216 + </Section>
217 +
218 + {/* Top tables */}
219 + {tops.map((t) => (
220 + <TopTable key={t.metric} t={t} year={year} sex={sex} scopeCode={scopeCode} prov={prov} geoSlug={geo.slug} />
221 + ))}
222 +
223 + {/* Trend */}
224 + <Section id="trend" kicker="Trend" title={`${EPI_METRIC_LABEL[trendMetric]} — top ${Math.min(8, trendIds.length)} cancers, all years`} description={`Top cancers of ${year} (${sex === 'all' ? 'both sexes' : humanize(sex)}) traced over every year published for ${geo.name}. Dashed lines mark estimated or projected values; the y axis starts at zero.`}>
225 + {trend.length ? (
226 + <>
227 + <TrendChart series={toSeries(trend)} unit={trend[0]!.unit} ariaLabel={`${EPI_METRIC_LABEL[trendMetric]} by year for the top ${trendIds.length} cancers in ${geo.name}`} yLabel={unitLabel(trend[0]!.unit)} />
228 + <p className="mt-1.5 flex flex-wrap items-center gap-1.5 text-[11.5px] text-ink-3">
229 + <SourceBadge p={{ sourceSlug: trend[0]!.source_slug }} compact />
230 + <ClaimBadge kind="observed" />
231 + {trend[0]!.standard_population ? <span>standard: {trend[0]!.standard_population}</span> : null}
232 + <span>{fmtInt(trend.length)} points</span>
233 + </p>
234 + </>
235 + ) : (
236 + <EmptyState compact>No multi-year series for this scope.</EmptyState>
237 + )}
238 + </Section>
239 +
240 + {/* Sources & freshness */}
241 + <Section id="sources" kicker="Provenance" title="Sources and freshness" description="What each source covers for this geography, exactly as ingested. Values are normalized (units, labels) but never changed.">
242 + <div className="ci-table-wrap">
243 + <table className="ci-table">
244 + <thead>
245 + <tr>
246 + <th>Source</th>
247 + <th>Metric</th>
248 + <th>Sex</th>
249 + <th className="num">Years</th>
250 + <th className="num">Cancers</th>
251 + <th>Estimate types</th>
252 + <th>Standard population</th>
253 + <th>Ingested</th>
254 + </tr>
255 + </thead>
256 + <tbody>
257 + {coverage.map((c) => (
258 + <tr key={`${c.source_slug}-${c.metric}-${c.sex}`}>
259 + <td>
260 + <SourceBadge p={{ sourceSlug: c.source_slug, sourceName: c.source_name }} compact />
261 + </td>
262 + <td>{EPI_METRIC_LABEL[c.metric] ?? c.metric}</td>
263 + <td>{c.sex === 'all' ? 'Both' : humanize(c.sex)}</td>
264 + <td className="num">
265 + {c.min_year}–{c.max_year} <span className="text-ink-3">({c.n_years})</span>
266 + </td>
267 + <td className="num">{fmtInt(c.n_cancers)}</td>
268 + <td>
269 + {c.estimate_types.map((t) => (
270 + <Badge key={t} tone={t === 'observed' ? 'ok' : 'warn'} className="mr-1">
271 + {t}
272 + </Badge>
273 + ))}
274 + </td>
275 + <td className="text-[12px] text-ink-3">{c.standard_population ?? '—'}</td>
276 + <td className="text-[12px] text-ink-3">{toDate(c.last_updated)?.toISOString().slice(0, 10) ?? '—'}</td>
277 + </tr>
278 + ))}
279 + </tbody>
280 + </table>
281 + </div>
282 + <ul className="mt-3 space-y-1.5 text-[13px]">
283 + {usedSources.map((s) => (
284 + <li key={s.slug} className="flex flex-wrap items-center gap-2">
285 + <Link className="ci-link" href={`/source/${s.slug}`}>
286 + {s.name}
287 + </Link>
288 + <Badge tone={s.license_status === 'approved' ? 'ok' : 'warn'}>license: {s.license_status}</Badge>
289 + {s.license ? <span className="text-[12px] text-ink-3">{s.license}</span> : null}
290 + </li>
291 + ))}
292 + </ul>
293 + <Note>
294 + {standardPop ? `Age-standardized rates use the ${standardPop}; they are not comparable with rates standardized to the World (Segi) population used by IARC. ` : ''}
295 + Global comparisons are not available: the IARC Global Cancer Observatory remains under license review and the SEER API awaits credentials (CLAUDE.md §10.3-10.4). Only geographies with licensed observations get a page; see <Link className="ci-link" href="/countries">/countries</Link>.
296 + </Note>
297 + <Freshness dataUpdatedAt={freshest} sourceVersion={coverage.length ? `${years[years.length - 1]}–${latestYear}` : null} extra={`${fmtInt(coverage.reduce((n, c) => n + c.n_years * c.n_cancers, 0))} observations approx. · ranks from snapshots geo=${scopeCode}`} />
298 + </Section>
299 + </article>
300 + );
301 +}
302 +
303 +function headerLede(geo: { name: string; kind: string }) {
304 + return `Cancer incidence and mortality for ${geo.name} as published by the source registry: per site group, per year and sex, with age-standardized rates where the source provides them. Population statistics describe groups, never individuals.`;
305 +}
306 +
307 +function toSeries(points: Awaited<ReturnType<typeof trendFor>>): TrendSeries[] {
308 + const map = new Map<string, TrendSeries>();
309 + for (const p of points) {
310 + if (!map.has(p.cancer_id)) map.set(p.cancer_id, { key: p.cancer_id, name: p.canonical_name, href: `/cancer/${p.slug}/statistics`, points: [], dashed: false });
311 + const s = map.get(p.cancer_id)!;
312 + s.points.push({ x: Number(p.year), y: Number(p.value) });
313 + if (p.estimate_type !== 'observed') s.dashed = true;
314 + }
315 + // Order series by latest value descending so the legend matches the table.
316 + return [...map.values()].sort((a, b) => (b.points.at(-1)?.y ?? 0) - (a.points.at(-1)?.y ?? 0));
317 +}
318 +
319 +function TopTable({ t, year, sex, scopeCode, prov, geoSlug }: { t: TopCancersResult; year: number; sex: Sex; scopeCode: string; prov: Awaited<ReturnType<typeof loadProvenance>>; geoSlug: string }) {
320 + const label = EPI_METRIC_LABEL[t.metric] ?? t.metric;
321 + const first = t.rows[0];
322 + const ranked = t.rows.some((r) => r.rank != null);
323 + const scopeKey = t.year != null ? `geo=${scopeCode}|sex=${sex}|age=all|year=${t.year}|level=top` : null;
324 + return (
325 + <Section id={t.metric} kicker="Top cancers" title={`By ${label.toLowerCase()}`} description={t.year == null ? undefined : `${t.year}${t.year !== year ? ` — latest year available for this metric (${year} not yet published by the source)` : ''} · ${sex === 'all' ? 'both sexes' : humanize(sex)} · all ages · ${fmtInt(t.rows.length)} site groups${first?.standard_population ? ` · ${first.standard_population}` : ''}`}>
326 + {!first ? (
327 + <EmptyState compact>No {label.toLowerCase()} observation for this geography and sex.</EmptyState>
328 + ) : (
329 + <>
330 + <div className="ci-table-wrap">
331 + <table className="ci-table">
332 + <thead>
333 + <tr>
334 + <th className="num">#</th>
335 + <th>Cancer</th>
336 + <th className="num">
337 + {label} ({unitLabel(first.unit)})
338 + </th>
339 + <th className="num">95% CI</th>
340 + <th>Type</th>
341 + <th>Rank in scope</th>
342 + <th>Site definition</th>
343 + <th>Source</th>
344 + </tr>
345 + </thead>
346 + <tbody>
347 + {t.rows.map((r, i) => (
348 + <tr key={r.cancer_id}>
349 + <td className="num">{i + 1}</td>
350 + <td>
351 + <Link className="ci-link" href={`/cancer/${r.slug}`}>
352 + {r.canonical_name}
353 + </Link>
354 + <span className="ml-1.5 text-[11px] text-ink-3">{humanize(r.entity_type)}</span>
355 + </td>
356 + <td className="num font-medium">{fmtValue(r.value, r.unit)}</td>
357 + <td className="num text-ink-3">{r.lower_ci != null && r.upper_ci != null ? `${fmtValue(r.lower_ci, r.unit)}–${fmtValue(r.upper_ci, r.unit)}` : '—'}</td>
358 + <td>
359 + <Badge tone={r.estimate_type === 'observed' ? 'ok' : 'warn'}>{r.estimate_type}</Badge>
360 + </td>
361 + <td className="text-[12.5px]">
362 + {r.rank != null && r.rank_scope_key ? (
363 + <Link className="ci-link" href={`/rankings/${t.metric}?scope=${encodeURIComponent(r.rank_scope_key)}`} title={r.rank_scope_key}>
364 + #{r.rank} of {fmtInt(r.eligible_entities)}
365 + </Link>
366 + ) : (
367 + <span className="text-ink-4" title="No current ranking snapshot for this metric and scope">no snapshot</span>
368 + )}
369 + </td>
370 + <td className="max-w-[260px] text-[11.5px] text-ink-3">{r.site_definition ?? '—'}</td>
371 + <td>
372 + <span className="inline-flex gap-1">
373 + <SourceBadge p={toInfo(prov.get(r.provenance_id)) ?? { sourceSlug: r.source_slug, sourceName: r.source_name }} compact />
374 + <ClaimBadge kind="observed" />
375 + </span>
376 + </td>
377 + </tr>
378 + ))}
379 + </tbody>
380 + </table>
381 + </div>
382 + {/* div, not p: the SourceBadge popover holds a <dl>, which would close a <p> in the HTML parser (hydration mismatch). */}
383 + <div className="mt-2 flex flex-wrap items-center gap-1.5 text-[12px] text-ink-3">
384 + <SourceBadge p={toInfo(prov.get(first.provenance_id)) ?? { sourceSlug: first.source_slug, sourceName: first.source_name }} />
385 + {ranked && scopeKey ? (
386 + <span>
387 + Rank from snapshot <span className="ci-mono">{scopeKey}</span> ·{' '}
388 + <Link className="ci-link" href={`/rankings/${t.metric}?scope=${encodeURIComponent(scopeKey)}`}>
389 + full ranking and "Why this rank?"
390 + </Link>
391 + </span>
392 + ) : (
393 + <span>"#" is the position in this table; no ranking snapshot covers this scope yet.</span>
394 + )}
395 + {t.rows.every((r) => r.estimate_type === 'observed') ? <ConfidenceBadge level="HIGH" /> : <ConfidenceBadge level="MEDIUM" />}
396 + <Link className="ci-link" href={`/country/${geoSlug}?sex=${sex}&year=${t.year}#trend`}>
397 + trend ↓
398 + </Link>
399 + </div>
400 + <Freshness dataUpdatedAt={t.rows.map((r) => toDate(r.updated_at)).filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? null} sourceVersion={prov.get(first.provenance_id)?.dataset_version ?? null} />
401 + </>
402 + )}
403 + </Section>
404 + );
405 +}
added apps/web/src/app/drug/[slug]/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading drug" />;
5 +}
modified apps/web/src/app/drug/[slug]/page.tsx +52 −14
@@ -6,35 +6,43 @@ import { PageHeader, Section, KV, Note } from '@/components/ui/section';
6 6 import { Badge } from '@/components/ui/badge';
7 7 import { EmptyState } from '@/components/ui/empty-state';
8 8 import { Freshness } from '@/components/ui/freshness';
9 +import { Pager } from '@/components/ui/pager';
9 10 import { ApprovalsTable } from '@/components/data/approvals-table';
10 11 import { EvidenceTable } from '@/components/data/evidence-table';
11 12 import { TrialTable } from '@/components/data/trial-list';
12 13 import { getDrugBySlug, approvalsForDrug } from '@/lib/queries/drugs';
13 −import { evidenceForDrug } from '@/lib/queries/evidence';
14 −import { trialsForDrug } from '@/lib/queries/trials';
14 +import { evidenceForDrug, evidenceForDrugCount, EVIDENCE_PAGE_SIZE } from '@/lib/queries/evidence';
15 +import { trialsForDrug, trialsForDrugCount, TRIAL_PAGE_SIZE } from '@/lib/queries/trials';
15 16 import { loadProvenance } from '@/lib/queries/provenance';
16 17 import { jsonLd, drugLd } from '@/lib/seo';
17 18 import { fmtInt, humanize } from '@/lib/format';
18 −import { str, type SP } from '@/lib/search-params';
19 +import { pageInfo } from '@/lib/pagination';
20 +import { str, int, withParams, type SP } from '@/lib/search-params';
19 21
20 22 export const revalidate = 3600;
21 23
22 24 export async function generateMetadata({ params }: { params: Promise<{ slug: string }> }): Promise<Metadata> {
23 25 const d = await getDrugBySlug((await params).slug);
24 − return d ? { title: `${d.name} — drug`, description: d.description ?? `${d.name}: regulatory approvals by jurisdiction, curated evidence and clinical trials.` } : { title: 'Drug' };
26 + return d ? { title: `${d.name} — drug`, description: d.description ?? `${d.name}: regulatory approvals by jurisdiction, curated evidence and clinical trials.`, alternates: { canonical: `/drug/${d.slug}` } } : { title: 'Drug' };
25 27 }
26 28
27 29 export default async function DrugPage({ params, searchParams }: { params: Promise<{ slug: string }>; searchParams: Promise<SP> }) {
28 30 const { slug } = await params;
29 31 const d = await getDrugBySlug(slug);
30 32 if (!d) notFound();
31 − const [approvals, evidence, trials] = await Promise.all([approvalsForDrug(d.id), evidenceForDrug(d.id), trialsForDrug(d.id)]);
33 + const sp = await searchParams;
34 + const [evTotal, tTotal] = await Promise.all([evidenceForDrugCount(d.id), trialsForDrugCount(d.id)]);
35 + const ev = pageInfo(int(sp, 'evPage', 1, 1, 100_000), EVIDENCE_PAGE_SIZE, evTotal);
36 + const tp = pageInfo(int(sp, 'tPage', 1, 1, 100_000), TRIAL_PAGE_SIZE, tTotal);
37 + const [approvals, evidence, trials] = await Promise.all([approvalsForDrug(d.id), evTotal ? evidenceForDrug(d.id, { page: ev.page, pageSize: ev.pageSize }) : Promise.resolve([]), tTotal ? trialsForDrug(d.id, { page: tp.page, pageSize: tp.pageSize }) : Promise.resolve([])]);
32 38 const prov = await loadProvenance([...approvals.map((a) => a.provenance_id), ...evidence.map((e) => e.provenance_id)]);
33 39 const jurisdictions = [...new Set(approvals.map((a) => a.jurisdiction))].sort();
34 − const wanted = approvals.length ? str(await searchParams, 'jurisdiction') : '';
40 + const wanted = approvals.length ? str(sp, 'jurisdiction') : '';
35 41 const selected = jurisdictions.includes(wanted) ? wanted : null;
36 42 const shown = selected ? approvals.filter((a) => a.jurisdiction === selected) : approvals;
37 43 const aliases = d.aliases ?? [];
44 + const current = { jurisdiction: selected ?? '', evPage: ev.page > 1 ? ev.page : '', tPage: tp.page > 1 ? tp.page : '' };
45 + const href = (o: Record<string, string | number | null | undefined>, hash?: string) => `/drug/${d.slug}${withParams(current, o)}${hash ? `#${hash}` : ''}`;
38 46
39 47 return (
40 48 <article>
@@ -72,31 +80,61 @@ export default async function DrugPage({ params, searchParams }: { params: Promi
72 80 {approvals.length ? (
73 81 <>
74 82 <nav aria-label="Jurisdiction" className="mb-3 flex flex-wrap gap-1.5 text-[12.5px]">
75 − <Link href={`/drug/${d.slug}`} className={`border px-2 py-0.5 no-underline ${!selected ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
83 + <Link href={href({ jurisdiction: '' }, 'approvals')} aria-current={!selected ? 'page' : undefined} className="ci-chip">
76 84 All
77 85 </Link>
78 86 {jurisdictions.map((j) => (
79 − <Link key={j} href={`/drug/${d.slug}?jurisdiction=${j}`} className={`ci-mono border px-2 py-0.5 no-underline ${selected === j ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
87 + <Link key={j} href={href({ jurisdiction: j }, 'approvals')} aria-current={selected === j ? 'page' : undefined} className="ci-chip ci-mono">
80 88 {j}
81 89 </Link>
82 90 ))}
83 91 </nav>
84 92 <ApprovalsTable rows={shown} prov={prov} showDrug={false} />
85 − <Freshness dataUpdatedAt={approvals.reduce((m, a) => (a.updated_at > m ? a.updated_at : m), approvals[0]!.updated_at)} />
93 + <Freshness dataUpdatedAt={approvals.reduce<Date | string | null>((m, a) => (m == null || String(a.updated_at) > String(m) ? a.updated_at : m), null)} />
86 94 </>
87 95 ) : (
88 − <EmptyState compact knows={[{ label: 'Curated evidence below' }, { label: 'Sources', href: '/sources' }]}>
96 + <EmptyState compact knows={[{ label: 'Curated evidence below', href: '#evidence' }, { label: 'Sources', href: '/sources' }]}>
89 97 No regulatory approval recorded. Absence here is not evidence of absence: only ingested jurisdictions are covered.
90 98 </EmptyState>
91 99 )}
92 100 </Section>
93 101
94 − <Section id="evidence" kicker="Curated evidence" title={`Clinical evidence (${fmtInt(evidence.length)})`} description="CIViC items in which this therapy appears, grouped by molecular profile. Cancer context per row.">
95 − {evidence.length ? <EvidenceTable items={evidence} prov={prov} showCancer /> : <EmptyState compact>No curated evidence item mentions this therapy yet.</EmptyState>}
102 + <Section id="evidence" kicker="Curated evidence" title={`Clinical evidence (${fmtInt(evTotal)})`} description={`CIViC items in which this therapy appears, grouped by cancer context, then molecular profile. ${EVIDENCE_PAGE_SIZE} items per page.`}>
103 + {evidence.length ? (
104 + <>
105 + <EvidenceTable
106 + items={evidence}
107 + prov={prov}
108 + showCancer
109 + summary={
110 + <>
111 + Showing {fmtInt(ev.from)}–{fmtInt(ev.to)} of {fmtInt(evTotal)} evidence items
112 + </>
113 + }
114 + />
115 + <Pager total={evTotal} pageSize={ev.pageSize} page={ev.page} hrefFor={(p) => href({ evPage: p > 1 ? p : '' }, 'evidence')} label="Evidence pages" noun="evidence items" />
116 + </>
117 + ) : (
118 + <EmptyState compact>No curated evidence item mentions this therapy yet.</EmptyState>
119 + )}
96 120 </Section>
97 121
98 − <Section id="trials" kicker="Clinical trials" title={`Trials with this intervention (${fmtInt(d.trial_count ?? trials.length)})`}>
99 − {trials.length ? <TrialTable rows={trials} /> : <EmptyState compact>No registered study lists this drug as an intervention yet.</EmptyState>}
122 + <Section id="trials" kicker="Clinical trials" title={`Trials with this intervention (${fmtInt(tTotal)})`} description={tTotal ? `Most recently updated first, ${TRIAL_PAGE_SIZE} per page.` : undefined}>
123 + {trials.length ? (
124 + <>
125 + <TrialTable
126 + rows={trials}
127 + summary={
128 + <>
129 + Showing {fmtInt(tp.from)}–{fmtInt(tp.to)} of {fmtInt(tTotal)} studies
130 + </>
131 + }
132 + />
133 + <Pager total={tTotal} pageSize={tp.pageSize} page={tp.page} hrefFor={(p) => href({ tPage: p > 1 ? p : '' }, 'trials')} label="Trial pages" noun="studies" />
134 + </>
135 + ) : (
136 + <EmptyState compact>No registered study lists this drug as an intervention yet.</EmptyState>
137 + )}
100 138 </Section>
101 139 </div>
102 140
added apps/web/src/app/drugs/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading drugs" />;
5 +}
modified apps/web/src/app/drugs/page.tsx +2 −1
@@ -9,7 +9,8 @@ import { fmtInt, humanize } from '@/lib/format';
9 9 import { str, int, withParams, type SP } from '@/lib/search-params';
10 10
11 11 export const metadata: Metadata = { title: 'Drugs', description: 'Oncology drugs with jurisdiction-aware approvals, curated evidence and trials.' };
12 −export const dynamic = 'force-dynamic';
12 +// Filtered list: rendered per request (searchParams); 600 s is the CDN/ISR hint shared by all list pages.
13 +export const revalidate = 600;
13 14 const PAGE_SIZE = 50;
14 15
15 16 export default async function DrugsPage({ searchParams }: { searchParams: Promise<SP> }) {
added apps/web/src/app/gene/[symbol]/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading gene" />;
5 +}
modified apps/web/src/app/gene/[symbol]/page.tsx +87 −37
@@ -6,31 +6,52 @@ import { PageHeader, Section, KV, Note } from '@/components/ui/section';
6 6 import { Badge } from '@/components/ui/badge';
7 7 import { EmptyState } from '@/components/ui/empty-state';
8 8 import { Freshness } from '@/components/ui/freshness';
9 +import { Pager } from '@/components/ui/pager';
9 10 import { EvidenceTable } from '@/components/data/evidence-table';
10 11 import { GeneFrequencyTable } from '@/components/data/frequency-table';
11 −import { getGeneBySymbol, variantsForGene, frequenciesForGene } from '@/lib/queries/genomics';
12 −import { evidenceForGene } from '@/lib/queries/evidence';
12 +import { getGeneBySymbol, variantsForGene, variantsForGeneCount, frequenciesForGene, VARIANT_PAGE_SIZE } from '@/lib/queries/genomics';
13 +import { evidenceForGene, evidenceForGeneCount, evidenceCancersForGene, EVIDENCE_PAGE_SIZE } from '@/lib/queries/evidence';
13 14 import { loadProvenance } from '@/lib/queries/provenance';
14 −import { recentPublicationsFor } from '@/lib/queries/publications';
15 +import { recentPublicationsFor, recentPublicationsForCount, PUBLICATION_PAGE_SIZE } from '@/lib/queries/publications';
15 16 import { PublicationList } from '@/components/data/publication-list';
16 17 import { fmtInt, humanize } from '@/lib/format';
18 +import { pageInfo } from '@/lib/pagination';
19 +import { int, withParams, type SP } from '@/lib/search-params';
17 20
18 21 export const revalidate = 3600;
22 +/** Cancer chips above the evidence table: the most-cited contexts only (every row still names its cancer). */
23 +const CANCER_CHIPS = 20;
19 24
20 25 export async function generateMetadata({ params }: { params: Promise<{ symbol: string }> }): Promise<Metadata> {
21 26 const g = await getGeneBySymbol((await params).symbol);
22 − return g ? { title: `${g.symbol} — gene`, description: `${g.symbol}${g.name ? ` (${g.name})` : ''}: curated cancer evidence, variants and cohort alteration frequencies.` } : { title: 'Gene' };
27 + return g ? { title: `${g.symbol} — gene`, description: `${g.symbol}${g.name ? ` (${g.name})` : ''}: curated cancer evidence, variants and cohort alteration frequencies.`, alternates: { canonical: `/gene/${g.symbol}` } } : { title: 'Gene' };
23 28 }
24 29
25 −export default async function GenePage({ params }: { params: Promise<{ symbol: string }> }) {
30 +export default async function GenePage({ params, searchParams }: { params: Promise<{ symbol: string }>; searchParams: Promise<SP> }) {
26 31 const { symbol } = await params;
27 32 const g = await getGeneBySymbol(symbol);
28 33 if (!g) notFound();
29 34 if (g.symbol !== symbol) permanentRedirect(`/gene/${g.symbol}`);
30 − const [variants, evidence, freqs, pubs] = await Promise.all([variantsForGene(g.id), evidenceForGene(g.id, g.symbol), frequenciesForGene(g.id, g.symbol), recentPublicationsFor('gene', [g.id], 20)]);
35 + const sp = await searchParams;
36 + const evPageReq = int(sp, 'evPage', 1, 1, 100_000);
37 + const vPageReq = int(sp, 'vPage', 1, 1, 100_000);
38 + const pPageReq = int(sp, 'pPage', 1, 1, 100_000);
39 +
40 + const [evTotal, vTotal, pTotal] = await Promise.all([evidenceForGeneCount(g.id, g.symbol), variantsForGeneCount(g.id), recentPublicationsForCount('gene', [g.id])]);
41 + const ev = pageInfo(evPageReq, EVIDENCE_PAGE_SIZE, evTotal);
42 + const vp = pageInfo(vPageReq, VARIANT_PAGE_SIZE, vTotal);
43 + const pp = pageInfo(pPageReq, PUBLICATION_PAGE_SIZE, pTotal);
44 + const [variants, evidence, freqs, pubs, cancersInEvidence] = await Promise.all([
45 + vTotal ? variantsForGene(g.id, { page: vp.page, pageSize: vp.pageSize }) : Promise.resolve([]),
46 + evTotal ? evidenceForGene(g.id, g.symbol, { page: ev.page, pageSize: ev.pageSize }) : Promise.resolve([]),
47 + frequenciesForGene(g.id, g.symbol),
48 + pTotal ? recentPublicationsFor('gene', [g.id], { page: pp.page, pageSize: pp.pageSize }) : Promise.resolve([]),
49 + evTotal ? evidenceCancersForGene(g.id, g.symbol, CANCER_CHIPS) : Promise.resolve([]),
50 + ]);
31 51 const prov = await loadProvenance([...evidence.map((e) => e.provenance_id), ...freqs.map((f) => f.provenance_id)]);
32 − const cancersInEvidence = new Map<string, { slug: string; name: string; n: number }>();
33 − for (const e of evidence) if (e.cancer_slug) cancersInEvidence.set(e.cancer_slug, { slug: e.cancer_slug, name: e.cancer_name ?? e.cancer_slug, n: (cancersInEvidence.get(e.cancer_slug)?.n ?? 0) + 1 });
52 +
53 + const current = { evPage: ev.page > 1 ? ev.page : '', vPage: vp.page > 1 ? vp.page : '', pPage: pp.page > 1 ? pp.page : '' };
54 + const href = (o: Record<string, string | number | null | undefined>, hash: string) => `/gene/${g.symbol}${withParams(current, o)}#${hash}`;
34 55
35 56 return (
36 57 <article>
@@ -70,22 +91,34 @@ export default async function GenePage({ params }: { params: Promise<{ symbol: s
70 91 </Section>
71 92 ) : null}
72 93
73 − <Section id="evidence" kicker="Curated evidence" title={`Clinical evidence (${fmtInt(evidence.length)})`} description="CIViC items involving this gene, grouped by molecular profile and therapy, with native levels and directions.">
74 − {evidence.length ? (
94 + <Section id="evidence" kicker="Curated evidence" title={`Clinical evidence (${fmtInt(evTotal)})`} description={`CIViC items involving this gene, grouped by molecular profile and therapy, with native levels and directions. ${EVIDENCE_PAGE_SIZE} items per page.`}>
95 + {evTotal ? (
75 96 <>
76 − {cancersInEvidence.size ? (
77 − <p className="mb-3 flex flex-wrap gap-1.5 text-[12.5px]">
78 − <span className="ci-kicker mr-1">Cancers</span>
79 − {[...cancersInEvidence.values()]
80 − .sort((a, b) => b.n - a.n)
81 − .map((c) => (
82 − <Link key={c.slug} href={`/cancer/${c.slug}/evidence`} className="border border-rule px-2 py-0.5 no-underline hover:border-accent">
83 − {c.name} <span className="ci-num text-ink-3">{c.n}</span>
84 − </Link>
97 + {cancersInEvidence.length ? (
98 + <nav aria-label="Cancers in evidence" className="mb-3">
99 + <ul className="m-0 flex list-none flex-wrap gap-1.5 p-0 text-[12.5px]">
100 + <li className="ci-kicker mr-1 self-center">{cancersInEvidence.length >= CANCER_CHIPS ? `Top ${CANCER_CHIPS} cancers` : 'Cancers'}</li>
101 + {cancersInEvidence.map((c) => (
102 + <li key={c.slug}>
103 + <Link href={`/cancer/${c.slug}/evidence`} className="ci-chip">
104 + {c.name} <span className="ci-num text-ink-3">{c.n}</span>
105 + </Link>
106 + </li>
85 107 ))}
86 − </p>
108 + </ul>
109 + </nav>
87 110 ) : null}
88 − <EvidenceTable items={evidence} prov={prov} showCancer />
111 + <EvidenceTable
112 + items={evidence}
113 + prov={prov}
114 + showCancer
115 + summary={
116 + <>
117 + Showing {fmtInt(ev.from)}–{fmtInt(ev.to)} of {fmtInt(evTotal)} evidence items
118 + </>
119 + }
120 + />
121 + <Pager total={evTotal} pageSize={ev.pageSize} page={ev.page} hrefFor={(p) => href({ evPage: p > 1 ? p : '' }, 'evidence')} label="Evidence pages" noun="evidence items" />
89 122 </>
90 123 ) : (
91 124 <EmptyState compact>No curated evidence item involves this gene yet.</EmptyState>
@@ -96,8 +129,22 @@ export default async function GenePage({ params }: { params: Promise<{ symbol: s
96 129 {freqs.length ? <GeneFrequencyTable rows={freqs} prov={prov} /> : <EmptyState compact>No cohort frequency recorded for this gene.</EmptyState>}
97 130 </Section>
98 131
99 − <Section id="publications" kicker="Literature" title="Linked publications">
100 − {pubs.length ? <PublicationList rows={pubs} /> : <EmptyState compact>No publication linked to this gene yet.</EmptyState>}
132 + <Section id="publications" kicker="Literature" title={`Linked publications (${fmtInt(pTotal)})`} description={pTotal ? `${PUBLICATION_PAGE_SIZE} per page, newest first.` : undefined}>
133 + {pubs.length ? (
134 + <>
135 + <PublicationList
136 + rows={pubs}
137 + summary={
138 + <>
139 + Showing {fmtInt(pp.from)}–{fmtInt(pp.to)} of {fmtInt(pTotal)} publications
140 + </>
141 + }
142 + />
143 + <Pager total={pTotal} pageSize={pp.pageSize} page={pp.page} hrefFor={(p) => href({ pPage: p > 1 ? p : '' }, 'publications')} label="Publication pages" noun="publications" />
144 + </>
145 + ) : (
146 + <EmptyState compact>No publication linked to this gene yet.</EmptyState>
147 + )}
101 148 </Section>
102 149 </div>
103 150
@@ -119,21 +166,24 @@ export default async function GenePage({ params }: { params: Promise<{ symbol: s
119 166 />
120 167 <Freshness dataUpdatedAt={g.updated_at} extra="source: hgnc" />
121 168 </Section>
122 − <Section id="variants" kicker="Variants" title={variants.length >= 100 ? `Variants (top ${fmtInt(variants.length)} by evidence — full list via the API)` : `Variants (${fmtInt(variants.length)})`} level={3}>
169 + <Section id="variants" kicker="Variants" title={`Variants (${fmtInt(vTotal)})`} description={vTotal > VARIANT_PAGE_SIZE ? `Sorted by evidence count, ${VARIANT_PAGE_SIZE} per page.` : undefined} level={3}>
123 170 {variants.length ? (
124 − <ul className="max-h-[480px] overflow-y-auto text-[13.5px]">
125 − {variants.map((v) => (
126 − <li key={v.id} className="flex items-baseline justify-between gap-2 border-b border-rule py-1">
127 − <Link className="ci-link" href={`/variant/${v.slug}`}>
128 − {v.name}
129 − </Link>
130 − <span className="text-[11.5px] text-ink-3">
131 − {v.variant_type ? humanize(v.variant_type) : ''}
132 − {v.evidence_count ? ` · ${v.evidence_count} ev.` : ''}
133 − </span>
134 − </li>
135 − ))}
136 − </ul>
171 + <>
172 + <ul className="ci-rows">
173 + {variants.map((v) => (
174 + <li key={v.id}>
175 + <Link className="ci-link" href={`/variant/${v.slug}`}>
176 + {v.name}
177 + </Link>
178 + <span>
179 + {v.variant_type ? humanize(v.variant_type) : ''}
180 + {v.evidence_count ? ` · ${v.evidence_count} ev.` : ''}
181 + </span>
182 + </li>
183 + ))}
184 + </ul>
185 + <Pager total={vTotal} pageSize={vp.pageSize} page={vp.page} hrefFor={(p) => href({ vPage: p > 1 ? p : '' }, 'variants')} label="Variant pages" noun="variants" className="text-[12px]" />
186 + </>
137 187 ) : (
138 188 <p className="text-[13px] text-ink-3">No variant entity recorded for this gene.</p>
139 189 )}
added apps/web/src/app/genes/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading genes" />;
5 +}
modified apps/web/src/app/genes/page.tsx +2 −1
@@ -9,7 +9,8 @@ import { fmtInt } from '@/lib/format';
9 9 import { str, int, bool, withParams, type SP } from '@/lib/search-params';
10 10
11 11 export const metadata: Metadata = { title: 'Genes', description: 'HGNC genes with curated cancer evidence, variants and cohort frequencies.' };
12 −export const dynamic = 'force-dynamic';
12 +// Filtered list: rendered per request (searchParams); 600 s is the CDN/ISR hint shared by all list pages.
13 +export const revalidate = 600;
13 14 const PAGE_SIZE = 50;
14 15
15 16 export default async function GenesPage({ searchParams }: { searchParams: Promise<SP> }) {
modified apps/web/src/app/globals.css +278 −0
@@ -322,3 +322,281 @@ pre.ci-code {
322 322 display: none !important;
323 323 }
324 324 }
325 +
326 +/* ------------------------------------------------------------------------------------------------
327 + Hardening pass (pagination, table weight, accessibility). Appended; nothing above is rewritten.
328 + ------------------------------------------------------------------------------------------------ */
329 +
330 +/* Badges: one short class per tone instead of a long utility string repeated on every table row
331 + (every className ships twice — HTML + RSC payload). Text always accompanies the colour. */
332 +.ci-badge {
333 + display: inline-flex;
334 + align-items: center;
335 + gap: 4px;
336 + border: 1px solid transparent;
337 + border-radius: var(--radius-sm);
338 + padding: 1px 6px;
339 + font-size: 11px;
340 + font-weight: 500;
341 + line-height: 16px;
342 + letter-spacing: 0.02em;
343 +}
344 +.ci-badge--neutral {
345 + background: var(--color-paper-3);
346 + color: var(--color-ink-2);
347 +}
348 +.ci-badge--accent {
349 + background: var(--color-accent-soft);
350 + color: var(--color-accent-2);
351 +}
352 +.ci-badge--warn {
353 + background: var(--color-warn-soft);
354 + color: var(--color-warn);
355 +}
356 +.ci-badge--danger {
357 + background: var(--color-danger-soft);
358 + color: var(--color-danger);
359 +}
360 +.ci-badge--ok {
361 + background: var(--color-ok-soft);
362 + color: var(--color-ok);
363 +}
364 +.ci-badge--outline {
365 + background: transparent;
366 + color: var(--color-ink-2);
367 + border-color: var(--color-rule-strong);
368 +}
369 +
370 +/* Source badge link (compact rows and popover trigger share one class) */
371 +.ci-src {
372 + font-family: var(--font-mono);
373 + display: inline-flex;
374 + align-items: center;
375 + border: 1px solid var(--color-rule-strong);
376 + border-radius: var(--radius-sm);
377 + background: var(--color-paper);
378 + padding: 0 4px;
379 + font-size: 10.5px;
380 + line-height: 16px;
381 + color: var(--color-ink-2);
382 + text-decoration: none;
383 + white-space: nowrap;
384 +}
385 +.ci-src:hover,
386 +.ci-src:focus-visible {
387 + border-color: var(--color-accent);
388 + color: var(--color-accent);
389 +}
390 +
391 +/* Pager */
392 +.ci-pager-link {
393 + display: inline-block;
394 + min-width: 28px;
395 + border: 1px solid var(--color-rule);
396 + padding: 4px 8px;
397 + text-align: center;
398 + color: var(--color-ink-2);
399 + text-decoration: none;
400 + line-height: 1.2;
401 +}
402 +.ci-pager-link:hover {
403 + border-color: var(--color-accent);
404 + color: var(--color-accent);
405 +}
406 +.ci-pager-link.is-current {
407 + border-color: var(--color-accent);
408 + background: var(--color-accent-soft);
409 + color: var(--color-accent-2);
410 + font-weight: 500;
411 +}
412 +.ci-pager-link.is-disabled {
413 + color: var(--color-ink-4);
414 + cursor: default;
415 +}
416 +
417 +/* Filter chips (status / phase / cohort / jurisdiction selectors) */
418 +.ci-chip {
419 + display: inline-block;
420 + border: 1px solid var(--color-rule);
421 + padding: 2px 8px;
422 + color: var(--color-ink-2);
423 + text-decoration: none;
424 + line-height: 1.35;
425 +}
426 +.ci-chip:hover {
427 + border-color: var(--color-accent);
428 +}
429 +.ci-chip[aria-current='page'],
430 +.ci-chip.is-selected {
431 + border-color: var(--color-accent);
432 + background: var(--color-accent-soft);
433 + color: var(--color-accent-2);
434 +}
435 +
436 +/* Focus visibility: links inside dense tables, tabs, chips and pager get a visible ring even when
437 + the surrounding element clips overflow (outline-offset 0 keeps the ring inside table cells). */
438 +.ci-tab:focus-visible,
439 +.ci-chip:focus-visible,
440 +.ci-pager-link:focus-visible,
441 +.ci-src:focus-visible,
442 +.ci-table a:focus-visible,
443 +.ci-table summary:focus-visible {
444 + outline: 2px solid var(--color-accent);
445 + outline-offset: 0;
446 + border-radius: 2px;
447 +}
448 +.ci-tabs a:focus-visible {
449 + outline-offset: -2px;
450 +}
451 +details > summary:focus-visible {
452 + outline: 2px solid var(--color-accent);
453 + outline-offset: 2px;
454 +}
455 +
456 +/* Table captions live above the table (visible text, not sr-only) — the caption is the place where
457 + the ONE provenance popover per table sits. */
458 +.ci-table caption {
459 + caption-side: top;
460 + text-align: left;
461 + padding: 0 0 6px;
462 + font-size: 12px;
463 + color: var(--color-ink-3);
464 +}
465 +.ci-table thead th[aria-sort] {
466 + cursor: default;
467 +}
468 +
469 +/* Skeletons (loading.tsx) — never look like data */
470 +.ci-skeleton {
471 + background: linear-gradient(90deg, var(--color-paper-3) 0%, var(--color-paper-2) 50%, var(--color-paper-3) 100%);
472 + background-size: 200% 100%;
473 + border-radius: 2px;
474 + animation: ci-shimmer 1.4s ease-in-out infinite;
475 +}
476 +@keyframes ci-shimmer {
477 + from {
478 + background-position: 200% 0;
479 + }
480 + to {
481 + background-position: -200% 0;
482 + }
483 +}
484 +@media (prefers-reduced-motion: reduce) {
485 + .ci-skeleton {
486 + animation: none;
487 + }
488 +}
489 +
490 +/* Keep the sticky provenance popover reachable by keyboard users: the panel becomes visible while
491 + any element inside is focused (focus-within already covers the trigger). */
492 +.ci-pop:focus-within > .ci-pop-panel {
493 + display: block;
494 +}
495 +
496 +/* Row-group header inside a single dense table (one <tbody> per variant / cancer group) */
497 +.ci-table tr.ci-group th {
498 + position: static;
499 + background: var(--color-paper-2);
500 + border-top: 1px solid var(--color-rule-strong);
501 + border-bottom: 1px solid var(--color-rule);
502 + padding: 7px 10px;
503 + text-align: left;
504 + font-family: var(--font-sans);
505 + font-size: 13.5px;
506 + font-weight: 500;
507 + letter-spacing: 0;
508 + text-transform: none;
509 + color: var(--color-ink);
510 + white-space: normal;
511 + overflow-wrap: anywhere;
512 +}
513 +.ci-table tbody:first-of-type tr.ci-group th {
514 + border-top: 0;
515 +}
516 +.ci-evidence td.w-t {
517 + min-width: 160px;
518 +}
519 +abbr.no-underline {
520 + text-decoration: none;
521 +}
522 +
523 +/* Compact list rows (e.g. variants on a gene page): link left, meta right */
524 +.ci-rows {
525 + margin: 0;
526 + padding: 0;
527 + list-style: none;
528 + font-size: 13.5px;
529 +}
530 +.ci-rows > li {
531 + display: flex;
532 + align-items: baseline;
533 + justify-content: space-between;
534 + gap: 8px;
535 + border-bottom: 1px solid var(--color-rule);
536 + padding: 4px 0;
537 +}
538 +.ci-rows > li > span {
539 + font-size: 11.5px;
540 + color: var(--color-ink-3);
541 + text-align: right;
542 +}
543 +
544 +/* Publication list */
545 +.ci-pubs {
546 + margin: 0;
547 + padding: 0;
548 + list-style: none;
549 +}
550 +.ci-pub {
551 + padding: 10px 0;
552 + border-bottom: 1px solid var(--color-rule);
553 + font-size: 13.5px;
554 +}
555 +.ci-pub > div {
556 + display: flex;
557 + flex-wrap: wrap;
558 + align-items: baseline;
559 + column-gap: 8px;
560 +}
561 +.ci-pub > p {
562 + margin-top: 2px;
563 + font-size: 12.5px;
564 + color: var(--color-ink-3);
565 +}
566 +
567 +/* Evidence table: link, level and details styles are scoped here instead of per-cell classes
568 + (50 rows × ~8 classes × 2 copies per page). */
569 +.ci-evidence a {
570 + color: var(--color-accent);
571 + text-decoration: underline;
572 + text-decoration-color: color-mix(in oklab, var(--color-accent) 35%, transparent);
573 +}
574 +.ci-evidence a:hover {
575 + text-decoration-color: var(--color-accent);
576 +}
577 +.ci-evidence tr.ci-group a.ci-mono {
578 + color: var(--color-ink-3);
579 + text-decoration: none;
580 +}
581 +.ci-evidence tr.ci-group a.ci-mono:hover {
582 + color: var(--color-accent);
583 +}
584 +.ci-evidence abbr {
585 + font-family: var(--font-mono);
586 + font-size: 0.86em;
587 + font-weight: 500;
588 + text-decoration: none;
589 +}
590 +.ci-evidence td.ev {
591 + max-width: 360px;
592 + font-size: 12.5px;
593 + color: var(--color-ink-2);
594 +}
595 +.ci-evidence td.ev summary {
596 + color: var(--color-accent);
597 + text-decoration: underline;
598 + text-decoration-color: color-mix(in oklab, var(--color-accent) 35%, transparent);
599 +}
600 +.ci-evidence td.ev details p {
601 + margin-top: 4px;
602 +}
modified apps/web/src/app/methodology/page.tsx +60 −4
@@ -20,9 +20,9 @@ export default async function MethodologyPage() {
20 20 <PageHeader kicker="Methodology" title="How the index is built" lede="CancerIndex separates layers — raw, normalized, canonical, derived, ranked — and keeps them separable. This page documents the rules applied at each step and lists every metric with its exact formula and version." />
21 21
22 22 <nav aria-label="On this page" className="flex flex-wrap gap-x-4 gap-y-1 border-y border-rule py-2 text-[13px]">
23 − {['layers', 'normalization', 'reconciliation', 'hierarchy', 'uncertainty', 'metrics', 'versioning', 'not-computed', 'limitations'].map((id) => (
23 + {['layers', 'normalization', 'reconciliation', 'hierarchy', 'uncertainty', 'metrics', 'versioning', 'country-scopes', 'cagr', 'compare', 'gap-caveat', 'not-computed', 'limitations'].map((id) => (
24 24 <a key={id} href={`#${id}`} className="ci-link">
25 − {humanize(id)}
25 + {id === 'cagr' ? 'CAGR' : humanize(id)}
26 26 </a>
27 27 ))}
28 28 </nav>
@@ -149,10 +149,66 @@ export default async function MethodologyPage() {
149 149 <p>A ranking snapshot is one metric × one scope × one formula version at one time. It stores the number of eligible entities, the list of source ids and a hash of its inputs; each row stores its lineage (observation ids, counters, formula inputs). Re-running the engine with identical inputs yields identical ranks. Changing a formula creates a new version; old snapshots remain queryable. Public identifiers (CI-CAN-…, CI-GENE-…) are minted once and never reused.</p>
150 150 </Section>
151 151
152 + <Section id="country-scopes" kicker="§47" title="Country scopes and data sources per geography">
153 + <p>
154 + Population statistics are stored per geography, year, sex and age group, exactly as published (<ClaimBadge kind="observed" />), and rankings are computed per scope key — <code>geo=&lt;ISO3&gt;|sex=&lt;all|male|female&gt;|age=all|year=&lt;YYYY&gt;|level=top</code>. A country page (<Link className="ci-link" href="/countries">/countries</Link>, <Link className="ci-link" href="/country/united-states">/country/united-states</Link>) exists only for geographies with at least one ingested observation; nothing is shown for the others.
155 + </p>
156 + <ul>
157 + <li>
158 + <strong>United States</strong> — <Link className="ci-link" href="/source/cdc-uscs">U.S. Cancer Statistics (CDC)</Link>: NPCR/SEER incidence and NVSS mortality, 1999 onwards, per USCS site group (ICD-O-3 / ICD-10 ranges shown as the site definition on every row), both sexes and by sex, all ages, age-standardized to the 2000 U.S. standard population (19 age groups). Incidence is published about one year later than mortality, so the latest year differs between metrics; the page states the year actually used. USCS site groups are mapped one-to-one to the top-level registry set; "Female Breast" / "Male Breast" / "Male and Female Breast" map to the same entity and are selected by the sex dimension.
159 + </li>
160 + <li>
161 + <strong>Why US only for now</strong> — the connectors that would cover other countries are not licensed for production: the IARC Global Cancer Observatory (GLOBOCAN, 185 countries) has license status <em>review</em> and the SEER API is <em>awaiting credentials</em> (CLAUDE.md §10.4, §142). Estimates are never substituted for registry counts, and no country is listed without observations.
162 + </li>
163 + <li>
164 + <strong>All-sites totals</strong> — a national total is displayed only when the source publishes an all-sites row. CancerIndex never sums site groups (definitions overlap across sources and years); when the row is absent the page says so and shows per-site figures only.
165 + </li>
166 + <li>
167 + <strong>Standard populations</strong> — rates standardized to the 2000 U.S. standard population are not comparable with World (Segi) standardized rates used by IARC; the standard is printed with every rate and rankings never mix standards.
168 + </li>
169 + </ul>
170 + </Section>
171 +
172 + <Section id="cagr" kicker="Derived on the fly" title='Compound annual growth rate (home page "Fastest-rising incidence")'>
173 + <p>
174 + The home module computes, for each cancer with a complete series, the compound annual growth rate of the age-standardized incidence rate over the last ten years published for the geography (both sexes, all ages):
175 + </p>
176 + <p className="my-2">
177 + <code>CAGR = (ASIR[end_year] / ASIR[start_year])^(1 / (end_year − start_year)) − 1</code>
178 + </p>
179 + <ul>
180 + <li>Window: the ten most recent distinct years with observations; the module is hidden when fewer than ten years exist. Cancers lacking a value at either endpoint, or with fewer than ten points in the window, are skipped — nothing is interpolated.</li>
181 + <li>Inputs: the two endpoint observations only (their provenance is kept on the row); the value is labelled <ClaimBadge kind="computed" /> with version <code>ci-asir-cagr-10y-v1</code>. It is a page-level derivation, not a stored metric and not a ranking snapshot.</li>
182 + <li>Interpretation: a rising rate may reflect true risk change, screening or diagnostic practice, coding or registry changes, or residual demographic effects. The module states the two endpoint values so the number can be reproduced.</li>
183 + </ul>
184 + </Section>
185 +
186 + <Section id="compare" kicker="§100" title="Compare page rules">
187 + <ul>
188 + <li>Two to four cancers are compared (<Link className="ci-link" href="/compare">/compare?ids=slug-1,slug-2</Link>); the selection lives in the URL so a comparison is shareable. Unknown or merged slugs are ignored and reported.</li>
189 + <li>
190 + <strong>Nearest top-level ancestor</strong> — registry figures exist only for the mutually exclusive top-level site groups. For an entity below that level (e.g. glioblastoma), the compare page and the overview "Key figures" strip walk every hierarchy type upwards and use the nearest active top-level ancestor (e.g. "Malignant Central Nervous System Neoplasm"), labelling the cell "figures for &lt;ancestor&gt; (registry level)". Those figures describe the whole site group, not the subtype. When no top-level ancestor exists the cell reads "no registry observation at this level".
191 + </li>
192 + <li>Counters (trials, publications, evidence, genes, cohorts, drugs) are shown for the entity itself and aggregate over its descendants; entities at different depths are therefore labelled with their depth and hierarchy position and should not be read as equivalent.</li>
193 + <li>Ranks show the entity's own current rank when one exists (top-level entities among the top-level set, others among all malignant entities), otherwise the ancestor's burden rank, labelled as such. Bars in the charts are proportional within one chart only; missing values render as a dash, never as zero.</li>
194 + </ul>
195 + </Section>
196 +
197 + <Section id="gap-caveat" kicker="§324" title="Trial gap and research gap — caveats">
198 + <p>
199 + Gap indexes are <em>burden percentile − activity percentile</em> within one scope (geography, year, sex, top level): <code>trial_gap = percentile(mortality_count) − percentile(active_trials)</code> and <code>research_gap = percentile(mortality_count) − percentile(publications_5y)</code>. Positive values flag cancers with high mortality burden and comparatively little registered activity. They are quantitative signals, not accusations.
200 + </p>
201 + <ul>
202 + <li>Trial counts aggregate a cancer and its NCIt descendants only. Registrations phrased at a broader level (e.g. "colorectal cancer" for colon and rectal cancer, or "head and neck cancer" for laryngeal and pharyngeal sites) are attributed to the broader entity, which can overstate the gap of narrower top-level sites.</li>
203 + <li>Literature counts are query-based per entity (the query is stored with each count) and are not aggregated over descendants; entities without a stored query are excluded from the research gap.</li>
204 + <li>Percentiles depend on the eligible set of the scope; a gap value is therefore comparable only with rows of the same snapshot. The number of eligible entities and the two percentiles are stored on every row and shown on the ranking page.</li>
205 + <li>Burden here is deaths in one geography (currently the United States); a cancer that is common elsewhere but rare in that geography will show a small burden percentile.</li>
206 + </ul>
207 + </Section>
208 +
152 209 <Section id="not-computed" kicker="Phase 1" title="What is not yet computed">
153 210 <ul>
154 − <li>Burden and lethality rankings (annual cases, deaths, age-standardized rates, mortality-to-incidence ratio, 5-year survival) require licensed registry observations. IARC / GLOBOCAN is under license review and SEER awaits credentials; nothing is displayed until observations exist.</li>
155 − <li>Gap indexes (trial gap, research gap) depend on burden data and therefore wait as well.</li>
211 + <li>Burden, lethality and gap rankings exist only for scopes with licensed registry observations — currently the United States (CDC U.S. Cancer Statistics), per year and sex, top level. Global scopes wait for IARC / GLOBOCAN (license review) and SEER (credentials); 5-year survival waits for survival observations.</li>
156 212 <li>Composite scores (an overall "impact" index) are deliberately absent: they would hide the choice of weights.</li>
157 213 <li>AI-generated summaries are not enabled; all text on entity pages is sourced from terminologies or written as fixed methodology.</li>
158 214 </ul>
modified apps/web/src/app/page.tsx +14 −2
@@ -10,7 +10,10 @@ import { previewSnapshot } from '@/lib/queries/rankings';
10 10 import { mostActiveResearch, mostCuratedEvidence } from '@/lib/queries/trials';
11 11 import { resolveTopLevel } from '@/lib/queries/cancers';
12 12 import { listSources } from '@/lib/queries/sources';
13 −import { fmtInt, fmtValue, fmtDate, scopeLabel, unitLabel, humanize } from '@/lib/format';
13 +import { fmtInt, fmtValue, fmtDate, scopeLabel, unitLabel, humanize, parseScopeKey } from '@/lib/format';
14 +import { BurdenModule } from '@/components/home/burden-module';
15 +import { RisingModule } from '@/components/home/rising-module';
16 +import { GapsModule } from '@/components/home/gaps-module';
14 17
15 18 export const revalidate = 900;
16 19
@@ -73,10 +76,13 @@ export default async function HomePage() {
73 76
74 77 <div className="mt-10 grid gap-10 lg:grid-cols-[1.35fr_1fr]">
75 78 <div className="space-y-10">
79 + {/* Cancer burden (US, latest year) — §105 "Cancer burden today" */}
80 + <BurdenModule slug="united-states" />
81 +
76 82 {/* Rankings preview */}
77 83 <Section
78 84 id="rankings"
79 − kicker="Global rankings"
85 + kicker={preview && parseScopeKey(preview.snapshot.scope_key).geo !== 'WORLD' ? `Rankings · ${parseScopeKey(preview.snapshot.scope_key).geo}` : 'Global rankings'}
80 86 title={preview ? preview.metric.name : 'Global rankings'}
81 87 description={preview ? `${scopeLabel(preview.snapshot.scope_key)} · formula ${preview.snapshot.formula_version} · generated ${fmtDate(preview.snapshot.generated_at)} · ${fmtInt(preview.snapshot.eligible_entities)} eligible entities` : undefined}
82 88 actions={
@@ -140,6 +146,12 @@ export default async function HomePage() {
140 146 )}
141 147 </Section>
142 148
149 + {/* Fastest-rising incidence (derived on the fly, §105/§328) */}
150 + <RisingModule slug="united-states" />
151 +
152 + {/* Largest trial / research gaps (§105, §265-266) */}
153 + <GapsModule geo="USA" />
154 +
143 155 {/* Active research */}
144 156 <Section id="active-research" kicker="Clinical research" title="Most active clinical research" description="Cancers with the most interventional trials in an active status (recruiting, not yet recruiting, enrolling by invitation, active not recruiting), counted over the entity and its descendants.">
145 157 {active.length ? (
modified apps/web/src/app/publication/[pmid]/page.tsx +30 −8
@@ -8,24 +8,30 @@ import { EmptyState } from '@/components/ui/empty-state';
8 8 import { Freshness } from '@/components/ui/freshness';
9 9 import { SourceBadge } from '@/components/ui/source-badge';
10 10 import { EvidenceTable } from '@/components/data/evidence-table';
11 +import { Pager } from '@/components/ui/pager';
11 12 import { getPublicationByPmid, publicationEntities } from '@/lib/queries/publications';
12 −import { evidenceForPublication } from '@/lib/queries/evidence';
13 +import { evidenceForPublication, evidenceForPublicationCount, EVIDENCE_PAGE_SIZE } from '@/lib/queries/evidence';
13 14 import { loadProvenance } from '@/lib/queries/provenance';
14 15 import { fmtDate, fmtInt, humanize, truncate } from '@/lib/format';
16 +import { pageInfo } from '@/lib/pagination';
17 +import { int, type SP } from '@/lib/search-params';
15 18
16 19 export const revalidate = 3600;
17 20
18 21 export async function generateMetadata({ params }: { params: Promise<{ pmid: string }> }): Promise<Metadata> {
19 22 const p = await getPublicationByPmid((await params).pmid);
20 − return p ? { title: `${truncate(p.title, 80)} — PMID ${p.pmid}`, description: truncate(p.abstract ?? p.title, 160), robots: p.retracted ? { index: false } : undefined } : { title: 'Publication' };
23 + return p ? { title: `${truncate(p.title, 80)} — PMID ${p.pmid}`, description: truncate(p.abstract ?? p.title, 160), robots: p.retracted ? { index: false } : undefined, alternates: { canonical: `/publication/${p.pmid}` } } : { title: 'Publication' };
21 24 }
22 25
23 −export default async function PublicationPage({ params }: { params: Promise<{ pmid: string }> }) {
26 +export default async function PublicationPage({ params, searchParams }: { params: Promise<{ pmid: string }>; searchParams: Promise<SP> }) {
24 27 const { pmid } = await params;
25 28 if (!/^\d{1,10}$/.test(pmid)) notFound();
26 29 const p = await getPublicationByPmid(pmid);
27 30 if (!p) notFound();
28 − const [edges, evidence] = await Promise.all([publicationEntities(p.id), evidenceForPublication(pmid)]);
31 + const sp = await searchParams;
32 + const evTotal = await evidenceForPublicationCount(pmid);
33 + const ev = pageInfo(int(sp, 'evPage', 1, 1, 100_000), EVIDENCE_PAGE_SIZE, evTotal);
34 + const [edges, evidence] = await Promise.all([publicationEntities(p.id), evTotal ? evidenceForPublication(pmid, { page: ev.page, pageSize: ev.pageSize }) : Promise.resolve([])]);
29 35 const prov = await loadProvenance(evidence.map((e) => e.provenance_id));
30 36 const validated = edges.filter((e) => e.status === 'validated');
31 37 const candidates = edges.filter((e) => e.status === 'candidate');
@@ -46,7 +52,7 @@ export default async function PublicationPage({ params }: { params: Promise<{ pm
46 52 <p className="mt-2 text-[13.5px] text-ink-2">
47 53 {p.authors.map((a) => a.name).join(', ') || 'Authors not recorded'}
48 54 </p>
49 − <p className="mt-1 flex flex-wrap items-center gap-2 text-[12.5px] text-ink-3">
55 + <div className="mt-1 flex flex-wrap items-center gap-2 text-[12.5px] text-ink-3">
50 56 {p.journal ? <span className="italic">{p.journal}</span> : null}
51 57 {p.pub_date ? <span>{fmtDate(p.pub_date)}</span> : p.pub_year ? <span>{p.pub_year}</span> : null}
52 58 <span className="ci-mono">PMID {p.pmid}</span>
@@ -63,7 +69,7 @@ export default async function PublicationPage({ params }: { params: Promise<{ pm
63 69 ))}
64 70 <SourceBadge p={{ sourceSlug: 'pubmed', sourceName: 'PubMed', retrievedAt: p.updated_at, ingestRunId: p.ingest_run_id }} />
65 71 <ClaimBadge kind="published" />
66 − </p>
72 + </div>
67 73 </PageHeader>
68 74
69 75 <div className="grid gap-8 lg:grid-cols-[1fr_320px]">
@@ -133,8 +139,24 @@ export default async function PublicationPage({ params }: { params: Promise<{ pm
133 139 ) : null}
134 140 </Section>
135 141
136 − <Section id="evidence" kicker="Curated evidence" title={`Evidence citing this paper (${fmtInt(evidence.length)})`}>
137 − {evidence.length ? <EvidenceTable items={evidence} prov={prov} showCancer /> : <EmptyState compact>No curated evidence item cites this publication.</EmptyState>}
142 + <Section id="evidence" kicker="Curated evidence" title={`Evidence citing this paper (${fmtInt(evTotal)})`} description={evTotal > EVIDENCE_PAGE_SIZE ? `${EVIDENCE_PAGE_SIZE} items per page.` : undefined}>
143 + {evidence.length ? (
144 + <>
145 + <EvidenceTable
146 + items={evidence}
147 + prov={prov}
148 + showCancer
149 + summary={
150 + <>
151 + Showing {fmtInt(ev.from)}–{fmtInt(ev.to)} of {fmtInt(evTotal)} evidence items
152 + </>
153 + }
154 + />
155 + <Pager total={evTotal} pageSize={ev.pageSize} page={ev.page} hrefFor={(pg) => `/publication/${p.pmid}${pg > 1 ? `?evPage=${pg}` : ''}#evidence`} label="Evidence pages" noun="evidence items" />
156 + </>
157 + ) : (
158 + <EmptyState compact>No curated evidence item cites this publication.</EmptyState>
159 + )}
138 160 </Section>
139 161 </div>
140 162
modified apps/web/src/app/sitemap.xml/route.ts +4 −2
@@ -1,9 +1,11 @@
1 1 import { indexXml, sitemapChunks } from '@/lib/sitemap';
2 +import { SITE_URL } from '@/lib/site';
2 3
3 4 export const dynamic = 'force-dynamic';
4 5
5 −/** Sitemap index → chunked sitemaps at /sitemap/<id> (cancers, genes, drugs, trials, sources). */
6 +/** Sitemap index → chunked sitemaps at /sitemap/<id> (cancers, genes, drugs, trials, sources) + /sitemap/countries. */
6 7 export async function GET() {
7 8 const chunks = await sitemapChunks();
8 − return new Response(indexXml([0, ...chunks.map((c) => c.id)]), { headers: { 'Content-Type': 'application/xml; charset=utf-8', 'Cache-Control': 'public, max-age=3600' } });
9 + const xml = indexXml([0, ...chunks.map((c) => c.id)]).replace('</sitemapindex>', ` <sitemap><loc>${SITE_URL}/sitemap/countries</loc><lastmod>${new Date().toISOString()}</lastmod></sitemap>\n</sitemapindex>`);
10 + return new Response(xml, { headers: { 'Content-Type': 'application/xml; charset=utf-8', 'Cache-Control': 'public, max-age=3600' } });
9 11 }
added apps/web/src/app/sitemap/countries/route.ts +21 −0
@@ -0,0 +1,21 @@
1 +import { urlsetXml, type Entry } from '@/lib/sitemap';
2 +import { geographySlugsForSitemap } from '@/lib/queries/geography';
3 +import { SITE_URL } from '@/lib/site';
4 +import { toDate } from '@/lib/format';
5 +
6 +export const dynamic = 'force-dynamic';
7 +
8 +/**
9 + * Sitemap chunk for geography routes: /countries, /compare and one /country/<slug> per geography that has
10 + * epidemiology observations. Static segment, so it takes precedence over /sitemap/[id] without touching it.
11 + */
12 +export async function GET() {
13 + const now = new Date();
14 + const geos = await geographySlugsForSitemap();
15 + const entries: Entry[] = [
16 + { url: `${SITE_URL}/countries`, lastModified: geos.map((g) => toDate(g.updated_at)).filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? now, changeFrequency: 'weekly', priority: 0.7 },
17 + { url: `${SITE_URL}/compare`, lastModified: now, changeFrequency: 'weekly', priority: 0.5 },
18 + ...geos.map((g) => ({ url: `${SITE_URL}/country/${g.slug}`, lastModified: toDate(g.updated_at) ?? now, changeFrequency: 'weekly' as const, priority: 0.7 })),
19 + ];
20 + return new Response(urlsetXml(entries), { headers: { 'Content-Type': 'application/xml; charset=utf-8', 'Cache-Control': 'public, max-age=3600' } });
21 +}
added apps/web/src/app/trial/[nct]/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading clinical trial" />;
5 +}
modified apps/web/src/app/trial/[nct]/page.tsx +43 −19
@@ -9,30 +9,36 @@ import { Freshness } from '@/components/ui/freshness';
9 9 import { SourceBadge } from '@/components/ui/source-badge';
10 10 import { JsonView } from '@/components/ui/json-view';
11 11 import { PublicationList } from '@/components/data/publication-list';
12 +import { Pager } from '@/components/ui/pager';
12 13 import { getTrialByNct, trialConditionsFor, trialInterventionsFor, trialLocationsByCountry } from '@/lib/queries/trials';
13 −import { publicationsForTrial } from '@/lib/queries/publications';
14 +import { publicationsForTrial, publicationsForTrialCount, PUBLICATION_PAGE_SIZE } from '@/lib/queries/publications';
14 15 import { fmtDate, fmtInt, humanize, phaseLabel } from '@/lib/format';
16 +import { pageInfo } from '@/lib/pagination';
17 +import { int, type SP } from '@/lib/search-params';
15 18
16 19 export const revalidate = 3600;
17 20
18 21 export async function generateMetadata({ params }: { params: Promise<{ nct: string }> }): Promise<Metadata> {
19 22 const t = await getTrialByNct((await params).nct);
20 − return t ? { title: `${t.nct_id} — ${t.brief_title}`, description: t.brief_summary ? t.brief_summary.slice(0, 160) : `${t.nct_id}: status, phase, conditions, interventions, locations and references.` } : { title: 'Trial' };
23 + return t ? { title: `${t.nct_id} — ${t.brief_title}`, description: t.brief_summary ? t.brief_summary.slice(0, 160) : `${t.nct_id}: status, phase, conditions, interventions, locations and references.`, alternates: { canonical: `/trial/${t.nct_id}` } } : { title: 'Trial' };
21 24 }
22 25
23 −export default async function TrialPage({ params }: { params: Promise<{ nct: string }> }) {
26 +export default async function TrialPage({ params, searchParams }: { params: Promise<{ nct: string }>; searchParams: Promise<SP> }) {
24 27 const { nct } = await params;
25 28 const t = await getTrialByNct(nct);
26 29 if (!t) notFound();
27 30 if (t.nct_id !== nct) permanentRedirect(`/trial/${t.nct_id}`);
28 − const [conditions, interventions, locations, pubs] = await Promise.all([trialConditionsFor(t.id), trialInterventionsFor(t.id), trialLocationsByCountry(t.id), publicationsForTrial(t.nct_id)]);
31 + const sp = await searchParams;
32 + const pubTotal = await publicationsForTrialCount(t.nct_id);
33 + const pp = pageInfo(int(sp, 'pPage', 1, 1, 100_000), PUBLICATION_PAGE_SIZE, pubTotal);
34 + const [conditions, interventions, locations, pubs] = await Promise.all([trialConditionsFor(t.id), trialInterventionsFor(t.id), trialLocationsByCountry(t.id), pubTotal ? publicationsForTrial(t.nct_id, { page: pp.page, pageSize: pp.pageSize }) : Promise.resolve([])]);
29 35 const elig = t.eligibility ?? {};
30 36 const eligText = typeof elig.criteria === 'string' ? elig.criteria : typeof elig.eligibilityCriteria === 'string' ? elig.eligibilityCriteria : null;
31 37
32 38 return (
33 39 <article>
34 40 <PageHeader kicker={`Clinical trial · ${t.study_type ? humanize(t.study_type) : 'study'}`} title={t.brief_title} lede={t.official_title && t.official_title !== t.brief_title ? t.official_title : undefined}>
35 − <p className="mt-2 flex flex-wrap items-center gap-2 text-[12.5px]">
41 + <div className="mt-2 flex flex-wrap items-center gap-2 text-[12.5px]">
36 42 <span className="ci-mono text-ink-2">{t.nct_id}</span>
37 43 <span className="ci-mono text-ink-3">{t.id}</span>
38 44 {t.acronym ? <Badge tone="outline">{t.acronym}</Badge> : null}
@@ -44,7 +50,7 @@ export default async function TrialPage({ params }: { params: Promise<{ nct: str
44 50 </a>
45 51 <SourceBadge p={{ sourceSlug: 'clinicaltrials', sourceName: 'ClinicalTrials.gov', retrievedAt: t.updated_at, ingestRunId: t.ingest_run_id, layer: 'normalized' }} />
46 52 <ClaimBadge kind="published" />
47 − </p>
53 + </div>
48 54 {t.why_stopped ? <Note tone="warn">Why stopped (as posted): {t.why_stopped}</Note> : null}
49 55 </PageHeader>
50 56
@@ -62,10 +68,12 @@ export default async function TrialPage({ params }: { params: Promise<{ nct: str
62 68 <table className="ci-table">
63 69 <thead>
64 70 <tr>
65 − <th>Condition (as posted)</th>
66 − <th>Mapped entity</th>
67 − <th>Match</th>
68 − <th className="num">Confidence</th>
71 + <th scope="col">Condition (as posted)</th>
72 + <th scope="col">Mapped entity</th>
73 + <th scope="col">Match</th>
74 + <th scope="col" className="num">
75 + Confidence
76 + </th>
69 77 </tr>
70 78 </thead>
71 79 <tbody>
@@ -109,10 +117,10 @@ export default async function TrialPage({ params }: { params: Promise<{ nct: str
109 117 <table className="ci-table">
110 118 <thead>
111 119 <tr>
112 − <th>Intervention</th>
113 − <th>Type</th>
114 − <th>Mapped drug</th>
115 − <th>Match</th>
120 + <th scope="col">Intervention</th>
121 + <th scope="col">Type</th>
122 + <th scope="col">Mapped drug</th>
123 + <th scope="col">Match</th>
116 124 </tr>
117 125 </thead>
118 126 <tbody>
@@ -189,8 +197,20 @@ export default async function TrialPage({ params }: { params: Promise<{ nct: str
189 197 )}
190 198 </Section>
191 199
192 − <Section id="references" kicker="References" title={`Publications (${fmtInt(pubs.length || t.references.length)})`}>
193 − {pubs.length ? <PublicationList rows={pubs} /> : null}
200 + <Section id="references" kicker="References" title={`Publications (${fmtInt(pubTotal || t.references.length)})`}>
201 + {pubs.length ? (
202 + <>
203 + <PublicationList
204 + rows={pubs}
205 + summary={
206 + <>
207 + Showing {fmtInt(pp.from)}–{fmtInt(pp.to)} of {fmtInt(pubTotal)} indexed publications citing this registration
208 + </>
209 + }
210 + />
211 + <Pager total={pubTotal} pageSize={pp.pageSize} page={pp.page} hrefFor={(p) => `/trial/${t.nct_id}${p > 1 ? `?pPage=${p}` : ''}#references`} label="Publication pages" noun="publications" />
212 + </>
213 + ) : null}
194 214 {t.references.length ? (
195 215 <ul className="mt-2 space-y-1 text-[13px]">
196 216 {t.references.map((r, i) => (
@@ -238,9 +258,13 @@ export default async function TrialPage({ params }: { params: Promise<{ nct: str
238 258 <table className="ci-table">
239 259 <thead>
240 260 <tr>
241 − <th>Country</th>
242 − <th className="num">Sites</th>
243 − <th className="num">Recruiting</th>
261 + <th scope="col">Country</th>
262 + <th scope="col" className="num">
263 + Sites
264 + </th>
265 + <th scope="col" className="num">
266 + Recruiting
267 + </th>
244 268 </tr>
245 269 </thead>
246 270 <tbody>
added apps/web/src/app/trials/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading clinical trials" />;
5 +}
modified apps/web/src/app/trials/page.tsx +23 −13
@@ -1,21 +1,22 @@
1 1 import type { Metadata } from 'next';
2 2 import { PageHeader } from '@/components/ui/section';
3 3 import { EmptyState } from '@/components/ui/empty-state';
4 −import { Pagination } from '@/components/ui/pagination';
4 +import { Pager } from '@/components/ui/pager';
5 5 import { TrialTable } from '@/components/data/trial-list';
6 6 import { Freshness } from '@/components/ui/freshness';
7 −import { listTrials, trialFacets } from '@/lib/queries/trials';
7 +import { listTrialRows, trialFacets, TRIAL_PAGE_SIZE } from '@/lib/queries/trials';
8 8 import { getCancerBySlug, getDescendantIds } from '@/lib/queries/cancers';
9 9 import { fmtInt, humanize, phaseLabel } from '@/lib/format';
10 +import { pageInfo } from '@/lib/pagination';
10 11 import { str, int, withParams, type SP } from '@/lib/search-params';
11 12
12 13 export const metadata: Metadata = { title: 'Clinical trials', description: 'ClinicalTrials.gov oncology studies mapped to the cancer taxonomy, filterable by status, phase, country and cancer.' };
13 −export const dynamic = 'force-dynamic';
14 −const PAGE_SIZE = 50;
14 +// Filtered list: rendered per request (searchParams); 600 s is the CDN/ISR hint shared by all list pages.
15 +export const revalidate = 600;
15 16
16 17 export default async function TrialsPage({ searchParams }: { searchParams: Promise<SP> }) {
17 18 const sp = await searchParams;
18 − const q = str(sp, 'q');
19 + const q = str(sp, 'q').slice(0, 200);
19 20 const status = str(sp, 'status');
20 21 const phase = str(sp, 'phase');
21 22 const country = str(sp, 'country');
@@ -23,15 +24,17 @@ export default async function TrialsPage({ searchParams }: { searchParams: Promi
23 24 const page = int(sp, 'page', 1, 1, 100_000);
24 25 const cancer = cancerSlug ? await getCancerBySlug(cancerSlug) : null;
25 26 const cancerIds = cancer ? await getDescendantIds(cancer.id) : null;
26 − const [facets, { rows, total }] = await Promise.all([trialFacets(cancerIds), listTrials({ q, status, phase, country, cancerIds, page, pageSize: PAGE_SIZE })]);
27 + const [facets, { rows, total }] = await Promise.all([trialFacets(cancerIds), listTrialRows({ q, status, phase, country, cancerIds, page, pageSize: TRIAL_PAGE_SIZE })]);
28 + const info = pageInfo(page, TRIAL_PAGE_SIZE, total);
27 29 const current = { q, status, phase, country, cancer: cancerSlug };
28 30 const href = (o: Record<string, string | number | null | undefined>) => `/trials${withParams(current, o)}`;
29 31 const anyTrials = facets.statuses.reduce((s, x) => s + x.n, 0);
32 + const filtered = Boolean(q || status || phase || country || cancer);
30 33
31 34 return (
32 35 <div>
33 36 <PageHeader kicker="Clinical trials" title="Clinical trials" lede="Studies registered on ClinicalTrials.gov whose conditions were reconciled to the cancer taxonomy. Status and phase are as posted by the registrant." />
34 − <form method="get" action="/trials" className="grid gap-2 border-y border-rule py-3 text-[13.5px] sm:grid-cols-2 lg:grid-cols-[2fr_1fr_1fr_1fr_1fr_auto]">
37 + <form method="get" action="/trials" className="grid gap-2 border-y border-rule py-3 text-[13.5px] sm:grid-cols-2 lg:grid-cols-[2fr_1fr_1fr_1fr_1fr_auto]" role="search" aria-label="Filter trials">
35 38 <label className="flex flex-col gap-1">
36 39 <span className="ci-kicker">NCT, title, acronym or sponsor</span>
37 40 <input name="q" defaultValue={q} className="border border-rule-strong bg-white px-2 py-1.5 outline-none focus:border-accent" />
@@ -80,12 +83,12 @@ export default async function TrialsPage({ searchParams }: { searchParams: Promi
80 83 </button>
81 84 </div>
82 85 </form>
83 − <p className="mt-3 text-[13px] text-ink-2">
84 − <span className="ci-num font-medium text-ink">{fmtInt(total)}</span> studies
86 + <p className="mt-3 text-[13px] text-ink-2" role="status">
87 + <span className="ci-num font-medium text-ink">{fmtInt(total)}</span> studies{filtered ? ' match' : ' indexed'}
85 88 {cancer ? (
86 89 <>
87 90 {' '}
88 − mapped to <span className="font-medium">{cancer.canonical_name}</span> and descendants
91 + · mapped to <span className="font-medium">{cancer.canonical_name}</span> and descendants
89 92 </>
90 93 ) : cancerSlug ? (
91 94 <span className="text-warn"> — unknown cancer slug "{cancerSlug}" (ignored)</span>
@@ -100,10 +103,17 @@ export default async function TrialsPage({ searchParams }: { searchParams: Promi
100 103 ) : (
101 104 <>
102 105 <div className="mt-3">
103 − <TrialTable rows={rows} />
106 + <TrialTable
107 + rows={rows}
108 + summary={
109 + <>
110 + Showing {fmtInt(info.from)}–{fmtInt(info.to)} of {fmtInt(total)} studies
111 + </>
112 + }
113 + />
104 114 </div>
105 − <Pagination page={page} pageSize={PAGE_SIZE} total={total} hrefFor={(p) => href({ page: p === 1 ? '' : p })} />
106 − <Freshness dataUpdatedAt={rows.reduce((m, t) => (t.updated_at > m ? t.updated_at : m), rows[0]!.updated_at)} extra="source: clinicaltrials" />
115 + <Pager page={info.page} pageSize={TRIAL_PAGE_SIZE} total={total} hrefFor={(p) => href({ page: p === 1 ? '' : p })} label="Trial pages" noun="studies" />
116 + <Freshness dataUpdatedAt={rows.reduce<Date | string | null>((m, t) => (m == null || String(t.updated_at) > String(m) ? t.updated_at : m), null)} extra="source: clinicaltrials" />
107 117 </>
108 118 )}
109 119 </div>
added apps/web/src/app/variant/[slug]/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading variant" />;
5 +}
modified apps/web/src/app/variant/[slug]/page.tsx +74 −69
@@ -6,35 +6,39 @@ import { PageHeader, Section, KV, Note } from '@/components/ui/section';
6 6 import { Badge, ClaimBadge } from '@/components/ui/badge';
7 7 import { EmptyState } from '@/components/ui/empty-state';
8 8 import { Freshness } from '@/components/ui/freshness';
9 −import { SourceBadge } from '@/components/ui/source-badge';
9 +import { SourceBadge, TableProvenance } from '@/components/ui/source-badge';
10 10 import { JsonView } from '@/components/ui/json-view';
11 +import { Pager } from '@/components/ui/pager';
11 12 import { EvidenceTable } from '@/components/data/evidence-table';
12 13 import { getVariantBySlug, variantAliases, clinicalSignificanceFor } from '@/lib/queries/genomics';
13 −import { evidenceForVariant, type EvidenceItem } from '@/lib/queries/evidence';
14 +import { evidenceForVariant, evidenceForVariantCount, EVIDENCE_PAGE_SIZE } from '@/lib/queries/evidence';
14 15 import { loadProvenance, toInfo } from '@/lib/queries/provenance';
15 16 import { fmtInt, humanize } from '@/lib/format';
17 +import { pageInfo } from '@/lib/pagination';
18 +import { int, type SP } from '@/lib/search-params';
16 19
17 20 export const revalidate = 3600;
18 21
19 22 export async function generateMetadata({ params }: { params: Promise<{ slug: string }> }): Promise<Metadata> {
20 23 const v = await getVariantBySlug((await params).slug);
21 − return v ? { title: `${v.gene_symbol ? `${v.gene_symbol} ` : ''}${v.name} — variant`, description: `Curated evidence by cancer and ClinVar interpretations for ${v.gene_symbol ?? ''} ${v.name}.` } : { title: 'Variant' };
24 + return v ? { title: `${v.gene_symbol ? `${v.gene_symbol} ` : ''}${v.name} — variant`, description: `Curated evidence by cancer and ClinVar interpretations for ${v.gene_symbol ?? ''} ${v.name}.`, alternates: { canonical: `/variant/${v.slug}` } } : { title: 'Variant' };
22 25 }
23 26
24 −export default async function VariantPage({ params }: { params: Promise<{ slug: string }> }) {
27 +const CLINVAR = { sourceSlug: 'clinvar', sourceName: 'ClinVar' };
28 +
29 +export default async function VariantPage({ params, searchParams }: { params: Promise<{ slug: string }>; searchParams: Promise<SP> }) {
25 30 const { slug } = await params;
26 31 const v = await getVariantBySlug(slug);
27 32 if (!v) notFound();
28 − const [aliases, evidence, clinsig] = await Promise.all([variantAliases(v.id), evidenceForVariant(v.id), clinicalSignificanceFor(v.id)]);
33 + const sp = await searchParams;
34 + const evTotal = await evidenceForVariantCount(v.id);
35 + const ev = pageInfo(int(sp, 'evPage', 1, 1, 100_000), EVIDENCE_PAGE_SIZE, evTotal);
36 + const [aliases, evidence, clinsig] = await Promise.all([variantAliases(v.id), evTotal ? evidenceForVariant(v.id, { page: ev.page, pageSize: ev.pageSize }) : Promise.resolve([]), clinicalSignificanceFor(v.id)]);
29 37 const prov = await loadProvenance([...evidence.map((e) => e.provenance_id), ...clinsig.map((c) => c.provenance_id)]);
30 − // Evidence grouped by cancer (§50)
31 − const byCancer = new Map<string, { slug: string | null; name: string; items: EvidenceItem[] }>();
32 − for (const e of evidence) {
33 − const k = e.cancer_id ?? `unmapped:${e.disease_name ?? 'unknown'}`;
34 − if (!byCancer.has(k)) byCancer.set(k, { slug: e.cancer_slug, name: e.cancer_name ?? e.disease_name ?? 'Unmapped disease', items: [] });
35 − byCancer.get(k)!.items.push(e);
36 − }
37 − const cancerGroups = [...byCancer.values()].sort((a, b) => b.items.length - a.items.length);
38 + // Evidence grouped by cancer (§50) — the query orders by cancer, so groups are contiguous within a page.
39 + const contexts = new Set(evidence.map((e) => e.cancer_id ?? `unmapped:${e.disease_name ?? 'unknown'}`)).size;
40 + const hrefFor = (p: number) => `/variant/${v.slug}${p > 1 ? `?evPage=${p}` : ''}#evidence`;
41 + const clinProv = clinsig[0] ? (toInfo(prov.get(clinsig[0].provenance_id)) ?? CLINVAR) : CLINVAR;
38 42
39 43 return (
40 44 <article>
@@ -63,26 +67,21 @@ export default async function VariantPage({ params }: { params: Promise<{ slug:
63 67
64 68 <div className="grid gap-8 lg:grid-cols-[1fr_320px]">
65 69 <div className="space-y-8">
66 − <Section id="evidence" kicker="Curated evidence" title={`Evidence by cancer (${fmtInt(evidence.length)} items)`} description="Grouped by cancer context first, then therapy. The same variant can be sensitizing in one cancer and irrelevant in another — contexts are never merged.">
67 − {cancerGroups.length ? (
68 − <div className="space-y-8">
69 − {cancerGroups.map((g) => (
70 − <div key={g.slug ?? g.name}>
71 − <h3 className="mb-2 text-lg">
72 − {g.slug ? (
73 − <Link href={`/cancer/${g.slug}/evidence`} className="ci-link">
74 − {g.name}
75 − </Link>
76 − ) : (
77 − <span className="text-ink-2">{g.name}</span>
78 − )}{' '}
79 − <span className="ci-num text-[13px] text-ink-3">{g.items.length}</span>
80 − {!g.slug ? <Badge tone="danger" className="ml-2">unmapped disease</Badge> : null}
81 − </h3>
82 − <EvidenceTable items={g.items} prov={prov} showVariant={false} />
83 − </div>
84 − ))}
85 − </div>
70 + <Section id="evidence" kicker="Curated evidence" title={`Evidence by cancer (${fmtInt(evTotal)} items)`} description={`Grouped by cancer context first, then therapy. The same variant can be sensitizing in one cancer and irrelevant in another — contexts are never merged. ${EVIDENCE_PAGE_SIZE} items per page; a cancer group may continue on the next page.`}>
71 + {evidence.length ? (
72 + <>
73 + <EvidenceTable
74 + items={evidence}
75 + prov={prov}
76 + groupBy="cancer"
77 + summary={
78 + <>
79 + Showing {fmtInt(ev.from)}–{fmtInt(ev.to)} of {fmtInt(evTotal)} evidence items across {contexts} cancer context{contexts === 1 ? '' : 's'} on this page
80 + </>
81 + }
82 + />
83 + <Pager total={evTotal} pageSize={ev.pageSize} page={ev.page} hrefFor={hrefFor} label="Evidence pages" noun="evidence items" />
84 + </>
86 85 ) : (
87 86 <EmptyState compact>No curated evidence item references this variant yet.</EmptyState>
88 87 )}
@@ -90,43 +89,49 @@ export default async function VariantPage({ params }: { params: Promise<{ slug:
90 89
91 90 <Section id="clinvar" kicker="ClinVar" title={`Clinical significance (${fmtInt(clinsig.length)})`} description="ClinVar interpretations are shown as structured records — significance, review status, star rating, conditions — never flattened into one word.">
92 91 {clinsig.length ? (
93 − <div className="ci-table-wrap">
94 − <table className="ci-table">
95 − <thead>
96 − <tr>
97 − <th>Variation</th>
98 − <th>Clinical significance</th>
99 − <th>Review status</th>
100 − <th className="num">Stars</th>
101 − <th>Conditions</th>
102 − <th>Origin</th>
103 − <th className="num">Submitters</th>
104 − <th>Last evaluated</th>
105 − <th>Source</th>
106 − </tr>
107 − </thead>
108 − <tbody>
109 − {clinsig.map((c) => (
110 − <tr key={c.id}>
111 − <td className="ci-mono">{c.clinvar_variation_id}</td>
112 − <td className="font-medium">{c.clinical_significance}</td>
113 − <td className="text-[12.5px]">{c.review_status ?? '—'}</td>
114 − <td className="num">{c.star_rating ?? '—'}</td>
115 − <td className="max-w-[300px] text-[12.5px]">{c.conditions.join('; ') || '—'}</td>
116 − <td className="text-[12.5px]">{c.origin_simple ?? '—'}</td>
117 − <td className="num">{c.number_submitters ?? '—'}</td>
118 − <td className="whitespace-nowrap text-[12.5px]">{c.last_evaluated ?? '—'}</td>
119 − <td>
120 − <span className="inline-flex gap-1">
121 − <SourceBadge p={toInfo(prov.get(c.provenance_id)) ?? { sourceSlug: 'clinvar', sourceName: 'ClinVar' }} />
122 − <ClaimBadge kind="curated" />
123 − </span>
124 − </td>
92 + <>
93 + <TableProvenance p={clinProv} claim={<ClaimBadge kind="curated" />}>
94 + {clinsig.length} interpretation{clinsig.length === 1 ? '' : 's'} · review status and star rating as assigned by ClinVar.
95 + </TableProvenance>
96 + <div className="ci-table-wrap">
97 + <table className="ci-table">
98 + <thead>
99 + <tr>
100 + <th scope="col">Variation</th>
101 + <th scope="col">Clinical significance</th>
102 + <th scope="col">Review status</th>
103 + <th scope="col" className="num">
104 + Stars
105 + </th>
106 + <th scope="col">Conditions</th>
107 + <th scope="col">Origin</th>
108 + <th scope="col" className="num">
109 + Submitters
110 + </th>
111 + <th scope="col">Last evaluated</th>
112 + <th scope="col">Source</th>
125 113 </tr>
126 − ))}
127 − </tbody>
128 − </table>
129 − </div>
114 + </thead>
115 + <tbody>
116 + {clinsig.map((c) => (
117 + <tr key={c.id}>
118 + <td className="ci-mono">{c.clinvar_variation_id}</td>
119 + <td className="font-medium">{c.clinical_significance}</td>
120 + <td className="text-[12.5px]">{c.review_status ?? '—'}</td>
121 + <td className="num">{c.star_rating ?? '—'}</td>
122 + <td className="max-w-[300px] text-[12.5px]">{c.conditions.join('; ') || '—'}</td>
123 + <td className="text-[12.5px]">{c.origin_simple ?? '—'}</td>
124 + <td className="num">{c.number_submitters ?? '—'}</td>
125 + <td className="whitespace-nowrap text-[12.5px]">{c.last_evaluated ?? '—'}</td>
126 + <td>
127 + <SourceBadge compact title={null} p={toInfo(prov.get(c.provenance_id)) ?? CLINVAR} />
128 + </td>
129 + </tr>
130 + ))}
131 + </tbody>
132 + </table>
133 + </div>
134 + </>
130 135 ) : (
131 136 <EmptyState compact>No ClinVar interpretation attached to this variant.</EmptyState>
132 137 )}
added apps/web/src/components/cancer/key-figures.tsx +201 −0
@@ -0,0 +1,201 @@
1 +import Link from 'next/link';
2 +import type { ReactNode } from 'react';
3 +import { ClaimBadge, type ClaimKind } from '@/components/ui/badge';
4 +import { SourceBadge } from '@/components/ui/source-badge';
5 +import { Freshness } from '@/components/ui/freshness';
6 +import { EmptyState } from '@/components/ui/empty-state';
7 +import { latestFiguresFor, nearestRegistryAncestor, type LatestFigure } from '@/lib/queries/epidemiology';
8 +import { rankingsForCancers, pickLatestScopes, bestRankFor } from '@/lib/queries/rankings';
9 +import { loadProvenance, toInfo } from '@/lib/queries/provenance';
10 +import { fmtInt, fmtValue, unitLabel, scopeLabel, toDate } from '@/lib/format';
11 +import type { CancerBundle } from './load';
12 +
13 +/**
14 + * "Key figures" strip for the cancer overview (SPEC §43, §309). Every tile shows value + unit, the period,
15 + * a source badge and — when a current ranking snapshot covers the entity — "rank #n of m (scope)".
16 + * Registry figures come from the entity's own observations or, for entities below the top level, from the
17 + * nearest top-level ancestor with an explicit note. Nothing is estimated or extrapolated here.
18 + */
19 +
20 +interface Tile {
21 + key: string;
22 + label: string;
23 + value: ReactNode;
24 + unit?: string | null;
25 + period: string;
26 + source: ReactNode;
27 + claim: ClaimKind;
28 + rank?: { rank: number; eligible: number; scope: string; href: string } | null;
29 + note?: string | null;
30 + href?: string;
31 +}
32 +
33 +const RANK_METRIC_FOR: Record<string, string> = { mortality_count: 'mortality_count', incidence_count: 'incidence_count', as_mortality_rate: 'as_mortality_rate' };
34 +
35 +export async function KeyFigures({ b }: { b: CancerBundle }) {
36 + const { cancer: c, counters } = b;
37 + const registry = c.top_level ? { id: c.id, slug: c.slug, canonical_name: c.canonical_name, depth: 0 } : await nearestRegistryAncestor(c.id);
38 + const rankIds = registry && registry.id !== c.id ? [c.id, registry.id] : [c.id];
39 + const [figures, ranksAll] = await Promise.all([registry ? latestFiguresFor([registry.id], 'USA', 'all') : Promise.resolve([] as LatestFigure[]), rankingsForCancers(rankIds)]);
40 + const ranks = pickLatestScopes(ranksAll);
41 + const prov = await loadProvenance(figures.map((f) => f.provenance_id));
42 + const byMetric = new Map(figures.map((f) => [f.metric, f]));
43 + const level: 'top' | 'all' = c.top_level ? 'top' : 'all';
44 +
45 + const rankOf = (metric: string, forId: string, geo?: string) => {
46 + const r = bestRankFor(
47 + ranks.filter((x) => x.cancer_id === forId),
48 + metric,
49 + { geo, level: forId === c.id ? level : 'top' },
50 + );
51 + return r ? { rank: r.rank, eligible: r.eligible_entities, scope: scopeLabel(r.scope_key), href: `/rankings/${r.metric_slug}?scope=${encodeURIComponent(r.scope_key)}` } : null;
52 + };
53 +
54 + const tiles: Tile[] = [];
55 + const epiNote = registry && registry.depth > 0 ? `figures shown for ${registry.canonical_name} (registry level)` : null;
56 + const epi: Array<[string, string]> = [
57 + ['mortality_count', 'US deaths'],
58 + ['incidence_count', 'US new cases'],
59 + ['as_mortality_rate', 'US age-standardized mortality'],
60 + ];
61 + for (const [metric, label] of epi) {
62 + const f = byMetric.get(metric);
63 + if (!f) continue;
64 + const p = toInfo(prov.get(f.provenance_id), 'normalized') ?? { sourceSlug: f.source_slug, sourceName: f.source_name };
65 + tiles.push({
66 + key: metric,
67 + label,
68 + value: fmtValue(f.value, f.unit),
69 + unit: f.unit,
70 + period: `${f.geography_name} · ${f.year_end && f.year_end !== f.year ? `${f.year}–${f.year_end}` : f.year} · ${f.sex === 'all' ? 'both sexes' : f.sex} · all ages${f.standard_population ? ` · ${f.standard_population.replace(/\s*\(.*\)$/, '')}` : ''}${f.estimate_type !== 'observed' ? ` · ${f.estimate_type}` : ''}`,
71 + source: <SourceBadge p={p} />,
72 + claim: 'observed',
73 + rank: registry ? rankOf(RANK_METRIC_FOR[metric] ?? metric, registry.id, f.iso3 ?? undefined) : null,
74 + note: epiNote,
75 + href: registry && registry.depth > 0 ? `/cancer/${registry.slug}/statistics` : `/cancer/${c.slug}/statistics`,
76 + });
77 + }
78 + const registryTilesMissing = tiles.length === 0;
79 +
80 + if (counters) {
81 + const src = (slug: string, name: string) => <SourceBadge p={{ sourceSlug: slug, sourceName: name, layer: 'derived', note: 'Counter computed by CancerIndex from ingested records; aggregates the entity and its descendants.' }} />;
82 + tiles.push(
83 + { key: 'active_trials', label: 'Active trials', value: fmtInt(counters.active_trial_count), unit: 'count', period: 'current registry status · entity + descendants', source: src('clinicaltrials', 'ClinicalTrials.gov'), claim: 'computed', rank: rankOf('active_trials', c.id), href: `/cancer/${c.slug}/trials` },
84 + { key: 'recruiting_trials', label: 'Recruiting trials', value: fmtInt(counters.recruiting_trial_count), unit: 'count', period: 'status RECRUITING · entity + descendants', source: src('clinicaltrials', 'ClinicalTrials.gov'), claim: 'computed', rank: rankOf('recruiting_trials', c.id), href: `/cancer/${c.slug}/trials?status=RECRUITING` },
85 + { key: 'evidence', label: 'Curated evidence items', value: fmtInt(counters.evidence_count), unit: 'count', period: 'accepted CIViC items · entity + descendants', source: src('civic', 'CIViC'), claim: 'computed', rank: rankOf('curated_evidence_items', c.id), href: `/cancer/${c.slug}/evidence` },
86 + { key: 'pubs12m', label: 'Publications, last 12 months', value: fmtInt(counters.publication_count_12m), unit: 'count', period: 'PubMed records · stored query · 12-month window', source: src('pubmed', 'PubMed'), claim: 'computed', rank: rankOf('publications_12m', c.id), href: `/cancer/${c.slug}/research` },
87 + { key: 'cohorts', label: 'Genomic cohorts', value: fmtInt(counters.cohort_count), unit: 'count', period: 'open GDC projects · entity + descendants', source: src('gdc', 'NCI GDC'), claim: 'computed', rank: rankOf('genomic_cohorts', c.id), href: `/cancer/${c.slug}/genomics` },
88 + );
89 + }
90 +
91 + const freshest = [...figures.map((f) => toDate(f.updated_at)), counters ? toDate(counters.updated_at) : null].filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? null;
92 +
93 + return (
94 + <section id="key-figures" aria-labelledby="key-figures-title" className="ci-rule pt-5">
95 + <div className="mb-3 flex flex-wrap items-end justify-between gap-2">
96 + <div>
97 + <p className="ci-kicker mb-1">Key figures</p>
98 + <h2 id="key-figures-title" className="text-xl sm:text-2xl">
99 + At a glance
100 + </h2>
101 + </div>
102 + <p className="text-[12px] text-ink-3">Each figure carries its unit, period and source; ranks link to "Why this rank?".</p>
103 + </div>
104 + {tiles.length === 0 ? (
105 + <EmptyState compact title="No key figure yet">
106 + No registry observation, trial, evidence, literature or cohort counter is attached to this entity. Counters are refreshed after each connector run.
107 + </EmptyState>
108 + ) : (
109 + <>
110 + {registryTilesMissing ? (
111 + <p className="mb-3 border-l-2 border-rule-strong pl-3 text-[12.5px] text-ink-3">
112 + {registry
113 + ? `No US registry observation for ${registry.depth > 0 ? `${registry.canonical_name} (nearest registry-level ancestor)` : 'this top-level site'} yet.`
114 + : 'No registry-level ancestor: burden figures are published for the mutually exclusive top-level site groups only, and this entity does not descend from one.'}{' '}
115 + Global figures (IARC / GLOBOCAN) stay under license review and SEER awaits credentials.
116 + </p>
117 + ) : null}
118 + <ul className="grid grid-cols-1 gap-px bg-rule sm:grid-cols-2 lg:grid-cols-4">
119 + {tiles.map((t) => (
120 + <li key={t.key} className="flex min-w-0 flex-col bg-paper px-3 py-2.5">
121 + <span className="text-[11.5px] font-medium uppercase tracking-wide text-ink-3">{t.label}</span>
122 + <span className="mt-0.5 flex flex-wrap items-baseline gap-x-1.5">
123 + <span className="ci-num font-display text-2xl text-ink">{t.value}</span>
124 + {t.unit ? <span className="text-[11.5px] text-ink-3">{unitLabel(t.unit)}</span> : null}
125 + </span>
126 + <span className="mt-0.5 text-[11.5px] leading-4 text-ink-3">{t.period}</span>
127 + <span className="mt-1 flex flex-wrap items-center gap-1.5">
128 + {t.source}
129 + <ClaimBadge kind={t.claim} />
130 + </span>
131 + {t.rank ? (
132 + <Link href={t.rank.href} className="ci-link mt-1 text-[12px]">
133 + rank #{t.rank.rank} of {fmtInt(t.rank.eligible)} <span className="text-ink-3">({t.rank.scope})</span>
134 + </Link>
135 + ) : (
136 + <span className="mt-1 text-[11.5px] text-ink-4">not in a current ranking snapshot</span>
137 + )}
138 + {t.note ? (
139 + <span className="mt-1 text-[11.5px] italic text-warn">
140 + {t.note}
141 + {registry && registry.depth > 0 ? (
142 + <>
143 + {' '}
144 + —{' '}
145 + <Link className="ci-link" href={`/cancer/${registry.slug}`}>
146 + open {registry.canonical_name}
147 + </Link>
148 + </>
149 + ) : null}
150 + </span>
151 + ) : null}
152 + </li>
153 + ))}
154 + </ul>
155 + <Freshness dataUpdatedAt={freshest} extra={`registry figures: ${figures.length ? `${figures[0]!.source_slug} · latest year available per metric` : 'none'} · counters aggregate over descendants`} />
156 + </>
157 + )}
158 + </section>
159 + );
160 +}
161 +
162 +export const COMPLETENESS_DIMENSIONS: Array<{ key: string; label: string; hint: string }> = [
163 + { key: 'epidemiology', label: 'Epidemiology', hint: 'At least one incidence/mortality observation attached to this entity' },
164 + { key: 'survival', label: 'Survival', hint: 'At least one survival observation' },
165 + { key: 'trials', label: 'Trials', hint: 'At least one ClinicalTrials.gov study mapped to the entity or a descendant' },
166 + { key: 'literature', label: 'Literature', hint: 'At least one PubMed record linked' },
167 + { key: 'genomics', label: 'Genomics', hint: 'At least one open genomic cohort (GDC project)' },
168 + { key: 'evidence', label: 'Evidence', hint: 'At least one accepted CIViC evidence item' },
169 + { key: 'therapies', label: 'Therapies', hint: 'At least one drug with a regulatory approval for the entity or a descendant' },
170 +];
171 +
172 +/** Data completeness row: the 7 dimensions stored in entity_counters.completeness as filled/empty squares with labels. */
173 +export function CompletenessRow({ completeness, computedAt }: { completeness: Record<string, number> | null | undefined; computedAt?: Date | string | null }) {
174 + const has = completeness && Object.keys(completeness).length > 0;
175 + const filled = COMPLETENESS_DIMENSIONS.filter((d) => has && Number(completeness![d.key] ?? 0) > 0).length;
176 + return (
177 + <div className="mt-3 border-t border-rule pt-2" aria-label={`Data completeness: ${filled} of ${COMPLETENESS_DIMENSIONS.length} dimensions`}>
178 + <p className="flex flex-wrap items-baseline justify-between gap-2 text-[12px] text-ink-3">
179 + <span className="ci-kicker">Data completeness</span>
180 + <span>
181 + <span className="ci-num text-ink">{filled}</span> / {COMPLETENESS_DIMENSIONS.length} dimensions{has ? '' : ' · counters not computed yet'}
182 + </span>
183 + </p>
184 + <ul className="mt-1.5 flex flex-wrap gap-x-4 gap-y-1.5 text-[12px]">
185 + {COMPLETENESS_DIMENSIONS.map((d) => {
186 + const on = has && Number(completeness![d.key] ?? 0) > 0;
187 + return (
188 + <li key={d.key} className="inline-flex items-center gap-1.5" title={d.hint}>
189 + <span aria-hidden className={`inline-block h-[10px] w-[10px] border ${on ? 'border-accent bg-accent' : 'border-rule-strong bg-transparent'}`} />
190 + <span className={on ? 'text-ink' : 'text-ink-3'}>
191 + {d.label}
192 + <span className="sr-only">{on ? ' (data present)' : ' (no data yet)'}</span>
193 + </span>
194 + </li>
195 + );
196 + })}
197 + </ul>
198 + {computedAt ? <p className="mt-1 text-[11px] text-ink-4">Computed from entity_counters.completeness · refreshed after each connector run</p> : null}
199 + </div>
200 + );
201 +}
modified apps/web/src/components/cancer/tabs/drugs.tsx +16 −22
@@ -5,31 +5,19 @@ import { Freshness } from '@/components/ui/freshness';
5 5 import { Badge } from '@/components/ui/badge';
6 6 import { ApprovalsTable } from '@/components/data/approvals-table';
7 7 import { approvalsForCancer } from '@/lib/queries/drugs';
8 −import { evidenceForCancer } from '@/lib/queries/evidence';
8 +import { therapyMentionsForCancer } from '@/lib/queries/evidence';
9 9 import { loadProvenance } from '@/lib/queries/provenance';
10 10 import { fmtInt } from '@/lib/format';
11 11 import type { CancerBundle } from '../load';
12 12
13 13 export async function DrugsTab({ b, jurisdiction }: { b: CancerBundle; jurisdiction: string | null }) {
14 − const [approvals, evidence] = await Promise.all([approvalsForCancer(b.descendants), evidenceForCancer(b.descendants)]);
14 + // Therapies mentioned in curated evidence (not approvals) — aggregated in SQL (no evidence rows
15 + // are shipped to the page), listed separately and never called "approved".
16 + const [approvals, therapies] = await Promise.all([approvalsForCancer(b.descendants), therapyMentionsForCancer(b.descendants)]);
15 17 const jurisdictions = [...new Set(approvals.map((a) => a.jurisdiction))].sort();
16 18 const selected = jurisdiction && jurisdictions.includes(jurisdiction) ? jurisdiction : null;
17 19 const shownApprovals = selected ? approvals.filter((a) => a.jurisdiction === selected) : approvals;
18 20
19 − // Therapies mentioned in curated evidence (not approvals) — listed separately, never called "approved".
20 − const therapyMap = new Map<string, { slug: string; name: string; n: number; sensitivity: number; resistance: number }>();
21 − for (const e of evidence) {
22 − // therapy_slugs and therapy_slug_names share one ORDER BY; therapy_ids/therapy_names follow CIViC's order and must not be zipped with them.
23 − (e.therapy_slugs ?? []).forEach((slug, i) => {
24 − const cur = therapyMap.get(slug) ?? { slug, name: e.therapy_slug_names?.[i] ?? slug, n: 0, sensitivity: 0, resistance: 0 };
25 − cur.n += 1;
26 − if ((e.significance ?? '').includes('SENSITIV')) cur.sensitivity += 1;
27 − if ((e.significance ?? '').includes('RESIST')) cur.resistance += 1;
28 − therapyMap.set(slug, cur);
29 − });
30 − }
31 − const therapies = [...therapyMap.values()].sort((a, c) => c.n - a.n);
32 −
33 21 if (approvals.length === 0 && therapies.length === 0) {
34 22 return (
35 23 <Section id="drugs" kicker="Drugs" title="Regulatory approvals and therapies in evidence">
@@ -46,11 +34,11 @@ export async function DrugsTab({ b, jurisdiction }: { b: CancerBundle; jurisdict
46 34 {approvals.length ? (
47 35 <>
48 36 <nav aria-label="Jurisdiction" className="mb-3 flex flex-wrap gap-1.5 text-[12.5px]">
49 − <Link href={`/cancer/${b.cancer.slug}/drugs`} aria-current={!selected ? 'page' : undefined} className={`border px-2 py-0.5 no-underline ${!selected ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
37 + <Link href={`/cancer/${b.cancer.slug}/drugs`} aria-current={!selected ? 'page' : undefined} className="ci-chip">
50 38 All jurisdictions
51 39 </Link>
52 40 {jurisdictions.map((j) => (
53 − <Link key={j} href={`/cancer/${b.cancer.slug}/drugs?jurisdiction=${j}`} aria-current={selected === j ? 'page' : undefined} className={`ci-mono border px-2 py-0.5 no-underline ${selected === j ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
41 + <Link key={j} href={`/cancer/${b.cancer.slug}/drugs?jurisdiction=${encodeURIComponent(j)}`} aria-current={selected === j ? 'page' : undefined} className="ci-chip ci-mono">
54 42 {j}
55 43 </Link>
56 44 ))}
@@ -69,10 +57,16 @@ export async function DrugsTab({ b, jurisdiction }: { b: CancerBundle; jurisdict
69 57 <table className="ci-table">
70 58 <thead>
71 59 <tr>
72 − <th>Therapy</th>
73 − <th className="num">Evidence items (count)</th>
74 − <th className="num">Sensitivity / response</th>
75 − <th className="num">Resistance</th>
60 + <th scope="col">Therapy</th>
61 + <th scope="col" className="num">
62 + Evidence items (count)
63 + </th>
64 + <th scope="col" className="num">
65 + Sensitivity / response
66 + </th>
67 + <th scope="col" className="num">
68 + Resistance
69 + </th>
76 70 </tr>
77 71 </thead>
78 72 <tbody>
modified apps/web/src/components/cancer/tabs/evidence.tsx +23 −8
@@ -1,15 +1,17 @@
1 1 import { Section, Note } from '@/components/ui/section';
2 2 import { EmptyState } from '@/components/ui/empty-state';
3 3 import { Freshness } from '@/components/ui/freshness';
4 +import { Pager } from '@/components/ui/pager';
4 5 import { EvidenceTable } from '@/components/data/evidence-table';
5 −import { evidenceForCancer } from '@/lib/queries/evidence';
6 +import { evidenceForCancer, evidenceForCancerCount, EVIDENCE_PAGE_SIZE } from '@/lib/queries/evidence';
6 7 import { loadProvenance } from '@/lib/queries/provenance';
7 8 import { fmtInt } from '@/lib/format';
9 +import { pageInfo } from '@/lib/pagination';
8 10 import type { CancerBundle } from '../load';
9 11
10 −export async function EvidenceTab({ b }: { b: CancerBundle }) {
11 − const items = await evidenceForCancer(b.descendants);
12 − if (items.length === 0) {
12 +export async function EvidenceTab({ b, evPage = 1 }: { b: CancerBundle; evPage?: number }) {
13 + const total = await evidenceForCancerCount(b.descendants);
14 + if (total === 0) {
13 15 return (
14 16 <Section id="evidence" kicker="Variants & evidence" title="Curated clinical evidence by variant and therapy">
15 17 <EmptyState knows={[{ label: 'Genomics tab', href: `/cancer/${b.cancer.slug}/genomics` }, { label: 'Drugs tab', href: `/cancer/${b.cancer.slug}/drugs` }]}>
@@ -18,14 +20,27 @@ export async function EvidenceTab({ b }: { b: CancerBundle }) {
18 20 </Section>
19 21 );
20 22 }
23 + const info = pageInfo(evPage, EVIDENCE_PAGE_SIZE, total);
24 + const items = await evidenceForCancer(b.descendants, { page: info.page, pageSize: info.pageSize });
21 25 const prov = await loadProvenance(items.map((e) => e.provenance_id));
22 − const accepted = items.filter((e) => e.status === 'ACCEPTED').length;
23 26 const descendantItems = items.filter((e) => e.cancer_id !== b.cancer.id).length;
24 − const latest = items.reduce((m, e) => (e.updated_at > m ? e.updated_at : m), items[0]!.updated_at);
27 + const latest = items.reduce<Date | string | null>((m, e) => (m == null || String(e.updated_at) > String(m) ? e.updated_at : m), null);
28 + const hrefFor = (p: number) => `/cancer/${b.cancer.slug}/evidence${p > 1 ? `?evPage=${p}` : ''}`;
25 29 return (
26 30 <div className="space-y-6">
27 − <Section id="evidence" kicker="Variants & evidence" title="Curated clinical evidence by variant and therapy" description={`${fmtInt(items.length)} evidence items (${fmtInt(accepted)} accepted${descendantItems ? `, ${fmtInt(descendantItems)} mapped to a descendant entity` : ''}). Grouped by molecular profile, then therapy.`}>
28 − <EvidenceTable items={items} prov={prov} showCancer={descendantItems > 0} />
31 + <Section id="evidence" kicker="Variants & evidence" title="Curated clinical evidence by variant and therapy" description={`${fmtInt(total)} evidence items mapped to this entity or its descendants, grouped by molecular profile, then therapy. ${EVIDENCE_PAGE_SIZE} items per page.`}>
32 + <EvidenceTable
33 + items={items}
34 + prov={prov}
35 + showCancer={descendantItems > 0}
36 + summary={
37 + <>
38 + Showing {fmtInt(info.from)}–{fmtInt(info.to)} of {fmtInt(total)} evidence items
39 + {descendantItems ? ` (${fmtInt(descendantItems)} on this page mapped to a descendant entity)` : ''}
40 + </>
41 + }
42 + />
43 + <Pager total={total} pageSize={info.pageSize} page={info.page} hrefFor={hrefFor} label="Evidence pages" noun="evidence items" />
29 44 <Freshness dataUpdatedAt={latest} extra="source: civic (CC0)" />
30 45 </Section>
31 46 <Note tone="warn">Evidence levels, directions and ratings are those assigned by CIViC curators. "Submitted" items have not completed curation review. This is not treatment guidance.</Note>
modified apps/web/src/components/cancer/tabs/genomics.tsx +51 −17
@@ -2,14 +2,18 @@ import Link from 'next/link';
2 2 import { Section, Note } from '@/components/ui/section';
3 3 import { EmptyState } from '@/components/ui/empty-state';
4 4 import { FrequencyTables } from '@/components/data/frequency-table';
5 −import { frequenciesForCancer } from '@/lib/queries/genomics';
5 +import { cohortsForCancer, frequenciesForCancer } from '@/lib/queries/genomics';
6 6 import { loadProvenance } from '@/lib/queries/provenance';
7 7 import { fmtInt } from '@/lib/format';
8 8 import type { CancerBundle } from '../load';
9 9
10 +/**
11 + * Cohort selector renders ONE cohort by default — the one with the largest number of cases
12 + * profiled — plus an explicit "All cohorts" option (`?cohort=all`). Cohorts are never pooled.
13 + */
10 14 export async function GenomicsTab({ b, cohort }: { b: CancerBundle; cohort: string | null }) {
11 − const rows = await frequenciesForCancer(b.descendants);
12 − if (rows.length === 0) {
15 + const cohorts = await cohortsForCancer(b.descendants);
16 + if (cohorts.length === 0) {
13 17 return (
14 18 <Section id="genomics" kicker="Genomics" title="Gene alteration frequencies by cohort">
15 19 <EmptyState knows={[{ label: 'Variants & evidence', href: `/cancer/${b.cancer.slug}/evidence` }, { label: 'Genes index', href: '/genes' }]}>
@@ -18,24 +22,54 @@ export async function GenomicsTab({ b, cohort }: { b: CancerBundle; cohort: stri
18 22 </Section>
19 23 );
20 24 }
25 + const withRows = cohorts.filter((c) => c.frequency_rows > 0);
26 + const defaultCohort = (withRows[0] ?? cohorts[0])!;
27 + const showAll = cohort === 'all';
28 + const selected = showAll ? null : cohorts.find((c) => c.cohort_id === cohort || c.study_id === cohort) ?? defaultCohort;
29 + const rows = await frequenciesForCancer(b.descendants, { cohortId: selected?.cohort_id ?? null });
21 30 const prov = await loadProvenance(rows.map((r) => r.provenance_id));
22 − const cohorts = [...new Map(rows.map((r) => [r.cohort_id, r])).values()];
23 − const selected = cohort && cohorts.some((c) => c.cohort_id === cohort) ? cohort : null;
31 + const base = `/cancer/${b.cancer.slug}/genomics`;
32 + const hrefFor = (c: (typeof cohorts)[number]) => (c.cohort_id === defaultCohort.cohort_id ? base : `${base}?cohort=${encodeURIComponent(c.study_id)}`);
33 +
24 34 return (
25 35 <div className="space-y-6">
26 − <Section id="genomics" kicker="Genomics" title="Gene alteration frequencies by cohort" description={`${fmtInt(cohorts.length)} cohort${cohorts.length === 1 ? '' : 's'} mapped to this entity or its descendants. Each frequency is affected / profiled within one cohort; cohorts are shown separately because case selection, sequencing depth and definitions differ.`}>
27 − <nav aria-label="Cohort" className="mb-4 flex flex-wrap gap-1.5 text-[12.5px]">
28 − <Link href={`/cancer/${b.cancer.slug}/genomics`} aria-current={!selected ? 'page' : undefined} className={`border px-2 py-0.5 no-underline ${!selected ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
29 − All cohorts
30 − </Link>
31 − {cohorts.map((c) => (
32 − <Link key={c.cohort_id} href={`/cancer/${b.cancer.slug}/genomics?cohort=${encodeURIComponent(c.cohort_id)}`} aria-current={selected === c.cohort_id ? 'page' : undefined} className={`border px-2 py-0.5 no-underline ${selected === c.cohort_id ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
33 − {c.study_id}
34 − {c.cancer_name && c.cancer_id !== b.cancer.id ? <span className="ml-1 text-ink-3">({c.cancer_name})</span> : null}
35 − </Link>
36 − ))}
36 + <Section
37 + id="genomics"
38 + kicker="Genomics"
39 + title="Gene alteration frequencies by cohort"
40 + description={`${fmtInt(cohorts.length)} cohort${cohorts.length === 1 ? '' : 's'} mapped to this entity or its descendants. Each frequency is affected / profiled within one cohort; cohorts are shown separately because case selection, sequencing depth and definitions differ. By default the cohort with the most cases profiled is shown.`}
41 + >
42 + <nav aria-label="Cohort" className="mb-4">
43 + <ul className="m-0 flex list-none flex-wrap gap-1.5 p-0 text-[12.5px]">
44 + {cohorts.map((c) => {
45 + const isCurrent = !showAll && selected?.cohort_id === c.cohort_id;
46 + return (
47 + <li key={c.cohort_id}>
48 + <Link href={hrefFor(c)} aria-current={isCurrent ? 'page' : undefined} className="ci-chip" title={`${c.cohort_name} · ${fmtInt(c.cases_profiled)} cases profiled · ${fmtInt(c.frequency_rows)} gene rows`}>
49 + {c.study_id}
50 + <span className="ci-num ml-1 text-ink-3">{fmtInt(c.cases_profiled)}</span>
51 + {c.cancer_name && c.cancer_id !== b.cancer.id ? <span className="ml-1 text-ink-3">({c.cancer_name})</span> : null}
52 + </Link>
53 + </li>
54 + );
55 + })}
56 + {cohorts.length > 1 ? (
57 + <li>
58 + <Link href={`${base}?cohort=all`} aria-current={showAll ? 'page' : undefined} className="ci-chip">
59 + All cohorts
60 + </Link>
61 + </li>
62 + ) : null}
63 + </ul>
64 + <p className="mt-1 text-[11.5px] text-ink-3">Chip number = cases profiled in that cohort.</p>
37 65 </nav>
38 − <FrequencyTables rows={rows} prov={prov} cohortFilter={selected} showCancer />
66 + {rows.length ? (
67 + <FrequencyTables rows={rows} prov={prov} showCancer />
68 + ) : (
69 + <EmptyState compact title="No frequency rows for this cohort">
70 + The cohort is mapped to this entity but no gene frequency has been ingested for it yet.
71 + </EmptyState>
72 + )}
39 73 </Section>
40 74 <Note>Cohorts mapped to a descendant (e.g. a subtype) are included because their cases belong to this entity by definition; the mapping and its match type are shown per cohort.</Note>
41 75 </div>
modified apps/web/src/components/cancer/tabs/overview.tsx +11 −2
@@ -7,6 +7,7 @@ import { Freshness } from '@/components/ui/freshness';
7 7 import { CompletenessDots } from '@/components/ui/completeness';
8 8 import { loadProvenance, toInfo } from '@/lib/queries/provenance';
9 9 import { fmtInt, humanize, isoDate } from '@/lib/format';
10 +import { KeyFigures, CompletenessRow } from '../key-figures';
10 11 import type { CancerBundle } from '../load';
11 12
12 13 export async function OverviewTab({ b }: { b: CancerBundle }) {
@@ -26,15 +27,22 @@ export async function OverviewTab({ b }: { b: CancerBundle }) {
26 27 ];
27 28
28 29 return (
30 + <div className="space-y-8">
31 + {/* Key figures strip (§43, §309): full width, above the definition. */}
32 + <div>
33 + <KeyFigures b={b} />
34 + <CompletenessRow completeness={counters?.completeness} computedAt={counters?.updated_at} />
35 + </div>
29 36 <div className="grid gap-8 lg:grid-cols-[1.5fr_1fr]">
30 37 <div className="space-y-8">
31 38 <Section id="definition" kicker="Definition" title="What this entity is">
32 39 {c.description ? (
33 40 <>
34 41 <p className="max-w-3xl text-[15px] leading-relaxed text-ink">{c.description}</p>
35 − <p className="mt-2 flex flex-wrap items-center gap-1.5 text-[12px] text-ink-3">
42 + {/* div, not p: the SourceBadge popover contains a <dl>, which the HTML parser uses to close an open <p> → hydration mismatch (React #418). */}
43 + <div className="mt-2 flex flex-wrap items-center gap-1.5 text-[12px] text-ink-3">
36 44 Definition text as published by {descProv ? <SourceBadge p={descProv} /> : 'the source terminology'} <ClaimBadge kind="curated" />
37 − </p>
45 + </div>
38 46 <Freshness dataUpdatedAt={prov.get(c.description_provenance_id ?? -1)?.retrieved_at ?? c.updated_at} sourceVersion={prov.get(c.description_provenance_id ?? -1)?.dataset_version ?? c.classification_version} />
39 47 </>
40 48 ) : (
@@ -192,5 +200,6 @@ export async function OverviewTab({ b }: { b: CancerBundle }) {
192 200 <Note>CancerIndex is a research and information platform. It does not diagnose and does not recommend treatment.</Note>
193 201 </aside>
194 202 </div>
203 + </div>
195 204 );
196 205 }
modified apps/web/src/components/cancer/tabs/rankings.tsx +20 −20
@@ -5,6 +5,7 @@ import { ClaimBadge, ConfidenceBadge } from '@/components/ui/badge';
5 5 import { JsonView } from '@/components/ui/json-view';
6 6 import { rankingsForCancer } from '@/lib/queries/rankings';
7 7 import { fmtDate, fmtInt, fmtValue, scopeLabel, unitLabel } from '@/lib/format';
8 +import { latestYearPerScope } from '@/lib/rankings-util';
8 9 import type { CancerBundle } from '../load';
9 10
10 11 function Delta({ rank, prev }: { rank: number; prev: number | null }) {
@@ -21,16 +22,7 @@ function Delta({ rank, prev }: { rank: number; prev: number | null }) {
21 22 export async function RankingsTab({ b }: { b: CancerBundle }) {
22 23 const allRows = await rankingsForCancer(b.cancer.id);
23 24 // One row per (metric, geography, sex, age, level): the latest year only. Historical years stay on the ranking pages.
24 − const latestByScope = new Map<string, (typeof allRows)[number]>();
25 − for (const r of allRows) {
26 − const year = Number(/year=(\d{4})/.exec(r.scope_key)?.[1] ?? 0);
27 − const key = `${r.metric_slug}|${r.scope_key.replace(/year=[^|]*/, '')}`;
28 − const cur = latestByScope.get(key);
29 − const curYear = cur ? Number(/year=(\d{4})/.exec(cur.scope_key)?.[1] ?? 0) : -1;
30 − if (!cur || year > curYear) latestByScope.set(key, r);
31 − }
32 − const rows = [...latestByScope.values()];
33 − const hiddenYears = allRows.length - rows.length;
25 + const { rows, hidden: hiddenYears } = latestYearPerScope(allRows);
34 26 if (rows.length === 0) {
35 27 return (
36 28 <Section id="rankings" kicker="Rankings" title="Why this rank?">
@@ -47,16 +39,24 @@ export async function RankingsTab({ b }: { b: CancerBundle }) {
47 39 <table className="ci-table">
48 40 <thead>
49 41 <tr>
50 − <th>Metric</th>
51 − <th>Scope</th>
52 − <th className="num">Rank / eligible</th>
53 − <th className="num">Δ</th>
54 − <th className="num">Value</th>
55 − <th className="num">Percentile</th>
56 − <th>Confidence</th>
57 − <th>Formula</th>
58 − <th>Generated</th>
59 − <th>Inputs</th>
42 + <th scope="col">Metric</th>
43 + <th scope="col">Scope</th>
44 + <th scope="col" className="num">
45 + Rank / eligible
46 + </th>
47 + <th scope="col" className="num">
48 + <abbr title="Change versus the previous snapshot">Δ</abbr>
49 + </th>
50 + <th scope="col" className="num">
51 + Value
52 + </th>
53 + <th scope="col" className="num">
54 + Percentile
55 + </th>
56 + <th scope="col">Confidence</th>
57 + <th scope="col">Formula</th>
58 + <th scope="col">Generated</th>
59 + <th scope="col">Inputs</th>
60 60 </tr>
61 61 </thead>
62 62 <tbody>
modified apps/web/src/components/cancer/tabs/research.tsx +38 −17
@@ -2,33 +2,43 @@ import { Section } from '@/components/ui/section';
2 2 import { EmptyState } from '@/components/ui/empty-state';
3 3 import { Freshness } from '@/components/ui/freshness';
4 4 import { ClaimBadge } from '@/components/ui/badge';
5 −import { SourceBadge } from '@/components/ui/source-badge';
5 +import { SourceBadge, TableProvenance } from '@/components/ui/source-badge';
6 +import { Pager } from '@/components/ui/pager';
6 7 import { PublicationList } from '@/components/data/publication-list';
7 −import { literatureCountsFor, recentPublicationsFor } from '@/lib/queries/publications';
8 +import { literatureCountsFor, recentPublicationsFor, recentPublicationsForCount, PUBLICATION_PAGE_SIZE } from '@/lib/queries/publications';
8 9 import { loadProvenance, toInfo } from '@/lib/queries/provenance';
9 10 import { fmtInt, isoDate } from '@/lib/format';
11 +import { pageInfo } from '@/lib/pagination';
10 12 import type { CancerBundle } from '../load';
11 13
12 14 const WINDOW_LABEL: Record<string, string> = { all: 'All time', '10y': 'Last 10 years', '5y': 'Last 5 years', '5y_prior': 'Preceding 5-year window', '12m': 'Last 12 months' };
13 15
14 −export async function ResearchTab({ b }: { b: CancerBundle }) {
15 − const [counts, pubs] = await Promise.all([literatureCountsFor(b.cancer.id), recentPublicationsFor('cancer', b.descendants, 30)]);
16 − const prov = await loadProvenance(counts.map((c) => c.provenance_id));
16 +export async function ResearchTab({ b, pPage = 1 }: { b: CancerBundle; pPage?: number }) {
17 + const [counts, pubTotal] = await Promise.all([literatureCountsFor(b.cancer.id), recentPublicationsForCount('cancer', b.descendants)]);
18 + const info = pageInfo(pPage, PUBLICATION_PAGE_SIZE, pubTotal);
19 + const [pubs, prov] = await Promise.all([pubTotal > 0 ? recentPublicationsFor('cancer', b.descendants, { page: info.page, pageSize: info.pageSize }) : Promise.resolve([]), loadProvenance(counts.map((c) => c.provenance_id))]);
20 + const countsProv = counts[0] ? toInfo(prov.get(counts[0].provenance_id), 'derived') : null;
21 + const hrefFor = (p: number) => `/cancer/${b.cancer.slug}/research${p > 1 ? `?pPage=${p}` : ''}#recent-publications`;
17 22 return (
18 23 <div className="space-y-8">
19 24 <Section id="literature-counts" kicker="Research activity" title="PubMed record counts" description="Each count is the number of PubMed records returned by the exact query shown, at the time it was run. Counts are research-activity signals, not quality measures.">
20 25 {counts.length ? (
21 26 <>
27 + <TableProvenance p={countsProv ?? { sourceSlug: 'pubmed', sourceName: 'PubMed', layer: 'derived' }} claim={<ClaimBadge kind="computed" />}>
28 + {counts.length} window{counts.length === 1 ? '' : 's'} · counts computed by CancerIndex from PubMed E-utilities; the exact query is stored with each count.
29 + </TableProvenance>
22 30 <div className="ci-table-wrap">
23 31 <table className="ci-table">
24 32 <thead>
25 33 <tr>
26 − <th>Window</th>
27 − <th>Period</th>
28 − <th className="num">Records (count)</th>
29 − <th>Exact query</th>
30 − <th>Computed</th>
31 − <th>Source</th>
34 + <th scope="col">Window</th>
35 + <th scope="col">Period</th>
36 + <th scope="col" className="num">
37 + Records (count)
38 + </th>
39 + <th scope="col">Exact query</th>
40 + <th scope="col">Computed</th>
41 + <th scope="col">Source</th>
32 42 </tr>
33 43 </thead>
34 44 <tbody>
@@ -42,10 +52,7 @@ export async function ResearchTab({ b }: { b: CancerBundle }) {
42 52 </td>
43 53 <td className="whitespace-nowrap text-[12.5px]">{isoDate(c.updated_at)}</td>
44 54 <td>
45 − <span className="inline-flex gap-1">
46 − <SourceBadge p={toInfo(prov.get(c.provenance_id), 'derived') ?? { sourceSlug: 'pubmed', sourceName: 'PubMed' }} />
47 − <ClaimBadge kind="computed" />
48 − </span>
55 + <SourceBadge compact title={null} p={toInfo(prov.get(c.provenance_id), 'derived') ?? { sourceSlug: 'pubmed', sourceName: 'PubMed' }} />
49 56 </td>
50 57 </tr>
51 58 ))}
@@ -60,8 +67,22 @@ export async function ResearchTab({ b }: { b: CancerBundle }) {
60 67 </EmptyState>
61 68 )}
62 69 </Section>
63 − <Section id="recent-publications" kicker="Literature" title="Linked publications" description="Publications linked to this entity or its descendants. Link method and validation status are shown; 'candidate' links have not been reviewed.">
64 − {pubs.length ? <PublicationList rows={pubs} /> : <EmptyState compact>No publication is linked to this entity yet.</EmptyState>}
70 + <Section id="recent-publications" kicker="Literature" title={`Linked publications (${fmtInt(pubTotal)})`} description={`Publications linked to this entity or its descendants, newest first, ${PUBLICATION_PAGE_SIZE} per page. Link method and validation status are shown; 'candidate' links have not been reviewed.`}>
71 + {pubs.length ? (
72 + <>
73 + <PublicationList
74 + rows={pubs}
75 + summary={
76 + <>
77 + Showing {fmtInt(info.from)}–{fmtInt(info.to)} of {fmtInt(pubTotal)} publications
78 + </>
79 + }
80 + />
81 + <Pager total={pubTotal} pageSize={PUBLICATION_PAGE_SIZE} page={info.page} hrefFor={hrefFor} label="Publication pages" noun="publications" />
82 + </>
83 + ) : (
84 + <EmptyState compact>No publication is linked to this entity yet.</EmptyState>
85 + )}
65 86 </Section>
66 87 </div>
67 88 );
modified apps/web/src/components/cancer/tabs/statistics.tsx +33 −19
@@ -2,13 +2,15 @@ import { Section, Note } from '@/components/ui/section';
2 2 import { EmptyState } from '@/components/ui/empty-state';
3 3 import { Freshness } from '@/components/ui/freshness';
4 4 import { Badge, ClaimBadge } from '@/components/ui/badge';
5 −import { SourceBadge } from '@/components/ui/source-badge';
5 +import { SourceBadge, TableProvenance } from '@/components/ui/source-badge';
6 6 import { LineChart, type Series } from '@/components/charts/line-chart';
7 7 import { epidemiologyFor, EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
8 8 import { loadProvenance, toInfo } from '@/lib/queries/provenance';
9 9 import { fmtValue, humanize, unitLabel } from '@/lib/format';
10 10 import type { CancerBundle } from '../load';
11 11
12 +const ROWS_PER_TABLE = 12;
13 +
12 14 export async function StatisticsTab({ b }: { b: CancerBundle }) {
13 15 const obs = await epidemiologyFor(b.cancer.id);
14 16 if (obs.length === 0) {
@@ -37,7 +39,7 @@ export async function StatisticsTab({ b }: { b: CancerBundle }) {
37 39 if (!groups.has(k)) groups.set(k, { geography: o.geography_name, metric: o.metric, unit: o.unit, rows: [] });
38 40 groups.get(k)!.rows.push(o);
39 41 }
40 − const latest = obs.reduce((m, o) => (o.updated_at > m ? o.updated_at : m), obs[0]!.updated_at);
42 + const latest = obs.reduce<Date | string | null>((m, o) => (m == null || String(o.updated_at) > String(m) ? o.updated_at : m), null);
41 43
42 44 return (
43 45 <div className="space-y-8">
@@ -51,25 +53,36 @@ export async function StatisticsTab({ b }: { b: CancerBundle }) {
51 53 }
52 54 const series = [...seriesMap.values()];
53 55 const multiYear = series.some((s) => s.points.length > 1);
56 + const sources = new Set(g.rows.map((r) => r.source_slug));
57 + const firstProv = toInfo(prov.get(g.rows[0]!.provenance_id)) ?? { sourceSlug: g.rows[0]!.source_slug, sourceName: g.rows[0]!.source_name };
58 + const shown = [...g.rows].sort((a, c) => c.year - a.year || a.sex.localeCompare(c.sex)).slice(0, ROWS_PER_TABLE);
54 59 return (
55 60 <Section key={`${g.geography}-${g.metric}`} id={`${g.metric}-${g.geography}`} kicker={g.geography} title={EPI_METRIC_LABEL[g.metric] ?? humanize(g.metric)} description={`Unit: ${unitLabel(g.unit)}${g.rows[0]?.standard_population ? ` · standard population: ${g.rows[0].standard_population}` : ''}`}>
56 61 {multiYear ? <LineChart series={series} unit={g.unit} ariaLabel={`${EPI_METRIC_LABEL[g.metric] ?? g.metric} in ${g.geography} by year`} /> : null}
57 − <div className={`ci-table-wrap ${multiYear ? 'mt-4' : ''}`}>
62 + <TableProvenance className={multiYear ? 'mt-4' : ''} p={firstProv} claim={<ClaimBadge kind="observed" />}>
63 + {g.rows.length} observation{g.rows.length === 1 ? '' : 's'}
64 + {sources.size > 1 ? ` from ${sources.size} sources` : ''} · observations differ by dataset and year: hover a row badge for its dataset and version.
65 + </TableProvenance>
66 + <div className="ci-table-wrap">
58 67 <table className="ci-table">
59 68 <thead>
60 69 <tr>
61 − <th>Year</th>
62 − <th>Sex</th>
63 − <th>Age group</th>
64 − <th className="num">Value ({unitLabel(g.unit)})</th>
65 − <th className="num">95% CI</th>
66 − <th>Type</th>
67 − <th>Site definition</th>
68 − <th>Source</th>
70 + <th scope="col">Year</th>
71 + <th scope="col">Sex</th>
72 + <th scope="col">Age group</th>
73 + <th scope="col" className="num">
74 + Value ({unitLabel(g.unit)})
75 + </th>
76 + <th scope="col" className="num">
77 + 95% CI
78 + </th>
79 + <th scope="col">Type</th>
80 + <th scope="col">Site definition</th>
81 + <th scope="col">Source</th>
69 82 </tr>
70 83 </thead>
71 84 <tbody>
72 − {[...g.rows].sort((a, c) => c.year - a.year || a.sex.localeCompare(c.sex)).slice(0, 12).map((r) => (
85 + {shown.map((r) => (
73 86 <tr key={r.id}>
74 87 <td className="ci-num">{r.year_end && r.year_end !== r.year ? `${r.year}–${r.year_end}` : r.year}</td>
75 88 <td>{humanize(r.sex)}</td>
@@ -81,22 +94,23 @@ export async function StatisticsTab({ b }: { b: CancerBundle }) {
81 94 </td>
82 95 <td className="max-w-[240px] text-[12px] text-ink-3">{r.site_definition ?? '—'}</td>
83 96 <td>
84 − <span className="inline-flex gap-1">
85 − <SourceBadge p={toInfo(prov.get(r.provenance_id)) ?? { sourceSlug: r.source_slug, sourceName: r.source_name }} />
86 − <ClaimBadge kind="observed" />
87 − </span>
97 + <SourceBadge compact p={toInfo(prov.get(r.provenance_id)) ?? { sourceSlug: r.source_slug, sourceName: r.source_name }} />
88 98 </td>
89 99 </tr>
90 100 ))}
91 101 </tbody>
92 102 </table>
93 103 </div>
94 − {g.rows.length > 12 ? (
104 + {g.rows.length > ROWS_PER_TABLE ? (
95 105 <p className="mt-1 text-[12px] text-ink-3">
96 − Showing the 12 most recent rows of {g.rows.length}; every year is plotted above and available via <a className="ci-link" href={`/api/v1/cancers/${b.cancer.id}/statistics`}>the API</a>.
106 + Showing the {ROWS_PER_TABLE} most recent rows of {g.rows.length}; every year is plotted above and available via{' '}
107 + <a className="ci-link" href={`/api/v1/cancers/${b.cancer.id}/statistics`}>
108 + the API
109 + </a>
110 + .
97 111 </p>
98 112 ) : null}
99 − <Freshness dataUpdatedAt={g.rows.reduce((m, r) => (r.updated_at > m ? r.updated_at : m), g.rows[0]!.updated_at)} sourceVersion={prov.get(g.rows[0]!.provenance_id)?.dataset_version ?? null} />
113 + <Freshness dataUpdatedAt={g.rows.reduce<Date | string | null>((m, r) => (m == null || String(r.updated_at) > String(m) ? r.updated_at : m), null)} sourceVersion={prov.get(g.rows[0]!.provenance_id)?.dataset_version ?? null} />
100 114 </Section>
101 115 );
102 116 })}
modified apps/web/src/components/cancer/tabs/survival.tsx +26 −15
@@ -2,7 +2,7 @@ import { Section, Note } from '@/components/ui/section';
2 2 import { EmptyState } from '@/components/ui/empty-state';
3 3 import { Freshness } from '@/components/ui/freshness';
4 4 import { ClaimBadge } from '@/components/ui/badge';
5 −import { SourceBadge } from '@/components/ui/source-badge';
5 +import { SourceBadge, TableProvenance } from '@/components/ui/source-badge';
6 6 import { survivalFor } from '@/lib/queries/epidemiology';
7 7 import { loadProvenance, toInfo } from '@/lib/queries/provenance';
8 8 import { fmtInt, fmtPct, fmtNum, humanize } from '@/lib/format';
@@ -31,22 +31,36 @@ export async function SurvivalTab({ b }: { b: CancerBundle }) {
31 31 <Note tone="warn">Population survival is not an individual prognosis. Figures apply to the cohort, period, stage and population declared by the source; treatment landscapes change after the diagnosis period shown.</Note>
32 32 {[...groups.entries()].map(([k, rs]) => {
33 33 const f = rs[0]!;
34 + const firstProv = toInfo(prov.get(f.provenance_id)) ?? { sourceSlug: f.source_slug, sourceName: f.source_name };
34 35 return (
35 36 <Section key={k} id={k.replace(/[^a-z0-9]+/gi, '-')} kicker={f.geography_name ?? 'Population'} title={`${humanize(f.survival_type)} survival${f.stage ? ` — ${f.stage}` : ' — all stages'}`} description={f.staging_system ? `Staging system: ${f.staging_system}` : undefined}>
37 + <TableProvenance p={firstProv} claim={<ClaimBadge kind="observed" />}>
38 + {rs.length} observation{rs.length === 1 ? '' : 's'} · diagnosis period, follow-up and method as published; hover a row badge for its dataset and version.
39 + </TableProvenance>
36 40 <div className="ci-table-wrap">
37 41 <table className="ci-table">
38 42 <thead>
39 43 <tr>
40 − <th>Diagnosis period</th>
41 − <th>Sex</th>
42 − <th>Age group</th>
43 − <th className="num">Follow-up (months)</th>
44 − <th className="num">Survival (%)</th>
45 − <th className="num">95% CI</th>
46 − <th className="num">Median (months)</th>
47 − <th className="num">Cohort (n)</th>
48 − <th>Method</th>
49 − <th>Source</th>
44 + <th scope="col">Diagnosis period</th>
45 + <th scope="col">Sex</th>
46 + <th scope="col">Age group</th>
47 + <th scope="col" className="num">
48 + Follow-up (months)
49 + </th>
50 + <th scope="col" className="num">
51 + Survival (%)
52 + </th>
53 + <th scope="col" className="num">
54 + 95% CI
55 + </th>
56 + <th scope="col" className="num">
57 + Median (months)
58 + </th>
59 + <th scope="col" className="num">
60 + Cohort (n)
61 + </th>
62 + <th scope="col">Method</th>
63 + <th scope="col">Source</th>
50 64 </tr>
51 65 </thead>
52 66 <tbody>
@@ -62,10 +76,7 @@ export async function SurvivalTab({ b }: { b: CancerBundle }) {
62 76 <td className="num">{r.cohort_size != null ? fmtInt(r.cohort_size) : '—'}</td>
63 77 <td className="text-[12px] text-ink-3">{r.method ?? '—'}</td>
64 78 <td>
65 − <span className="inline-flex gap-1">
66 − <SourceBadge p={toInfo(prov.get(r.provenance_id)) ?? { sourceSlug: r.source_slug, sourceName: r.source_name }} />
67 − <ClaimBadge kind="observed" />
68 − </span>
79 + <SourceBadge compact p={toInfo(prov.get(r.provenance_id)) ?? { sourceSlug: r.source_slug, sourceName: r.source_name }} />
69 80 </td>
70 81 </tr>
71 82 ))}
modified apps/web/src/components/cancer/tabs/trials.tsx +27 −18
@@ -2,18 +2,18 @@ import Link from 'next/link';
2 2 import { Section } from '@/components/ui/section';
3 3 import { EmptyState } from '@/components/ui/empty-state';
4 4 import { Freshness } from '@/components/ui/freshness';
5 −import { Pagination } from '@/components/ui/pagination';
5 +import { Pager } from '@/components/ui/pager';
6 6 import { TrialTable } from '@/components/data/trial-list';
7 −import { listTrials, trialFacets } from '@/lib/queries/trials';
7 +import { listTrialRows, trialFacets, TRIAL_PAGE_SIZE } from '@/lib/queries/trials';
8 8 import { fmtInt, humanize, phaseLabel } from '@/lib/format';
9 +import { pageInfo } from '@/lib/pagination';
9 10 import { withParams } from '@/lib/search-params';
10 11 import type { CancerBundle } from '../load';
11 12
12 −const PAGE_SIZE = 25;
13 −
14 13 export async function TrialsTab({ b, status, phase, page }: { b: CancerBundle; status: string; phase: string; page: number }) {
15 14 const facets = await trialFacets(b.descendants);
16 − const { rows, total } = await listTrials({ q: '', status, phase, country: '', cancerIds: b.descendants, page, pageSize: PAGE_SIZE });
15 + const { rows, total } = await listTrialRows({ q: '', status, phase, country: '', cancerIds: b.descendants, page, pageSize: TRIAL_PAGE_SIZE });
16 + const info = pageInfo(page, TRIAL_PAGE_SIZE, total);
17 17 const base = `/cancer/${b.cancer.slug}/trials`;
18 18 const href = (o: Record<string, string | number | null | undefined>) => `${base}${withParams({ status, phase }, o)}`;
19 19 const anyTrials = facets.statuses.reduce((s, x) => s + x.n, 0);
@@ -28,40 +28,49 @@ export async function TrialsTab({ b, status, phase, page }: { b: CancerBundle; s
28 28 );
29 29 }
30 30
31 + const chip = (isCurrent: boolean) => ({ className: 'ci-chip', 'aria-current': isCurrent ? ('page' as const) : undefined });
32 +
31 33 return (
32 − <Section id="trials" kicker="Clinical trials" title="Registered studies" description={`${fmtInt(anyTrials)} studies whose conditions map to this entity or one of its descendants. Status and phase are as posted on ClinicalTrials.gov.`}>
34 + <Section id="trials" kicker="Clinical trials" title="Registered studies" description={`${fmtInt(anyTrials)} studies whose conditions map to this entity or one of its descendants. Status and phase are as posted on ClinicalTrials.gov. ${TRIAL_PAGE_SIZE} studies per page.`}>
33 35 <div className="mb-3 flex flex-wrap gap-4 text-[12.5px]">
34 − <div className="flex flex-wrap items-center gap-1.5">
36 + <nav aria-label="Filter by status" className="flex flex-wrap items-center gap-1.5">
35 37 <span className="ci-kicker mr-1">Status</span>
36 − <Link href={href({ status: '', page: '' })} className={`border px-2 py-0.5 no-underline ${!status ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
38 + <Link href={href({ status: '', page: '' })} {...chip(!status)}>
37 39 All
38 40 </Link>
39 − <Link href={href({ status: 'active', page: '' })} className={`border px-2 py-0.5 no-underline ${status === 'active' ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
41 + <Link href={href({ status: 'active', page: '' })} {...chip(status === 'active')}>
40 42 Active (any)
41 43 </Link>
42 44 {facets.statuses.map((s) => (
43 − <Link key={s.k} href={href({ status: s.k, page: '' })} className={`border px-2 py-0.5 no-underline ${status === s.k ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
45 + <Link key={s.k} href={href({ status: s.k, page: '' })} {...chip(status === s.k)}>
44 46 {humanize(s.k)} <span className="ci-num text-ink-3">{fmtInt(s.n)}</span>
45 47 </Link>
46 48 ))}
47 − </div>
48 − <div className="flex flex-wrap items-center gap-1.5">
49 + </nav>
50 + <nav aria-label="Filter by phase" className="flex flex-wrap items-center gap-1.5">
49 51 <span className="ci-kicker mr-1">Phase</span>
50 − <Link href={href({ phase: '', page: '' })} className={`border px-2 py-0.5 no-underline ${!phase ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
52 + <Link href={href({ phase: '', page: '' })} {...chip(!phase)}>
51 53 All
52 54 </Link>
53 55 {facets.phases.map((p) => (
54 − <Link key={p.k} href={href({ phase: p.k, page: '' })} className={`border px-2 py-0.5 no-underline ${phase === p.k ? 'border-accent bg-accent-soft text-accent-2' : 'border-rule text-ink-2 hover:border-accent'}`}>
56 + <Link key={p.k} href={href({ phase: p.k, page: '' })} {...chip(phase === p.k)}>
55 57 {phaseLabel(p.k)} <span className="ci-num text-ink-3">{fmtInt(p.n)}</span>
56 58 </Link>
57 59 ))}
58 − </div>
60 + </nav>
59 61 </div>
60 62 {rows.length ? (
61 63 <>
62 − <TrialTable rows={rows} />
63 − <Pagination page={page} pageSize={PAGE_SIZE} total={total} hrefFor={(p) => href({ page: p === 1 ? '' : p })} />
64 − <Freshness dataUpdatedAt={rows.reduce((m, t) => (t.updated_at > m ? t.updated_at : m), rows[0]!.updated_at)} sourceUpdatedAt={rows[0]?.last_update_posted_date ?? null} extra="source: clinicaltrials" />
64 + <TrialTable
65 + rows={rows}
66 + summary={
67 + <>
68 + Showing {fmtInt(info.from)}–{fmtInt(info.to)} of {fmtInt(total)} studies{status || phase ? ' matching the filters' : ''}
69 + </>
70 + }
71 + />
72 + <Pager total={total} pageSize={TRIAL_PAGE_SIZE} page={info.page} hrefFor={(p) => href({ page: p === 1 ? '' : p })} label="Trial pages" noun="studies" />
73 + <Freshness dataUpdatedAt={rows.reduce<Date | string | null>((m, t) => (m == null || String(t.updated_at) > String(m) ? t.updated_at : m), null)} sourceUpdatedAt={rows[0]?.last_update_posted_date ?? null} extra="source: clinicaltrials" />
65 74 </>
66 75 ) : (
67 76 <EmptyState compact title="No study matches these filters" />
added apps/web/src/components/charts/compare-bars.tsx +54 −0
@@ -0,0 +1,54 @@
1 +import { fmtValue, unitLabel } from '@/lib/format';
2 +
3 +export interface CompareBarDatum {
4 + label: string;
5 + value: number | null; // null = no figure (rendered as "—", never as zero)
6 + note?: string | null; // e.g. "2024 · observed" or "registry level: Lung cancer"
7 + href?: string;
8 + muted?: boolean;
9 +}
10 +
11 +/**
12 + * Compact horizontal bar list for the compare page (one small chart per metric). Bars are plain
13 + * divs so the block stays responsive on narrow screens; the number always accompanies the bar and
14 + * missing values are shown as a dash, never as an empty bar of length zero.
15 + */
16 +export function CompareBars({ title, unit, data, caption }: { title: string; unit?: string | null; data: CompareBarDatum[]; caption?: React.ReactNode }) {
17 + const values = data.map((d) => d.value).filter((v): v is number => v != null && Number.isFinite(v));
18 + const max = Math.max(...values, Number.EPSILON);
19 + return (
20 + <figure className="min-w-0 border-t border-rule-strong pt-2">
21 + <figcaption className="mb-1.5 flex items-baseline justify-between gap-2 text-[12.5px]">
22 + <span className="font-medium text-ink">{title}</span>
23 + {unit ? <span className="text-[11px] text-ink-3">{unitLabel(unit)}</span> : null}
24 + </figcaption>
25 + <ol className="space-y-1.5" aria-label={`${title} — bar comparison`}>
26 + {data.map((d, i) => {
27 + const has = d.value != null && Number.isFinite(d.value);
28 + const pct = has ? Math.max(1, (Math.abs(d.value!) / max) * 100) : 0;
29 + return (
30 + <li key={`${d.label}-${i}`} className="text-[12.5px]">
31 + <div className="flex items-baseline justify-between gap-2">
32 + <span className="min-w-0 truncate text-ink-2">
33 + {d.href ? (
34 + <a className="ci-link" href={d.href}>
35 + {d.label}
36 + </a>
37 + ) : (
38 + d.label
39 + )}
40 + </span>
41 + <span className="ci-num shrink-0 text-ink">{has ? fmtValue(d.value, unit) : '—'}</span>
42 + </div>
43 + <div className="mt-0.5 h-[7px] w-full bg-paper-3" role="presentation">
44 + {has ? <div className="h-full" style={{ width: `${pct}%`, background: d.muted ? 'var(--color-ink-4)' : 'var(--color-accent)' }} /> : null}
45 + </div>
46 + {d.note ? <div className="mt-0.5 text-[11px] text-ink-3">{d.note}</div> : null}
47 + </li>
48 + );
49 + })}
50 + </ol>
51 + {caption ? <p className="mt-1.5 text-[11.5px] text-ink-3">{caption}</p> : null}
52 + </figure>
53 + );
54 +}
added apps/web/src/components/charts/trend-chart.tsx +115 −0
@@ -0,0 +1,115 @@
1 +import { fmtValue } from '@/lib/format';
2 +
3 +export interface TrendSeries {
4 + key: string;
5 + name: string;
6 + href?: string;
7 + points: Array<{ x: number; y: number }>;
8 + dashed?: boolean; // estimated / projected values
9 +}
10 +
11 +/**
12 + * Eight-way colour set that stays distinguishable in print and for common colour-vision deficiencies;
13 + * shape (dash) and the legend label carry the meaning too — colour is never the only carrier.
14 + */
15 +const TREND_COLORS = ['#0f5f63', '#8a4b0a', '#8b1e2d', '#2d5f2e', '#4a4a46', '#5b3a8c', '#b5651d', '#1f5fa8'];
16 +
17 +/**
18 + * Multi-series yearly line chart in pure SVG for country trend panels (up to 8 series). Zero-based y axis,
19 + * light grid, year ticks, end-of-line labels when space allows, plus a text legend with the last value.
20 + */
21 +export function TrendChart({ series, unit, ariaLabel, height = 300, yLabel }: { series: TrendSeries[]; unit?: string | null; ariaLabel: string; height?: number; yLabel?: string }) {
22 + const shown = series.filter((s) => s.points.length > 0).slice(0, TREND_COLORS.length);
23 + const all = shown.flatMap((s) => s.points);
24 + if (all.length === 0) return null;
25 + const xs = all.map((p) => p.x);
26 + const xMin = Math.min(...xs);
27 + const xMax = Math.max(...xs);
28 + const yMax = Math.max(...all.map((p) => p.y), Number.EPSILON) * 1.08;
29 + const width = 760;
30 + const pad = { l: 58, r: 16, t: 12, b: 30 };
31 + const iw = width - pad.l - pad.r;
32 + const ih = height - pad.t - pad.b;
33 + const sx = (x: number) => pad.l + (xMax === xMin ? iw / 2 : ((x - xMin) / (xMax - xMin)) * iw);
34 + const sy = (y: number) => pad.t + ih - (y / yMax) * ih;
35 + const yTicks = 5;
36 + const span = xMax - xMin;
37 + const step = span <= 8 ? 1 : span <= 16 ? 2 : span <= 30 ? 5 : 10;
38 + const xTicks: number[] = [];
39 + for (let x = Math.ceil(xMin / step) * step; x <= xMax; x += step) xTicks.push(x);
40 + if (!xTicks.includes(xMin)) xTicks.unshift(xMin);
41 + if (!xTicks.includes(xMax)) xTicks.push(xMax);
42 +
43 + return (
44 + <figure className="w-full">
45 + <div className="overflow-x-auto">
46 + <svg viewBox={`0 0 ${width} ${height}`} width="100%" role="img" aria-label={ariaLabel} className="block min-w-[520px]">
47 + <title>{ariaLabel}</title>
48 + {Array.from({ length: yTicks + 1 }, (_, i) => {
49 + const v = (yMax / yTicks) * i;
50 + const y = sy(v);
51 + return (
52 + <g key={i}>
53 + <line x1={pad.l} x2={width - pad.r} y1={y} y2={y} stroke="var(--color-rule)" strokeWidth="1" />
54 + <text x={pad.l - 6} y={y + 4} textAnchor="end" fontSize="11" fill="var(--color-ink-3)" style={{ fontVariantNumeric: 'tabular-nums' }}>
55 + {fmtValue(v, unit)}
56 + </text>
57 + </g>
58 + );
59 + })}
60 + {yLabel ? (
61 + <text x={pad.l} y={pad.t - 2} fontSize="10.5" fill="var(--color-ink-3)">
62 + {yLabel}
63 + </text>
64 + ) : null}
65 + {xTicks.map((x) => (
66 + <g key={x}>
67 + <line x1={sx(x)} x2={sx(x)} y1={pad.t + ih} y2={pad.t + ih + 4} stroke="var(--color-rule-strong)" />
68 + <text x={sx(x)} y={height - 9} textAnchor="middle" fontSize="11" fill="var(--color-ink-3)">
69 + {x}
70 + </text>
71 + </g>
72 + ))}
73 + {shown.map((s, si) => {
74 + const pts = [...s.points].sort((a, b) => a.x - b.x);
75 + const color = TREND_COLORS[si % TREND_COLORS.length];
76 + const d = pts.map((p, i) => `${i === 0 ? 'M' : 'L'}${sx(p.x).toFixed(1)},${sy(p.y).toFixed(1)}`).join(' ');
77 + return (
78 + <g key={s.key}>
79 + <path d={d} fill="none" stroke={color} strokeWidth="1.75" strokeDasharray={s.dashed ? '4 3' : undefined} strokeLinejoin="round" />
80 + {pts.map((p) => (
81 + <circle key={p.x} cx={sx(p.x)} cy={sy(p.y)} r="2" fill={color}>
82 + <title>{`${s.name} — ${p.x}: ${fmtValue(p.y, unit)}`}</title>
83 + </circle>
84 + ))}
85 + </g>
86 + );
87 + })}
88 + </svg>
89 + </div>
90 + <figcaption className="mt-1.5 grid grid-cols-1 gap-x-4 gap-y-1 text-[12px] text-ink-2 sm:grid-cols-2 lg:grid-cols-4">
91 + {shown.map((s, si) => {
92 + const last = [...s.points].sort((a, b) => a.x - b.x).at(-1);
93 + return (
94 + <span key={s.key} className="inline-flex min-w-0 items-baseline gap-1.5">
95 + <span className="inline-block h-[3px] w-4 shrink-0 self-center" style={{ background: s.dashed ? 'transparent' : TREND_COLORS[si % TREND_COLORS.length], borderTop: s.dashed ? `2px dashed ${TREND_COLORS[si % TREND_COLORS.length]}` : undefined }} aria-hidden />
96 + {s.href ? (
97 + <a className="ci-link truncate" href={s.href}>
98 + {s.name}
99 + </a>
100 + ) : (
101 + <span className="truncate">{s.name}</span>
102 + )}
103 + {last ? (
104 + <span className="ci-num shrink-0 text-ink-3">
105 + {fmtValue(last.y, unit)} <span className="text-[10.5px]">({last.x})</span>
106 + </span>
107 + ) : null}
108 + {s.dashed ? <span className="text-[10.5px] italic text-warn">estimated</span> : null}
109 + </span>
110 + );
111 + })}
112 + </figcaption>
113 + </figure>
114 + );
115 +}
modified apps/web/src/components/data/approvals-table.tsx +74 −62
@@ -1,77 +1,89 @@
1 1 import Link from 'next/link';
2 2 import { Badge, ClaimBadge, StatusBadge } from '@/components/ui/badge';
3 −import { SourceBadge } from '@/components/ui/source-badge';
3 +import { SourceBadge, TableProvenance } from '@/components/ui/source-badge';
4 4 import type { ApprovalRow } from '@/lib/queries/drugs';
5 5 import type { ProvRow } from '@/lib/queries/provenance';
6 6 import { toInfo } from '@/lib/queries/provenance';
7 −import { fmtDate } from '@/lib/format';
7 +import { fmtDate, fmtInt } from '@/lib/format';
8 8
9 −/** Jurisdiction-aware regulatory status (§13). Never a bare "approved". */
9 +/**
10 + * Jurisdiction-aware regulatory status (§13). Never a bare "approved". Approvals may come from
11 + * different authorities' datasets, so rows keep a compact badge with a dataset · version title;
12 + * the caption carries one full popover (first row's provenance) and the claim label.
13 + */
10 14 export function ApprovalsTable({ rows, prov, showDrug = true, showCancer = true }: { rows: ApprovalRow[]; prov: Map<number, ProvRow>; showDrug?: boolean; showCancer?: boolean }) {
15 + const first = rows[0];
16 + const caption = first ? (toInfo(prov.get(first.provenance_id)) ?? { sourceSlug: first.source_slug, sourceName: first.source_name }) : null;
17 + const sources = new Set(rows.map((r) => r.source_slug));
11 18 return (
12 − <div className="ci-table-wrap">
13 − <table className="ci-table">
14 − <thead>
15 − <tr>
16 − {showDrug ? <th>Drug</th> : null}
17 − <th>Jurisdiction · authority</th>
18 − {showCancer ? <th>Cancer</th> : null}
19 − <th>Indication</th>
20 − <th>Status</th>
21 − <th>Approval date</th>
22 − <th>Source</th>
23 − </tr>
24 − </thead>
25 − <tbody>
26 − {rows.map((a) => (
27 − <tr key={a.id}>
28 − {showDrug ? (
19 + <div>
20 + {caption ? (
21 + <TableProvenance p={caption} claim={<ClaimBadge kind="regulatory" />}>
22 + {fmtInt(rows.length)} approval record{rows.length === 1 ? '' : 's'}
23 + {sources.size > 1 ? ` from ${sources.size} sources (hover a row badge for its dataset)` : ''} · authority, jurisdiction and indication text as published.
24 + </TableProvenance>
25 + ) : null}
26 + <div className="ci-table-wrap">
27 + <table className="ci-table">
28 + <thead>
29 + <tr>
30 + {showDrug ? <th scope="col">Drug</th> : null}
31 + <th scope="col">Jurisdiction · authority</th>
32 + {showCancer ? <th scope="col">Cancer</th> : null}
33 + <th scope="col">Indication</th>
34 + <th scope="col">Status</th>
35 + <th scope="col">Approval date</th>
36 + <th scope="col">Source</th>
37 + </tr>
38 + </thead>
39 + <tbody>
40 + {rows.map((a) => (
41 + <tr key={a.id}>
42 + {showDrug ? (
43 + <td>
44 + <Link className="ci-link" href={`/drug/${a.drug_slug}`}>
45 + {a.drug_name}
46 + </Link>
47 + </td>
48 + ) : null}
49 + <td className="whitespace-nowrap">
50 + <span className="ci-mono font-medium">{a.jurisdiction}</span> <span className="text-ink-3">{a.authority}</span>
51 + </td>
52 + {showCancer ? (
53 + <td>
54 + {a.tumor_agnostic ? <Badge tone="accent">Tumor-agnostic</Badge> : null}
55 + {a.cancer_slug ? (
56 + <Link className="ci-link" href={`/cancer/${a.cancer_slug}`}>
57 + {a.cancer_name}
58 + </Link>
59 + ) : !a.tumor_agnostic ? (
60 + <span className="text-ink-3">—</span>
61 + ) : null}
62 + </td>
63 + ) : null}
64 + <td className="min-w-[280px] max-w-[520px] text-[12.5px]">
65 + {a.indication}
66 + <span className="mt-0.5 flex flex-wrap gap-1">
67 + {a.line_of_therapy ? <Badge tone="outline">{a.line_of_therapy}</Badge> : null}
68 + {a.disease_stage ? <Badge tone="outline">{a.disease_stage}</Badge> : null}
69 + {a.accelerated ? <Badge tone="warn">accelerated</Badge> : null}
70 + {a.conditional ? <Badge tone="warn">conditional</Badge> : null}
71 + {a.biomarker_ids.length ? <Badge tone="outline">biomarker-restricted</Badge> : null}
72 + </span>
73 + </td>
29 74 <td>
30 − <Link className="ci-link" href={`/drug/${a.drug_slug}`}>
31 − {a.drug_name}
32 − </Link>
75 + <StatusBadge status={a.status} />
76 + {a.withdrawal_date ? <span className="block text-[11px] text-danger">withdrawn {fmtDate(a.withdrawal_date)}</span> : null}
33 77 </td>
34 − ) : null}
35 − <td className="whitespace-nowrap">
36 − <span className="ci-mono font-medium">{a.jurisdiction}</span> <span className="text-ink-3">{a.authority}</span>
37 − </td>
38 − {showCancer ? (
78 + <td className="whitespace-nowrap">{fmtDate(a.approval_date)}</td>
39 79 <td>
40 − {a.tumor_agnostic ? <Badge tone="accent">Tumor-agnostic</Badge> : null}
41 − {a.cancer_slug ? (
42 − <Link className="ci-link" href={`/cancer/${a.cancer_slug}`}>
43 − {a.cancer_name}
44 − </Link>
45 − ) : !a.tumor_agnostic ? (
46 − <span className="text-ink-3">—</span>
47 − ) : null}
80 + <SourceBadge compact p={toInfo(prov.get(a.provenance_id)) ?? { sourceSlug: a.source_slug, sourceName: a.source_name }} />
48 81 </td>
49 − ) : null}
50 − <td className="min-w-[280px] max-w-[520px] text-[12.5px]">
51 − {a.indication}
52 − <span className="mt-0.5 flex flex-wrap gap-1">
53 − {a.line_of_therapy ? <Badge tone="outline">{a.line_of_therapy}</Badge> : null}
54 − {a.disease_stage ? <Badge tone="outline">{a.disease_stage}</Badge> : null}
55 − {a.accelerated ? <Badge tone="warn">accelerated</Badge> : null}
56 − {a.conditional ? <Badge tone="warn">conditional</Badge> : null}
57 − {a.biomarker_ids.length ? <Badge tone="outline">biomarker-restricted</Badge> : null}
58 − </span>
59 − </td>
60 − <td>
61 − <StatusBadge status={a.status} />
62 − {a.withdrawal_date ? <span className="block text-[11px] text-danger">withdrawn {fmtDate(a.withdrawal_date)}</span> : null}
63 − </td>
64 − <td className="whitespace-nowrap">{fmtDate(a.approval_date)}</td>
65 − <td>
66 − <span className="inline-flex gap-1">
67 − <SourceBadge p={toInfo(prov.get(a.provenance_id)) ?? { sourceSlug: a.source_slug, sourceName: a.source_name }} />
68 − <ClaimBadge kind="regulatory" />
69 − </span>
70 − </td>
71 − </tr>
72 − ))}
73 − </tbody>
74 − </table>
82 + </tr>
83 + ))}
84 + </tbody>
85 + </table>
86 + </div>
75 87 </div>
76 88 );
77 89 }
modified apps/web/src/components/data/evidence-table.tsx +166 −123
@@ -1,154 +1,197 @@
1 1 import Link from 'next/link';
2 −import { Badge, ClaimBadge, MatchBadge, StatusBadge } from '@/components/ui/badge';
3 −import { SourceBadge } from '@/components/ui/source-badge';
4 −import { EVIDENCE_LEVEL_LABEL, groupEvidence, type EvidenceItem } from '@/lib/queries/evidence';
2 +import { Fragment, type ReactNode } from 'react';
3 +import { Badge, ClaimBadge, MatchBadge, StatusBadge, isExactMatch } from '@/components/ui/badge';
4 +import { SourceBadge, TableProvenance } from '@/components/ui/source-badge';
5 +import { EVIDENCE_LEVEL_LABEL, EVIDENCE_DESCRIPTION_CHARS, groupEvidence, therapyKey, type EvidenceItem, type EvidenceGroupBy } from '@/lib/queries/evidence';
5 6 import type { ProvRow } from '@/lib/queries/provenance';
6 7 import { toInfo } from '@/lib/queries/provenance';
7 8 import { humanize } from '@/lib/format';
8 9
9 −function DirectionBadge({ e }: { e: EvidenceItem }) {
10 +const CIVIC = { sourceSlug: 'civic', sourceName: 'CIViC' };
11 +
12 +function Direction({ e }: { e: EvidenceItem }) {
10 13 const dir = e.evidence_direction ?? '—';
11 14 const sig = e.significance ?? '';
12 15 const tone = dir === 'DOES_NOT_SUPPORT' ? 'warn' : sig.includes('RESIST') ? 'danger' : sig.includes('SENSITIV') ? 'ok' : 'neutral';
13 16 return (
14 − <span className="inline-flex flex-wrap gap-1">
15 − <Badge tone={tone} title="Evidence direction as curated at the source">
16 − {humanize(dir)}
17 − </Badge>
18 − {sig ? <Badge tone="outline">{humanize(sig.replace('SENSITIVITYRESPONSE', 'SENSITIVITY_RESPONSE'))}</Badge> : null}
19 − </span>
17 + <>
18 + <Badge tone={tone}>{humanize(dir)}</Badge>
19 + {sig ? (
20 + <>
21 + {' '}
22 + <Badge tone="outline">{humanize(sig.replace('SENSITIVITYRESPONSE', 'SENSITIVITY_RESPONSE'))}</Badge>
23 + </>
24 + ) : null}
25 + </>
20 26 );
21 27 }
22 28
23 29 /**
24 − * CIViC evidence grouped by variant → therapy. Levels, directions and significance are shown in
25 − * their native form (§3, never collapsed to works/doesn't).
30 + * CIViC evidence in ONE dense table. Rows are grouped (by molecular profile → therapy, or by cancer
31 + * on variant pages) with a `<tbody>` per group and a row-group header, instead of one table per
32 + * group: a table header is rendered once, which halves the markup for pages with many small groups
33 + * (every element ships twice — HTML + RSC payload). Levels, directions and significance are shown
34 + * in their native form (§3, never collapsed to works/doesn't). One provenance popover for the whole
35 + * table (all rows come from one CIViC release); rows carry a compact badge.
36 + *
37 + * `summary` replaces the default "N evidence items" sentence — pass the page window when the caller
38 + * paginates ("Showing 1–50 of 405 evidence items").
26 39 */
27 −export function EvidenceTable({ items, prov, showCancer = false, showVariant = true }: { items: EvidenceItem[]; prov: Map<number, ProvRow>; showCancer?: boolean; showVariant?: boolean }) {
28 − const groups = groupEvidence(items);
29 − // One provenance popover for the whole table (all rows come from one CIViC release); rows carry a compact badge.
40 +export function EvidenceTable({ items, prov, showCancer = false, groupBy = 'variant', summary }: { items: EvidenceItem[]; prov: Map<number, ProvRow>; showCancer?: boolean; groupBy?: EvidenceGroupBy; summary?: ReactNode }) {
41 + const groups = groupEvidence(items, groupBy);
30 42 const first = items[0] ? toInfo(prov.get(items[0].provenance_id)) : null;
43 + const cancerCol = showCancer && groupBy !== 'cancer';
44 + const profileCol = groupBy === 'cancer';
45 + const cols = 7 + (cancerCol ? 1 : 0) + (profileCol ? 1 : 0);
31 46 return (
32 − <div className="space-y-6">
33 − <p className="flex flex-wrap items-center gap-2 text-[12px] text-ink-3">
34 − <SourceBadge p={first ?? { sourceSlug: 'civic', sourceName: 'CIViC' }} />
35 − <ClaimBadge kind="curated" />
36 − {items.length} evidence item{items.length === 1 ? '' : 's'} · levels, directions and significance as curated at the source; each row links to its CIViC record.
37 − </p>
38 − {groups.map((g) => (
39 − <section key={g.key} aria-label={g.label}>
40 − {showVariant ? (
41 − <h3 className="mb-1.5 flex flex-wrap items-baseline gap-2 text-[15px]">
42 − {g.slug ? (
43 − <Link href={`/variant/${g.slug}`} className="ci-link font-sans font-medium">
44 − {g.label}
45 − </Link>
46 − ) : (
47 − <span className="font-sans font-medium">{g.label}</span>
48 − )}
49 − {g.genes.map((s) => (
50 − <Link key={s} href={`/gene/${s}`} className="ci-mono text-[12px] text-ink-3 hover:text-accent">
51 − {s}
52 − </Link>
53 − ))}
54 − </h3>
55 − ) : null}
56 − <div className="ci-table-wrap">
57 − <table className="ci-table">
58 − <thead>
59 − <tr>
60 − <th>Therapy</th>
61 − {showCancer ? <th>Cancer</th> : null}
62 − <th>Type</th>
63 − <th>Level</th>
64 − <th>Direction · significance</th>
65 − <th className="num">Rating (1–5)</th>
66 − <th>Status</th>
67 − <th>Evidence</th>
68 − <th>Source</th>
47 + <div>
48 + <TableProvenance p={first ?? CIVIC} claim={<ClaimBadge kind="curated" />}>
49 + {summary ?? `${items.length} evidence item${items.length === 1 ? '' : 's'}`} · levels, directions and significance as curated at the source; each row links to its CIViC record.
50 + </TableProvenance>
51 + <div className="ci-table-wrap">
52 + <table className="ci-table ci-evidence">
53 + <thead>
54 + <tr>
55 + {profileCol ? <th scope="col">Molecular profile</th> : null}
56 + <th scope="col">Therapy</th>
57 + {cancerCol ? <th scope="col">Cancer</th> : null}
58 + <th scope="col">Type</th>
59 + <th scope="col" title="CIViC evidence level: A validated · B clinical · C case study · D preclinical · E inferential">
60 + Level
61 + </th>
62 + <th scope="col" title="Evidence direction and clinical significance as curated at the source">
63 + Direction · significance
64 + </th>
65 + <th scope="col" className="num">
66 + Rating (1–5)
67 + </th>
68 + <th scope="col">Status</th>
69 + <th scope="col">Evidence</th>
70 + <th scope="col">Source</th>
71 + </tr>
72 + </thead>
73 + {groups.map((g) => (
74 + <tbody key={g.key}>
75 + {groupBy !== 'none' ? (
76 + <tr className="ci-group">
77 + <th scope="rowgroup" colSpan={cols}>
78 + {g.href ? (
79 + <Link href={g.href}>
80 + {g.label}
81 + </Link>
82 + ) : (
83 + <span>{g.label}</span>
84 + )}
85 + {g.genes.length > 1
86 + ? g.genes.map((s) => (
87 + <Link key={s} href={`/gene/${s}`} className="ci-mono ml-2 text-[12px] text-ink-3 hover:text-accent">
88 + {s}
89 + </Link>
90 + ))
91 + : null}
92 + <span className="ci-num ml-2 text-[12px] text-ink-3">{g.items.length}</span>
93 + {g.unmapped ? (
94 + <Badge tone="danger" className="ml-2">
95 + unmapped disease
96 + </Badge>
97 + ) : null}
98 + </th>
69 99 </tr>
70 − </thead>
71 − <tbody>
72 − {[...g.therapies.entries()].flatMap(([therapy, es]) =>
73 − es.map((e, i) => (
74 − <tr key={e.id}>
75 − <td className="min-w-[160px]">
76 − {i === 0 ? (
77 − e.therapy_slugs?.length ? (
78 − e.therapy_slugs.map((s, j) => (
79 − <span key={s}>
80 − {j > 0 ? ' + ' : ''}
81 − <Link className="ci-link" href={`/drug/${s}`}>
82 − {e.therapy_slug_names?.[j] ?? s}
83 − </Link>
84 − </span>
85 − ))
86 − ) : (
87 − <span className={therapy.startsWith('(') ? 'text-ink-3' : ''}>{therapy}</span>
88 − )
100 + ) : null}
101 + {g.items.map((e, i) => {
102 + const prevKey = i > 0 ? therapyKey(g.items[i - 1]!) : null;
103 + const sameTherapy = prevKey === therapyKey(e) && groupBy === 'variant';
104 + const desc = e.description ?? '';
105 + const cut = desc.length >= EVIDENCE_DESCRIPTION_CHARS;
106 + return (
107 + <tr key={e.id}>
108 + {profileCol ? <td className="w-t text-[12.5px]">{e.molecular_profile_name ?? e.name ?? '—'}</td> : null}
109 + <td className="w-t">
110 + {sameTherapy ? (
111 + <span className="text-ink-4" aria-label="same therapy as the row above">
112 + 〃
113 + </span>
114 + ) : e.therapy_slugs?.length ? (
115 + e.therapy_slugs.map((s, j) => (
116 + <Fragment key={s}>
117 + {j > 0 ? ' + ' : ''}
118 + <Link href={`/drug/${s}`}>
119 + {e.therapy_slug_names?.[j] ?? s}
120 + </Link>
121 + </Fragment>
122 + ))
123 + ) : (
124 + <span className="text-ink-3">{therapyLabel(e)}</span>
125 + )}
126 + {e.therapy_interaction_type && !sameTherapy ? <span className="block text-[11px] text-ink-3">{humanize(e.therapy_interaction_type)}</span> : null}
127 + </td>
128 + {cancerCol ? (
129 + <td className="w-t">
130 + {e.cancer_slug ? (
131 + <Link href={`/cancer/${e.cancer_slug}`}>
132 + {e.cancer_name}
133 + </Link>
89 134 ) : (
90 − <span className="text-ink-4">〃</span>
135 + <span className="text-ink-3">{e.disease_name ?? '—'}</span>
91 136 )}
92 − {e.therapy_interaction_type && i === 0 ? <span className="block text-[11px] text-ink-3">{humanize(e.therapy_interaction_type)}</span> : null}
93 − </td>
94 − {showCancer ? (
95 − <td className="min-w-[160px]">
96 − {e.cancer_slug ? (
97 − <Link className="ci-link" href={`/cancer/${e.cancer_slug}`}>
98 − {e.cancer_name}
99 − </Link>
100 − ) : (
101 − <span className="text-ink-3">{e.disease_name ?? '—'}</span>
102 − )}
103 − {e.cancer_match_type && e.cancer_match_type !== 'EXACT_IDENTIFIER' ? <MatchBadge matchType={e.cancer_match_type} className="ml-1" /> : null}
104 − </td>
105 − ) : null}
106 − <td>
107 − <Badge>{humanize(e.evidence_type)}</Badge>
108 − </td>
109 − <td>
110 − <span className="ci-mono font-medium" title={e.evidence_level ? EVIDENCE_LEVEL_LABEL[e.evidence_level] : undefined}>
111 − {e.evidence_level ?? '—'}
112 − </span>
137 + {/* Exact mappings are the norm; only non-exact ones (alias, broader, probabilistic, unresolved) carry the caveat badge. */}
138 + {!isExactMatch(e.cancer_match_type) ? <MatchBadge matchType={e.cancer_match_type} className="ml-1" /> : null}
113 139 </td>
114 − <td>
115 − <DirectionBadge e={e} />
116 − </td>
117 − <td className="num">{e.evidence_rating ?? '—'}</td>
118 − <td>
119 − <StatusBadge status={e.status} />
120 − </td>
121 − <td className="max-w-[360px] text-[12.5px] text-ink-2">
122 − <details>
123 − <summary className="ci-link">EID{e.civic_id}</summary>
124 − <p className="mt-1">{e.description ? `${e.description}${e.description.length >= 320 ? '…' : ''}` : 'No description at source.'}{e.description && e.description.length >= 320 ? <span className="text-ink-3"> (full text at CIViC)</span> : null}</p>
140 + ) : null}
141 + <td>
142 + <Badge>{humanize(e.evidence_type)}</Badge>
143 + </td>
144 + <td>
145 + <abbr {...(e.evidence_level && EVIDENCE_LEVEL_LABEL[e.evidence_level] ? { title: EVIDENCE_LEVEL_LABEL[e.evidence_level] } : {})}>
146 + {e.evidence_level ?? '—'}
147 + </abbr>
148 + </td>
149 + <td>
150 + <Direction e={e} />
151 + </td>
152 + <td className="num">{e.evidence_rating ?? '—'}</td>
153 + <td>
154 + <StatusBadge status={e.status} />
155 + </td>
156 + <td className="ev">
157 + <details>
158 + <summary>EID{e.civic_id}</summary>
159 + <p>
160 + {desc ? `${desc}${cut ? '…' : ''}` : 'No description at source.'}
161 + {cut ? <span className="text-ink-3"> (full text at CIViC)</span> : null}
162 + </p>
163 + <p>
125 164 {e.pmid ? (
126 − <p className="mt-1">
165 + <>
127 166 PMID{' '}
128 − <a className="ci-link" href={`https://pubmed.ncbi.nlm.nih.gov/${e.pmid}/`} target="_blank" rel="noopener noreferrer">
167 + <a href={`https://pubmed.ncbi.nlm.nih.gov/${e.pmid}/`} target="_blank">
129 168 {e.pmid}
130 169 </a>
131 170 {e.source_citation ? <span className="text-ink-3"> · {e.source_citation}</span> : null}
132 − </p>
171 + {' · '}
172 + </>
133 173 ) : null}
134 − <a className="ci-link mt-1 inline-block" href={`https://civicdb.org/evidence/${e.civic_id}/summary`} target="_blank" rel="noopener noreferrer">
174 + <a href={`https://civicdb.org/evidence/${e.civic_id}/summary`} target="_blank">
135 175 Open in CIViC
136 176 </a>
137 − </details>
138 − </td>
139 − <td>
140 − <span className="inline-flex flex-wrap gap-1">
141 − <SourceBadge compact p={toInfo(prov.get(e.provenance_id)) ?? { sourceSlug: 'civic', sourceName: 'CIViC' }} />
142 − </span>
143 − </td>
144 − </tr>
145 − )),
146 − )}
147 − </tbody>
148 − </table>
149 − </div>
150 − </section>
151 − ))}
177 + </p>
178 + </details>
179 + </td>
180 + <td>
181 + <SourceBadge compact title={null} p={toInfo(prov.get(e.provenance_id)) ?? CIVIC} />
182 + </td>
183 + </tr>
184 + );
185 + })}
186 + </tbody>
187 + ))}
188 + </table>
189 + </div>
152 190 </div>
153 191 );
154 192 }
193 +
194 +function therapyLabel(e: EvidenceItem): string {
195 + if (e.therapy_names.length) return e.therapy_names.join(' + ');
196 + return e.evidence_type === 'PREDICTIVE' ? 'Unspecified therapy' : `(${(e.evidence_type ?? 'evidence').toLowerCase()})`;
197 +}
modified apps/web/src/components/data/frequency-table.tsx +95 −65
@@ -1,13 +1,17 @@
1 1 import Link from 'next/link';
2 −import { Badge, ClaimBadge, MatchBadge } from '@/components/ui/badge';
3 −import { SourceBadge } from '@/components/ui/source-badge';
2 +import { Badge, ClaimBadge, MatchBadge, isExactMatch } from '@/components/ui/badge';
3 +import { SourceBadge, TableProvenance } from '@/components/ui/source-badge';
4 4 import { Freshness } from '@/components/ui/freshness';
5 5 import type { FreqRow } from '@/lib/queries/genomics';
6 6 import type { ProvRow } from '@/lib/queries/provenance';
7 7 import { toInfo } from '@/lib/queries/provenance';
8 8 import { fmtInt, fmtPct, humanize } from '@/lib/format';
9 9
10 −/** Gene alteration frequencies with mandatory denominators (§260-261), one table per cohort. */
10 +/**
11 + * Gene alteration frequencies with mandatory denominators (§260-261), one table per cohort.
12 + * Every row of a cohort table shares one provenance (one cohort, one release), so the popover is
13 + * rendered once per cohort in the caption; rows carry the compact badge.
14 + */
11 15 export function FrequencyTables({ rows, prov, cohortFilter, showCancer = false }: { rows: FreqRow[]; prov: Map<number, ProvRow>; cohortFilter?: string | null; showCancer?: boolean }) {
12 16 const cohorts = new Map<string, FreqRow[]>();
13 17 for (const r of rows) {
@@ -19,6 +23,7 @@ export function FrequencyTables({ rows, prov, cohortFilter, showCancer = false }
19 23 <div className="space-y-8">
20 24 {[...cohorts.entries()].map(([cohortId, rs]) => {
21 25 const c = rs[0]!;
26 + const info = toInfo(prov.get(c.provenance_id)) ?? { sourceSlug: c.source_slug, sourceName: c.source_name, datasetVersion: c.data_release };
22 27 return (
23 28 <section key={cohortId} aria-label={c.cohort_name}>
24 29 <header className="mb-1.5 flex flex-wrap items-baseline justify-between gap-2">
@@ -51,17 +56,28 @@ export function FrequencyTables({ rows, prov, cohortFilter, showCancer = false }
51 56 ) : null}
52 57 </p>
53 58 </header>
59 + <TableProvenance p={info} claim={<ClaimBadge kind="observed" />}>
60 + {rs.length} gene{rs.length === 1 ? '' : 's'} · frequency = affected / profiled, as published by the cohort.
61 + </TableProvenance>
54 62 <div className="ci-table-wrap">
55 63 <table className="ci-table">
56 64 <thead>
57 65 <tr>
58 − <th className="num">#</th>
59 − <th>Gene</th>
60 − <th>Alteration</th>
61 − <th className="num">Affected (n)</th>
62 − <th className="num">Profiled (n)</th>
63 − <th className="num">Frequency (%)</th>
64 − <th>Source</th>
66 + <th scope="col" className="num">
67 + #
68 + </th>
69 + <th scope="col">Gene</th>
70 + <th scope="col">Alteration</th>
71 + <th scope="col" className="num">
72 + Affected (n)
73 + </th>
74 + <th scope="col" className="num">
75 + Profiled (n)
76 + </th>
77 + <th scope="col" className="num">
78 + Frequency (%)
79 + </th>
80 + <th scope="col">Source</th>
65 81 </tr>
66 82 </thead>
67 83 <tbody>
@@ -80,17 +96,14 @@ export function FrequencyTables({ rows, prov, cohortFilter, showCancer = false }
80 96 <td className="num">{fmtInt(r.cases_profiled)}</td>
81 97 <td className="num font-medium">{fmtPct(r.frequency, 1)}</td>
82 98 <td>
83 − <span className="inline-flex gap-1">
84 − <SourceBadge p={toInfo(prov.get(r.provenance_id)) ?? { sourceSlug: r.source_slug, sourceName: r.source_name }} />
85 − <ClaimBadge kind="observed" />
86 − </span>
99 + <SourceBadge compact title={null} p={info} />
87 100 </td>
88 101 </tr>
89 102 ))}
90 103 </tbody>
91 104 </table>
92 105 </div>
93 − <Freshness dataUpdatedAt={c.updated_at} sourceVersion={c.data_release} extra="frequency = affected / profiled, as published by the cohort" />
106 + <Freshness dataUpdatedAt={c.updated_at} sourceVersion={c.data_release} />
94 107 </section>
95 108 );
96 109 })}
@@ -98,60 +111,77 @@ export function FrequencyTables({ rows, prov, cohortFilter, showCancer = false }
98 111 );
99 112 }
100 113
101 −/** Gene-centric view: one compact row per cohort (used on /gene/[symbol]); denominators stay mandatory. */
114 +/**
115 + * Gene-centric view: one compact row per cohort (used on /gene/[symbol]); denominators stay mandatory.
116 + * Rows come from different cohorts (different releases), so each compact badge carries a `title`
117 + * with dataset · version · retrieved; the caption shows one popover for the source.
118 + */
102 119 export function GeneFrequencyTable({ rows, prov }: { rows: FreqRow[]; prov: Map<number, ProvRow> }) {
103 120 const sorted = [...rows].sort((a, b) => b.frequency - a.frequency);
121 + const first = sorted[0];
122 + const caption = first ? (toInfo(prov.get(first.provenance_id)) ?? { sourceSlug: first.source_slug, sourceName: first.source_name }) : null;
104 123 return (
105 − <div className="ci-table-wrap">
106 − <table className="ci-table">
107 − <thead>
108 − <tr>
109 − <th>Cohort</th>
110 − <th>Mapped cancer</th>
111 − <th>Alteration</th>
112 − <th className="num">Affected (n)</th>
113 − <th className="num">Profiled (n)</th>
114 − <th className="num">Frequency (%)</th>
115 − <th className="num">Rank in cohort</th>
116 − <th>Source</th>
117 − </tr>
118 − </thead>
119 − <tbody>
120 − {sorted.map((r) => (
121 − <tr key={r.id}>
122 − <td>
123 − <span className="font-medium">{r.cohort_name}</span> <span className="ci-mono text-[11.5px] text-ink-3">{r.study_id}</span>
124 − </td>
125 − <td>
126 − {r.cancer_slug ? (
127 − <>
128 − <Link className="ci-link" href={`/cancer/${r.cancer_slug}`}>
129 − {r.cancer_name}
130 − </Link>{' '}
131 − <MatchBadge matchType={r.cancer_match_type} />
132 − </>
133 − ) : (
134 − <span className="text-ink-3">unmapped</span>
135 − )}
136 − </td>
137 − <td>
138 − <Badge tone="outline">{humanize(r.alteration_type)}</Badge>
139 − </td>
140 − <td className="num">{fmtInt(r.cases_affected)}</td>
141 − <td className="num">{fmtInt(r.cases_profiled)}</td>
142 − <td className="num font-medium">{fmtPct(r.frequency, 1)}</td>
143 − <td className="num text-ink-3">{r.rank ?? '—'}</td>
144 − <td>
145 − <span className="inline-flex gap-1">
146 − <SourceBadge p={toInfo(prov.get(r.provenance_id)) ?? { sourceSlug: r.source_slug, sourceName: r.source_name }} />
147 − <ClaimBadge kind="observed" />
148 − </span>
149 − </td>
124 + <div>
125 + {caption ? (
126 + <TableProvenance p={caption} claim={<ClaimBadge kind="observed" />}>
127 + {sorted.length} cohort row{sorted.length === 1 ? '' : 's'} · frequency = affected / profiled, as published by each cohort; cohorts are not pooled. Hover a row badge for its release.
128 + </TableProvenance>
129 + ) : null}
130 + <div className="ci-table-wrap">
131 + <table className="ci-table">
132 + <thead>
133 + <tr>
134 + <th scope="col">Cohort</th>
135 + <th scope="col">Mapped cancer</th>
136 + <th scope="col">Alteration</th>
137 + <th scope="col" className="num">
138 + Affected (n)
139 + </th>
140 + <th scope="col" className="num">
141 + Profiled (n)
142 + </th>
143 + <th scope="col" className="num">
144 + Frequency (%)
145 + </th>
146 + <th scope="col" className="num">
147 + Rank in cohort
148 + </th>
149 + <th scope="col">Source</th>
150 150 </tr>
151 − ))}
152 − </tbody>
153 − </table>
154 − <p className="mt-1 text-[12px] text-ink-3">frequency = affected / profiled, as published by each cohort; cohorts are not pooled.</p>
151 + </thead>
152 + <tbody>
153 + {sorted.map((r) => (
154 + <tr key={r.id}>
155 + <td>
156 + <span className="font-medium">{r.cohort_name}</span> <span className="ci-mono text-[11.5px] text-ink-3">{r.study_id}</span>
157 + </td>
158 + <td>
159 + {r.cancer_slug ? (
160 + <>
161 + <Link className="ci-link" href={`/cancer/${r.cancer_slug}`}>
162 + {r.cancer_name}
163 + </Link>
164 + {!isExactMatch(r.cancer_match_type) ? <MatchBadge matchType={r.cancer_match_type} className="ml-1" /> : null}
165 + </>
166 + ) : (
167 + <span className="text-ink-3">unmapped</span>
168 + )}
169 + </td>
170 + <td>
171 + <Badge tone="outline">{humanize(r.alteration_type)}</Badge>
172 + </td>
173 + <td className="num">{fmtInt(r.cases_affected)}</td>
174 + <td className="num">{fmtInt(r.cases_profiled)}</td>
175 + <td className="num font-medium">{fmtPct(r.frequency, 1)}</td>
176 + <td className="num text-ink-3">{r.rank ?? '—'}</td>
177 + <td>
178 + <SourceBadge compact p={toInfo(prov.get(r.provenance_id)) ?? { sourceSlug: r.source_slug, sourceName: r.source_name, dataset: r.cohort_name, datasetVersion: r.data_release }} />
179 + </td>
180 + </tr>
181 + ))}
182 + </tbody>
183 + </table>
184 + </div>
155 185 </div>
156 186 );
157 187 }
modified apps/web/src/components/data/publication-list.tsx +51 −37
@@ -1,42 +1,56 @@
1 1 import Link from 'next/link';
2 −import { Badge } from '@/components/ui/badge';
3 −import type { PublicationRow } from '@/lib/queries/publications';
4 −import { fmtDate } from '@/lib/format';
2 +import type { ReactNode } from 'react';
3 +import { Badge, ClaimBadge } from '@/components/ui/badge';
4 +import { SourceBadge, TableProvenance, type ProvenanceInfo } from '@/components/ui/source-badge';
5 +import type { PublicationListRow } from '@/lib/queries/publications';
6 +import { fmtDate, fmtInt } from '@/lib/format';
5 7
6 −export function PublicationList({ rows }: { rows: Array<PublicationRow & { method?: string; edge_status?: string }> }) {
8 +const PUBMED: ProvenanceInfo = { sourceSlug: 'pubmed', sourceName: 'PubMed (NLM)', dataset: 'PubMed E-utilities', layer: 'normalized' };
9 +
10 +/**
11 + * Publication list. Bibliographic rows all come from PubMed, so one provenance popover sits in the
12 + * caption and each row carries a compact badge. Link method / validation status stay per row.
13 + * Layout classes live in globals.css (.ci-pub) — the list is repeated 25× per page.
14 + */
15 +export function PublicationList({ rows, summary }: { rows: PublicationListRow[]; summary?: ReactNode }) {
16 + const latest = rows.reduce<Date | string | null>((m, p) => (m == null || String(p.updated_at) > String(m) ? p.updated_at : m), null);
7 17 return (
8 − <ol className="divide-y divide-rule">
9 − {rows.map((p) => (
10 − <li key={p.id} className="py-2.5 text-[13.5px]">
11 − <div className="flex flex-wrap items-baseline gap-x-2">
12 − {p.pmid ? (
13 − <Link href={`/publication/${p.pmid}`} className="ci-link font-medium">
14 − {p.title}
15 − </Link>
16 − ) : (
17 − <span className="font-medium">{p.title}</span>
18 − )}
19 − {p.retracted ? <Badge tone="danger">Retracted</Badge> : null}
20 − {p.is_preprint ? <Badge tone="warn">Preprint</Badge> : null}
21 − </div>
22 − <p className="mt-0.5 text-[12.5px] text-ink-3">
23 − {p.authors
24 − .slice(0, 3)
25 − .map((a) => a.name)
26 − .join(', ')}
27 − {p.authors.length > 3 ? ' et al.' : ''}
28 − {p.journal_iso || p.journal ? ` · ${p.journal_iso ?? p.journal}` : ''}
29 − {p.pub_date ? ` · ${fmtDate(p.pub_date)}` : p.pub_year ? ` · ${p.pub_year}` : ''}
30 − {p.pmid ? <span className="ci-mono"> · PMID {p.pmid}</span> : null}
31 − {p.edge_status ? (
32 − <>
33 − {' '}
34 − · <Badge tone={p.edge_status === 'validated' ? 'ok' : 'warn'}>{p.edge_status}</Badge> {p.method ? <span className="ci-mono">{p.method}</span> : null}
35 − </>
36 − ) : null}
37 − </p>
38 − </li>
39 − ))}
40 − </ol>
18 + <div>
19 + <TableProvenance p={{ ...PUBMED, retrievedAt: latest }} claim={<ClaimBadge kind="published" />}>
20 + {summary ?? `${fmtInt(rows.length)} publication${rows.length === 1 ? '' : 's'}`} · bibliographic data as recorded by PubMed; abstracts are not reproduced.
21 + </TableProvenance>
22 + <ol className="ci-pubs">
23 + {rows.map((p) => (
24 + <li key={p.id} className="ci-pub">
25 + <div>
26 + {p.pmid ? (
27 + <Link href={`/publication/${p.pmid}`} className="ci-link font-medium">
28 + {p.title}
29 + </Link>
30 + ) : (
31 + <span className="font-medium">{p.title}</span>
32 + )}
33 + {p.retracted ? <Badge tone="danger">Retracted</Badge> : null}
34 + {p.is_preprint ? <Badge tone="warn">Preprint</Badge> : null}
35 + </div>
36 + <p>
37 + {p.authors.map((a) => a.name).join(', ')}
38 + {p.author_count > 3 ? ' et al.' : ''}
39 + {p.journal_iso || p.journal ? ` · ${p.journal_iso ?? p.journal}` : ''}
40 + {p.pub_date ? ` · ${fmtDate(p.pub_date)}` : p.pub_year ? ` · ${p.pub_year}` : ''}
41 + {p.pmid ? ` · PMID ${p.pmid}` : ''}
42 + {p.edge_status ? (
43 + <>
44 + {' · '}
45 + <Badge tone={p.edge_status === 'validated' ? 'ok' : 'warn'}>{p.edge_status}</Badge>
46 + {p.method ? <span className="ci-mono"> {p.method}</span> : null}
47 + </>
48 + ) : null}{' '}
49 + <SourceBadge compact title={null} p={PUBMED} />
50 + </p>
51 + </li>
52 + ))}
53 + </ol>
54 + </div>
41 55 );
42 56 }
modified apps/web/src/components/data/trial-list.tsx +71 −45
@@ -1,53 +1,79 @@
1 1 import Link from 'next/link';
2 −import { Badge, StatusBadge } from '@/components/ui/badge';
3 −import type { TrialRow } from '@/lib/queries/trials';
2 +import type { ReactNode } from 'react';
3 +import { Badge, ClaimBadge, StatusBadge } from '@/components/ui/badge';
4 +import { SourceBadge, TableProvenance, type ProvenanceInfo } from '@/components/ui/source-badge';
5 +import type { TrialListRow } from '@/lib/queries/trials';
4 6 import { fmtDate, fmtInt, phaseLabel } from '@/lib/format';
5 7
6 −export function TrialTable({ rows }: { rows: TrialRow[] }) {
8 +const CT: ProvenanceInfo = { sourceSlug: 'clinicaltrials', sourceName: 'ClinicalTrials.gov', dataset: 'ClinicalTrials.gov API v2 studies', layer: 'normalized' };
9 +
10 +/**
11 + * Dense trial table. All rows come from one registry, so the provenance popover is rendered once in
12 + * the caption (with the latest retrieval date of the page) and each row carries a compact badge.
13 + */
14 +export function TrialTable({ rows, summary }: { rows: TrialListRow[]; summary?: ReactNode }) {
15 + const latest = rows.reduce<Date | string | null>((m, t) => (m == null || String(t.updated_at) > String(m) ? t.updated_at : m), null);
7 16 return (
8 − <div className="ci-table-wrap">
9 − <table className="ci-table">
10 − <thead>
11 − <tr>
12 − <th>NCT</th>
13 − <th>Title</th>
14 − <th>Status</th>
15 − <th>Phase</th>
16 − <th className="num">Enrollment (n)</th>
17 − <th>Sponsor</th>
18 − <th className="num">Countries</th>
19 − <th>Last update</th>
20 − </tr>
21 − </thead>
22 − <tbody>
23 − {rows.map((t) => (
24 − <tr key={t.id}>
25 − <td>
26 − <Link href={`/trial/${t.nct_id}`} className="ci-mono ci-link">
27 − {t.nct_id}
28 − </Link>
29 − </td>
30 − <td className="min-w-[280px] max-w-[520px]">
31 − <Link href={`/trial/${t.nct_id}`} className="text-ink no-underline hover:text-accent">
32 − {t.brief_title}
33 − </Link>
34 − {t.acronym ? <span className="ml-1 text-[12px] text-ink-3">({t.acronym})</span> : null}
35 − </td>
36 − <td>
37 − <StatusBadge status={t.overall_status} />
38 − </td>
39 − <td className="whitespace-nowrap">{t.phases.length ? t.phases.map(phaseLabel).join(' / ') : <span className="text-ink-4">—</span>}</td>
40 − <td className="num">{t.enrollment_count == null ? '—' : fmtInt(t.enrollment_count)}</td>
41 − <td className="max-w-[220px] truncate text-[12.5px]" title={t.lead_sponsor ?? ''}>
42 − {t.lead_sponsor ?? '—'}
43 − {t.lead_sponsor_class ? <Badge tone="outline" className="ml-1">{t.lead_sponsor_class}</Badge> : null}
44 − </td>
45 − <td className="num">{t.countries.length}</td>
46 − <td className="whitespace-nowrap text-[12.5px]">{fmtDate(t.last_update_posted_date)}</td>
17 + <div>
18 + <TableProvenance p={{ ...CT, retrievedAt: latest }} claim={<ClaimBadge kind="published" />}>
19 + {summary ?? `${fmtInt(rows.length)} stud${rows.length === 1 ? 'y' : 'ies'}`} · status and phase as posted by the registrant; each row links to the study record.
20 + </TableProvenance>
21 + <div className="ci-table-wrap">
22 + <table className="ci-table">
23 + <thead>
24 + <tr>
25 + <th scope="col">NCT</th>
26 + <th scope="col">Title</th>
27 + <th scope="col">Status</th>
28 + <th scope="col">Phase</th>
29 + <th scope="col" className="num">
30 + Enrollment (n)
31 + </th>
32 + <th scope="col">Sponsor</th>
33 + <th scope="col" className="num">
34 + Countries
35 + </th>
36 + <th scope="col">Last update</th>
37 + <th scope="col">Source</th>
47 38 </tr>
48 − ))}
49 − </tbody>
50 − </table>
39 + </thead>
40 + <tbody>
41 + {rows.map((t) => (
42 + <tr key={t.id}>
43 + <td>
44 + <Link href={`/trial/${t.nct_id}`} className="ci-mono ci-link">
45 + {t.nct_id}
46 + </Link>
47 + </td>
48 + <td className="min-w-[280px] max-w-[520px]">
49 + <Link href={`/trial/${t.nct_id}`} className="text-ink no-underline hover:text-accent">
50 + {t.brief_title}
51 + </Link>
52 + {t.acronym ? <span className="ml-1 text-[12px] text-ink-3">({t.acronym})</span> : null}
53 + </td>
54 + <td>
55 + <StatusBadge status={t.overall_status} />
56 + </td>
57 + <td className="whitespace-nowrap">{t.phases.length ? t.phases.map(phaseLabel).join(' / ') : <span className="text-ink-4">—</span>}</td>
58 + <td className="num">{t.enrollment_count == null ? '—' : fmtInt(t.enrollment_count)}</td>
59 + <td className="max-w-[220px] truncate text-[12.5px]" title={t.lead_sponsor ?? ''}>
60 + {t.lead_sponsor ?? '—'}
61 + {t.lead_sponsor_class ? (
62 + <Badge tone="outline" className="ml-1">
63 + {t.lead_sponsor_class}
64 + </Badge>
65 + ) : null}
66 + </td>
67 + <td className="num">{t.countries.length}</td>
68 + <td className="whitespace-nowrap text-[12.5px]">{fmtDate(t.last_update_posted_date)}</td>
69 + <td>
70 + <SourceBadge compact title={null} p={CT} />
71 + </td>
72 + </tr>
73 + ))}
74 + </tbody>
75 + </table>
76 + </div>
51 77 </div>
52 78 );
53 79 }
added apps/web/src/components/home/burden-module.tsx +117 −0
@@ -0,0 +1,117 @@
1 +import Link from 'next/link';
2 +import { Section } from '@/components/ui/section';
3 +import { EmptyState } from '@/components/ui/empty-state';
4 +import { Freshness } from '@/components/ui/freshness';
5 +import { Badge, ClaimBadge } from '@/components/ui/badge';
6 +import { SourceBadge } from '@/components/ui/source-badge';
7 +import { getGeographyBySlug, topCancersFor, yearsFor, type TopCancersResult } from '@/lib/queries/geography';
8 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
9 +import { fmtInt, fmtValue, unitLabel, toDate } from '@/lib/format';
10 +
11 +/**
12 + * Home module "Cancer burden in the United States (latest year)" (§105, §328): top 5 by deaths and top 5 by
13 + * age-standardized incidence, straight from observations, with year and source on every row.
14 + */
15 +export async function BurdenModule({ slug = 'united-states' }: { slug?: string }) {
16 + const geo = await getGeographyBySlug(slug);
17 + const years = geo ? await yearsFor(geo.id) : [];
18 + const latest = years[0];
19 + const [deaths, asir] = geo && latest ? await Promise.all([topCancersFor(geo, 'mortality_count', latest, 'all', 5), topCancersFor(geo, 'as_incidence_rate', latest, 'all', 5)]) : [null, null];
20 + const has = !!(deaths?.rows.length || asir?.rows.length);
21 + const freshest = [...(deaths?.rows ?? []), ...(asir?.rows ?? [])].map((r) => toDate(r.updated_at)).filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? null;
22 +
23 + return (
24 + <Section
25 + id="us-burden"
26 + kicker="Cancer burden"
27 + title={`Cancer burden in the United States${latest ? ` (${latest})` : ''}`}
28 + description="Registry observations as published by the source, both sexes, all ages, per site group. No all-sites total is shown unless the source publishes one."
29 + actions={
30 + <Link href={`/country/${slug}`} className="ci-link">
31 + Country page →
32 + </Link>
33 + }
34 + >
35 + {!has ? (
36 + <EmptyState title="No US observation yet">
37 + Burden figures appear once a licensed registry connector has ingested observations for the United States. IARC / GLOBOCAN remains under license review; SEER awaits credentials.
38 + <div className="mt-1">
39 + <Link href="/countries" className="ci-link">
40 + Countries with data
41 + </Link>
42 + </div>
43 + </EmptyState>
44 + ) : (
45 + <>
46 + <div className="grid gap-6 md:grid-cols-2">
47 + {[deaths, asir].map((res) => (res ? <TopFive key={res.metric} res={res} sex="all" slug={slug} /> : null))}
48 + </div>
49 + <Freshness dataUpdatedAt={freshest} extra={`source: ${deaths?.rows[0]?.source_slug ?? asir?.rows[0]?.source_slug ?? 'registry'} · per-site rows, not summed`} />
50 + </>
51 + )}
52 + </Section>
53 + );
54 +}
55 +
56 +function TopFive({ res, sex, slug }: { res: TopCancersResult; sex: string; slug: string }) {
57 + const first = res.rows[0];
58 + if (!first) {
59 + return (
60 + <div>
61 + <p className="ci-kicker mb-1">{EPI_METRIC_LABEL[res.metric] ?? res.metric}</p>
62 + <EmptyState compact>No {EPI_METRIC_LABEL[res.metric]?.toLowerCase() ?? res.metric} observation for {res.requestedYear}.</EmptyState>
63 + </div>
64 + );
65 + }
66 + const allObserved = res.rows.every((r) => r.estimate_type === 'observed');
67 + return (
68 + <div className="min-w-0">
69 + <p className="mb-1 flex flex-wrap items-baseline justify-between gap-x-2">
70 + <span className="ci-kicker">Top 5 · {EPI_METRIC_LABEL[res.metric] ?? res.metric}</span>
71 + <span className="text-[11.5px] text-ink-3">
72 + {res.year}
73 + {res.year !== res.requestedYear ? ` (latest available; ${res.requestedYear} not yet published)` : ''} · {unitLabel(first.unit)}
74 + </span>
75 + </p>
76 + <div className="ci-table-wrap">
77 + <table className="ci-table">
78 + <thead>
79 + <tr>
80 + <th className="num">#</th>
81 + <th>Cancer</th>
82 + <th className="num">{unitLabel(first.unit)}</th>
83 + <th>Type</th>
84 + </tr>
85 + </thead>
86 + <tbody>
87 + {res.rows.map((r, i) => (
88 + <tr key={r.cancer_id}>
89 + <td className="num">{r.rank ?? i + 1}</td>
90 + <td>
91 + <Link className="ci-link" href={`/cancer/${r.slug}`}>
92 + {r.canonical_name}
93 + </Link>
94 + </td>
95 + <td className="num font-medium">{fmtValue(r.value, r.unit)}</td>
96 + <td>
97 + <Badge tone={r.estimate_type === 'observed' ? 'ok' : 'warn'}>{r.estimate_type}</Badge>
98 + </td>
99 + </tr>
100 + ))}
101 + </tbody>
102 + </table>
103 + </div>
104 + {/* div, not p: the SourceBadge popover contains a <dl>, which the HTML parser would use to close a <p> (hydration mismatch). */}
105 + <div className="mt-1.5 flex flex-wrap items-center gap-1.5 text-[11.5px] text-ink-3">
106 + <SourceBadge p={{ sourceSlug: first.source_slug, sourceName: first.source_name, dataset: first.site_definition ? 'USCS site groups' : null, layer: 'normalized' }} />
107 + <ClaimBadge kind="observed" />
108 + {first.standard_population ? <span>standard: {first.standard_population}</span> : null}
109 + {res.rows.some((r) => r.rank == null) ? <span>rank = position in this table when no snapshot covers the year</span> : <span>rank from snapshot {first.rank_scope_key ? <span className="ci-mono">{first.rank_scope_key}</span> : null}</span>}
110 + {!allObserved ? <span className="italic text-warn">includes estimated values</span> : null}
111 + <Link className="ci-link" href={`/country/${slug}?sex=${sex}&year=${res.year}#${res.metric}`}>
112 + full table ({fmtInt(res.rows.length)} shown) →
113 + </Link>
114 + </div>
115 + </div>
116 + );
117 +}
added apps/web/src/components/home/gaps-module.tsx +95 −0
@@ -0,0 +1,95 @@
1 +import Link from 'next/link';
2 +import { Section } from '@/components/ui/section';
3 +import { EmptyState } from '@/components/ui/empty-state';
4 +import { Freshness } from '@/components/ui/freshness';
5 +import { ClaimBadge, ConfidenceBadge } from '@/components/ui/badge';
6 +import { gapRankings } from '@/lib/queries/rankings';
7 +import { fmtDate, fmtInt, fmtValue, scopeLabel } from '@/lib/format';
8 +
9 +/**
10 + * Home module "Largest trial gaps / research gaps" (§105, §328, §265-266): the gap indexes from the current
11 + * ranking snapshots for a geography. Rendered only when snapshots exist; each row exposes the two percentiles
12 + * that produced the value (lineage stored on the ranking row).
13 + */
14 +export async function GapsModule({ geo = 'USA', limit = 8 }: { geo?: string; limit?: number }) {
15 + const gaps = await gapRankings(geo, limit);
16 + return (
17 + <Section
18 + id="gaps"
19 + kicker="Unmet need"
20 + title="Largest trial and research gaps"
21 + description="Burden percentile minus activity percentile within the same scope. Positive values flag cancers with high mortality burden but comparatively few active trials (trial gap) or little recent literature (research gap). A quantitative signal, not an accusation."
22 + actions={
23 + <Link href="/rankings" className="ci-link">
24 + All rankings →
25 + </Link>
26 + }
27 + >
28 + {gaps.length === 0 ? (
29 + <EmptyState title="Gap indexes not yet computed">
30 + Trial and research gap indexes require a burden snapshot (deaths per top-level cancer for one geography and year) plus trial and literature counters. Nothing is shown until such a snapshot exists.
31 + </EmptyState>
32 + ) : (
33 + <div className="grid gap-6 lg:grid-cols-2">
34 + {gaps.map(({ metric, snapshot, rows }) => (
35 + <div key={metric.slug} className="min-w-0">
36 + <p className="mb-1 flex flex-wrap items-baseline justify-between gap-x-2">
37 + <Link href={`/rankings/${metric.slug}?scope=${encodeURIComponent(snapshot.scope_key)}`} className="ci-link font-medium">
38 + {metric.name}
39 + </Link>
40 + <span className="text-[11.5px] text-ink-3">{scopeLabel(snapshot.scope_key)}</span>
41 + </p>
42 + <div className="ci-table-wrap">
43 + <table className="ci-table">
44 + <thead>
45 + <tr>
46 + <th className="num">#</th>
47 + <th>Cancer</th>
48 + <th className="num">Gap (pct. points)</th>
49 + <th className="num">Deaths</th>
50 + <th className="num">{metric.slug === 'trial_gap' ? 'Active trials' : 'Publications 5y'}</th>
51 + <th>Confidence</th>
52 + </tr>
53 + </thead>
54 + <tbody>
55 + {rows.map((r) => (
56 + <tr key={r.id}>
57 + <td className="num">{r.rank}</td>
58 + <td>
59 + <Link className="ci-link" href={`/cancer/${r.slug}/rankings`}>
60 + {r.canonical_name}
61 + </Link>
62 + </td>
63 + <td className={`num font-medium ${r.value > 0 ? 'text-danger' : ''}`}>{fmtValue(r.value, r.unit)}</td>
64 + <td className="num">{fmtInt(r.inputs.deaths as number | undefined)}</td>
65 + <td className="num">{fmtInt(r.inputs.activity as number | undefined)}</td>
66 + <td>
67 + <ConfidenceBadge level={r.confidence} />
68 + </td>
69 + </tr>
70 + ))}
71 + </tbody>
72 + </table>
73 + </div>
74 + <p className="mt-1.5 flex flex-wrap items-center gap-1.5 text-[11.5px] text-ink-3">
75 + <ClaimBadge kind="computed" />
76 + <span>
77 + formula <code className="ci-mono">{metric.formula}</code> · <span className="ci-mono">{snapshot.formula_version}</span>
78 + </span>
79 + <span>sources: {snapshot.source_ids.join(', ')}</span>
80 + <span>{fmtInt(snapshot.eligible_entities)} eligible</span>
81 + </p>
82 + <p className="mt-1 text-[11.5px] text-ink-3">
83 + {metric.slug === 'trial_gap' ? 'Caveat: trial counts aggregate a cancer and its NCIt descendants; trials registered at a broader level (e.g. "colorectal") are attributed to the broader entity and can overstate the gap of narrower sites.' : 'Caveat: literature counts are query-based per entity and not aggregated over descendants.'}{' '}
84 + <Link className="ci-link" href="/methodology#gap-caveat">
85 + Read the caveat
86 + </Link>
87 + </p>
88 + <Freshness dataUpdatedAt={snapshot.generated_at} extra={`snapshot generated ${fmtDate(snapshot.generated_at)}`} />
89 + </div>
90 + ))}
91 + </div>
92 + )}
93 + </Section>
94 + );
95 +}
added apps/web/src/components/home/rising-module.tsx +89 −0
@@ -0,0 +1,89 @@
1 +import Link from 'next/link';
2 +import { Section } from '@/components/ui/section';
3 +import { EmptyState } from '@/components/ui/empty-state';
4 +import { Freshness } from '@/components/ui/freshness';
5 +import { Badge, ClaimBadge } from '@/components/ui/badge';
6 +import { SourceBadge } from '@/components/ui/source-badge';
7 +import { getGeographyBySlug, cagrFor, CAGR_FORMULA, CAGR_FORMULA_VERSION } from '@/lib/queries/geography';
8 +import { fmtNum, fmtValue, fmtInt, toDate } from '@/lib/format';
9 +
10 +/**
11 + * Home module "Fastest-rising incidence" (§105, §328): ASIR compound annual growth rate over the last ten
12 + * available years, computed on the fly from observations. Shown only when ≥ 10 years exist; labelled as
13 + * derived, with the formula and both endpoints on every row so the number can be reproduced.
14 + */
15 +export async function RisingModule({ slug = 'united-states', limit = 8 }: { slug?: string; limit?: number }) {
16 + const geo = await getGeographyBySlug(slug);
17 + const res = geo ? await cagrFor(geo.id, 'as_incidence_rate', 'all', 10) : null;
18 + const rows = res?.rows.slice(0, limit) ?? [];
19 + const first = rows[0];
20 + return (
21 + <Section
22 + id="rising"
23 + kicker="Trend"
24 + title={`Fastest-rising incidence${geo ? ` · ${geo.name}` : ''}`}
25 + description={res ? `Age-standardized incidence rate, both sexes, all ages: compound annual growth rate between ${res.startYear} and ${res.endYear} (the last 10 years published). Derived by CancerIndex from registry observations; not a source figure.` : 'Age-standardized incidence rate: compound annual growth rate over the last 10 published years.'}
26 + actions={
27 + geo ? (
28 + <Link href={`/country/${slug}#trend`} className="ci-link">
29 + Trend chart →
30 + </Link>
31 + ) : undefined
32 + }
33 + >
34 + {!res || rows.length === 0 ? (
35 + <EmptyState title="Not enough years to compute a trend">
36 + A 10-year growth rate is computed only when at least ten distinct years of age-standardized incidence exist for the geography{res ? ` (${res.yearsAvailable} available)` : ''}. Nothing is extrapolated from shorter series.
37 + </EmptyState>
38 + ) : (
39 + <>
40 + <div className="ci-table-wrap">
41 + <table className="ci-table">
42 + <thead>
43 + <tr>
44 + <th>Cancer</th>
45 + <th className="num">CAGR (%/yr)</th>
46 + <th className="num">{res.startYear} (per 100,000)</th>
47 + <th className="num">{res.endYear} (per 100,000)</th>
48 + <th className="num">Years</th>
49 + </tr>
50 + </thead>
51 + <tbody>
52 + {rows.map((r) => (
53 + <tr key={r.cancer_id}>
54 + <td>
55 + <Link className="ci-link" href={`/cancer/${r.slug}/statistics`}>
56 + {r.canonical_name}
57 + </Link>
58 + {r.estimate_types.some((t) => t !== 'observed') ? <Badge tone="warn" className="ml-1.5">includes estimates</Badge> : null}
59 + </td>
60 + <td className={`num font-medium ${r.cagr > 0 ? 'text-danger' : r.cagr < 0 ? 'text-ok' : ''}`}>
61 + {r.cagr > 0 ? '+' : ''}
62 + {fmtNum(r.cagr * 100, 2)}
63 + </td>
64 + <td className="num">{fmtValue(r.start_value, r.unit)}</td>
65 + <td className="num">{fmtValue(r.end_value, r.unit)}</td>
66 + <td className="num text-ink-3">{fmtInt(r.n_years)}</td>
67 + </tr>
68 + ))}
69 + </tbody>
70 + </table>
71 + </div>
72 + <div className="mt-2 flex flex-wrap items-center gap-1.5 text-[12px] text-ink-3">
73 + <ClaimBadge kind="computed" />
74 + {first ? <SourceBadge p={{ sourceSlug: first.source_slug, layer: 'derived', note: 'Inputs are the two endpoint observations (start and end year) of the source series.' }} /> : null}
75 + <span>
76 + formula: <code className="ci-mono">{CAGR_FORMULA}</code>
77 + </span>
78 + <span className="ci-mono">{CAGR_FORMULA_VERSION}</span>
79 + {first?.standard_population ? <span>standard: {first.standard_population}</span> : null}
80 + </div>
81 + <p className="mt-1 text-[12px] text-ink-3">
82 + Rising incidence can reflect true risk changes, screening or diagnostic practice, coding changes or population ageing beyond what standardization removes; the rate says nothing about causes. Falling rates are listed on the <Link className="ci-link" href="/methodology#cagr">methodology page</Link> definition.
83 + </p>
84 + <Freshness dataUpdatedAt={rows.map((r) => toDate(r.updated_at)).filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? null} sourceVersion={`${res.startYear}–${res.endYear}`} extra={`${fmtInt(res.rows.length)} cancers with a complete 10-year series`} />
85 + </>
86 + )}
87 + </Section>
88 + );
89 +}
modified apps/web/src/components/ui/badge.tsx +16 −7
@@ -2,18 +2,22 @@ import type { ReactNode } from 'react';
2 2
3 3 type Tone = 'neutral' | 'accent' | 'warn' | 'danger' | 'ok' | 'outline';
4 4
5 +// Tones map to short CSS classes (globals.css `.ci-badge--*`) instead of long Tailwind strings: dense
6 +// tables repeat badges hundreds of times and every className is shipped twice (HTML + RSC payload).
5 7 const TONES: Record<Tone, string> = {
6 − neutral: 'bg-paper-3 text-ink-2 border-transparent',
7 − accent: 'bg-accent-soft text-accent-2 border-transparent',
8 − warn: 'bg-warn-soft text-warn border-transparent',
9 − danger: 'bg-danger-soft text-danger border-transparent',
10 − ok: 'bg-ok-soft text-ok border-transparent',
11 − outline: 'bg-transparent text-ink-2 border-rule-strong',
8 + neutral: 'ci-badge--neutral',
9 + accent: 'ci-badge--accent',
10 + warn: 'ci-badge--warn',
11 + danger: 'ci-badge--danger',
12 + ok: 'ci-badge--ok',
13 + outline: 'ci-badge--outline',
12 14 };
13 15
14 16 export function Badge({ children, tone = 'neutral', title, className = '', mono = false }: { children: ReactNode; tone?: Tone; title?: string; className?: string; mono?: boolean }) {
17 + // Only spread `title` when set: an undefined prop is serialized as "$undefined" in the RSC payload.
18 + const extra = title ? { title } : {};
15 19 return (
16 − <span title={title} className={`inline-flex items-center gap-1 rounded-sm border px-1.5 py-[1px] text-[11px] font-medium leading-4 tracking-wide ${mono ? 'ci-mono' : ''} ${TONES[tone]} ${className}`}>
20 + <span {...extra} className={`ci-badge ${TONES[tone]}${mono ? ' ci-mono' : ''}${className ? ` ${className}` : ''}`}>
17 21 {children}
18 22 </span>
19 23 );
@@ -80,6 +84,11 @@ export function ConfidenceBadge({ level, className = '' }: { level: string | nul
80 84 );
81 85 }
82 86
87 +/** True for mappings made through a shared identifier or a curated/ontology one-to-one match. */
88 +export function isExactMatch(matchType: string | null | undefined): boolean {
89 + return matchType === 'EXACT_IDENTIFIER' || matchType === 'CURATED_EXACT' || matchType === 'ONTOLOGY_EXACT';
90 +}
91 +
83 92 /** Entity-mapping confidence (CLAUDE.md §221). */
84 93 export function MatchBadge({ matchType, className = '' }: { matchType: string | null | undefined; className?: string }) {
85 94 const m = matchType ?? 'UNRESOLVED';
added apps/web/src/components/ui/pager.tsx +86 −0
@@ -0,0 +1,86 @@
1 +import Link from 'next/link';
2 +import { fmtInt } from '@/lib/format';
3 +import { pageInfo, pageWindow } from '@/lib/pagination';
4 +
5 +/**
6 + * Server-side pager (URL state, §295). Renders "Showing a–b of N", previous/next links with
7 + * rel=prev/next, and a compact page window. Pure links: works without JavaScript and is crawlable.
8 + */
9 +export function Pager({
10 + total,
11 + pageSize,
12 + page,
13 + hrefFor,
14 + label = 'Pagination',
15 + noun,
16 + className = '',
17 +}: {
18 + total: number;
19 + pageSize: number;
20 + page: number;
21 + /** URL for a given page (1-based). Page 1 should drop the param so canonical URLs stay clean. */
22 + hrefFor: (page: number) => string;
23 + /** aria-label for the <nav>; use a distinct label when a page has several pagers. */
24 + label?: string;
25 + /** Row noun for the summary, e.g. "evidence items". */
26 + noun?: string;
27 + className?: string;
28 +}) {
29 + const info = pageInfo(page, pageSize, total);
30 + if (total <= 0) return null;
31 + const linkCls = 'ci-pager-link';
32 + const disabledCls = 'ci-pager-link is-disabled';
33 + return (
34 + <nav aria-label={label} className={`ci-pager mt-3 flex flex-wrap items-center justify-between gap-2 text-[13px] text-ink-2 ${className}`}>
35 + <p className="m-0">
36 + Showing <span className="ci-num">{fmtInt(info.from)}</span>–<span className="ci-num">{fmtInt(info.to)}</span> of <span className="ci-num">{fmtInt(total)}</span>
37 + {noun ? ` ${noun}` : ''}
38 + </p>
39 + {info.pageCount > 1 ? (
40 + <ul className="m-0 flex list-none items-center gap-1 p-0">
41 + <li>
42 + {info.hasPrev ? (
43 + <Link className={linkCls} href={hrefFor(info.page - 1)} rel="prev" aria-label={`Previous page (${info.page - 1})`}>
44 + ← Previous
45 + </Link>
46 + ) : (
47 + <span className={disabledCls} aria-disabled="true">
48 + ← Previous
49 + </span>
50 + )}
51 + </li>
52 + {pageWindow(info.page, info.pageCount).map((p, i) =>
53 + p == null ? (
54 + <li key={`gap-${i}`} aria-hidden className="px-1 text-ink-4">
55 + …
56 + </li>
57 + ) : (
58 + <li key={p}>
59 + {p === info.page ? (
60 + <span className="ci-pager-link is-current ci-num" aria-current="page" aria-label={`Page ${p}, current page`}>
61 + {p}
62 + </span>
63 + ) : (
64 + <Link className={`${linkCls} ci-num`} href={hrefFor(p)} aria-label={`Page ${p}`}>
65 + {p}
66 + </Link>
67 + )}
68 + </li>
69 + ),
70 + )}
71 + <li>
72 + {info.hasNext ? (
73 + <Link className={linkCls} href={hrefFor(info.page + 1)} rel="next" aria-label={`Next page (${info.page + 1})`}>
74 + Next →
75 + </Link>
76 + ) : (
77 + <span className={disabledCls} aria-disabled="true">
78 + Next →
79 + </span>
80 + )}
81 + </li>
82 + </ul>
83 + ) : null}
84 + </nav>
85 + );
86 +}
modified apps/web/src/components/ui/pagination.tsx +6 −32
@@ -1,34 +1,8 @@
1 −import Link from 'next/link';
2 −import { fmtInt } from '@/lib/format';
1 +import { Pager } from './pager';
3 2
4 −export function Pagination({ page, pageSize, total, hrefFor }: { page: number; pageSize: number; total: number; hrefFor: (page: number) => string }) {
5 − const pages = Math.max(1, Math.ceil(total / pageSize));
6 − const from = total === 0 ? 0 : (page - 1) * pageSize + 1;
7 − const to = Math.min(total, page * pageSize);
8 − return (
9 − <nav aria-label="Pagination" className="mt-3 flex flex-wrap items-center justify-between gap-2 text-[13px] text-ink-2">
10 − <p>
11 − Showing <span className="ci-num">{fmtInt(from)}</span>–<span className="ci-num">{fmtInt(to)}</span> of <span className="ci-num">{fmtInt(total)}</span>
12 − </p>
13 − <div className="flex items-center gap-1">
14 − {page > 1 ? (
15 − <Link className="border border-rule px-2 py-1 hover:border-accent hover:text-accent" href={hrefFor(page - 1)} rel="prev">
16 − ← Previous
17 − </Link>
18 − ) : (
19 − <span className="border border-rule px-2 py-1 text-ink-4">← Previous</span>
20 − )}
21 − <span className="px-2">
22 − Page <span className="ci-num">{page}</span> / <span className="ci-num">{pages}</span>
23 − </span>
24 − {page < pages ? (
25 − <Link className="border border-rule px-2 py-1 hover:border-accent hover:text-accent" href={hrefFor(page + 1)} rel="next">
26 − Next →
27 − </Link>
28 − ) : (
29 − <span className="border border-rule px-2 py-1 text-ink-4">Next →</span>
30 − )}
31 − </div>
32 − </nav>
33 − );
3 +/** Backwards-compatible alias: existing list pages use <Pagination>; new code should use <Pager>. */
4 +export function Pagination({ page, pageSize, total, hrefFor, label }: { page: number; pageSize: number; total: number; hrefFor: (page: number) => string; label?: string }) {
5 + return <Pager page={page} pageSize={pageSize} total={total} hrefFor={hrefFor} label={label} />;
34 6 }
7 +
8 +export { Pager };
added apps/web/src/components/ui/skeleton.tsx +55 −0
@@ -0,0 +1,55 @@
1 +/**
2 + * Loading skeletons for route-level loading.tsx files. Neutral blocks only (no fake numbers, no
3 + * placeholder text that could be mistaken for data — CLAUDE.md §281). `aria-busy` + a visually
4 + * hidden status line announce the loading state to assistive technology.
5 + */
6 +export function Skeleton({ className = '', w, h = 14 }: { className?: string; w?: string | number; h?: number }) {
7 + return <span aria-hidden className={`ci-skeleton block ${className}`} style={{ width: w ?? '100%', height: h }} />;
8 +}
9 +
10 +export function SkeletonText({ lines = 3, className = '' }: { lines?: number; className?: string }) {
11 + return (
12 + <div className={`space-y-2 ${className}`} aria-hidden>
13 + {Array.from({ length: lines }, (_, i) => (
14 + <Skeleton key={i} w={i === lines - 1 ? '62%' : '100%'} />
15 + ))}
16 + </div>
17 + );
18 +}
19 +
20 +export function SkeletonTable({ rows = 8, cols = 6, className = '' }: { rows?: number; cols?: number; className?: string }) {
21 + return (
22 + <div className={`ci-table-wrap ${className}`} aria-hidden>
23 + <div className="grid gap-x-3 gap-y-0 px-2" style={{ gridTemplateColumns: `repeat(${cols}, minmax(0, 1fr))` }}>
24 + {Array.from({ length: cols }, (_, i) => (
25 + <Skeleton key={`h${i}`} h={10} className="my-3" w="60%" />
26 + ))}
27 + {Array.from({ length: rows * cols }, (_, i) => (
28 + <Skeleton key={i} h={12} className="my-[9px]" w={`${55 + ((i * 17) % 40)}%`} />
29 + ))}
30 + </div>
31 + </div>
32 + );
33 +}
34 +
35 +export function PageSkeleton({ title = 'Loading', tabs = 0, table = true }: { title?: string; tabs?: number; table?: boolean }) {
36 + return (
37 + <div aria-busy="true" className="pt-6">
38 + <p className="sr-only" role="status">
39 + {title}…
40 + </p>
41 + <Skeleton h={10} w={120} className="mb-3" />
42 + <Skeleton h={34} w="55%" className="mb-3" />
43 + <Skeleton h={12} w="35%" className="mb-6" />
44 + {tabs > 0 ? (
45 + <div className="mb-6 flex gap-5 border-b border-rule pb-2" aria-hidden>
46 + {Array.from({ length: tabs }, (_, i) => (
47 + <Skeleton key={i} h={12} w={64 + ((i * 23) % 40)} />
48 + ))}
49 + </div>
50 + ) : null}
51 + <SkeletonText lines={2} className="mb-6 max-w-2xl" />
52 + {table ? <SkeletonTable /> : null}
53 + </div>
54 + );
55 +}
modified apps/web/src/components/ui/source-badge.tsx +38 −9
@@ -17,27 +17,41 @@ export interface ProvenanceInfo {
17 17 note?: string | null;
18 18 }
19 19
20 +/** One-line provenance summary for `title` attributes on compact badges (dataset · version · retrieved). */
21 +export function provenanceTitle(p: ProvenanceInfo): string {
22 + const bits = [p.sourceName ?? p.sourceSlug];
23 + if (p.dataset) bits.push(p.dataset);
24 + if (p.datasetVersion) bits.push(`version ${p.datasetVersion}`);
25 + if (p.retrievedAt) bits.push(`retrieved ${fmtDate(p.retrievedAt)}`);
26 + if (p.layer === 'derived' || p.layer === 'ranked') bits.push('computed by CancerIndex');
27 + return bits.join(' · ');
28 +}
29 +
20 30 /**
21 31 * Small source badge; hover / focus reveals the provenance popover (source, dataset, version,
22 32 * retrieved date, raw vs normalized, link). CSS-only so it works inside server components.
33 + *
34 + * `compact`: link only, no popover — for dense tables. When every row shares one source/dataset the
35 + * popover is shown once in the table caption; when provenance genuinely differs per row (e.g.
36 + * epidemiology observations from different datasets/years) pass `title` (or let it default to a
37 + * dataset · version · retrieved summary) so the detail stays one hover away without the markup.
23 38 */
24 −export function SourceBadge({ p, className = '', compact = false }: { p: ProvenanceInfo; className?: string; compact?: boolean }) {
25 − // compact: link only, no popover — for dense tables where every row shares one source/dataset (the
26 − // popover is then shown once in the table caption).
39 +export function SourceBadge({ p, className = '', compact = false, title }: { p: ProvenanceInfo; className?: string; compact?: boolean; title?: string | null }) {
27 40 if (compact) {
41 + // The visible slug is the link text; the "Source" column header gives the context and `title`
42 + // (dataset · version · retrieved) is exposed as the accessible description. No aria-label
43 + // duplicate: every attribute here is repeated hundreds of times per page (HTML + RSC payload).
44 + const t = title === null ? undefined : (title ?? (p.dataset || p.datasetVersion ? provenanceTitle(p) : undefined));
45 + const extra = t ? { title: t } : {};
28 46 return (
29 − <Link href={`/source/${p.sourceSlug}`} className={`ci-mono inline-flex items-center rounded-sm border border-rule-strong bg-paper px-1 py-[0px] text-[10.5px] leading-4 text-ink-2 no-underline hover:border-accent hover:text-accent focus:border-accent ${className}`} aria-label={`Source: ${p.sourceName ?? p.sourceSlug}`}>
47 + <Link href={`/source/${p.sourceSlug}`} className={`ci-src${className ? ` ${className}` : ''}`} {...extra}>
30 48 {p.sourceSlug}
31 49 </Link>
32 50 );
33 51 }
34 52 return (
35 53 <span className={`ci-pop ${className}`}>
36 − <Link
37 − href={`/source/${p.sourceSlug}`}
38 − className="ci-mono inline-flex items-center rounded-sm border border-rule-strong bg-paper px-1 py-[0px] text-[10.5px] leading-4 text-ink-2 no-underline hover:border-accent hover:text-accent focus:border-accent"
39 − aria-label={`Source: ${p.sourceName ?? p.sourceSlug}. Open provenance.`}
40 − >
54 + <Link href={`/source/${p.sourceSlug}`} className="ci-src" aria-label={`Source: ${p.sourceName ?? p.sourceSlug}. Open provenance.`}>
41 55 {p.sourceSlug}
42 56 </Link>
43 57 <span className="ci-pop-panel" role="tooltip">
@@ -100,3 +114,18 @@ export function SourceBadge({ p, className = '', compact = false }: { p: Provena
100 114 </span>
101 115 );
102 116 }
117 +
118 +/**
119 + * Table caption used by every dense list: ONE provenance popover for the whole table, the claim
120 + * label and a count sentence. Rows then carry compact badges only.
121 + */
122 +export function TableProvenance({ p, claim, children, className = '' }: { p: ProvenanceInfo; claim?: React.ReactNode; children?: React.ReactNode; className?: string }) {
123 + return (
124 + // <div>, not <p>: the popover contains a <dl>, which the HTML parser would close a <p> on (React #418 in production).
125 + <div className={`mb-2 flex flex-wrap items-center gap-2 text-[12px] text-ink-3 ${className}`}>
126 + <SourceBadge p={p} />
127 + {claim}
128 + {children ? <span>{children}</span> : null}
129 + </div>
130 + );
131 +}
modified apps/web/src/components/ui/tabs.tsx +17 −7
@@ -7,18 +7,28 @@ export interface TabDef {
7 7 count?: number | null;
8 8 }
9 9
10 −/** Sticky, horizontally scrollable tab bar (server component; the current tab is a route segment). */
10 +/**
11 + * Sticky, horizontally scrollable tab bar (server component; the current tab is a route segment).
12 + * These are links to distinct URLs, so the correct semantics are a labelled <nav> with
13 + * aria-current="page" — not the ARIA tabs pattern, which implies in-page panel switching.
14 + */
11 15 export function Tabs({ tabs, current, ariaLabel }: { tabs: TabDef[]; current: string; ariaLabel: string }) {
12 16 return (
13 17 <nav aria-label={ariaLabel} className="ci-tabs -mx-4 px-4 sm:-mx-6 sm:px-6">
14 − <div className="flex">
18 + <ul className="m-0 flex list-none p-0">
15 19 {tabs.map((t) => (
16 − <Link key={t.key} href={t.href} className="ci-tab" aria-current={t.key === current ? 'page' : undefined} scroll={false}>
17 − {t.label}
18 − {t.count != null && t.count > 0 ? <span className="ci-num ml-1 text-[11px] text-ink-3">{t.count}</span> : null}
19 − </Link>
20 + <li key={t.key}>
21 + <Link href={t.href} className="ci-tab" {...(t.key === current ? { 'aria-current': 'page' as const } : {})} scroll={false}>
22 + {t.label}
23 + {t.count != null && t.count > 0 ? (
24 + <span className="ci-num ml-1 text-[11px] text-ink-3" aria-label={`${t.count} items`}>
25 + {t.count}
26 + </span>
27 + ) : null}
28 + </Link>
29 + </li>
20 30 ))}
21 − </div>
31 + </ul>
22 32 </nav>
23 33 );
24 34 }
added apps/web/src/lib/db-errors.ts +28 −0
@@ -0,0 +1,28 @@
1 +/** Pure error classification for database failures (no server-only import so it is unit-testable). */
2 +
3 +/** Thrown when the database itself is unreachable (connection refused, auth, pool exhausted…). */
4 +export class DatabaseUnavailableError extends Error {
5 + readonly code: string | undefined;
6 + constructor(cause: unknown) {
7 + const e = cause as { message?: string; code?: string } | undefined;
8 + super(`Database unavailable${e?.code ? ` (${e.code})` : ''}: ${e?.message ?? String(cause)}`);
9 + this.name = 'DatabaseUnavailableError';
10 + this.code = e?.code;
11 + }
12 +}
13 +
14 +// Node socket errors + postgres.js connection errors + PostgreSQL SQLSTATE classes 08 (connection
15 +// exception), 28 (auth), 3D (bad database), 53 (insufficient resources) and 57P (operator intervention).
16 +const CONNECTION_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'EHOSTUNREACH', 'EPIPE', 'EAI_AGAIN', 'CONNECT_TIMEOUT', 'CONNECTION_CLOSED', 'CONNECTION_ENDED', 'CONNECTION_DESTROYED']);
17 +const CONNECTION_SQLSTATE = /^(08|28|3D|53|57P)/;
18 +const CONNECTION_MESSAGE = /ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|connection (refused|terminated|closed|ended)|CONNECT_TIMEOUT|too many connections|password authentication failed/i;
19 +
20 +/** True when the error means "the database is unreachable" rather than "this query/table is wrong". */
21 +export function isConnectionError(err: unknown, depth = 0): boolean {
22 + if (!err || typeof err !== 'object' || depth > 4) return false;
23 + const e = err as { code?: unknown; errno?: unknown; message?: unknown; cause?: unknown };
24 + const code = typeof e.code === 'string' ? e.code : typeof e.errno === 'string' ? e.errno : '';
25 + if (code && (CONNECTION_CODES.has(code) || CONNECTION_SQLSTATE.test(code))) return true;
26 + if (typeof e.message === 'string' && CONNECTION_MESSAGE.test(e.message)) return true;
27 + return e.cause != null && e.cause !== err ? isConnectionError(e.cause, depth + 1) : false;
28 +}
modified apps/web/src/lib/db.ts +13 −2
@@ -52,14 +52,25 @@ export const {
52 52 entityCounters,
53 53 } = schema;
54 54
55 +import { DatabaseUnavailableError, isConnectionError } from '@/lib/db-errors';
56 +export { DatabaseUnavailableError, isConnectionError };
57 +
55 58 /**
56 − * Run a query and swallow "relation does not exist" style errors so pages render an EmptyState
57 − * instead of crashing when a table has not been created on this environment yet.
59 + * Run a query and swallow query-level errors ("relation does not exist", bad column, cast…) so
60 + * pages render an EmptyState instead of crashing when a table has not been created on this
61 + * environment yet. Connection-level errors are NOT swallowed: they are rethrown as
62 + * DatabaseUnavailableError so the route's error.tsx renders (and nothing misleading — a 404 or
63 + * an empty state — gets cached by ISR while the database is down).
58 64 */
59 65 export async function safe<T>(fn: () => Promise<T>, fallback: T): Promise<T> {
60 66 try {
61 67 return await fn();
62 68 } catch (err) {
69 + if (err instanceof DatabaseUnavailableError) throw err;
70 + if (isConnectionError(err)) {
71 + if (process.env.NODE_ENV !== 'production') console.error('[cancerindex/web] database unavailable:', (err as Error).message);
72 + throw new DatabaseUnavailableError(err);
73 + }
63 74 if (process.env.NODE_ENV !== 'production') console.error('[cancerindex/web] query failed:', (err as Error).message);
64 75 return fallback;
65 76 }
added apps/web/src/lib/pagination.ts +62 −0
@@ -0,0 +1,62 @@
1 +/** Pure pagination math shared by server components, query helpers and tests (§295 URL state). */
2 +
3 +export interface PageInfo {
4 + /** 1-based current page, clamped into [1, pageCount]. */
5 + page: number;
6 + pageSize: number;
7 + total: number;
8 + pageCount: number;
9 + /** SQL OFFSET for the clamped page. */
10 + offset: number;
11 + /** 1-based index of the first row shown (0 when total = 0). */
12 + from: number;
13 + /** 1-based index of the last row shown (0 when total = 0). */
14 + to: number;
15 + hasPrev: boolean;
16 + hasNext: boolean;
17 +}
18 +
19 +export function pageCount(total: number, pageSize: number): number {
20 + if (!Number.isFinite(total) || total <= 0) return 1;
21 + const size = Math.max(1, Math.floor(pageSize));
22 + return Math.max(1, Math.ceil(total / size));
23 +}
24 +
25 +/** Clamp a requested page into the valid range for `total` rows (always ≥ 1). */
26 +export function clampPage(page: number, total: number, pageSize: number): number {
27 + const n = Number.isFinite(page) ? Math.floor(page) : 1;
28 + return Math.min(pageCount(total, pageSize), Math.max(1, n));
29 +}
30 +
31 +export function offsetFor(page: number, pageSize: number): number {
32 + return (Math.max(1, Math.floor(page)) - 1) * Math.max(1, Math.floor(pageSize));
33 +}
34 +
35 +export function pageInfo(page: number, pageSize: number, total: number): PageInfo {
36 + const size = Math.max(1, Math.floor(pageSize));
37 + const count = pageCount(total, size);
38 + const p = clampPage(page, total, size);
39 + const from = total === 0 ? 0 : (p - 1) * size + 1;
40 + const to = total === 0 ? 0 : Math.min(total, p * size);
41 + return { page: p, pageSize: size, total, pageCount: count, offset: (p - 1) * size, from, to, hasPrev: p > 1, hasNext: p < count };
42 +}
43 +
44 +/**
45 + * Compact list of page numbers to show as links: first, last, current ±1, with `null` gaps.
46 + * e.g. page 7 of 20 → [1, null, 6, 7, 8, null, 20].
47 + */
48 +export function pageWindow(page: number, count: number, radius = 1): Array<number | null> {
49 + if (count <= 1) return [1];
50 + const want = new Set<number>([1, count]);
51 + for (let i = page - radius; i <= page + radius; i++) if (i >= 1 && i <= count) want.add(i);
52 + const sorted = [...want].sort((a, b) => a - b);
53 + const out: Array<number | null> = [];
54 + for (let i = 0; i < sorted.length; i++) {
55 + const cur = sorted[i]!;
56 + const prev = sorted[i - 1];
57 + if (prev != null && cur - prev === 2) out.push(prev + 1);
58 + else if (prev != null && cur - prev > 2) out.push(null);
59 + out.push(cur);
60 + }
61 + return out;
62 +}
added apps/web/src/lib/queries/compare.ts +79 −0
@@ -0,0 +1,79 @@
1 +import 'server-only';
2 +import { run, sql, safe } from '@/lib/db';
3 +import type { CancerCore, Counters } from '@/lib/queries/cancers';
4 +import { latestFiguresFor, nearestRegistryAncestor, type LatestFigure, type RegistryAncestor } from '@/lib/queries/epidemiology';
5 +import { rankingsForCancers, pickLatestScopes, type CancerRanking } from '@/lib/queries/rankings';
6 +
7 +/**
8 + * Compare engine (SPEC §100): side-by-side facts for 2–4 cancers. Only facts that exist in the database are
9 + * shown; registry figures for entities below the top level fall back to the nearest top-level ancestor and
10 + * are labelled as such (see /methodology#compare).
11 + */
12 +
13 +export const COMPARE_MIN = 2;
14 +export const COMPARE_MAX = 4;
15 +
16 +export interface CompareEntity {
17 + cancer: CancerCore;
18 + parents: Array<{ slug: string; canonical_name: string; hierarchy_type: string }>;
19 + counters: Counters | null;
20 + registry: RegistryAncestor | null; // entity whose observations are shown (self when top-level)
21 + figures: Map<string, LatestFigure>; // metric → latest observation (sex=all, USA)
22 + ranks: Array<CancerRanking & { cancer_id: string }>; // latest year per scope
23 +}
24 +
25 +export function parseCompareIds(raw: string | undefined): string[] {
26 + if (!raw) return [];
27 + const seen = new Set<string>();
28 + const out: string[] = [];
29 + for (const part of raw.split(',')) {
30 + const s = part.trim().toLowerCase();
31 + if (!/^[a-z0-9][a-z0-9-]{0,120}$/.test(s) || seen.has(s)) continue;
32 + seen.add(s);
33 + out.push(s);
34 + if (out.length >= COMPARE_MAX) break;
35 + }
36 + return out;
37 +}
38 +
39 +export async function resolveCancers(slugs: string[]): Promise<CancerCore[]> {
40 + if (slugs.length === 0) return [];
41 + const rows = await safe(() => run<CancerCore>(sql`SELECT * FROM cancers WHERE slug IN (${sql.join(slugs.map((s) => sql`${s}`), sql`, `)}) AND status <> 'merged'`), [] as CancerCore[]);
42 + const bySlug = new Map(rows.map((r) => [r.slug, r]));
43 + return slugs.map((s) => bySlug.get(s)).filter((r): r is CancerCore => !!r);
44 +}
45 +
46 +export async function loadCompare(slugs: string[]): Promise<CompareEntity[]> {
47 + const cancers = await resolveCancers(slugs);
48 + if (cancers.length === 0) return [];
49 + const ids = cancers.map((c) => c.id);
50 + const [parents, counters, ancestors] = await Promise.all([
51 + safe(
52 + () =>
53 + run<{ child_id: string; slug: string; canonical_name: string; hierarchy_type: string }>(sql`
54 + SELECT h.child_id, p.slug, p.canonical_name, h.hierarchy_type FROM cancer_hierarchy h JOIN cancers p ON p.id = h.parent_id
55 + WHERE h.child_id IN (${sql.join(ids.map((i) => sql`${i}`), sql`, `)}) ORDER BY h.hierarchy_type, p.canonical_name`),
56 + [],
57 + ),
58 + safe(() => run<Counters & { entity_id: string }>(sql`SELECT * FROM entity_counters WHERE entity_type = 'cancer' AND entity_id IN (${sql.join(ids.map((i) => sql`${i}`), sql`, `)})`), []),
59 + Promise.all(cancers.map((c) => (c.top_level ? Promise.resolve<RegistryAncestor | null>({ id: c.id, slug: c.slug, canonical_name: c.canonical_name, depth: 0 }) : nearestRegistryAncestor(c.id)))),
60 + ]);
61 + const registryIds = [...new Set(ancestors.filter((a): a is RegistryAncestor => !!a).map((a) => a.id))];
62 + const rankIds = [...new Set([...ids, ...registryIds])];
63 + const [figures, ranks] = await Promise.all([latestFiguresFor(registryIds, 'USA', 'all'), rankingsForCancers(rankIds)]);
64 + const latestRanks = pickLatestScopes(ranks);
65 + const countersById = new Map(counters.map((c) => [c.entity_id, c]));
66 + return cancers.map((c, i) => {
67 + const registry = ancestors[i] ?? null;
68 + const figs = new Map<string, LatestFigure>();
69 + if (registry) for (const f of figures) if (f.cancer_id === registry.id) figs.set(f.metric, f);
70 + return {
71 + cancer: c,
72 + parents: parents.filter((p) => p.child_id === c.id).slice(0, 4),
73 + counters: countersById.get(c.id) ?? null,
74 + registry,
75 + figures: figs,
76 + ranks: latestRanks.filter((r) => r.cancer_id === c.id || (registry && registry.depth > 0 && r.cancer_id === registry.id && /^(mortality_count|incidence_count|as_mortality_rate|as_incidence_rate|mortality_incidence_ratio|trial_gap|research_gap)$/.test(r.metric_slug))),
77 + };
78 + });
79 +}
modified apps/web/src/lib/queries/epidemiology.ts +75 −0
@@ -73,6 +73,81 @@ export async function survivalFor(cancerId: string, limit = 1000): Promise<SurvO
73 73 );
74 74 }
75 75
76 +// ---------- Registry-level figures (overview hero, compare page, home) ----------
77 +
78 +export interface RegistryAncestor {
79 + id: string;
80 + slug: string;
81 + canonical_name: string;
82 + depth: number; // hierarchy steps from the requested entity (0 = the entity itself)
83 +}
84 +
85 +/**
86 + * Nearest ancestor that belongs to the mutually exclusive top-level registry set (§246-247), walking every
87 + * hierarchy type upwards. Returns the entity itself (depth 0) when it is top-level; null when no top-level
88 + * ancestor exists (e.g. non-malignant branches).
89 + */
90 +export async function nearestRegistryAncestor(cancerId: string): Promise<RegistryAncestor | null> {
91 + const rows = await safe(
92 + () =>
93 + run<RegistryAncestor>(sql`
94 + WITH RECURSIVE up AS (
95 + SELECT c.id, 0 AS depth, ARRAY[c.id]::varchar[] AS path FROM cancers c WHERE c.id = ${cancerId}
96 + UNION ALL
97 + SELECT h.parent_id, up.depth + 1, up.path || h.parent_id FROM up JOIN cancer_hierarchy h ON h.child_id = up.id
98 + WHERE up.depth < 12 AND NOT (h.parent_id = ANY(up.path))
99 + )
100 + SELECT c.id, c.slug, c.canonical_name, min(up.depth)::int AS depth
101 + FROM up JOIN cancers c ON c.id = up.id
102 + WHERE c.top_level AND c.status = 'active'
103 + GROUP BY c.id, c.slug, c.canonical_name ORDER BY depth ASC LIMIT 1`),
104 + [] as RegistryAncestor[],
105 + );
106 + return rows[0] ?? null;
107 +}
108 +
109 +export interface LatestFigure {
110 + cancer_id: string;
111 + metric: string;
112 + year: number;
113 + year_end: number | null;
114 + sex: string;
115 + value: number;
116 + unit: string;
117 + lower_ci: number | null;
118 + upper_ci: number | null;
119 + estimate_type: string;
120 + standard_population: string | null;
121 + site_definition: string | null;
122 + geography_name: string;
123 + geography_slug: string;
124 + iso3: string | null;
125 + source_slug: string;
126 + source_name: string;
127 + provenance_id: number;
128 + updated_at: Date | string;
129 +}
130 +
131 +/**
132 + * Latest-year observation per (cancer, metric) for one geography (ISO3) and sex, all ages.
133 + * One row per cancer × metric; the year may differ between metrics (incidence usually lags mortality).
134 + */
135 +export async function latestFiguresFor(cancerIds: string[], iso3 = 'USA', sex = 'all'): Promise<LatestFigure[]> {
136 + if (cancerIds.length === 0) return [];
137 + return safe(
138 + () =>
139 + run<LatestFigure>(sql`
140 + SELECT DISTINCT ON (o.cancer_id, o.metric) o.cancer_id, o.metric, o.year, o.year_end, o.sex, o.value, o.unit, o.lower_ci, o.upper_ci, o.estimate_type, o.standard_population, o.site_definition,
141 + g.name AS geography_name, g.slug AS geography_slug, g.iso3, s.slug AS source_slug, s.name AS source_name, o.provenance_id, o.updated_at
142 + FROM epidemiology_observations o JOIN geographies g ON g.id = o.geography_id JOIN sources s ON s.id = o.source_id
143 + WHERE g.iso3 = ${iso3} AND o.sex = ${sex} AND o.age_group = 'all'
144 + AND o.metric IN ('mortality_count','incidence_count','as_mortality_rate','as_incidence_rate')
145 + AND o.cancer_id IN (${sql.join(cancerIds.map((i) => sql`${i}`), sql`, `)})
146 + ORDER BY o.cancer_id, o.metric, o.year DESC, (o.estimate_type = 'observed') DESC, o.updated_at DESC`),
147 + [] as LatestFigure[],
148 + );
149 +}
150 +
76 151 export const EPI_METRIC_LABEL: Record<string, string> = {
77 152 incidence_count: 'New cases',
78 153 incidence_rate: 'Incidence rate (crude)',
modified apps/web/src/lib/queries/evidence.ts +127 −31
@@ -10,9 +10,7 @@ export interface EvidenceItem {
10 10 gene_symbols: string[];
11 11 gene_ids: string[];
12 12 variant_ids: string[];
13 − civic_variant_ids: number[];
14 13 disease_name: string | null;
15 − doid: string | null;
16 14 cancer_id: string | null;
17 15 cancer_slug: string | null;
18 16 cancer_name: string | null;
@@ -26,12 +24,11 @@ export interface EvidenceItem {
26 24 significance: string | null;
27 25 evidence_rating: number | null;
28 26 status: string | null;
27 + /** Excerpt (EVIDENCE_DESCRIPTION_CHARS), never the full text. */
29 28 description: string | null;
30 29 pmid: string | null;
31 30 source_citation: string | null;
32 − phenotypes: string[];
33 31 provenance_id: number;
34 − ingest_run_id: string | null;
35 32 updated_at: Date;
36 33 variant_slugs: string[] | null;
37 34 variant_names: string[] | null;
@@ -40,34 +37,110 @@ export interface EvidenceItem {
40 37 therapy_slug_names: string[] | null;
41 38 }
42 39
40 +/** Description excerpt length shipped per row (the full text is one click away at CIViC). */
41 +export const EVIDENCE_DESCRIPTION_CHARS = 200;
42 +
43 43 const SELECT = sql`
44 − SELECT e.*, c.slug AS cancer_slug, c.canonical_name AS cancer_name,
44 + SELECT e.id, e.civic_id, e.name, e.molecular_profile_id, e.molecular_profile_name, e.gene_symbols, e.gene_ids, e.variant_ids, e.disease_name, e.cancer_id, e.cancer_match_type,
45 + e.therapy_names, e.therapy_ids, e.therapy_interaction_type, e.evidence_type, e.evidence_level, e.evidence_direction, e.significance, e.evidence_rating, e.status,
46 + e.pmid, e.source_citation, e.provenance_id, e.updated_at,
47 + c.slug AS cancer_slug, c.canonical_name AS cancer_name,
45 48 (SELECT array_agg(v.slug ORDER BY v.slug) FROM variants v WHERE v.id = ANY(e.variant_ids)) AS variant_slugs,
46 49 (SELECT array_agg(coalesce(v.gene_symbol || ' ', '') || v.name ORDER BY v.slug) FROM variants v WHERE v.id = ANY(e.variant_ids)) AS variant_names,
47 50 (SELECT array_agg(d.slug ORDER BY d.slug) FROM drugs d WHERE d.id = ANY(e.therapy_ids)) AS therapy_slugs,
48 51 (SELECT array_agg(d.name ORDER BY d.slug) FROM drugs d WHERE d.id = ANY(e.therapy_ids)) AS therapy_slug_names,
49 − left(e.description, 320) AS description
52 + left(e.description, ${EVIDENCE_DESCRIPTION_CHARS}) AS description
50 53 FROM civic_evidence_items e LEFT JOIN cancers c ON c.id = e.cancer_id`;
51 54
52 −export async function evidenceForCancer(cancerIds: string[], limit = 300): Promise<EvidenceItem[]> {
55 +export const EVIDENCE_PAGE_SIZE = 50;
56 +
57 +export interface Page {
58 + page: number;
59 + pageSize: number;
60 +}
61 +
62 +/**
63 + * Stable, group-preserving order so server-side pages never split a (profile → therapy) group at
64 + * random: molecular profile, then therapy, then level, then the CIViC id. `showCancer` contexts
65 + * (gene / drug / publication pages) sort by cancer first so the per-cancer grouping stays contiguous.
66 + */
67 +const ORDER_BY_PROFILE = sql`ORDER BY coalesce(e.molecular_profile_name, e.name) NULLS LAST, array_to_string(e.therapy_names, '+'), e.evidence_level NULLS LAST, e.evidence_rating DESC NULLS LAST, e.civic_id`;
68 +const ORDER_BY_CANCER = sql`ORDER BY c.canonical_name NULLS LAST, e.disease_name NULLS LAST, coalesce(e.molecular_profile_name, e.name) NULLS LAST, array_to_string(e.therapy_names, '+'), e.evidence_level NULLS LAST, e.civic_id`;
69 +
70 +function pageClause(p: Page) {
71 + return sql`LIMIT ${p.pageSize} OFFSET ${(Math.max(1, p.page) - 1) * p.pageSize}`;
72 +}
73 +
74 +async function count(where: ReturnType<typeof sql>): Promise<number> {
75 + const r = await safe(() => run<{ n: string }>(sql`SELECT count(*) AS n FROM civic_evidence_items e WHERE ${where}`), [{ n: '0' }]);
76 + return Number(r[0]?.n ?? 0);
77 +}
78 +
79 +const whereCancer = (cancerIds: string[]) => sql`e.cancer_id IN (${sql.join(cancerIds.map((i) => sql`${i}`), sql`, `)})`;
80 +const whereVariant = (variantId: string) => sql`${variantId} = ANY(e.variant_ids)`;
81 +const whereGene = (geneId: string, symbol: string) => sql`(${geneId} = ANY(e.gene_ids) OR ${symbol} = ANY(e.gene_symbols))`;
82 +const whereDrug = (drugId: string) => sql`${drugId} = ANY(e.therapy_ids)`;
83 +const wherePmid = (pmid: string) => sql`e.pmid = ${pmid}`;
84 +
85 +export async function evidenceForCancer(cancerIds: string[], p: Page = { page: 1, pageSize: EVIDENCE_PAGE_SIZE }): Promise<EvidenceItem[]> {
86 + if (cancerIds.length === 0) return [];
87 + return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE ${whereCancer(cancerIds)} ${ORDER_BY_PROFILE} ${pageClause(p)}`), [] as EvidenceItem[]);
88 +}
89 +export async function evidenceForCancerCount(cancerIds: string[]): Promise<number> {
90 + if (cancerIds.length === 0) return 0;
91 + return count(whereCancer(cancerIds));
92 +}
93 +
94 +/** Lightweight rows for the Drugs tab "therapies in evidence" aggregate: no description, no joins. */
95 +export async function therapyMentionsForCancer(cancerIds: string[]): Promise<Array<{ slug: string; name: string; n: number; sensitivity: number; resistance: number }>> {
53 96 if (cancerIds.length === 0) return [];
54 − return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE e.cancer_id IN (${sql.join(cancerIds.map((i) => sql`${i}`), sql`, `)}) ORDER BY e.evidence_level NULLS LAST, e.evidence_rating DESC NULLS LAST, e.civic_id LIMIT ${limit}`), [] as EvidenceItem[]);
97 + const rows = await safe(
98 + () =>
99 + run<{ slug: string; name: string; n: string; sensitivity: string; resistance: string }>(sql`
100 + SELECT d.slug, d.name, count(*) AS n,
101 + count(*) FILTER (WHERE e.significance ILIKE '%SENSITIV%') AS sensitivity,
102 + count(*) FILTER (WHERE e.significance ILIKE '%RESIST%') AS resistance
103 + FROM civic_evidence_items e JOIN drugs d ON d.id = ANY(e.therapy_ids)
104 + WHERE ${whereCancer(cancerIds)} GROUP BY d.slug, d.name ORDER BY n DESC, d.name`),
105 + [],
106 + );
107 + return rows.map((r) => ({ slug: r.slug, name: r.name, n: Number(r.n), sensitivity: Number(r.sensitivity), resistance: Number(r.resistance) }));
55 108 }
56 109
57 −export async function evidenceForVariant(variantId: string, limit = 200): Promise<EvidenceItem[]> {
58 − return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE ${variantId} = ANY(e.variant_ids) ORDER BY c.canonical_name NULLS LAST, e.evidence_level NULLS LAST, e.civic_id LIMIT ${limit}`), [] as EvidenceItem[]);
110 +export async function evidenceForVariant(variantId: string, p: Page = { page: 1, pageSize: EVIDENCE_PAGE_SIZE }): Promise<EvidenceItem[]> {
111 + return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE ${whereVariant(variantId)} ${ORDER_BY_CANCER} ${pageClause(p)}`), [] as EvidenceItem[]);
112 +}
113 +export async function evidenceForVariantCount(variantId: string): Promise<number> {
114 + return count(whereVariant(variantId));
59 115 }
60 116
61 −export async function evidenceForGene(geneId: string, symbol: string, limit = 150): Promise<EvidenceItem[]> {
62 − return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE ${geneId} = ANY(e.gene_ids) OR ${symbol} = ANY(e.gene_symbols) ORDER BY c.canonical_name NULLS LAST, e.evidence_level NULLS LAST, e.civic_id LIMIT ${limit}`), [] as EvidenceItem[]);
117 +export async function evidenceForGene(geneId: string, symbol: string, p: Page = { page: 1, pageSize: EVIDENCE_PAGE_SIZE }): Promise<EvidenceItem[]> {
118 + return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE ${whereGene(geneId, symbol)} ${ORDER_BY_PROFILE} ${pageClause(p)}`), [] as EvidenceItem[]);
119 +}
120 +export async function evidenceForGeneCount(geneId: string, symbol: string): Promise<number> {
121 + return count(whereGene(geneId, symbol));
122 +}
123 +/** Cancers appearing in a gene's evidence with item counts (facet chips above the paginated table). */
124 +export async function evidenceCancersForGene(geneId: string, symbol: string, limit = 40): Promise<Array<{ slug: string; name: string; n: number }>> {
125 + const rows = await safe(
126 + () => run<{ slug: string; name: string; n: string }>(sql`SELECT c.slug, c.canonical_name AS name, count(*) AS n FROM civic_evidence_items e JOIN cancers c ON c.id = e.cancer_id WHERE ${whereGene(geneId, symbol)} GROUP BY c.slug, c.canonical_name ORDER BY n DESC, c.canonical_name LIMIT ${limit}`),
127 + [],
128 + );
129 + return rows.map((r) => ({ slug: r.slug, name: r.name, n: Number(r.n) }));
63 130 }
64 131
65 −export async function evidenceForDrug(drugId: string, limit = 150): Promise<EvidenceItem[]> {
66 − return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE ${drugId} = ANY(e.therapy_ids) ORDER BY c.canonical_name NULLS LAST, e.evidence_level NULLS LAST, e.civic_id LIMIT ${limit}`), [] as EvidenceItem[]);
132 +export async function evidenceForDrug(drugId: string, p: Page = { page: 1, pageSize: EVIDENCE_PAGE_SIZE }): Promise<EvidenceItem[]> {
133 + return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE ${whereDrug(drugId)} ${ORDER_BY_CANCER} ${pageClause(p)}`), [] as EvidenceItem[]);
134 +}
135 +export async function evidenceForDrugCount(drugId: string): Promise<number> {
136 + return count(whereDrug(drugId));
67 137 }
68 138
69 −export async function evidenceForPublication(pmid: string, limit = 200): Promise<EvidenceItem[]> {
70 − return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE e.pmid = ${pmid} ORDER BY e.civic_id LIMIT ${limit}`), [] as EvidenceItem[]);
139 +export async function evidenceForPublication(pmid: string, p: Page = { page: 1, pageSize: EVIDENCE_PAGE_SIZE }): Promise<EvidenceItem[]> {
140 + return safe(() => run<EvidenceItem>(sql`${SELECT} WHERE ${wherePmid(pmid)} ${ORDER_BY_CANCER} ${pageClause(p)}`), [] as EvidenceItem[]);
141 +}
142 +export async function evidenceForPublicationCount(pmid: string): Promise<number> {
143 + return count(wherePmid(pmid));
71 144 }
72 145
73 146 export const EVIDENCE_LEVEL_LABEL: Record<string, string> = {
@@ -78,21 +151,44 @@ export const EVIDENCE_LEVEL_LABEL: Record<string, string> = {
78 151 E: 'E — Inferential association',
79 152 };
80 153
81 −/** Group evidence items by variant → therapy (never collapsed to works/doesn't work). */
82 −export function groupEvidence(items: EvidenceItem[]) {
83 − const byVariant = new Map<string, { key: string; label: string; slug: string | null; genes: string[]; therapies: Map<string, EvidenceItem[]> }>();
154 +export type EvidenceGroupBy = 'variant' | 'cancer' | 'none';
155 +
156 +export interface EvidenceGroup {
157 + key: string;
158 + label: string;
159 + href: string | null;
160 + genes: string[];
161 + unmapped: boolean;
162 + items: EvidenceItem[];
163 +}
164 +
165 +/** Therapy key used to collapse consecutive rows of the same therapy inside a group (〃 marker). */
166 +export function therapyKey(e: EvidenceItem): string {
167 + return e.therapy_names.length ? e.therapy_names.join(' + ') : e.evidence_type === 'PREDICTIVE' ? 'Unspecified therapy' : `(${(e.evidence_type ?? 'evidence').toLowerCase()})`;
168 +}
169 +
170 +/**
171 + * Group evidence items (never collapsed to works/doesn't work). Groups and rows keep the incoming
172 + * (SQL) order so a paginated table reads the same way as the query orders it; within a variant
173 + * group, rows of the same therapy stay adjacent because the query orders by therapy.
174 + */
175 +export function groupEvidence(items: EvidenceItem[], by: EvidenceGroupBy = 'variant'): EvidenceGroup[] {
176 + if (by === 'none') return items.length ? [{ key: 'all', label: '', href: null, genes: [], unmapped: false, items }] : [];
177 + const groups = new Map<string, EvidenceGroup>();
84 178 for (const e of items) {
85 − const vLabel = e.variant_names?.join(' + ') ?? e.molecular_profile_name ?? e.name ?? `CIViC EID${e.civic_id}`;
86 − const vKey = e.variant_ids.length ? e.variant_ids.join('+') : `mp-${e.molecular_profile_id ?? e.civic_id}`;
87 − if (!byVariant.has(vKey)) byVariant.set(vKey, { key: vKey, label: vLabel, slug: e.variant_slugs?.length === 1 ? e.variant_slugs[0]! : null, genes: e.gene_symbols, therapies: new Map() });
88 − const g = byVariant.get(vKey)!;
89 − const tKey = e.therapy_names.length ? e.therapy_names.join(' + ') : e.evidence_type === 'PREDICTIVE' ? 'Unspecified therapy' : `(${(e.evidence_type ?? 'evidence').toLowerCase()})`;
90 − if (!g.therapies.has(tKey)) g.therapies.set(tKey, []);
91 − g.therapies.get(tKey)!.push(e);
179 + let key: string;
180 + let g: EvidenceGroup | undefined;
181 + if (by === 'cancer') {
182 + key = e.cancer_id ?? `unmapped:${e.disease_name ?? 'unknown'}`;
183 + g = groups.get(key);
184 + if (!g) g = { key, label: e.cancer_name ?? e.disease_name ?? 'Unmapped disease', href: e.cancer_slug ? `/cancer/${e.cancer_slug}/evidence` : null, genes: [], unmapped: !e.cancer_slug, items: [] };
185 + } else {
186 + key = e.variant_ids.length ? e.variant_ids.join('+') : `mp-${e.molecular_profile_id ?? e.civic_id}`;
187 + g = groups.get(key);
188 + if (!g) g = { key, label: e.variant_names?.join(' + ') ?? e.molecular_profile_name ?? e.name ?? `CIViC EID${e.civic_id}`, href: e.variant_slugs?.length === 1 ? `/variant/${e.variant_slugs[0]!}` : null, genes: e.gene_symbols, unmapped: false, items: [] };
189 + }
190 + groups.set(key, g);
191 + g.items.push(e);
92 192 }
93 − return [...byVariant.values()].sort((a, b) => {
94 − const na = [...a.therapies.values()].reduce((s, x) => s + x.length, 0);
95 − const nb = [...b.therapies.values()].reduce((s, x) => s + x.length, 0);
96 − return nb - na || a.label.localeCompare(b.label);
97 − });
193 + return [...groups.values()];
98 194 }
modified apps/web/src/lib/queries/genomics.ts +60 −5
@@ -81,9 +81,49 @@ export interface FreqRow {
81 81 updated_at: Date;
82 82 }
83 83
84 −/** Gene frequencies for a cancer (and descendants), grouped per cohort by the caller (§260-261). */
85 −export async function frequenciesForCancer(cancerIds: string[], limitPerCohort = 30): Promise<FreqRow[]> {
84 +export interface CohortSummary {
85 + cohort_id: string;
86 + study_id: string;
87 + cohort_name: string;
88 + program: string | null;
89 + data_release: string | null;
90 + cases_with_ssm: number | null;
91 + case_count: number | null;
92 + cancer_id: string | null;
93 + cancer_slug: string | null;
94 + cancer_name: string | null;
95 + cancer_match_type: string | null;
96 + /** Largest denominator among the cohort's frequency rows — drives the default cohort choice. */
97 + cases_profiled: number;
98 + frequency_rows: number;
99 +}
100 +
101 +/** Cohorts mapped to a cancer (and descendants), largest denominator first (§260-261). */
102 +export async function cohortsForCancer(cancerIds: string[]): Promise<CohortSummary[]> {
86 103 if (cancerIds.length === 0) return [];
104 + const rows = await safe(
105 + () =>
106 + run<CohortSummary & { cases_profiled: string; frequency_rows: string }>(sql`
107 + SELECT gc.id AS cohort_id, gc.study_id, gc.name AS cohort_name, gc.program, gc.data_release, gc.cases_with_ssm, gc.case_count, gc.cancer_id, gc.cancer_match_type,
108 + c.slug AS cancer_slug, c.canonical_name AS cancer_name,
109 + coalesce(max(f.cases_profiled), 0) AS cases_profiled, count(f.id) AS frequency_rows
110 + FROM genomic_cohorts gc LEFT JOIN cancers c ON c.id = gc.cancer_id LEFT JOIN cancer_gene_frequencies f ON f.cohort_id = gc.id
111 + WHERE gc.cancer_id IN (${sql.join(cancerIds.map((i) => sql`${i}`), sql`, `)})
112 + GROUP BY gc.id, c.slug, c.canonical_name
113 + ORDER BY cases_profiled DESC, frequency_rows DESC, gc.name`),
114 + [],
115 + );
116 + return rows.map((r) => ({ ...r, cases_profiled: Number(r.cases_profiled), frequency_rows: Number(r.frequency_rows) }));
117 +}
118 +
119 +/**
120 + * Gene frequencies for a cancer (and descendants), grouped per cohort by the caller (§260-261).
121 + * `cohortId` restricts to one cohort (the default UI renders the largest cohort only).
122 + */
123 +export async function frequenciesForCancer(cancerIds: string[], opts: { cohortId?: string | null; limitPerCohort?: number } = {}): Promise<FreqRow[]> {
124 + if (cancerIds.length === 0) return [];
125 + const limitPerCohort = opts.limitPerCohort ?? 30;
126 + const cohortFilter = opts.cohortId ? sql`AND gc.id = ${opts.cohortId}` : sql``;
87 127 return safe(
88 128 () =>
89 129 run<FreqRow>(sql`
@@ -93,7 +133,7 @@ export async function frequenciesForCancer(cancerIds: string[], limitPerCohort =
93 133 row_number() OVER (PARTITION BY f.cohort_id, f.alteration_type ORDER BY f.frequency DESC) AS rn
94 134 FROM cancer_gene_frequencies f JOIN genomic_cohorts gc ON gc.id = f.cohort_id JOIN sources s ON s.id = gc.source_id
95 135 LEFT JOIN cancers c ON c.id = gc.cancer_id
96 − WHERE gc.cancer_id IN (${sql.join(cancerIds.map((i) => sql`${i}`), sql`, `)})
136 + WHERE gc.cancer_id IN (${sql.join(cancerIds.map((i) => sql`${i}`), sql`, `)}) ${cohortFilter}
97 137 ) x WHERE rn <= ${limitPerCohort} ORDER BY cohort_name, alteration_type, frequency DESC`),
98 138 [] as FreqRow[],
99 139 );
@@ -142,8 +182,23 @@ export async function getVariantBySlug(slug: string): Promise<VariantRow | null>
142 182 return rows[0] ?? null;
143 183 }
144 184
145 −export async function variantsForGene(geneId: string, limit = 100): Promise<VariantRow[]> {
146 − return safe(() => run<VariantRow>(sql`SELECT v.*, (SELECT count(*) FROM civic_evidence_items e WHERE v.id = ANY(e.variant_ids))::int AS evidence_count FROM variants v WHERE v.gene_id = ${geneId} ORDER BY evidence_count DESC, v.name LIMIT ${limit}`), [] as VariantRow[]);
185 +export const VARIANT_PAGE_SIZE = 50;
186 +
187 +/** Compact variant rows for the gene page list (no coordinates JSON, no HGVS strings). */
188 +export type VariantListRow = Pick<VariantRow, 'id' | 'slug' | 'name' | 'variant_type' | 'evidence_count'>;
189 +
190 +export async function variantsForGene(geneId: string, p: { page: number; pageSize: number } = { page: 1, pageSize: VARIANT_PAGE_SIZE }): Promise<VariantListRow[]> {
191 + return safe(
192 + () =>
193 + run<VariantListRow>(sql`
194 + SELECT v.id, v.slug, v.name, v.variant_type, (SELECT count(*) FROM civic_evidence_items e WHERE v.id = ANY(e.variant_ids))::int AS evidence_count
195 + FROM variants v WHERE v.gene_id = ${geneId} ORDER BY evidence_count DESC, v.name LIMIT ${p.pageSize} OFFSET ${(Math.max(1, p.page) - 1) * p.pageSize}`),
196 + [] as VariantListRow[],
197 + );
198 +}
199 +export async function variantsForGeneCount(geneId: string): Promise<number> {
200 + const r = await safe(() => run<{ n: string }>(sql`SELECT count(*) AS n FROM variants v WHERE v.gene_id = ${geneId}`), [{ n: '0' }]);
201 + return Number(r[0]?.n ?? 0);
147 202 }
148 203
149 204 export async function variantAliases(variantId: string): Promise<string[]> {
added apps/web/src/lib/queries/geography.ts +310 −0
@@ -0,0 +1,310 @@
1 +import 'server-only';
2 +import { run, sql, safe } from '@/lib/db';
3 +
4 +/**
5 + * Geography (country) queries for /countries and /country/[slug] (SPEC §47).
6 + * Everything is read straight from epidemiology_observations; nothing is estimated or extrapolated.
7 + * Only geographies with at least one observation are listed.
8 + */
9 +
10 +export const SEXES = ['all', 'male', 'female'] as const;
11 +export type Sex = (typeof SEXES)[number];
12 +
13 +export const BURDEN_METRICS = ['mortality_count', 'incidence_count', 'as_mortality_rate', 'as_incidence_rate'] as const;
14 +export type BurdenMetric = (typeof BURDEN_METRICS)[number];
15 +
16 +export interface GeographyRow {
17 + id: string;
18 + slug: string;
19 + name: string;
20 + kind: string;
21 + iso2: string | null;
22 + iso3: string | null;
23 + who_region: string | null;
24 + population: number | null;
25 + population_year: number | null;
26 + parent_slug: string | null;
27 + parent_name: string | null;
28 +}
29 +
30 +export interface CountryListRow extends GeographyRow {
31 + n_obs: number;
32 + n_cancers: number;
33 + min_year: number;
34 + max_year: number;
35 + sources: Array<{ slug: string; name: string }>;
36 + last_updated: Date | string;
37 +}
38 +
39 +export const WHO_REGION_LABEL: Record<string, string> = {
40 + 'who-afro': 'WHO African Region',
41 + 'who-amro': 'WHO Region of the Americas',
42 + 'who-searo': 'WHO South-East Asia Region',
43 + 'who-euro': 'WHO European Region',
44 + 'who-emro': 'WHO Eastern Mediterranean Region',
45 + 'who-wpro': 'WHO Western Pacific Region',
46 +};
47 +
48 +/** Geographies of any kind that carry at least one epidemiology observation. */
49 +export async function listGeographiesWithObservations(): Promise<CountryListRow[]> {
50 + return safe(
51 + () =>
52 + run<CountryListRow>(sql`
53 + SELECT g.id, g.slug, g.name, g.kind, g.iso2, g.iso3, g.who_region, g.population, g.population_year,
54 + pg.slug AS parent_slug, pg.name AS parent_name,
55 + a.n_obs::int AS n_obs, a.n_cancers::int AS n_cancers, a.min_year::int AS min_year, a.max_year::int AS max_year, a.last_updated,
56 + (SELECT json_agg(json_build_object('slug', s.slug, 'name', s.name) ORDER BY s.name)
57 + FROM sources s WHERE s.id IN (SELECT DISTINCT o2.source_id FROM epidemiology_observations o2 WHERE o2.geography_id = g.id)) AS sources
58 + FROM geographies g
59 + LEFT JOIN geographies pg ON pg.id = g.parent_id
60 + JOIN (
61 + SELECT geography_id, count(*) AS n_obs, count(DISTINCT cancer_id) AS n_cancers, min(year) AS min_year, max(coalesce(year_end, year)) AS max_year, max(updated_at) AS last_updated
62 + FROM epidemiology_observations GROUP BY geography_id
63 + ) a ON a.geography_id = g.id
64 + ORDER BY (g.kind = 'world') DESC, (g.kind = 'country') DESC, g.name`),
65 + [] as CountryListRow[],
66 + );
67 +}
68 +
69 +export async function getGeographyBySlug(slug: string): Promise<GeographyRow | null> {
70 + const rows = await safe(
71 + () =>
72 + run<GeographyRow>(sql`
73 + SELECT g.id, g.slug, g.name, g.kind, g.iso2, g.iso3, g.who_region, g.population, g.population_year, pg.slug AS parent_slug, pg.name AS parent_name
74 + FROM geographies g LEFT JOIN geographies pg ON pg.id = g.parent_id WHERE g.slug = ${slug} LIMIT 1`),
75 + [] as GeographyRow[],
76 + );
77 + return rows[0] ?? null;
78 +}
79 +
80 +/** Scope key used by the ranking engine for a geography (ISO3, else upper-cased slug). */
81 +export function geographyScopeCode(g: { iso3: string | null; slug: string }): string {
82 + return g.iso3 ?? g.slug.toUpperCase();
83 +}
84 +
85 +export interface CoverageRow {
86 + metric: string;
87 + sex: string;
88 + min_year: number;
89 + max_year: number;
90 + n_years: number;
91 + n_cancers: number;
92 + source_slug: string;
93 + source_name: string;
94 + estimate_types: string[];
95 + standard_population: string | null;
96 + last_updated: Date | string;
97 +}
98 +
99 +/** What the source covers for this geography: per metric × sex, the year span and entity count. */
100 +export async function coverageFor(geographyId: string): Promise<CoverageRow[]> {
101 + return safe(
102 + () =>
103 + run<CoverageRow>(sql`
104 + SELECT o.metric, o.sex, min(o.year)::int AS min_year, max(o.year)::int AS max_year, count(DISTINCT o.year)::int AS n_years, count(DISTINCT o.cancer_id)::int AS n_cancers,
105 + s.slug AS source_slug, s.name AS source_name, array_agg(DISTINCT o.estimate_type) AS estimate_types, max(o.standard_population) AS standard_population, max(o.updated_at) AS last_updated
106 + FROM epidemiology_observations o JOIN sources s ON s.id = o.source_id
107 + WHERE o.geography_id = ${geographyId} AND o.age_group = 'all'
108 + GROUP BY o.metric, o.sex, s.slug, s.name ORDER BY o.metric, o.sex`),
109 + [] as CoverageRow[],
110 + );
111 +}
112 +
113 +/** Distinct years with any observation for the geography (descending). */
114 +export async function yearsFor(geographyId: string): Promise<number[]> {
115 + const rows = await safe(() => run<{ year: number }>(sql`SELECT DISTINCT year FROM epidemiology_observations WHERE geography_id = ${geographyId} ORDER BY year DESC`), [] as Array<{ year: number }>);
116 + return rows.map((r) => Number(r.year));
117 +}
118 +
119 +/**
120 + * "All cancer sites" observation, if the source publishes one (e.g. USCS "All Cancer Sites Combined").
121 + * Detected from the site definition or a cancer entity flagged as the all-sites aggregate; null when absent —
122 + * the page then says so instead of summing per-site rows (sites overlap and sources differ in inclusion).
123 + */
124 +export interface AllSitesObs {
125 + metric: string;
126 + year: number;
127 + sex: string;
128 + value: number;
129 + unit: string;
130 + estimate_type: string;
131 + site_definition: string | null;
132 + source_slug: string;
133 + source_name: string;
134 + provenance_id: number;
135 + cancer_slug: string;
136 + cancer_name: string;
137 +}
138 +export async function allSitesObservations(geographyId: string, year: number, sex: Sex): Promise<AllSitesObs[]> {
139 + return safe(
140 + () =>
141 + run<AllSitesObs>(sql`
142 + SELECT o.metric, o.year, o.sex, o.value, o.unit, o.estimate_type, o.site_definition, s.slug AS source_slug, s.name AS source_name, o.provenance_id, c.slug AS cancer_slug, c.canonical_name AS cancer_name
143 + FROM epidemiology_observations o JOIN sources s ON s.id = o.source_id JOIN cancers c ON c.id = o.cancer_id
144 + WHERE o.geography_id = ${geographyId} AND o.year = ${year} AND o.sex = ${sex} AND o.age_group = 'all'
145 + AND (o.site_definition ILIKE '%all cancer sites%' OR o.site_definition ILIKE '%all sites combined%' OR o.site_definition ILIKE '%all malignant neoplasms%')
146 + ORDER BY o.metric`),
147 + [] as AllSitesObs[],
148 + );
149 +}
150 +
151 +export interface TopCancerRow {
152 + cancer_id: string;
153 + slug: string;
154 + canonical_name: string;
155 + entity_type: string;
156 + value: number;
157 + unit: string;
158 + lower_ci: number | null;
159 + upper_ci: number | null;
160 + estimate_type: string;
161 + site_definition: string | null;
162 + standard_population: string | null;
163 + year: number;
164 + year_end: number | null;
165 + source_slug: string;
166 + source_name: string;
167 + provenance_id: number;
168 + updated_at: Date | string;
169 + rank: number | null;
170 + eligible_entities: number | null;
171 + rank_scope_key: string | null;
172 +}
173 +
174 +export interface TopCancersResult {
175 + metric: BurdenMetric;
176 + requestedYear: number;
177 + year: number | null; // actual year used (latest ≤ requested with data for this metric/sex), null if none
178 + rows: TopCancerRow[];
179 +}
180 +
181 +/**
182 + * Top cancers for one metric in a geography/year/sex. If the requested year has no data for this metric
183 + * (e.g. incidence lags mortality by a year), the latest earlier year is used and reported in `year`.
184 + * Ranks come from the current ranking snapshot whose scope matches (metric_slug + scope_key), when one exists.
185 + */
186 +export async function topCancersFor(geo: { id: string; iso3: string | null; slug: string }, metric: BurdenMetric, requestedYear: number, sex: Sex, limit = 40): Promise<TopCancersResult> {
187 + const yr = await safe(
188 + () => run<{ y: number | null }>(sql`SELECT max(year) AS y FROM epidemiology_observations WHERE geography_id = ${geo.id} AND metric = ${metric} AND sex = ${sex} AND age_group = 'all' AND year <= ${requestedYear}`),
189 + [{ y: null }],
190 + );
191 + const year = yr[0]?.y == null ? null : Number(yr[0].y);
192 + if (year == null) return { metric, requestedYear, year: null, rows: [] };
193 + const scopeKey = `geo=${geographyScopeCode(geo)}|sex=${sex}|age=all|year=${year}|level=top`;
194 + const rows = await safe(
195 + () =>
196 + run<TopCancerRow>(sql`
197 + SELECT DISTINCT ON (o.cancer_id) o.cancer_id, c.slug, c.canonical_name, c.entity_type, o.value, o.unit, o.lower_ci, o.upper_ci, o.estimate_type, o.site_definition, o.standard_population,
198 + o.year, o.year_end, s.slug AS source_slug, s.name AS source_name, o.provenance_id, o.updated_at,
199 + r.rank, r.eligible_entities, r.scope_key AS rank_scope_key
200 + FROM epidemiology_observations o
201 + JOIN cancers c ON c.id = o.cancer_id
202 + JOIN sources s ON s.id = o.source_id
203 + LEFT JOIN rankings r ON r.cancer_id = o.cancer_id AND r.metric_slug = ${metric} AND r.scope_key = ${scopeKey}
204 + AND r.snapshot_id = (SELECT id FROM ranking_snapshots rs WHERE rs.metric_slug = ${metric} AND rs.scope_key = ${scopeKey} AND rs.is_current ORDER BY rs.generated_at DESC LIMIT 1)
205 + WHERE o.geography_id = ${geo.id} AND o.metric = ${metric} AND o.sex = ${sex} AND o.age_group = 'all' AND o.year = ${year}
206 + ORDER BY o.cancer_id, (c.top_level) DESC, o.value DESC`),
207 + [] as TopCancerRow[],
208 + );
209 + rows.sort((a, b) => Number(b.value) - Number(a.value) || a.canonical_name.localeCompare(b.canonical_name));
210 + return { metric, requestedYear, year, rows: rows.slice(0, limit) };
211 +}
212 +
213 +export interface TrendPoint {
214 + cancer_id: string;
215 + slug: string;
216 + canonical_name: string;
217 + year: number;
218 + value: number;
219 + lower_ci: number | null;
220 + upper_ci: number | null;
221 + estimate_type: string;
222 + unit: string;
223 + standard_population: string | null;
224 + source_slug: string;
225 +}
226 +
227 +/** Full yearly series of one metric for a set of cancers (all years available in the geography). */
228 +export async function trendFor(geographyId: string, metric: BurdenMetric, sex: Sex, cancerIds: string[]): Promise<TrendPoint[]> {
229 + if (cancerIds.length === 0) return [];
230 + return safe(
231 + () =>
232 + run<TrendPoint>(sql`
233 + SELECT o.cancer_id, c.slug, c.canonical_name, o.year, o.value, o.lower_ci, o.upper_ci, o.estimate_type, o.unit, o.standard_population, s.slug AS source_slug
234 + FROM epidemiology_observations o JOIN cancers c ON c.id = o.cancer_id JOIN sources s ON s.id = o.source_id
235 + WHERE o.geography_id = ${geographyId} AND o.metric = ${metric} AND o.sex = ${sex} AND o.age_group = 'all'
236 + AND o.cancer_id IN (${sql.join(cancerIds.map((i) => sql`${i}`), sql`, `)})
237 + ORDER BY c.canonical_name, o.year`),
238 + [] as TrendPoint[],
239 + );
240 +}
241 +
242 +export interface CagrRow {
243 + cancer_id: string;
244 + slug: string;
245 + canonical_name: string;
246 + start_year: number;
247 + end_year: number;
248 + start_value: number;
249 + end_value: number;
250 + n_years: number;
251 + cagr: number; // fraction per year, e.g. 0.021 = +2.1 %/yr
252 + unit: string;
253 + estimate_types: string[];
254 + source_slug: string;
255 + standard_population: string | null;
256 + start_provenance_id: number;
257 + end_provenance_id: number;
258 + updated_at: Date | string;
259 +}
260 +
261 +export const CAGR_FORMULA = 'CAGR = (value[end_year] / value[start_year])^(1 / (end_year − start_year)) − 1';
262 +export const CAGR_FORMULA_VERSION = 'ci-asir-cagr-10y-v1 (derived on the fly, not a stored metric)';
263 +
264 +/**
265 + * Compound annual growth rate of a metric over the last `window` available years, per cancer, computed on the
266 + * fly from observations (derived value; formula shown on the page). Returns nothing unless at least `window`
267 + * distinct years exist for the geography. Cancers with fewer than `window` points in the window are skipped.
268 + */
269 +export async function cagrFor(geographyId: string, metric: BurdenMetric, sex: Sex, window = 10): Promise<{ rows: CagrRow[]; startYear: number; endYear: number; yearsAvailable: number } | null> {
270 + const years = await safe(
271 + () => run<{ year: number }>(sql`SELECT DISTINCT year FROM epidemiology_observations WHERE geography_id = ${geographyId} AND metric = ${metric} AND sex = ${sex} AND age_group = 'all' ORDER BY year DESC`),
272 + [] as Array<{ year: number }>,
273 + );
274 + const ys = years.map((r) => Number(r.year));
275 + if (ys.length < window) return null;
276 + const endYear = ys[0]!;
277 + const startYear = ys[window - 1]!;
278 + const rows = await safe(
279 + () =>
280 + run<CagrRow>(sql`
281 + WITH w AS (
282 + SELECT o.cancer_id, o.year, o.value, o.unit, o.estimate_type, o.provenance_id, o.standard_population, o.source_id, o.updated_at
283 + FROM epidemiology_observations o
284 + WHERE o.geography_id = ${geographyId} AND o.metric = ${metric} AND o.sex = ${sex} AND o.age_group = 'all' AND o.year BETWEEN ${startYear} AND ${endYear}
285 + ), agg AS (
286 + SELECT cancer_id, count(DISTINCT year) AS n_years, min(year) AS y0, max(year) AS y1, array_agg(DISTINCT estimate_type) AS estimate_types, max(unit) AS unit, max(standard_population) AS standard_population, max(source_id) AS source_id, max(updated_at) AS updated_at
287 + FROM w GROUP BY cancer_id
288 + )
289 + SELECT a.cancer_id, c.slug, c.canonical_name, a.y0::int AS start_year, a.y1::int AS end_year, w0.value AS start_value, w1.value AS end_value, a.n_years::int AS n_years,
290 + CASE WHEN w0.value > 0 AND a.y1 > a.y0 THEN power(w1.value / w0.value, 1.0 / (a.y1 - a.y0)) - 1 ELSE NULL END AS cagr,
291 + a.unit, a.estimate_types, s.slug AS source_slug, a.standard_population, w0.provenance_id AS start_provenance_id, w1.provenance_id AS end_provenance_id, a.updated_at
292 + FROM agg a
293 + JOIN cancers c ON c.id = a.cancer_id
294 + JOIN sources s ON s.id = a.source_id
295 + JOIN w w0 ON w0.cancer_id = a.cancer_id AND w0.year = a.y0
296 + JOIN w w1 ON w1.cancer_id = a.cancer_id AND w1.year = a.y1
297 + WHERE a.n_years >= ${window} AND a.y0 = ${startYear} AND a.y1 = ${endYear} AND w0.value > 0
298 + ORDER BY cagr DESC NULLS LAST`),
299 + [] as CagrRow[],
300 + );
301 + return { rows: rows.filter((r) => r.cagr != null && Number.isFinite(Number(r.cagr))).map((r) => ({ ...r, cagr: Number(r.cagr), start_value: Number(r.start_value), end_value: Number(r.end_value) })), startYear, endYear, yearsAvailable: ys.length };
302 +}
303 +
304 +/** Geography slugs for the sitemap chunk (only those with observations). */
305 +export async function geographySlugsForSitemap(): Promise<Array<{ slug: string; updated_at: Date | string }>> {
306 + return safe(
307 + () => run<{ slug: string; updated_at: Date | string }>(sql`SELECT g.slug, max(o.updated_at) AS updated_at FROM geographies g JOIN epidemiology_observations o ON o.geography_id = g.id GROUP BY g.slug ORDER BY g.slug`),
308 + [] as Array<{ slug: string; updated_at: Date | string }>,
309 + );
310 +}
modified apps/web/src/lib/queries/publications.ts +38 −9
@@ -73,18 +73,47 @@ export async function literatureCountsFor(cancerId: string): Promise<LitCount[]>
73 73 return safe(() => run<LitCount>(sql`SELECT * FROM literature_counts WHERE cancer_id = ${cancerId} ORDER BY CASE window_key WHEN 'all' THEN 0 WHEN '10y' THEN 1 WHEN '5y' THEN 2 WHEN '5y_prior' THEN 3 WHEN '12m' THEN 4 ELSE 5 END, window_key`), [] as LitCount[]);
74 74 }
75 75
76 −export async function recentPublicationsFor(entityType: string, entityIds: string[], limit = 25): Promise<Array<PublicationRow & { method: string; edge_status: string }>> {
76 +export const PUBLICATION_PAGE_SIZE = 25;
77 +
78 +/** Columns the publication lists render (never the abstract or the full MeSH/author arrays). */
79 +export type PublicationListRow = Pick<PublicationRow, 'id' | 'pmid' | 'title' | 'journal' | 'journal_iso' | 'pub_date' | 'pub_year' | 'is_preprint' | 'retracted' | 'updated_at' | 'ingest_run_id'> & {
80 + /** First three authors only. */
81 + authors: Array<{ name: string }>;
82 + author_count: number;
83 + method?: string;
84 + edge_status?: string;
85 +};
86 +
87 +const LIST_COLUMNS = sql`p.id, p.pmid, p.title, p.journal, p.journal_iso, p.pub_date, p.pub_year, p.is_preprint, p.retracted, p.updated_at, p.ingest_run_id,
88 + (SELECT coalesce(jsonb_agg(jsonb_build_object('name', a->>'name')), '[]'::jsonb) FROM (SELECT a FROM jsonb_array_elements(p.authors) WITH ORDINALITY x(a, i) ORDER BY i LIMIT 3) s) AS authors,
89 + jsonb_array_length(p.authors) AS author_count`;
90 +
91 +const whereEntity = (entityType: string, entityIds: string[]) => sql`e.entity_type = ${entityType} AND e.entity_id IN (${sql.join(entityIds.map((i) => sql`${i}`), sql`, `)}) AND e.status <> 'rejected'`;
92 +
93 +/** Publications linked to any of the entities, newest first, one row per publication (best edge status). */
94 +export async function recentPublicationsFor(entityType: string, entityIds: string[], p: { page: number; pageSize: number } = { page: 1, pageSize: PUBLICATION_PAGE_SIZE }): Promise<PublicationListRow[]> {
77 95 if (entityIds.length === 0) return [];
78 96 return safe(
79 97 () =>
80 − run<PublicationRow & { method: string; edge_status: string }>(sql`
81 − SELECT DISTINCT ON (p.id) p.*, e.method, e.status AS edge_status FROM publication_entity_edges e JOIN publications p ON p.id = e.publication_id
82 − WHERE e.entity_type = ${entityType} AND e.entity_id IN (${sql.join(entityIds.map((i) => sql`${i}`), sql`, `)}) AND e.status <> 'rejected'
83 − ORDER BY p.id, e.status`),
84 − [] as Array<PublicationRow & { method: string; edge_status: string }>,
85 − ).then((rows) => rows.sort((a, b) => (b.pub_date ?? '').localeCompare(a.pub_date ?? '')).slice(0, limit));
98 + run<PublicationListRow>(sql`
99 + SELECT ${LIST_COLUMNS}, x.method, x.edge_status FROM (
100 + SELECT DISTINCT ON (e.publication_id) e.publication_id, e.method, e.status AS edge_status
101 + FROM publication_entity_edges e WHERE ${whereEntity(entityType, entityIds)} ORDER BY e.publication_id, e.status
102 + ) x JOIN publications p ON p.id = x.publication_id
103 + ORDER BY p.pub_date DESC NULLS LAST, p.id LIMIT ${p.pageSize} OFFSET ${(Math.max(1, p.page) - 1) * p.pageSize}`),
104 + [] as PublicationListRow[],
105 + );
106 +}
107 +export async function recentPublicationsForCount(entityType: string, entityIds: string[]): Promise<number> {
108 + if (entityIds.length === 0) return 0;
109 + const r = await safe(() => run<{ n: string }>(sql`SELECT count(DISTINCT e.publication_id) AS n FROM publication_entity_edges e WHERE ${whereEntity(entityType, entityIds)}`), [{ n: '0' }]);
110 + return Number(r[0]?.n ?? 0);
86 111 }
87 112
88 −export async function publicationsForTrial(nctId: string, limit = 50): Promise<PublicationRow[]> {
89 − return safe(() => run<PublicationRow>(sql`SELECT * FROM publications WHERE ${nctId} = ANY(nct_ids) ORDER BY pub_date DESC NULLS LAST LIMIT ${limit}`), [] as PublicationRow[]);
113 +export async function publicationsForTrial(nctId: string, p: { page: number; pageSize: number } = { page: 1, pageSize: PUBLICATION_PAGE_SIZE }): Promise<PublicationListRow[]> {
114 + return safe(() => run<PublicationListRow>(sql`SELECT ${LIST_COLUMNS} FROM publications p WHERE ${nctId} = ANY(p.nct_ids) ORDER BY p.pub_date DESC NULLS LAST, p.id LIMIT ${p.pageSize} OFFSET ${(Math.max(1, p.page) - 1) * p.pageSize}`), [] as PublicationListRow[]);
115 +}
116 +export async function publicationsForTrialCount(nctId: string): Promise<number> {
117 + const r = await safe(() => run<{ n: string }>(sql`SELECT count(*) AS n FROM publications p WHERE ${nctId} = ANY(p.nct_ids)`), [{ n: '0' }]);
118 + return Number(r[0]?.n ?? 0);
90 119 }
modified apps/web/src/lib/queries/rankings.ts +68 −0
@@ -138,6 +138,74 @@ export async function previewSnapshot(preferred: string[]): Promise<{ snapshot:
138 138 return null;
139 139 }
140 140
141 +/**
142 + * Current ranking rows for several cancers at once (compare page, overview hero). Same shape as
143 + * rankingsForCancer; callers keep the latest year per (metric, scope-without-year) with pickLatestScopes.
144 + */
145 +export async function rankingsForCancers(cancerIds: string[]): Promise<Array<CancerRanking & { cancer_id: string }>> {
146 + if (cancerIds.length === 0) return [];
147 + return safe(
148 + () =>
149 + run<CancerRanking & { cancer_id: string }>(sql`
150 + SELECT r.id, r.cancer_id, r.metric_slug, m.name AS metric_name, r.unit, r.scope_key, s.formula_version, m.formula, r.rank, r.previous_rank, r.eligible_entities, r.value, r.confidence, r.percentile, r.inputs, s.generated_at, s.inputs_hash, s.source_ids
151 + FROM rankings r JOIN ranking_snapshots s ON s.id = r.snapshot_id AND s.is_current JOIN metric_definitions m ON m.slug = r.metric_slug
152 + WHERE r.cancer_id IN (${sql.join(cancerIds.map((i) => sql`${i}`), sql`, `)}) ORDER BY m.category, m.name, s.scope_key`),
153 + [],
154 + );
155 +}
156 +
157 +/** Keep, per (cancer, metric, scope without year), only the latest year. Ties keep the first row. */
158 +export function pickLatestScopes<T extends { metric_slug: string; scope_key: string; cancer_id?: string }>(rows: T[]): T[] {
159 + const yearOf = (k: string) => Number(/year=(\d{4})/.exec(k)?.[1] ?? 0);
160 + const out = new Map<string, T>();
161 + for (const r of rows) {
162 + const key = `${r.cancer_id ?? ''}|${r.metric_slug}|${r.scope_key.replace(/year=[^|]*/, '')}`;
163 + const cur = out.get(key);
164 + if (!cur || yearOf(r.scope_key) > yearOf(cur.scope_key)) out.set(key, r);
165 + }
166 + return [...out.values()];
167 +}
168 +
169 +/**
170 + * Pick the most relevant current rank for one metric: prefer the requested geography, then the requested
171 + * entity level (top for top-level entities, all otherwise), then the latest year.
172 + */
173 +export function bestRankFor<T extends { metric_slug: string; scope_key: string }>(rows: T[], metricSlug: string, opts: { geo?: string; level?: 'top' | 'all' } = {}): T | null {
174 + const cands = rows.filter((r) => r.metric_slug === metricSlug);
175 + if (cands.length === 0) return null;
176 + const score = (k: string) => {
177 + let s = 0;
178 + if (opts.geo && k.includes(`geo=${opts.geo}|`)) s += 100;
179 + if (opts.level && k.endsWith(`level=${opts.level}`)) s += 10;
180 + s += Number(/year=(\d{4})/.exec(k)?.[1] ?? 0) / 10_000;
181 + return s;
182 + };
183 + return [...cands].sort((a, b) => score(b.scope_key) - score(a.scope_key))[0] ?? null;
184 +}
185 +
186 +/** Latest current snapshot of a metric for a geography code (e.g. USA), top level. */
187 +export async function latestSnapshotForGeo(slug: string, geo: string, sex = 'all'): Promise<Snapshot | null> {
188 + const rows = await safe(
189 + () => run<Snapshot>(sql`SELECT * FROM ranking_snapshots WHERE metric_slug = ${slug} AND is_current AND geography = ${geo} AND sex = ${sex} AND entity_level = 'top' ORDER BY year DESC NULLS LAST, generated_at DESC LIMIT 1`),
190 + [] as Snapshot[],
191 + );
192 + return rows[0] ?? null;
193 +}
194 +
195 +/** Home module: the gap indexes (trial gap, research gap) for a geography, top rows with lineage. */
196 +export async function gapRankings(geo: string, limit = 8): Promise<Array<{ metric: MetricDef; snapshot: Snapshot; rows: RankingRow[] }>> {
197 + const out: Array<{ metric: MetricDef; snapshot: Snapshot; rows: RankingRow[] }> = [];
198 + for (const slug of ['trial_gap', 'research_gap']) {
199 + const snapshot = await latestSnapshotForGeo(slug, geo);
200 + if (!snapshot) continue;
201 + const metric = await getMetric(slug);
202 + if (!metric) continue;
203 + const rows = await rankingRows(snapshot.id, limit);
204 + if (rows.length) out.push({ metric, snapshot, rows });
205 + }
206 + return out;
207 +}
208 +
141 209 export async function listSnapshots(limit = 200): Promise<Array<Snapshot & { metric_name: string; row_count: number }>> {
142 210 return safe(
143 211 () => run<Snapshot & { metric_name: string; row_count: number }>(sql`SELECT s.*, m.name AS metric_name, (SELECT count(*) FROM rankings r WHERE r.snapshot_id = s.id)::int AS row_count FROM ranking_snapshots s JOIN metric_definitions m ON m.id = s.metric_id ORDER BY s.generated_at DESC LIMIT ${limit}`),
modified apps/web/src/lib/queries/search.ts +4 −1
@@ -2,6 +2,7 @@ import 'server-only';
2 2 import { run, sql, safe } from '@/lib/db';
3 3 import { normalizeLabel } from '@cancerindex/shared';
4 4 import type { SearchHit } from '@/components/layout/command-palette';
5 +import { matchFromTier, sortHits } from '@/lib/search-util';
5 6
6 7 /**
7 8 * Entity search (§294, §312). Ordering: exact canonical name > exact alias/identifier > prefix >
@@ -68,5 +69,7 @@ export async function searchEntities(qRaw: string, limit = 20): Promise<SearchHi
68 69 SELECT * FROM cand ORDER BY tier, score DESC, title LIMIT ${limit}`),
69 70 [] as Row[],
70 71 );
71 − return rows.map((r) => ({ type: r.type, id: r.id, title: r.title, subtitle: r.subtitle, href: r.href, match: r.tier === 0 ? 'exact' : r.tier === 1 ? (r.type === 'gene' || r.type === 'trial' || r.type === 'publication' || r.type === 'variant' ? 'identifier' : 'alias') : r.tier === 2 ? 'prefix' : 'trigram' }));
72 + // The SQL ORDER BY already applies the tier/score/title order; sortHits re-applies the same pure
73 + // comparator so the ordering contract is one function (unit-tested) rather than two copies.
74 + return sortHits(rows.map((r) => ({ ...r, score: Number(r.score) }))).map((r) => ({ type: r.type, id: r.id, title: r.title, subtitle: r.subtitle, href: r.href, match: matchFromTier(r.tier, r.type) }));
72 75 }
modified apps/web/src/lib/queries/trials.ts +23 −2
@@ -121,8 +121,29 @@ export async function trialLocationsByCountry(trialId: string): Promise<Array<{
121 121 return rows.map((r) => ({ country: r.country, n: Number(r.n), recruiting: Number(r.recruiting) }));
122 122 }
123 123
124 −export async function trialsForDrug(drugId: string, limit = 100): Promise<TrialRow[]> {
125 − return safe(() => run<TrialRow>(sql`SELECT t.* FROM clinical_trials t WHERE EXISTS (SELECT 1 FROM trial_interventions ti WHERE ti.trial_id = t.id AND ti.drug_id = ${drugId}) ORDER BY t.last_update_posted_date DESC NULLS LAST LIMIT ${limit}`), [] as TrialRow[]);
124 +export const TRIAL_PAGE_SIZE = 50;
125 +
126 +/** Columns the trial tables actually render (the full row carries arms, outcomes, eligibility JSON…). */
127 +export type TrialListRow = Pick<TrialRow, 'id' | 'nct_id' | 'brief_title' | 'acronym' | 'phases' | 'overall_status' | 'enrollment_count' | 'lead_sponsor' | 'lead_sponsor_class' | 'countries' | 'last_update_posted_date' | 'updated_at'>;
128 +const LIST_COLUMNS = sql`t.id, t.nct_id, t.brief_title, t.acronym, t.phases, t.overall_status, t.enrollment_count, t.lead_sponsor, t.lead_sponsor_class, t.countries, t.last_update_posted_date, t.updated_at`;
129 +
130 +export async function listTrialRows(f: TrialFilters): Promise<{ rows: TrialListRow[]; total: number }> {
131 + if (f.cancerIds && f.cancerIds.length === 0) return { rows: [], total: 0 };
132 + const [total, rows] = await Promise.all([
133 + safe(() => run<{ n: string }>(sql`SELECT count(*) AS n FROM clinical_trials t WHERE ${trialWhere(f)}`), [{ n: '0' }]),
134 + safe(() => run<TrialListRow>(sql`SELECT ${LIST_COLUMNS} FROM clinical_trials t WHERE ${trialWhere(f)} ORDER BY t.last_update_posted_date DESC NULLS LAST, t.nct_id LIMIT ${f.pageSize} OFFSET ${(Math.max(1, f.page) - 1) * f.pageSize}`), [] as TrialListRow[]),
135 + ]);
136 + return { rows, total: Number(total[0]?.n ?? 0) };
137 +}
138 +
139 +const whereDrug = (drugId: string) => sql`EXISTS (SELECT 1 FROM trial_interventions ti WHERE ti.trial_id = t.id AND ti.drug_id = ${drugId})`;
140 +
141 +export async function trialsForDrug(drugId: string, p: { page: number; pageSize: number } = { page: 1, pageSize: TRIAL_PAGE_SIZE }): Promise<TrialListRow[]> {
142 + return safe(() => run<TrialListRow>(sql`SELECT ${LIST_COLUMNS} FROM clinical_trials t WHERE ${whereDrug(drugId)} ORDER BY t.last_update_posted_date DESC NULLS LAST, t.nct_id LIMIT ${p.pageSize} OFFSET ${(Math.max(1, p.page) - 1) * p.pageSize}`), [] as TrialListRow[]);
143 +}
144 +export async function trialsForDrugCount(drugId: string): Promise<number> {
145 + const r = await safe(() => run<{ n: string }>(sql`SELECT count(*) AS n FROM clinical_trials t WHERE ${whereDrug(drugId)}`), [{ n: '0' }]);
146 + return Number(r[0]?.n ?? 0);
126 147 }
127 148
128 149 /** Aggregate view for the home "Most active clinical research" module. */
added apps/web/src/lib/rankings-util.ts +37 −0
@@ -0,0 +1,37 @@
1 +/** Pure helpers for ranking rows (no DB, no server-only) — shared by the cancer Rankings tab and tests. */
2 +
3 +export interface ScopedRankingRow {
4 + metric_slug: string;
5 + scope_key: string;
6 +}
7 +
8 +/** Year encoded in a scope key ("…|year=2021|…"); 0 when absent or "latest". */
9 +export function scopeYear(scopeKey: string): number {
10 + const m = /year=(\d{4})/.exec(scopeKey);
11 + return m ? Number(m[1]) : 0;
12 +}
13 +
14 +/** Scope key with the year component removed — identifies the (geo, sex, age, level) scope across years. */
15 +export function scopeWithoutYear(scopeKey: string): string {
16 + return scopeKey
17 + .split('|')
18 + .filter((part) => !part.startsWith('year='))
19 + .join('|');
20 +}
21 +
22 +/**
23 + * Keep one row per (metric, scope-without-year): the latest year. Earlier yearly snapshots are
24 + * left to the ranking pages. Input order is preserved for the surviving rows; ties on year keep
25 + * the first row seen.
26 + */
27 +export function latestYearPerScope<T extends ScopedRankingRow>(rows: T[]): { rows: T[]; hidden: number } {
28 + const latest = new Map<string, { row: T; year: number; index: number }>();
29 + rows.forEach((r, index) => {
30 + const key = `${r.metric_slug}|${scopeWithoutYear(r.scope_key)}`;
31 + const year = scopeYear(r.scope_key);
32 + const cur = latest.get(key);
33 + if (!cur || year > cur.year) latest.set(key, { row: r, year, index: cur?.index ?? index });
34 + });
35 + const kept = [...latest.values()].sort((a, b) => a.index - b.index).map((x) => x.row);
36 + return { rows: kept, hidden: rows.length - kept.length };
37 +}
added apps/web/src/lib/search-util.ts +34 −0
@@ -0,0 +1,34 @@
1 +/** Pure part of entity search (§294, §312): tier → match label and the final ordering. */
2 +
3 +export type SearchType = 'cancer' | 'gene' | 'variant' | 'drug' | 'trial' | 'publication' | 'source';
4 +export type SearchMatch = 'exact' | 'alias' | 'prefix' | 'trigram' | 'identifier';
5 +
6 +export interface RankedHit {
7 + type: SearchType;
8 + title: string;
9 + /** 0 exact · 1 alias/identifier · 2 prefix · 3 trigram */
10 + tier: number;
11 + /** pg_trgm similarity (0..1), higher is better */
12 + score: number;
13 +}
14 +
15 +const IDENTIFIER_TYPES: ReadonlySet<SearchType> = new Set(['gene', 'trial', 'publication', 'variant']);
16 +
17 +/** Tier 1 means "matched an identifier" for code-like entities and "matched an alias" for the rest. */
18 +export function matchFromTier(tier: number, type: SearchType): SearchMatch {
19 + if (tier <= 0) return 'exact';
20 + if (tier === 1) return IDENTIFIER_TYPES.has(type) ? 'identifier' : 'alias';
21 + if (tier === 2) return 'prefix';
22 + return 'trigram';
23 +}
24 +
25 +/** Ordering: exact > alias/identifier > prefix > trigram, then similarity desc, then title A→Z. */
26 +export function compareHits(a: RankedHit, b: RankedHit): number {
27 + if (a.tier !== b.tier) return a.tier - b.tier;
28 + if (a.score !== b.score) return b.score - a.score;
29 + return a.title.localeCompare(b.title, 'en', { sensitivity: 'base' });
30 +}
31 +
32 +export function sortHits<T extends RankedHit>(hits: T[]): T[] {
33 + return [...hits].sort(compareHits);
34 +}
modified apps/web/src/lib/site.ts +2 −0
@@ -9,6 +9,8 @@ export const NAV = [
9 9 { href: '/cancers', label: 'Cancers' },
10 10 { href: '/taxonomy', label: 'Taxonomy' },
11 11 { href: '/rankings', label: 'Rankings' },
12 + { href: '/countries', label: 'Countries' },
13 + { href: '/compare', label: 'Compare' },
12 14 { href: '/genes', label: 'Genes' },
13 15 { href: '/drugs', label: 'Drugs' },
14 16 { href: '/trials', label: 'Trials' },
added apps/web/test/db-errors.test.ts +40 −0
@@ -0,0 +1,40 @@
1 +import { describe, it, expect } from 'vitest';
2 +import { isConnectionError } from '@/lib/db-errors';
3 +
4 +const err = (props: Record<string, unknown>) => Object.assign(new Error(typeof props.message === 'string' ? props.message : 'boom'), props);
5 +
6 +describe('isConnectionError', () => {
7 + it('recognises Node socket errors', () => {
8 + expect(isConnectionError(err({ code: 'ECONNREFUSED', message: 'connect ECONNREFUSED 127.0.0.1:5433' }))).toBe(true);
9 + expect(isConnectionError(err({ code: 'ECONNRESET' }))).toBe(true);
10 + expect(isConnectionError(err({ code: 'ETIMEDOUT' }))).toBe(true);
11 + expect(isConnectionError(err({ errno: 'ENOTFOUND' }))).toBe(true);
12 + });
13 + it('recognises postgres.js connection-level errors', () => {
14 + expect(isConnectionError(err({ code: 'CONNECT_TIMEOUT', message: 'write CONNECT_TIMEOUT 127.0.0.1:5433' }))).toBe(true);
15 + expect(isConnectionError(err({ code: 'CONNECTION_CLOSED' }))).toBe(true);
16 + expect(isConnectionError(err({ code: 'CONNECTION_ENDED' }))).toBe(true);
17 + });
18 + it('recognises SQLSTATE classes for connection, auth, bad database, resources and shutdown', () => {
19 + expect(isConnectionError(err({ code: '08006' }))).toBe(true);
20 + expect(isConnectionError(err({ code: '28P01', message: 'password authentication failed' }))).toBe(true);
21 + expect(isConnectionError(err({ code: '3D000', message: 'database "nope" does not exist' }))).toBe(true);
22 + expect(isConnectionError(err({ code: '53300', message: 'too many connections' }))).toBe(true);
23 + expect(isConnectionError(err({ code: '57P01', message: 'terminating connection due to administrator command' }))).toBe(true);
24 + });
25 + it('does NOT flag query-level errors (missing table / column / bad cast) — those become empty states', () => {
26 + expect(isConnectionError(err({ code: '42P01', message: 'relation "foo" does not exist' }))).toBe(false);
27 + expect(isConnectionError(err({ code: '42703', message: 'column "bar" does not exist' }))).toBe(false);
28 + expect(isConnectionError(err({ code: '22P02', message: 'invalid input syntax for type integer' }))).toBe(false);
29 + expect(isConnectionError(new Error('some application bug'))).toBe(false);
30 + });
31 + it('looks through `cause` chains and message text', () => {
32 + expect(isConnectionError(err({ message: 'query failed', cause: err({ code: 'ECONNREFUSED' }) }))).toBe(true);
33 + expect(isConnectionError(err({ message: 'connection terminated unexpectedly' }))).toBe(true);
34 + });
35 + it('ignores non-objects', () => {
36 + expect(isConnectionError(null)).toBe(false);
37 + expect(isConnectionError('ECONNREFUSED')).toBe(false);
38 + expect(isConnectionError(undefined)).toBe(false);
39 + });
40 +});
added apps/web/test/format.test.ts +121 −0
@@ -0,0 +1,121 @@
1 +import { describe, it, expect } from 'vitest';
2 +import { fmtValue, fmtPct, fmtInt, fmtNum, scopeLabel, unitLabel, parseScopeKey, toDate, isoDate, humanize, phaseLabel, truncate, pluralize } from '@/lib/format';
3 +
4 +describe('fmtValue', () => {
5 + it('formats counts as integers with thousands separators', () => {
6 + expect(fmtValue(1234567, 'count')).toBe('1,234,567');
7 + expect(fmtValue('42', 'count')).toBe('42');
8 + expect(fmtValue(3.7, 'count')).toBe('4');
9 + });
10 + it('formats rates per 100k with one decimal', () => {
11 + expect(fmtValue(12.345, 'per_100k')).toBe('12.3');
12 + expect(fmtValue(0, 'per_100k')).toBe('0');
13 + });
14 + it('formats ratios with two decimals and probabilities as percentages', () => {
15 + expect(fmtValue(0.4567, 'ratio')).toBe('0.46');
16 + expect(fmtValue(0.4567, 'probability')).toBe('45.7%');
17 + expect(fmtValue(1, 'probability')).toBe('100%');
18 + });
19 + it('signs percentile points', () => {
20 + expect(fmtValue(3.21, 'percentile_points')).toBe('+3.2');
21 + expect(fmtValue(-3.21, 'percentile_points')).toBe('-3.2');
22 + expect(fmtValue(0, 'percentile_points')).toBe('0');
23 + });
24 + it('falls back sensibly for unknown units', () => {
25 + expect(fmtValue(12345.6, null)).toBe('12,346');
26 + expect(fmtValue(1.2345, 'unknown')).toBe('1.23');
27 + });
28 + it('never invents a number for missing values', () => {
29 + expect(fmtValue(null, 'count')).toBe('—');
30 + expect(fmtValue(undefined, 'ratio')).toBe('—');
31 + expect(fmtValue('abc', 'count')).toBe('—');
32 + expect(fmtValue(Number.NaN, 'per_100k')).toBe('—');
33 + });
34 +});
35 +
36 +describe('fmtPct / fmtInt / fmtNum', () => {
37 + it('fmtPct multiplies by 100 and respects digits', () => {
38 + expect(fmtPct(0.12345)).toBe('12.3%');
39 + expect(fmtPct(0.12345, 2)).toBe('12.35%');
40 + expect(fmtPct(null)).toBe('—');
41 + expect(fmtPct(Number.POSITIVE_INFINITY)).toBe('—');
42 + });
43 + it('fmtInt handles strings, empties and non-finite values', () => {
44 + expect(fmtInt('1000')).toBe('1,000');
45 + expect(fmtInt('')).toBe('—');
46 + expect(fmtInt(null)).toBe('—');
47 + expect(fmtInt('x')).toBe('—');
48 + });
49 + it('fmtNum caps at 3 decimals', () => {
50 + expect(fmtNum(1.23456, 3)).toBe('1.235');
51 + expect(fmtNum(1.23456, 9)).toBe('1.235');
52 + expect(fmtNum(1.5, 0)).toBe('2');
53 + });
54 +});
55 +
56 +describe('scope keys', () => {
57 + it('parses a scope key into a record', () => {
58 + expect(parseScopeKey('geo=WORLD|sex=all|age=all|year=latest|level=top')).toEqual({ geo: 'WORLD', sex: 'all', age: 'all', year: 'latest', level: 'top' });
59 + expect(parseScopeKey('')).toEqual({});
60 + expect(parseScopeKey('geo=USA|broken')).toEqual({ geo: 'USA' });
61 + });
62 + it('labels the default world scope', () => {
63 + expect(scopeLabel('geo=WORLD|sex=all|age=all|year=latest|level=top')).toBe('World · both sexes · all ages · latest · top-level cancers');
64 + });
65 + it('labels a specific scope', () => {
66 + expect(scopeLabel('geo=USA|sex=female|age=65+|year=2021|level=all')).toBe('USA · Female · ages 65+ · 2021 · all malignant entities');
67 + expect(scopeLabel('geo=USA|sex=all|age=all|year=2021|level=subtype')).toContain('subtype level');
68 + });
69 +});
70 +
71 +describe('unitLabel', () => {
72 + it('maps known units', () => {
73 + expect(unitLabel('count')).toBe('count');
74 + expect(unitLabel('per_100k')).toBe('per 100,000');
75 + expect(unitLabel('ratio')).toBe('ratio');
76 + expect(unitLabel('probability')).toBe('%');
77 + expect(unitLabel('percentile_points')).toBe('percentile points');
78 + });
79 + it('passes unknown units through and empties for null', () => {
80 + expect(unitLabel('years')).toBe('years');
81 + expect(unitLabel(null)).toBe('');
82 + expect(unitLabel(undefined)).toBe('');
83 + });
84 +});
85 +
86 +describe('dates (drizzle execute returns timestamp strings)', () => {
87 + it('parses postgres timestamp strings with a numeric offset', () => {
88 + const d = toDate('2026-09-08 05:07:09.317-04');
89 + expect(d).not.toBeNull();
90 + expect(d!.toISOString()).toBe('2026-09-08T09:07:09.317Z');
91 + });
92 + it('parses plain dates as UTC midnight and rejects garbage', () => {
93 + expect(isoDate('2024-02-29')).toBe('2024-02-29');
94 + expect(toDate('not a date')).toBeNull();
95 + expect(toDate('')).toBeNull();
96 + expect(toDate(null)).toBeNull();
97 + expect(isoDate(new Date(Number.NaN))).toBe('—');
98 + });
99 +});
100 +
101 +describe('labels', () => {
102 + it('humanizes enums', () => {
103 + expect(humanize('NOT_YET_RECRUITING')).toBe('Not Yet Recruiting');
104 + expect(humanize('adenocarcinoma_nos')).toBe('Adenocarcinoma NOS');
105 + expect(humanize(null)).toBe('—');
106 + });
107 + it('labels phases', () => {
108 + expect(phaseLabel('PHASE3')).toBe('Phase 3');
109 + expect(phaseLabel('EARLY_PHASE1')).toBe('Early Phase 1');
110 + expect(phaseLabel('NA')).toBe('N/A');
111 + expect(phaseLabel('SOMETHING_ELSE')).toBe('Something Else');
112 + });
113 + it('truncates on word boundaries and pluralizes', () => {
114 + expect(truncate('The quick brown fox jumps over the lazy dog', 20)).toBe('The quick brown fox…');
115 + expect(truncate('short', 20)).toBe('short');
116 + expect(truncate(null, 5)).toBe('');
117 + expect(pluralize(1, 'study', 'studies')).toBe('study');
118 + expect(pluralize(2, 'study', 'studies')).toBe('studies');
119 + expect(pluralize(0, 'gene')).toBe('genes');
120 + });
121 +});
added apps/web/test/pagination.test.ts +87 −0
@@ -0,0 +1,87 @@
1 +import { describe, it, expect } from 'vitest';
2 +import { pageCount, clampPage, offsetFor, pageInfo, pageWindow } from '@/lib/pagination';
3 +
4 +describe('pageCount', () => {
5 + it('is at least 1', () => {
6 + expect(pageCount(0, 50)).toBe(1);
7 + expect(pageCount(-5, 50)).toBe(1);
8 + expect(pageCount(Number.NaN, 50)).toBe(1);
9 + });
10 + it('rounds up', () => {
11 + expect(pageCount(50, 50)).toBe(1);
12 + expect(pageCount(51, 50)).toBe(2);
13 + expect(pageCount(405, 50)).toBe(9);
14 + expect(pageCount(3000, 50)).toBe(60);
15 + });
16 + it('guards against a zero page size', () => {
17 + expect(pageCount(10, 0)).toBe(10);
18 + });
19 +});
20 +
21 +describe('clampPage', () => {
22 + it('clamps into [1, pageCount]', () => {
23 + expect(clampPage(0, 405, 50)).toBe(1);
24 + expect(clampPage(-3, 405, 50)).toBe(1);
25 + expect(clampPage(9, 405, 50)).toBe(9);
26 + expect(clampPage(10, 405, 50)).toBe(9);
27 + expect(clampPage(99999, 405, 50)).toBe(9);
28 + expect(clampPage(Number.NaN, 405, 50)).toBe(1);
29 + expect(clampPage(2.7, 405, 50)).toBe(2);
30 + });
31 + it('is 1 when there are no rows', () => {
32 + expect(clampPage(7, 0, 50)).toBe(1);
33 + });
34 +});
35 +
36 +describe('offsetFor', () => {
37 + it('computes SQL offsets', () => {
38 + expect(offsetFor(1, 50)).toBe(0);
39 + expect(offsetFor(2, 50)).toBe(50);
40 + expect(offsetFor(9, 25)).toBe(200);
41 + expect(offsetFor(0, 50)).toBe(0);
42 + });
43 +});
44 +
45 +describe('pageInfo', () => {
46 + it('describes a middle page', () => {
47 + const p = pageInfo(3, 50, 405);
48 + expect(p).toMatchObject({ page: 3, pageSize: 50, total: 405, pageCount: 9, offset: 100, from: 101, to: 150, hasPrev: true, hasNext: true });
49 + });
50 + it('describes the last, partial page', () => {
51 + const p = pageInfo(9, 50, 405);
52 + expect(p).toMatchObject({ page: 9, offset: 400, from: 401, to: 405, hasPrev: true, hasNext: false });
53 + });
54 + it('clamps an out-of-range request to the last page (never an empty page)', () => {
55 + const p = pageInfo(50, 50, 405);
56 + expect(p.page).toBe(9);
57 + expect(p.offset).toBe(400);
58 + });
59 + it('handles an empty result', () => {
60 + const p = pageInfo(1, 50, 0);
61 + expect(p).toMatchObject({ page: 1, pageCount: 1, offset: 0, from: 0, to: 0, hasPrev: false, hasNext: false });
62 + });
63 + it('handles a single page', () => {
64 + const p = pageInfo(1, 25, 12);
65 + expect(p).toMatchObject({ page: 1, pageCount: 1, from: 1, to: 12, hasPrev: false, hasNext: false });
66 + });
67 +});
68 +
69 +describe('pageWindow', () => {
70 + it('collapses to [1] for one page', () => {
71 + expect(pageWindow(1, 1)).toEqual([1]);
72 + });
73 + it('shows first, last and neighbours with gaps', () => {
74 + expect(pageWindow(7, 20)).toEqual([1, null, 6, 7, 8, null, 20]);
75 + });
76 + it('fills a gap of exactly one page instead of an ellipsis', () => {
77 + expect(pageWindow(3, 20)).toEqual([1, 2, 3, 4, null, 20]);
78 + expect(pageWindow(18, 20)).toEqual([1, null, 17, 18, 19, 20]);
79 + });
80 + it('lists every page when there are few', () => {
81 + expect(pageWindow(2, 4)).toEqual([1, 2, 3, 4]);
82 + expect(pageWindow(1, 2)).toEqual([1, 2]);
83 + });
84 + it('honours a wider radius', () => {
85 + expect(pageWindow(10, 30, 2)).toEqual([1, null, 8, 9, 10, 11, 12, null, 30]);
86 + });
87 +});
added apps/web/test/rankings-util.test.ts +52 −0
@@ -0,0 +1,52 @@
1 +import { describe, it, expect } from 'vitest';
2 +import { latestYearPerScope, scopeYear, scopeWithoutYear } from '@/lib/rankings-util';
3 +
4 +const row = (metric_slug: string, scope_key: string, id: number) => ({ id, metric_slug, scope_key });
5 +
6 +describe('scopeYear / scopeWithoutYear', () => {
7 + it('extracts the year and strips it from the key', () => {
8 + expect(scopeYear('geo=WORLD|sex=all|age=all|year=2021|level=top')).toBe(2021);
9 + expect(scopeYear('geo=WORLD|sex=all|age=all|year=latest|level=top')).toBe(0);
10 + expect(scopeYear('')).toBe(0);
11 + expect(scopeWithoutYear('geo=WORLD|sex=all|age=all|year=2021|level=top')).toBe('geo=WORLD|sex=all|age=all|level=top');
12 + expect(scopeWithoutYear('geo=USA|level=all')).toBe('geo=USA|level=all');
13 + });
14 +});
15 +
16 +describe('latestYearPerScope', () => {
17 + it('keeps one row per metric + scope: the latest year', () => {
18 + const rows = [
19 + row('mortality_count', 'geo=USA|sex=all|age=all|year=2019|level=top', 1),
20 + row('mortality_count', 'geo=USA|sex=all|age=all|year=2021|level=top', 2),
21 + row('mortality_count', 'geo=USA|sex=all|age=all|year=2020|level=top', 3),
22 + ];
23 + const r = latestYearPerScope(rows);
24 + expect(r.rows.map((x) => x.id)).toEqual([2]);
25 + expect(r.hidden).toBe(2);
26 + });
27 + it('treats different metrics, geographies and sexes as separate scopes', () => {
28 + const rows = [
29 + row('mortality_count', 'geo=USA|sex=all|age=all|year=2020|level=top', 1),
30 + row('incidence_count', 'geo=USA|sex=all|age=all|year=2020|level=top', 2),
31 + row('mortality_count', 'geo=WORLD|sex=all|age=all|year=2020|level=top', 3),
32 + row('mortality_count', 'geo=USA|sex=female|age=all|year=2020|level=top', 4),
33 + row('mortality_count', 'geo=USA|sex=all|age=all|year=2018|level=top', 5),
34 + ];
35 + const r = latestYearPerScope(rows);
36 + expect(r.rows.map((x) => x.id)).toEqual([1, 2, 3, 4]);
37 + expect(r.hidden).toBe(1);
38 + });
39 + it('keeps input order for surviving rows (first appearance of the scope)', () => {
40 + const rows = [row('a', 'geo=X|year=2019', 1), row('b', 'geo=X|year=2021', 2), row('a', 'geo=X|year=2021', 3)];
41 + expect(latestYearPerScope(rows).rows.map((x) => x.id)).toEqual([3, 2]);
42 + });
43 + it('keeps yearless ("latest") scopes as their own rows and prefers a dated year over none', () => {
44 + const rows = [row('active_trials', 'geo=WORLD|sex=all|age=all|year=latest|level=top', 1), row('mortality_count', 'geo=USA|year=latest', 2), row('mortality_count', 'geo=USA|year=2021', 3)];
45 + const r = latestYearPerScope(rows);
46 + expect(r.rows.map((x) => x.id)).toEqual([1, 3]);
47 + expect(r.hidden).toBe(1);
48 + });
49 + it('handles empty input', () => {
50 + expect(latestYearPerScope([])).toEqual({ rows: [], hidden: 0 });
51 + });
52 +});
added apps/web/test/search-util.test.ts +50 −0
@@ -0,0 +1,50 @@
1 +import { describe, it, expect } from 'vitest';
2 +import { matchFromTier, compareHits, sortHits, type RankedHit } from '@/lib/search-util';
3 +
4 +const hit = (title: string, tier: number, score: number, type: RankedHit['type'] = 'cancer'): RankedHit => ({ type, title, tier, score });
5 +
6 +describe('matchFromTier', () => {
7 + it('maps tiers to match labels', () => {
8 + expect(matchFromTier(0, 'cancer')).toBe('exact');
9 + expect(matchFromTier(2, 'drug')).toBe('prefix');
10 + expect(matchFromTier(3, 'cancer')).toBe('trigram');
11 + expect(matchFromTier(7, 'cancer')).toBe('trigram');
12 + });
13 + it('calls tier 1 an identifier for code-like entities and an alias otherwise', () => {
14 + expect(matchFromTier(1, 'gene')).toBe('identifier');
15 + expect(matchFromTier(1, 'trial')).toBe('identifier');
16 + expect(matchFromTier(1, 'publication')).toBe('identifier');
17 + expect(matchFromTier(1, 'variant')).toBe('identifier');
18 + expect(matchFromTier(1, 'cancer')).toBe('alias');
19 + expect(matchFromTier(1, 'drug')).toBe('alias');
20 + expect(matchFromTier(1, 'source')).toBe('alias');
21 + });
22 +});
23 +
24 +describe('compareHits / sortHits', () => {
25 + it('orders exact > alias/identifier > prefix > trigram regardless of score', () => {
26 + const sorted = sortHits([hit('Glioma', 3, 0.99), hit('Glioblastoma', 0, 0.2), hit('GBM', 1, 0.5), hit('Glioblastoma multiforme', 2, 0.9)]);
27 + expect(sorted.map((h) => h.title)).toEqual(['Glioblastoma', 'GBM', 'Glioblastoma multiforme', 'Glioma']);
28 + });
29 + it('breaks tier ties by similarity, descending', () => {
30 + const sorted = sortHits([hit('B', 3, 0.3), hit('A', 3, 0.9), hit('C', 3, 0.6)]);
31 + expect(sorted.map((h) => h.title)).toEqual(['A', 'C', 'B']);
32 + });
33 + it('breaks remaining ties alphabetically, case-insensitively', () => {
34 + const sorted = sortHits([hit('beta', 2, 0.5), hit('Alpha', 2, 0.5), hit('gamma', 2, 0.5)]);
35 + expect(sorted.map((h) => h.title)).toEqual(['Alpha', 'beta', 'gamma']);
36 + });
37 + it('is a proper comparator (antisymmetric, zero for equal keys)', () => {
38 + const a = hit('Same', 1, 0.5, 'gene');
39 + const b = hit('Same', 1, 0.5, 'drug');
40 + expect(compareHits(a, b)).toBe(0);
41 + const c = hit('Other', 0, 0.1);
42 + expect(Math.sign(compareHits(a, c))).toBe(-Math.sign(compareHits(c, a)));
43 + });
44 + it('does not mutate the input', () => {
45 + const input = [hit('b', 3, 0.1), hit('a', 0, 0.1)];
46 + const copy = [...input];
47 + sortHits(input);
48 + expect(input).toEqual(copy);
49 + });
50 +});
modified apps/web/vitest.config.ts +1 −1
@@ -2,6 +2,6 @@ import { defineConfig } from 'vitest/config';
2 2 import path from 'node:path';
3 3
4 4 export default defineConfig({
5 − test: { include: ['src/**/*.test.ts'], environment: 'node', passWithNoTests: true },
5 + test: { include: ['src/**/*.test.ts', 'test/**/*.test.ts'], environment: 'node', passWithNoTests: true },
6 6 resolve: { alias: { '@': path.resolve(__dirname, 'src') } },
7 7 });
modified deploy/README.md +23 −1
@@ -11,6 +11,7 @@ these are the files and the procedure.
11 11 | `cancerindex-web` (PM2) | `~/apps/cancerindex/apps/web`, `next start -p 8250 -H 0.0.0.0` | 8250 (public via ngrok) |
12 12 | `cancerindex-api` (PM2) | `~/apps/cancerindex`, `tsx apps/api/src/server.ts` | 8251 (127.0.0.1) |
13 13 | `cancerindex-worker` (PM2) | `~/apps/cancerindex`, `tsx workers/main.ts` | — |
14 +| `cancerindex-backup` (PM2, cron `20 5 * * *`, `autorestart: false`) | `~/apps/cancerindex`, `/bin/bash deploy/backup.sh` | — |
14 15 | `cancerindex-ngrok` | `www.cancerindex.io` → 8250 (reserved domain) | — |
15 16 | PostgreSQL 17 | database `cancerindex` on the node (extensions `pg_trgm`, `unaccent`, `vector`) | 5432 |
16 17
@@ -79,7 +80,28 @@ Preferred node: **M4M64b** (Postgres 17 + pgvector, pnpm, redis, 772 GB disk). A
79 80 - **Move node**: `mld move cancerindex --to <node>` — the data lake (`data/raw`) is excluded from sync;
80 81 copy it separately (`rsync -a M4M64b:~/apps/cancerindex/data/raw/ <node>:~/apps/cancerindex/data/raw/`)
81 82 and dump/restore the database (`pg_dump -Fc cancerindex`).
82 −- **Backups**: `pg_dump -Fc cancerindex` + `data/raw` (append-only). Both are needed for TRACE.
83 +- **Backups** (§172): the PM2 process `cancerindex-backup` runs `deploy/backup.sh` every day at
84 + 05:20 UTC (`cron_restart`, `autorestart: false` — PM2 restarts it on the cron only). It writes
85 + `pg_dump -Fc` to `~/apps/cancerindex/backups/cancerindex-YYYYMMDD-HHMM.dump`, verifies the archive
86 + with `pg_restore --list` (core tables must be present), keeps the last **14 daily + 8 weekly**
87 + dumps, logs to `logs/backup.log` and exits non-zero on any failure (`pm2 ls` shows it `errored`;
88 + `pm2 logs cancerindex-backup`). Run it by hand with `bash deploy/backup.sh`.
89 + The raw data lake (`data/raw`, append-only gzip JSON Lines) is **not** in the dump — mirror it with
90 + `rsync -a ~/apps/cancerindex/data/raw/ <nas-or-node>:/…/cancerindex-raw/`. Both are needed for TRACE.
91 +- **Restore test / disaster recovery**: `bash deploy/restore.sh backups/<file>.dump` creates
92 + `cancerindex_restore_<timestamp>` on the same server (never overwrites `cancerindex`), restores in
93 + parallel (`--jobs 4`), prints row counts. To promote: stop the PM2 processes, `ALTER DATABASE
94 + cancerindex RENAME TO cancerindex_old; ALTER DATABASE cancerindex_restore_… RENAME TO cancerindex;`
95 + (or point `DATABASE_URL` at the restored database), start the processes, run `pnpm cix doctor`.
96 + Test a restore after every schema migration.
97 +- **Readiness / health**: `pnpm cix doctor` (env, database + extensions + pending migrations, table
98 + sizes, every connector's health / last success / last run / anomaly / drift / cursor, stale
99 + connectors, unresolved labels, `data/raw` disk usage, ranking freshness, open alerts; exit 1 on a
100 + hard failure) and `pnpm cix alerts` (open alerts; `ack <id>` / `resolve <id>`). The API exposes
101 + `GET /v1/admin/alerts` (admin token).
102 +- **Stopping a run cleanly**: `pm2 stop cancerindex-worker` sends SIGTERM — the active connector
103 + saves its cursor and marks the run `aborted`; the next scheduled run resumes at the last completed
104 + page (docs/connectors/README.md).
83 105 - **Trigger a run**: `pnpm cix run <id>` on the node, or `POST /v1/admin/connectors/<id>/run` with
84 106 `x-admin-token`, or `tsx workers/cli.ts run <id>`.
85 107 - **Pause a connector**: `POST /v1/admin/connectors/<id>/pause` (worker skips scheduled runs).
added deploy/backup.sh +82 −0
@@ -0,0 +1,82 @@
1 +#!/usr/bin/env bash
2 +# CancerIndex — PostgreSQL backup (CLAUDE.md §172).
3 +#
4 +# pg_dump -Fc <database> → $BACKUP_DIR/cancerindex-YYYYMMDD-HHMM.dump, verified with pg_restore --list,
5 +# retention: last 14 daily dumps + 8 weekly (the newest dump of each ISO week), everything else pruned.
6 +# Logs to $LOG_DIR/backup.log; exits non-zero on any failure (PM2 shows the process as errored).
7 +#
8 +# Usage (on the node): cd ~/apps/cancerindex && bash deploy/backup.sh
9 +# DATABASE_URL postgres://localhost:5432/cancerindex (default)
10 +# BACKUP_DIR ~/apps/cancerindex/backups (default)
11 +# LOG_DIR <repo>/logs (default)
12 +# KEEP_DAILY=14 KEEP_WEEKLY=8
13 +#
14 +# Scheduled by PM2 (`cancerindex-backup`, cron_restart "20 5 * * *", autorestart false) — see the mld manifest.
15 +# The raw data lake (data/raw) is append-only and NOT part of this dump: rsync it separately (deploy/README.md).
16 +set -uo pipefail
17 +
18 +cd "$(dirname "$0")/.." || exit 1
19 +DATABASE_URL="${DATABASE_URL:-postgres://localhost:5432/cancerindex}"
20 +BACKUP_DIR="${BACKUP_DIR:-$HOME/apps/cancerindex/backups}"
21 +LOG_DIR="${LOG_DIR:-$PWD/logs}"
22 +KEEP_DAILY="${KEEP_DAILY:-14}"
23 +KEEP_WEEKLY="${KEEP_WEEKLY:-8}"
24 +PREFIX="cancerindex-"
25 +
26 +mkdir -p "$BACKUP_DIR" "$LOG_DIR" || { echo "cannot create $BACKUP_DIR / $LOG_DIR" >&2; exit 1; }
27 +LOG="$LOG_DIR/backup.log"
28 +ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }
29 +log() { echo "[$(ts)] $*" | tee -a "$LOG"; }
30 +fail() { log "ERROR: $*"; exit 1; }
31 +
32 +command -v pg_dump >/dev/null || fail "pg_dump not found in PATH"
33 +command -v pg_restore >/dev/null || fail "pg_restore not found in PATH"
34 +
35 +STAMP="$(date +%Y%m%d-%H%M)"
36 +OUT="$BACKUP_DIR/${PREFIX}${STAMP}.dump"
37 +TMP="$OUT.partial"
38 +DB_LABEL="$(printf '%s' "$DATABASE_URL" | sed -E 's#//([^:@/]+):[^@/]+@#//\1:***@#')"
39 +
40 +log "== backup start db=$DB_LABEL → $OUT"
41 +t0=$(date +%s)
42 +if ! pg_dump -Fc --no-owner --no-acl -d "$DATABASE_URL" -f "$TMP" 2>>"$LOG"; then
43 + rm -f "$TMP"
44 + fail "pg_dump failed (see $LOG)"
45 +fi
46 +mv "$TMP" "$OUT" || fail "cannot move $TMP → $OUT"
47 +
48 +# Verification: the archive must be a readable custom-format dump listing our core tables.
49 +if ! LIST="$(pg_restore --list "$OUT" 2>>"$LOG")"; then
50 + fail "pg_restore --list failed on $OUT — dump is corrupt"
51 +fi
52 +TABLES="$(printf '%s\n' "$LIST" | grep -c 'TABLE DATA' || true)"
53 +for t in cancers provenance ingest_runs; do
54 + printf '%s\n' "$LIST" | grep -q "TABLE DATA public $t " || fail "table $t missing from dump listing"
55 +done
56 +SIZE="$(du -h "$OUT" | cut -f1)"
57 +log "dump ok: $SIZE, $TABLES tables with data, $(( $(date +%s) - t0 )) s"
58 +
59 +# Retention: keep the newest $KEEP_DAILY dumps, plus the newest dump of each of the last $KEEP_WEEKLY ISO weeks.
60 +KEEP_FILE="$(mktemp)"
61 +# shellcheck disable=SC2012
62 +ls -1 "$BACKUP_DIR"/${PREFIX}*.dump 2>/dev/null | sort -r | head -n "$KEEP_DAILY" >"$KEEP_FILE"
63 +# (no associative arrays: macOS /bin/bash is 3.2)
64 +SEEN_WEEKS=" "
65 +weeks=0
66 +for f in $(ls -1 "$BACKUP_DIR"/${PREFIX}*.dump 2>/dev/null | sort -r); do
67 + base="$(basename "$f" .dump)"; d="${base#"$PREFIX"}"; d="${d%%-*}" # YYYYMMDD
68 + if date -j -f %Y%m%d "$d" +%G-%V >/dev/null 2>&1; then wk="$(date -j -f %Y%m%d "$d" +%G-%V)"; else wk="$(date -d "$d" +%G-%V 2>/dev/null || echo "$d")"; fi
69 + case "$SEEN_WEEKS" in *" $wk "*) continue ;; esac
70 + if [ "$weeks" -lt "$KEEP_WEEKLY" ]; then
71 + SEEN_WEEKS="$SEEN_WEEKS$wk "; weeks=$((weeks + 1)); echo "$f" >>"$KEEP_FILE"
72 + fi
73 +done
74 +pruned=0
75 +for f in "$BACKUP_DIR"/${PREFIX}*.dump; do
76 + [ -e "$f" ] || continue
77 + if ! grep -qxF "$f" "$KEEP_FILE"; then rm -f "$f" && pruned=$((pruned + 1)) && log "pruned $(basename "$f")"; fi
78 +done
79 +rm -f "$KEEP_FILE" "$BACKUP_DIR"/${PREFIX}*.dump.partial
80 +TOTAL="$(du -sh "$BACKUP_DIR" | cut -f1)"
81 +log "retention: kept $(ls -1 "$BACKUP_DIR"/${PREFIX}*.dump | wc -l | tr -d ' ') dumps ($TOTAL), pruned $pruned"
82 +log "== backup done"
modified deploy/mld-manifest.cancerindex.json +19 −0
@@ -131,6 +131,25 @@
131 131 "cron_restart": null,
132 132 "autorestart": true,
133 133 "max_memory_restart": "3G"
134 + },
135 + {
136 + "name": "cancerindex-backup",
137 + "manager": "pm2",
138 + "script": "/bin/bash",
139 + "args": ["deploy/backup.sh"],
140 + "interpreter": null,
141 + "cwd": "{{HOME}}/apps/cancerindex",
142 + "env": {
143 + "DATABASE_URL": "postgres://localhost:5432/cancerindex",
144 + "BACKUP_DIR": "{{HOME}}/apps/cancerindex/backups",
145 + "LOG_DIR": "{{HOME}}/apps/cancerindex/logs",
146 + "KEEP_DAILY": "14",
147 + "KEEP_WEEKLY": "8",
148 + "PATH": "/opt/homebrew/opt/postgresql@17/bin:/opt/homebrew/bin:/usr/bin:/bin"
149 + },
150 + "cron_restart": "20 5 * * *",
151 + "autorestart": false,
152 + "max_memory_restart": null
134 153 }
135 154 ]
136 155 }
added deploy/restore.sh +60 −0
@@ -0,0 +1,60 @@
1 +#!/usr/bin/env bash
2 +# CancerIndex — restore a pg_dump custom archive into a NEW database (CLAUDE.md §172 "test restore").
3 +#
4 +# bash deploy/restore.sh <dump-file> [--target <dbname>] [--jobs N]
5 +#
6 +# Creates `cancerindex_restore_<YYYYMMDD-HHMMSS>` (or --target) on the same server as PG_URL /
7 +# DATABASE_URL, restores into it and prints row counts of the core tables. It NEVER touches the
8 +# production database: promoting a restore is a deliberate manual step (rename databases, or point
9 +# DATABASE_URL at the restored one, after stopping the PM2 processes).
10 +set -uo pipefail
11 +
12 +DUMP="${1:-}"; shift || true
13 +[ -n "$DUMP" ] && [ -f "$DUMP" ] || { echo "usage: $0 <dump-file> [--target dbname] [--jobs N]" >&2; exit 2; }
14 +TARGET=""; JOBS="${RESTORE_JOBS:-4}"
15 +while [ $# -gt 0 ]; do
16 + case "$1" in
17 + --target) TARGET="$2"; shift 2 ;;
18 + --jobs) JOBS="$2"; shift 2 ;;
19 + *) echo "unknown option $1" >&2; exit 2 ;;
20 + esac
21 +done
22 +
23 +SRC_URL="${DATABASE_URL:-postgres://localhost:5432/cancerindex}"
24 +# Server URL without the database name (…/dbname → …/), used for createdb + the target connection.
25 +SERVER_URL="$(printf '%s' "$SRC_URL" | sed -E 's#^(postgres(ql)?://[^/]*)/[^?]*#\1/#')"
26 +TARGET="${TARGET:-cancerindex_restore_$(date +%Y%m%d-%H%M%S)}"
27 +case "$TARGET" in cancerindex|postgres|template0|template1) echo "refusing to restore into '$TARGET'" >&2; exit 2 ;; esac
28 +TARGET_URL="${SERVER_URL}${TARGET}"
29 +
30 +command -v pg_restore >/dev/null || { echo "pg_restore not found" >&2; exit 1; }
31 +pg_restore --list "$DUMP" >/dev/null || { echo "not a readable pg_dump archive: $DUMP" >&2; exit 1; }
32 +
33 +echo "== restoring $(basename "$DUMP") → $TARGET (server ${SERVER_URL%/})"
34 +if psql -d "${SERVER_URL}postgres" -Atc "SELECT 1 FROM pg_database WHERE datname = '$TARGET'" | grep -q 1; then
35 + echo "database $TARGET already exists — choose another --target" >&2; exit 1
36 +fi
37 +psql -d "${SERVER_URL}postgres" -v ON_ERROR_STOP=1 -qc "CREATE DATABASE \"$TARGET\"" || exit 1
38 +psql -d "$TARGET_URL" -v ON_ERROR_STOP=1 -qc "CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS unaccent;" || exit 1
39 +psql -d "$TARGET_URL" -qc "CREATE EXTENSION IF NOT EXISTS vector" 2>/dev/null || echo "note: pgvector not available — vector columns will fail to restore if present"
40 +
41 +t0=$(date +%s)
42 +# --no-owner/--no-acl: the dump was taken that way; -j parallel restore for large tables.
43 +if ! pg_restore --no-owner --no-acl --exit-on-error -j "$JOBS" -d "$TARGET_URL" "$DUMP"; then
44 + echo "pg_restore FAILED — database $TARGET left in place for inspection (drop it with: dropdb $TARGET)" >&2
45 + exit 1
46 +fi
47 +echo "restored in $(( $(date +%s) - t0 )) s"
48 +echo "== row counts ($TARGET)"
49 +psql -d "$TARGET_URL" -Atc "
50 + SELECT rpad(t, 28) || count FROM (
51 + SELECT 'cancers' AS t, count(*)::text AS count FROM cancers UNION ALL
52 + SELECT 'provenance', count(*)::text FROM provenance UNION ALL
53 + SELECT 'source_records', count(*)::text FROM source_records UNION ALL
54 + SELECT 'ingest_runs', count(*)::text FROM ingest_runs UNION ALL
55 + SELECT 'clinical_trials', count(*)::text FROM clinical_trials UNION ALL
56 + SELECT 'rankings', count(*)::text FROM rankings
57 + ) x"
58 +echo "== done. Inspect with: psql $TARGET_URL"
59 +echo " To promote: stop PM2 processes, then rename databases (ALTER DATABASE … RENAME TO …) or point DATABASE_URL at $TARGET."
60 +echo " To discard: dropdb $TARGET"
modified docs/ARCHITECTURE.md +44 −0
@@ -149,6 +149,50 @@ places it on the preferred node (M4M64b: Postgres 17 + pgvector, pnpm, large dis
149 149 and starts three PM2 processes plus the ngrok tunnel. `deploy/first-run.sh` bootstraps the data in
150 150 the recommended connector order. Details in `deploy/README.md`.
151 151
152 +## 7b. Operations: checkpoints, anomaly guard, alerts, backups, doctor
153 +
154 +Reference: `docs/connectors/README.md` (runtime guarantees), `deploy/README.md` (day-2 procedures),
155 +`docs/schema-changes-ops.md` (ops tables and indexes added outside the generated migrations).
156 +
157 +- **Checkpoints (§90).** `RunContext` persists `connector_cursors.cursor` (and
158 + `ingest_runs.cursor_after`) automatically every `manifest.checkpointEvery` upserted records
159 + (default 2 000) or 60 s when the cursor changed, on `ctx.saveCursor()`, and on SIGTERM/SIGINT
160 + (`runConnector` installs handlers for the duration of the run: first signal → cursor saved, run
161 + `aborted`, connector stops at its next `shouldStop()`; second signal → immediate exit). A killed
162 + run therefore resumes from the last completed page; payload hashes make the overlap idempotent.
163 + `RawLake` serialises raw writes through one promise chain (single shared drain, no listener
164 + pile-up) and can `flush()` gzip blocks before a kill.
165 +- **Anomaly guard (§171).** `ctx.guardCount(entity, total)` compares a declared total with the
166 + previous successful run's `records_fetched` (`anomalyGuard.minRatioOfPrevious`, default 0.5) and
167 + refuses to continue: run `failed` + `ingest_runs.anomaly`, critical alert, previous data untouched.
168 + Wired into OncoTree, NCIt and the ClinicalTrials.gov full crawl; connectors keep their fixed floors
169 + (HGNC, GDC, USCS, CIViC…). Nothing is ever mass-deleted on a shrunken response.
170 +- **Alerts (§170).** Table `system_alerts` (`kind`, `severity info|warn|critical`, `connector_id`,
171 + `message`, `detail`, `first_seen_at`, `last_seen_at`, `count`, `status open|acknowledged|resolved`),
172 + deduplicated on kind + connector + message. Raised by `runConnector` (failure, anomaly, aborted,
173 + schema drift) and by the worker's hourly health probe (`source_failing`, `source_stale` = no
174 + success within 2× the cron interval of an active connector); resolved on the next success.
175 + Surfaces: `pnpm cix alerts`, `pnpm cix doctor`, `GET /v1/admin/alerts`.
176 +- **Backups (§172).** PM2 process `cancerindex-backup` (`cron_restart "20 5 * * *"`,
177 + `autorestart: false`) runs `deploy/backup.sh`: `pg_dump -Fc` to
178 + `~/apps/cancerindex/backups/cancerindex-YYYYMMDD-HHMM.dump`, verified with `pg_restore --list`,
179 + 14 daily + 8 weekly retained, `logs/backup.log`, non-zero exit on failure. `deploy/restore.sh
180 + <dump>` restores into a fresh `cancerindex_restore_<ts>` database (never over production). The raw
181 + lake (`data/raw`) is mirrored separately with rsync; both are needed for TRACE.
182 +- **Doctor.** `pnpm cix doctor` prints the readiness report (environment, database + extensions +
183 + pending migrations vs `drizzle.__drizzle_migrations`, table sizes, every connector's status /
184 + license / health / last success age / last run status-anomaly-drift / cursor summary / stale flag,
185 + unresolved-label backlog, `data/raw` disk usage and free space, ranking-snapshot freshness, open
186 + alerts) and exits 1 on a hard failure (unreachable database, missing required extension, pending
187 + migration, unwritable data dir, missing ops schema, critical alert).
188 +- **Indexes.** Indexes that drizzle cannot express or that were added by the performance review are
189 + created idempotently in `packages/database/src/migrate.ts` after the SQL migrations
190 + (`createPerformanceIndexes`): GIN on `civic_evidence_items.gene_ids/variant_ids/therapy_ids`
191 + (the counters use the containment form `@> ARRAY[id]::text[]`), `trial_conditions(cancer_id,
192 + trial_id)`, `clinical_trials(overall_status, study_type)` and `(overall_status,
193 + last_update_posted_date)`, `cancer_aliases(normalized text_pattern_ops)`,
194 + `ingest_runs(connector_id, status, started_at)`.
195 +
152 196 ## 8. Phase 1 boundaries
153 197
154 198 - Rankings are count-based (trials, literature, curated evidence, genes, cohorts) for all entities
modified docs/SECURITY.md +19 −0
@@ -69,6 +69,25 @@ API-key owner e-mails.
69 69 (`traceValue`).
70 70 - Schema changes go through Drizzle migrations reviewed by the integrator; no manual production
71 71 mutation.
72 +- Interrupted runs are safe: SIGTERM/SIGINT persist the cursor and mark the run `aborted`; the raw
73 + lake is flushed so referenced payloads stay readable; a killed run resumes from its last checkpoint
74 + (docs/connectors/README.md). A shrunken upstream response trips the anomaly guard (`ingest_runs.anomaly`,
75 + critical alert) instead of propagating deletions.
76 +- Operational alerts (`system_alerts`) carry connector ids, messages and small JSON details only —
77 + never credentials or request headers. `GET /v1/admin/alerts` is behind the admin token.
78 +
79 +### Backups and recovery (§172)
80 +
81 +- Daily `pg_dump -Fc` (PM2 `cancerindex-backup`, 05:20 UTC) to `~/apps/cancerindex/backups`,
82 + verified with `pg_restore --list`, 14 daily + 8 weekly retained, log in `logs/backup.log`. Dumps
83 + contain no end-user PII beyond API-key owner e-mails and only sha256 hashes of API keys, but they
84 + do contain the full canonical/derived data: keep the backups directory readable by the service
85 + user only and mirror it (with `data/raw`) to the NAS over the LAN, never to a public bucket.
86 +- `deploy/restore.sh` always restores into a new `cancerindex_restore_<ts>` database; promoting a
87 + restore is an explicit operator action (rename databases after stopping PM2). Test a restore after
88 + every schema migration.
89 +- `pnpm cix doctor` is the post-incident checklist: environment, database, pending migrations,
90 + connector freshness and open alerts in one exit code.
72 91
73 92 ### Dependencies and runtime
74 93
added docs/connectors/README.md +92 −0
@@ -0,0 +1,92 @@
1 +# Connectors — runtime guarantees (restart, checkpoints, anomaly guard, alerts)
2 +
3 +One page per connector lives next to this file (`clinvar.md`, `clinicaltrials.md`, …). This page
4 +documents what the SDK (`packages/connectors/src/sdk/run.ts`) guarantees for **every** connector,
5 +so a connector author only has to respect three rules.
6 +
7 +## Rules for connector authors
8 +
9 +1. **Mutate `ctx.cursor` only at a safe point** — after a page / batch is fully persisted
10 + (`upsertSourceRecord`, canonical rows, provenance). The SDK may persist the cursor at any moment
11 + after that; a cursor must never be ahead of the data.
12 +2. **Call `ctx.shouldStop()` between pages** and return when it is true (time budget, record cap or
13 + abort request). Do not catch and swallow the return — the run status is decided by `runConnector`.
14 +3. **Use the SDK writers** (`ctx.upsertSourceRecord`, `ctx.upsertSourceRecordsBatch`) or, when you
15 + keep your own batch writer (ClinVar), call `await ctx.checkpoint(n)` after each persisted batch.
16 +
17 +## What the SDK guarantees
18 +
19 +### Cursor persistence (CLAUDE.md §90)
20 +
21 +| Moment | What is written | Where |
22 +|---|---|---|
23 +| Run start | `cursor_before` (the cursor the run starts from), `last_attempt_at` | `ingest_runs`, `connector_cursors` |
24 +| Automatic checkpoint | current `ctx.cursor` **if it changed** since the last save — every `manifest.checkpointEvery` upserted records (default 2 000) **or** every 60 s, evaluated in `upsertSourceRecord` / `upsertSourceRecordsBatch` / `ctx.checkpoint()` | `connector_cursors.cursor`, `ingest_runs.cursor_after` |
25 +| `await ctx.saveCursor()` | immediately, serialised (concurrent calls share one write) | same |
26 +| SIGTERM / SIGINT | cursor saved, run marked `aborted` (provisionally, then finalised when `sync()` returns), raw-lake gzip streams flushed | same + `system_alerts` |
27 +| End of run (`succeeded`, `partial`, `failed`, `aborted`) | final cursor, counters, log, schema drift, anomaly | `ingest_runs`, `connector_cursors` |
28 +
29 +`dry_run` and `probe` modes never write the cursor (`ctx.persistsCursor === false`), so a smoke run
30 +cannot move a production cursor. `--mode full` and `--reset-cursor` start from `{}`.
31 +
32 +### Restart semantics
33 +
34 +- **Killed run (SIGKILL, OOM, power loss)**: the next run resumes from the last checkpoint —
35 + at most `checkpointEvery` records (or 60 s of work) are re-fetched; records are hashed
36 + (`source_records.payload_hash`), so re-processing is idempotent (`unchanged`, no duplicates).
37 +- **SIGTERM / SIGINT (PM2 stop/restart, Ctrl-C)**: the first signal makes `shouldStop()` return
38 + true, saves the cursor and marks the run `aborted`; the connector returns at its next page
39 + boundary and `runConnector` finalises the run. A second signal exits the process immediately
40 + (status and cursor are already persisted). PM2's default `kill_timeout` is 1.6 s; the provisional
41 + bookkeeping is done first precisely so that a hard kill after it still leaves a consistent state.
42 + Handlers are installed for the duration of the run only (`handleSignals: false` disables them).
43 +- **Time budget** (`--max-minutes`, `CI_MAX_RUN_MINUTES`): status `partial`, cursor saved; the
44 + worker's next scheduled run (or `first-run.sh`'s retry loop) continues.
45 +- **Failure**: status `failed`, cursor saved as it was at the last safe point, health `failing`.
46 +- **Anomaly** (see below): status `failed` with `ingest_runs.anomaly` set; nothing destructive
47 + happened, the previous data stays published.
48 +
49 +### Anomaly guard (CLAUDE.md §171)
50 +
51 +`await ctx.guardCount(entity, declaredTotal)` compares a total the source declares (or the number
52 +of parsed records) with the previous **successful** run's `records_fetched` for the connector. If
53 +`declaredTotal < previous × manifest.anomalyGuard.minRatioOfPrevious` (default 0.5) it throws
54 +`AnomalyError`, sets `ctx.anomaly`, and `runConnector` records the run as `failed` with
55 +`ingest_runs.anomaly`, health `failing`, and a **critical** `anomaly` alert. The first run of a
56 +connector is accepted as the baseline. Wired in: OncoTree (tumour types), NCIt (concepts +
57 +qualified states + retired), ClinicalTrials.gov (`totalCount` of a full crawl); connectors with a
58 +fixed floor (HGNC ≥ 30 k rows, GDC ≥ 40 projects, USCS ≥ 20 k rows…) keep their own checks — any
59 +error message starting with `anomaly:` is also recorded in `ingest_runs.anomaly`.
60 +
61 +### Schema drift (CLAUDE.md §25)
62 +
63 +Observed field names/types per entity are stored in `connector_field_stats`; new fields or type
64 +changes are listed in `ingest_runs.schema_drift`, set health to `degraded` and raise a `schema_drift`
65 +alert (resolved automatically on the next drift-free run).
66 +
67 +### Alerts (CLAUDE.md §170)
68 +
69 +`system_alerts` rows (`packages/database/src/alerts.ts`: `raiseAlert`, `resolveAlerts`,
70 +`listAlerts`) are deduplicated on kind + connector + message while open/acknowledged (`count`
71 +increments). Raised by the SDK: `connector_failure` (warn), `anomaly` (critical),
72 +`connector_aborted` (info, or warn when credentials are missing), `schema_drift` (warn). Raised by
73 +the worker's hourly health probe: `source_failing`, `source_stale` (no success within 2× the
74 +schedule interval). A successful run resolves the connector's failure/aborted/anomaly/stale/failing
75 +alerts. Read them with `pnpm cix alerts`, `pnpm cix doctor`, or `GET /v1/admin/alerts`.
76 +
77 +## Operator cheat-sheet
78 +
79 +```bash
80 +pnpm cix run clinvar --max-minutes 30 # partial → cursor saved → run again to continue
81 +pnpm cix run clinvar --mode backfill # reprocess a dataset version already marked complete
82 +pnpm cix run oncotree --reset-cursor # start from scratch (idempotent thanks to payload hashes)
83 +pnpm cix doctor # readiness: env, DB, migrations, connectors, stale, alerts
84 +pnpm cix alerts [--status resolved] ; pnpm cix alerts ack 12 ; pnpm cix alerts resolve 12
85 +psql -c "SELECT connector_id, cursor, last_success_at, health FROM connector_cursors"
86 +psql -c "SELECT id, status, records_fetched, anomaly, cursor_after FROM ingest_runs WHERE connector_id='clinvar' ORDER BY started_at DESC LIMIT 5"
87 +```
88 +
89 +Tests: `packages/connectors/src/sdk/run.test.ts` (fake paged connector: checkpoint every 5 records,
90 +SIGTERM at page 3 → `aborted` + resume from page 3, anomaly guard, alert dedupe, doctor) and
91 +`lake.test.ts` (5 000 concurrent raw writes, single drain listener). The database-backed tests run
92 +only when `CI_TEST_DATABASE_URL` (or `DATABASE_URL`) is set.
added docs/connectors/cbioportal.md +63 −0
@@ -0,0 +1,63 @@
1 +# Connector `cbioportal` — cBioPortal for Cancer Genomics (public studies)
2 +
3 +| | |
4 +|---|---|
5 +| Source | https://www.cbioportal.org — public instance operated by Memorial Sloan Kettering Cancer Center with the cBioPortal consortium (Dana-Farber, Princess Margaret, Children's Hospital of Philadelphia, The Hyve, Bilkent…) |
6 +| Tier / category | 1 / genomics (CLAUDE.md §10.8 — cohorts beyond TCGA, original study IDs preserved) |
7 +| Access | REST `https://www.cbioportal.org/api`, no authentication, OpenAPI 3 (`/api/v3/api-docs`, Swagger UI `/api/swagger-ui/index.html`) |
8 +| Docs verified | 2026-09-08 — OpenAPI document, https://docs.cbioportal.org/user-guide/faq/ (What is cBioPortal, license, citation), live API |
9 +| License | **ODC Open Database License (ODC-ODbL)** unless otherwise noted per study, attribution to the original studies required — `licenseStatus: approved`, `redistribution: attribution`, `commercialUse: restricted` (some studies restrict commercial use) |
10 +| Code | `packages/connectors/src/connectors/cbioportal/` (`manifest.ts`, `normalize.ts`, `index.ts`, `cbioportal.test.ts`, `fixtures/`) |
11 +
12 +## License and citation (quoted)
13 +
14 +FAQ, "Do I need permission or a license to use the cBioPortal logo or figures or data?" (https://docs.cbioportal.org/user-guide/faq/):
15 +
16 +> You are free to use figures from cBioPortal, there is no copyright permission required. However you must provide proper citation for the cBioPortal and the original data source as described above. Unless otherwise noted, data in cBioPortal are available under the ODC Open Database License with no restrictions on the use of the data, as long as you properly give attribution to the original studies. There are some studies that restrict the commercial use of the data, but that will be explicitly mentioned in the study information.
17 +
18 +FAQ, "What is the cBioPortal for Cancer Genomics?":
19 +
20 +> The cBioPortal for Cancer Genomics is an open-access, open-source resource for interactive exploration of multidimensional cancer genomics data sets.
21 +
22 +FAQ, "How do I cite the cBioPortal?": Cerami E et al., *The cBio Cancer Genomics Portal: An Open Platform for Exploring Multidimensional Cancer Genomics Data*, Cancer Discov 2012 (PMID 22588877); Gao J et al., *Integrative analysis of complex cancer genomics and clinical profiles using the cBioPortal*, Sci Signal 2013 (PMID 23550210); de Bruijn I et al., *Analysis and Visualization of Longitudinal Genomic and Clinical Data from the AACR Project GENIE Biopharma Collaborative in cBioPortal*, Cancer Res 2023 (PMID 37668528). "Remember also to cite the source of the data if you are using a publicly available dataset."
23 +
24 +Consequences for CancerIndex: the portal citation is the manifest `attribution`; every cohort's provenance carries the study's own `citation` and `pmid` (417 of 545 studies have one) and a publication stub is created for the PMID; the study `description` (where commercial-use notices appear) is kept on the source record; only aggregate per-gene counts are stored (no sample- or patient-level data).
25 +
26 +## Verified endpoints
27 +
28 +| Endpoint | Verified behaviour (2026-09-08) |
29 +|---|---|
30 +| `GET /api/studies?pageSize=1000&pageNumber=0&projection=DETAILED` | 545 public studies (one page; `pageNumber` paging supported). Fields: `studyId`, `cancerTypeId` (OncoTree code, lower case; `mixed` for pan-cancer cohorts — 48 studies), `name`, `description`, `publicStudy`, `pmid`, `citation`, `importDate "2026-07-28 17:54:40"`, `allSampleCount`, **`sequencedSampleCount`** (samples with mutation data), `cnaSampleCount`, `referenceGenome` (473 hg19, 72 hg38), `cancerType{id,name,parent}`. 539/545 have `sequencedSampleCount > 0`. |
31 +| `GET /api/cancer-types?pageSize=10000` | 897 OncoTree nodes `{cancerTypeId, name, shortName, parent}`; root `tissue`; `mixed` → parent `other`. |
32 +| `GET /api/studies/{studyId}/molecular-profiles` | e.g. `msk_impact_2017_mutations` (`molecularAlterationType MUTATION_EXTENDED`, datatype MAF), `_cna`, `_structural_variants`. |
33 +| `GET /api/sample-lists/{studyId}_sequenced` | `{category all_cases_with_mutation_data, sampleCount, sampleIds[]}` — `msk_impact_2017_sequenced` 10,945 = `sequencedSampleCount`; `acc_tcga_sequenced` 90 = `sequencedSampleCount`. The study-level count is therefore used as the denominator source without downloading sample ids (the SUMMARY projection of `/studies/{id}/sample-lists` has no `sampleCount`). |
34 +| `POST /api/mutated-genes/fetch` body `{"studyIds":["msk_impact_2017"]}` (StudyViewFilter) | Array of `AlterationCountByGene`: `hugoGeneSymbol`, `entrezGeneId`, **`numberOfAlteredCases`** (distinct samples with ≥ 1 mutation in the gene), **`numberOfProfiledCases`** (samples profiled for that gene — gene-panel aware: MSK-IMPACT genes on IMPACT341+410 → 10,945; genes only on IMPACT410 → 8,136; 4 genes → 0), `totalCount` (mutations), `matchingGenePanelIds`. TP53 4,538/10,945, KRAS 1,643, TERT 1,460, PIK3CA 1,355, APC 1,121 — identical to the study summary "Mutated Genes" table. ~1 s for 10,945 samples. |
35 +| Alternatives inspected | `/api/molecular-profiles/{id}/mutations/fetch` (per-mutation rows, `pageSize` ≤ 10,000,000 — hundreds of thousands of rows for large studies) and `/api/molecular-profiles/{id}/mutations?sampleListId=…&entrezGeneId=…` (one gene at a time) exist but are far more expensive; `/api/mutation-counts-by-gene` does not exist; `/api/studies/{id}/significantly-mutated-genes` is MutSig only. `/api/mutated-genes/fetch` is the cheapest endpoint returning per-gene counts with a denominator. |
36 +
37 +Health check: `GET /api/studies?pageSize=1&projection=SUMMARY` → healthy when one study is returned.
38 +
39 +## Ingestion
40 +
41 +1. `/studies` (DETAILED, paged) + `/cancer-types`. Anomaly guard: < 100 studies → refuse to persist. `datasetVersion = cbioportal-<latest importDate>`.
42 +2. Studies are processed in `studyId` order; `ctx.cursor.lastStudyId` is saved after each study (restartable, time budget honoured between studies); a completed pass starts over on the next schedule.
43 +3. **Cohort** (`genomic_cohorts`, unique on `source_id + study_id`): `studyId`, `name`, `program` = first token of the name's trailing parenthetical (`MSK-IMPACT Clinical Sequencing Cohort (MSK, Nat Med 2017)` → **MSK**; `… (TCGA, PanCancer Atlas)` → **TCGA**), `primarySites` = OncoTree node just below the `tissue` root (luad → nsclc → lung → **Lung**), `diseaseTypes` = cancer-type name, `caseCount = allSampleCount`, `casesWithSsm = sequencedSampleCount`, `dataRelease = importDate (date)`, `accessLevel open`, `url https://www.cbioportal.org/study/summary?id=<studyId>`, provenance per study (`sourceUrl /api/studies/<id>`, `dataset 'cBioPortal public studies'`, `pmid`, `cohortSize`, `population '<studyId> (<program>)'`, methodology with the study citation). An unchanged study keeps its previous provenance row.
44 +4. **Cancer reconciliation**: `resolver.byCode('oncotree', CANCERTYPEID.toUpperCase())` → `EXACT_IDENTIFIER`; fallback `byLabel(cancer-type name)` (match type recorded, and the code stored in `cancer_codes` with that match type); `cancerTypeId = mixed` → `cancerId null`, `cancerMatchType 'UNRESOLVED'` (pan-cancer, not queued — by design); other misses → `unresolved_labels` with the OncoTree lineage.
45 +5. **Publication stub** for `pmid` (`publications`, `publicationTypes ['stub']`, title = citation) + `publication_entity_edges` to the cohort's cancer (`method cbioportal_study`) — enriched later by the PubMed connector.
46 +6. **Frequencies** (`cancer_gene_frequencies`, unique on `cohort_id + gene_symbol + alteration_type`) for studies with `sequencedSampleCount > 0` **and** a `MUTATION_EXTENDED` profile (`/molecular-profiles` and `/mutated-genes/fetch` are requested concurrently — concurrency 2): genes ranked by `numberOfAlteredCases` (desc, symbol asc), **top 200 per study** (`TOP_GENES_PER_STUDY`; WES studies return 7,000–18,000 genes), genes with `numberOfProfiledCases = 0` dropped. `alterationType 'ssm'`, `casesAffected = numberOfAlteredCases`, `casesProfiled = numberOfProfiledCases` (gene-panel aware; equals the `<studyId>_sequenced` count for exome studies), `frequency` validated by `validateFrequency()` (rejected rows counted, never stored), `rank`, `dataRelease`, `geneId` via the shared `GeneCache` (Entrez id filled when empty), `cancerId` of the cohort. Provenance per study (`sourceUrl /api/mutated-genes/fetch`, `dataset 'cBioPortal mutated genes by study'`, `pmid`, `cohortSize = sequencedSampleCount`, methodology: *distinct samples with ≥ 1 mutation in gene / samples profiled for the gene (numberOfProfiledCases; = samples in `<studyId>_sequenced` for whole-exome studies)*, profile id, top-N note). Source record `mutated_genes` per study with the ranked list.
47 +7. Studies without a mutation profile are cohorts without frequencies (`noMutationProfile` in the run summary); no per-mutation paging is ever needed, so no sample cap applies.
48 +
49 +TCGA cohorts also present in GDC are kept as **separate cohorts** (different processing: Firehose Legacy vs PanCancer Atlas vs GDC harmonized) and flagged by `program = TCGA`; they are never pooled with GDC frequencies.
50 +
51 +`dry_run` fetches studies + cancer types and the top-5 mutated genes of `acc_tcga` without writing.
52 +
53 +## Observed run (cancerindex_b, 2026-09-08)
54 +
55 +See the run summary appended below (10-record smoke, then 20-minute run).
56 +
57 +## Limitations / notes
58 +
59 +- Mutation counts follow the portal's default study-view filters (all mutation types the portal shows in "Mutated Genes"; germline/somatic and driver/VUS filters untouched); counts are per **sample**, not per patient.
60 +- Only `alteration_type = 'ssm'`; CNA (`/api/cna-genes/fetch`) and structural-variant counts are not ingested yet.
61 +- Top 200 genes per study; long-tail genes are not stored (the source record keeps the ranked list only up to the cap).
62 +- `mixed` studies (48, incl. MSK-IMPACT 2017) have no cancer; their frequencies are cohort-level only and never roll up to a cancer.
63 +- Some studies restrict commercial use (noted in their description on the portal) — review before any commercial redistribution of those cohorts; `commercialUse: restricted` at the manifest level reflects this.
added docs/connectors/chembl.md +73 −0
@@ -0,0 +1,73 @@
1 +# Connector `chembl` — ChEMBL (EMBL-EBI)
2 +
3 +| | |
4 +|---|---|
5 +| Source | https://www.ebi.ac.uk/chembl/ — European Molecular Biology Laboratory, European Bioinformatics Institute |
6 +| Tier / category | 1 / drugs |
7 +| Access | REST, `https://www.ebi.ac.uk/chembl/api/data/<resource>.json`, no authentication |
8 +| Docs verified | 2026-09-08 — https://chembl.gitbook.io/chembl-interface-documentation/web-services/chembl-data-web-services + live probes (status, molecule/search, mechanism, target, drug_indication) |
9 +| License | **CC BY-SA 3.0** — `licenseStatus: approved`, `redistribution: attribution` (share-alike noted) |
10 +| Code | `packages/connectors/src/connectors/chembl/` (`manifest.ts`, `api.ts`, `index.ts`, `chembl.test.ts`, `fixtures/`) |
11 +
12 +## License (quoted)
13 +
14 +About page (https://chembl.gitbook.io/chembl-interface-documentation/about, fetched 2026-09-08):
15 +
16 +> The ChEMBL data is made available on a Creative Commons Attribution-Share Alike 3.0 Unported License.
17 +
18 +Attribution: "ChEMBL, EMBL-EBI, CC BY-SA 3.0" on `/source/chembl` and in the API `sources` envelope; every run records the release (`datasetVersion`, e.g. `ChEMBL_37`, release DOIs exist on the Downloads page). Share-alike: the fields and edges derived from ChEMBL carry ChEMBL provenance so a redistributed extract can be published under a compatible licence. Citation: Zdrazil B et al., *The ChEMBL Database in 2023*, Nucleic Acids Res 2024 (PMID 37933841).
19 +
20 +## Verified endpoints (2026-09-08, ChEMBL_37 released 2026-05-01)
21 +
22 +| Endpoint | Used for | Notes |
23 +|---|---|---|
24 +| `status.json` | health, `datasetVersion` | `{status: "UP", chembl_db_version: "ChEMBL_37", chembl_release_date, disinct_compounds: 2921148, targets: 18552}` |
25 +| `molecule/search.json?q=<name>&limit=20` | identity | full molecule objects (`pref_name`, `molecule_type`, `max_phase`, `first_approval`, `molecule_synonyms[{molecule_synonym, syn_type}]`, `molecule_hierarchy.parent_chembl_id`, `atc_classifications`, `cross_references`, …) + `page_meta` |
26 +| `mechanism.json?parent_molecule_chembl_id=<id>&limit=200` | mechanisms | `mechanism_of_action`, `action_type`, `target_chembl_id`, `max_phase`, `mechanism_refs`. **Attached to the dosed form**: `molecule_chembl_id=CHEMBL941` (imatinib) returns 0, the parent filter returns the 4 mechanisms recorded on CHEMBL1642 (mesylate) |
27 +| `target/<id>.json` | targets → genes | `target_type` (SINGLE PROTEIN, CHIMERIC PROTEIN, PROTEIN FAMILY, PROTEIN COMPLEX…), `organism`, `target_components[].accession` (UniProt) and `target_component_synonyms` with `syn_type GENE_SYMBOL` |
28 +| `drug_indication.json?parent_molecule_chembl_id=<id>&limit=500&offset=` | indications | `mesh_id`, `mesh_heading`, `efo_id`/`efo_term` (EFO or MONDO), `max_phase_for_ind` (**string** "3.0"; mechanism `max_phase` is a number — both coerced) |
29 +
30 +No published rate limit ("reasonable use") → manifest 3 req/s, 2 in flight; targets are cached for the run. During development the API returned HTTP 500 for several minutes (EBI outage) — the SDK retries 5xx with back-off and the run fails cleanly if the outage persists; the cursor lets the next run resume.
31 +
32 +## Scope — enrichment, not import
33 +
34 +Only existing `drugs` rows (CIViC therapies today, trial interventions later) are looked up; the 2.9 M ChEMBL compounds are never crawled. Names that cannot be molecules — regimens (`… Regimen`), combinations (`A/B`), procedures (radiation, surgery, transplantation) — are skipped and counted (`isNonMoleculeName`).
35 +
36 +## Sync design
37 +
38 +Drugs are processed in `(name, id)` order; `ctx.cursor = {pass, after: {name, id}, processed, completedAt}` is flushed every 25 drugs so a run stopped by the time budget (`--max-minutes`) or a crash resumes after the last processed drug. A completed pass starts a fresh pass on the next run (full re-check against the current release; every write is idempotent).
39 +
40 +Per drug:
41 +
42 +1. **Identity** — `molecule/search.json?q=<name>` then `pickExactMatch()`: the hit whose `pref_name`, or one of its `molecule_synonyms`, equals our name after `normalizeLabel` (case/punctuation-insensitive). Ties: `pref_name` beats synonym → higher `max_phase` → parent molecule (`molecule_hierarchy.parent_chembl_id` = itself) → lowest ChEMBL number. Examples: "Osimertinib" → CHEMBL3353410 (free base, not the mesylate); "Imatinib" → CHEMBL941, "Imatinib Mesylate" → CHEMBL1642 (both exist as CIViC therapies); "Trastuzumab" → CHEMBL1201585 (not the ADCs). No exact hit → `unresolved_labels` (`entity_kind drug`, context = total hits + top 5 `{chemblId, prefName, maxPhase, type}`) — **never a fuzzy assignment**.
43 +2. **Mechanisms** for the parent molecule, deduplicated by (mechanism, target, action type) — ChEMBL repeats a mechanism per salt/dosed form.
44 +3. **Targets**: each `target_chembl_id` → `target/<id>.json` (cached) → human protein components → `GENE_SYMBOL` synonyms → `GeneCache.ensure(symbol)` (shared with CIViC/ClinVar/GDC; HGNC-style symbol check). Chimeric proteins (Bcr/Abl → ABL1 + BCR) and protein families (ABL → ABL1 + ABL2) expand to every component gene. Non-human targets yield nothing.
45 +4. **Indications** for the parent molecule (paged), deduplicated by MeSH UI with the highest `max_phase_for_ind`; EFO/MONDO ids accumulated.
46 +5. **Raw record** `ctx.upsertSourceRecord('molecule', <ChEMBL id>, {molecule, parentChemblId, matchedBy, firstApproval, atcClassifications, mechanisms, targets, indications})` → raw lake `data/raw/chembl/<date>/molecule/`. `first_approval` and ATC codes live **only here**: a first-approval year is not a jurisdictional approval record (`drug_approvals` is fed by regulatory connectors, CLAUDE.md §13).
47 +6. **Provenance**: one row per (drug, molecule) — `sourceRecordId` = ChEMBL id, `sourceUrl https://www.ebi.ac.uk/chembl/compound_report_card/<id>/`, `dataset ChEMBL`, `datasetVersion ChEMBL_37`, `evidenceType database`, `accessLevel open`, `license CC BY-SA 3.0` — reused while the source record is unchanged (looked up from the drug's existing edges).
48 +7. **`drugs` row**: `chembl_id`; `kind` from `molecule_type` (Small molecule → `small_molecule`, Antibody → `monoclonal_antibody`, Antibody drug conjugate → `adc`, Cell → `cell_therapy`, Gene → `gene_therapy`, Protein / Enzyme / Oligonucleotide / … → `other`; `Unknown` keeps the existing value); `mechanism` = distinct `mechanism_of_action` strings joined with "; "; `development_status` from `max_phase` (4 → `approved`, 3 → `phase_3`, 2 → `phase_2`, 1 → `phase_1`, 0.5 → `early_phase_1`, null/−1 untouched) and **never downgraded** (a regulatory `approved` survives a ChEMBL phase 3); `target_gene_ids` = union with the new gene ids (`sql.raw('ARRAY[…]')`). A `change_events` row records the ChEMBL id assignment.
49 +8. **`drug_aliases`** from `molecule_synonyms`: `TRADE_NAME → brand`, `INN/USAN/BAN/JAN/USP → generic`, `RESEARCH_CODE → development_code`, anything else → `synonym`; the drug's own name is skipped, duplicates collapse per (normalized, type).
50 +9. **`knowledge_edges`**
51 + - drug **TARGETS** gene — one edge per gene, `evidenceLevel` = ChEMBL `action_type`(s) joined with `|` (INHIBITOR, ANTAGONIST, BINDING AGENT…), `direction` null, `cancerContextIds []`, `evidenceCategory curated_evidence`, `sourceRecordId` = ChEMBL id, `provenanceIds [provenance]`.
52 + - drug **INVESTIGATED_FOR** cancer — for each indication whose `mesh_id` resolves through `resolver.byCode('mesh', ui)` (i.e. `cancer_codes system 'mesh'` written by the `mesh` connector; run `mesh` first): `evidenceLevel` = `max_phase_for_ind` as text ("4", "3", …, never re-scaled), `direction 'unknown'`, `cancerContextIds [cancer]`, `evidenceCategory curated_evidence`. Unresolved MeSH ids go to `unresolved_labels` (`entity_kind cancer`, context `{meshId, efoIds, chemblId, drugId, drugName, maxPhaseForIndication}`) **only when the heading is a neoplasm** — known MeSH neoplasm descriptor (mesh source records / codes) or `NEOPLASM_HEADING_RE` — so rheumatoid arthritis, malaria or pulmonary hypertension indications (imatinib) do not pollute the cancer queue; they are counted as `indicationsNonCancer`.
53 + - Upsert on the unique key (source/target/type/source_id/source_record_id); `supportCount` stays 1, `lastSeenAt` refreshed.
54 +
55 +`evidenceCategory` note: the task brief suggested `database` for indication edges; `knowledge_edges.evidence_category` is the scientific safety label set (observed_data | published_evidence | curated_evidence | regulatory_status | clinical_guideline | computed_metric, CLAUDE.md §3/§12) and ChEMBL indications are manually curated from labels and trial registries, so they are labelled `curated_evidence`; the *provenance* row carries `evidenceType database`. Change `evidenceCategory` in `index.ts` if the integrator prefers a new label.
56 +
57 +## Tests (`chembl.test.ts`, fixtures from live responses, trimmed)
58 +
59 +Search pages (osimertinib, imatinib, trastuzumab, empty, page with a malformed molecule), mechanisms (osimertinib single, imatinib four mechanisms, duplicate dosed forms), targets (single proteins EGFR/ABL1/KIT/ERBB2/PDGFRB, chimeric Bcr/Abl, ABL family, non-human), indications (dedupe by UI with max phase, string-phase coercion, multi-page merge, malformed rows), vocabulary mappings (kind, phase, synonym type, non-molecule names), manifest licence.
60 +
61 +## Observed run (cancerindex_c, 2026-09-08, `--max-minutes 15`, 661 CIViC drugs)
62 +
63 +See the table below — filled from the run log of `ING-CHEMBL-20260908-*`.
64 +
65 +OBSERVED_RUN_TABLE
66 +
67 +## Limitations / notes
68 +
69 +- Identity is exact-name only: CIViC names that differ from ChEMBL `pref_name`/synonyms (e.g. investigational codes written differently, salts named "X Hydrochloride" vs ChEMBL "X HYDROCHLORIDE" are fine, but "Anti-PD-1 antibody" is not) stay unresolved and appear in the queue with the top hits for a curator.
70 +- Mechanisms/indications come from the parent molecule, so a salt row (e.g. "Imatinib Mesylate") receives the same mechanism and indication edges as its parent — both rows keep their own ChEMBL id.
71 +- `max_phase_for_ind` reflects the highest trial phase ChEMBL curated for that indication, not an approval; approvals are regulatory records (FDA/EMA connectors).
72 +- Targets without a human protein component (organisms, viral proteins, "unchecked" targets) produce no gene edge; non-protein targets are skipped.
73 +- The connector depends on the `mesh` connector for indication edges; run order: `ncit-evs → oncotree → hgnc → civic → mesh → chembl`.
added docs/connectors/mesh.md +102 −0
@@ -0,0 +1,102 @@
1 +# Connector `mesh` — MeSH (Medical Subject Headings, NLM)
2 +
3 +| | |
4 +|---|---|
5 +| Source | https://www.nlm.nih.gov/mesh/ — U.S. National Library of Medicine |
6 +| Tier / category | 1 / terminology |
7 +| Access | SPARQL over HTTP GET, `https://id.nlm.nih.gov/mesh/sparql?query=…&format=JSON&inference=true`, no authentication; fallback `https://nlmpubs.nlm.nih.gov/projects/mesh/MESH_FILES/xmlmesh/desc<year>.xml` |
8 +| Docs verified | 2026-09-08 — https://hhs.github.io/meshrdf/ (RDF model), live predicate probe on D001943, terms page |
9 +| License | **Public domain (U.S. Government work)** under the NLM Terms and Conditions — acknowledgment requested — `licenseStatus: approved`, `redistribution: attribution` |
10 +| Code | `packages/connectors/src/connectors/mesh/` (`manifest.ts`, `sparql.ts`, `xml.ts`, `index.ts`, `mesh.test.ts`, `fixtures/`) |
11 +
12 +## Terms (quoted)
13 +
14 +NLM Terms and Conditions (https://www.nlm.nih.gov/databases/download/terms_and_conditions.html, fetched 2026-09-08):
15 +
16 +> acknowledge NLM as the source of the data by including the phrase "Courtesy of the U.S. National Library of Medicine"
17 +
18 +> not indicate or imply that NLM has endorsed its products/services/applications
19 +
20 +Redistributors must "maintain the most current version of all distributed data" or make known "in a clear and conspicuous manner" that their product does not reflect NLM's latest data. "No charges, usage fees or royalties are paid to NLM." NLM web policies (https://www.nlm.nih.gov/web_policies.html): "Works produced by the U.S. government are not subject to copyright protection in the United States."
21 +
22 +CancerIndex shows "Courtesy of the U.S. National Library of Medicine", the MeSH year (`datasetVersion`, e.g. `MeSH 2026`) and the retrieval date on the `/source/mesh` page and in the API `sources` envelope.
23 +
24 +## What MeSH is for in CancerIndex
25 +
26 +MeSH is the indexing vocabulary of PubMed, **not** a disease ontology: it never creates canonical cancers (CLAUDE.md §340). The connector gives existing canonical cancers their MeSH descriptor UI so that
27 +
28 +1. the PubMed connector can build MeSH-anchored literature queries — `("Breast Neoplasms"[mh])` instead of title/abstract phrases — and
29 +2. publication → cancer edges resolve by UI (`resolver.byCode('mesh', ui)`) or descriptor name (`byLabel(name, {allowMeshInversion: true})`).
30 +
31 +## Verified endpoint behaviour (2026-09-08)
32 +
33 +- Descriptor model (probed on `mesh:D001943` "Breast Neoplasms"): `rdfs:label`, `meshv:identifier`, `meshv:treeNumber` → resource whose `rdfs:label` is the tree number (`C04.588.180`, `C17.800.090.500`), `meshv:preferredConcept` / `meshv:concept` → Concept (`meshv:scopeNote`, `meshv:term` / `meshv:preferredTerm`) → Term (`meshv:prefLabel`, `meshv:altLabel`), `meshv:dateIntroduced`, `meshv:broaderDescriptor`.
34 +- The default dataset spans every year graph (`<http://id.nlm.nih.gov/mesh/2015>` … `/2026`, 30,512 topical descriptors in 2026): a naive C04 count returns 8,989 rows. Queries are restricted with `FROM <http://id.nlm.nih.gov/mesh>` (current vocabulary) → **703 descriptors under C04**.
35 +- The endpoint answers SPARQL-XML as `text/plain` when the `Accept` header lists `application/json`, ignoring `format=JSON`; with `Accept: */*` it honours the parameter. The connector overrides the SDK default header per request.
36 +- One page (LIMIT 1000) of the grouped descriptor query returns in ~1.2 s (≈ 0.8 MB JSON).
37 +- Year graph detection: `ASK { GRAPH <…/mesh/2027> { … } }` then `/2026` → `datasetVersion = "MeSH 2026"`.
38 +- XML fallback: `desc2026.xml`, Last-Modified 2026-08-12, ~313 MB.
39 +
40 +## Scope
41 +
42 +Tree-number prefixes fetched (`TREE_PREFIXES` in `sparql.ts`):
43 +
44 +| Prefix | Content | Descriptors (MeSH 2026) |
45 +|---|---|---|
46 +| `C04` | Neoplasms — whole tree (by site, by histologic type, syndromes, precancerous conditions, experimental neoplasms) | 703 |
47 +| `C15.378.190.6*` | Bone Marrow Diseases → Myelodysplastic Syndromes (.625), Myeloproliferative Disorders (.636), MDS/MPN (.615) | 31 outside C04 |
48 +| `C20.683` | Immunoproliferative Disorders (lymphoproliferative disorders, paraproteinemias) | — |
49 +
50 +Total 734 distinct descriptors. The two extra subtrees contain non-neoplastic entries (Infectious Mononucleosis, Hypergammaglobulinemia, Anemia, Refractory…): those are mapped when the resolver knows them and otherwise **skipped silently** — only C04 misses enter the curation queue.
51 +
52 +## Sync design
53 +
54 +1. `detectYear` → `datasetVersion`.
55 +2. `fetchViaSparql`: paged grouped query (`descriptorsQuery(offset)`, LIMIT 1000): per descriptor the heading, all tree numbers, the entry terms of the **preferred concept** (`?terms`), the entry terms of the **other concepts** (`?related`), scope note, `dateIntroduced`. Rows failing the `Descriptor` schema are counted (`validationFailures`, `rejected`) and logged, never dropped silently. If the SPARQL route throws, `fetchViaXml` streams the descriptor XML, splits it on `</DescriptorRecord>` boundaries and parses each record with fast-xml-parser (`PreferredConceptYN` separates synonyms from other concepts; records outside the prefixes are discarded). `dry_run` stops after fetching.
56 +3. Anomaly guard: fewer than 500 descriptors → the run fails without writing (CLAUDE.md §171).
57 +4. One `provenance` row per run (dataset "MeSH descriptors — Neoplasms tree …", `evidenceType expert_curation`, `accessLevel open`).
58 +5. Per descriptor: `ctx.upsertSourceRecord('descriptor', <UI>, {ui, label, treeNumbers, terms, relatedTerms, scopeNote, dateIntroduced, meshYear, inC04})` (raw lake `data/raw/mesh/<date>/descriptor/`), then reconciliation (below), then `cancer_codes` / `cancer_aliases` writes.
59 +6. Cursor `{year, route, descriptors, syncedAt}`; the run is idempotent (`onConflictDoNothing` everywhere, source records `unchanged` on re-runs).
60 +
61 +### Reconciliation (CLAUDE.md §69, §221) — `mapDescriptor()`
62 +
63 +| Step | Input | Accepted | Written |
64 +|---|---|---|---|
65 +| 1 | heading, `resolver.byLabel(label, {allowMeshInversion: true})` ("Carcinoma, Non-Small-Cell Lung" → "Non-Small-Cell Lung Carcinoma") | `ONTOLOGY_EXACT` / `CURATED_EXACT` / `ALIAS` with confidence ≥ 0.9 | `cancer_codes (system 'mesh', code = UI, match_type = resolver type)` + heading alias + entry-term aliases |
66 +| 1b | heading → `CURATED_BROADER` (qualified state / lineage) | kept as fallback | `cancer_codes` with `match_type CURATED_BROADER`, **no aliases** (useful for PubMed queries at the broader level) |
67 +| 2 | each entry term **of the preferred concept**, same call | exact grades ≥ 0.9 | as step 1 but `match_type ALIAS` (confidence capped at 0.9) |
68 +| 3 | nothing | — | C04 descriptors → `unresolved_labels` (`entity_kind cancer`, context `{meshUi, treeNumbers, entryTerms, scopeNote}`, fuzzy suggestion from `resolver.suggest` for the curator); outside C04 → counted as skipped |
69 +
70 +Why preferred-concept terms only: MeSH lists the terms of *narrower and related concepts* as entry terms of a descriptor ("Adenocarcinoma, Basal Cell" and "Carcinoma, Tubular" under Adenocarcinoma, "Glioblastoma, Giant Cell" under Glioblastoma, "Breast Cancer" under Breast Neoplasms). Using them for mapping sent D000230 Adenocarcinoma to *Basal Cell Adenocarcinoma* and D005909 Glioblastoma to *Giant Cell Glioblastoma* in a first run. They are now stored in the raw record (`relatedTerms`) and never used for mapping or aliases.
71 +
72 +### Aliases
73 +
74 +- The heading is stored as `alias_type 'mesh_heading'`, `source_terminology 'MeSH'`. A dedicated type guarantees the row exists even when NCIt already has an identical `synonym` (unique key is cancer + normalized + type): the PubMed connector selects aliases whose `source_terminology` matches `/mesh/i` to build `"<heading>"[mh]` queries, and its `[tiab]` term ranking ignores unknown alias types, so `mesh_heading` never leaks into title/abstract queries.
75 +- Entry terms of the preferred concept are stored as `alias_type 'synonym'`, `source_terminology 'MeSH'`, after `aliasWorthyTerms()`: ≥ 3 characters, not the heading, not already present for that cancer under any type, and not a permutation of a kept term ("Neoplasm, Breast" / "Neoplasms, Breast" / "Breast Neoplasm" all normalize to the same un-inverted string as "Breast Neoplasms").
76 +- `CURATED_BROADER` mappings get the code only.
77 +
78 +## Observed run (cancerindex_c, 2026-09-08, NCIt 26.08e + OncoTree + CIViC/ClinVar/GDC loaded — 10,192 cancers)
79 +
80 +| | |
81 +|---|---|
82 +| Duration | 7 s (3 SPARQL requests: 2 year probes + 1 page) |
83 +| Descriptors fetched | **734** (703 in C04, 31 in the hematologic/immunoproliferative subtrees) — 0 malformed rows |
84 +| Mapped | **428** — ONTOLOGY_EXACT 219, ALIAS 202, CURATED_EXACT 3, CURATED_BROADER 4 |
85 +| C04 resolution | **419 / 703 (59.6 %)**; 10 cancers received two descriptors (e.g. two MeSH descriptors for one NCIt concept) |
86 +| Aliases written | 424 `mesh_heading` (414 distinct cancers now MeSH-anchorable for PubMed) + 860 `synonym` (source_terminology MeSH) |
87 +| Unresolved (C04, queued) | 284; outside-C04 skipped 22 |
88 +| Spot checks | D009369 Neoplasms → Neoplasm (ONTOLOGY_EXACT); D008175 Lung Neoplasms → Lung Neoplasm; D002289 Carcinoma, Non-Small-Cell Lung → Lung Non-Small Cell Carcinoma (ALIAS via inversion); D015464 → Chronic Myeloid Leukemia, BCR-ABL1 Positive; D001943 Breast Neoplasms → Breast Neoplasm (ALIAS via "Breast Tumor"); D005909 Glioblastoma → Glioblastoma; D018285 Klatskin Tumor → Hilar Cholangiocarcinoma; D001913 Bowen's Disease → Squamous Cell Carcinoma In Situ; D006394 Hemangiosarcoma → Angiosarcoma |
89 +
90 +Top 20 unresolved descriptors (alphabetical, each seen once): Abdominal Neoplasms, Aberrant Crypt Foci, ACTH Syndrome Ectopic, Adenocarcinoma in Situ, Adenocarcinoma Bronchiolo-Alveolar, Adenofibroma, Adenoma Acidophil, Adenoma Basophil, Adenoma Chromophobe, Adenomatosis Pulmonary, Adenomatous Polyposis Coli, Anal Gland Neoplasms, Anaplasia, Angiokeratoma, Anti-N-Methyl-D-Aspartate Receptor Encephalitis, Arachnoid Cysts, Atypical Squamous Cells of the Cervix, Avian Leukosis, Barrett Esophagus, Basal Cell Nevus Syndrome. Three groups: (a) non-neoplastic or veterinary descriptors filed under C04 (Anaplasia, Arachnoid Cysts, Avian Leukosis, paraneoplastic syndromes) — correctly unmapped; (b) heading strings that are **ambiguous in this development database** because OncoTree ran before NCIt and minted duplicates ("Adenocarcinoma, NOS" next to NCIt "Adenocarcinoma", "Sarcoma, NOS", "Teratoma", "Paraganglioma"): the resolver refuses ambiguous aliases, so Adenocarcinoma (D000230), Sarcoma (D012509 → resolved via entry term to Soft Tissue Sarcoma) and Teratoma stay open or land on a sibling; with NCIt loaded first the OncoTree nodes map onto NCIt concepts and these headings resolve exactly; (c) genuine vocabulary gaps for the curation queue (Adenoma, Acidophil; Anal Gland Neoplasms; Adenocarcinoma, Bronchiolo-Alveolar — the WHO-retired term).
91 +
92 +### Effect on PubMed
93 +
94 +Before this connector `literature_counts` queries were all `[tiab]`. After it, `pubmed` `loadTargets()` sees 424 cancers with MeSH names; see `docs/connectors/pubmed.md` and the run report in the integration notes for the `[mh]` share measured on the development database.
95 +
96 +## Limitations / notes
97 +
98 +- MeSH descriptors cover benign and malignant neoplasms alike; the descriptor is attached to whichever canonical concept the heading names (Breast Neoplasms → Breast Neoplasm, not Malignant Breast Neoplasm). Literature counts anchored on such headings therefore include benign-tumour literature — the query string is stored with every count (CLAUDE.md §251).
99 +- Tree numbers are stored in the source record only; a `hierarchy_type 'mesh'` dimension is a future addition.
100 +- Supplementary Concept Records (C-numbers used by PubMed for rare tumours) are not fetched.
101 +- The XML fallback has been exercised on a fixture, not on the 313 MB file; it exists so a SPARQL outage does not block the monthly run.
102 +- Alert bookkeeping in `scripts/ci.ts` warns when `system_alerts` is absent on a development database created before that table existed (`drizzle-kit push` fixes it locally).
added docs/connectors/openfda.md +87 −0
@@ -0,0 +1,87 @@
1 +# Connector `openfda` — Drugs@FDA applications and drug labels via openFDA
2 +
3 +| | |
4 +|---|---|
5 +| Source | https://open.fda.gov — U.S. Food and Drug Administration (openFDA), datasets **Drugs@FDA** (`/drug/drugsfda.json`) and **drug labeling / SPL** (`/drug/label.json`) |
6 +| Tier / category | 3 / regulatory (CLAUDE.md §13 — country-aware regulatory status, never a bare `approved = true`) |
7 +| Access | REST `https://api.fda.gov`, no authentication; optional `OPENFDA_API_KEY` (query parameter `api_key`) |
8 +| Docs verified | 2026-09-08 — https://open.fda.gov/apis/ (query syntax), https://open.fda.gov/apis/authentication/ (limits), https://open.fda.gov/apis/drug/drugsfda/, https://open.fda.gov/apis/drug/label/, live API |
9 +| License | **Public domain, CC0 1.0 Universal** (https://open.fda.gov/license/, https://open.fda.gov/terms/) — `licenseStatus: approved`, `commercialUse: allowed`, `redistribution: allowed` |
10 +| Code | `packages/connectors/src/connectors/openfda/` (`manifest.ts`, `normalize.ts`, `index.ts`, `openfda.test.ts`, `fixtures/`) |
11 +
12 +## Terms, license and disclaimer (quoted)
13 +
14 +Terms of service, "Data Rights and Usage" (https://open.fda.gov/terms/):
15 +
16 +> Unless otherwise noted, the content, data, documentation, code, and related materials on openFDA is public domain and made available with a Creative Commons CC0 1.0 Universal dedication. In short, FDA waives all rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law. You can copy, modify, distribute, and perform the work, even for commercial purposes, all without asking permission. FDA makes no warranties about the work, and disclaims liability for all uses of the work, to the fullest extent permitted by applicable law. Some data on openFDA may not be public domain, such as copies of copyrightable works made available to the FDA by private entities.
17 +
18 +License page (https://open.fda.gov/license/): same CC0 1.0 dedication, plus "When using or citing the work, you should not imply endorsement by the author or the affirmer." (The GMDN device-terminology restriction on that page does not apply to Drugs@FDA / SPL data.)
19 +
20 +Disclaimer — returned in `meta.disclaimer` of **every** API response and repeated on the Drugs@FDA endpoint page ("Responsible use of the data"); stored verbatim in `manifest.termsNotes`:
21 +
22 +> Do not rely on openFDA to make decisions regarding medical care. While we make every effort to ensure that data is accurate, you should assume all results are unvalidated. We may limit or otherwise restrict your access to the API in line with our Terms of Service.
23 +
24 +> Always speak to your health provider about the risks and benefits of FDA-regulated products.
25 +
26 +CancerIndex mirrors this: regulatory status is shown as a dated, sourced fact per application (evidence label *regulatory*), never as a treatment recommendation.
27 +
28 +Rate limits (https://open.fda.gov/apis/authentication/, verified 2026-09-08):
29 +
30 +> With no API key: 240 requests per minute, per IP address. 1,000 requests per day, per IP address. With an API key: 240 requests per minute, per key. 120,000 requests per day, per key.
31 +
32 +The connector runs at 3 req/s (180/min) sequentially. Without `OPENFDA_API_KEY` it stops itself after ~950 requests in a run (`OPENFDA_DAILY_BUDGET_NO_KEY`) and resumes from its cursor on the next run; a full pass over ~660 drugs needs roughly 900–1,100 requests, so a key (free, https://open.fda.gov/apis/authentication/) is recommended for the weekly schedule. The key is never written to provenance URLs (`applicationProvenanceUrl`).
33 +
34 +## Verified endpoints
35 +
36 +| Endpoint | Verified behaviour (2026-09-08) |
37 +|---|---|
38 +| `GET /drug/drugsfda.json?search=openfda.generic_name:"osimertinib"&limit=100&skip=0` | `results[]` = applications: `application_number` (NDA208065), `sponsor_name`, `openfda{generic_name[], brand_name[], manufacturer_name[], substance_name[], unii[], rxcui[], pharm_class_epc[], pharm_class_moa[]…}`, `products[]{product_number, brand_name, active_ingredients[{name,strength}], dosage_form, route, marketing_status, reference_drug}`, `submissions[]{submission_type ORIG\|SUPPL, submission_number, submission_status AP\|TA\|…, submission_status_date YYYYMMDD, submission_class_code (TYPE 1…, EFFICACY, LABELING, MANUF (CMC)…), review_priority PRIORITY\|STANDARD, submission_property_type[{code Orphan}], application_docs[{url,type Label\|Letter\|Review…}]}`; `meta{disclaimer, terms, license, last_updated "2026-09-04", results{skip,limit,total}}`. Osimertinib: 23 submissions = 1 ORIG (TYPE 1, AP 2015-11-13, PRIORITY), 8 SUPPL/EFFICACY, 13 SUPPL/LABELING, 1 SUPPL/MANUF (CMC). |
39 +| `GET /drug/drugsfda.json?search=openfda.brand_name:"tagrisso"+openfda.substance_name:"tagrisso"&limit=100` | OR search (space/`+` between terms) — fallback for single-token names. |
40 +| `GET /drug/label.json?search=openfda.application_number:"NDA208065"&limit=1` | Current SPL: `set_id`, `id`, `version` (36), `effective_time` (20240925), `indications_and_usage[]` (one text blob: highlights with one indication per `( 1.N )` reference, then the full 1.1…1.N subsections), `openfda.*`; `meta.last_updated 2026-09-07`. |
41 +| No match | HTTP **404** `{"error":{"code":"NOT_FOUND","message":"No matches found!"}}` — treated as an empty result (`isNoMatch`), not as a failure; other statuses go through the SDK retry/backoff. |
42 +| Phrase search noise | `generic_name:"trastuzumab"` also returns ADO-TRASTUZUMAB EMTANSINE (BLA125427), FAM-TRASTUZUMAB DERUXTECAN-NXKI (BLA761139), PERTUZUMAB, TRASTUZUMAB, AND HYALURONIDASE-ZZXF (BLA761170), five biosimilars (TRASTUZUMAB-ANNS…) and HERCEPTIN (BLA103792). `generic_name:"imatinib"` → 12 applications (NDA021588 Gleevec, NDA219097 Imkeldi, 10 ANDAs). `generic_name:"pembrolizumab"` → BLA125514 Keytruda + BLA761467 Keytruda Qlex (pembrolizumab + berahyaluronidase alfa). |
43 +
44 +Health check: `GET /drug/drugsfda.json?limit=1` → healthy when `meta.last_updated` is present.
45 +
46 +## Ingestion
47 +
48 +Scope (bounded, CLAUDE.md §227): every row of the local `drugs` table (CIViC therapies today, trial interventions later), ordered by id; `ctx.cursor.lastDrugId` is saved after each drug so a run stopped by the time budget or the daily quota resumes exactly where it stopped; a completed pass starts a new one on the next schedule.
49 +
50 +1. **Skip class / regimen names** (`ATR Inhibitor`, `Adjuvant Chemotherapy`, `Abemaciclib Regimen`…, regex `CLASS_NAME_RE` on multi-word names) — queued in `unresolved_labels` (`entity_kind drug`, `context.reason class_or_regimen_name`) without spending a request.
51 +2. **Search** `openfda.generic_name:"<name>"` (paged, ≤ 3 pages of 100); on no match, `brand_name OR substance_name` for single-token names, then ≤ 2 brand-like drug aliases via `openfda.brand_name`. Names with no accepted application → `unresolved_labels` (`reason no_drugsfda_match`, `{source:'openfda'}`).
52 +3. **Strict post-filter** (`classifyApplication`): an application is accepted only when its generic name / active ingredient is the molecule itself — exact, salt form (`IMATINIB MESYLATE`), biologic four-letter suffix (`TRASTUZUMAB-ANNS`) or FDA proper-name prefix (`FAM-TRASTUZUMAB DERUXTECAN` for "trastuzumab deruxtecan"). Fixed-dose combinations (several `substance_name`s, or "X, Y, and Z" generic names) and other molecules are `unmatched`. Kinds: `reference` (NDA/BLA of the molecule), `generic_or_biosimilar` (ANDA, or every matching name carries a biologic suffix). A drug named after an ADC (e.g. "Trastuzumab Emtansine") matches only under its own name.
53 +4. **Source records** `application` (payload = full application) for every accepted application, `label` (set_id) for approval-bearing ones; `drugs.unii` filled when empty; `openfda.brand_name[]` → `drug_aliases` (`alias_type brand`).
54 +5. **Approval rows** (`drug_approvals`, `jurisdiction US`, `authority FDA`, `status approved`) from the **reference** applications; when a molecule only exists as generics/biosimilars, the earliest-approved one bears the rows (others remain source records — ten identical ANDA rows would add no regulatory information). Events (`approvalEvents`): submissions with `submission_status = AP` and `submission_type = ORIG` → `approvalType 'ORIG'`; `SUPPL` with `submission_class_code = EFFICACY` → `approvalType 'SUPPL'`. Labeling / manufacturing supplements and tentative approvals (TA) are not approval events. `approvalDate = submission_status_date` (YYYYMMDD → YYYY-MM-DD). `applicationNumber` on every row; `raw` keeps submission number, review priority, class code, label/letter document URLs and the idempotency `key`.
55 + - **ORIG rows** carry the label's *Indications and Usage*. The highlights are split into one bullet per `( 1.N )` reference (`splitIndications`; fallbacks: `•` bullets, numbered subsections, whole text). Each bullet goes through the conservative dictionary pass (below): a bullet naming exactly one cancer becomes its own row (`cancerId`, `indication` = bullet text ≤ 4,000 chars, `raw.cancer_match = {matchType:'PROBABILISTIC', via:'label text dictionary', alias}`); the remaining bullets (0 or ≥ 2 cancers, tumor-agnostic) are joined into one row with `cancerId null` (`raw.cancer_match.matchType 'UNRESOLVED'`). When openFDA has no label for the application the ORIG row says so in `indication`.
56 + - **SUPPL rows**: `indication = "Efficacy supplement <date> (see label)"`, `cancerId null` (the supplement letter, not the current label, defines what changed — parsing it is out of scope).
57 + - **`accelerated`** stays `null` unless the bullet text says "accelerated approval" (then `true`). `review_priority = PRIORITY` is a *review track* (priority review), **not** accelerated approval (21 CFR 314 subpart H) and is recorded in `raw.reviewPriority` only. Withdrawn accelerated approvals are not detected (limitation).
58 + - **`tumorAgnostic`** is `true` only when a bullet says "solid tumors" together with a biomarker (NTRK, MSI-H/dMMR, TMB-H, BRAF V600E, RET fusion, HER2, KRAS G12C…); such bullets are never mapped to the generic "Solid Neoplasm" concept.
59 +6. **Provenance**: one row per application and run when anything changed (`sourceUrl https://api.fda.gov/drug/drugsfda.json?search=application_number:"…"`, `dataset 'Drugs@FDA via openFDA'`, `datasetVersion 'drugsfda-<meta.last_updated>'`, `evidenceType regulatory`, `accessLevel open`, `publishedAt` = first approval date, `updatedAtSource` = label `meta.last_updated`, methodology text). Unchanged application + unchanged label + all rows present → the previous provenance row is reused (no duplicate evidence per weekly run).
60 +7. **Knowledge edges** for rows with a `cancerId`: `drug —APPROVED_FOR→ cancer`, `evidenceCategory regulatory_status`, `evidenceLevel 'FDA ORIG' | 'FDA SUPPL'`, `direction supports`, `cancerContextIds [cancerId]`, `sourceRecordId = application_number`, `provenanceIds [provenance]` (unique per drug/cancer/application, refreshed not incremented).
61 +8. **Idempotency**: rows of an application are keyed by `raw.key` (`ORIG|<submission>|b<bullet>`, `ORIG|<submission>|rest`, `SUPPL|<submission>`); existing keys are updated, new keys inserted, keys that no longer derive from the freshly fetched payload are deleted **within that application only** (never a mass delete on a shrunken response — the payload was fetched successfully).
62 +
63 +`dry_run` probes osimertinib, pembrolizumab and imatinib and logs applications, kinds and event counts without writing.
64 +
65 +## Cancer dictionary (conservative, PROBABILISTIC)
66 +
67 +Built once per run from `cancer_aliases` of active, malignant concepts with `top_level = true` or `entity_type IN (cancer, cancer_family, hematologic_malignancy, subtype, histology, molecular_subtype)` — the spec's shorter list (cancer / cancer_family / hematologic_malignancy) was widened because NCIt models NSCLC, HNSCC, urothelial carcinoma, glioblastoma, colorectal cancer, CML (BCR-ABL1+) as `subtype` / `molecular_subtype`; without them most oncology labels would stay unmapped. Rules:
68 +
69 +- alias length ≥ 6 characters, or ≥ 4 for curated `abbreviation` aliases (NSCLC, HNSCC, GIST, DLBCL); 3-letter abbreviations (AML, CML, ALL) are excluded;
70 +- generic words are stop-listed (`cancer`, `tumor`, `carcinoma`, `solid tumor`, `leukemia`, `lymphoma`, `sarcoma`, `adenocarcinoma`, `squamous cell carcinoma`, `carcinoma in situ`, `metastatic disease`…, `GENERIC_ALIAS_STOPLIST`);
71 +- an alias shared by several concepts is kept only when `CancerResolver.byLabel` disambiguates it (preferred name, curated display name, broadest concept of one lineage — e.g. "breast cancer" → Malignant Breast Neoplasm); otherwise it is dropped (`ambiguousDropped`);
72 +- matching is a whole-word n-gram lookup over `normalizeLabel(bullet)`; overlapping mentions resolve to the **longest** span ("metastatic breast cancer" beats "breast cancer"), then the bullet is mapped only if **exactly one** distinct cancer remains. "Ph+ CML in blast crisis" → CML; "HES and/or CEL" (two cancers) → null; "melanoma or breast cancer" → null.
73 +
74 +Every mapped row is `PROBABILISTIC` (text mention, not an identifier) and says so in `raw.cancer_match`; curators can review `drug_approvals WHERE cancer_id IS NULL` (`raw.bullets[].distinctCancers`).
75 +
76 +## Observed run (cancerindex_b, 2026-09-08)
77 +
78 +See the run summary appended below after the first 15-minute run.
79 +
80 +## Limitations / notes
81 +
82 +- One pass costs ~1.5 requests per drug (+1 label per approval-bearing application); without an API key the 1,000/day IP quota is the binding constraint (self-stop at 950, cursor saved).
83 +- Efficacy supplements are dated but not mapped to cancers (no label diff); the current label's bullets are attached to the ORIG event date, so a bullet added by a 2024 supplement is stored under the original approval row set with the ORIG date — the supplement rows carry the actual dates. A future refinement can match `application_docs` label PDFs per supplement.
84 +- `accelerated` relies on label wording ("approved under accelerated approval"); conversions to regular approval and withdrawals are not tracked yet (Project Confirm — spec §13 FDA Oncology, later).
85 +- ANDA generics / biosimilars of a molecule that already has a reference NDA/BLA are kept as source records only.
86 +- Line of therapy, disease stage and biomarker ids (`lineOfTherapy`, `diseaseStage`, `biomarkerIds`) are not extracted (Phase 2).
87 +- Only US / FDA. Health Canada, EMA, MHRA, PMDA are separate connectors (spec §13).
added docs/schema-changes-ops.md +95 −0
@@ -0,0 +1,95 @@
1 +# Schema changes — ops layer (2026-09-08)
2 +
3 +Developed with `drizzle-kit push` on `cancerindex_a` (CLAUDE.md conventions: parallel agents do not
4 +commit migration files). **The integrator generates the migration** with `pnpm db:generate` after
5 +merging; the expected DDL is listed here so the generated file can be reviewed against it.
6 +
7 +## 1. New table `system_alerts` (CLAUDE.md §170)
8 +
9 +Schema: `packages/database/src/schema/ext-ops.ts` (exported from `schema/index.ts`). Helpers:
10 +`packages/database/src/alerts.ts` (`raiseAlert`, `raiseAlertSafe`, `resolveAlerts`, `listAlerts`).
11 +
12 +```sql
13 +CREATE TABLE "system_alerts" (
14 + "id" bigserial PRIMARY KEY NOT NULL,
15 + "kind" text NOT NULL, -- connector_failure | connector_aborted | anomaly | schema_drift | source_failing | source_stale | …
16 + "severity" text DEFAULT 'warn' NOT NULL, -- info | warn | critical
17 + "connector_id" text,
18 + "message" text NOT NULL,
19 + "detail" jsonb DEFAULT '{}'::jsonb NOT NULL,
20 + "first_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
21 + "last_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
22 + "count" integer DEFAULT 1 NOT NULL,
23 + "status" text DEFAULT 'open' NOT NULL, -- open | acknowledged | resolved
24 + "resolved_at" timestamp with time zone
25 +);
26 +CREATE INDEX "system_alerts_status_idx" ON "system_alerts" USING btree ("status","kind","connector_id");
27 +CREATE INDEX "system_alerts_seen_idx" ON "system_alerts" USING btree ("last_seen_at");
28 +```
29 +
30 +Deduplication (kind + connector_id + message among open/acknowledged rows) is done in `raiseAlert`,
31 +not by a unique index, so historical resolved rows with the same message can coexist.
32 +
33 +Writers: `runConnector` (failed / anomaly / aborted / schema drift, resolves on success), the worker
34 +health probe (`source_failing`, `source_stale`). Readers: `pnpm cix alerts`, `pnpm cix doctor`,
35 +`GET /v1/admin/alerts`. Manual triage: `pnpm cix alerts ack <id>` / `resolve <id>`.
36 +
37 +## 2. Manifest field `checkpointEvery`
38 +
39 +`ConnectorManifest.checkpointEvery` (zod, default 2000) — stored inside `sources.manifest` jsonb;
40 +no DDL. Drives the automatic mid-run cursor checkpoint (docs/connectors/README.md).
41 +
42 +## 3. Performance indexes (raw SQL, `packages/database/src/migrate.ts` → `createPerformanceIndexes`)
43 +
44 +Idempotent `CREATE INDEX IF NOT EXISTS`, executed after the SQL migrations by `pnpm db:migrate`
45 +(already the pattern for the trigram indexes). They are *not* part of the drizzle schema on purpose
46 +(GIN / opclass indexes are not expressible there), so `drizzle-kit generate` will not emit them; the
47 +post-migration block recreates them idempotently after every `pnpm db:migrate`, which also runs in the
48 +mld `post_sync` hook on deploy.
49 +
50 +```sql
51 +CREATE INDEX IF NOT EXISTS civic_evidence_gene_ids_gin ON civic_evidence_items USING gin (gene_ids);
52 +CREATE INDEX IF NOT EXISTS civic_evidence_variant_ids_gin ON civic_evidence_items USING gin (variant_ids);
53 +CREATE INDEX IF NOT EXISTS civic_evidence_therapy_ids_gin ON civic_evidence_items USING gin (therapy_ids);
54 +CREATE INDEX IF NOT EXISTS trial_conditions_cancer_trial_idx ON trial_conditions (cancer_id, trial_id);
55 +CREATE INDEX IF NOT EXISTS clinical_trials_status_type_idx ON clinical_trials (overall_status, study_type);
56 +CREATE INDEX IF NOT EXISTS cancer_aliases_norm_pattern_idx ON cancer_aliases (normalized text_pattern_ops);
57 +CREATE INDEX IF NOT EXISTS ingest_runs_connector_status_idx ON ingest_runs (connector_id, status, started_at DESC);
58 +DROP INDEX IF EXISTS clinical_trials_status_updated_idx; -- tried and rejected, see below
59 +```
60 +
61 +### Measurements (cancerindex_a, PostgreSQL 17.9, Apple Silicon; 9 629 cancers, 17 218 hierarchy edges,
62 +33 687 aliases, 45 045 genes, 126 107 trials, 278 132 trial_conditions, 11 968 CIViC items; EXPLAIN ANALYZE, warm cache)
63 +
64 +| Query shape | Before | After | Note |
65 +|---|---|---|---|
66 +| Gene-level counters: per-gene evidence counts over all genes (`counters.ts`), containment form + GIN | 42 557 ms (`@>` seq scan) / 25 963 ms (`= ANY`) | **55 ms** | `x = ANY(array_col)` cannot use GIN → counters rewritten to `gene_ids @> ARRAY[g.id]::text[]` (equivalent) |
67 +| `refreshCounters()` end to end (`pnpm cix counters`, 9 510 cancers + 725 genes, two runs each) | 96.0 s / 96.0 s | **5.3 s / 6.6 s** | three `= ANY` full scans over 45 045 genes removed; the remaining time is the recursive descendants + trial_map ≈ 2.6 s |
68 +| CIViC items for one gene (`gene_ids @> ARRAY[id]`) | 1.65 ms | 0.19 ms | API `genes.ts` still uses `= ANY` (1.9 ms, unchanged) — switch to `@>` when touching that route |
69 +| Global active-interventional count (`overall_status = ANY(...) AND study_type`) | 18.3 ms | 1.1 ms | index-only scan on `clinical_trials_status_type_idx` |
70 +| Alias lookup `cancers?q=` (exact OR prefix OR word LIKE) | 3.5 ms | 0.5 ms | BitmapOr over `_norm_idx` + `_norm_pattern_idx` + trigram |
71 +| Alias search (exact OR prefix OR `%` similarity) | 48.2 ms | 3.7 ms | prefix branch now indexable under en_US collation |
72 +| Recursive descendants of a top-level cancer (`descendantIds`) | 1.9 ms | 0.8 ms | unchanged plan (`cancer_hierarchy_uq` index-only); noise |
73 +| Counters `trial_map` (all cancers × descendants → trials) | 2 561 ms | 2 644 ms | dominated by the recursive CTE + DISTINCT; `trial_conditions_cancer_trial_idx` gives an index-only probe per descendant but no net change at this size |
74 +| CIViC evidence for a scope by status | 1.1 ms | 1.1 ms | `civic_evidence_cancer_idx` already sufficient |
75 +| Trials for a scope, `EXISTS` semi-join + status + `ORDER BY … LIMIT 20` | 2.1 ms | **116 ms with** `(overall_status, last_update_posted_date)` → index rejected; 2 ms without | ordered-index-walk trap for small scopes |
76 +
77 +Trial scope queries re-measured with a representative scope (Malignant Central Nervous System
78 +Neoplasm — the top-level cancer with the most mapped trial conditions in this database), without
79 +(`DROP INDEX` inside a rolled-back transaction) and with `trial_conditions_cancer_trial_idx`:
80 +
81 +| Query shape (scope = CNS, 192 descendants) | Without | With |
82 +|---|---|---|
83 +| Trials for the scope, `EXISTS` semi-join + `overall_status = 'RECRUITING'` + `ORDER BY last_update_posted_date LIMIT 20` (API `trials.ts`) | 35.2 ms | 21.5 ms |
84 +| Same, JOIN form with active-status list and `count(*) OVER()` (API `cancers.ts`) | 18.3 ms | 19.0 ms (noise) |
85 +| Status facet for the scope (web trials page) | 19.3 ms | 16.9 ms |
86 +
87 +The gain is modest at 278 k conditions (index-only probe instead of heap fetches per descendant);
88 +it grows with `trial_conditions` size and it removes the heap I/O on a cold cache.
89 +
90 +### Recommendations outside this change set (routes not touched here)
91 +
92 +- `apps/api/src/routes/genes.ts` / `drugs.ts`: replace `X = ANY(e.gene_ids|variant_ids|therapy_ids)`
93 + with `e.gene_ids @> ARRAY[X]::text[]` to use the new GIN indexes.
94 +- Keep `ORDER BY last_update_posted_date` lists without a status-ordered index; if large scopes get
95 + slow, paginate with a keyset on `(last_update_posted_date, id)` inside the semi-join instead.
added packages/connectors/src/connectors/cbioportal/cbioportal.test.ts +109 −0
@@ -0,0 +1,109 @@
1 +import { readFileSync } from 'node:fs';
2 +import path from 'node:path';
3 +import { describe, expect, it } from 'vitest';
4 +import { validateFrequency } from '../../sdk/validate.js';
5 +import { TOP_GENES_PER_STUDY, manifest, molecularProfilesUrl, mutatedGenesUrl, studiesUrl, studyUrl } from './manifest.js';
6 +import { CancerType, MolecularProfile, MutatedGene, SampleList, Study, findMutationProfile, importDateToRelease, mutatedGenesBody, oncotreeCode, programFromStudyName, rankMutatedGenes, typeLineage } from './normalize.js';
7 +
8 +const fx = (name: string) => JSON.parse(readFileSync(path.join(import.meta.dirname, 'fixtures', name), 'utf8')) as unknown;
9 +
10 +describe('cbioportal manifest', () => {
11 + it('is an attribution-required ODbL genomics source with verified docs', () => {
12 + expect(manifest.id).toBe('cbioportal');
13 + expect(manifest.category).toBe('genomics');
14 + expect(manifest.licenseStatus).toBe('approved');
15 + expect(manifest.redistribution).toBe('attribution');
16 + expect(manifest.license).toMatch(/ODC Open Database License/);
17 + expect(manifest.documentationVerifiedAt).toBe('2026-09-08');
18 + expect(manifest.rateLimits).toMatchObject({ requestsPerSecond: 3, maxConcurrency: 2 });
19 + expect(studiesUrl(0)).toBe('https://www.cbioportal.org/api/studies?pageSize=1000&pageNumber=0&projection=DETAILED');
20 + expect(molecularProfilesUrl('acc_tcga')).toBe('https://www.cbioportal.org/api/studies/acc_tcga/molecular-profiles');
21 + expect(mutatedGenesUrl()).toBe('https://www.cbioportal.org/api/mutated-genes/fetch');
22 + expect(studyUrl('msk_impact_2017')).toBe('https://www.cbioportal.org/study/summary?id=msk_impact_2017');
23 + expect(mutatedGenesBody('acc_tcga')).toEqual({ studyIds: ['acc_tcga'] });
24 + });
25 +});
26 +
27 +describe('studies and cancer types', () => {
28 + const studies = (fx('studies-detailed.json') as unknown[]).map((s) => Study.parse(s));
29 + const types = new Map((fx('cancer-types.json') as unknown[]).map((c) => CancerType.parse(c)).map((c) => [c.cancerTypeId, c]));
30 +
31 + it('parses the DETAILED studies page with sequenced sample counts and citations', () => {
32 + const msk = studies.find((s) => s.studyId === 'msk_impact_2017')!;
33 + expect(msk).toMatchObject({ cancerTypeId: 'mixed', pmid: '28481359', citation: 'Zehir et al. Nat Med 2017', allSampleCount: 10945, sequencedSampleCount: 10945 });
34 + expect(importDateToRelease(msk.importDate)).toBe('2026-07-28');
35 + expect(programFromStudyName(msk.name)).toBe('MSK');
36 + expect(programFromStudyName('Adrenocortical Carcinoma (TCGA, Firehose Legacy)')).toBe('TCGA');
37 + expect(programFromStudyName('Lung Adenocarcinoma (TCGA, PanCancer Atlas)')).toBe('TCGA');
38 + expect(programFromStudyName('Some cohort without source')).toBeNull();
39 + expect(programFromStudyName('Adenoid Cystic Carcinoma Project (J Clin Invest 2019)')).toBeNull(); // citation, not a source
40 + expect(importDateToRelease(undefined)).toBeNull();
41 + });
42 + it('maps cancerTypeId to an upper-case OncoTree code; mixed / other are never codes', () => {
43 + expect(oncotreeCode('acc')).toBe('ACC');
44 + expect(oncotreeCode('luad')).toBe('LUAD');
45 + expect(oncotreeCode('mixed')).toBeNull();
46 + expect(oncotreeCode('other')).toBeNull();
47 + expect(oncotreeCode('tissue')).toBeNull();
48 + });
49 + it('derives disease type and primary site by walking cancer-type parents to the tissue root', () => {
50 + expect(typeLineage('luad', types)).toMatchObject({ diseaseType: 'Lung Adenocarcinoma', primarySite: 'Lung', chain: ['luad', 'nsclc', 'lung'] });
51 + expect(typeLineage('acc', types)).toMatchObject({ diseaseType: 'Adrenocortical Carcinoma', primarySite: 'Adrenal Gland' });
52 + expect(typeLineage('mixed', types)).toMatchObject({ diseaseType: 'Mixed Cancer Types', primarySite: 'Other' });
53 + expect(typeLineage('nope', types)).toMatchObject({ diseaseType: null, primarySite: null, chain: ['nope'] });
54 + });
55 + it('rejects a malformed study and accepts an empty page', () => {
56 + expect(Study.safeParse({ name: 'x' }).success).toBe(false);
57 + expect(Study.safeParse({ studyId: 'x', cancerTypeId: 'acc', name: 'x', allSampleCount: -1 }).success).toBe(false);
58 + expect((fx('studies-detailed.json') as unknown[]).length).toBe(6);
59 + expect(([] as unknown[]).map((s) => Study.parse(s))).toEqual([]);
60 + });
61 +});
62 +
63 +describe('molecular profiles and sample lists', () => {
64 + it('finds the <studyId>_mutations profile', () => {
65 + const profiles = (fx('molecular-profiles-acc_tcga.json') as unknown[]).map((p) => MolecularProfile.parse(p));
66 + expect(profiles.length).toBeGreaterThan(3);
67 + expect(findMutationProfile(profiles, 'acc_tcga')?.molecularProfileId).toBe('acc_tcga_mutations');
68 + expect(findMutationProfile(profiles, 'acc_tcga')?.molecularAlterationType).toBe('MUTATION_EXTENDED');
69 + });
70 + it('a study without a mutation profile yields null (frequencies skipped)', () => {
71 + const profiles = (fx('molecular-profiles-no-mutations.json') as unknown[]).map((p) => MolecularProfile.parse(p));
72 + expect(profiles.every((p) => p.molecularAlterationType !== 'MUTATION_EXTENDED')).toBe(true);
73 + expect(findMutationProfile(profiles, 'angs_project_painter_2018')).toBeNull();
74 + });
75 + it('the _sequenced sample list carries the denominator', () => {
76 + const list = SampleList.parse(fx('sample-list-acc_tcga_sequenced.json'));
77 + expect(list).toMatchObject({ sampleListId: 'acc_tcga_sequenced', category: 'all_cases_with_mutation_data', sampleCount: 90 });
78 + const acc = (fx('studies-detailed.json') as unknown[]).map((s) => Study.parse(s)).find((s) => s.studyId === 'acc_tcga')!;
79 + expect(acc.sequencedSampleCount).toBe(list.sampleCount); // study.sequencedSampleCount == sample list count
80 + });
81 +});
82 +
83 +describe('mutated genes → frequencies with denominators', () => {
84 + it('ranks acc_tcga genes by altered samples with the WES denominator', () => {
85 + const genes = (fx('mutated-genes-acc_tcga.json') as unknown[]).map((g) => MutatedGene.parse(g));
86 + const ranked = rankMutatedGenes(genes, TOP_GENES_PER_STUDY);
87 + expect(ranked[0]).toMatchObject({ hugoGeneSymbol: 'ZFPM1', numberOfAlteredCases: 47, numberOfProfiledCases: 90, rank: 1 });
88 + expect(ranked.every((g, i) => i === 0 || g.numberOfAlteredCases <= ranked[i - 1]!.numberOfAlteredCases)).toBe(true);
89 + expect(ranked.every((g) => validateFrequency({ casesAffected: g.numberOfAlteredCases, casesProfiled: g.numberOfProfiledCases }).length === 0)).toBe(true);
90 + expect(ranked[0]!.numberOfAlteredCases / ranked[0]!.numberOfProfiledCases).toBeCloseTo(0.522, 3);
91 + });
92 + it('MSK-IMPACT: panel-aware denominators (IMPACT341 vs IMPACT410 genes) and zero-profiled genes dropped', () => {
93 + const genes = (fx('mutated-genes-msk_impact_2017.json') as unknown[]).map((g) => MutatedGene.parse(g));
94 + expect(genes.some((g) => g.numberOfProfiledCases === 0)).toBe(true);
95 + const ranked = rankMutatedGenes(genes, 5);
96 + expect(ranked).toHaveLength(5);
97 + expect(ranked.map((g) => g.hugoGeneSymbol)).toEqual(['TP53', 'KRAS', 'TERT', 'PIK3CA', 'APC']);
98 + expect(ranked[0]).toMatchObject({ numberOfAlteredCases: 4538, numberOfProfiledCases: 10945 });
99 + const all = rankMutatedGenes(genes, TOP_GENES_PER_STUDY);
100 + expect(all.some((g) => g.numberOfProfiledCases === 8136)).toBe(true); // gene only on the 410-gene panel
101 + expect(all.every((g) => g.numberOfProfiledCases > 0)).toBe(true);
102 + });
103 + it('validateFrequency rejects numerator > denominator and malformed gene rows are refused', () => {
104 + expect(validateFrequency({ casesAffected: 10, casesProfiled: 5 })).toHaveLength(1);
105 + expect(MutatedGene.safeParse({ hugoGeneSymbol: 'TP53', numberOfAlteredCases: 1 }).success).toBe(false);
106 + expect(MutatedGene.safeParse({ numberOfAlteredCases: 1, numberOfProfiledCases: 2 }).success).toBe(false);
107 + expect(rankMutatedGenes([], 10)).toEqual([]);
108 + });
109 +});
added packages/connectors/src/connectors/cbioportal/fixtures/cancer-types.json +93 −0
@@ -0,0 +1,93 @@
1 +[
2 + {
3 + "cancerTypeId": "acc",
4 + "name": "Adrenocortical Carcinoma",
5 + "dedicatedColor": "Purple",
6 + "shortName": "ACC",
7 + "parent": "adrenal_gland"
8 + },
9 + {
10 + "cancerTypeId": "adrenal_gland",
11 + "name": "Adrenal Gland",
12 + "dedicatedColor": "Purple",
13 + "shortName": "ADRENAL_GLAND",
14 + "parent": "tissue"
15 + },
16 + {
17 + "cancerTypeId": "angs",
18 + "name": "Angiosarcoma",
19 + "dedicatedColor": "LightYellow",
20 + "shortName": "ANGS",
21 + "parent": "soft_tissue"
22 + },
23 + {
24 + "cancerTypeId": "brca",
25 + "name": "Invasive Breast Carcinoma",
26 + "dedicatedColor": "HotPink",
27 + "shortName": "BRCA",
28 + "parent": "breast"
29 + },
30 + {
31 + "cancerTypeId": "breast",
32 + "name": "Breast",
33 + "dedicatedColor": "HotPink",
34 + "shortName": "BREAST",
35 + "parent": "tissue"
36 + },
37 + {
38 + "cancerTypeId": "luad",
39 + "name": "Lung Adenocarcinoma",
40 + "dedicatedColor": "Gainsboro",
41 + "shortName": "LUAD",
42 + "parent": "nsclc"
43 + },
44 + {
45 + "cancerTypeId": "lung",
46 + "name": "Lung",
47 + "dedicatedColor": "Gainsboro",
48 + "shortName": "LUNG",
49 + "parent": "tissue"
50 + },
51 + {
52 + "cancerTypeId": "mixed",
53 + "name": "Mixed Cancer Types",
54 + "dedicatedColor": "Black",
55 + "shortName": "MIXED",
56 + "parent": "other"
57 + },
58 + {
59 + "cancerTypeId": "nsclc",
60 + "name": "Non-Small Cell Lung Cancer",
61 + "dedicatedColor": "Gainsboro",
62 + "shortName": "NSCLC",
63 + "parent": "lung"
64 + },
65 + {
66 + "cancerTypeId": "other",
67 + "name": "Other",
68 + "dedicatedColor": "Black",
69 + "shortName": "OTHER",
70 + "parent": "tissue"
71 + },
72 + {
73 + "cancerTypeId": "prad",
74 + "name": "Prostate Adenocarcinoma",
75 + "dedicatedColor": "Cyan",
76 + "shortName": "PRAD",
77 + "parent": "prostate"
78 + },
79 + {
80 + "cancerTypeId": "prostate",
81 + "name": "Prostate",
82 + "dedicatedColor": "Cyan",
83 + "shortName": "PROSTATE",
84 + "parent": "tissue"
85 + },
86 + {
87 + "cancerTypeId": "soft_tissue",
88 + "name": "Soft Tissue",
89 + "dedicatedColor": "LightYellow",
90 + "shortName": "SOFT_TISSUE",
91 + "parent": "tissue"
92 + }
93 +]
\ No newline at end of file
added packages/connectors/src/connectors/cbioportal/fixtures/molecular-profiles-acc_tcga.json +1 −0
@@ -0,0 +1 @@
1 +[{"molecularProfileId":"acc_tcga_gistic","studyId":"acc_tcga","molecularAlterationType":"COPY_NUMBER_ALTERATION","datatype":"DISCRETE","name":"Putative copy-number alterations from GISTIC","description":"Putative copy-number calls on 90 cases determined using GISTIC 2.0. Values: -2 = homozygous deletion; -1 = hemizygous deletion; 0 = neutral / no change; 1 = gain; 2 = high level amplification.","showProfileInAnalysisTab":true,"patientLevel":false},{"molecularProfileId":"acc_tcga_linear_CNA","studyId":"acc_tcga","molecularAlterationType":"COPY_NUMBER_ALTERATION","datatype":"CONTINUOUS","name":"Capped relative linear copy-number values","description":"Capped relative linear copy-number values for each gene (from Affymetrix SNP6).","showProfileInAnalysisTab":false,"patientLevel":false},{"molecularProfileId":"acc_tcga_methylation_hm450","studyId":"acc_tcga","molecularAlterationType":"METHYLATION","datatype":"CONTINUOUS","name":"Methylation (HM450)","description":"Methylation (HM450) beta-values for genes in 80 cases. For genes with multiple methylation probes, the probe most anti-correlated with expression.","showProfileInAnalysisTab":false,"patientLevel":false},{"molecularProfileId":"acc_tcga_mutations","studyId":"acc_tcga","molecularAlterationType":"MUTATION_EXTENDED","datatype":"MAF","name":"Mutations","description":"Mutation data from whole exome sequencing. Mutation packager: ACC/20160128/gdac.broadinstitute.org_ACC.Mutation_Packager_Calls.Level_3.2016012800.0.0.tar.gz.","showProfileInAnalysisTab":true,"patientLevel":false},{"molecularProfileId":"acc_tcga_rna_seq_v2_mrna","studyId":"acc_tcga","molecularAlterationType":"MRNA_EXPRESSION","datatype":"CONTINUOUS","name":"mRNA expression (RNA Seq V2 RSEM)","description":"mRNA gene expression (RNA Seq V2 RSEM)","showProfileInAnalysisTab":false,"patientLevel":false},{"molecularProfileId":"acc_tcga_rna_seq_v2_mrna_median_Zscores","studyId":"acc_tcga","molecularAlterationType":"MRNA_EXPRESSION","datatype":"Z-SCORE","name":"mRNA expression z-scores relative to diploid samples (RNA Seq V2 RSEM)","description":"mRNA expression z-scores (RNA Seq V2 RSEM) compared to the expression distribution of each gene tumors that are diploid for this gene.","showProfileInAnalysisTab":true,"patientLevel":false},{"molecularProfileId":"acc_tcga_rna_seq_v2_mrna_median_all_sample_Zscores","studyId":"acc_tcga","molecularAlterationType":"MRNA_EXPRESSION","datatype":"Z-SCORE","name":"mRNA expression z-scores relative to all samples (log RNA Seq V2 RSEM)","description":"Log-transformed mRNA expression z-scores compared to the expression distribution of all samples (RNA Seq V2 RSEM).","showProfileInAnalysisTab":true,"patientLevel":false},{"molecularProfileId":"acc_tcga_rppa","studyId":"acc_tcga","molecularAlterationType":"PROTEIN_LEVEL","datatype":"LOG2-VALUE","name":"Protein expression (RPPA)","description":"Protein expression measured by reverse-phase protein array","showProfileInAnalysisTab":false,"patientLevel":false},{"molecularProfileId":"acc_tcga_rppa_Zscores","studyId":"acc_tcga","molecularAlterationType":"PROTEIN_LEVEL","datatype":"Z-SCORE","name":"Protein expression z-scores (RPPA)","description":"Protein expression, measured by reverse-phase protein array, z-scores","showProfileInAnalysisTab":true,"patientLevel":false}]
\ No newline at end of file
added packages/connectors/src/connectors/cbioportal/fixtures/molecular-profiles-no-mutations.json +12 −0
@@ -0,0 +1,12 @@
1 +[
2 + {
3 + "molecularProfileId": "angs_project_painter_2018_gistic",
4 + "studyId": "angs_project_painter_2018",
5 + "molecularAlterationType": "COPY_NUMBER_ALTERATION",
6 + "datatype": "DISCRETE",
7 + "name": "Putative copy-number alterations from GISTIC",
8 + "description": "Putative copy-number from GISTIC 2.0. Values: -2 = homozygous deletion; -1 = hemizygous deletion; 0 = neutral / no change; 1 = gain; 2 = high level amplification.",
9 + "showProfileInAnalysisTab": true,
10 + "patientLevel": false
11 + }
12 +]
\ No newline at end of file
added packages/connectors/src/connectors/cbioportal/fixtures/mutated-genes-acc_tcga.json +229 −0
@@ -0,0 +1,229 @@
1 +[
2 + {
3 + "numberOfAlteredCases": 47,
4 + "numberOfAlteredCasesOnPanel": 47,
5 + "totalCount": 112,
6 + "numberOfProfiledCases": 90,
7 + "matchingGenePanelIds": [
8 + "WES"
9 + ],
10 + "entrezGeneId": 161882,
11 + "hugoGeneSymbol": "ZFPM1",
12 + "qValue": 2.399999957884008e-13,
13 + "entrezGeneIds": [
14 + 161882
15 + ],
16 + "hugoGeneSymbols": [
17 + "ZFPM1"
18 + ],
19 + "uniqueEventKey": "ZFPM1"
20 + },
21 + {
22 + "numberOfAlteredCases": 35,
23 + "numberOfAlteredCasesOnPanel": 35,
24 + "totalCount": 50,
25 + "numberOfProfiledCases": 90,
26 + "matchingGenePanelIds": [
27 + "WES"
28 + ],
29 + "entrezGeneId": 727897,
30 + "hugoGeneSymbol": "MUC5B",
31 + "qValue": 2.399999957884008e-13,
32 + "entrezGeneIds": [
33 + 727897
34 + ],
35 + "hugoGeneSymbols": [
36 + "MUC5B"
37 + ],
38 + "uniqueEventKey": "MUC5B"
39 + },
40 + {
41 + "numberOfAlteredCases": 34,
42 + "numberOfAlteredCasesOnPanel": 34,
43 + "totalCount": 35,
44 + "numberOfProfiledCases": 90,
45 + "matchingGenePanelIds": [
46 + "WES"
47 + ],
48 + "entrezGeneId": 2617,
49 + "hugoGeneSymbol": "GARS1",
50 + "qValue": 2.399999957884008e-13,
51 + "entrezGeneIds": [
52 + 2617
53 + ],
54 + "hugoGeneSymbols": [
55 + "GARS1"
56 + ],
57 + "uniqueEventKey": "GARS1"
58 + },
59 + {
60 + "numberOfAlteredCases": 33,
61 + "numberOfAlteredCasesOnPanel": 33,
62 + "totalCount": 34,
63 + "numberOfProfiledCases": 90,
64 + "matchingGenePanelIds": [
65 + "WES"
66 + ],
67 + "entrezGeneId": 340385,
68 + "hugoGeneSymbol": "ZNF517",
69 + "qValue": 2.399999957884008e-13,
70 + "entrezGeneIds": [
71 + 340385
72 + ],
73 + "hugoGeneSymbols": [
74 + "ZNF517"
75 + ],
76 + "uniqueEventKey": "ZNF517"
77 + },
78 + {
79 + "numberOfAlteredCases": 31,
80 + "numberOfAlteredCasesOnPanel": 31,
81 + "totalCount": 54,
82 + "numberOfProfiledCases": 90,
83 + "matchingGenePanelIds": [
84 + "WES"
85 + ],
86 + "entrezGeneId": 26018,
87 + "hugoGeneSymbol": "LRIG1",
88 + "qValue": 2.399999957884008e-13,
89 + "entrezGeneIds": [
90 + 26018
91 + ],
92 + "hugoGeneSymbols": [
93 + "LRIG1"
94 + ],
95 + "uniqueEventKey": "LRIG1"
96 + },
97 + {
98 + "numberOfAlteredCases": 28,
99 + "numberOfAlteredCasesOnPanel": 28,
100 + "totalCount": 33,
101 + "numberOfProfiledCases": 90,
102 + "matchingGenePanelIds": [
103 + "WES"
104 + ],
105 + "entrezGeneId": 4583,
106 + "hugoGeneSymbol": "MUC2",
107 + "qValue": 2.399999957884008e-13,
108 + "entrezGeneIds": [
109 + 4583
110 + ],
111 + "hugoGeneSymbols": [
112 + "MUC2"
113 + ],
114 + "uniqueEventKey": "MUC2"
115 + },
116 + {
117 + "numberOfAlteredCases": 27,
118 + "numberOfAlteredCasesOnPanel": 27,
119 + "totalCount": 27,
120 + "numberOfProfiledCases": 90,
121 + "matchingGenePanelIds": [
122 + "WES"
123 + ],
124 + "entrezGeneId": 114294,
125 + "hugoGeneSymbol": "LACTB",
126 + "qValue": 2.399999957884008e-13,
127 + "entrezGeneIds": [
128 + 114294
129 + ],
130 + "hugoGeneSymbols": [
131 + "LACTB"
132 + ],
133 + "uniqueEventKey": "LACTB"
134 + },
135 + {
136 + "numberOfAlteredCases": 27,
137 + "numberOfAlteredCasesOnPanel": 27,
138 + "totalCount": 32,
139 + "numberOfProfiledCases": 90,
140 + "matchingGenePanelIds": [
141 + "WES"
142 + ],
143 + "entrezGeneId": 84033,
144 + "hugoGeneSymbol": "OBSCN",
145 + "qValue": 2.399999957884008e-13,
146 + "entrezGeneIds": [
147 + 84033
148 + ],
149 + "hugoGeneSymbols": [
150 + "OBSCN"
151 + ],
152 + "uniqueEventKey": "OBSCN"
153 + },
154 + {
155 + "numberOfAlteredCases": 26,
156 + "numberOfAlteredCasesOnPanel": 26,
157 + "totalCount": 26,
158 + "numberOfProfiledCases": 90,
159 + "matchingGenePanelIds": [
160 + "WES"
161 + ],
162 + "entrezGeneId": 92922,
163 + "hugoGeneSymbol": "CCDC102A",
164 + "qValue": 2.399999957884008e-13,
165 + "entrezGeneIds": [
166 + 92922
167 + ],
168 + "hugoGeneSymbols": [
169 + "CCDC102A"
170 + ],
171 + "uniqueEventKey": "CCDC102A"
172 + },
173 + {
174 + "numberOfAlteredCases": 26,
175 + "numberOfAlteredCasesOnPanel": 26,
176 + "totalCount": 26,
177 + "numberOfProfiledCases": 90,
178 + "matchingGenePanelIds": [
179 + "WES"
180 + ],
181 + "entrezGeneId": 4985,
182 + "hugoGeneSymbol": "OPRD1",
183 + "qValue": 2.399999957884008e-13,
184 + "entrezGeneIds": [
185 + 4985
186 + ],
187 + "hugoGeneSymbols": [
188 + "OPRD1"
189 + ],
190 + "uniqueEventKey": "OPRD1"
191 + },
192 + {
193 + "numberOfAlteredCases": 26,
194 + "numberOfAlteredCasesOnPanel": 26,
195 + "totalCount": 26,
196 + "numberOfProfiledCases": 90,
197 + "matchingGenePanelIds": [
198 + "WES"
199 + ],
200 + "entrezGeneId": 134548,
201 + "hugoGeneSymbol": "SOWAHA",
202 + "entrezGeneIds": [
203 + 134548
204 + ],
205 + "hugoGeneSymbols": [
206 + "SOWAHA"
207 + ],
208 + "uniqueEventKey": "SOWAHA"
209 + },
210 + {
211 + "numberOfAlteredCases": 25,
212 + "numberOfAlteredCasesOnPanel": 25,
213 + "totalCount": 25,
214 + "numberOfProfiledCases": 90,
215 + "matchingGenePanelIds": [
216 + "WES"
217 + ],
218 + "entrezGeneId": 64222,
219 + "hugoGeneSymbol": "TOR3A",
220 + "qValue": 2.399999957884008e-13,
221 + "entrezGeneIds": [
222 + 64222
223 + ],
224 + "hugoGeneSymbols": [
225 + "TOR3A"
226 + ],
227 + "uniqueEventKey": "TOR3A"
228 + }
229 +]
\ No newline at end of file
added packages/connectors/src/connectors/cbioportal/fixtures/mutated-genes-msk_impact_2017.json +206 −0
@@ -0,0 +1,206 @@
1 +[
2 + {
3 + "numberOfAlteredCases": 4538,
4 + "numberOfAlteredCasesOnPanel": 4538,
5 + "totalCount": 4954,
6 + "numberOfProfiledCases": 10945,
7 + "matchingGenePanelIds": [
8 + "IMPACT341",
9 + "IMPACT410"
10 + ],
11 + "entrezGeneId": 7157,
12 + "hugoGeneSymbol": "TP53",
13 + "entrezGeneIds": [
14 + 7157
15 + ],
16 + "hugoGeneSymbols": [
17 + "TP53"
18 + ],
19 + "uniqueEventKey": "TP53"
20 + },
21 + {
22 + "numberOfAlteredCases": 1643,
23 + "numberOfAlteredCasesOnPanel": 1643,
24 + "totalCount": 1670,
25 + "numberOfProfiledCases": 10945,
26 + "matchingGenePanelIds": [
27 + "IMPACT341",
28 + "IMPACT410"
29 + ],
30 + "entrezGeneId": 3845,
31 + "hugoGeneSymbol": "KRAS",
32 + "entrezGeneIds": [
33 + 3845
34 + ],
35 + "hugoGeneSymbols": [
36 + "KRAS"
37 + ],
38 + "uniqueEventKey": "KRAS"
39 + },
40 + {
41 + "numberOfAlteredCases": 1460,
42 + "numberOfAlteredCasesOnPanel": 1460,
43 + "totalCount": 1549,
44 + "numberOfProfiledCases": 10945,
45 + "matchingGenePanelIds": [
46 + "IMPACT341",
47 + "IMPACT410"
48 + ],
49 + "entrezGeneId": 7015,
50 + "hugoGeneSymbol": "TERT",
51 + "entrezGeneIds": [
52 + 7015
53 + ],
54 + "hugoGeneSymbols": [
55 + "TERT"
56 + ],
57 + "uniqueEventKey": "TERT"
58 + },
59 + {
60 + "numberOfAlteredCases": 1355,
61 + "numberOfAlteredCasesOnPanel": 1355,
62 + "totalCount": 1517,
63 + "numberOfProfiledCases": 10945,
64 + "matchingGenePanelIds": [
65 + "IMPACT341",
66 + "IMPACT410"
67 + ],
68 + "entrezGeneId": 5290,
69 + "hugoGeneSymbol": "PIK3CA",
70 + "entrezGeneIds": [
71 + 5290
72 + ],
73 + "hugoGeneSymbols": [
74 + "PIK3CA"
75 + ],
76 + "uniqueEventKey": "PIK3CA"
77 + },
78 + {
79 + "numberOfAlteredCases": 1121,
80 + "numberOfAlteredCasesOnPanel": 1121,
81 + "totalCount": 1674,
82 + "numberOfProfiledCases": 10945,
83 + "matchingGenePanelIds": [
84 + "IMPACT341",
85 + "IMPACT410"
86 + ],
87 + "entrezGeneId": 324,
88 + "hugoGeneSymbol": "APC",
89 + "entrezGeneIds": [
90 + 324
91 + ],
92 + "hugoGeneSymbols": [
93 + "APC"
94 + ],
95 + "uniqueEventKey": "APC"
96 + },
97 + {
98 + "numberOfAlteredCases": 875,
99 + "numberOfAlteredCasesOnPanel": 875,
100 + "totalCount": 1072,
101 + "numberOfProfiledCases": 10945,
102 + "matchingGenePanelIds": [
103 + "IMPACT341",
104 + "IMPACT410"
105 + ],
106 + "entrezGeneId": 8289,
107 + "hugoGeneSymbol": "ARID1A",
108 + "entrezGeneIds": [
109 + 8289
110 + ],
111 + "hugoGeneSymbols": [
112 + "ARID1A"
113 + ],
114 + "uniqueEventKey": "ARID1A"
115 + },
116 + {
117 + "numberOfAlteredCases": 851,
118 + "numberOfAlteredCasesOnPanel": 851,
119 + "totalCount": 1210,
120 + "numberOfProfiledCases": 10945,
121 + "matchingGenePanelIds": [
122 + "IMPACT341",
123 + "IMPACT410"
124 + ],
125 + "entrezGeneId": 8085,
126 + "hugoGeneSymbol": "KMT2D",
127 + "entrezGeneIds": [
128 + 8085
129 + ],
130 + "hugoGeneSymbols": [
131 + "KMT2D"
132 + ],
133 + "uniqueEventKey": "KMT2D"
134 + },
135 + {
136 + "numberOfAlteredCases": 665,
137 + "numberOfAlteredCasesOnPanel": 665,
138 + "totalCount": 764,
139 + "numberOfProfiledCases": 10945,
140 + "matchingGenePanelIds": [
141 + "IMPACT341",
142 + "IMPACT410"
143 + ],
144 + "entrezGeneId": 5728,
145 + "hugoGeneSymbol": "PTEN",
146 + "entrezGeneIds": [
147 + 5728
148 + ],
149 + "hugoGeneSymbols": [
150 + "PTEN"
151 + ],
152 + "uniqueEventKey": "PTEN"
153 + },
154 + {
155 + "numberOfAlteredCases": 359,
156 + "numberOfAlteredCasesOnPanel": 356,
157 + "totalCount": 466,
158 + "numberOfProfiledCases": 8136,
159 + "matchingGenePanelIds": [
160 + "IMPACT410"
161 + ],
162 + "entrezGeneId": 463,
163 + "hugoGeneSymbol": "ZFHX3",
164 + "entrezGeneIds": [
165 + 463
166 + ],
167 + "hugoGeneSymbols": [
168 + "ZFHX3"
169 + ],
170 + "uniqueEventKey": "ZFHX3"
171 + },
172 + {
173 + "numberOfAlteredCases": 292,
174 + "numberOfAlteredCasesOnPanel": 291,
175 + "totalCount": 362,
176 + "numberOfProfiledCases": 8136,
177 + "matchingGenePanelIds": [
178 + "IMPACT410"
179 + ],
180 + "entrezGeneId": 23269,
181 + "hugoGeneSymbol": "MGA",
182 + "entrezGeneIds": [
183 + 23269
184 + ],
185 + "hugoGeneSymbols": [
186 + "MGA"
187 + ],
188 + "uniqueEventKey": "MGA"
189 + },
190 + {
191 + "numberOfAlteredCases": 14,
192 + "numberOfAlteredCasesOnPanel": 0,
193 + "totalCount": 14,
194 + "numberOfProfiledCases": 0,
195 + "matchingGenePanelIds": [],
196 + "entrezGeneId": 81608,
197 + "hugoGeneSymbol": "FIP1L1",
198 + "entrezGeneIds": [
199 + 81608
200 + ],
201 + "hugoGeneSymbols": [
202 + "FIP1L1"
203 + ],
204 + "uniqueEventKey": "FIP1L1"
205 + }
206 +]
\ No newline at end of file
added packages/connectors/src/connectors/cbioportal/fixtures/sample-list-acc_tcga_sequenced.json +13 −0
@@ -0,0 +1,13 @@
1 +{
2 + "sampleListId": "acc_tcga_sequenced",
3 + "studyId": "acc_tcga",
4 + "category": "all_cases_with_mutation_data",
5 + "name": "Samples with mutation data",
6 + "description": "Samples with mutation data (90 samples)",
7 + "sampleCount": 90,
8 + "sampleIds": [
9 + "TCGA-OR-A5J1-01",
10 + "TCGA-OR-A5J2-01",
11 + "TCGA-OR-A5J3-01"
12 + ]
13 +}
\ No newline at end of file
added packages/connectors/src/connectors/cbioportal/fixtures/studies-detailed.json +255 −0
@@ -0,0 +1,255 @@
1 +[
2 + {
3 + "studyId": "prad_broad_2013",
4 + "cancerTypeId": "prad",
5 + "name": "Prostate Adenocarcinoma (Broad/Cornell, Cell 2013)",
6 + "description": "Whole genome sequencing of 57 prostate tumor and matches normal tissues.",
7 + "publicStudy": true,
8 + "pmid": "23622249",
9 + "citation": "Baca et al. Cell 2013",
10 + "groups": "PUBLIC",
11 + "status": 0,
12 + "importDate": "2026-01-13 09:51:16",
13 + "allSampleCount": 82,
14 + "sequencedSampleCount": 57,
15 + "cnaSampleCount": 56,
16 + "mrnaRnaSeqSampleCount": 0,
17 + "mrnaRnaSeqV2SampleCount": 0,
18 + "mrnaMicroarraySampleCount": 0,
19 + "miRnaSampleCount": 0,
20 + "methylationHm27SampleCount": 0,
21 + "rppaSampleCount": 0,
22 + "massSpectrometrySampleCount": 0,
23 + "completeSampleCount": 0,
24 + "referenceGenome": "hg19",
25 + "treatmentCount": 0,
26 + "structuralVariantCount": 28,
27 + "cancerType": {
28 + "id": "prad",
29 + "name": "Prostate Adenocarcinoma",
30 + "dedicatedColor": "Cyan",
31 + "shortName": "PRAD",
32 + "parent": "prostate"
33 + },
34 + "readPermission": true,
35 + "resourceCounts": []
36 + },
37 + {
38 + "studyId": "acc_tcga",
39 + "cancerTypeId": "acc",
40 + "name": "Adrenocortical Carcinoma (TCGA, Firehose Legacy)",
41 + "description": "TCGA Adrenocortical Carcinoma. Source data from <A HREF=\"http://gdac.broadinstitute.org/runs/stddata__2016_01_28/data/AC",
42 + "publicStudy": true,
43 + "groups": "PUBLIC",
44 + "status": 0,
45 + "importDate": "2026-01-05 12:46:31",
46 + "allSampleCount": 92,
47 + "sequencedSampleCount": 90,
48 + "cnaSampleCount": 90,
49 + "mrnaRnaSeqSampleCount": 0,
50 + "mrnaRnaSeqV2SampleCount": 79,
51 + "mrnaMicroarraySampleCount": 0,
52 + "miRnaSampleCount": 0,
53 + "methylationHm27SampleCount": 0,
54 + "rppaSampleCount": 46,
55 + "massSpectrometrySampleCount": 0,
56 + "completeSampleCount": 75,
57 + "referenceGenome": "hg19",
58 + "treatmentCount": 0,
59 + "structuralVariantCount": 0,
60 + "cancerType": {
61 + "id": "acc",
62 + "name": "Adrenocortical Carcinoma",
63 + "dedicatedColor": "Purple",
64 + "shortName": "ACC",
65 + "parent": "adrenal_gland"
66 + },
67 + "readPermission": true,
68 + "resourceCounts": []
69 + },
70 + {
71 + "studyId": "luad_tcga_pan_can_atlas_2018",
72 + "cancerTypeId": "luad",
73 + "name": "Lung Adenocarcinoma (TCGA, PanCancer Atlas)",
74 + "description": "Lung Adenocarcinoma TCGA PanCancer data. The original data is <a href=\"https://gdc.cancer.gov/about-data/publications/pa",
75 + "publicStudy": true,
76 + "pmid": "29625048,29596782,29622463,29617662,29625055,29625050,29617662,30643250,32214244,29625049,29850653,36334560",
77 + "citation": "TCGA, Cell 2018",
78 + "groups": "PUBLIC;PANCAN",
79 + "status": 0,
80 + "importDate": "2026-06-04 18:48:28",
81 + "allSampleCount": 566,
82 + "sequencedSampleCount": 566,
83 + "cnaSampleCount": 511,
84 + "mrnaRnaSeqSampleCount": 0,
85 + "mrnaRnaSeqV2SampleCount": 510,
86 + "mrnaMicroarraySampleCount": 0,
87 + "miRnaSampleCount": 0,
88 + "methylationHm27SampleCount": 0,
89 + "rppaSampleCount": 360,
90 + "massSpectrometrySampleCount": 0,
91 + "completeSampleCount": 503,
92 + "referenceGenome": "hg19",
93 + "treatmentCount": 202,
94 + "structuralVariantCount": 403,
95 + "cancerType": {
96 + "id": "luad",
97 + "name": "Lung Adenocarcinoma",
98 + "dedicatedColor": "Gainsboro",
99 + "shortName": "LUAD",
100 + "parent": "nsclc"
101 + },
102 + "readPermission": true,
103 + "resourceCounts": [
104 + {
105 + "resourceId": "IDC_OHIF_CT",
106 + "displayName": "Computed Tomography",
107 + "description": "Computed Tomography",
108 + "resourceType": "PATIENT",
109 + "priority": "1",
110 + "openByDefault": true,
111 + "cancerStudyIdentifier": "luad_tcga_pan_can_atlas_2018",
112 + "sampleCount": 60,
113 + "patientCount": 60
114 + },
115 + {
116 + "resourceId": "IDC_OHIF_NM",
117 + "displayName": "Nuclear Medicine",
118 + "description": "Nuclear Medicine",
119 + "resourceType": "PATIENT",
120 + "priority": "1",
121 + "openByDefault": true,
122 + "cancerStudyIdentifier": "luad_tcga_pan_can_atlas_2018",
123 + "sampleCount": 2,
124 + "patientCount": 2
125 + },
126 + {
127 + "resourceId": "IDC_OHIF_PT",
128 + "displayName": "Positron Emission Tomography",
129 + "description": "Positron Emission Tomography",
130 + "resourceType": "PATIENT",
131 + "priority": "1",
132 + "openByDefault": true,
133 + "cancerStudyIdentifier": "luad_tcga_pan_can_atlas_2018",
134 + "sampleCount": 23,
135 + "patientCount": 23
136 + },
137 + {
138 + "resourceId": "IDC_SLIM",
139 + "displayName": "Slide Microscopy",
140 + "description": "Slide Microscopy",
141 + "resourceType": "PATIENT",
142 + "priority": "1",
143 + "openByDefault": true,
144 + "cancerStudyIdentifier": "luad_tcga_pan_can_atlas_2018",
145 + "sampleCount": 514,
146 + "patientCount": 514
147 + }
148 + ]
149 + },
150 + {
151 + "studyId": "mixed_allen_2018",
152 + "cancerTypeId": "mixed",
153 + "name": "MSS Mixed Solid Tumors (Broad/Dana-Farber, Nat Genet 2018)",
154 + "description": "Whole exome sequencing of tumor/normal pairs and corresponding clinical outcomes from patients treated with immune check",
155 + "publicStudy": true,
156 + "pmid": "30150660",
157 + "citation": "Miao et al. Nat Genet 2018",
158 + "groups": "",
159 + "status": 0,
160 + "importDate": "2026-01-08 15:10:03",
161 + "allSampleCount": 249,
162 + "sequencedSampleCount": 249,
163 + "cnaSampleCount": 0,
164 + "mrnaRnaSeqSampleCount": 0,
165 + "mrnaRnaSeqV2SampleCount": 0,
166 + "mrnaMicroarraySampleCount": 0,
167 + "miRnaSampleCount": 0,
168 + "methylationHm27SampleCount": 0,
169 + "rppaSampleCount": 0,
170 + "massSpectrometrySampleCount": 0,
171 + "completeSampleCount": 0,
172 + "referenceGenome": "hg19",
173 + "treatmentCount": 0,
174 + "structuralVariantCount": 0,
175 + "cancerType": {
176 + "id": "mixed",
177 + "name": "Mixed Cancer Types",
178 + "dedicatedColor": "Black",
179 + "shortName": "MIXED",
180 + "parent": "other"
181 + },
182 + "readPermission": true,
183 + "resourceCounts": []
184 + },
185 + {
186 + "studyId": "msk_impact_2017",
187 + "cancerTypeId": "mixed",
188 + "name": "MSK-IMPACT Clinical Sequencing Cohort (MSK, Nat Med 2017)",
189 + "description": "Targeted sequencing of 10,000 clinical cases using the MSK-IMPACT assay",
190 + "publicStudy": true,
191 + "pmid": "28481359",
192 + "citation": "Zehir et al. Nat Med 2017",
193 + "groups": "PUBLIC",
194 + "status": 0,
195 + "importDate": "2026-07-28 17:54:40",
196 + "allSampleCount": 10945,
197 + "sequencedSampleCount": 10945,
198 + "cnaSampleCount": 10945,
199 + "mrnaRnaSeqSampleCount": 0,
200 + "mrnaRnaSeqV2SampleCount": 0,
201 + "mrnaMicroarraySampleCount": 0,
202 + "miRnaSampleCount": 0,
203 + "methylationHm27SampleCount": 0,
204 + "rppaSampleCount": 0,
205 + "massSpectrometrySampleCount": 0,
206 + "completeSampleCount": 0,
207 + "referenceGenome": "hg19",
208 + "treatmentCount": 0,
209 + "structuralVariantCount": 1667,
210 + "cancerType": {
211 + "id": "mixed",
212 + "name": "Mixed Cancer Types",
213 + "dedicatedColor": "Black",
214 + "shortName": "MIXED",
215 + "parent": "other"
216 + },
217 + "readPermission": true,
218 + "resourceCounts": []
219 + },
220 + {
221 + "studyId": "angs_project_painter_2018",
222 + "cancerTypeId": "angs",
223 + "name": "The Angiosarcoma Project - Count Me In (Nature Medicine, 2020)",
224 + "description": "The Angiosarcoma Project is an ongoing patient-driven initiative. This archived Angiosarcoma Project dataset was analyze",
225 + "publicStudy": true,
226 + "pmid": "32042194",
227 + "citation": "Painter et al. Nat Med 2020",
228 + "groups": "",
229 + "status": 0,
230 + "importDate": "2026-01-05 13:46:04",
231 + "allSampleCount": 48,
232 + "sequencedSampleCount": 48,
233 + "cnaSampleCount": 48,
234 + "mrnaRnaSeqSampleCount": 0,
235 + "mrnaRnaSeqV2SampleCount": 0,
236 + "mrnaMicroarraySampleCount": 0,
237 + "miRnaSampleCount": 0,
238 + "methylationHm27SampleCount": 0,
239 + "rppaSampleCount": 0,
240 + "massSpectrometrySampleCount": 0,
241 + "completeSampleCount": 0,
242 + "referenceGenome": "hg19",
243 + "treatmentCount": 0,
244 + "structuralVariantCount": 0,
245 + "cancerType": {
246 + "id": "angs",
247 + "name": "Angiosarcoma",
248 + "dedicatedColor": "LightYellow",
249 + "shortName": "ANGS",
250 + "parent": "soft_tissue"
251 + },
252 + "readPermission": true,
253 + "resourceCounts": []
254 + }
255 +]
\ No newline at end of file
added packages/connectors/src/connectors/cbioportal/index.ts +320 −0
@@ -0,0 +1,320 @@
1 +import { and, eq } from 'drizzle-orm';
2 +import { normalizeLabel } from '@cancerindex/shared';
3 +import { cancerCodes, cancerGeneFrequencies, genomicCohorts, mintId, publicationEntityEdges, publications } from '@cancerindex/database';
4 +import { CancerResolver, type CancerMatch } from '@cancerindex/ontology';
5 +import { Connector, type ConnectorHealth, type RunContext } from '../../sdk/run.js';
6 +import { validateFrequency } from '../../sdk/validate.js';
7 +import { GeneCache } from '../civic/genes.js';
8 +import { CBIO_API, MIN_EXPECTED_STUDIES, STUDIES_PAGE_SIZE, TOP_GENES_PER_STUDY, cancerTypesUrl, manifest, molecularProfilesUrl, mutatedGenesUrl, studiesUrl, studyApiUrl, studyUrl } from './manifest.js';
9 +import { CancerType, MolecularProfile, MutatedGene, Study, findMutationProfile, importDateToRelease, mutatedGenesBody, oncotreeCode, programFromStudyName, rankMutatedGenes, typeLineage } from './normalize.js';
10 +
11 +interface CbioCursor {
12 + pass?: string;
13 + startedAt?: string;
14 + completedAt?: string;
15 + lastStudyId?: string;
16 + stats?: Stats;
17 +}
18 +
19 +interface Stats {
20 + studies: number;
21 + cohorts: number;
22 + resolved: number;
23 + mixed: number;
24 + unresolved: string[];
25 + withMutationData: number;
26 + noMutationProfile: number;
27 + frequencies: number;
28 + rejectedFrequencies: number;
29 + publications: number;
30 +}
31 +
32 +interface CohortResolution {
33 + cancerId: string | null;
34 + matchType: string;
35 + via: string;
36 +}
37 +
38 +/**
39 + * cBioPortal connector — public studies as genomic cohorts + per-gene mutation frequencies with
40 + * explicit denominators (CLAUDE.md §10.8, §261-262):
41 + * 1. GET /studies (DETAILED) + /cancer-types → cohorts (OncoTree code → cancer, EXACT_IDENTIFIER).
42 + * 2. Per study with mutation data: GET /studies/{id}/molecular-profiles (must expose a
43 + * MUTATION_EXTENDED profile) and POST /mutated-genes/fetch {studyIds:[id]} → top genes by
44 + * altered samples; casesProfiled = numberOfProfiledCases (gene-panel aware; equals the
45 + * `<studyId>_sequenced` sample count for WES/WGS studies).
46 + * Studies are processed in studyId order; ctx.cursor.lastStudyId makes the run restartable.
47 + */
48 +export class CbioportalConnector extends Connector {
49 + readonly manifest = manifest;
50 +
51 + async healthCheck(ctx: RunContext): Promise<ConnectorHealth> {
52 + try {
53 + const res = await ctx.http.json<unknown>(`${CBIO_API}/studies?pageSize=1&projection=SUMMARY`);
54 + if (!Array.isArray(res) || res.length !== 1) return { status: 'degraded', detail: 'unexpected /studies payload' };
55 + return { status: 'healthy', detail: `studies endpoint OK (${(res[0] as { studyId?: string }).studyId ?? '?'})` };
56 + } catch (e) {
57 + return { status: 'failing', detail: (e as Error).message };
58 + }
59 + }
60 +
61 + async sync(ctx: RunContext): Promise<void> {
62 + const studies = await this.fetchStudies(ctx);
63 + const typeRows = await ctx.http.json<unknown[]>(cancerTypesUrl());
64 + const types = new Map<string, CancerType>();
65 + for (const raw of typeRows) {
66 + const p = CancerType.safeParse(raw);
67 + if (p.success) types.set(p.data.cancerTypeId, p.data);
68 + else ctx.counters.validationFailures++;
69 + }
70 + ctx.info(`fetched ${studies.length} public studies and ${types.size} cancer types`);
71 + if (studies.length < MIN_EXPECTED_STUDIES) throw new Error(`anomaly: only ${studies.length} cBioPortal studies returned (expected ~545) — refusing to persist (CLAUDE.md §171)`);
72 + const latestImport = studies.map((s) => importDateToRelease(s.importDate)).filter((d): d is string => !!d).sort().pop();
73 + ctx.datasetVersion = `cbioportal-${latestImport ?? new Date().toISOString().slice(0, 10)}`;
74 +
75 + if (ctx.mode === 'dry_run') {
76 + for (const s of studies) {
77 + ctx.observe('study', s);
78 + ctx.counters.fetched++;
79 + }
80 + const sample = studies.find((s) => s.studyId === 'acc_tcga') ?? studies[0]!;
81 + const genes = rankMutatedGenes((await ctx.http.postJson<unknown[]>(mutatedGenesUrl(), mutatedGenesBody(sample.studyId))).map((g) => MutatedGene.parse(g)), 5);
82 + ctx.info(`[dry_run] ${sample.studyId}: sequenced ${sample.sequencedSampleCount ?? '?'}, top genes ${genes.map((g) => `${g.hugoGeneSymbol}=${g.numberOfAlteredCases}/${g.numberOfProfiledCases}`).join(', ')}`);
83 + return;
84 + }
85 +
86 + const state = ctx.cursor as CbioCursor;
87 + if (!state.pass || state.completedAt) {
88 + for (const k of Object.keys(state)) delete (state as Record<string, unknown>)[k];
89 + state.pass = new Date().toISOString().slice(0, 10);
90 + state.startedAt = new Date().toISOString();
91 + ctx.info('starting a fresh cBioPortal pass');
92 + } else ctx.info(`resuming cBioPortal pass ${state.pass} after study ${state.lastStudyId ?? '(start)'}`);
93 + const stats: Stats = state.stats ?? { studies: 0, cohorts: 0, resolved: 0, mixed: 0, unresolved: [], withMutationData: 0, noMutationProfile: 0, frequencies: 0, rejectedFrequencies: 0, publications: 0 };
94 + state.stats = stats;
95 +
96 + const resolver = new CancerResolver(ctx.db);
97 + await resolver.warm();
98 + const geneCache = new GeneCache(ctx.db);
99 + const pubCache = new Map<string, string>();
100 + const sorted = [...studies].sort((a, b) => a.studyId.localeCompare(b.studyId));
101 +
102 + for (const study of sorted) {
103 + if (state.lastStudyId && study.studyId <= state.lastStudyId) continue;
104 + if (ctx.shouldStop()) {
105 + ctx.info(`stopping after study ${state.lastStudyId ?? '(start)'} — ${stats.studies} studies this pass; cursor saved`);
106 + return;
107 + }
108 + await this.processStudy(ctx, study, types, resolver, geneCache, pubCache, stats);
109 + state.lastStudyId = study.studyId;
110 + stats.studies++;
111 + }
112 + state.completedAt = new Date().toISOString();
113 + ctx.info(`cBioPortal pass complete: studies ${stats.studies}, cohorts ${stats.cohorts}, resolved ${stats.resolved}, mixed (pan-cancer, unmappable) ${stats.mixed}, unresolved ${stats.unresolved.length}, with mutation data ${stats.withMutationData}, without mutation profile ${stats.noMutationProfile}, frequencies ${stats.frequencies} (rejected ${stats.rejectedFrequencies}), publication stubs ${stats.publications}, genes created ${geneCache.created}`, { unresolved: stats.unresolved.slice(0, 40) });
114 + }
115 +
116 + /* ---------------------------------------------------------------------------------------------- */
117 +
118 + private async fetchStudies(ctx: RunContext): Promise<Study[]> {
119 + const out: Study[] = [];
120 + for (let page = 0; page < 20; page++) {
121 + const raw = await ctx.http.json<unknown[]>(studiesUrl(page));
122 + if (!Array.isArray(raw)) throw new Error('unexpected /studies payload (not an array)');
123 + for (const r of raw) {
124 + const p = Study.safeParse(r);
125 + if (!p.success) {
126 + ctx.counters.validationFailures++;
127 + ctx.counters.rejected++;
128 + ctx.warn(`invalid study record: ${p.error.issues[0]?.path.join('.')} ${p.error.issues[0]?.message}`);
129 + continue;
130 + }
131 + if (p.data.publicStudy === false) continue;
132 + out.push(p.data);
133 + }
134 + if (raw.length < STUDIES_PAGE_SIZE) break;
135 + }
136 + return out;
137 + }
138 +
139 + /** OncoTree code (EXACT_IDENTIFIER) → cancer-type name (alias) → unresolved; "mixed" is never mapped. */
140 + private resolveStudy(resolver: CancerResolver, study: Study, types: Map<string, CancerType>): CohortResolution {
141 + const code = oncotreeCode(study.cancerTypeId);
142 + if (!code) return { cancerId: null, matchType: 'UNRESOLVED', via: `cancerTypeId "${study.cancerTypeId}" (pan-cancer / mixed)` };
143 + const byCode: CancerMatch | null = resolver.byCode('oncotree', code);
144 + if (byCode) return { cancerId: byCode.cancerId, matchType: byCode.matchType, via: byCode.via };
145 + const name = types.get(study.cancerTypeId)?.name ?? study.cancerType?.name;
146 + const byLabel = name ? resolver.byLabel(name) : null;
147 + if (byLabel) return { cancerId: byLabel.cancerId, matchType: byLabel.matchType, via: byLabel.via };
148 + return { cancerId: null, matchType: 'UNRESOLVED', via: `oncotree:${code} unknown${name ? `, label "${name}" unmatched` : ''}` };
149 + }
150 +
151 + private async processStudy(ctx: RunContext, study: Study, types: Map<string, CancerType>, resolver: CancerResolver, geneCache: GeneCache, pubCache: Map<string, string>, stats: Stats): Promise<void> {
152 + const res = this.resolveStudy(resolver, study, types);
153 + const code = oncotreeCode(study.cancerTypeId);
154 + const lineage = typeLineage(study.cancerTypeId, types);
155 + const typeName = types.get(study.cancerTypeId)?.name ?? study.cancerType?.name ?? study.cancerTypeId;
156 + if (res.cancerId) stats.resolved++;
157 + else if (!code) stats.mixed++;
158 + else {
159 + stats.unresolved.push(`${study.studyId} ${study.cancerTypeId} "${typeName}"`);
160 + await ctx.recordUnresolved('cancer', typeName, normalizeLabel(typeName), { cbioportalStudyId: study.studyId, cancerTypeId: study.cancerTypeId, oncotreeCode: code, lineage: lineage.chain });
161 + }
162 + const program = programFromStudyName(study.name);
163 + const dataRelease = importDateToRelease(study.importDate);
164 +
165 + // Source record first: an unchanged study keeps its previous provenance row.
166 + const [existing] = await ctx.db.select({ id: genomicCohorts.id, provenanceId: genomicCohorts.provenanceId }).from(genomicCohorts).where(and(eq(genomicCohorts.sourceId, ctx.sourceId), eq(genomicCohorts.studyId, study.studyId))).limit(1);
167 + const cohortId = existing?.id ?? (await mintId(ctx.db, 'STUDY'));
168 + const rec = await ctx.upsertSourceRecord('study', study.studyId, study, { canonicalType: 'genomic_cohort', canonicalId: cohortId, sourceUpdatedAt: dataRelease ? new Date(dataRelease) : null });
169 + const provenanceId =
170 + rec.status === 'unchanged' && existing?.provenanceId
171 + ? existing.provenanceId
172 + : await ctx.addProvenance({
173 + sourceRecordId: study.studyId,
174 + sourceUrl: studyApiUrl(study.studyId),
175 + dataset: 'cBioPortal public studies',
176 + pmid: study.pmid ?? undefined,
177 + evidenceType: 'cohort',
178 + accessLevel: 'open',
179 + cohortSize: study.allSampleCount ?? undefined,
180 + population: `${study.studyId}${program ? ` (${program})` : ''}`,
181 + methodology: `cBioPortal study ${study.studyId}: ${study.name}${study.citation ? `; citation: ${study.citation}` : ''}${study.pmid ? ` (PMID ${study.pmid})` : ''}; importDate ${study.importDate ?? '?'}; reference genome ${study.referenceGenome ?? '?'}`,
182 + updatedAt: dataRelease ?? undefined,
183 + });
184 +
185 + const cohortValues = {
186 + name: study.name,
187 + program,
188 + primarySites: lineage.primarySite ? [lineage.primarySite] : [],
189 + diseaseTypes: lineage.diseaseType ? [lineage.diseaseType] : [],
190 + cancerId: res.cancerId,
191 + cancerMatchType: res.cancerId ? res.matchType : 'UNRESOLVED',
192 + caseCount: study.allSampleCount ?? null,
193 + casesWithSsm: study.sequencedSampleCount ?? null,
194 + dataRelease,
195 + accessLevel: 'open',
196 + url: studyUrl(study.studyId),
197 + provenanceId,
198 + updatedAt: new Date(),
199 + };
200 + await ctx.db
201 + .insert(genomicCohorts)
202 + .values({ id: cohortId, sourceId: ctx.sourceId, studyId: study.studyId, ...cohortValues })
203 + .onConflictDoUpdate({ target: [genomicCohorts.sourceId, genomicCohorts.studyId], set: cohortValues });
204 + if (!existing) await ctx.recordChange('genomic_cohort', cohortId, 'created', `Created from cBioPortal study ${study.studyId} (${study.name})`);
205 + stats.cohorts++;
206 + // A code resolved through its label is still a usable cross-reference (match type recorded).
207 + if (res.cancerId && code && res.matchType !== 'EXACT_IDENTIFIER') await ctx.db.insert(cancerCodes).values({ cancerId: res.cancerId, system: 'oncotree', code, matchType: res.matchType, sourceId: ctx.sourceId }).onConflictDoNothing();
208 +
209 + // Publication stub for the study's paper (enriched later by the PubMed connector).
210 + if (study.pmid && /^\d{1,9}$/.test(study.pmid)) {
211 + const pubId = await this.ensurePublication(ctx, study.pmid, study, pubCache);
212 + if (pubId) {
213 + stats.publications++;
214 + if (res.cancerId) await ctx.db.insert(publicationEntityEdges).values({ publicationId: pubId, entityType: 'cancer', entityId: res.cancerId, method: 'cbioportal_study', confidence: 1, status: 'validated', sourceId: ctx.sourceId, ingestRunId: ctx.runId }).onConflictDoNothing();
215 + }
216 + }
217 +
218 + // Mutation frequencies
219 + const sequenced = study.sequencedSampleCount ?? 0;
220 + if (sequenced <= 0) return;
221 + const [profilesRaw, genesRaw] = await Promise.all([ctx.http.json<unknown[]>(molecularProfilesUrl(study.studyId)), ctx.http.postJson<unknown[]>(mutatedGenesUrl(), mutatedGenesBody(study.studyId))]);
222 + const profiles: MolecularProfile[] = [];
223 + for (const r of Array.isArray(profilesRaw) ? profilesRaw : []) {
224 + const p = MolecularProfile.safeParse(r);
225 + if (p.success) profiles.push(p.data);
226 + else ctx.counters.validationFailures++;
227 + }
228 + const mutationProfile = findMutationProfile(profiles, study.studyId);
229 + if (!mutationProfile) {
230 + stats.noMutationProfile++;
231 + ctx.info(`${study.studyId}: sequencedSampleCount ${sequenced} but no MUTATION_EXTENDED profile — frequencies skipped`);
232 + return;
233 + }
234 + stats.withMutationData++;
235 + const genes: MutatedGene[] = [];
236 + for (const r of Array.isArray(genesRaw) ? genesRaw : []) {
237 + const p = MutatedGene.safeParse(r);
238 + if (p.success) genes.push(p.data);
239 + else {
240 + ctx.counters.validationFailures++;
241 + ctx.counters.rejected++;
242 + }
243 + }
244 + const ranked = rankMutatedGenes(genes, TOP_GENES_PER_STUDY);
245 + const freqRec = await ctx.upsertSourceRecord('mutated_genes', study.studyId, { studyId: study.studyId, molecularProfileId: mutationProfile.molecularProfileId, sequencedSampleCount: sequenced, totalGenes: genes.length, genes: ranked }, { canonicalType: 'genomic_cohort', canonicalId: cohortId });
246 + const [prevFreq] = await ctx.db.select({ provenanceId: cancerGeneFrequencies.provenanceId }).from(cancerGeneFrequencies).where(and(eq(cancerGeneFrequencies.cohortId, cohortId), eq(cancerGeneFrequencies.alterationType, 'ssm'))).limit(1);
247 + const freqProvenanceId =
248 + freqRec.status === 'unchanged' && prevFreq
249 + ? prevFreq.provenanceId
250 + : await ctx.addProvenance({
251 + sourceRecordId: study.studyId,
252 + sourceUrl: mutatedGenesUrl(),
253 + dataset: 'cBioPortal mutated genes by study',
254 + pmid: study.pmid ?? undefined,
255 + evidenceType: 'cohort',
256 + accessLevel: 'open',
257 + cohortSize: sequenced,
258 + population: `${study.studyId}${program ? ` (${program})` : ''}`,
259 + methodology: `POST /api/mutated-genes/fetch {studyIds:["${study.studyId}"]} (portal default filters): distinct samples with ≥1 mutation in gene / samples profiled for the gene (numberOfProfiledCases, gene-panel aware; = samples in ${study.studyId}_sequenced for whole-exome studies, ${sequenced}); profile ${mutationProfile.molecularProfileId}; top ${TOP_GENES_PER_STUDY} genes by altered samples`,
260 + });
261 + const rejected: string[] = [];
262 + for (const g of ranked) {
263 + const issues = validateFrequency({ casesAffected: g.numberOfAlteredCases, casesProfiled: g.numberOfProfiledCases });
264 + if (issues.length) {
265 + // Mixed-panel studies occasionally report more altered than profiled samples for a gene
266 + // (portal gene-panel matrix inconsistency): never stored, counted as validation failures.
267 + ctx.counters.validationFailures++;
268 + ctx.counters.rejected++;
269 + stats.rejectedFrequencies++;
270 + rejected.push(`${g.hugoGeneSymbol} ${g.numberOfAlteredCases}/${g.numberOfProfiledCases}`);
271 + continue;
272 + }
273 + const geneId = await geneCache.ensure(g.hugoGeneSymbol, { ncbiGeneId: g.entrezGeneId != null && g.entrezGeneId > 0 ? String(g.entrezGeneId) : null });
274 + if (!geneId) await ctx.recordUnresolved('gene', g.hugoGeneSymbol, normalizeLabel(g.hugoGeneSymbol), { cbioportalStudyId: study.studyId, entrezGeneId: g.entrezGeneId });
275 + const values = {
276 + cancerId: res.cancerId,
277 + geneId,
278 + casesAffected: g.numberOfAlteredCases,
279 + casesProfiled: g.numberOfProfiledCases,
280 + frequency: g.numberOfAlteredCases / g.numberOfProfiledCases,
281 + rank: g.rank,
282 + dataRelease,
283 + provenanceId: freqProvenanceId,
284 + updatedAt: new Date(),
285 + };
286 + await ctx.db
287 + .insert(cancerGeneFrequencies)
288 + .values({ cohortId, geneSymbol: g.hugoGeneSymbol, alterationType: 'ssm', ...values })
289 + .onConflictDoUpdate({ target: [cancerGeneFrequencies.cohortId, cancerGeneFrequencies.geneSymbol, cancerGeneFrequencies.alterationType], set: values });
290 + stats.frequencies++;
291 + }
292 + if (rejected.length) ctx.warn(`${study.studyId}: ${rejected.length} gene(s) rejected by validateFrequency (altered > profiled): ${rejected.slice(0, 8).join(', ')}${rejected.length > 8 ? '…' : ''}`);
293 + }
294 +
295 + /** Minimal publication stub (publicationTypes ['stub']); never fabricates an abstract. */
296 + private async ensurePublication(ctx: RunContext, pmid: string, study: Study, cache: Map<string, string>): Promise<string | null> {
297 + const cached = cache.get(pmid);
298 + if (cached) return cached;
299 + const [existing] = await ctx.db.select({ id: publications.id }).from(publications).where(eq(publications.pmid, pmid)).limit(1);
300 + let id = existing?.id;
301 + if (!id) {
302 + const minted = await mintId(ctx.db, 'PUB');
303 + const yearMatch = /(19|20)\d{2}/.exec(study.citation ?? '');
304 + const inserted = await ctx.db
305 + .insert(publications)
306 + .values({ id: minted, pmid, title: study.citation?.trim() || `PMID ${pmid}`, pubYear: yearMatch ? Number(yearMatch[0]) : null, publicationTypes: ['stub'], ingestRunId: ctx.runId })
307 + .onConflictDoNothing({ target: publications.pmid })
308 + .returning({ id: publications.id });
309 + id = inserted[0]?.id;
310 + if (!id) {
311 + const [again] = await ctx.db.select({ id: publications.id }).from(publications).where(eq(publications.pmid, pmid)).limit(1);
312 + id = again?.id;
313 + }
314 + }
315 + if (id) cache.set(pmid, id);
316 + return id ?? null;
317 + }
318 +}
319 +
320 +export const connector = new CbioportalConnector();
added packages/connectors/src/connectors/cbioportal/manifest.ts +96 −0
@@ -0,0 +1,96 @@
1 +import { defineManifest } from '../../sdk/manifest.js';
2 +
3 +/**
4 + * Docs verified 2026-09-08 — OpenAPI https://www.cbioportal.org/api/v3/api-docs (Swagger UI
5 + * https://www.cbioportal.org/api/swagger-ui/index.html), user guide FAQ
6 + * https://docs.cbioportal.org/user-guide/faq/ and the live public instance:
7 + * - GET /api/studies?pageSize=1000&projection=DETAILED → 545 public studies {studyId, cancerTypeId,
8 + * name, description, publicStudy, pmid, citation, importDate "2026-07-28 17:54:40", allSampleCount,
9 + * sequencedSampleCount (samples with mutation data = the `<studyId>_sequenced` sample list),
10 + * cnaSampleCount, referenceGenome, cancerType{id,name,parent}}. 539/545 have sequencedSampleCount > 0.
11 + * - GET /api/cancer-types?pageSize=10000 → 897 OncoTree nodes {cancerTypeId, name, shortName, parent}
12 + * (cancerTypeId = OncoTree code in lower case; "mixed" = several cancer types, "tissue" = root).
13 + * - GET /api/studies/{studyId}/molecular-profiles → profiles; mutations = molecularAlterationType
14 + * MUTATION_EXTENDED, id `<studyId>_mutations`.
15 + * - GET /api/sample-lists/{studyId}_sequenced → {sampleListId, category all_cases_with_mutation_data,
16 + * sampleCount, sampleIds[]} (cross-checked: msk_impact_2017 10,945 = sequencedSampleCount).
17 + * - POST /api/mutated-genes/fetch (Study View, body StudyViewFilter {studyIds:[…]}) → per gene
18 + * {hugoGeneSymbol, entrezGeneId, numberOfAlteredCases (distinct samples with ≥1 mutation),
19 + * numberOfProfiledCases (samples profiled for that gene — panel-aware), totalCount (mutations),
20 + * matchingGenePanelIds}. This is the endpoint behind the study summary "Mutated Genes" table and the
21 + * cheapest way to get per-gene counts with a denominator (1 request per study; MSK-IMPACT 10,945
22 + * samples in ~1 s). It is used instead of paging per-mutation lists.
23 + *
24 + * License / terms — FAQ "Do I need permission or a license to use the cBioPortal logo or figures or
25 + * data?" (https://docs.cbioportal.org/user-guide/faq/): "Unless otherwise noted, data in cBioPortal
26 + * are available under the ODC Open Database License with no restrictions on the use of the data, as
27 + * long as you properly give attribution to the original studies. There are some studies that
28 + * restrict the commercial use of the data, but that will be explicitly mentioned in the study
29 + * information." FAQ "What is the cBioPortal for Cancer Genomics?": "an open-access, open-source
30 + * resource for interactive exploration of multidimensional cancer genomics data sets". FAQ "How do I
31 + * cite the cBioPortal?": Cerami et al. Cancer Discov 2012 (PMID 22588877); Gao et al. Sci Signal
32 + * 2013 (PMID 23550210); de Bruijn et al. Cancer Res 2023 (PMID 37668528); "Remember also to cite the
33 + * source of the data if you are using a publicly available dataset." → per-study `citation`/`pmid`
34 + * stored on every cohort provenance.
35 + */
36 +export const manifest = defineManifest({
37 + id: 'cbioportal',
38 + name: 'cBioPortal for Cancer Genomics (public studies)',
39 + organization: 'Memorial Sloan Kettering Cancer Center and the cBioPortal consortium',
40 + category: 'genomics',
41 + tier: 1,
42 + description:
43 + 'Public cancer genomics cohorts beyond TCGA (MSK-IMPACT, GENIE-derived publications, institutional and consortium studies): every public study becomes a genomic cohort (OncoTree cancer type, sample counts, publication), and its per-gene mutation frequencies are ingested with explicit denominators (samples profiled for the gene / samples with mutation data). Cohorts overlapping GDC (TCGA) are kept as separate, never pooled, cohorts.',
44 + homepage: 'https://www.cbioportal.org',
45 + docsUrl: 'https://www.cbioportal.org/api/swagger-ui/index.html',
46 + termsUrl: 'https://docs.cbioportal.org/user-guide/faq/',
47 + access: { type: 'rest', auth: 'none', baseUrl: 'https://www.cbioportal.org/api' },
48 + license: 'ODC Open Database License (ODC-ODbL) unless otherwise noted per study; attribution to the original studies required',
49 + licenseStatus: 'approved',
50 + commercialUse: 'restricted',
51 + redistribution: 'attribution',
52 + attribution: 'Data from cBioPortal for Cancer Genomics (Cerami et al. 2012, PMID 22588877; Gao et al. 2013, PMID 23550210; de Bruijn et al. 2023, PMID 37668528) and from the original studies cited on each cohort.',
53 + termsReviewedAt: '2026-09-08',
54 + termsNotes: 'ODC-ODbL with attribution to the original studies; "some studies … restrict the commercial use of the data, but that will be explicitly mentioned in the study information" — the study description is kept on the source record so such notices are reviewable; commercialUse therefore marked restricted at the source level. Only aggregate per-gene counts are stored (no sample-level or patient-level data).',
55 + updateFrequency: 'Continuous (new studies monthly); study importDate recorded as dataRelease',
56 + expectedLatency: 'Days after a study import',
57 + // Studies are processed in studyId order and the last processed id is saved in ctx.cursor, so an
58 + // interrupted run resumes (CLAUDE.md §90). A completed pass starts over next time.
59 + supportsIncrementalSync: true,
60 + entities: ['genomic_cohorts', 'cancer_gene_frequencies', 'genes', 'publications'],
61 + metrics: ['gene_mutation_frequency'],
62 + rateLimits: { requestsPerSecond: 3, maxConcurrency: 2, notes: 'No published limit; 2 requests per study (~1,100 per full pass).' },
63 + rawRetention: 'full',
64 + documentationVerifiedAt: '2026-09-08',
65 + status: 'active',
66 + schedule: '0 5 2 * *',
67 +});
68 +
69 +export const CBIO_API = manifest.access.baseUrl!;
70 +export const STUDIES_PAGE_SIZE = 1000;
71 +/** Per-study cap on stored gene frequencies (ranked by altered samples); WES studies return 15k+ genes. */
72 +export const TOP_GENES_PER_STUDY = 200;
73 +/** Anomaly guard: the public portal lists ~545 studies (2026-09-08). */
74 +export const MIN_EXPECTED_STUDIES = 100;
75 +
76 +export function studiesUrl(pageNumber = 0): string {
77 + return `${CBIO_API}/studies?pageSize=${STUDIES_PAGE_SIZE}&pageNumber=${pageNumber}&projection=DETAILED`;
78 +}
79 +export function cancerTypesUrl(): string {
80 + return `${CBIO_API}/cancer-types?pageSize=10000`;
81 +}
82 +export function molecularProfilesUrl(studyId: string): string {
83 + return `${CBIO_API}/studies/${encodeURIComponent(studyId)}/molecular-profiles`;
84 +}
85 +export function sampleListUrl(sampleListId: string): string {
86 + return `${CBIO_API}/sample-lists/${encodeURIComponent(sampleListId)}`;
87 +}
88 +export function mutatedGenesUrl(): string {
89 + return `${CBIO_API}/mutated-genes/fetch`;
90 +}
91 +export function studyUrl(studyId: string): string {
92 + return `https://www.cbioportal.org/study/summary?id=${encodeURIComponent(studyId)}`;
93 +}
94 +export function studyApiUrl(studyId: string): string {
95 + return `${CBIO_API}/studies/${encodeURIComponent(studyId)}`;
96 +}
added packages/connectors/src/connectors/cbioportal/normalize.ts +141 −0
@@ -0,0 +1,141 @@
1 +import { z } from 'zod';
2 +
3 +/* ------------------------------------------------------------------------------------------------
4 + * Response shapes (verified against the live API on 2026-09-08; unknown fields kept for the lake).
5 + * ---------------------------------------------------------------------------------------------- */
6 +
7 +export const CancerType = z.object({
8 + cancerTypeId: z.string(),
9 + name: z.string(),
10 + shortName: z.string().optional(),
11 + parent: z.string().optional(),
12 + dedicatedColor: z.string().optional(),
13 +});
14 +export type CancerType = z.infer<typeof CancerType>;
15 +
16 +export const Study = z
17 + .object({
18 + studyId: z.string(),
19 + cancerTypeId: z.string(),
20 + name: z.string(),
21 + description: z.string().optional(),
22 + publicStudy: z.boolean().optional(),
23 + pmid: z.string().optional(),
24 + citation: z.string().optional(),
25 + groups: z.string().optional(),
26 + importDate: z.string().optional(), // "2026-07-28 17:54:40"
27 + allSampleCount: z.number().int().nonnegative().optional(),
28 + sequencedSampleCount: z.number().int().nonnegative().optional(),
29 + cnaSampleCount: z.number().int().nonnegative().optional(),
30 + referenceGenome: z.string().optional(),
31 + cancerType: CancerType.partial().optional(),
32 + })
33 + .passthrough();
34 +export type Study = z.infer<typeof Study>;
35 +
36 +export const MolecularProfile = z
37 + .object({
38 + molecularProfileId: z.string(),
39 + studyId: z.string(),
40 + molecularAlterationType: z.string(), // MUTATION_EXTENDED | COPY_NUMBER_ALTERATION | …
41 + datatype: z.string().optional(),
42 + name: z.string().optional(),
43 + description: z.string().optional(),
44 + })
45 + .passthrough();
46 +export type MolecularProfile = z.infer<typeof MolecularProfile>;
47 +
48 +export const SampleList = z
49 + .object({
50 + sampleListId: z.string(),
51 + studyId: z.string(),
52 + category: z.string().optional(),
53 + name: z.string().optional(),
54 + sampleCount: z.number().int().nonnegative().optional(),
55 + sampleIds: z.array(z.string()).optional(),
56 + })
57 + .passthrough();
58 +
59 +export const MutatedGene = z
60 + .object({
61 + hugoGeneSymbol: z.string(),
62 + entrezGeneId: z.number().int().optional(),
63 + numberOfAlteredCases: z.number().int().nonnegative(),
64 + numberOfProfiledCases: z.number().int().nonnegative(),
65 + totalCount: z.number().int().nonnegative().optional(),
66 + matchingGenePanelIds: z.array(z.string()).optional(),
67 + qValue: z.number().optional(),
68 + })
69 + .passthrough();
70 +export type MutatedGene = z.infer<typeof MutatedGene>;
71 +
72 +/** Body of POST /api/mutated-genes/fetch (StudyViewFilter restricted to one study; portal default filters). */
73 +export function mutatedGenesBody(studyId: string): { studyIds: string[] } {
74 + return { studyIds: [studyId] };
75 +}
76 +
77 +/** The study's mutation profile (`<studyId>_mutations`, type MUTATION_EXTENDED), if any. */
78 +export function findMutationProfile(profiles: MolecularProfile[], studyId: string): MolecularProfile | null {
79 + const byId = profiles.find((p) => p.molecularProfileId === `${studyId}_mutations`);
80 + if (byId) return byId;
81 + return profiles.find((p) => p.molecularAlterationType === 'MUTATION_EXTENDED') ?? null;
82 +}
83 +
84 +/** "2026-07-28 17:54:40" → "2026-07-28". */
85 +export function importDateToRelease(importDate: string | undefined): string | null {
86 + if (!importDate) return null;
87 + const m = /^(\d{4}-\d{2}-\d{2})/.exec(importDate.trim());
88 + return m ? m[1]! : null;
89 +}
90 +
91 +/**
92 + * Program / data source from the study name's parenthetical: "MSK-IMPACT Clinical Sequencing Cohort
93 + * (MSK, Nat Med 2017)" → "MSK"; "Adrenocortical Carcinoma (TCGA, Firehose Legacy)" → "TCGA";
94 + * "Lung Adenocarcinoma (TCGA, PanCancer Atlas)" → "TCGA". Null when the name has no parenthetical.
95 + */
96 +export function programFromStudyName(name: string): string | null {
97 + const m = /\(([^()]*)\)\s*$/.exec(name.trim());
98 + if (!m) return null;
99 + const first = m[1]!.split(/[,;]/)[0]!.trim();
100 + // "(J Clin Invest 2019)" / "(Nat Med 2017)" is a citation, not a data source.
101 + if (/\b(19|20)\d{2}\b/.test(first)) return null;
102 + return first.length > 0 && first.length <= 40 ? first : null;
103 +}
104 +
105 +/** OncoTree code as stored in cancer_codes (upper case; "mixed" is not a code). */
106 +export function oncotreeCode(cancerTypeId: string): string | null {
107 + const id = cancerTypeId.trim();
108 + if (!id || /^(mixed|other|tissue)$/i.test(id)) return null;
109 + return id.toUpperCase();
110 +}
111 +
112 +export interface TypeLineage {
113 + diseaseType: string | null;
114 + primarySite: string | null;
115 + chain: string[]; // cancerTypeId path from the node up to (excluding) "tissue"
116 +}
117 +
118 +/** Walk cancer-type parents up to the tissue root: the node just below "tissue" is the primary site. */
119 +export function typeLineage(cancerTypeId: string, types: Map<string, CancerType>): TypeLineage {
120 + const chain: string[] = [];
121 + let cur: string | undefined = cancerTypeId;
122 + const seen = new Set<string>();
123 + while (cur && cur !== 'tissue' && !seen.has(cur) && chain.length < 12) {
124 + seen.add(cur);
125 + chain.push(cur);
126 + cur = types.get(cur)?.parent;
127 + }
128 + const node = types.get(cancerTypeId);
129 + const siteId = chain.length ? chain[chain.length - 1]! : null;
130 + const site = siteId ? types.get(siteId) : undefined;
131 + return { diseaseType: node?.name ?? null, primarySite: site?.name ?? null, chain };
132 +}
133 +
134 +/** Rank genes by altered samples (desc), symbol (asc); drop genes without a positive denominator. */
135 +export function rankMutatedGenes(genes: MutatedGene[], top: number): Array<MutatedGene & { rank: number }> {
136 + return genes
137 + .filter((g) => g.numberOfProfiledCases > 0 && g.numberOfAlteredCases > 0)
138 + .sort((a, b) => b.numberOfAlteredCases - a.numberOfAlteredCases || a.hugoGeneSymbol.localeCompare(b.hugoGeneSymbol))
139 + .slice(0, top)
140 + .map((g, i) => ({ ...g, rank: i + 1 }));
141 +}
modified packages/connectors/src/connectors/cdc-uscs/index.ts +7 −1
@@ -114,6 +114,7 @@ export class CdcUscsConnector extends Connector {
114 114 }
115 115 let observations = 0;
116 116 let suppressed = 0;
117 + let sexSpecificRateSkipped = 0;
117 118 let notAvailable = 0;
118 119 for (const [key, rows] of groups) {
119 120 if (ctx.shouldStop()) break;
@@ -137,12 +138,17 @@ export class CdcUscsConnector extends Connector {
137 138 if (await upsertObservation(ctx, { ...base, metric, value: r.count, unit: 'count' })) observations++;
138 139 }
139 140 if (r.rateMarker == null && r.ageAdjustedRate != null) {
141 + // Sex-specific sites: the "Male and Female" rate uses the sex-specific population → not an all-sex rate (CLAUDE.md §114, §116).
142 + if (def.sexSpecific && r.sex === 'all') {
143 + sexSpecificRateSkipped++;
144 + continue;
145 + }
140 146 const metric: EpiObservationInput['metric'] = isIncidence ? 'as_incidence_rate' : 'as_mortality_rate';
141 147 if (await upsertObservation(ctx, { ...base, metric, value: r.ageAdjustedRate, unit: 'per_100k', lowerCi: r.ciLower, upperCi: r.ciUpper, standardPopulation: STANDARD_POPULATION })) observations++;
142 148 }
143 149 }
144 150 }
145 − ctx.info(`observations written: ${observations}; suppressed cells skipped: ${suppressed}; not-available cells skipped: ${notAvailable}`);
151 + ctx.info(`observations written: ${observations}; suppressed cells skipped: ${suppressed}; not-available cells skipped: ${notAvailable}; all-sex rates skipped for sex-specific sites: ${sexSpecificRateSkipped}`);
146 152 ctx.cursor = { zipUrl: url, sha256, datasetVersion, syncedAt: new Date().toISOString() };
147 153 }
148 154
modified packages/connectors/src/connectors/cdc-uscs/sites.ts +14 −8
@@ -25,6 +25,12 @@ export interface UscsSiteDefinition {
25 25 icdo3: string; // SEER site recode, ICD-O-3 topography (+ histology where defining)
26 26 /** Restrict the SEX values accepted for this label (breast is split into sex-specific labels). */
27 27 sexes?: Array<'Male and Female' | 'Female' | 'Male'>;
28 + /**
29 + * Sex-specific site: USCS computes the "Male and Female" *rate* on the population of that sex
30 + * (technical notes: "rates for sex-specific cancers are calculated using the relevant sex population"),
31 + * so it is NOT comparable with all-sex rates of other sites. Counts are kept; all-sex rates are skipped.
32 + */
33 + sexSpecific?: 'male' | 'female';
28 34 note?: string;
29 35 }
30 36
@@ -47,14 +53,14 @@ export const USCS_SITES: UscsSiteDefinition[] = [
47 53 { label: 'Male and Female Breast', lookupCodes: ['C50'], relation: 'exact', icd10: 'C50', icdo3: 'C500-C509', sexes: ['Male and Female'] },
48 54 { label: 'Female Breast', lookupCodes: ['C50'], relation: 'exact', icd10: 'C50', icdo3: 'C500-C509', sexes: ['Female'] },
49 55 { label: 'Male Breast', lookupCodes: ['C50'], relation: 'exact', icd10: 'C50', icdo3: 'C500-C509', sexes: ['Male'] },
50 − { label: 'Vulva', lookupCodes: ['C51'], relation: 'exact', icd10: 'C51', icdo3: 'C510-C519' },
51 − { label: 'Vagina', lookupCodes: ['C52'], relation: 'exact', icd10: 'C52', icdo3: 'C529' },
52 − { label: 'Cervix', lookupCodes: ['C53'], relation: 'exact', icd10: 'C53', icdo3: 'C530-C539' },
53 − { label: 'Corpus', lookupCodes: ['C54'], relation: 'exact', icd10: 'C54', icdo3: 'C540-C549' },
54 − { label: 'Ovary', lookupCodes: ['C56'], relation: 'exact', icd10: 'C56', icdo3: 'C569' },
55 − { label: 'Penis', lookupCodes: ['C60'], relation: 'exact', icd10: 'C60', icdo3: 'C600-C609' },
56 − { label: 'Prostate', lookupCodes: ['C61'], relation: 'exact', icd10: 'C61', icdo3: 'C619' },
57 − { label: 'Testis', lookupCodes: ['C62'], relation: 'exact', icd10: 'C62', icdo3: 'C620-C629' },
56 + { label: 'Vulva', sexSpecific: 'female', lookupCodes: ['C51'], relation: 'exact', icd10: 'C51', icdo3: 'C510-C519' },
57 + { label: 'Vagina', sexSpecific: 'female', lookupCodes: ['C52'], relation: 'exact', icd10: 'C52', icdo3: 'C529' },
58 + { label: 'Cervix', sexSpecific: 'female', lookupCodes: ['C53'], relation: 'exact', icd10: 'C53', icdo3: 'C530-C539' },
59 + { label: 'Corpus', sexSpecific: 'female', lookupCodes: ['C54'], relation: 'exact', icd10: 'C54', icdo3: 'C540-C549' },
60 + { label: 'Ovary', sexSpecific: 'female', lookupCodes: ['C56'], relation: 'exact', icd10: 'C56', icdo3: 'C569' },
61 + { label: 'Penis', sexSpecific: 'male', lookupCodes: ['C60'], relation: 'exact', icd10: 'C60', icdo3: 'C600-C609' },
62 + { label: 'Prostate', sexSpecific: 'male', lookupCodes: ['C61'], relation: 'exact', icd10: 'C61', icdo3: 'C619' },
63 + { label: 'Testis', sexSpecific: 'male', lookupCodes: ['C62'], relation: 'exact', icd10: 'C62', icdo3: 'C620-C629' },
58 64 { label: 'Kidney and Renal Pelvis', lookupCodes: ['C64', 'C65'], relation: 'exact', icd10: 'C64-C65', icdo3: 'C649, C659' },
59 65 { label: 'Urinary Bladder', lookupCodes: ['C67'], relation: 'exact', icd10: 'C67', icdo3: 'C670-C679' },
60 66 { label: 'Brain and Other Nervous System', lookupCodes: ['C70', 'C71', 'C72'], relation: 'exact', icd10: 'C70, C71, C72', icdo3: 'C700-C709, C710-C719, C720-C729' },
added packages/connectors/src/connectors/chembl/api.ts +350 −0
@@ -0,0 +1,350 @@
1 +import { z } from 'zod';
2 +import { normalizeLabel } from '@cancerindex/shared';
3 +
4 +export const BASE_URL = 'https://www.ebi.ac.uk/chembl/api/data';
5 +export const compoundUrl = (chemblId: string) => `https://www.ebi.ac.uk/chembl/compound_report_card/${chemblId}/`;
6 +export const targetUrl = (chemblId: string) => `https://www.ebi.ac.uk/chembl/target_report_card/${chemblId}/`;
7 +
8 +/* ------------------------------------------------------------------------------------------------ */
9 +/* Payload schemas (fields we rely on; everything else passes through into the raw record) */
10 +
11 +const numberish = z.union([z.number(), z.string().regex(/^-?\d+(\.\d+)?$/).transform(Number)]).nullable().optional();
12 +
13 +export const PageMeta = z.object({ limit: z.number(), offset: z.number(), total_count: z.number(), next: z.string().nullable().optional(), previous: z.string().nullable().optional() });
14 +
15 +export const Synonym = z.object({ molecule_synonym: z.string(), syn_type: z.string().nullable().optional(), synonyms: z.string().nullable().optional() }).passthrough();
16 +
17 +export const Molecule = z
18 + .object({
19 + molecule_chembl_id: z.string().regex(/^CHEMBL\d+$/),
20 + pref_name: z.string().nullable().optional(),
21 + molecule_type: z.string().nullable().optional(),
22 + max_phase: numberish,
23 + first_approval: z.number().nullable().optional(),
24 + molecule_synonyms: z.array(Synonym).nullable().optional(),
25 + molecule_hierarchy: z.object({ parent_chembl_id: z.string().nullable().optional(), molecule_chembl_id: z.string().nullable().optional(), active_chembl_id: z.string().nullable().optional() }).nullable().optional(),
26 + atc_classifications: z.array(z.string()).nullable().optional(),
27 + indication_class: z.string().nullable().optional(),
28 + withdrawn_flag: z.boolean().nullable().optional(),
29 + black_box_warning: z.union([z.number(), z.boolean()]).nullable().optional(),
30 + cross_references: z.array(z.object({ xref_src: z.string(), xref_id: z.string(), xref_name: z.string().nullable().optional() }).passthrough()).nullable().optional(),
31 + })
32 + .passthrough();
33 +export type Molecule = z.infer<typeof Molecule>;
34 +
35 +export const SearchPage = z.object({ page_meta: PageMeta, molecules: z.array(z.unknown()) });
36 +
37 +export const Mechanism = z
38 + .object({
39 + mec_id: z.number().nullable().optional(),
40 + molecule_chembl_id: z.string(),
41 + parent_molecule_chembl_id: z.string().nullable().optional(),
42 + mechanism_of_action: z.string().nullable().optional(),
43 + action_type: z.string().nullable().optional(),
44 + target_chembl_id: z.string().nullable().optional(),
45 + max_phase: numberish,
46 + direct_interaction: z.union([z.number(), z.boolean()]).nullable().optional(),
47 + disease_efficacy: z.union([z.number(), z.boolean()]).nullable().optional(),
48 + mechanism_comment: z.string().nullable().optional(),
49 + mechanism_refs: z.array(z.object({ ref_type: z.string().nullable().optional(), ref_id: z.string().nullable().optional(), ref_url: z.string().nullable().optional() }).passthrough()).nullable().optional(),
50 + })
51 + .passthrough();
52 +export type Mechanism = z.infer<typeof Mechanism>;
53 +export const MechanismPage = z.object({ page_meta: PageMeta, mechanisms: z.array(z.unknown()) });
54 +
55 +export const Target = z
56 + .object({
57 + target_chembl_id: z.string(),
58 + pref_name: z.string().nullable().optional(),
59 + target_type: z.string().nullable().optional(),
60 + organism: z.string().nullable().optional(),
61 + tax_id: z.number().nullable().optional(),
62 + target_components: z
63 + .array(
64 + z
65 + .object({
66 + accession: z.string().nullable().optional(),
67 + component_type: z.string().nullable().optional(),
68 + component_description: z.string().nullable().optional(),
69 + target_component_synonyms: z.array(z.object({ component_synonym: z.string(), syn_type: z.string().nullable().optional() }).passthrough()).nullable().optional(),
70 + })
71 + .passthrough(),
72 + )
73 + .nullable()
74 + .optional(),
75 + })
76 + .passthrough();
77 +export type Target = z.infer<typeof Target>;
78 +
79 +export const Indication = z
80 + .object({
81 + drugind_id: z.number().nullable().optional(),
82 + molecule_chembl_id: z.string(),
83 + parent_molecule_chembl_id: z.string().nullable().optional(),
84 + mesh_id: z.string().regex(/^[CD]\d{6,9}$/),
85 + mesh_heading: z.string(),
86 + efo_id: z.string().nullable().optional(),
87 + efo_term: z.string().nullable().optional(),
88 + max_phase_for_ind: numberish,
89 + })
90 + .passthrough();
91 +export type Indication = z.infer<typeof Indication>;
92 +export const IndicationPage = z.object({ page_meta: PageMeta, drug_indications: z.array(z.unknown()) });
93 +
94 +export const Status = z.object({ status: z.string(), chembl_db_version: z.string(), chembl_release_date: z.string().optional(), disinct_compounds: z.number().optional(), targets: z.number().optional() }).passthrough();
95 +
96 +/* ------------------------------------------------------------------------------------------------ */
97 +/* URLs */
98 +
99 +export const searchUrl = (base: string, name: string, limit = 20) => `${base}/molecule/search.json?q=${encodeURIComponent(name)}&limit=${limit}`;
100 +export const mechanismsUrl = (base: string, parentId: string) => `${base}/mechanism.json?parent_molecule_chembl_id=${encodeURIComponent(parentId)}&limit=200`;
101 +export const targetFetchUrl = (base: string, targetId: string) => `${base}/target/${encodeURIComponent(targetId)}.json`;
102 +export const indicationsUrl = (base: string, parentId: string, offset = 0) => `${base}/drug_indication.json?parent_molecule_chembl_id=${encodeURIComponent(parentId)}&limit=500&offset=${offset}`;
103 +export const statusUrl = (base: string) => `${base}/status.json`;
104 +
105 +/* ------------------------------------------------------------------------------------------------ */
106 +/* Pure mapping helpers */
107 +
108 +/** Split a search page into valid molecules and malformed rows (never dropped silently). */
109 +export function parseMolecules(page: unknown): { molecules: Molecule[]; malformed: number; total: number } {
110 + const p = SearchPage.safeParse(page);
111 + if (!p.success) throw new Error(`malformed search response: ${p.error.issues[0]?.message}`);
112 + const molecules: Molecule[] = [];
113 + let malformed = 0;
114 + for (const raw of p.data.molecules) {
115 + const m = Molecule.safeParse(raw);
116 + if (m.success) molecules.push(m.data);
117 + else malformed++;
118 + }
119 + return { molecules, malformed, total: p.data.page_meta.total_count };
120 +}
121 +
122 +export interface ExactMatch {
123 + molecule: Molecule;
124 + by: 'pref_name' | 'synonym';
125 + matchedText: string;
126 +}
127 +
128 +const fold = (s: string) => normalizeLabel(s);
129 +
130 +/**
131 + * Exact-name assignment only (never fuzzy): the hit whose pref_name — or, failing that, one of its
132 + * synonyms — equals our name after normalization. Ties: pref_name beats synonym, then higher
133 + * max_phase, then the parent molecule (hierarchy parent = itself), then the lowest ChEMBL number.
134 + */
135 +export function pickExactMatch(name: string, molecules: Molecule[]): ExactMatch | null {
136 + const target = fold(name);
137 + if (!target) return null;
138 + const candidates: Array<ExactMatch & { phase: number; isParent: boolean; num: number }> = [];
139 + for (const m of molecules) {
140 + let hit: ExactMatch | null = null;
141 + if (m.pref_name && fold(m.pref_name) === target) hit = { molecule: m, by: 'pref_name', matchedText: m.pref_name };
142 + else {
143 + const syn = (m.molecule_synonyms ?? []).find((s) => fold(s.molecule_synonym) === target);
144 + if (syn) hit = { molecule: m, by: 'synonym', matchedText: syn.molecule_synonym };
145 + }
146 + if (!hit) continue;
147 + const parent = m.molecule_hierarchy?.parent_chembl_id;
148 + candidates.push({ ...hit, phase: m.max_phase ?? -2, isParent: !parent || parent === m.molecule_chembl_id, num: Number(m.molecule_chembl_id.slice(6)) });
149 + }
150 + if (!candidates.length) return null;
151 + candidates.sort((a, b) => (a.by === b.by ? 0 : a.by === 'pref_name' ? -1 : 1) || b.phase - a.phase || Number(b.isParent) - Number(a.isParent) || a.num - b.num);
152 + const { molecule, by, matchedText } = candidates[0]!;
153 + return { molecule, by, matchedText };
154 +}
155 +
156 +/** ChEMBL molecule_type → drugs.kind vocabulary (null = leave the existing value untouched). */
157 +export function mapKind(moleculeType: string | null | undefined): string | null {
158 + switch ((moleculeType ?? '').trim()) {
159 + case 'Small molecule':
160 + return 'small_molecule';
161 + case 'Antibody':
162 + return 'monoclonal_antibody';
163 + case 'Antibody drug conjugate':
164 + return 'adc';
165 + case 'Cell':
166 + return 'cell_therapy';
167 + case 'Gene':
168 + return 'gene_therapy';
169 + case 'Protein':
170 + case 'Enzyme':
171 + case 'Oligonucleotide':
172 + case 'Oligosaccharide':
173 + case 'Polymer':
174 + case 'Inorganic':
175 + return 'other';
176 + case 'Unknown':
177 + case '':
178 + return null;
179 + default:
180 + return 'other';
181 + }
182 +}
183 +
184 +/** ChEMBL max_phase → drugs.development_status (null = unknown / not set). */
185 +export function mapPhase(maxPhase: number | null | undefined): string | null {
186 + if (maxPhase == null || maxPhase < 0.5) return null;
187 + if (maxPhase >= 4) return 'approved';
188 + if (maxPhase >= 3) return 'phase_3';
189 + if (maxPhase >= 2) return 'phase_2';
190 + if (maxPhase >= 1) return 'phase_1';
191 + return 'early_phase_1';
192 +}
193 +const PHASE_RANK: Record<string, number> = { early_phase_1: 1, phase_1: 2, phase_2: 3, phase_3: 4, approved: 5 };
194 +/** Never downgrade a status recorded by a regulatory source. */
195 +export function higherStatus(existing: string | null | undefined, incoming: string | null): string | null {
196 + if (!incoming) return existing ?? null;
197 + if (!existing) return incoming;
198 + return (PHASE_RANK[incoming] ?? 0) > (PHASE_RANK[existing] ?? 0) ? incoming : existing;
199 +}
200 +
201 +/** ChEMBL syn_type → drug_aliases.alias_type. */
202 +export function mapSynonymType(synType: string | null | undefined): 'brand' | 'generic' | 'development_code' | 'synonym' {
203 + switch ((synType ?? '').toUpperCase()) {
204 + case 'TRADE_NAME':
205 + return 'brand';
206 + case 'INN':
207 + case 'USAN':
208 + case 'BAN':
209 + case 'JAN':
210 + case 'USP':
211 + return 'generic';
212 + case 'RESEARCH_CODE':
213 + return 'development_code';
214 + default:
215 + return 'synonym';
216 + }
217 +}
218 +
219 +export interface AliasSpec {
220 + alias: string;
221 + normalized: string;
222 + aliasType: ReturnType<typeof mapSynonymType>;
223 +}
224 +
225 +/** Distinct aliases from ChEMBL synonyms; the drug's own name is skipped (already its generic alias). */
226 +export function aliasesFromMolecule(m: Molecule, drugName: string): AliasSpec[] {
227 + const own = fold(drugName);
228 + const seen = new Set<string>();
229 + const out: AliasSpec[] = [];
230 + const push = (alias: string, aliasType: AliasSpec['aliasType']) => {
231 + const a = alias.trim();
232 + const normalized = fold(a);
233 + if (!normalized || normalized === own || a.length > 200) return;
234 + const key = `${normalized}|${aliasType}`;
235 + if (seen.has(key)) return;
236 + seen.add(key);
237 + out.push({ alias: a, normalized, aliasType });
238 + };
239 + for (const s of m.molecule_synonyms ?? []) push(s.molecule_synonym, mapSynonymType(s.syn_type));
240 + return out;
241 +}
242 +
243 +export interface MechanismSpec {
244 + mechanismOfAction: string | null;
245 + actionType: string | null;
246 + targetChemblId: string | null;
247 + maxPhase: number | null;
248 + comment: string | null;
249 + refs: Array<{ type: string | null; id: string | null; url: string | null }>;
250 +}
251 +
252 +/** Parse + dedupe mechanisms (ChEMBL repeats one mechanism per dosed form: free base, salts). */
253 +export function dedupeMechanisms(page: unknown): { mechanisms: MechanismSpec[]; malformed: number } {
254 + const p = MechanismPage.safeParse(page);
255 + if (!p.success) throw new Error(`malformed mechanism response: ${p.error.issues[0]?.message}`);
256 + const seen = new Map<string, MechanismSpec>();
257 + let malformed = 0;
258 + for (const raw of p.data.mechanisms) {
259 + const r = Mechanism.safeParse(raw);
260 + if (!r.success) {
261 + malformed++;
262 + continue;
263 + }
264 + const m = r.data;
265 + const key = `${(m.mechanism_of_action ?? '').toLowerCase()}|${m.target_chembl_id ?? ''}|${m.action_type ?? ''}`;
266 + const prev = seen.get(key);
267 + const spec: MechanismSpec = {
268 + mechanismOfAction: m.mechanism_of_action?.trim() || null,
269 + actionType: m.action_type ?? null,
270 + targetChemblId: m.target_chembl_id ?? null,
271 + maxPhase: m.max_phase ?? null,
272 + comment: m.mechanism_comment ?? null,
273 + refs: (m.mechanism_refs ?? []).map((x) => ({ type: x.ref_type ?? null, id: x.ref_id ?? null, url: x.ref_url ?? null })),
274 + };
275 + if (!prev) seen.set(key, spec);
276 + else if ((spec.maxPhase ?? -2) > (prev.maxPhase ?? -2)) seen.set(key, { ...spec, refs: [...prev.refs, ...spec.refs] });
277 + }
278 + return { mechanisms: [...seen.values()], malformed };
279 +}
280 +
281 +/** Human-readable mechanism text for drugs.mechanism ("A; B"). */
282 +export function mechanismText(mechs: MechanismSpec[]): string | null {
283 + const parts = [...new Set(mechs.map((m) => m.mechanismOfAction).filter((s): s is string => !!s))];
284 + return parts.length ? parts.join('; ') : null;
285 +}
286 +
287 +/** HGNC-style gene symbols of the human protein components of a target (GENE_SYMBOL synonyms). */
288 +export function geneSymbolsFromTarget(t: Target): string[] {
289 + if (t.organism && !/^homo sapiens$/i.test(t.organism)) return [];
290 + const out = new Set<string>();
291 + for (const c of t.target_components ?? []) {
292 + if (c.component_type && !/protein/i.test(c.component_type)) continue;
293 + for (const s of c.target_component_synonyms ?? []) if (s.syn_type === 'GENE_SYMBOL') out.add(s.component_synonym.trim());
294 + }
295 + return [...out];
296 +}
297 +
298 +export interface IndicationSpec {
299 + meshId: string;
300 + meshHeading: string;
301 + efoIds: string[];
302 + maxPhase: number | null;
303 +}
304 +
305 +/** Parse + dedupe indications by MeSH descriptor (highest phase wins; EFO/MONDO ids accumulated). */
306 +export function dedupeIndications(page: unknown): { indications: IndicationSpec[]; malformed: number; next: string | null } {
307 + const p = IndicationPage.safeParse(page);
308 + if (!p.success) throw new Error(`malformed drug_indication response: ${p.error.issues[0]?.message}`);
309 + const byMesh = new Map<string, IndicationSpec>();
310 + let malformed = 0;
311 + for (const raw of p.data.drug_indications) {
312 + const r = Indication.safeParse(raw);
313 + if (!r.success) {
314 + malformed++;
315 + continue;
316 + }
317 + const i = r.data;
318 + const prev = byMesh.get(i.mesh_id);
319 + const phase = i.max_phase_for_ind ?? null;
320 + if (!prev) byMesh.set(i.mesh_id, { meshId: i.mesh_id, meshHeading: i.mesh_heading, efoIds: i.efo_id ? [i.efo_id] : [], maxPhase: phase });
321 + else {
322 + if (i.efo_id && !prev.efoIds.includes(i.efo_id)) prev.efoIds.push(i.efo_id);
323 + if ((phase ?? -2) > (prev.maxPhase ?? -2)) prev.maxPhase = phase;
324 + }
325 + }
326 + return { indications: [...byMesh.values()], malformed, next: p.data.page_meta.next ?? null };
327 +}
328 +
329 +/** Merge indication pages (dedupe across pages). */
330 +export function mergeIndications(pages: IndicationSpec[][]): IndicationSpec[] {
331 + const byMesh = new Map<string, IndicationSpec>();
332 + for (const page of pages)
333 + for (const i of page) {
334 + const prev = byMesh.get(i.meshId);
335 + if (!prev) byMesh.set(i.meshId, { ...i, efoIds: [...i.efoIds] });
336 + else {
337 + for (const e of i.efoIds) if (!prev.efoIds.includes(e)) prev.efoIds.push(e);
338 + if ((i.maxPhase ?? -2) > (prev.maxPhase ?? -2)) prev.maxPhase = i.maxPhase;
339 + }
340 + }
341 + return [...byMesh.values()];
342 +}
343 +
344 +/** Regimens, combinations and procedures are not molecules — ChEMBL cannot describe them. */
345 +export function isNonMoleculeName(name: string): boolean {
346 + return /\bregimen\b/i.test(name) || name.includes('/') || /\b(therapy|radiation|radiotherapy|transplant(ation)?|surgery|chemotherapy|placebo)\b/i.test(name);
347 +}
348 +
349 +/** Heuristic used only to decide whether an unresolved indication heading belongs in the *cancer* curation queue. */
350 +export const NEOPLASM_HEADING_RE = /neoplasm|cancer|carcinoma|tumou?r|leuk(a)?emia|lymphoma|sarcoma|melanoma|myeloma|glioma|blastoma|adenoma|mesothelioma|myelodysplastic|myeloproliferative|macroglobulinemia|plasmacytoma|thymoma|seminoma|teratoma|histiocytosis|mycosis fungoides|sezary|polycythemia vera|thrombocythemia|myelofibrosis|mastocytosis|gammopathy/i;
added packages/connectors/src/connectors/chembl/chembl.test.ts +213 −0
@@ -0,0 +1,213 @@
1 +import { readFileSync } from 'node:fs';
2 +import path from 'node:path';
3 +import { describe, expect, it } from 'vitest';
4 +import { manifest } from './manifest.js';
5 +import { NEOPLASM_HEADING_RE, Status, Target, aliasesFromMolecule, dedupeIndications, dedupeMechanisms, geneSymbolsFromTarget, higherStatus, indicationsUrl, isNonMoleculeName, mapKind, mapPhase, mapSynonymType, mechanismText, mechanismsUrl, mergeIndications, parseMolecules, pickExactMatch, searchUrl } from './api.js';
6 +
7 +const fx = (name: string) => JSON.parse(readFileSync(path.join(import.meta.dirname, 'fixtures', name), 'utf8')) as unknown;
8 +const search = (drug: string) => parseMolecules(fx(`search-${drug}.json`));
9 +const target = (id: string) => Target.parse(fx(`target-${id}.json`));
10 +
11 +describe('chembl manifest', () => {
12 + it('is approved CC BY-SA 3.0 with attribution + share-alike note and verified docs', () => {
13 + expect(manifest.id).toBe('chembl');
14 + expect(manifest.licenseStatus).toBe('approved');
15 + expect(manifest.license).toBe('CC BY-SA 3.0');
16 + expect(manifest.redistribution).toBe('attribution');
17 + expect(manifest.termsNotes).toMatch(/share-alike/i);
18 + expect(manifest.documentationVerifiedAt).toBe('2026-09-08');
19 + expect(manifest.rateLimits).toMatchObject({ requestsPerSecond: 3, maxConcurrency: 2 });
20 + });
21 +});
22 +
23 +describe('URLs', () => {
24 + it('encodes names and uses the parent molecule for mechanisms / indications', () => {
25 + expect(searchUrl('https://x/api/data', '5-Fluorouracil/Leucovorin')).toBe('https://x/api/data/molecule/search.json?q=5-Fluorouracil%2FLeucovorin&limit=20');
26 + expect(mechanismsUrl('https://x', 'CHEMBL941')).toContain('mechanism.json?parent_molecule_chembl_id=CHEMBL941');
27 + expect(indicationsUrl('https://x', 'CHEMBL941', 500)).toContain('drug_indication.json?parent_molecule_chembl_id=CHEMBL941&limit=500&offset=500');
28 + });
29 +});
30 +
31 +describe('search page → exact match', () => {
32 + it('osimertinib: picks the free base by pref_name over the mesylate (synonym) and the unnamed hits', () => {
33 + const { molecules, malformed, total } = search('osimertinib');
34 + expect(malformed).toBe(0);
35 + expect(total).toBe(3);
36 + const hit = pickExactMatch('Osimertinib', molecules)!;
37 + expect(hit.molecule.molecule_chembl_id).toBe('CHEMBL3353410');
38 + expect(hit.by).toBe('pref_name');
39 + expect(hit.molecule.max_phase).toBe(4);
40 + expect(hit.molecule.first_approval).toBe(2015);
41 + });
42 + it('imatinib: pref_name IMATINIB wins over IMATINIB MESYLATE; "Imatinib Mesylate" matches the salt', () => {
43 + const { molecules } = search('imatinib');
44 + expect(pickExactMatch('Imatinib', molecules)!.molecule.molecule_chembl_id).toBe('CHEMBL941');
45 + expect(pickExactMatch('imatinib mesylate', molecules)!.molecule.molecule_chembl_id).toBe('CHEMBL1642');
46 + // brand as synonym on both forms (ChEMBL lists GLEEVEC on the parent and the mesylate) → parent wins the tie
47 + const gleevec = pickExactMatch('Gleevec', molecules)!;
48 + expect(gleevec.by).toBe('synonym');
49 + expect(gleevec.molecule.molecule_chembl_id).toBe('CHEMBL941');
50 + // brand only on the salt → the salt
51 + expect(pickExactMatch('Imatinib accord', molecules)!.molecule.molecule_chembl_id).toBe('CHEMBL1642');
52 + });
53 + it('trastuzumab: the antibody, not its ADCs', () => {
54 + const { molecules } = search('trastuzumab');
55 + const hit = pickExactMatch('Trastuzumab', molecules)!;
56 + expect(hit.molecule.molecule_chembl_id).toBe('CHEMBL1201585');
57 + expect(hit.molecule.molecule_type).toBe('Antibody');
58 + expect(pickExactMatch('Trastuzumab Deruxtecan', molecules)!.molecule.molecule_chembl_id).toBe('CHEMBL4297844');
59 + });
60 + it('no exact match → null (never fuzzy-assigned); empty page → null', () => {
61 + const { molecules } = search('trastuzumab');
62 + expect(pickExactMatch('Trastuzumab biosimilar XYZ', molecules)).toBeNull();
63 + const none = search('none');
64 + expect(none.total).toBe(0);
65 + expect(pickExactMatch('Zzqxnotadrug99', none.molecules)).toBeNull();
66 + });
67 + it('counts malformed molecules instead of dropping the page', () => {
68 + const { molecules, malformed } = search('fluorouracil');
69 + expect(malformed).toBe(1);
70 + expect(molecules).toHaveLength(1);
71 + expect(pickExactMatch('Fluorouracil', molecules)!.molecule.molecule_chembl_id).toBe('CHEMBL185');
72 + });
73 + it('rejects a malformed page (server error body)', () => {
74 + expect(() => parseMolecules({ error_message: 'Internal Server Error' })).toThrow(/malformed search response/);
75 + });
76 +});
77 +
78 +describe('vocabulary mapping', () => {
79 + it('molecule_type → kind (Unknown keeps the existing value)', () => {
80 + expect(mapKind('Small molecule')).toBe('small_molecule');
81 + expect(mapKind('Antibody')).toBe('monoclonal_antibody');
82 + expect(mapKind('Antibody drug conjugate')).toBe('adc');
83 + expect(mapKind('Protein')).toBe('other');
84 + expect(mapKind('Oligonucleotide')).toBe('other');
85 + expect(mapKind('Unknown')).toBeNull();
86 + expect(mapKind(null)).toBeNull();
87 + });
88 + it('max_phase → development_status, never downgrading', () => {
89 + expect(mapPhase(4)).toBe('approved');
90 + expect(mapPhase(3)).toBe('phase_3');
91 + expect(mapPhase(2)).toBe('phase_2');
92 + expect(mapPhase(1)).toBe('phase_1');
93 + expect(mapPhase(0.5)).toBe('early_phase_1');
94 + expect(mapPhase(null)).toBeNull();
95 + expect(mapPhase(-1)).toBeNull();
96 + expect(higherStatus('approved', 'phase_3')).toBe('approved');
97 + expect(higherStatus('phase_1', 'phase_3')).toBe('phase_3');
98 + expect(higherStatus(null, null)).toBeNull();
99 + });
100 + it('syn_type → alias_type', () => {
101 + expect(mapSynonymType('TRADE_NAME')).toBe('brand');
102 + expect(mapSynonymType('INN')).toBe('generic');
103 + expect(mapSynonymType('RESEARCH_CODE')).toBe('development_code');
104 + expect(mapSynonymType('OTHER')).toBe('synonym');
105 + expect(mapSynonymType(null)).toBe('synonym');
106 + });
107 + it('aliases from synonyms: brand Tagrisso, research code AZD9291, own name skipped, deduped per type', () => {
108 + const { molecules } = search('osimertinib');
109 + const osi = pickExactMatch('Osimertinib', molecules)!.molecule;
110 + const aliases = aliasesFromMolecule(osi, 'Osimertinib');
111 + expect(aliases).toContainEqual({ alias: 'Tagrisso', normalized: 'tagrisso', aliasType: 'brand' });
112 + expect(aliases).toContainEqual({ alias: 'AZD-9291', normalized: 'azd 9291', aliasType: 'development_code' });
113 + expect(aliases.find((a) => a.normalized === 'osimertinib')).toBeUndefined();
114 + expect(new Set(aliases.map((a) => `${a.normalized}|${a.aliasType}`)).size).toBe(aliases.length);
115 + });
116 + it('non-molecule names are skipped (regimens, combinations, procedures)', () => {
117 + expect(isNonMoleculeName('Abemaciclib Regimen')).toBe(true);
118 + expect(isNonMoleculeName('Pertuzumab/Trastuzumab Regimen')).toBe(true);
119 + expect(isNonMoleculeName('3-Dimensional Conformal Radiation Therapy')).toBe(true);
120 + expect(isNonMoleculeName('Osimertinib')).toBe(false);
121 + expect(isNonMoleculeName('5-Fluoro-2-Deoxycytidine')).toBe(false);
122 + });
123 +});
124 +
125 +describe('mechanisms', () => {
126 + it('osimertinib: one EGFR inhibitor mechanism with refs', () => {
127 + const { mechanisms, malformed } = dedupeMechanisms(fx('mechanism-osimertinib.json'));
128 + expect(malformed).toBe(0);
129 + expect(mechanisms).toHaveLength(1);
130 + expect(mechanisms[0]).toMatchObject({ mechanismOfAction: 'Epidermal growth factor receptor erbB1 inhibitor', actionType: 'INHIBITOR', targetChemblId: 'CHEMBL203', maxPhase: 4 });
131 + expect(mechanisms[0]!.refs.some((r) => r.type === 'PubMed')).toBe(true);
132 + expect(mechanismText(mechanisms)).toBe('Epidermal growth factor receptor erbB1 inhibitor');
133 + });
134 + it('imatinib: four mechanisms attached to the mesylate, joined into one mechanism text', () => {
135 + const { mechanisms } = dedupeMechanisms(fx('mechanism-imatinib.json'));
136 + expect(mechanisms).toHaveLength(4);
137 + expect(mechanisms.map((m) => m.targetChemblId).sort()).toEqual(['CHEMBL1862', 'CHEMBL1913', 'CHEMBL1936', 'CHEMBL2096618']);
138 + expect(mechanisms.every((m) => m.actionType === 'INHIBITOR')).toBe(true);
139 + expect(mechanismText(mechanisms)).toBe('Tyrosine-protein kinase ABL inhibitor; Platelet-derived growth factor receptor beta inhibitor; Stem cell growth factor receptor inhibitor; Bcr/Abl fusion protein inhibitor');
140 + });
141 + it('dedupes the same mechanism repeated for several dosed forms', () => {
142 + const raw = fx('mechanism-imatinib.json') as { mechanisms: unknown[] };
143 + const doubled = { page_meta: { limit: 20, offset: 0, total_count: 8 }, mechanisms: [...raw.mechanisms, ...raw.mechanisms.map((m) => ({ ...(m as object), molecule_chembl_id: 'CHEMBL941' }))] };
144 + expect(dedupeMechanisms(doubled).mechanisms).toHaveLength(4);
145 + });
146 + it('empty and malformed pages', () => {
147 + expect(dedupeMechanisms({ page_meta: { limit: 20, offset: 0, total_count: 0 }, mechanisms: [] })).toEqual({ mechanisms: [], malformed: 0 });
148 + expect(dedupeMechanisms({ page_meta: { limit: 20, offset: 0, total_count: 1 }, mechanisms: [{ mechanism_of_action: 'x' }] }).malformed).toBe(1);
149 + expect(() => dedupeMechanisms({ mechanisms: 'nope' })).toThrow(/malformed mechanism response/);
150 + expect(mechanismText([])).toBeNull();
151 + });
152 +});
153 +
154 +describe('targets → gene symbols', () => {
155 + it('single protein: EGFR, ABL1, KIT, ERBB2, PDGFRB', () => {
156 + expect(geneSymbolsFromTarget(target('CHEMBL203'))).toEqual(['EGFR']);
157 + expect(geneSymbolsFromTarget(target('CHEMBL1862'))).toEqual(['ABL1']);
158 + expect(geneSymbolsFromTarget(target('CHEMBL1936'))).toEqual(['KIT']);
159 + expect(geneSymbolsFromTarget(target('CHEMBL1824'))).toEqual(['ERBB2']);
160 + expect(geneSymbolsFromTarget(target('CHEMBL1913'))).toEqual(['PDGFRB']);
161 + });
162 + it('chimeric protein (Bcr/Abl) and protein family (ABL) expand to every component gene', () => {
163 + expect(geneSymbolsFromTarget(target('CHEMBL2096618')).sort()).toEqual(['ABL1', 'BCR']);
164 + expect(geneSymbolsFromTarget(target('CHEMBL2111414')).sort()).toEqual(['ABL1', 'ABL2']);
165 + });
166 + it('non-human targets yield no genes', () => {
167 + const t = { ...target('CHEMBL203'), organism: 'Mus musculus' };
168 + expect(geneSymbolsFromTarget(t)).toEqual([]);
169 + });
170 +});
171 +
172 +describe('indications', () => {
173 + it('osimertinib: deduped by MeSH UI with the highest phase (Lung Neoplasms listed twice)', () => {
174 + const { indications, malformed, next } = dedupeIndications(fx('indication-osimertinib.json'));
175 + expect(malformed).toBe(0);
176 + expect(next).toBeNull();
177 + expect(indications).toHaveLength(10); // 12 rows: Lung Neoplasms and Neoplasms each listed twice (EFO + MONDO)
178 + const lung = indications.find((i) => i.meshId === 'D008175')!;
179 + expect(lung).toMatchObject({ meshHeading: 'Lung Neoplasms', maxPhase: 3 });
180 + expect(lung.efoIds.sort()).toEqual(['MONDO:0008903', 'MONDO:0021117']);
181 + expect(indications.find((i) => i.meshId === 'D002289')!.maxPhase).toBe(3); // "3.0" string coerced
182 + expect(indications.find((i) => i.meshId === 'D009369')!.maxPhase).toBe(4);
183 + });
184 + it('imatinib: 52 rows collapse to distinct descriptors; non-cancer headings are recognisable', () => {
185 + const { indications } = dedupeIndications(fx('indication-imatinib.json'));
186 + expect(indications.length).toBeLessThan(52);
187 + expect(indications.length).toBeGreaterThan(20);
188 + expect(NEOPLASM_HEADING_RE.test('Arthritis, Rheumatoid')).toBe(false);
189 + expect(NEOPLASM_HEADING_RE.test('Hypertension, Pulmonary')).toBe(false);
190 + expect(NEOPLASM_HEADING_RE.test('Leukemia, Myelogenous, Chronic, BCR-ABL Positive')).toBe(true);
191 + expect(NEOPLASM_HEADING_RE.test('Hypereosinophilic Syndrome')).toBe(false);
192 + expect(NEOPLASM_HEADING_RE.test('Desmoplastic Small Round Cell Tumor')).toBe(true);
193 + });
194 + it('merges pages and keeps the max phase across them', () => {
195 + const merged = mergeIndications([
196 + [{ meshId: 'D008175', meshHeading: 'Lung Neoplasms', efoIds: ['EFO:1'], maxPhase: 2 }],
197 + [{ meshId: 'D008175', meshHeading: 'Lung Neoplasms', efoIds: ['EFO:2'], maxPhase: 3 }, { meshId: 'D009369', meshHeading: 'Neoplasms', efoIds: [], maxPhase: null }],
198 + ]);
199 + expect(merged).toHaveLength(2);
200 + expect(merged[0]).toEqual({ meshId: 'D008175', meshHeading: 'Lung Neoplasms', efoIds: ['EFO:1', 'EFO:2'], maxPhase: 3 });
201 + });
202 + it('malformed rows counted, malformed page rejected', () => {
203 + expect(dedupeIndications({ page_meta: { limit: 20, offset: 0, total_count: 1 }, drug_indications: [{ mesh_id: 'not-a-ui', mesh_heading: 'x', molecule_chembl_id: 'CHEMBL1' }] }).malformed).toBe(1);
204 + expect(() => dedupeIndications({ drug_indications: null })).toThrow(/malformed drug_indication response/);
205 + });
206 +});
207 +
208 +describe('status', () => {
209 + it('parses the status payload', () => {
210 + const s = Status.parse({ activities: 24527044, chembl_db_version: 'ChEMBL_37', chembl_release_date: '2026-05-01', compound_records: 3824604, disinct_compounds: 2921148, publications: 101100, status: 'UP', targets: 18552 });
211 + expect(s.chembl_db_version).toBe('ChEMBL_37');
212 + });
213 +});
added packages/connectors/src/connectors/chembl/fixtures/indication-imatinib.json +895 −0
@@ -0,0 +1,895 @@
1 +{
2 + "page_meta": {
3 + "limit": 100,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 52
8 + },
9 + "drug_indications": [
10 + {
11 + "drugind_id": 68560,
12 + "molecule_chembl_id": "CHEMBL941",
13 + "parent_molecule_chembl_id": "CHEMBL941",
14 + "mesh_id": "D054198",
15 + "mesh_heading": "Precursor Cell Lymphoblastic Leukemia-Lymphoma",
16 + "efo_id": "EFO:0000220",
17 + "efo_term": "acute lymphoblastic leukemia",
18 + "max_phase_for_ind": "3.0",
19 + "indication_refs": [
20 + {
21 + "ref_id": "NCT00130195,NCT00137111,NCT00149136,NCT00287105,NCT00376467,NCT00476190,NCT00973752,NCT02081378,NCT02881086,NCT03589326,NCT04307576,NCT04688983,NCT04722848,NCT06051409,NCT06061094",
22 + "ref_type": "ClinicalTrials",
23 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00130195%20NCT00137111%20NCT00149136%20NCT00287105%20NCT00376467%20NCT00476190%20NCT00973752%20NCT02081378%20NCT02881086%20NCT03589326%20NCT04307576%20NCT04688983%20NCT04722848%20NCT06051409%20NCT06061094"
24 + }
25 + ]
26 + },
27 + {
28 + "drugind_id": 68561,
29 + "molecule_chembl_id": "CHEMBL941",
30 + "parent_molecule_chembl_id": "CHEMBL941",
31 + "mesh_id": "D015470",
32 + "mesh_heading": "Leukemia, Myeloid, Acute",
33 + "efo_id": "EFO:0000222",
34 + "efo_term": "acute myeloid leukemia",
35 + "max_phase_for_ind": "2.0",
36 + "indication_refs": [
37 + {
38 + "ref_id": "NCT00707408,NCT02638428",
39 + "ref_type": "ClinicalTrials",
40 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00707408%20NCT02638428"
41 + }
42 + ]
43 + },
44 + {
45 + "drugind_id": 68565,
46 + "molecule_chembl_id": "CHEMBL941",
47 + "parent_molecule_chembl_id": "CHEMBL941",
48 + "mesh_id": "D015464",
49 + "mesh_heading": "Leukemia, Myelogenous, Chronic, BCR-ABL Positive",
50 + "efo_id": "EFO:0000339",
51 + "efo_term": "chronic myelogenous leukemia",
52 + "max_phase_for_ind": "3.0",
53 + "indication_refs": [
54 + {
55 + "ref_id": "NCT00103844,NCT00219739,NCT00237120,NCT00297570,NCT00320190,NCT00324077,NCT00327262,NCT00333840,NCT00428909,NCT00471497,NCT00481247,NCT00510926,NCT00514969,NCT00574873,NCT00718263,NCT00760877,NCT00802841,NCT00852566,NCT00858806,NCT01011998,NCT01188889,NCT01216085,NCT01275196,NCT01400074,NCT01460693,NCT01503502,NCT01593254,NCT01650805,NCT01795716,NCT01804985,NCT01819389,NCT02001818,NCT02081378,NCT02130557,NCT02174445,NCT02204644,NCT02269267,NCT02326311,NCT02480608,NCT03193281,NCT03228303,NCT03515018,NCT03578367,NCT03654768,NCT03722420,NCT03906292,NCT04070443,NCT04126681,NCT04147533,NCT04578847,NCT04838041,NCT04971226,NCT05413915",
56 + "ref_type": "ClinicalTrials",
57 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00103844%20NCT00219739%20NCT00237120%20NCT00297570%20NCT00320190%20NCT00324077%20NCT00327262%20NCT00333840%20NCT00428909%20NCT00471497%20NCT00481247%20NCT00510926%20NCT00514969%20NCT00574873%20NCT00718263%20NCT00760877%20NCT00802841%20NCT00852566%20NCT00858806%20NCT01011998%20NCT01188889%20NCT01216085%20NCT01275196%20NCT01400074%20NCT01460693%20NCT01503502%20NCT01593254%20NCT01650805%20NCT01795716%20NCT01804985%20NCT01819389%20NCT02001818%20NCT02081378%20NCT02130557%20NCT02174445%20NCT02204644%20NCT02269267%20NCT02326311%20NCT02480608%20NCT03193281%20NCT03228303%20NCT03515018%20NCT03578367%20NCT03654768%20NCT03722420%20NCT03906292%20NCT04070443%20NCT04126681%20NCT04147533%20NCT04578847%20NCT04838041%20NCT04971226%20NCT05413915"
58 + }
59 + ]
60 + },
61 + {
62 + "drugind_id": 68566,
63 + "molecule_chembl_id": "CHEMBL941",
64 + "parent_molecule_chembl_id": "CHEMBL941",
65 + "mesh_id": "D002292",
66 + "mesh_heading": "Carcinoma, Renal Cell",
67 + "efo_id": "EFO:0000349",
68 + "efo_term": "clear cell renal carcinoma",
69 + "max_phase_for_ind": "1.0",
70 + "indication_refs": [
71 + {
72 + "ref_id": "NCT00193258",
73 + "ref_type": "ClinicalTrials",
74 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00193258"
75 + }
76 + ]
77 + },
78 + {
79 + "drugind_id": 68567,
80 + "molecule_chembl_id": "CHEMBL941",
81 + "parent_molecule_chembl_id": "CHEMBL941",
82 + "mesh_id": "D015179",
83 + "mesh_heading": "Colorectal Neoplasms",
84 + "efo_id": "EFO:0000365",
85 + "efo_term": "colorectal adenocarcinoma",
86 + "max_phase_for_ind": "1.0",
87 + "indication_refs": [
88 + {
89 + "ref_id": "NCT00784446",
90 + "ref_type": "ClinicalTrials",
91 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00784446"
92 + }
93 + ]
94 + },
95 + {
96 + "drugind_id": 68568,
97 + "molecule_chembl_id": "CHEMBL941",
98 + "parent_molecule_chembl_id": "CHEMBL941",
99 + "mesh_id": "D005350",
100 + "mesh_heading": "Fibroma",
101 + "efo_id": "EFO:0000497",
102 + "efo_term": "fibromatosis",
103 + "max_phase_for_ind": "2.0",
104 + "indication_refs": [
105 + {
106 + "ref_id": "NCT01137916,NCT02495519",
107 + "ref_type": "ClinicalTrials",
108 + "ref_url": "https://clinicaltrials.gov/search?term=NCT01137916%20NCT02495519"
109 + }
110 + ]
111 + },
112 + {
113 + "drugind_id": 68569,
114 + "molecule_chembl_id": "CHEMBL941",
115 + "parent_molecule_chembl_id": "CHEMBL941",
116 + "mesh_id": "D013274",
117 + "mesh_heading": "Stomach Neoplasms",
118 + "efo_id": "EFO:0000503",
119 + "efo_term": "gastric adenocarcinoma",
120 + "max_phase_for_ind": "2.0",
121 + "indication_refs": [
122 + {
123 + "ref_id": "NCT00209079",
124 + "ref_type": "ClinicalTrials",
125 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00209079"
126 + }
127 + ]
128 + },
129 + {
130 + "drugind_id": 68570,
131 + "molecule_chembl_id": "CHEMBL941",
132 + "parent_molecule_chembl_id": "CHEMBL941",
133 + "mesh_id": "D005909",
134 + "mesh_heading": "Glioblastoma",
135 + "efo_id": "EFO:0000519",
136 + "efo_term": "glioblastoma multiforme",
137 + "max_phase_for_ind": "2.0",
138 + "indication_refs": [
139 + {
140 + "ref_id": "NCT00290771",
141 + "ref_type": "ClinicalTrials",
142 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00290771"
143 + }
144 + ]
145 + },
146 + {
147 + "drugind_id": 68571,
148 + "molecule_chembl_id": "CHEMBL941",
149 + "parent_molecule_chembl_id": "CHEMBL941",
150 + "mesh_id": "D007938",
151 + "mesh_heading": "Leukemia",
152 + "efo_id": "EFO:0000565",
153 + "efo_term": "leukemia",
154 + "max_phase_for_ind": "3.0",
155 + "indication_refs": [
156 + {
157 + "ref_id": "NCT00250042,NCT00362466,NCT00415857,NCT00982488,NCT01126814,NCT01319981,NCT02272777",
158 + "ref_type": "ClinicalTrials",
159 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00250042%20NCT00362466%20NCT00415857%20NCT00982488%20NCT01126814%20NCT01319981%20NCT02272777"
160 + }
161 + ]
162 + },
163 + {
164 + "drugind_id": 68572,
165 + "molecule_chembl_id": "CHEMBL941",
166 + "parent_molecule_chembl_id": "CHEMBL941",
167 + "mesh_id": "D009369",
168 + "mesh_heading": "Neoplasms",
169 + "efo_id": "EFO:0000616",
170 + "efo_term": "neoplasm",
171 + "max_phase_for_ind": "3.0",
172 + "indication_refs": [
173 + {
174 + "ref_id": "L01EA01",
175 + "ref_type": "ATC",
176 + "ref_url": "https://www.whocc.no/atc_ddd_index/?code=L01EA01"
177 + }
178 + ]
179 + },
180 + {
181 + "drugind_id": 68573,
182 + "molecule_chembl_id": "CHEMBL941",
183 + "parent_molecule_chembl_id": "CHEMBL941",
184 + "mesh_id": "D001172",
185 + "mesh_heading": "Arthritis, Rheumatoid",
186 + "efo_id": "EFO:0000685",
187 + "efo_term": "rheumatoid arthritis",
188 + "max_phase_for_ind": "2.0",
189 + "indication_refs": [
190 + {
191 + "ref_id": "NCT00154336",
192 + "ref_type": "ClinicalTrials",
193 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00154336"
194 + }
195 + ]
196 + },
197 + {
198 + "drugind_id": 68574,
199 + "molecule_chembl_id": "CHEMBL941",
200 + "parent_molecule_chembl_id": "CHEMBL941",
201 + "mesh_id": "D012509",
202 + "mesh_heading": "Sarcoma",
203 + "efo_id": "EFO:0000691",
204 + "efo_term": "sarcoma",
205 + "max_phase_for_ind": "3.0",
206 + "indication_refs": [
207 + {
208 + "ref_id": "NCT00116935,NCT02413736",
209 + "ref_type": "ClinicalTrials",
210 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00116935%20NCT02413736"
211 + }
212 + ]
213 + },
214 + {
215 + "drugind_id": 68575,
216 + "molecule_chembl_id": "CHEMBL941",
217 + "parent_molecule_chembl_id": "CHEMBL941",
218 + "mesh_id": "D012595",
219 + "mesh_heading": "Scleroderma, Systemic",
220 + "efo_id": "EFO:0000717",
221 + "efo_term": "systemic scleroderma",
222 + "max_phase_for_ind": "2.0",
223 + "indication_refs": [
224 + {
225 + "ref_id": "NCT00573326",
226 + "ref_type": "ClinicalTrials",
227 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00573326"
228 + }
229 + ]
230 + },
231 + {
232 + "drugind_id": 68576,
233 + "molecule_chembl_id": "CHEMBL941",
234 + "parent_molecule_chembl_id": "CHEMBL941",
235 + "mesh_id": "D008545",
236 + "mesh_heading": "Melanoma",
237 + "efo_id": "EFO:0000756",
238 + "efo_term": "melanoma",
239 + "max_phase_for_ind": "2.0",
240 + "indication_refs": [
241 + {
242 + "ref_id": "NCT00421317,NCT00424515,NCT01782508,NCT04598009",
243 + "ref_type": "ClinicalTrials",
244 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00421317%20NCT00424515%20NCT01782508%20NCT04598009"
245 + }
246 + ]
247 + },
248 + {
249 + "drugind_id": 68578,
250 + "molecule_chembl_id": "CHEMBL941",
251 + "parent_molecule_chembl_id": "CHEMBL941",
252 + "mesh_id": "D006976",
253 + "mesh_heading": "Hypertension, Pulmonary",
254 + "efo_id": "EFO:0001361",
255 + "efo_term": "pulmonary arterial hypertension",
256 + "max_phase_for_ind": "3.0",
257 + "indication_refs": [
258 + {
259 + "ref_id": "NCT01117987,NCT01392469,NCT01392495",
260 + "ref_type": "ClinicalTrials",
261 + "ref_url": "https://clinicaltrials.gov/search?term=NCT01117987%20NCT01392469%20NCT01392495"
262 + }
263 + ]
264 + },
265 + {
266 + "drugind_id": 68579,
267 + "molecule_chembl_id": "CHEMBL941",
268 + "parent_molecule_chembl_id": "CHEMBL941",
269 + "mesh_id": "D005354",
270 + "mesh_heading": "Fibrosarcoma",
271 + "efo_id": "EFO:0002087",
272 + "efo_term": "fibrosarcoma",
273 + "max_phase_for_ind": "1.0",
274 + "indication_refs": [
275 + {
276 + "ref_id": "NCT00122473",
277 + "ref_type": "ClinicalTrials",
278 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00122473"
279 + }
280 + ]
281 + },
282 + {
283 + "drugind_id": 68580,
284 + "molecule_chembl_id": "CHEMBL941",
285 + "parent_molecule_chembl_id": "CHEMBL941",
286 + "mesh_id": "D008545",
287 + "mesh_heading": "Melanoma",
288 + "efo_id": "EFO:0002617",
289 + "efo_term": "metastatic melanoma",
290 + "max_phase_for_ind": "2.0",
291 + "indication_refs": [
292 + {
293 + "ref_id": "NCT00402662,NCT00881049",
294 + "ref_type": "ClinicalTrials",
295 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00402662%20NCT00881049"
296 + }
297 + ]
298 + },
299 + {
300 + "drugind_id": 68583,
301 + "molecule_chembl_id": "CHEMBL941",
302 + "parent_molecule_chembl_id": "CHEMBL941",
303 + "mesh_id": "D017728",
304 + "mesh_heading": "Lymphoma, Large-Cell, Anaplastic",
305 + "efo_id": "EFO:0003032",
306 + "efo_term": "anaplastic large cell lymphoma",
307 + "max_phase_for_ind": "1.0",
308 + "indication_refs": [
309 + {
310 + "ref_id": "NCT02462538",
311 + "ref_type": "ClinicalTrials",
312 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02462538"
313 + }
314 + ]
315 + },
316 + {
317 + "drugind_id": 68584,
318 + "molecule_chembl_id": "CHEMBL941",
319 + "parent_molecule_chembl_id": "CHEMBL941",
320 + "mesh_id": "D013274",
321 + "mesh_heading": "Stomach Neoplasms",
322 + "efo_id": "EFO:0003897",
323 + "efo_term": "stomach neoplasm",
324 + "max_phase_for_ind": "2.0",
325 + "indication_refs": [
326 + {
327 + "ref_id": "NCT03170180",
328 + "ref_type": "ClinicalTrials",
329 + "ref_url": "https://clinicaltrials.gov/search?term=NCT03170180"
330 + }
331 + ]
332 + },
333 + {
334 + "drugind_id": 68585,
335 + "molecule_chembl_id": "CHEMBL941",
336 + "parent_molecule_chembl_id": "CHEMBL941",
337 + "mesh_id": "D006255",
338 + "mesh_heading": "Rhinitis, Allergic, Seasonal",
339 + "efo_id": "EFO:0003956",
340 + "efo_term": "seasonal allergic rhinitis",
341 + "max_phase_for_ind": "1.0",
342 + "indication_refs": [
343 + {
344 + "ref_id": "NCT00426179",
345 + "ref_type": "ClinicalTrials",
346 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00426179"
347 + }
348 + ]
349 + },
350 + {
351 + "drugind_id": 68586,
352 + "molecule_chembl_id": "CHEMBL941",
353 + "parent_molecule_chembl_id": "CHEMBL941",
354 + "mesh_id": "D003110",
355 + "mesh_heading": "Colonic Neoplasms",
356 + "efo_id": "EFO:0004288",
357 + "efo_term": "colonic neoplasm",
358 + "max_phase_for_ind": "2.0",
359 + "indication_refs": [
360 + {
361 + "ref_id": "NCT02685046",
362 + "ref_type": "ClinicalTrials",
363 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02685046"
364 + }
365 + ]
366 + },
367 + {
368 + "drugind_id": 68588,
369 + "molecule_chembl_id": "CHEMBL941",
370 + "parent_molecule_chembl_id": "CHEMBL941",
371 + "mesh_id": "D055371",
372 + "mesh_heading": "Acute Lung Injury",
373 + "efo_id": "EFO:0004610",
374 + "efo_term": "acute lung injury",
375 + "max_phase_for_ind": "1.0",
376 + "indication_refs": [
377 + {
378 + "ref_id": "NCT03328117",
379 + "ref_type": "ClinicalTrials",
380 + "ref_url": "https://clinicaltrials.gov/search?term=NCT03328117"
381 + }
382 + ]
383 + },
384 + {
385 + "drugind_id": 68590,
386 + "molecule_chembl_id": "CHEMBL941",
387 + "parent_molecule_chembl_id": "CHEMBL941",
388 + "mesh_id": "D008228",
389 + "mesh_heading": "Lymphoma, Non-Hodgkin",
390 + "efo_id": "EFO:0005952",
391 + "efo_term": "non-Hodgkins lymphoma",
392 + "max_phase_for_ind": "1.0",
393 + "indication_refs": [
394 + {
395 + "ref_id": "NCT02431403",
396 + "ref_type": "ClinicalTrials",
397 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02431403"
398 + }
399 + ]
400 + },
401 + {
402 + "drugind_id": 68591,
403 + "molecule_chembl_id": "CHEMBL941",
404 + "parent_molecule_chembl_id": "CHEMBL941",
405 + "mesh_id": "D016778",
406 + "mesh_heading": "Malaria, Falciparum",
407 + "efo_id": "EFO:0007444",
408 + "efo_term": "Plasmodium falciparum malaria",
409 + "max_phase_for_ind": "2.0",
410 + "indication_refs": [
411 + {
412 + "ref_id": "NCT02614404,NCT03697668",
413 + "ref_type": "ClinicalTrials",
414 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02614404%20NCT03697668"
415 + }
416 + ]
417 + },
418 + {
419 + "drugind_id": 68592,
420 + "molecule_chembl_id": "CHEMBL941",
421 + "parent_molecule_chembl_id": "CHEMBL941",
422 + "mesh_id": "D058405",
423 + "mesh_heading": "Desmoplastic Small Round Cell Tumor",
424 + "efo_id": "EFO:1000895",
425 + "efo_term": "desmoplastic small round cell tumor",
426 + "max_phase_for_ind": "1.0",
427 + "indication_refs": [
428 + {
429 + "ref_id": "NCT00417807",
430 + "ref_type": "ClinicalTrials",
431 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00417807"
432 + }
433 + ]
434 + },
435 + {
436 + "drugind_id": 111878,
437 + "molecule_chembl_id": "CHEMBL941",
438 + "parent_molecule_chembl_id": "CHEMBL941",
439 + "mesh_id": "D013964",
440 + "mesh_heading": "Thyroid Neoplasms",
441 + "efo_id": "EFO:0000641",
442 + "efo_term": "papillary thyroid carcinoma",
443 + "max_phase_for_ind": "2.0",
444 + "indication_refs": [
445 + {
446 + "ref_id": "NCT03469011",
447 + "ref_type": "ClinicalTrials",
448 + "ref_url": "https://clinicaltrials.gov/search?term=NCT03469011"
449 + }
450 + ]
451 + },
452 + {
453 + "drugind_id": 111879,
454 + "molecule_chembl_id": "CHEMBL941",
455 + "parent_molecule_chembl_id": "CHEMBL941",
456 + "mesh_id": "D020521",
457 + "mesh_heading": "Stroke",
458 + "efo_id": "EFO:0000712",
459 + "efo_term": "stroke",
460 + "max_phase_for_ind": "3.0",
461 + "indication_refs": [
462 + {
463 + "ref_id": "NCT03639922",
464 + "ref_type": "ClinicalTrials",
465 + "ref_url": "https://clinicaltrials.gov/search?term=NCT03639922"
466 + }
467 + ]
468 + },
469 + {
470 + "drugind_id": 111880,
471 + "molecule_chembl_id": "CHEMBL941",
472 + "parent_molecule_chembl_id": "CHEMBL941",
473 + "mesh_id": "D013945",
474 + "mesh_heading": "Thymoma",
475 + "efo_id": "EFO:1000576",
476 + "efo_term": "Thymic Carcinoma",
477 + "max_phase_for_ind": "1.0",
478 + "indication_refs": [
479 + {
480 + "ref_id": "NCT00314873",
481 + "ref_type": "ClinicalTrials",
482 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00314873"
483 + }
484 + ]
485 + },
486 + {
487 + "drugind_id": 111881,
488 + "molecule_chembl_id": "CHEMBL941",
489 + "parent_molecule_chembl_id": "CHEMBL941",
490 + "mesh_id": "D017681",
491 + "mesh_heading": "Hypereosinophilic Syndrome",
492 + "efo_id": "EFO:1001467",
493 + "efo_term": "Hypereosinophilic syndrome",
494 + "max_phase_for_ind": "2.0",
495 + "indication_refs": [
496 + {
497 + "ref_id": "NCT00044304",
498 + "ref_type": "ClinicalTrials",
499 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00044304"
500 + }
501 + ]
502 + },
503 + {
504 + "drugind_id": 123392,
505 + "molecule_chembl_id": "CHEMBL941",
506 + "parent_molecule_chembl_id": "CHEMBL941",
507 + "mesh_id": "D054198",
508 + "mesh_heading": "Precursor Cell Lymphoblastic Leukemia-Lymphoma",
509 + "efo_id": "MONDO:0000873",
510 + "efo_term": "lymphoblastic lymphoma",
511 + "max_phase_for_ind": "3.0",
512 + "indication_refs": [
513 + {
514 + "ref_id": "NCT02881086",
515 + "ref_type": "ClinicalTrials",
516 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02881086"
517 + }
518 + ]
519 + },
520 + {
521 + "drugind_id": 123393,
522 + "molecule_chembl_id": "CHEMBL941",
523 + "parent_molecule_chembl_id": "CHEMBL941",
524 + "mesh_id": "D013964",
525 + "mesh_heading": "Thyroid Neoplasms",
526 + "efo_id": "MONDO:0002108",
527 + "efo_term": "thyroid cancer",
528 + "max_phase_for_ind": "2.0",
529 + "indication_refs": [
530 + {
531 + "ref_id": "NCT00115739",
532 + "ref_type": "ClinicalTrials",
533 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00115739"
534 + }
535 + ]
536 + },
537 + {
538 + "drugind_id": 123394,
539 + "molecule_chembl_id": "CHEMBL941",
540 + "parent_molecule_chembl_id": "CHEMBL941",
541 + "mesh_id": "D007951",
542 + "mesh_heading": "Leukemia, Myeloid",
543 + "efo_id": "MONDO:0004643",
544 + "efo_term": "myeloid leukemia",
545 + "max_phase_for_ind": "3.0",
546 + "indication_refs": [
547 + {
548 + "ref_id": "NCT00519090",
549 + "ref_type": "ClinicalTrials",
550 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00519090"
551 + }
552 + ]
553 + },
554 + {
555 + "drugind_id": 123395,
556 + "molecule_chembl_id": "CHEMBL941",
557 + "parent_molecule_chembl_id": "CHEMBL941",
558 + "mesh_id": "D006976",
559 + "mesh_heading": "Hypertension, Pulmonary",
560 + "efo_id": "MONDO:0005149",
561 + "efo_term": "pulmonary hypertension",
562 + "max_phase_for_ind": "3.0",
563 + "indication_refs": [
564 + {
565 + "ref_id": "NCT01568645",
566 + "ref_type": "ClinicalTrials",
567 + "ref_url": "https://clinicaltrials.gov/search?term=NCT01568645"
568 + }
569 + ]
570 + },
571 + {
572 + "drugind_id": 123396,
573 + "molecule_chembl_id": "CHEMBL941",
574 + "parent_molecule_chembl_id": "CHEMBL941",
575 + "mesh_id": "D001943",
576 + "mesh_heading": "Breast Neoplasms",
577 + "efo_id": "MONDO:0007254",
578 + "efo_term": "breast cancer",
579 + "max_phase_for_ind": "2.0",
580 + "indication_refs": [
581 + {
582 + "ref_id": "NCT00193180",
583 + "ref_type": "ClinicalTrials",
584 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00193180"
585 + }
586 + ]
587 + },
588 + {
589 + "drugind_id": 123398,
590 + "molecule_chembl_id": "CHEMBL941",
591 + "parent_molecule_chembl_id": "CHEMBL941",
592 + "mesh_id": "D008175",
593 + "mesh_heading": "Lung Neoplasms",
594 + "efo_id": "MONDO:0008903",
595 + "efo_term": "lung cancer",
596 + "max_phase_for_ind": "2.0",
597 + "indication_refs": [
598 + {
599 + "ref_id": "NCT00193349",
600 + "ref_type": "ClinicalTrials",
601 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00193349"
602 + }
603 + ]
604 + },
605 + {
606 + "drugind_id": 123399,
607 + "molecule_chembl_id": "CHEMBL941",
608 + "parent_molecule_chembl_id": "CHEMBL941",
609 + "mesh_id": "D006086",
610 + "mesh_heading": "Graft vs Host Disease",
611 + "efo_id": "MONDO:0013730",
612 + "efo_term": "graft versus host disease",
613 + "max_phase_for_ind": "3.0",
614 + "indication_refs": [
615 + {
616 + "ref_id": "NCT00760981,NCT03112603",
617 + "ref_type": "ClinicalTrials",
618 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00760981%20NCT03112603"
619 + }
620 + ]
621 + },
622 + {
623 + "drugind_id": 123400,
624 + "molecule_chembl_id": "CHEMBL941",
625 + "parent_molecule_chembl_id": "CHEMBL941",
626 + "mesh_id": "D019337",
627 + "mesh_heading": "Hematologic Neoplasms",
628 + "efo_id": "MONDO:0044881",
629 + "efo_term": "hematopoietic and lymphoid cell neoplasm",
630 + "max_phase_for_ind": "1.0",
631 + "indication_refs": [
632 + {
633 + "ref_id": "NCT02352558",
634 + "ref_type": "ClinicalTrials",
635 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02352558"
636 + }
637 + ]
638 + },
639 + {
640 + "drugind_id": 128956,
641 + "molecule_chembl_id": "CHEMBL941",
642 + "parent_molecule_chembl_id": "CHEMBL941",
643 + "mesh_id": "D013119",
644 + "mesh_heading": "Spinal Cord Injuries",
645 + "efo_id": "EFO:1001919",
646 + "efo_term": "Spinal cord injury",
647 + "max_phase_for_ind": "2.0",
648 + "indication_refs": [
649 + {
650 + "ref_id": "NCT02363361",
651 + "ref_type": "ClinicalTrials",
652 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02363361"
653 + }
654 + ]
655 + },
656 + {
657 + "drugind_id": 128957,
658 + "molecule_chembl_id": "CHEMBL941",
659 + "parent_molecule_chembl_id": "CHEMBL941",
660 + "mesh_id": "D045169",
661 + "mesh_heading": "Severe Acute Respiratory Syndrome",
662 + "efo_id": "MONDO:0100096",
663 + "efo_term": "COVID-19",
664 + "max_phase_for_ind": "3.0",
665 + "indication_refs": [
666 + {
667 + "ref_id": "NCT02735707,NCT04330690,NCT04346147,NCT04394416",
668 + "ref_type": "ClinicalTrials",
669 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02735707%20NCT04330690%20NCT04346147%20NCT04394416"
670 + }
671 + ]
672 + },
673 + {
674 + "drugind_id": 135858,
675 + "molecule_chembl_id": "CHEMBL941",
676 + "parent_molecule_chembl_id": "CHEMBL941",
677 + "mesh_id": "D018222",
678 + "mesh_heading": "Desmoid Tumors",
679 + "efo_id": "EFO:0009907",
680 + "efo_term": "Desmoid-type fibromatosis",
681 + "max_phase_for_ind": "2.0",
682 + "indication_refs": [
683 + {
684 + "ref_id": "NCT01137916,NCT03802084",
685 + "ref_type": "ClinicalTrials",
686 + "ref_url": "https://clinicaltrials.gov/search?term=NCT01137916%20NCT03802084"
687 + }
688 + ]
689 + },
690 + {
691 + "drugind_id": 141152,
692 + "molecule_chembl_id": "CHEMBL941",
693 + "parent_molecule_chembl_id": "CHEMBL941",
694 + "mesh_id": "D008103",
695 + "mesh_heading": "Liver Cirrhosis",
696 + "efo_id": "EFO:0001422",
697 + "efo_term": "cirrhosis of liver",
698 + "max_phase_for_ind": "1.0",
699 + "indication_refs": [
700 + {
701 + "ref_id": "NCT05224128",
702 + "ref_type": "ClinicalTrials",
703 + "ref_url": "https://clinicaltrials.gov/search?term=NCT05224128"
704 + }
705 + ]
706 + },
707 + {
708 + "drugind_id": 141153,
709 + "molecule_chembl_id": "CHEMBL941",
710 + "parent_molecule_chembl_id": "CHEMBL941",
711 + "mesh_id": "D002817",
712 + "mesh_heading": "Chordoma",
713 + "efo_id": "MONDO:0008978",
714 + "efo_term": "chordoma",
715 + "max_phase_for_ind": "2.0",
716 + "indication_refs": [
717 + {
718 + "ref_id": "NCT00150072,NCT01175109",
719 + "ref_type": "ClinicalTrials",
720 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00150072%20NCT01175109"
721 + }
722 + ]
723 + },
724 + {
725 + "drugind_id": 141154,
726 + "molecule_chembl_id": "CHEMBL941",
727 + "parent_molecule_chembl_id": "CHEMBL941",
728 + "mesh_id": "D046152",
729 + "mesh_heading": "Gastrointestinal Stromal Tumors",
730 + "efo_id": "MONDO:0011719",
731 + "efo_term": "gastrointestinal stromal tumor",
732 + "max_phase_for_ind": "3.0",
733 + "indication_refs": [
734 + {
735 + "ref_id": "NCT00441155,NCT00751036,NCT00785785,NCT00812240,NCT00940563,NCT01151852,NCT01275222,NCT01294202,NCT01541709,NCT02260505,NCT02268435,NCT02365441,NCT03609424,NCT04933669,NCT05009927,NCT05152472,NCT05245968,NCT05493215",
736 + "ref_type": "ClinicalTrials",
737 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00441155%20NCT00751036%20NCT00785785%20NCT00812240%20NCT00940563%20NCT01151852%20NCT01275222%20NCT01294202%20NCT01541709%20NCT02260505%20NCT02268435%20NCT02365441%20NCT03609424%20NCT04933669%20NCT05009927%20NCT05152472%20NCT05245968%20NCT05493215"
738 + }
739 + ]
740 + },
741 + {
742 + "drugind_id": 141155,
743 + "molecule_chembl_id": "CHEMBL941",
744 + "parent_molecule_chembl_id": "CHEMBL941",
745 + "mesh_id": "D014376",
746 + "mesh_heading": "Tuberculosis",
747 + "efo_id": "MONDO:0018076",
748 + "efo_term": "tuberculosis",
749 + "max_phase_for_ind": "2.0",
750 + "indication_refs": [
751 + {
752 + "ref_id": "NCT03891901",
753 + "ref_type": "ClinicalTrials",
754 + "ref_url": "https://clinicaltrials.gov/search?term=NCT03891901"
755 + }
756 + ]
757 + },
758 + {
759 + "drugind_id": 146376,
760 + "molecule_chembl_id": "CHEMBL941",
761 + "parent_molecule_chembl_id": "CHEMBL941",
762 + "mesh_id": "D002294",
763 + "mesh_heading": "Carcinoma, Squamous Cell",
764 + "efo_id": "EFO:0000181",
765 + "efo_term": "head and neck squamous cell carcinoma",
766 + "max_phase_for_ind": "0.5",
767 + "indication_refs": [
768 + {
769 + "ref_id": "NCT05816785",
770 + "ref_type": "ClinicalTrials",
771 + "ref_url": "https://clinicaltrials.gov/search?term=NCT05816785"
772 + }
773 + ]
774 + },
775 + {
776 + "drugind_id": 146377,
777 + "molecule_chembl_id": "CHEMBL941",
778 + "parent_molecule_chembl_id": "CHEMBL941",
779 + "mesh_id": "D011014",
780 + "mesh_heading": "Pneumonia",
781 + "efo_id": "EFO:0003106",
782 + "efo_term": "pneumonia",
783 + "max_phase_for_ind": "3.0",
784 + "indication_refs": [
785 + {
786 + "ref_id": "NCT02735707",
787 + "ref_type": "ClinicalTrials",
788 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02735707"
789 + }
790 + ]
791 + },
792 + {
793 + "drugind_id": 146378,
794 + "molecule_chembl_id": "CHEMBL941",
795 + "parent_molecule_chembl_id": "CHEMBL941",
796 + "mesh_id": "D007251",
797 + "mesh_heading": "Influenza, Human",
798 + "efo_id": "EFO:0007328",
799 + "efo_term": "influenza",
800 + "max_phase_for_ind": "3.0",
801 + "indication_refs": [
802 + {
803 + "ref_id": "NCT02735707",
804 + "ref_type": "ClinicalTrials",
805 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02735707"
806 + }
807 + ]
808 + },
809 + {
810 + "drugind_id": 146379,
811 + "molecule_chembl_id": "CHEMBL941",
812 + "parent_molecule_chembl_id": "CHEMBL941",
813 + "mesh_id": "D010190",
814 + "mesh_heading": "Pancreatic Neoplasms",
815 + "efo_id": "EFO:1000359",
816 + "efo_term": "Malignant Pancreatic Neoplasm",
817 + "max_phase_for_ind": "1.0",
818 + "indication_refs": [
819 + {
820 + "ref_id": "NCT01048320",
821 + "ref_type": "ClinicalTrials",
822 + "ref_url": "https://clinicaltrials.gov/search?term=NCT01048320"
823 + }
824 + ]
825 + },
826 + {
827 + "drugind_id": 146380,
828 + "molecule_chembl_id": "CHEMBL941",
829 + "parent_molecule_chembl_id": "CHEMBL941",
830 + "mesh_id": "D009369",
831 + "mesh_heading": "Neoplasms",
832 + "efo_id": "MONDO:0004992",
833 + "efo_term": "cancer",
834 + "max_phase_for_ind": "3.0",
835 + "indication_refs": [
836 + {
837 + "ref_id": "NCT00611689,NCT02461849,NCT06119789",
838 + "ref_type": "ClinicalTrials",
839 + "ref_url": "https://clinicaltrials.gov/search?term=NCT00611689%20NCT02461849%20NCT06119789"
840 + }
841 + ]
842 + },
843 + {
844 + "drugind_id": 153478,
845 + "molecule_chembl_id": "CHEMBL941",
846 + "parent_molecule_chembl_id": "CHEMBL941",
847 + "mesh_id": "D007945",
848 + "mesh_heading": "Leukemia, Lymphoid",
849 + "efo_id": "EFO:0004289",
850 + "efo_term": "lymphoid leukemia",
851 + "max_phase_for_ind": "2.0",
852 + "indication_refs": [
853 + {
854 + "ref_id": "NCT06336395",
855 + "ref_type": "ClinicalTrials",
856 + "ref_url": "https://clinicaltrials.gov/search?term=NCT06336395"
857 + }
858 + ]
859 + },
860 + {
861 + "drugind_id": 153479,
862 + "molecule_chembl_id": "CHEMBL941",
863 + "parent_molecule_chembl_id": "CHEMBL941",
864 + "mesh_id": "D006258",
865 + "mesh_heading": "Head and Neck Neoplasms",
866 + "efo_id": "EFO:0006859",
867 + "efo_term": "head and neck malignant neoplasia",
868 + "max_phase_for_ind": "0.5",
869 + "indication_refs": [
870 + {
871 + "ref_id": "NCT05816785",
872 + "ref_type": "ClinicalTrials",
873 + "ref_url": "https://clinicaltrials.gov/search?term=NCT05816785"
874 + }
875 + ]
876 + },
877 + {
878 + "drugind_id": 153480,
879 + "molecule_chembl_id": "CHEMBL941",
880 + "parent_molecule_chembl_id": "CHEMBL941",
881 + "mesh_id": "D015179",
882 + "mesh_heading": "Colorectal Neoplasms",
883 + "efo_id": "MONDO:0005575",
884 + "efo_term": "colorectal cancer",
885 + "max_phase_for_ind": "1.0",
886 + "indication_refs": [
887 + {
888 + "ref_id": "NCT01271166",
889 + "ref_type": "ClinicalTrials",
890 + "ref_url": "https://clinicaltrials.gov/search?term=NCT01271166"
891 + }
892 + ]
893 + }
894 + ]
895 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/indication-osimertinib.json +215 −0
@@ -0,0 +1,215 @@
1 +{
2 + "page_meta": {
3 + "limit": 100,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 12
8 + },
9 + "drug_indications": [
10 + {
11 + "drugind_id": 50352,
12 + "molecule_chembl_id": "CHEMBL3353410",
13 + "parent_molecule_chembl_id": "CHEMBL3353410",
14 + "mesh_id": "D002289",
15 + "mesh_heading": "Carcinoma, Non-Small-Cell Lung",
16 + "efo_id": "EFO:0003060",
17 + "efo_term": "non-small cell lung carcinoma",
18 + "max_phase_for_ind": "3.0",
19 + "indication_refs": [
20 + {
21 + "ref_id": "NCT01802632,NCT02094261,NCT02143466,NCT02157883,NCT02163733,NCT02179671,NCT02197247,NCT02228369,NCT02296125,NCT02317016,NCT02411448,NCT02442349,NCT02454933,NCT02496663,NCT02503722,NCT02511106,NCT02529995,NCT02664935,NCT02771314,NCT02789345,NCT02803203,NCT02811354,NCT02824952,NCT02841579,NCT02856893,NCT02954523,NCT02971501,NCT03050411,NCT03122717,NCT03133546,NCT03239340,NCT03255083,NCT03257124,NCT03381274,NCT03392246,NCT03394118,NCT03410043,NCT03414814,NCT03424759,NCT03433469,NCT03434418,NCT03455829,NCT03463525,NCT03521154,NCT03532698,NCT03535363,NCT03543683,NCT03586453,NCT03667820,NCT03784599,NCT03810066,NCT03810807,NCT03831932,NCT03833154,NCT03940703,NCT03944772,NCT04029350,NCT04035486,NCT04141644,NCT04181060,NCT04285671,NCT04335292,NCT04351555,NCT04410796,NCT04479306,NCT04486833,NCT04487080,NCT04541407,NCT04563871,NCT04606771,NCT04695925,NCT04720976,NCT04721015,NCT04762199,NCT04765059,NCT04769388,NCT04772235,NCT04780568,NCT04811001,NCT04816214,NCT04870190,NCT04908956,NCT04959981,NCT04974879,NCT05011487,NCT05015608,NCT05017025,NCT05020769,NCT05089916,NCT05104281,NCT05120349,NCT05163249,NCT05166616,NCT05261399,NCT05281406,NCT05298176,NCT05382728,NCT05401110,NCT05493501,NCT05507606,NCT05526755,NCT05546866,NCT05642572,NCT05686434,NCT05693090,NCT05773092,NCT05801029,NCT05816252,NCT06014827,NCT06032936,NCT06067776,NCT06093503,NCT06206850,NCT06306456,NCT06319950,NCT06350097,NCT06362980,NCT06380348,NCT06391944,NCT06417814,NCT06436144,NCT06477055,NCT06486142",
22 + "ref_type": "ClinicalTrials",
23 + "ref_url": "https://clinicaltrials.gov/search?term=NCT01802632%20NCT02094261%20NCT02143466%20NCT02157883%20NCT02163733%20NCT02179671%20NCT02197247%20NCT02228369%20NCT02296125%20NCT02317016%20NCT02411448%20NCT02442349%20NCT02454933%20NCT02496663%20NCT02503722%20NCT02511106%20NCT02529995%20NCT02664935%20NCT02771314%20NCT02789345%20NCT02803203%20NCT02811354%20NCT02824952%20NCT02841579%20NCT02856893%20NCT02954523%20NCT02971501%20NCT03050411%20NCT03122717%20NCT03133546%20NCT03239340%20NCT03255083%20NCT03257124%20NCT03381274%20NCT03392246%20NCT03394118%20NCT03410043%20NCT03414814%20NCT03424759%20NCT03433469%20NCT03434418%20NCT03455829%20NCT03463525%20NCT03521154%20NCT03532698%20NCT03535363%20NCT03543683%20NCT03586453%20NCT03667820%20NCT03784599%20NCT03810066%20NCT03810807%20NCT03831932%20NCT03833154%20NCT03940703%20NCT03944772%20NCT04029350%20NCT04035486%20NCT04141644%20NCT04181060%20NCT04285671%20NCT04335292%20NCT04351555%20NCT04410796%20NCT04479306%20NCT04486833%20NCT04487080%20NCT04541407%20NCT04563871%20NCT04606771%20NCT04695925%20NCT04720976%20NCT04721015%20NCT04762199%20NCT04765059%20NCT04769388%20NCT04772235%20NCT04780568%20NCT04811001%20NCT04816214%20NCT04870190%20NCT04908956%20NCT04959981%20NCT04974879%20NCT05011487%20NCT05015608%20NCT05017025%20NCT05020769%20NCT05089916%20NCT05104281%20NCT05120349%20NCT05163249%20NCT05166616%20NCT05261399%20NCT05281406%20NCT05298176%20NCT05382728%20NCT05401110%20NCT05493501%20NCT05507606%20NCT05526755%20NCT05546866%20NCT05642572%20NCT05686434%20NCT05693090%20NCT05773092%20NCT05801029%20NCT05816252%20NCT06014827%20NCT06032936%20NCT06067776%20NCT06093503%20NCT06206850%20NCT06306456%20NCT06319950%20NCT06350097%20NCT06362980%20NCT06380348%20NCT06391944%20NCT06417814%20NCT06436144%20NCT06477055%20NCT06486142"
24 + }
25 + ]
26 + },
27 + {
28 + "drugind_id": 52825,
29 + "molecule_chembl_id": "CHEMBL3353410",
30 + "parent_molecule_chembl_id": "CHEMBL3353410",
31 + "mesh_id": "D009369",
32 + "mesh_heading": "Neoplasms",
33 + "efo_id": "EFO:0000616",
34 + "efo_term": "neoplasm",
35 + "max_phase_for_ind": "4.0",
36 + "indication_refs": [
37 + {
38 + "ref_id": "L01EB04",
39 + "ref_type": "ATC",
40 + "ref_url": "https://www.whocc.no/atc_ddd_index/?code=L01EB04"
41 + }
42 + ]
43 + },
44 + {
45 + "drugind_id": 71327,
46 + "molecule_chembl_id": "CHEMBL3353410",
47 + "parent_molecule_chembl_id": "CHEMBL3353410",
48 + "mesh_id": "D000230",
49 + "mesh_heading": "Adenocarcinoma",
50 + "efo_id": "EFO:0000228",
51 + "efo_term": "adenocarcinoma",
52 + "max_phase_for_ind": "2.0",
53 + "indication_refs": [
54 + {
55 + "ref_id": "NCT02664935",
56 + "ref_type": "ClinicalTrials",
57 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02664935"
58 + }
59 + ]
60 + },
61 + {
62 + "drugind_id": 71329,
63 + "molecule_chembl_id": "CHEMBL3353410",
64 + "parent_molecule_chembl_id": "CHEMBL3353410",
65 + "mesh_id": "D002294",
66 + "mesh_heading": "Carcinoma, Squamous Cell",
67 + "efo_id": "EFO:0000707",
68 + "efo_term": "squamous cell carcinoma",
69 + "max_phase_for_ind": "2.0",
70 + "indication_refs": [
71 + {
72 + "ref_id": "NCT02664935",
73 + "ref_type": "ClinicalTrials",
74 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02664935"
75 + }
76 + ]
77 + },
78 + {
79 + "drugind_id": 112550,
80 + "molecule_chembl_id": "CHEMBL3353410",
81 + "parent_molecule_chembl_id": "CHEMBL3353410",
82 + "mesh_id": "D002277",
83 + "mesh_heading": "Carcinoma",
84 + "efo_id": "EFO:0000313",
85 + "efo_term": "carcinoma",
86 + "max_phase_for_ind": "2.0",
87 + "indication_refs": [
88 + {
89 + "ref_id": "NCT03778229",
90 + "ref_type": "ClinicalTrials",
91 + "ref_url": "https://clinicaltrials.gov/search?term=NCT03778229"
92 + }
93 + ]
94 + },
95 + {
96 + "drugind_id": 112551,
97 + "molecule_chembl_id": "CHEMBL3353410",
98 + "parent_molecule_chembl_id": "CHEMBL3353410",
99 + "mesh_id": "D005909",
100 + "mesh_heading": "Glioblastoma",
101 + "efo_id": "EFO:0000519",
102 + "efo_term": "glioblastoma multiforme",
103 + "max_phase_for_ind": "2.0",
104 + "indication_refs": [
105 + {
106 + "ref_id": "NCT03732352",
107 + "ref_type": "ClinicalTrials",
108 + "ref_url": "https://clinicaltrials.gov/search?term=NCT03732352"
109 + }
110 + ]
111 + },
112 + {
113 + "drugind_id": 124490,
114 + "molecule_chembl_id": "CHEMBL3353410",
115 + "parent_molecule_chembl_id": "CHEMBL3353410",
116 + "mesh_id": "D008175",
117 + "mesh_heading": "Lung Neoplasms",
118 + "efo_id": "MONDO:0008903",
119 + "efo_term": "lung cancer",
120 + "max_phase_for_ind": "3.0",
121 + "indication_refs": [
122 + {
123 + "ref_id": "NCT02474355,NCT02504346,NCT02736513,NCT02803203,NCT02917993,NCT02997501,NCT03455829,NCT03567642,NCT03755102,NCT03804580,NCT03891615,NCT04285671,NCT04545710,NCT04591002,NCT04988607,NCT05166616,NCT05215951,NCT06067776,NCT06194448",
124 + "ref_type": "ClinicalTrials",
125 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02474355%20NCT02504346%20NCT02736513%20NCT02803203%20NCT02917993%20NCT02997501%20NCT03455829%20NCT03567642%20NCT03755102%20NCT03804580%20NCT03891615%20NCT04285671%20NCT04545710%20NCT04591002%20NCT04988607%20NCT05166616%20NCT05215951%20NCT06067776%20NCT06194448"
126 + }
127 + ]
128 + },
129 + {
130 + "drugind_id": 124491,
131 + "molecule_chembl_id": "CHEMBL3353410",
132 + "parent_molecule_chembl_id": "CHEMBL3353410",
133 + "mesh_id": "D008175",
134 + "mesh_heading": "Lung Neoplasms",
135 + "efo_id": "MONDO:0021117",
136 + "efo_term": "lung neoplasm",
137 + "max_phase_for_ind": "3.0",
138 + "indication_refs": [
139 + {
140 + "ref_id": "NCT02769286",
141 + "ref_type": "ClinicalTrials",
142 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02769286"
143 + }
144 + ]
145 + },
146 + {
147 + "drugind_id": 141841,
148 + "molecule_chembl_id": "CHEMBL3353410",
149 + "parent_molecule_chembl_id": "CHEMBL3353410",
150 + "mesh_id": "D001932",
151 + "mesh_heading": "Brain Neoplasms",
152 + "efo_id": "EFO:0003833",
153 + "efo_term": "brain neoplasm",
154 + "max_phase_for_ind": "1.0",
155 + "indication_refs": [
156 + {
157 + "ref_id": "NCT05120960",
158 + "ref_type": "ClinicalTrials",
159 + "ref_url": "https://clinicaltrials.gov/search?term=NCT05120960"
160 + }
161 + ]
162 + },
163 + {
164 + "drugind_id": 146788,
165 + "molecule_chembl_id": "CHEMBL3353410",
166 + "parent_molecule_chembl_id": "CHEMBL3353410",
167 + "mesh_id": "D009369",
168 + "mesh_heading": "Neoplasms",
169 + "efo_id": "MONDO:0004992",
170 + "efo_term": "cancer",
171 + "max_phase_for_ind": "4.0",
172 + "indication_refs": [
173 + {
174 + "ref_id": "NCT02099058,NCT02151981,NCT05629234",
175 + "ref_type": "ClinicalTrials",
176 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02099058%20NCT02151981%20NCT05629234"
177 + }
178 + ]
179 + },
180 + {
181 + "drugind_id": 154202,
182 + "molecule_chembl_id": "CHEMBL3353410",
183 + "parent_molecule_chembl_id": "CHEMBL3353410",
184 + "mesh_id": "D000077192",
185 + "mesh_heading": "Adenocarcinoma of Lung",
186 + "efo_id": "EFO:0005288",
187 + "efo_term": "non-small cell lung adenocarcinoma",
188 + "max_phase_for_ind": "2.0",
189 + "indication_refs": [
190 + {
191 + "ref_id": "NCT02759835,NCT05528458",
192 + "ref_type": "ClinicalTrials",
193 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02759835%20NCT05528458"
194 + }
195 + ]
196 + },
197 + {
198 + "drugind_id": 154203,
199 + "molecule_chembl_id": "CHEMBL3353410",
200 + "parent_molecule_chembl_id": "CHEMBL3353410",
201 + "mesh_id": "D015179",
202 + "mesh_heading": "Colorectal Neoplasms",
203 + "efo_id": "MONDO:0005575",
204 + "efo_term": "colorectal cancer",
205 + "max_phase_for_ind": "2.0",
206 + "indication_refs": [
207 + {
208 + "ref_id": "NCT02959749",
209 + "ref_type": "ClinicalTrials",
210 + "ref_url": "https://clinicaltrials.gov/search?term=NCT02959749"
211 + }
212 + ]
213 + }
214 + ]
215 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/mechanism-imatinib.json +126 −0
@@ -0,0 +1,126 @@
1 +{
2 + "page_meta": {
3 + "limit": 20,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 4
8 + },
9 + "mechanisms": [
10 + {
11 + "action_type": "INHIBITOR",
12 + "binding_site_comment": null,
13 + "direct_interaction": 1,
14 + "disease_efficacy": 1,
15 + "max_phase": 4,
16 + "mec_id": 304,
17 + "mechanism_comment": "Mechanism_for_indication: Precursor Cell Lymphoblastic Leukemia-Lymphoma, Leukemia, Myelogenous, Chronic, BCR-ABL Positive",
18 + "mechanism_of_action": "Tyrosine-protein kinase ABL inhibitor",
19 + "mechanism_refs": [
20 + {
21 + "ref_id": "setid=211ef2da-2868-4a77-8055-1cb2cd78e24b#section-13",
22 + "ref_type": "DailyMed",
23 + "ref_url": "http://dailymed.nlm.nih.gov/dailymed/lookup.cfm?setid=211ef2da-2868-4a77-8055-1cb2cd78e24b#section-13"
24 + },
25 + {
26 + "ref_id": "Bcr-Abl_tyrosine-kinase_inhibitor",
27 + "ref_type": "Wikipedia",
28 + "ref_url": "http://en.wikipedia.org/wiki/Bcr-Abl_tyrosine-kinase_inhibitor"
29 + }
30 + ],
31 + "molecular_mechanism": 1,
32 + "molecule_chembl_id": "CHEMBL1642",
33 + "parent_molecule_chembl_id": "CHEMBL941",
34 + "record_id": 1343743,
35 + "selectivity_comment": null,
36 + "site_id": null,
37 + "target_chembl_id": "CHEMBL1862",
38 + "variant_sequence": null
39 + },
40 + {
41 + "action_type": "INHIBITOR",
42 + "binding_site_comment": null,
43 + "direct_interaction": 1,
44 + "disease_efficacy": 1,
45 + "max_phase": 4,
46 + "mec_id": 418,
47 + "mechanism_comment": "Mechanism_for_indication: Dermatofibrosarcoma, Hypereosinophilic Syndrome, Myelodysplastic-Myeloproliferative Diseases",
48 + "mechanism_of_action": "Platelet-derived growth factor receptor beta inhibitor",
49 + "mechanism_refs": [
50 + {
51 + "ref_id": "setid=211ef2da-2868-4a77-8055-1cb2cd78e24b#section-13",
52 + "ref_type": "DailyMed",
53 + "ref_url": "http://dailymed.nlm.nih.gov/dailymed/lookup.cfm?setid=211ef2da-2868-4a77-8055-1cb2cd78e24b#section-13"
54 + },
55 + {
56 + "ref_id": "Bcr-Abl_tyrosine-kinase_inhibitor",
57 + "ref_type": "Wikipedia",
58 + "ref_url": "http://en.wikipedia.org/wiki/Bcr-Abl_tyrosine-kinase_inhibitor"
59 + }
60 + ],
61 + "molecular_mechanism": 1,
62 + "molecule_chembl_id": "CHEMBL1642",
63 + "parent_molecule_chembl_id": "CHEMBL941",
64 + "record_id": 1343743,
65 + "selectivity_comment": null,
66 + "site_id": null,
67 + "target_chembl_id": "CHEMBL1913",
68 + "variant_sequence": null
69 + },
70 + {
71 + "action_type": "INHIBITOR",
72 + "binding_site_comment": null,
73 + "direct_interaction": 1,
74 + "disease_efficacy": 1,
75 + "max_phase": 4,
76 + "mec_id": 451,
77 + "mechanism_comment": "Mechanism_for_indication: Gastrointestinal Stromal Tumors",
78 + "mechanism_of_action": "Stem cell growth factor receptor inhibitor",
79 + "mechanism_refs": [
80 + {
81 + "ref_id": "setid=211ef2da-2868-4a77-8055-1cb2cd78e24b#section-13",
82 + "ref_type": "DailyMed",
83 + "ref_url": "http://dailymed.nlm.nih.gov/dailymed/lookup.cfm?setid=211ef2da-2868-4a77-8055-1cb2cd78e24b#section-13"
84 + },
85 + {
86 + "ref_id": "Bcr-Abl_tyrosine-kinase_inhibitor",
87 + "ref_type": "Wikipedia",
88 + "ref_url": "http://en.wikipedia.org/wiki/Bcr-Abl_tyrosine-kinase_inhibitor"
89 + }
90 + ],
91 + "molecular_mechanism": 1,
92 + "molecule_chembl_id": "CHEMBL1642",
93 + "parent_molecule_chembl_id": "CHEMBL941",
94 + "record_id": 1343743,
95 + "selectivity_comment": null,
96 + "site_id": null,
97 + "target_chembl_id": "CHEMBL1936",
98 + "variant_sequence": null
99 + },
100 + {
101 + "action_type": "INHIBITOR",
102 + "binding_site_comment": null,
103 + "direct_interaction": 1,
104 + "disease_efficacy": 1,
105 + "max_phase": 4,
106 + "mec_id": 8630,
107 + "mechanism_comment": "Mechanism_for_indication: Precursor Cell Lymphoblastic Leukemia-Lymphoma, Leukemia, Myelogenous, Chronic, BCR-ABL Positive",
108 + "mechanism_of_action": "Bcr/Abl fusion protein inhibitor",
109 + "mechanism_refs": [
110 + {
111 + "ref_id": "label/2008/021588s024lbl.pdf",
112 + "ref_type": "FDA",
113 + "ref_url": "https://www.accessdata.fda.gov/drugsatfda_docs/label/2008/021588s024lbl.pdf"
114 + }
115 + ],
116 + "molecular_mechanism": 1,
117 + "molecule_chembl_id": "CHEMBL1642",
118 + "parent_molecule_chembl_id": "CHEMBL941",
119 + "record_id": 1343743,
120 + "selectivity_comment": null,
121 + "site_id": null,
122 + "target_chembl_id": "CHEMBL2096618",
123 + "variant_sequence": null
124 + }
125 + ]
126 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/mechanism-osimertinib.json +46 −0
@@ -0,0 +1,46 @@
1 +{
2 + "page_meta": {
3 + "limit": 20,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 1
8 + },
9 + "mechanisms": [
10 + {
11 + "action_type": "INHIBITOR",
12 + "binding_site_comment": null,
13 + "direct_interaction": 1,
14 + "disease_efficacy": 1,
15 + "max_phase": 4,
16 + "mec_id": 5260,
17 + "mechanism_comment": "Binds irreversibly to certain mutant forms of EGFR (T790M, L858R, and exon 19 deletion)",
18 + "mechanism_of_action": "Epidermal growth factor receptor erbB1 inhibitor",
19 + "mechanism_refs": [
20 + {
21 + "ref_id": "setid=5e81b4a7-b971-45e1-9c31-29cea8c87ce7",
22 + "ref_type": "DailyMed",
23 + "ref_url": "http://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=5e81b4a7-b971-45e1-9c31-29cea8c87ce7"
24 + },
25 + {
26 + "ref_id": "24893891",
27 + "ref_type": "PubMed",
28 + "ref_url": "http://europepmc.org/abstract/MED/24893891"
29 + },
30 + {
31 + "ref_id": "25271963",
32 + "ref_type": "PubMed",
33 + "ref_url": "http://europepmc.org/abstract/MED/25271963"
34 + }
35 + ],
36 + "molecular_mechanism": 1,
37 + "molecule_chembl_id": "CHEMBL3353410",
38 + "parent_molecule_chembl_id": "CHEMBL3353410",
39 + "record_id": 2473534,
40 + "selectivity_comment": null,
41 + "site_id": null,
42 + "target_chembl_id": "CHEMBL203",
43 + "variant_sequence": null
44 + }
45 + ]
46 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/mechanism-trastuzumab.json +36 −0
@@ -0,0 +1,36 @@
1 +{
2 + "page_meta": {
3 + "limit": 20,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 1
8 + },
9 + "mechanisms": [
10 + {
11 + "action_type": "INHIBITOR",
12 + "binding_site_comment": null,
13 + "direct_interaction": 1,
14 + "disease_efficacy": 1,
15 + "max_phase": 4,
16 + "mec_id": 127,
17 + "mechanism_comment": null,
18 + "mechanism_of_action": "Receptor protein-tyrosine kinase erbB-2 inhibitor",
19 + "mechanism_refs": [
20 + {
21 + "ref_id": "setid=492dbdb2-077e-4064-bff3-372d6af0a7a2",
22 + "ref_type": "DailyMed",
23 + "ref_url": "http://dailymed.nlm.nih.gov/dailymed/lookup.cfm?setid=492dbdb2-077e-4064-bff3-372d6af0a7a2"
24 + }
25 + ],
26 + "molecular_mechanism": 1,
27 + "molecule_chembl_id": "CHEMBL1201585",
28 + "parent_molecule_chembl_id": "CHEMBL1201585",
29 + "record_id": 1343415,
30 + "selectivity_comment": null,
31 + "site_id": null,
32 + "target_chembl_id": "CHEMBL1824",
33 + "variant_sequence": null
34 + }
35 + ]
36 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/search-fluorouracil.json +230 −0
@@ -0,0 +1,230 @@
1 +{
2 + "page_meta": {
3 + "limit": 20,
4 + "offset": 0,
5 + "total_count": 2,
6 + "next": null,
7 + "previous": null
8 + },
9 + "molecules": [
10 + {
11 + "molecule_chembl_id": "CHEMBL185",
12 + "pref_name": "FLUOROURACIL",
13 + "molecule_type": "Small molecule",
14 + "max_phase": "4.0",
15 + "first_approval": 1962,
16 + "molecule_synonyms": [
17 + {
18 + "molecule_synonym": "5-fluorouracil",
19 + "syn_type": "OTHER",
20 + "synonyms": "5-Fluorouracil"
21 + },
22 + {
23 + "molecule_synonym": "5-fluoruracil",
24 + "syn_type": "OTHER",
25 + "synonyms": "5-FLUORURACIL"
26 + },
27 + {
28 + "molecule_synonym": "5 FU",
29 + "syn_type": "RESEARCH_CODE",
30 + "synonyms": "5 FU"
31 + },
32 + {
33 + "molecule_synonym": "5-FU",
34 + "syn_type": "RESEARCH_CODE",
35 + "synonyms": "5-FU"
36 + },
37 + {
38 + "molecule_synonym": "5.f.u.",
39 + "syn_type": "TRADE_NAME",
40 + "synonyms": "5.F.U."
41 + },
42 + {
43 + "molecule_synonym": "Accusite",
44 + "syn_type": "TRADE_NAME",
45 + "synonyms": "ACCUSITE"
46 + },
47 + {
48 + "molecule_synonym": "Actikerall",
49 + "syn_type": "TRADE_NAME",
50 + "synonyms": "ACTIKERALL"
51 + },
52 + {
53 + "molecule_synonym": "Adrucil",
54 + "syn_type": "TRADE_NAME",
55 + "synonyms": "ADRUCIL"
56 + },
57 + {
58 + "molecule_synonym": "Carac",
59 + "syn_type": "TRADE_NAME",
60 + "synonyms": "CARAC"
61 + },
62 + {
63 + "molecule_synonym": "Efudex",
64 + "syn_type": "TRADE_NAME",
65 + "synonyms": "EFUDEX"
66 + },
67 + {
68 + "molecule_synonym": "Efudix",
69 + "syn_type": "TRADE_NAME",
70 + "synonyms": "EFUDIX"
71 + },
72 + {
73 + "molecule_synonym": "Fluoroplex",
74 + "syn_type": "TRADE_NAME",
75 + "synonyms": "FLUOROPLEX"
76 + },
77 + {
78 + "molecule_synonym": "Fluorouracil",
79 + "syn_type": "ATC",
80 + "synonyms": "FLUOROURACIL"
81 + },
82 + {
83 + "molecule_synonym": "Fluorouracil",
84 + "syn_type": "BAN",
85 + "synonyms": "FLUOROURACIL"
86 + },
87 + {
88 + "molecule_synonym": "Fluorouracil",
89 + "syn_type": "BNF",
90 + "synonyms": "FLUOROURACIL"
91 + },
92 + {
93 + "molecule_synonym": "Fluorouracil",
94 + "syn_type": "FDA",
95 + "synonyms": "FLUOROURACIL"
96 + },
97 + {
98 + "molecule_synonym": "Fluorouracil",
99 + "syn_type": "INN",
100 + "synonyms": "FLUOROURACIL"
101 + },
102 + {
103 + "molecule_synonym": "Fluorouracil",
104 + "syn_type": "JAN",
105 + "synonyms": "FLUOROURACIL"
106 + },
107 + {
108 + "molecule_synonym": "Fluorouracil",
109 + "syn_type": "MERCK_INDEX",
110 + "synonyms": "FLUOROURACIL"
111 + },
112 + {
113 + "molecule_synonym": "Fluorouracil",
114 + "syn_type": "OTHER",
115 + "synonyms": "FLUOROURACIL"
116 + },
117 + {
118 + "molecule_synonym": "Fluorouracil",
119 + "syn_type": "TRADE_NAME",
120 + "synonyms": "FLUOROURACIL"
121 + },
122 + {
123 + "molecule_synonym": "Fluorouracil",
124 + "syn_type": "USAN",
125 + "synonyms": "FLUOROURACIL"
126 + },
127 + {
128 + "molecule_synonym": "Fluorouracil",
129 + "syn_type": "USP",
130 + "synonyms": "FLUOROURACIL"
131 + },
132 + {
133 + "molecule_synonym": "Fluorouracil component of folfox",
134 + "syn_type": "TRADE_NAME",
135 + "synonyms": "FLUOROURACIL COMPONENT OF FOLFOX"
136 + },
137 + {
138 + "molecule_synonym": "Fluorouracilo",
139 + "syn_type": "INN_SPANISH",
140 + "synonyms": "FLUOROURACILO"
141 + },
142 + {
143 + "molecule_synonym": "Fluoro-uracil roche",
144 + "syn_type": "TRADE_NAME",
145 + "synonyms": "FLUORO-URACIL ROCHE"
146 + },
147 + {
148 + "molecule_synonym": "Fluorouracilum",
149 + "syn_type": "OTHER",
150 + "synonyms": "FLUOROURACILUM"
151 + },
152 + {
153 + "molecule_synonym": "Fluorouricil",
154 + "syn_type": "OTHER",
155 + "synonyms": "FLUOROURICIL"
156 + },
157 + {
158 + "molecule_synonym": "Fluracil",
159 + "syn_type": "OTHER",
160 + "synonyms": "FLURACIL"
161 + },
162 + {
163 + "molecule_synonym": "NSC-19893",
164 + "syn_type": "RESEARCH_CODE",
165 + "synonyms": "NSC-19893"
166 + },
167 + {
168 + "molecule_synonym": "Phthoruracil",
169 + "syn_type": "OTHER",
170 + "synonyms": "PHTHORURACIL"
171 + },
172 + {
173 + "molecule_synonym": "Queroplex",
174 + "syn_type": "OTHER",
175 + "synonyms": "QUEROPLEX"
176 + },
177 + {
178 + "molecule_synonym": "RO 2-9757",
179 + "syn_type": "RESEARCH_CODE",
180 + "synonyms": "RO 2-9757"
181 + },
182 + {
183 + "molecule_synonym": "RO-2-9757",
184 + "syn_type": "RESEARCH_CODE",
185 + "synonyms": "RO-2-9757"
186 + },
187 + {
188 + "molecule_synonym": "RO-29757",
189 + "syn_type": "RESEARCH_CODE",
190 + "synonyms": "RO-29757"
191 + },
192 + {
193 + "molecule_synonym": "Tolak",
194 + "syn_type": "TRADE_NAME",
195 + "synonyms": "TOLAK"
196 + }
197 + ],
198 + "molecule_hierarchy": {
199 + "active_chembl_id": "CHEMBL185",
200 + "molecule_chembl_id": "CHEMBL185",
201 + "parent_chembl_id": "CHEMBL185"
202 + },
203 + "atc_classifications": [
204 + "L01BC02",
205 + "L01BC52"
206 + ],
207 + "withdrawn_flag": false,
208 + "black_box_warning": 0,
209 + "oral": false,
210 + "parenteral": true,
211 + "topical": true,
212 + "prodrug": 0,
213 + "therapeutic_flag": true,
214 + "usan_stem": "-racil; -uracil",
215 + "usan_year": 1962,
216 + "score": 30.0,
217 + "cross_references": [
218 + {
219 + "xref_id": "fluorouracil",
220 + "xref_name": "fluorouracil",
221 + "xref_src": "DailyMed"
222 + }
223 + ]
224 + },
225 + {
226 + "pref_name": "BROKEN",
227 + "molecule_type": "Small molecule"
228 + }
229 + ]
230 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/search-imatinib.json +366 −0
@@ -0,0 +1,366 @@
1 +{
2 + "page_meta": {
3 + "limit": 20,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 3
8 + },
9 + "molecules": [
10 + {
11 + "molecule_chembl_id": "CHEMBL941",
12 + "pref_name": "IMATINIB",
13 + "molecule_type": "Small molecule",
14 + "max_phase": "4.0",
15 + "first_approval": 2001,
16 + "molecule_synonyms": [
17 + {
18 + "molecule_synonym": "CGP057148B",
19 + "syn_type": "PND",
20 + "synonyms": "CGP057148B"
21 + },
22 + {
23 + "molecule_synonym": "CGP-57148B",
24 + "syn_type": "PND",
25 + "synonyms": "CGP-57148B"
26 + },
27 + {
28 + "molecule_synonym": "Glamox",
29 + "syn_type": "OTHER",
30 + "synonyms": "GLAMOX"
31 + },
32 + {
33 + "molecule_synonym": "GLAMOX",
34 + "syn_type": "PND",
35 + "synonyms": "GLAMOX"
36 + },
37 + {
38 + "molecule_synonym": "GLEEVEC",
39 + "syn_type": "PND",
40 + "synonyms": "Gleevec"
41 + },
42 + {
43 + "molecule_synonym": "GLEEVEC,STI-571",
44 + "syn_type": "PND",
45 + "synonyms": "Gleevec,STI-571"
46 + },
47 + {
48 + "molecule_synonym": "GLIVEC",
49 + "syn_type": "PND",
50 + "synonyms": "Glivec"
51 + },
52 + {
53 + "molecule_synonym": "Imatinib",
54 + "syn_type": "ATC",
55 + "synonyms": "IMATINIB"
56 + },
57 + {
58 + "molecule_synonym": "Imatinib",
59 + "syn_type": "BAN",
60 + "synonyms": "IMATINIB"
61 + },
62 + {
63 + "molecule_synonym": "Imatinib",
64 + "syn_type": "INN",
65 + "synonyms": "IMATINIB"
66 + },
67 + {
68 + "molecule_synonym": "Imatinib",
69 + "syn_type": "MERCK_INDEX",
70 + "synonyms": "IMATINIB"
71 + },
72 + {
73 + "molecule_synonym": "Imatinib",
74 + "syn_type": "OTHER",
75 + "synonyms": "IMATINIB"
76 + },
77 + {
78 + "molecule_synonym": "IMATINIB",
79 + "syn_type": "PND",
80 + "synonyms": "IMATINIB"
81 + },
82 + {
83 + "molecule_synonym": "IMATINIB MESYLATE",
84 + "syn_type": "PND",
85 + "synonyms": "Imatinib mesylate"
86 + },
87 + {
88 + "molecule_synonym": "IMATINIB MESYLATEGLEEVECGLIVECIMATINIBSTI571CGP 571484-[(4-METHYL-1-PIPERAZINYL)METHYL]-N-[4-METHYL-3-[[4-(3-PYRIDINYL)-2-PYRIMIDINYL]AMINO]-PHENYL]BENZAMIDE",
89 + "syn_type": "PND",
90 + "synonyms": "Imatinib mesylateGleevecGlivecImatinibSTI571CGP 571484-[(4-Methyl-1-piperazinyl)methyl]-N-[4-methyl-3-[[4-(3-pyridinyl)-2-pyrimidinyl]amino]-phenyl]benzamide"
91 + },
92 + {
93 + "molecule_synonym": "IMATINIB (STI571)",
94 + "syn_type": "PND",
95 + "synonyms": "Imatinib (STI571)"
96 + },
97 + {
98 + "molecule_synonym": "NSC-743414",
99 + "syn_type": "PND",
100 + "synonyms": "NSC-743414"
101 + },
102 + {
103 + "molecule_synonym": "NSC-743414",
104 + "syn_type": "RESEARCH_CODE",
105 + "synonyms": "NSC-743414"
106 + },
107 + {
108 + "molecule_synonym": "NSC-759854",
109 + "syn_type": "PND",
110 + "synonyms": "NSC-759854"
111 + },
112 + {
113 + "molecule_synonym": "NSC-759854",
114 + "syn_type": "RESEARCH_CODE",
115 + "synonyms": "NSC-759854"
116 + },
117 + {
118 + "molecule_synonym": "ST-1571",
119 + "syn_type": "PND",
120 + "synonyms": "ST-1571"
121 + },
122 + {
123 + "molecule_synonym": "STI-571",
124 + "syn_type": "PND",
125 + "synonyms": "STI-571"
126 + },
127 + {
128 + "molecule_synonym": "STI571",
129 + "syn_type": "PND",
130 + "synonyms": "STI571"
131 + },
132 + {
133 + "molecule_synonym": "STI571,GLEEVEC",
134 + "syn_type": "PND",
135 + "synonyms": "STI571,Gleevec"
136 + }
137 + ],
138 + "molecule_hierarchy": {
139 + "active_chembl_id": "CHEMBL941",
140 + "molecule_chembl_id": "CHEMBL941",
141 + "parent_chembl_id": "CHEMBL941"
142 + },
143 + "atc_classifications": [
144 + "L01EA01"
145 + ],
146 + "withdrawn_flag": false,
147 + "black_box_warning": 0,
148 + "oral": true,
149 + "parenteral": false,
150 + "topical": false,
151 + "prodrug": 0,
152 + "therapeutic_flag": true,
153 + "usan_stem": "-tinib",
154 + "usan_year": null,
155 + "score": 36.0,
156 + "cross_references": []
157 + },
158 + {
159 + "molecule_chembl_id": "CHEMBL1642",
160 + "pref_name": "IMATINIB MESYLATE",
161 + "molecule_type": "Small molecule",
162 + "max_phase": "4.0",
163 + "first_approval": 2001,
164 + "molecule_synonyms": [
165 + {
166 + "molecule_synonym": "Gleevec",
167 + "syn_type": "TRADE_NAME",
168 + "synonyms": "GLEEVEC"
169 + },
170 + {
171 + "molecule_synonym": "Glivec",
172 + "syn_type": "TRADE_NAME",
173 + "synonyms": "GLIVEC"
174 + },
175 + {
176 + "molecule_synonym": "Imatinib accord",
177 + "syn_type": "OTHER",
178 + "synonyms": "IMATINIB ACCORD"
179 + },
180 + {
181 + "molecule_synonym": "Imatinib accord",
182 + "syn_type": "TRADE_NAME",
183 + "synonyms": "IMATINIB ACCORD"
184 + },
185 + {
186 + "molecule_synonym": "Imatinib actavis",
187 + "syn_type": "TRADE_NAME",
188 + "synonyms": "IMATINIB ACTAVIS"
189 + },
190 + {
191 + "molecule_synonym": "Imatinib (as mesilate)",
192 + "syn_type": "OTHER",
193 + "synonyms": "IMATINIB (AS MESILATE)"
194 + },
195 + {
196 + "molecule_synonym": "Imatinib koanaa",
197 + "syn_type": "TRADE_NAME",
198 + "synonyms": "IMATINIB KOANAA"
199 + },
200 + {
201 + "molecule_synonym": "Imatinib medac",
202 + "syn_type": "OTHER",
203 + "synonyms": "IMATINIB MEDAC"
204 + },
205 + {
206 + "molecule_synonym": "Imatinib medac",
207 + "syn_type": "TRADE_NAME",
208 + "synonyms": "IMATINIB MEDAC"
209 + },
210 + {
211 + "molecule_synonym": "Imatinib mesilate",
212 + "syn_type": "BNF",
213 + "synonyms": "IMATINIB MESILATE"
214 + },
215 + {
216 + "molecule_synonym": "Imatinib mesilate",
217 + "syn_type": "JAN",
218 + "synonyms": "IMATINIB MESILATE"
219 + },
220 + {
221 + "molecule_synonym": "Imatinib mesilate",
222 + "syn_type": "OTHER",
223 + "synonyms": "IMATINIB MESILATE"
224 + },
225 + {
226 + "molecule_synonym": "Imatinib mesylate",
227 + "syn_type": "EMA",
228 + "synonyms": "IMATINIB MESYLATE"
229 + },
230 + {
231 + "molecule_synonym": "Imatinib mesylate",
232 + "syn_type": "FDA",
233 + "synonyms": "IMATINIB MESYLATE"
234 + },
235 + {
236 + "molecule_synonym": "Imatinib mesylate",
237 + "syn_type": "OTHER",
238 + "synonyms": "IMATINIB MESYLATE"
239 + },
240 + {
241 + "molecule_synonym": "Imatinib mesylate",
242 + "syn_type": "TRADE_NAME",
243 + "synonyms": "IMATINIB MESYLATE"
244 + },
245 + {
246 + "molecule_synonym": "Imatinib mesylate",
247 + "syn_type": "USAN",
248 + "synonyms": "IMATINIB MESYLATE"
249 + },
250 + {
251 + "molecule_synonym": "Imatinib methane sulfonate",
252 + "syn_type": "OTHER",
253 + "synonyms": "IMATINIB METHANE SULFONATE"
254 + },
255 + {
256 + "molecule_synonym": "Imatinib methanesulfonate",
257 + "syn_type": "MERCK_INDEX",
258 + "synonyms": "IMATINIB METHANESULFONATE"
259 + },
260 + {
261 + "molecule_synonym": "Imatinib teva",
262 + "syn_type": "TRADE_NAME",
263 + "synonyms": "IMATINIB TEVA"
264 + },
265 + {
266 + "molecule_synonym": "Imatinib teva b.v.",
267 + "syn_type": "TRADE_NAME",
268 + "synonyms": "IMATINIB TEVA B.V."
269 + },
270 + {
271 + "molecule_synonym": "NSC-716051",
272 + "syn_type": "RESEARCH_CODE",
273 + "synonyms": "NSC-716051"
274 + },
275 + {
276 + "molecule_synonym": "QTI-571",
277 + "syn_type": "RESEARCH_CODE",
278 + "synonyms": "QTI-571"
279 + },
280 + {
281 + "molecule_synonym": "Qti571",
282 + "syn_type": "OTHER",
283 + "synonyms": "QTI571"
284 + },
285 + {
286 + "molecule_synonym": "QTI571",
287 + "syn_type": "RESEARCH_CODE",
288 + "synonyms": "QTI571"
289 + },
290 + {
291 + "molecule_synonym": "STI 571",
292 + "syn_type": "RESEARCH_CODE",
293 + "synonyms": "STI 571"
294 + },
295 + {
296 + "molecule_synonym": "STI-571",
297 + "syn_type": "RESEARCH_CODE",
298 + "synonyms": "STI-571"
299 + },
300 + {
301 + "molecule_synonym": "STI571",
302 + "syn_type": "RESEARCH_CODE",
303 + "synonyms": "STI571"
304 + }
305 + ],
306 + "molecule_hierarchy": {
307 + "active_chembl_id": "CHEMBL941",
308 + "molecule_chembl_id": "CHEMBL1642",
309 + "parent_chembl_id": "CHEMBL941"
310 + },
311 + "atc_classifications": [],
312 + "withdrawn_flag": false,
313 + "black_box_warning": 0,
314 + "oral": true,
315 + "parenteral": false,
316 + "topical": false,
317 + "prodrug": 0,
318 + "therapeutic_flag": true,
319 + "usan_stem": "-tinib",
320 + "usan_year": null,
321 + "score": 36.0,
322 + "cross_references": [
323 + {
324 + "xref_id": "human/EPAR/imatinib-teva-bv",
325 + "xref_name": "imatinib-teva-bv",
326 + "xref_src": "EMA"
327 + },
328 + {
329 + "xref_id": "human/EPAR/imatinib-actavis",
330 + "xref_name": "imatinib-actavis",
331 + "xref_src": "EMA"
332 + },
333 + {
334 + "xref_id": "human/EPAR/imatinib-teva",
335 + "xref_name": "imatinib-teva",
336 + "xref_src": "EMA"
337 + }
338 + ]
339 + },
340 + {
341 + "molecule_chembl_id": "CHEMBL3040018",
342 + "pref_name": null,
343 + "molecule_type": "Small molecule",
344 + "max_phase": null,
345 + "first_approval": null,
346 + "molecule_synonyms": [],
347 + "molecule_hierarchy": {
348 + "active_chembl_id": "CHEMBL3040018",
349 + "molecule_chembl_id": "CHEMBL3040018",
350 + "parent_chembl_id": "CHEMBL3040018"
351 + },
352 + "atc_classifications": [],
353 + "withdrawn_flag": false,
354 + "black_box_warning": 0,
355 + "oral": false,
356 + "parenteral": false,
357 + "topical": false,
358 + "prodrug": -1,
359 + "therapeutic_flag": false,
360 + "usan_stem": null,
361 + "usan_year": null,
362 + "score": 17.0,
363 + "cross_references": []
364 + }
365 + ]
366 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/search-none.json +10 −0
@@ -0,0 +1,10 @@
1 +{
2 + "page_meta": {
3 + "limit": 20,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 0
8 + },
9 + "molecules": []
10 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/search-osimertinib.json +226 −0
@@ -0,0 +1,226 @@
1 +{
2 + "page_meta": {
3 + "limit": 20,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 3
8 + },
9 + "molecules": [
10 + {
11 + "molecule_chembl_id": "CHEMBL3545063",
12 + "pref_name": "OSIMERTINIB MESYLATE",
13 + "molecule_type": "Small molecule",
14 + "max_phase": "4.0",
15 + "first_approval": 2015,
16 + "molecule_synonyms": [
17 + {
18 + "molecule_synonym": "AZD9291 mesylate",
19 + "syn_type": "RESEARCH_CODE",
20 + "synonyms": "AZD9291 mesylate"
21 + },
22 + {
23 + "molecule_synonym": "AZD-9291 MESYLATE",
24 + "syn_type": "RESEARCH_CODE",
25 + "synonyms": "AZD-9291 MESYLATE"
26 + },
27 + {
28 + "molecule_synonym": "Mereletinib mesilate",
29 + "syn_type": "OTHER",
30 + "synonyms": "MERELETINIB MESILATE"
31 + },
32 + {
33 + "molecule_synonym": "Mereletinib mesylate",
34 + "syn_type": "OTHER",
35 + "synonyms": "MERELETINIB MESYLATE"
36 + },
37 + {
38 + "molecule_synonym": "Osimertinib mesilate",
39 + "syn_type": "EMA",
40 + "synonyms": "OSIMERTINIB MESILATE"
41 + },
42 + {
43 + "molecule_synonym": "Osimertinib mesilate",
44 + "syn_type": "JAN",
45 + "synonyms": "OSIMERTINIB MESILATE"
46 + },
47 + {
48 + "molecule_synonym": "Osimertinib mesilate",
49 + "syn_type": "OTHER",
50 + "synonyms": "OSIMERTINIB MESILATE"
51 + },
52 + {
53 + "molecule_synonym": "Osimertinib mesylate",
54 + "syn_type": "EMA",
55 + "synonyms": "OSIMERTINIB MESYLATE"
56 + },
57 + {
58 + "molecule_synonym": "Osimertinib mesylate",
59 + "syn_type": "FDA",
60 + "synonyms": "OSIMERTINIB MESYLATE"
61 + },
62 + {
63 + "molecule_synonym": "Osimertinib mesylate",
64 + "syn_type": "MERCK_INDEX",
65 + "synonyms": "OSIMERTINIB MESYLATE"
66 + },
67 + {
68 + "molecule_synonym": "Osimertinib mesylate",
69 + "syn_type": "OTHER",
70 + "synonyms": "OSIMERTINIB MESYLATE"
71 + },
72 + {
73 + "molecule_synonym": "Osimertinib mesylate",
74 + "syn_type": "USAN",
75 + "synonyms": "OSIMERTINIB MESYLATE"
76 + },
77 + {
78 + "molecule_synonym": "Tagrisso",
79 + "syn_type": "TRADE_NAME",
80 + "synonyms": "TAGRISSO"
81 + }
82 + ],
83 + "molecule_hierarchy": {
84 + "active_chembl_id": "CHEMBL3353410",
85 + "molecule_chembl_id": "CHEMBL3545063",
86 + "parent_chembl_id": "CHEMBL3353410"
87 + },
88 + "atc_classifications": [],
89 + "withdrawn_flag": false,
90 + "black_box_warning": 0,
91 + "oral": true,
92 + "parenteral": false,
93 + "topical": false,
94 + "prodrug": 0,
95 + "therapeutic_flag": true,
96 + "usan_stem": "-tinib",
97 + "usan_year": 2015,
98 + "score": 38.0,
99 + "cross_references": [
100 + {
101 + "xref_id": "human/EPAR/tagrisso",
102 + "xref_name": "tagrisso",
103 + "xref_src": "EMA"
104 + },
105 + {
106 + "xref_id": "osimertinib%20mesylate",
107 + "xref_name": "osimertinib mesylate",
108 + "xref_src": "DailyMed"
109 + }
110 + ]
111 + },
112 + {
113 + "molecule_chembl_id": "CHEMBL3353410",
114 + "pref_name": "OSIMERTINIB",
115 + "molecule_type": "Small molecule",
116 + "max_phase": "4.0",
117 + "first_approval": 2015,
118 + "molecule_synonyms": [
119 + {
120 + "molecule_synonym": "Azd-9291",
121 + "syn_type": "OTHER",
122 + "synonyms": "AZD-9291"
123 + },
124 + {
125 + "molecule_synonym": "AZD-9291",
126 + "syn_type": "RESEARCH_CODE",
127 + "synonyms": "AZD-9291"
128 + },
129 + {
130 + "molecule_synonym": "AZD9291",
131 + "syn_type": "RESEARCH_CODE",
132 + "synonyms": "AZD9291"
133 + },
134 + {
135 + "molecule_synonym": "AZD-9291 FREE BASE",
136 + "syn_type": "RESEARCH_CODE",
137 + "synonyms": "AZD-9291 FREE BASE"
138 + },
139 + {
140 + "molecule_synonym": "Mereletinib",
141 + "syn_type": "OTHER",
142 + "synonyms": "MERELETINIB"
143 + },
144 + {
145 + "molecule_synonym": "Osimertinib",
146 + "syn_type": "ATC",
147 + "synonyms": "OSIMERTINIB"
148 + },
149 + {
150 + "molecule_synonym": "Osimertinib",
151 + "syn_type": "BNF",
152 + "synonyms": "OSIMERTINIB"
153 + },
154 + {
155 + "molecule_synonym": "Osimertinib",
156 + "syn_type": "INN",
157 + "synonyms": "OSIMERTINIB"
158 + },
159 + {
160 + "molecule_synonym": "Osimertinib",
161 + "syn_type": "MERCK_INDEX",
162 + "synonyms": "OSIMERTINIB"
163 + },
164 + {
165 + "molecule_synonym": "Osimertinib",
166 + "syn_type": "OTHER",
167 + "synonyms": "OSIMERTINIB"
168 + },
169 + {
170 + "molecule_synonym": "Osimertinib",
171 + "syn_type": "USAN",
172 + "synonyms": "OSIMERTINIB"
173 + },
174 + {
175 + "molecule_synonym": "Tagrisso",
176 + "syn_type": "TRADE_NAME",
177 + "synonyms": "TAGRISSO"
178 + }
179 + ],
180 + "molecule_hierarchy": {
181 + "active_chembl_id": "CHEMBL3353410",
182 + "molecule_chembl_id": "CHEMBL3353410",
183 + "parent_chembl_id": "CHEMBL3353410"
184 + },
185 + "atc_classifications": [
186 + "L01EB04"
187 + ],
188 + "withdrawn_flag": false,
189 + "black_box_warning": 0,
190 + "oral": true,
191 + "parenteral": false,
192 + "topical": false,
193 + "prodrug": 0,
194 + "therapeutic_flag": true,
195 + "usan_stem": "-tinib",
196 + "usan_year": 2015,
197 + "score": 34.0,
198 + "cross_references": []
199 + },
200 + {
201 + "molecule_chembl_id": "CHEMBL4472934",
202 + "pref_name": null,
203 + "molecule_type": "Unknown",
204 + "max_phase": null,
205 + "first_approval": null,
206 + "molecule_synonyms": [],
207 + "molecule_hierarchy": {
208 + "active_chembl_id": "CHEMBL3353410",
209 + "molecule_chembl_id": "CHEMBL4472934",
210 + "parent_chembl_id": "CHEMBL3353410"
211 + },
212 + "atc_classifications": [],
213 + "withdrawn_flag": false,
214 + "black_box_warning": 0,
215 + "oral": false,
216 + "parenteral": false,
217 + "topical": false,
218 + "prodrug": -1,
219 + "therapeutic_flag": false,
220 + "usan_stem": null,
221 + "usan_year": null,
222 + "score": 32.0,
223 + "cross_references": []
224 + }
225 + ]
226 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/search-trastuzumab.json +707 −0
@@ -0,0 +1,707 @@
1 +{
2 + "page_meta": {
3 + "limit": 20,
4 + "next": null,
5 + "offset": 0,
6 + "previous": null,
7 + "total_count": 4
8 + },
9 + "molecules": [
10 + {
11 + "molecule_chembl_id": "CHEMBL1201585",
12 + "pref_name": "TRASTUZUMAB",
13 + "molecule_type": "Antibody",
14 + "max_phase": "4.0",
15 + "first_approval": 1998,
16 + "molecule_synonyms": [
17 + {
18 + "molecule_synonym": "4D5V8",
19 + "syn_type": "RESEARCH_CODE",
20 + "synonyms": "4D5V8"
21 + },
22 + {
23 + "molecule_synonym": "ABP 980",
24 + "syn_type": "RESEARCH_CODE",
25 + "synonyms": "ABP 980"
26 + },
27 + {
28 + "molecule_synonym": "ABP-980",
29 + "syn_type": "RESEARCH_CODE",
30 + "synonyms": "ABP-980"
31 + },
32 + {
33 + "molecule_synonym": "ABP980",
34 + "syn_type": "RESEARCH_CODE",
35 + "synonyms": "ABP980"
36 + },
37 + {
38 + "molecule_synonym": "Abp-980 (trastuzumab biosimilar)",
39 + "syn_type": "OTHER",
40 + "synonyms": "ABP-980 (TRASTUZUMAB BIOSIMILAR)"
41 + },
42 + {
43 + "molecule_synonym": "Bmab-200",
44 + "syn_type": "OTHER",
45 + "synonyms": "BMAB-200"
46 + },
47 + {
48 + "molecule_synonym": "Canhera",
49 + "syn_type": "OTHER",
50 + "synonyms": "CANHERA"
51 + },
52 + {
53 + "molecule_synonym": "Dmb-3111",
54 + "syn_type": "OTHER",
55 + "synonyms": "DMB-3111"
56 + },
57 + {
58 + "molecule_synonym": "DMB-3111 (TRASTUZUMAB BIOSIMILAR)",
59 + "syn_type": "RESEARCH_CODE",
60 + "synonyms": "DMB-3111 (TRASTUZUMAB BIOSIMILAR)"
61 + },
62 + {
63 + "molecule_synonym": "EG-12014",
64 + "syn_type": "RESEARCH_CODE",
65 + "synonyms": "EG-12014"
66 + },
67 + {
68 + "molecule_synonym": "EG12014",
69 + "syn_type": "RESEARCH_CODE",
70 + "synonyms": "EG12014"
71 + },
72 + {
73 + "molecule_synonym": "Eg12014 trastuzumab biosimilar",
74 + "syn_type": "OTHER",
75 + "synonyms": "EG12014 TRASTUZUMAB BIOSIMILAR"
76 + },
77 + {
78 + "molecule_synonym": "Eg-12014 (trastuzumab biosimilar)",
79 + "syn_type": "OTHER",
80 + "synonyms": "EG-12014 (TRASTUZUMAB BIOSIMILAR)"
81 + },
82 + {
83 + "molecule_synonym": "Herceptin",
84 + "syn_type": "TRADE_NAME",
85 + "synonyms": "HERCEPTIN"
86 + },
87 + {
88 + "molecule_synonym": "Hercessi",
89 + "syn_type": "OTHER",
90 + "synonyms": "HERCESSI"
91 + },
92 + {
93 + "molecule_synonym": "Herwenda",
94 + "syn_type": "TRADE_NAME",
95 + "synonyms": "HERWENDA"
96 + },
97 + {
98 + "molecule_synonym": "Herzuma",
99 + "syn_type": "OTHER",
100 + "synonyms": "HERZUMA"
101 + },
102 + {
103 + "molecule_synonym": "Herzuma",
104 + "syn_type": "TRADE_NAME",
105 + "synonyms": "HERZUMA"
106 + },
107 + {
108 + "molecule_synonym": "Hlx 02",
109 + "syn_type": "OTHER",
110 + "synonyms": "HLX 02"
111 + },
112 + {
113 + "molecule_synonym": "HLX-02",
114 + "syn_type": "RESEARCH_CODE",
115 + "synonyms": "HLX-02"
116 + },
117 + {
118 + "molecule_synonym": "Hlx02",
119 + "syn_type": "OTHER",
120 + "synonyms": "HLX02"
121 + },
122 + {
123 + "molecule_synonym": "Hlx-02 trastuzumab biosimilar",
124 + "syn_type": "OTHER",
125 + "synonyms": "HLX-02 TRASTUZUMAB BIOSIMILAR"
126 + },
127 + {
128 + "molecule_synonym": "Kadcyla",
129 + "syn_type": "TRADE_NAME",
130 + "synonyms": "KADCYLA"
131 + },
132 + {
133 + "molecule_synonym": "Kanjinti",
134 + "syn_type": "OTHER",
135 + "synonyms": "KANJINTI"
136 + },
137 + {
138 + "molecule_synonym": "Kanjinti",
139 + "syn_type": "TRADE_NAME",
140 + "synonyms": "KANJINTI"
141 + },
142 + {
143 + "molecule_synonym": "Ogivri",
144 + "syn_type": "OTHER",
145 + "synonyms": "OGIVRI"
146 + },
147 + {
148 + "molecule_synonym": "Ogivri",
149 + "syn_type": "TRADE_NAME",
150 + "synonyms": "OGIVRI"
151 + },
152 + {
153 + "molecule_synonym": "Ontruzant",
154 + "syn_type": "OTHER",
155 + "synonyms": "ONTRUZANT"
156 + },
157 + {
158 + "molecule_synonym": "Ontruzant",
159 + "syn_type": "TRADE_NAME",
160 + "synonyms": "ONTRUZANT"
161 + },
162 + {
163 + "molecule_synonym": "Pf-05280014",
164 + "syn_type": "OTHER",
165 + "synonyms": "PF-05280014"
166 + },
167 + {
168 + "molecule_synonym": "R-597",
169 + "syn_type": "RESEARCH_CODE",
170 + "synonyms": "R-597"
171 + },
172 + {
173 + "molecule_synonym": "Rhumab her2",
174 + "syn_type": "OTHER",
175 + "synonyms": "RHUMAB HER2"
176 + },
177 + {
178 + "molecule_synonym": "RHUMABHER2",
179 + "syn_type": "RESEARCH_CODE",
180 + "synonyms": "RHUMABHER2"
181 + },
182 + {
183 + "molecule_synonym": "RO-0452317",
184 + "syn_type": "RESEARCH_CODE",
185 + "synonyms": "RO-0452317"
186 + },
187 + {
188 + "molecule_synonym": "RO0452317",
189 + "syn_type": "RESEARCH_CODE",
190 + "synonyms": "RO0452317"
191 + },
192 + {
193 + "molecule_synonym": "Sb-3 (trastuzumab biosimilar)",
194 + "syn_type": "OTHER",
195 + "synonyms": "SB-3 (TRASTUZUMAB BIOSIMILAR)"
196 + },
197 + {
198 + "molecule_synonym": "SYD-977",
199 + "syn_type": "RESEARCH_CODE",
200 + "synonyms": "SYD-977"
201 + },
202 + {
203 + "molecule_synonym": "SYD977",
204 + "syn_type": "RESEARCH_CODE",
205 + "synonyms": "SYD977"
206 + },
207 + {
208 + "molecule_synonym": "Trastuzumab",
209 + "syn_type": "ATC",
210 + "synonyms": "TRASTUZUMAB"
211 + },
212 + {
213 + "molecule_synonym": "Trastuzumab",
214 + "syn_type": "BAN",
215 + "synonyms": "TRASTUZUMAB"
216 + },
217 + {
218 + "molecule_synonym": "Trastuzumab",
219 + "syn_type": "BNF",
220 + "synonyms": "TRASTUZUMAB"
221 + },
222 + {
223 + "molecule_synonym": "Trastuzumab",
224 + "syn_type": "EMA",
225 + "synonyms": "TRASTUZUMAB"
226 + },
227 + {
228 + "molecule_synonym": "Trastuzumab",
229 + "syn_type": "FDA",
230 + "synonyms": "TRASTUZUMAB"
231 + },
232 + {
233 + "molecule_synonym": "Trastuzumab",
234 + "syn_type": "INN",
235 + "synonyms": "TRASTUZUMAB"
236 + },
237 + {
238 + "molecule_synonym": "Trastuzumab",
239 + "syn_type": "MERCK_INDEX",
240 + "synonyms": "TRASTUZUMAB"
241 + },
242 + {
243 + "molecule_synonym": "Trastuzumab",
244 + "syn_type": "OTHER",
245 + "synonyms": "TRASTUZUMAB"
246 + },
247 + {
248 + "molecule_synonym": "Trastuzumab",
249 + "syn_type": "USAN",
250 + "synonyms": "TRASTUZUMAB"
251 + },
252 + {
253 + "molecule_synonym": "Trastuzumab anns",
254 + "syn_type": "OTHER",
255 + "synonyms": "TRASTUZUMAB ANNS"
256 + },
257 + {
258 + "molecule_synonym": "Trastuzumab-anns",
259 + "syn_type": "OTHER",
260 + "synonyms": "TRASTUZUMAB-ANNS"
261 + },
262 + {
263 + "molecule_synonym": "Trastuzumab beta",
264 + "syn_type": "INN",
265 + "synonyms": "TRASTUZUMAB BETA"
266 + },
267 + {
268 + "molecule_synonym": "Trastuzumab biosimilar (abp-980)",
269 + "syn_type": "OTHER",
270 + "synonyms": "TRASTUZUMAB BIOSIMILAR (ABP-980)"
271 + },
272 + {
273 + "molecule_synonym": "Trastuzumab biosimilar (amgen)",
274 + "syn_type": "OTHER",
275 + "synonyms": "TRASTUZUMAB BIOSIMILAR (AMGEN)"
276 + },
277 + {
278 + "molecule_synonym": "Trastuzumab biosimilar (ct-p6)",
279 + "syn_type": "OTHER",
280 + "synonyms": "TRASTUZUMAB BIOSIMILAR (CT-P6)"
281 + },
282 + {
283 + "molecule_synonym": "Trastuzumab biosimilar (pfizer)",
284 + "syn_type": "OTHER",
285 + "synonyms": "TRASTUZUMAB BIOSIMILAR (PFIZER)"
286 + },
287 + {
288 + "molecule_synonym": "Trastuzumab biosimilar (sb-3)",
289 + "syn_type": "OTHER",
290 + "synonyms": "TRASTUZUMAB BIOSIMILAR (SB-3)"
291 + },
292 + {
293 + "molecule_synonym": "Trastuzumab component of herceptin hylecta",
294 + "syn_type": "TRADE_NAME",
295 + "synonyms": "TRASTUZUMAB COMPONENT OF HERCEPTIN HYLECTA"
296 + },
297 + {
298 + "molecule_synonym": "Trastuzumab component of phesgo",
299 + "syn_type": "TRADE_NAME",
300 + "synonyms": "TRASTUZUMAB COMPONENT OF PHESGO"
301 + },
302 + {
303 + "molecule_synonym": "Trastuzumab dkst",
304 + "syn_type": "OTHER",
305 + "synonyms": "TRASTUZUMAB DKST"
306 + },
307 + {
308 + "molecule_synonym": "Trastuzumab-dkst",
309 + "syn_type": "OTHER",
310 + "synonyms": "TRASTUZUMAB-DKST"
311 + },
312 + {
313 + "molecule_synonym": "Trastuzumab dttb",
314 + "syn_type": "OTHER",
315 + "synonyms": "TRASTUZUMAB DTTB"
316 + },
317 + {
318 + "molecule_synonym": "Trastuzumab-dttb",
319 + "syn_type": "OTHER",
320 + "synonyms": "TRASTUZUMAB-DTTB"
321 + },
322 + {
323 + "molecule_synonym": "Trastuzumab (herceptin)",
324 + "syn_type": "OTHER",
325 + "synonyms": "TRASTUZUMAB (HERCEPTIN)"
326 + },
327 + {
328 + "molecule_synonym": "Trastuzumab-oysk",
329 + "syn_type": "OTHER",
330 + "synonyms": "TRASTUZUMAB-OYSK"
331 + },
332 + {
333 + "molecule_synonym": "Trastuzumab-pfizer",
334 + "syn_type": "OTHER",
335 + "synonyms": "TRASTUZUMAB-PFIZER"
336 + },
337 + {
338 + "molecule_synonym": "Trastuzumab pkrb",
339 + "syn_type": "OTHER",
340 + "synonyms": "TRASTUZUMAB PKRB"
341 + },
342 + {
343 + "molecule_synonym": "Trastuzumab-pkrb",
344 + "syn_type": "OTHER",
345 + "synonyms": "TRASTUZUMAB-PKRB"
346 + },
347 + {
348 + "molecule_synonym": "Trastuzumab qyyp",
349 + "syn_type": "OTHER",
350 + "synonyms": "TRASTUZUMAB QYYP"
351 + },
352 + {
353 + "molecule_synonym": "Trastuzumab-qyyp",
354 + "syn_type": "OTHER",
355 + "synonyms": "TRASTUZUMAB-QYYP"
356 + },
357 + {
358 + "molecule_synonym": "Trastuzumab-strf",
359 + "syn_type": "OTHER",
360 + "synonyms": "TRASTUZUMAB-STRF"
361 + },
362 + {
363 + "molecule_synonym": "Trastuzumab-zzxf",
364 + "syn_type": "OTHER",
365 + "synonyms": "TRASTUZUMAB-ZZXF"
366 + },
367 + {
368 + "molecule_synonym": "Trazimera",
369 + "syn_type": "OTHER",
370 + "synonyms": "TRAZIMERA"
371 + },
372 + {
373 + "molecule_synonym": "Trazimera",
374 + "syn_type": "TRADE_NAME",
375 + "synonyms": "TRAZIMERA"
376 + },
377 + {
378 + "molecule_synonym": "Zercepac",
379 + "syn_type": "OTHER",
380 + "synonyms": "ZERCEPAC"
381 + },
382 + {
383 + "molecule_synonym": "Zercepac",
384 + "syn_type": "TRADE_NAME",
385 + "synonyms": "ZERCEPAC"
386 + }
387 + ],
388 + "molecule_hierarchy": {
389 + "active_chembl_id": "CHEMBL1201585",
390 + "molecule_chembl_id": "CHEMBL1201585",
391 + "parent_chembl_id": "CHEMBL1201585"
392 + },
393 + "atc_classifications": [
394 + "L01FD01"
395 + ],
396 + "withdrawn_flag": false,
397 + "black_box_warning": 1,
398 + "oral": false,
399 + "parenteral": true,
400 + "topical": false,
401 + "prodrug": 0,
402 + "therapeutic_flag": true,
403 + "usan_stem": "-mab; -mab",
404 + "usan_year": null,
405 + "score": 37.0,
406 + "cross_references": [
407 + {
408 + "xref_id": "trastuzumab-dkst",
409 + "xref_name": "trastuzumab-dkst",
410 + "xref_src": "DailyMed"
411 + },
412 + {
413 + "xref_id": "human/EPAR/ogivri",
414 + "xref_name": "ogivri",
415 + "xref_src": "EMA"
416 + },
417 + {
418 + "xref_id": "human/EPAR/herwenda",
419 + "xref_name": "herwenda",
420 + "xref_src": "EMA"
421 + }
422 + ]
423 + },
424 + {
425 + "molecule_chembl_id": "CHEMBL1743082",
426 + "pref_name": "TRASTUZUMAB EMTANSINE",
427 + "molecule_type": "Antibody drug conjugate",
428 + "max_phase": "4.0",
429 + "first_approval": 2013,
430 + "molecule_synonyms": [
431 + {
432 + "molecule_synonym": "Ado-trastuzumab emtansine",
433 + "syn_type": "FDA",
434 + "synonyms": "ADO-TRASTUZUMAB EMTANSINE"
435 + },
436 + {
437 + "molecule_synonym": "Kadcyla",
438 + "syn_type": "TRADE_NAME",
439 + "synonyms": "KADCYLA"
440 + },
441 + {
442 + "molecule_synonym": "PRO-132365",
443 + "syn_type": "RESEARCH_CODE",
444 + "synonyms": "PRO-132365"
445 + },
446 + {
447 + "molecule_synonym": "RG-3502",
448 + "syn_type": "RESEARCH_CODE",
449 + "synonyms": "RG-3502"
450 + },
451 + {
452 + "molecule_synonym": "T-dm1",
453 + "syn_type": "OTHER",
454 + "synonyms": "T-DM1"
455 + },
456 + {
457 + "molecule_synonym": "TDM-1",
458 + "syn_type": "RESEARCH_CODE",
459 + "synonyms": "TDM-1"
460 + },
461 + {
462 + "molecule_synonym": "TDM1",
463 + "syn_type": "RESEARCH_CODE",
464 + "synonyms": "TDM1"
465 + },
466 + {
467 + "molecule_synonym": "Trastuzumab emtansina",
468 + "syn_type": "INN_SPANISH",
469 + "synonyms": "TRASTUZUMAB EMTANSINA"
470 + },
471 + {
472 + "molecule_synonym": "Trastuzumab emtansine",
473 + "syn_type": "ATC",
474 + "synonyms": "TRASTUZUMAB EMTANSINE"
475 + },
476 + {
477 + "molecule_synonym": "Trastuzumab emtansine",
478 + "syn_type": "EMA",
479 + "synonyms": "TRASTUZUMAB EMTANSINE"
480 + },
481 + {
482 + "molecule_synonym": "Trastuzumab emtansine",
483 + "syn_type": "INN",
484 + "synonyms": "TRASTUZUMAB EMTANSINE"
485 + },
486 + {
487 + "molecule_synonym": "Trastuzumab emtansine",
488 + "syn_type": "OTHER",
489 + "synonyms": "TRASTUZUMAB EMTANSINE"
490 + },
491 + {
492 + "molecule_synonym": "Trastuzumab emtansine",
493 + "syn_type": "USAN",
494 + "synonyms": "TRASTUZUMAB EMTANSINE"
495 + },
496 + {
497 + "molecule_synonym": "TRASTUZUMAB-MCC-DM1 T-DM1",
498 + "syn_type": "RESEARCH_CODE",
499 + "synonyms": "TRASTUZUMAB-MCC-DM1 T-DM1"
500 + }
501 + ],
502 + "molecule_hierarchy": {
503 + "active_chembl_id": "CHEMBL1743082",
504 + "molecule_chembl_id": "CHEMBL1743082",
505 + "parent_chembl_id": "CHEMBL1743082"
506 + },
507 + "atc_classifications": [
508 + "L01FD03"
509 + ],
510 + "withdrawn_flag": false,
511 + "black_box_warning": 1,
512 + "oral": false,
513 + "parenteral": true,
514 + "topical": false,
515 + "prodrug": 0,
516 + "therapeutic_flag": true,
517 + "usan_stem": "-mab; -mab; -tansine",
518 + "usan_year": 2009,
519 + "score": 35.0,
520 + "cross_references": [
521 + {
522 + "xref_id": "ado-trastuzumab%20emtansine",
523 + "xref_name": "ado-trastuzumab emtansine",
524 + "xref_src": "DailyMed"
525 + },
526 + {
527 + "xref_id": "human/EPAR/kadcyla",
528 + "xref_name": "kadcyla",
529 + "xref_src": "EMA"
530 + }
531 + ]
532 + },
533 + {
534 + "molecule_chembl_id": "CHEMBL4297844",
535 + "pref_name": "TRASTUZUMAB DERUXTECAN",
536 + "molecule_type": "Antibody drug conjugate",
537 + "max_phase": "4.0",
538 + "first_approval": 2019,
539 + "molecule_synonyms": [
540 + {
541 + "molecule_synonym": "DS-8201",
542 + "syn_type": "RESEARCH_CODE",
543 + "synonyms": "DS-8201"
544 + },
545 + {
546 + "molecule_synonym": "DS-8201a",
547 + "syn_type": "RESEARCH_CODE",
548 + "synonyms": "DS-8201a"
549 + },
550 + {
551 + "molecule_synonym": "Ds-8201a",
552 + "syn_type": "OTHER",
553 + "synonyms": "DS-8201A"
554 + },
555 + {
556 + "molecule_synonym": "Enhertu",
557 + "syn_type": "TRADE_NAME",
558 + "synonyms": "Enhertu"
559 + },
560 + {
561 + "molecule_synonym": "Fam-trastuzumab deruxtecan-nxki",
562 + "syn_type": "FDA",
563 + "synonyms": "FAM-TRASTUZUMAB DERUXTECAN-NXKI"
564 + },
565 + {
566 + "molecule_synonym": "Trastuzumab deruxtecan",
567 + "syn_type": "ATC",
568 + "synonyms": "TRASTUZUMAB DERUXTECAN"
569 + },
570 + {
571 + "molecule_synonym": "Trastuzumab deruxtecan",
572 + "syn_type": "EMA",
573 + "synonyms": "TRASTUZUMAB DERUXTECAN"
574 + },
575 + {
576 + "molecule_synonym": "Trastuzumab deruxtecan",
577 + "syn_type": "INN",
578 + "synonyms": "TRASTUZUMAB DERUXTECAN"
579 + },
580 + {
581 + "molecule_synonym": "Trastuzumab deruxtecan",
582 + "syn_type": "OTHER",
583 + "synonyms": "TRASTUZUMAB DERUXTECAN"
584 + },
585 + {
586 + "molecule_synonym": "Trastuzumab deruxtecan",
587 + "syn_type": "USAN",
588 + "synonyms": "TRASTUZUMAB DERUXTECAN"
589 + },
590 + {
591 + "molecule_synonym": "Trastuzumab deruxtecan nxki",
592 + "syn_type": "OTHER",
593 + "synonyms": "TRASTUZUMAB DERUXTECAN NXKI"
594 + },
595 + {
596 + "molecule_synonym": "Trastuzumab deruxtecan-nxki",
597 + "syn_type": "OTHER",
598 + "synonyms": "TRASTUZUMAB DERUXTECAN-NXKI"
599 + }
600 + ],
601 + "molecule_hierarchy": {
602 + "active_chembl_id": "CHEMBL4297844",
603 + "molecule_chembl_id": "CHEMBL4297844",
604 + "parent_chembl_id": "CHEMBL4297844"
605 + },
606 + "atc_classifications": [
607 + "L01FD04"
608 + ],
609 + "withdrawn_flag": false,
610 + "black_box_warning": 1,
611 + "oral": false,
612 + "parenteral": true,
613 + "topical": false,
614 + "prodrug": 0,
615 + "therapeutic_flag": true,
616 + "usan_stem": "-mab; -mab; -tecan",
617 + "usan_year": null,
618 + "score": 35.0,
619 + "cross_references": [
620 + {
621 + "xref_id": "human/EPAR/enhertu",
622 + "xref_name": "enhertu",
623 + "xref_src": "EMA"
624 + },
625 + {
626 + "xref_id": "TRASTUZUMAB%20DERUXTECAN",
627 + "xref_name": "TRASTUZUMAB DERUXTECAN",
628 + "xref_src": "USAN"
629 + },
630 + {
631 + "xref_id": "fam-trastuzumab%20deruxtecan-nxki",
632 + "xref_name": "fam-trastuzumab deruxtecan-nxki",
633 + "xref_src": "DailyMed"
634 + }
635 + ]
636 + },
637 + {
638 + "molecule_chembl_id": "CHEMBL4298178",
639 + "pref_name": "TRASTUZUMAB DUOCARMAZINE",
640 + "molecule_type": "Antibody drug conjugate",
641 + "max_phase": "3.0",
642 + "first_approval": null,
643 + "molecule_synonyms": [
644 + {
645 + "molecule_synonym": "SYD-985",
646 + "syn_type": "RESEARCH_CODE",
647 + "synonyms": "SYD-985"
648 + },
649 + {
650 + "molecule_synonym": "SYD985",
651 + "syn_type": "RESEARCH_CODE",
652 + "synonyms": "SYD985"
653 + },
654 + {
655 + "molecule_synonym": "Trastuzumab duocarmazina",
656 + "syn_type": "INN_SPANISH",
657 + "synonyms": "TRASTUZUMAB DUOCARMAZINA"
658 + },
659 + {
660 + "molecule_synonym": "Trastuzumab duocarmazine",
661 + "syn_type": "ATC",
662 + "synonyms": "TRASTUZUMAB DUOCARMAZINE"
663 + },
664 + {
665 + "molecule_synonym": "Trastuzumab duocarmazine",
666 + "syn_type": "INN",
667 + "synonyms": "TRASTUZUMAB DUOCARMAZINE"
668 + },
669 + {
670 + "molecule_synonym": "Trastuzumab duocarmazine",
671 + "syn_type": "OTHER",
672 + "synonyms": "TRASTUZUMAB DUOCARMAZINE"
673 + },
674 + {
675 + "molecule_synonym": "Trastuzumab duocarmazine",
676 + "syn_type": "USAN",
677 + "synonyms": "TRASTUZUMAB DUOCARMAZINE"
678 + }
679 + ],
680 + "molecule_hierarchy": {
681 + "active_chembl_id": "CHEMBL4298178",
682 + "molecule_chembl_id": "CHEMBL4298178",
683 + "parent_chembl_id": "CHEMBL4298178"
684 + },
685 + "atc_classifications": [
686 + "L01FD05"
687 + ],
688 + "withdrawn_flag": false,
689 + "black_box_warning": 0,
690 + "oral": false,
691 + "parenteral": false,
692 + "topical": false,
693 + "prodrug": 0,
694 + "therapeutic_flag": false,
695 + "usan_stem": "-mab; -mab",
696 + "usan_year": 2018,
697 + "score": 34.0,
698 + "cross_references": [
699 + {
700 + "xref_id": "https://searchusan.ama-assn.org/finder/usan/search/TRASTUZUMAB%20DUOCARMAZINE/relevant/1/",
701 + "xref_name": "TRASTUZUMAB DUOCARMAZINE",
702 + "xref_src": "USAN"
703 + }
704 + ]
705 + }
706 + ]
707 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/target-CHEMBL1824.json +49 −0
@@ -0,0 +1,49 @@
1 +{
2 + "target_chembl_id": "CHEMBL1824",
3 + "pref_name": "Receptor tyrosine-protein kinase erbB-2",
4 + "target_type": "SINGLE PROTEIN",
5 + "organism": "Homo sapiens",
6 + "tax_id": 9606,
7 + "target_components": [
8 + {
9 + "accession": "P04626",
10 + "component_id": 120,
11 + "component_type": "PROTEIN",
12 + "component_description": "Receptor tyrosine-protein kinase erbB-2",
13 + "target_component_synonyms": [
14 + {
15 + "component_synonym": "ERBB2",
16 + "syn_type": "GENE_SYMBOL"
17 + },
18 + {
19 + "component_synonym": "HER2",
20 + "syn_type": "GENE_SYMBOL_OTHER"
21 + },
22 + {
23 + "component_synonym": "MLN19",
24 + "syn_type": "GENE_SYMBOL_OTHER"
25 + },
26 + {
27 + "component_synonym": "NEU",
28 + "syn_type": "GENE_SYMBOL_OTHER"
29 + },
30 + {
31 + "component_synonym": "NGL",
32 + "syn_type": "GENE_SYMBOL_OTHER"
33 + },
34 + {
35 + "component_synonym": "CD_antigen=CD340",
36 + "syn_type": "UNIPROT"
37 + },
38 + {
39 + "component_synonym": "Metastatic lymph node gene 19 protein",
40 + "syn_type": "UNIPROT"
41 + },
42 + {
43 + "component_synonym": "MLN 19",
44 + "syn_type": "UNIPROT"
45 + }
46 + ]
47 + }
48 + ]
49 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/target-CHEMBL1862.json +41 −0
@@ -0,0 +1,41 @@
1 +{
2 + "target_chembl_id": "CHEMBL1862",
3 + "pref_name": "Tyrosine-protein kinase ABL1",
4 + "target_type": "SINGLE PROTEIN",
5 + "organism": "Homo sapiens",
6 + "tax_id": 9606,
7 + "target_components": [
8 + {
9 + "accession": "P00519",
10 + "component_id": 173,
11 + "component_type": "PROTEIN",
12 + "component_description": "Tyrosine-protein kinase ABL1",
13 + "target_component_synonyms": [
14 + {
15 + "component_synonym": "ABL",
16 + "syn_type": "GENE_SYMBOL_OTHER"
17 + },
18 + {
19 + "component_synonym": "ABL1",
20 + "syn_type": "GENE_SYMBOL"
21 + },
22 + {
23 + "component_synonym": "JTK7",
24 + "syn_type": "GENE_SYMBOL_OTHER"
25 + },
26 + {
27 + "component_synonym": "Abelson murine leukemia viral oncogene homolog 1",
28 + "syn_type": "UNIPROT"
29 + },
30 + {
31 + "component_synonym": "Abelson tyrosine-protein kinase 1",
32 + "syn_type": "UNIPROT"
33 + },
34 + {
35 + "component_synonym": "p150",
36 + "syn_type": "UNIPROT"
37 + }
38 + ]
39 + }
40 + ]
41 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/target-CHEMBL1913.json +41 −0
@@ -0,0 +1,41 @@
1 +{
2 + "target_chembl_id": "CHEMBL1913",
3 + "pref_name": "Platelet-derived growth factor receptor beta",
4 + "target_type": "SINGLE PROTEIN",
5 + "organism": "Homo sapiens",
6 + "tax_id": 9606,
7 + "target_components": [
8 + {
9 + "accession": "P09619",
10 + "component_id": 227,
11 + "component_type": "PROTEIN",
12 + "component_description": "Platelet-derived growth factor receptor beta",
13 + "target_component_synonyms": [
14 + {
15 + "component_synonym": "PDGFR",
16 + "syn_type": "GENE_SYMBOL_OTHER"
17 + },
18 + {
19 + "component_synonym": "PDGFR1",
20 + "syn_type": "GENE_SYMBOL_OTHER"
21 + },
22 + {
23 + "component_synonym": "PDGFRB",
24 + "syn_type": "GENE_SYMBOL"
25 + },
26 + {
27 + "component_synonym": "Beta platelet-derived growth factor receptor",
28 + "syn_type": "UNIPROT"
29 + },
30 + {
31 + "component_synonym": "Beta-type platelet-derived growth factor receptor",
32 + "syn_type": "UNIPROT"
33 + },
34 + {
35 + "component_synonym": "CD140 antigen-like family member B",
36 + "syn_type": "UNIPROT"
37 + }
38 + ]
39 + }
40 + ]
41 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/target-CHEMBL1936.json +37 −0
@@ -0,0 +1,37 @@
1 +{
2 + "target_chembl_id": "CHEMBL1936",
3 + "pref_name": "Mast/stem cell growth factor receptor Kit",
4 + "target_type": "SINGLE PROTEIN",
5 + "organism": "Homo sapiens",
6 + "tax_id": 9606,
7 + "target_components": [
8 + {
9 + "accession": "P10721",
10 + "component_id": 255,
11 + "component_type": "PROTEIN",
12 + "component_description": "Mast/stem cell growth factor receptor Kit",
13 + "target_component_synonyms": [
14 + {
15 + "component_synonym": "KIT",
16 + "syn_type": "GENE_SYMBOL"
17 + },
18 + {
19 + "component_synonym": "SCFR",
20 + "syn_type": "GENE_SYMBOL_OTHER"
21 + },
22 + {
23 + "component_synonym": "CD_antigen=CD117",
24 + "syn_type": "UNIPROT"
25 + },
26 + {
27 + "component_synonym": "Mast/stem cell growth factor receptor Kit",
28 + "syn_type": "UNIPROT"
29 + },
30 + {
31 + "component_synonym": "p145 c-kit",
32 + "syn_type": "UNIPROT"
33 + }
34 + ]
35 + }
36 + ]
37 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/target-CHEMBL203.json +45 −0
@@ -0,0 +1,45 @@
1 +{
2 + "target_chembl_id": "CHEMBL203",
3 + "pref_name": "Epidermal growth factor receptor",
4 + "target_type": "SINGLE PROTEIN",
5 + "organism": "Homo sapiens",
6 + "tax_id": 9606,
7 + "target_components": [
8 + {
9 + "accession": "P00533",
10 + "component_id": 147,
11 + "component_type": "PROTEIN",
12 + "component_description": "Epidermal growth factor receptor",
13 + "target_component_synonyms": [
14 + {
15 + "component_synonym": "EGFR",
16 + "syn_type": "GENE_SYMBOL"
17 + },
18 + {
19 + "component_synonym": "ERBB",
20 + "syn_type": "GENE_SYMBOL_OTHER"
21 + },
22 + {
23 + "component_synonym": "ERBB1",
24 + "syn_type": "GENE_SYMBOL_OTHER"
25 + },
26 + {
27 + "component_synonym": "HER1",
28 + "syn_type": "GENE_SYMBOL_OTHER"
29 + },
30 + {
31 + "component_synonym": "Epidermal growth factor receptor",
32 + "syn_type": "UNIPROT"
33 + },
34 + {
35 + "component_synonym": "Proto-oncogene c-ErbB-1",
36 + "syn_type": "UNIPROT"
37 + },
38 + {
39 + "component_synonym": "Receptor tyrosine-protein kinase erbB-1",
40 + "syn_type": "UNIPROT"
41 + }
42 + ]
43 + }
44 + ]
45 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/target-CHEMBL2096618.json +69 −0
@@ -0,0 +1,69 @@
1 +{
2 + "target_chembl_id": "CHEMBL2096618",
3 + "pref_name": "Bcr/Abl fusion protein",
4 + "target_type": "CHIMERIC PROTEIN",
5 + "organism": "Homo sapiens",
6 + "tax_id": 9606,
7 + "target_components": [
8 + {
9 + "accession": "P00519",
10 + "component_id": 173,
11 + "component_type": "PROTEIN",
12 + "component_description": "Tyrosine-protein kinase ABL1",
13 + "target_component_synonyms": [
14 + {
15 + "component_synonym": "ABL",
16 + "syn_type": "GENE_SYMBOL_OTHER"
17 + },
18 + {
19 + "component_synonym": "ABL1",
20 + "syn_type": "GENE_SYMBOL"
21 + },
22 + {
23 + "component_synonym": "JTK7",
24 + "syn_type": "GENE_SYMBOL_OTHER"
25 + },
26 + {
27 + "component_synonym": "Abelson murine leukemia viral oncogene homolog 1",
28 + "syn_type": "UNIPROT"
29 + },
30 + {
31 + "component_synonym": "Abelson tyrosine-protein kinase 1",
32 + "syn_type": "UNIPROT"
33 + },
34 + {
35 + "component_synonym": "p150",
36 + "syn_type": "UNIPROT"
37 + }
38 + ]
39 + },
40 + {
41 + "accession": "P11274",
42 + "component_id": 3450,
43 + "component_type": "PROTEIN",
44 + "component_description": "Breakpoint cluster region protein",
45 + "target_component_synonyms": [
46 + {
47 + "component_synonym": "BCR",
48 + "syn_type": "GENE_SYMBOL"
49 + },
50 + {
51 + "component_synonym": "BCR1",
52 + "syn_type": "GENE_SYMBOL_OTHER"
53 + },
54 + {
55 + "component_synonym": "D22S11",
56 + "syn_type": "GENE_SYMBOL_OTHER"
57 + },
58 + {
59 + "component_synonym": "Breakpoint cluster region protein",
60 + "syn_type": "UNIPROT"
61 + },
62 + {
63 + "component_synonym": "Renal carcinoma antigen NY-REN-26",
64 + "syn_type": "UNIPROT"
65 + }
66 + ]
67 + }
68 + ]
69 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/fixtures/target-CHEMBL2111414.json +73 −0
@@ -0,0 +1,73 @@
1 +{
2 + "target_chembl_id": "CHEMBL2111414",
3 + "pref_name": "Tyrosine-protein kinase ABL",
4 + "target_type": "PROTEIN FAMILY",
5 + "organism": "Homo sapiens",
6 + "tax_id": 9606,
7 + "target_components": [
8 + {
9 + "accession": "P00519",
10 + "component_id": 173,
11 + "component_type": "PROTEIN",
12 + "component_description": "Tyrosine-protein kinase ABL1",
13 + "target_component_synonyms": [
14 + {
15 + "component_synonym": "ABL",
16 + "syn_type": "GENE_SYMBOL_OTHER"
17 + },
18 + {
19 + "component_synonym": "ABL1",
20 + "syn_type": "GENE_SYMBOL"
21 + },
22 + {
23 + "component_synonym": "JTK7",
24 + "syn_type": "GENE_SYMBOL_OTHER"
25 + },
26 + {
27 + "component_synonym": "Abelson murine leukemia viral oncogene homolog 1",
28 + "syn_type": "UNIPROT"
29 + },
30 + {
31 + "component_synonym": "Abelson tyrosine-protein kinase 1",
32 + "syn_type": "UNIPROT"
33 + },
34 + {
35 + "component_synonym": "p150",
36 + "syn_type": "UNIPROT"
37 + }
38 + ]
39 + },
40 + {
41 + "accession": "P42684",
42 + "component_id": 2333,
43 + "component_type": "PROTEIN",
44 + "component_description": "Tyrosine-protein kinase ABL2",
45 + "target_component_synonyms": [
46 + {
47 + "component_synonym": "ABL2",
48 + "syn_type": "GENE_SYMBOL"
49 + },
50 + {
51 + "component_synonym": "ABLL",
52 + "syn_type": "GENE_SYMBOL_OTHER"
53 + },
54 + {
55 + "component_synonym": "ARG",
56 + "syn_type": "GENE_SYMBOL_OTHER"
57 + },
58 + {
59 + "component_synonym": "Abelson murine leukemia viral oncogene homolog 2",
60 + "syn_type": "UNIPROT"
61 + },
62 + {
63 + "component_synonym": "Abelson-related gene protein",
64 + "syn_type": "UNIPROT"
65 + },
66 + {
67 + "component_synonym": "Abelson tyrosine-protein kinase 2",
68 + "syn_type": "UNIPROT"
69 + }
70 + ]
71 + }
72 + ]
73 +}
\ No newline at end of file
added packages/connectors/src/connectors/chembl/index.ts +385 −0
@@ -0,0 +1,385 @@
1 +import { and, eq, isNotNull, sql } from 'drizzle-orm';
2 +import { normalizeLabel } from '@cancerindex/shared';
3 +import { cancerCodes, connectorCursors, drugAliases, drugs, knowledgeEdges, sourceRecords, sources } from '@cancerindex/database';
4 +import { CancerResolver } from '@cancerindex/ontology';
5 +import { Connector, type ConnectorHealth, type RunContext } from '../../sdk/run.js';
6 +import { GENE_SYMBOL_RE } from '../../sdk/validate.js';
7 +import { GeneCache } from '../civic/genes.js';
8 +import { manifest } from './manifest.js';
9 +import {
10 + NEOPLASM_HEADING_RE,
11 + Status,
12 + Target,
13 + aliasesFromMolecule,
14 + compoundUrl,
15 + dedupeIndications,
16 + dedupeMechanisms,
17 + geneSymbolsFromTarget,
18 + higherStatus,
19 + indicationsUrl,
20 + isNonMoleculeName,
21 + mapKind,
22 + mapPhase,
23 + mechanismText,
24 + mechanismsUrl,
25 + mergeIndications,
26 + parseMolecules,
27 + pickExactMatch,
28 + searchUrl,
29 + statusUrl,
30 + targetFetchUrl,
31 + type IndicationSpec,
32 + type MechanismSpec,
33 + type Molecule,
34 +} from './api.js';
35 +
36 +const BASE = manifest.access.baseUrl!;
37 +const CURSOR_FLUSH_EVERY = 25;
38 +
39 +interface ChemblCursor {
40 + pass?: string;
41 + startedAt?: string;
42 + completedAt?: string;
43 + /** Last drug processed in (name, id) order — the next run resumes after it. */
44 + after?: { name: string; id: string } | null;
45 + processed?: number;
46 +}
47 +
48 +interface DrugRow {
49 + id: string;
50 + name: string;
51 + kind: string | null;
52 + developmentStatus: string | null;
53 + chemblId: string | null;
54 + targetGeneIds: string[];
55 +}
56 +
57 +interface TargetInfo {
58 + target: Target | null;
59 + symbols: string[];
60 + geneIds: string[];
61 +}
62 +
63 +interface Stats {
64 + considered: number;
65 + skippedNonMolecule: number;
66 + matched: number;
67 + matchedByPrefName: number;
68 + matchedBySynonym: number;
69 + unresolved: number;
70 + aliases: number;
71 + mechanisms: number;
72 + targetEdges: number;
73 + genesLinked: number;
74 + indications: number;
75 + indicationEdges: number;
76 + indicationsUnresolvedCancer: number;
77 + indicationsNonCancer: number;
78 + malformed: number;
79 +}
80 +
81 +/**
82 + * ChEMBL connector — enrichment of existing `drugs` rows (never an import of the compound universe).
83 + * For each drug, in (name, id) order: molecule/search → exact-name pick (pref_name or synonym,
84 + * case-insensitive; never fuzzy) → mechanisms + indications fetched for the *parent* molecule
85 + * (ChEMBL attaches them to dosed forms such as salts) → targets → HGNC genes.
86 + * Writes drugs.chembl_id / kind / mechanism / development_status / target_gene_ids, drug_aliases,
87 + * knowledge_edges drug TARGETS gene and drug INVESTIGATED_FOR cancer (MeSH-coded indications
88 + * resolved through cancer_codes system "mesh", i.e. after the mesh connector).
89 + */
90 +export class ChemblConnector extends Connector {
91 + readonly manifest = manifest;
92 +
93 + async healthCheck(ctx: RunContext): Promise<ConnectorHealth> {
94 + try {
95 + const s = Status.parse(await ctx.http.json(statusUrl(BASE)));
96 + return s.status === 'UP' ? { status: 'healthy', detail: `${s.chembl_db_version} (${s.chembl_release_date ?? '?'})` } : { status: 'degraded', detail: `status ${s.status}` };
97 + } catch (e) {
98 + return { status: 'failing', detail: (e as Error).message };
99 + }
100 + }
101 +
102 + async sync(ctx: RunContext): Promise<void> {
103 + const status = Status.parse(await ctx.http.json(statusUrl(BASE)));
104 + ctx.datasetVersion = status.chembl_db_version;
105 + ctx.info(`ChEMBL ${status.chembl_db_version} (release ${status.chembl_release_date ?? '?'})`);
106 +
107 + const state = ctx.cursor as ChemblCursor;
108 + if (!state.pass || state.completedAt) {
109 + for (const k of Object.keys(state)) delete (state as Record<string, unknown>)[k];
110 + state.pass = new Date().toISOString().slice(0, 10);
111 + state.startedAt = new Date().toISOString();
112 + state.after = null;
113 + state.processed = 0;
114 + ctx.info('starting a fresh ChEMBL pass over the drug list');
115 + } else ctx.info(`resuming ChEMBL pass ${state.pass} after "${state.after?.name ?? ''}" (${state.processed ?? 0} processed)`);
116 +
117 + const rows = await this.loadDrugs(ctx, state.after ?? null, ctx.mode === 'dry_run' ? 10 : undefined);
118 + ctx.info(`${rows.length} drugs to process`);
119 + if (ctx.mode === 'dry_run') return this.dryRun(ctx, rows);
120 +
121 + const resolver = new CancerResolver(ctx.db);
122 + await resolver.warm();
123 + const geneCache = new GeneCache(ctx.db);
124 + const targetCache = new Map<string, TargetInfo>();
125 + const neoplasmMesh = await this.loadNeoplasmMeshSet(ctx);
126 + const stats: Stats = { considered: 0, skippedNonMolecule: 0, matched: 0, matchedByPrefName: 0, matchedBySynonym: 0, unresolved: 0, aliases: 0, mechanisms: 0, targetEdges: 0, genesLinked: 0, indications: 0, indicationEdges: 0, indicationsUnresolvedCancer: 0, indicationsNonCancer: 0, malformed: 0 };
127 +
128 + let sinceFlush = 0;
129 + let completed = true;
130 + for (const drug of rows) {
131 + if (ctx.shouldStop()) {
132 + completed = false;
133 + break;
134 + }
135 + stats.considered++;
136 + try {
137 + await this.processDrug(ctx, drug, { resolver, geneCache, targetCache, neoplasmMesh, stats });
138 + } catch (e) {
139 + // One bad drug must not kill the pass; the error is logged and the drug is retried next pass.
140 + ctx.counters.rejected++;
141 + ctx.warn(`drug ${drug.id} "${drug.name}": ${(e as Error).message}`);
142 + }
143 + state.after = { name: drug.name, id: drug.id };
144 + state.processed = (state.processed ?? 0) + 1;
145 + if (++sinceFlush >= CURSOR_FLUSH_EVERY) {
146 + sinceFlush = 0;
147 + await this.persistCursor(ctx);
148 + }
149 + }
150 + if (completed) state.completedAt = new Date().toISOString();
151 + ctx.info(
152 + `ChEMBL pass ${completed ? 'complete' : 'paused (time budget)'}: considered ${stats.considered}, skipped non-molecule ${stats.skippedNonMolecule}, matched ${stats.matched} (pref_name ${stats.matchedByPrefName}, synonym ${stats.matchedBySynonym}), unresolved ${stats.unresolved}; aliases ${stats.aliases}; mechanisms ${stats.mechanisms} → TARGETS edges ${stats.targetEdges} (${stats.genesLinked} gene links); indications ${stats.indications} → INVESTIGATED_FOR edges ${stats.indicationEdges}, cancer headings unresolved ${stats.indicationsUnresolvedCancer}, non-cancer ${stats.indicationsNonCancer}; malformed payloads ${stats.malformed}`,
153 + { ...stats },
154 + );
155 + }
156 +
157 + /* ---------------------------------------------------------------------------------------------- */
158 +
159 + private async loadDrugs(ctx: RunContext, after: { name: string; id: string } | null, limit?: number): Promise<DrugRow[]> {
160 + const where = after ? sql`(${drugs.name}, ${drugs.id}) > (${after.name}, ${after.id})` : sql`true`;
161 + const q = ctx.db.select({ id: drugs.id, name: drugs.name, kind: drugs.kind, developmentStatus: drugs.developmentStatus, chemblId: drugs.chemblId, targetGeneIds: drugs.targetGeneIds }).from(drugs).where(where).orderBy(drugs.name, drugs.id);
162 + return limit ? await q.limit(limit) : await q;
163 + }
164 +
165 + /** MeSH UIs known to be neoplasm descriptors (mesh connector source records + mapped codes). */
166 + private async loadNeoplasmMeshSet(ctx: RunContext): Promise<Set<string>> {
167 + const set = new Set<string>();
168 + for (const r of await ctx.db.select({ code: cancerCodes.code }).from(cancerCodes).where(eq(cancerCodes.system, 'mesh'))) set.add(r.code);
169 + const [meshSource] = await ctx.db.select({ id: sources.id }).from(sources).where(eq(sources.slug, 'mesh')).limit(1);
170 + if (meshSource) for (const r of await ctx.db.select({ id: sourceRecords.sourceRecordId }).from(sourceRecords).where(and(eq(sourceRecords.sourceId, meshSource.id), eq(sourceRecords.entityKind, 'descriptor')))) set.add(r.id);
171 + return set;
172 + }
173 +
174 + private async persistCursor(ctx: RunContext): Promise<void> {
175 + await ctx.db.update(connectorCursors).set({ cursor: ctx.cursor, updatedAt: new Date() }).where(eq(connectorCursors.connectorId, manifest.id));
176 + }
177 +
178 + private async dryRun(ctx: RunContext, rows: DrugRow[]): Promise<void> {
179 + for (const d of rows) {
180 + if (isNonMoleculeName(d.name)) {
181 + ctx.info(`[dry_run] ${d.name}: skipped (not a molecule)`);
182 + continue;
183 + }
184 + const page = parseMolecules(await ctx.http.json(searchUrl(BASE, d.name)));
185 + ctx.counters.fetched++;
186 + const hit = pickExactMatch(d.name, page.molecules);
187 + ctx.info(`[dry_run] ${d.name}: ${page.total} hits → ${hit ? `${hit.molecule.molecule_chembl_id} ${hit.molecule.pref_name} (by ${hit.by})` : 'no exact match'}`);
188 + }
189 + }
190 +
191 + private async processDrug(ctx: RunContext, drug: DrugRow, env: { resolver: CancerResolver; geneCache: GeneCache; targetCache: Map<string, TargetInfo>; neoplasmMesh: Set<string>; stats: Stats }): Promise<void> {
192 + const { resolver, geneCache, targetCache, neoplasmMesh, stats } = env;
193 + if (isNonMoleculeName(drug.name)) {
194 + stats.skippedNonMolecule++;
195 + return;
196 + }
197 + // 1. Identity — exact name only.
198 + const page = parseMolecules(await ctx.http.json(searchUrl(BASE, drug.name)));
199 + stats.malformed += page.malformed;
200 + ctx.counters.validationFailures += page.malformed;
201 + const hit = pickExactMatch(drug.name, page.molecules);
202 + if (!hit) {
203 + stats.unresolved++;
204 + await ctx.recordUnresolved('drug', drug.name, normalizeLabel(drug.name), {
205 + drugId: drug.id,
206 + totalHits: page.total,
207 + topHits: page.molecules.slice(0, 5).map((m) => ({ chemblId: m.molecule_chembl_id, prefName: m.pref_name ?? null, maxPhase: m.max_phase ?? null, type: m.molecule_type ?? null })),
208 + });
209 + return;
210 + }
211 + stats.matched++;
212 + if (hit.by === 'pref_name') stats.matchedByPrefName++;
213 + else stats.matchedBySynonym++;
214 + const mol = hit.molecule;
215 + const chemblId = mol.molecule_chembl_id;
216 + const parentId = mol.molecule_hierarchy?.parent_chembl_id || chemblId;
217 +
218 + // 2. Mechanisms (+ targets) and indications, for the parent molecule (covers salts / dosed forms).
219 + const mech = dedupeMechanisms(await ctx.http.json(mechanismsUrl(BASE, parentId)));
220 + stats.malformed += mech.malformed;
221 + const indications = await this.fetchIndications(ctx, parentId, stats);
222 + const targets = new Map<string, TargetInfo>();
223 + for (const m of mech.mechanisms) {
224 + if (!m.targetChemblId || targets.has(m.targetChemblId)) continue;
225 + targets.set(m.targetChemblId, await this.resolveTarget(ctx, m.targetChemblId, targetCache, geneCache));
226 + }
227 + stats.mechanisms += mech.mechanisms.length;
228 + stats.indications += indications.length;
229 +
230 + // 3. Raw record (identity, phase, first approval, ATC, mechanisms, targets, indications).
231 + const payload = {
232 + molecule: mol,
233 + parentChemblId: parentId,
234 + matchedBy: hit.by,
235 + matchedText: hit.matchedText,
236 + firstApproval: mol.first_approval ?? null,
237 + atcClassifications: mol.atc_classifications ?? [],
238 + mechanisms: mech.mechanisms,
239 + targets: Object.fromEntries([...targets].map(([id, t]) => [id, { prefName: t.target?.pref_name ?? null, targetType: t.target?.target_type ?? null, organism: t.target?.organism ?? null, geneSymbols: t.symbols }])),
240 + indications,
241 + };
242 + const rec = await ctx.upsertSourceRecord('molecule', chemblId, payload, { canonicalType: 'drug', canonicalId: drug.id });
243 + const provenanceId = await this.provenanceFor(ctx, drug.id, chemblId, rec.status === 'unchanged');
244 +
245 + // 4. Drug row.
246 + const kind = mapKind(mol.molecule_type) ?? drug.kind;
247 + const developmentStatus = higherStatus(drug.developmentStatus, mapPhase(mol.max_phase ?? null));
248 + const mechanism = mechanismText(mech.mechanisms);
249 + const geneIds = [...new Set([...drug.targetGeneIds, ...[...targets.values()].flatMap((t) => t.geneIds)])];
250 + await ctx.db
251 + .update(drugs)
252 + .set({
253 + chemblId,
254 + kind,
255 + developmentStatus,
256 + mechanism: mechanism ?? sql`${drugs.mechanism}`,
257 + targetGeneIds: geneIds.length ? sql`${sql.raw(`ARRAY[${geneIds.map((g) => `'${g.replace(/'/g, '')}'`).join(',')}]::text[]`)}` : sql`${drugs.targetGeneIds}`,
258 + updatedAt: new Date(),
259 + })
260 + .where(eq(drugs.id, drug.id));
261 + if (drug.chemblId !== chemblId) await ctx.recordChange('drug', drug.id, 'updated', `ChEMBL ${chemblId} (${mol.pref_name ?? ''}, matched by ${hit.by})`, { chemblId: drug.chemblId }, { chemblId });
262 +
263 + // 5. Aliases.
264 + for (const a of aliasesFromMolecule(mol, drug.name)) {
265 + const ins = await ctx.db.insert(drugAliases).values({ drugId: drug.id, alias: a.alias, normalized: a.normalized, aliasType: a.aliasType, sourceId: ctx.sourceId }).onConflictDoNothing().returning({ id: drugAliases.id });
266 + if (ins.length) stats.aliases++;
267 + }
268 +
269 + // 6. drug TARGETS gene — one edge per gene; action types of all mechanisms hitting it are joined.
270 + const byGene = new Map<string, { actionTypes: Set<string>; mechanisms: MechanismSpec[] }>();
271 + for (const m of mech.mechanisms) {
272 + const t = m.targetChemblId ? targets.get(m.targetChemblId) : undefined;
273 + for (const g of t?.geneIds ?? []) {
274 + const e = byGene.get(g) ?? { actionTypes: new Set<string>(), mechanisms: [] };
275 + if (m.actionType) e.actionTypes.add(m.actionType);
276 + e.mechanisms.push(m);
277 + byGene.set(g, e);
278 + }
279 + }
280 + for (const [geneId, e] of byGene) {
281 + await this.upsertEdge(ctx, { sourceEntityType: 'drug', sourceEntityId: drug.id, targetEntityType: 'gene', targetEntityId: geneId, relationshipType: 'TARGETS', cancerContextIds: [], direction: null, evidenceLevel: [...e.actionTypes].sort().join('|') || null, evidenceCategory: 'curated_evidence', sourceRecordId: chemblId, provenanceId });
282 + stats.targetEdges++;
283 + }
284 + stats.genesLinked += byGene.size;
285 +
286 + // 7. drug INVESTIGATED_FOR cancer — MeSH-coded indications resolved by UI.
287 + for (const ind of indications) {
288 + const cancer = resolver.byCode('mesh', ind.meshId);
289 + if (!cancer) {
290 + if (neoplasmMesh.has(ind.meshId) || NEOPLASM_HEADING_RE.test(ind.meshHeading)) {
291 + stats.indicationsUnresolvedCancer++;
292 + await ctx.recordUnresolved('cancer', ind.meshHeading, normalizeLabel(ind.meshHeading), { meshId: ind.meshId, efoIds: ind.efoIds, chemblId, drugId: drug.id, drugName: drug.name, maxPhaseForIndication: ind.maxPhase });
293 + } else stats.indicationsNonCancer++;
294 + continue;
295 + }
296 + await this.upsertEdge(ctx, {
297 + sourceEntityType: 'drug',
298 + sourceEntityId: drug.id,
299 + targetEntityType: 'cancer',
300 + targetEntityId: cancer.cancerId,
301 + relationshipType: 'INVESTIGATED_FOR',
302 + cancerContextIds: [cancer.cancerId],
303 + direction: 'unknown',
304 + evidenceLevel: ind.maxPhase != null ? String(ind.maxPhase) : null, // ChEMBL max_phase_for_ind, never re-scaled
305 + evidenceCategory: 'curated_evidence',
306 + sourceRecordId: chemblId,
307 + provenanceId,
308 + });
309 + stats.indicationEdges++;
310 + }
311 + }
312 +
313 + private async fetchIndications(ctx: RunContext, parentId: string, stats: Stats): Promise<IndicationSpec[]> {
314 + const pages: IndicationSpec[][] = [];
315 + for (let offset = 0, guard = 0; guard < 20; guard++) {
316 + const page = dedupeIndications(await ctx.http.json(indicationsUrl(BASE, parentId, offset)));
317 + stats.malformed += page.malformed;
318 + pages.push(page.indications);
319 + if (!page.next) break;
320 + offset += 500;
321 + }
322 + return mergeIndications(pages);
323 + }
324 +
325 + private async resolveTarget(ctx: RunContext, targetId: string, cache: Map<string, TargetInfo>, geneCache: GeneCache): Promise<TargetInfo> {
326 + const cached = cache.get(targetId);
327 + if (cached) return cached;
328 + let info: TargetInfo = { target: null, symbols: [], geneIds: [] };
329 + try {
330 + const parsed = Target.safeParse(await ctx.http.json(targetFetchUrl(BASE, targetId)));
331 + if (!parsed.success) {
332 + ctx.counters.validationFailures++;
333 + ctx.warn(`malformed target ${targetId}: ${parsed.error.issues[0]?.message}`);
334 + } else {
335 + const symbols = geneSymbolsFromTarget(parsed.data);
336 + const geneIds: string[] = [];
337 + for (const s of symbols) {
338 + if (!GENE_SYMBOL_RE.test(s)) continue;
339 + const id = await geneCache.ensure(s);
340 + if (id) geneIds.push(id);
341 + else await ctx.recordUnresolved('gene', s, normalizeLabel(s), { targetChemblId: targetId });
342 + }
343 + info = { target: parsed.data, symbols, geneIds };
344 + }
345 + } catch (e) {
346 + ctx.warn(`target ${targetId} fetch failed: ${(e as Error).message}`);
347 + }
348 + cache.set(targetId, info);
349 + return info;
350 + }
351 +
352 + /** One provenance row per (drug, ChEMBL molecule) — reused while the source record is unchanged. */
353 + private async provenanceFor(ctx: RunContext, drugId: string, chemblId: string, unchanged: boolean): Promise<number> {
354 + if (unchanged) {
355 + const [edge] = await ctx.db
356 + .select({ provenanceIds: knowledgeEdges.provenanceIds })
357 + .from(knowledgeEdges)
358 + .where(and(eq(knowledgeEdges.sourceEntityType, 'drug'), eq(knowledgeEdges.sourceEntityId, drugId), eq(knowledgeEdges.sourceId, ctx.sourceId), eq(knowledgeEdges.sourceRecordId, chemblId), isNotNull(knowledgeEdges.provenanceIds)))
359 + .limit(1);
360 + const existing = edge?.provenanceIds?.[0];
361 + if (existing) return existing;
362 + }
363 + return ctx.addProvenance({
364 + sourceRecordId: chemblId,
365 + sourceUrl: compoundUrl(chemblId),
366 + dataset: 'ChEMBL',
367 + datasetVersion: ctx.datasetVersion,
368 + evidenceType: 'database',
369 + accessLevel: 'open',
370 + license: 'CC BY-SA 3.0',
371 + methodology: 'ChEMBL manually curated mechanism of action (target, action type), drug indications (MeSH/EFO, max phase per indication) and molecule metadata; exact-name identity match',
372 + });
373 + }
374 +
375 + private async upsertEdge(ctx: RunContext, e: { sourceEntityType: string; sourceEntityId: string; targetEntityType: string; targetEntityId: string; relationshipType: string; cancerContextIds: string[]; direction: string | null; evidenceLevel: string | null; evidenceCategory: string; sourceRecordId: string; provenanceId: number }): Promise<void> {
376 + const set = { cancerContextIds: e.cancerContextIds, direction: e.direction, evidenceLevel: e.evidenceLevel, evidenceCategory: e.evidenceCategory, status: 'active', provenanceIds: [e.provenanceId], lastSeenAt: new Date() };
377 + await ctx.db
378 + .insert(knowledgeEdges)
379 + .values({ sourceEntityType: e.sourceEntityType, sourceEntityId: e.sourceEntityId, targetEntityType: e.targetEntityType, targetEntityId: e.targetEntityId, relationshipType: e.relationshipType, sourceId: ctx.sourceId, sourceRecordId: e.sourceRecordId, supportCount: 1, ...set })
380 + .onConflictDoUpdate({ target: [knowledgeEdges.sourceEntityType, knowledgeEdges.sourceEntityId, knowledgeEdges.targetEntityType, knowledgeEdges.targetEntityId, knowledgeEdges.relationshipType, knowledgeEdges.sourceId, knowledgeEdges.sourceRecordId], set });
381 + }
382 +}
383 +
384 +export const connector = new ChemblConnector();
385 +export type { Molecule };
added packages/connectors/src/connectors/chembl/manifest.ts +55 −0
@@ -0,0 +1,55 @@
1 +import { defineManifest } from '../../sdk/manifest.js';
2 +
3 +/**
4 + * Docs verified 2026-09-08:
5 + * - ChEMBL Data Web Services (https://chembl.gitbook.io/chembl-interface-documentation/web-services/chembl-data-web-services):
6 + * base https://www.ebi.ac.uk/chembl/api/data/, resources molecule / mechanism / target /
7 + * drug_indication / status, `.json` suffix, pagination `limit`/`offset` with `page_meta`
8 + * {limit, offset, total_count, next, previous}, filters `field__exact|iexact|in|…`, search
9 + * `molecule/search.json?q=`. No published rate limit ("reasonable use") → 3 req/s, concurrency 2.
10 + * - Live checks: status.json → ChEMBL_37 (release 2026-05-01, 2,921,148 distinct compounds);
11 + * molecule/search.json?q=osimertinib → CHEMBL3353410 (OSIMERTINIB, Small molecule, max_phase 4,
12 + * first_approval 2015, synonyms with syn_type TRADE_NAME/RESEARCH_CODE/INN/…);
13 + * mechanism.json?molecule_chembl_id=CHEMBL3353410 → mechanism_of_action, action_type,
14 + * target_chembl_id CHEMBL203, max_phase, mechanism_refs. Mechanisms and indications are attached
15 + * to the *dosed* form (imatinib → CHEMBL1642 mesylate), so both are fetched with
16 + * `parent_molecule_chembl_id=<parent>` (4 mechanisms for imatinib, 0 with molecule_chembl_id).
17 + * target/CHEMBL203.json → target_components[].accession (UniProt) + target_component_synonyms
18 + * (syn_type GENE_SYMBOL). drug_indication max_phase_for_ind is a string ("3.0") while
19 + * mechanism.max_phase is a number — both coerced.
20 + * - License (https://chembl.gitbook.io/chembl-interface-documentation/about, fetched 2026-09-08):
21 + * "The ChEMBL data is made available on a Creative Commons Attribution-Share Alike 3.0 Unported
22 + * License." Releases carry a DOI (Downloads page) for version citation.
23 + */
24 +export const manifest = defineManifest({
25 + id: 'chembl',
26 + name: 'ChEMBL (EMBL-EBI)',
27 + organization: 'European Molecular Biology Laboratory — European Bioinformatics Institute (EMBL-EBI)',
28 + category: 'drugs',
29 + tier: 1,
30 + description:
31 + 'Manually curated database of bioactive molecules with drug-like properties. CancerIndex enriches its existing drugs (from CIViC and trials) — never the 2.9 M compound universe — with the ChEMBL identifier, molecule type, mechanism(s) of action, clinical development phase, synonyms (brands, INN, research codes), protein targets mapped to HGNC genes (drug TARGETS gene edges) and MeSH-coded drug indications resolved to canonical cancers (drug INVESTIGATED_FOR cancer edges). First-approval years and ATC codes stay in the raw source record: jurisdictional approvals come from regulatory connectors (CLAUDE.md §13).',
32 + homepage: 'https://www.ebi.ac.uk/chembl/',
33 + docsUrl: 'https://chembl.gitbook.io/chembl-interface-documentation/web-services/chembl-data-web-services',
34 + termsUrl: 'https://chembl.gitbook.io/chembl-interface-documentation/about',
35 + access: { type: 'rest', auth: 'none', baseUrl: 'https://www.ebi.ac.uk/chembl/api/data' },
36 + license: 'CC BY-SA 3.0',
37 + licenseStatus: 'approved',
38 + commercialUse: 'allowed',
39 + redistribution: 'attribution',
40 + attribution: 'ChEMBL, EMBL-EBI (https://www.ebi.ac.uk/chembl/), Creative Commons Attribution-ShareAlike 3.0 Unported. Zdrazil B et al., Nucleic Acids Res 2024 (PMID 37933841).',
41 + termsReviewedAt: '2026-09-08',
42 + termsNotes:
43 + 'CC BY-SA 3.0: attribution to ChEMBL/EMBL-EBI on every /source page and in API `sources`; share-alike applies to redistributed *derivatives of ChEMBL data* (the drug enrichment fields and edges carry ChEMBL provenance so they can be distributed under a compatible licence). Release DOIs are available for version citation; the dataset version (e.g. ChEMBL_37) is recorded on every run.',
44 + updateFrequency: 'Release every ~4 months (ChEMBL_37 = 2026-05-01)',
45 + expectedLatency: 'Days after a release (next scheduled run)',
46 + // Full crawl of our drug list each pass; the (name, id) cursor makes an interrupted pass resumable.
47 + supportsIncrementalSync: true,
48 + entities: ['drugs', 'drug_aliases', 'knowledge_edges', 'genes'],
49 + rateLimits: { requestsPerSecond: 3, maxConcurrency: 2, notes: 'No published limit; EMBL-EBI asks for reasonable use — 3 req/s, 2 in flight, targets cached per run.' },
50 + retryPolicy: { maxRetries: 4, baseDelayMs: 1000, maxDelayMs: 30_000 },
51 + rawRetention: 'full',
52 + documentationVerifiedAt: '2026-09-08',
53 + status: 'active',
54 + schedule: '0 5 * * 1',
55 +});
modified packages/connectors/src/connectors/clinicaltrials/index.ts +3 −2
@@ -1,6 +1,6 @@
1 1 import { and, desc, eq, sql } from 'drizzle-orm';
2 2 import { normalizeLabel } from '@cancerindex/shared';
3 −import { clinicalTrials, connectorCursors, ingestRuns, mintId, trialConditions, trialInterventions, trialLocations, unresolvedLabels } from '@cancerindex/database';
3 +import { clinicalTrials, ingestRuns, mintId, trialConditions, trialInterventions, trialLocations, unresolvedLabels } from '@cancerindex/database';
4 4 import { CancerResolver } from '@cancerindex/ontology';
5 5 import { Connector, type ConnectorHealth, type RunContext } from '../../sdk/run.js';
6 6 import { HttpError } from '../../sdk/http.js';
@@ -154,6 +154,7 @@ export class ClinicalTrialsConnector extends Connector {
154 154 if (state.rangeStart === null && previousFullTotal && page.totalCount < previousFullTotal * manifest.anomalyGuard.minRatioOfPrevious) {
155 155 throw new Error(`anomaly: totalCount ${page.totalCount} < ${manifest.anomalyGuard.minRatioOfPrevious * 100}% of previous full crawl (${previousFullTotal}) — refusing to continue (CLAUDE.md §171)`);
156 156 }
157 + if (state.rangeStart === null) await ctx.guardCount('study', page.totalCount); // full crawl total vs previous successful run (CLAUDE.md §171)
157 158 }
158 159 const pageUnresolved: Array<{ mapping: ConditionMapping; nctId: string; mesh: string[] }> = [];
159 160 for (const study of page.studies) {
@@ -214,7 +215,7 @@ export class ClinicalTrialsConnector extends Connector {
214 215 }
215 216
216 217 private async persistCursor(ctx: RunContext): Promise<void> {
217 − await ctx.db.update(connectorCursors).set({ cursor: ctx.cursor, updatedAt: new Date() }).where(eq(connectorCursors.connectorId, manifest.id));
218 + await ctx.saveCursor(); // SDK checkpoint: connector_cursors + ingest_runs.cursor_after (CLAUDE.md §90)
218 219 }
219 220
220 221 /** Normalize + validate + reconcile + persist one study. Returns null when rejected. */
modified packages/connectors/src/connectors/clinvar/index.ts +1 −0
@@ -146,6 +146,7 @@ export class ClinvarConnector extends Connector {
146 146 cursor.lineOffset = lineNo;
147 147 cursor.kept = (cursor.kept ?? 0) + r.rows;
148 148 cursor.scanned = lineNo;
149 + await ctx.checkpoint(r.rows); // mid-run cursor persistence (every checkpointEvery rows / 60 s)
149 150 };
150 151
151 152 for await (const line of rl) {
added packages/connectors/src/connectors/mesh/fixtures/desc-records.xml +67 −0
@@ -0,0 +1,67 @@
1 +<?xml version="1.0"?>
2 +<!DOCTYPE DescriptorRecordSet SYSTEM "https://www.nlm.nih.gov/databases/dtd/nlmdescriptorrecordset_20260101.dtd">
3 +<DescriptorRecordSet LanguageCode="eng">
4 +<DescriptorRecord DescriptorClass="1">
5 + <DescriptorUI>D001943</DescriptorUI>
6 + <DescriptorName><String>Breast Neoplasms</String></DescriptorName>
7 + <DateCreated><Year>1999</Year><Month>01</Month><Day>01</Day></DateCreated>
8 + <DateRevised><Year>2018</Year><Month>06</Month><Day>14</Day></DateRevised>
9 + <DateEstablished><Year>1966</Year><Month>01</Month><Day>01</Day></DateEstablished>
10 + <AllowableQualifiersList><AllowableQualifier><QualifierReferredTo><QualifierUI>Q000097</QualifierUI><QualifierName><String>blood</String></QualifierName></QualifierReferredTo><Abbreviation>BL</Abbreviation></AllowableQualifier></AllowableQualifiersList>
11 + <Annotation>human only; BREAST NEOPLASMS, MALE is also available</Annotation>
12 + <HistoryNote>66</HistoryNote>
13 + <TreeNumberList><TreeNumber>C04.588.180</TreeNumber><TreeNumber>C17.800.090.500</TreeNumber></TreeNumberList>
14 + <ConceptList>
15 + <Concept PreferredConceptYN="Y">
16 + <ConceptUI>M0002910</ConceptUI>
17 + <ConceptName><String>Breast Neoplasms</String></ConceptName>
18 + <ScopeNote>Tumors or cancer of the human BREAST.</ScopeNote>
19 + <TermList>
20 + <Term ConceptPreferredTermYN="Y" IsPermutedTermYN="N" LexicalTag="NON" RecordPreferredTermYN="Y"><TermUI>T005543</TermUI><String>Breast Neoplasms</String></Term>
21 + <Term ConceptPreferredTermYN="N" IsPermutedTermYN="Y" LexicalTag="NON" RecordPreferredTermYN="N"><TermUI>T005543</TermUI><String>Neoplasms, Breast</String></Term>
22 + <Term ConceptPreferredTermYN="N" IsPermutedTermYN="N" LexicalTag="NON" RecordPreferredTermYN="N"><TermUI>T005542</TermUI><String>Breast Tumors</String></Term>
23 + </TermList>
24 + </Concept>
25 + <Concept PreferredConceptYN="N">
26 + <ConceptUI>M0002909</ConceptUI>
27 + <ConceptName><String>Breast Cancer</String></ConceptName>
28 + <ConceptRelationList><ConceptRelation RelationName="NRW"><Concept1UI>M0002910</Concept1UI><Concept2UI>M0002909</Concept2UI></ConceptRelation></ConceptRelationList>
29 + <TermList>
30 + <Term ConceptPreferredTermYN="Y" IsPermutedTermYN="N" LexicalTag="NON" RecordPreferredTermYN="N"><TermUI>T005540</TermUI><String>Breast Cancer</String></Term>
31 + <Term ConceptPreferredTermYN="N" IsPermutedTermYN="Y" LexicalTag="NON" RecordPreferredTermYN="N"><TermUI>T005540</TermUI><String>Cancer, Breast</String></Term>
32 + </TermList>
33 + </Concept>
34 + </ConceptList>
35 +</DescriptorRecord>
36 +<DescriptorRecord DescriptorClass="1">
37 + <DescriptorUI>D000001</DescriptorUI>
38 + <DescriptorName><String>Calcimycin</String></DescriptorName>
39 + <DateEstablished><Year>1974</Year><Month>11</Month><Day>19</Day></DateEstablished>
40 + <TreeNumberList><TreeNumber>D03.633.100.221.173</TreeNumber></TreeNumberList>
41 + <ConceptList>
42 + <Concept PreferredConceptYN="Y">
43 + <ConceptUI>M0000001</ConceptUI>
44 + <ConceptName><String>Calcimycin</String></ConceptName>
45 + <TermList>
46 + <Term ConceptPreferredTermYN="Y" IsPermutedTermYN="N" LexicalTag="NON" RecordPreferredTermYN="Y"><TermUI>T000002</TermUI><String>Calcimycin</String></Term>
47 + </TermList>
48 + </Concept>
49 + </ConceptList>
50 +</DescriptorRecord>
51 +<DescriptorRecord DescriptorClass="1">
52 + <DescriptorUI>D009190</DescriptorUI>
53 + <DescriptorName><String>Myelodysplastic Syndromes</String></DescriptorName>
54 + <TreeNumberList><TreeNumber>C15.378.190.625</TreeNumber></TreeNumberList>
55 + <ConceptList>
56 + <Concept PreferredConceptYN="Y">
57 + <ConceptUI>M0014315</ConceptUI>
58 + <ConceptName><String>Myelodysplastic Syndromes</String></ConceptName>
59 + <ScopeNote>Clonal hematopoietic stem cell disorders characterized by dysplasia in one or more hematopoietic cell lineages.</ScopeNote>
60 + <TermList>
61 + <Term ConceptPreferredTermYN="Y" IsPermutedTermYN="N" LexicalTag="NON" RecordPreferredTermYN="Y"><TermUI>T027543</TermUI><String>Myelodysplastic Syndromes</String></Term>
62 + <Term ConceptPreferredTermYN="N" IsPermutedTermYN="N" LexicalTag="NON" RecordPreferredTermYN="N"><TermUI>T027544</TermUI><String>Dysmyelopoietic Syndromes</String></Term>
63 + </TermList>
64 + </Concept>
65 + </ConceptList>
66 +</DescriptorRecord>
67 +</DescriptorRecordSet>
added packages/connectors/src/connectors/mesh/fixtures/descriptors-page.json +365 −0
@@ -0,0 +1,365 @@
1 +{
2 + "head": {
3 + "vars": [
4 + "d",
5 + "label",
6 + "scope",
7 + "introduced",
8 + "trees",
9 + "terms",
10 + "related"
11 + ]
12 + },
13 + "results": {
14 + "bindings": [
15 + {
16 + "d": {
17 + "type": "uri",
18 + "value": "http://id.nlm.nih.gov/mesh/D000230"
19 + },
20 + "label": {
21 + "type": "literal",
22 + "xml:lang": "en",
23 + "value": "Adenocarcinoma"
24 + },
25 + "scope": {
26 + "type": "literal",
27 + "xml:lang": "en",
28 + "value": "A malignant epithelial tumor with a glandular organization."
29 + },
30 + "introduced": {
31 + "type": "literal",
32 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
33 + "value": "1966-01-01"
34 + },
35 + "trees": {
36 + "type": "literal",
37 + "value": "C04.557.470.200.025"
38 + },
39 + "terms": {
40 + "type": "literal",
41 + "value": "Adenocarcinoma|Adenocarcinomas|Adenoma, Malignant|Adenomas, Malignant|Malignant Adenoma|Malignant Adenomas"
42 + },
43 + "related": {
44 + "type": "literal",
45 + "value": "Adenocarcinoma, Basal Cell|Adenocarcinoma, Granular Cell|Adenocarcinoma, Oxyphilic|Adenocarcinoma, Tubular|Adenocarcinomas, Basal Cell|Adenocarcinomas, Granular Cell|Adenocarcinomas, Oxyphilic|Adenocarcinomas, Tubular|Basal Cell Adenocarcinoma|Basal Cell Adenocarcinomas|Carcinoma, Cribriform|Carcinoma, Granular Cell|Carcinoma, Tubular|Carcinomas, Cribriform|Carcinomas, Granular Cell|Carcinomas, Tubular|Cribriform Carcinoma|Cribriform Carcinomas|Granular Cell Adenocarcinoma|Granular Cell Adenocarcinomas|Granular Cell Carcinoma|Granular Cell Carcinomas|Oxyphilic Adenocarcinoma|Oxyphilic Adenocarcinomas|Tubular Adenocarcinoma|Tubular Adenocarcinomas|Tubular Carcinoma|Tubular Carcinomas"
46 + }
47 + },
48 + {
49 + "d": {
50 + "type": "uri",
51 + "value": "http://id.nlm.nih.gov/mesh/D001943"
52 + },
53 + "label": {
54 + "type": "literal",
55 + "xml:lang": "en",
56 + "value": "Breast Neoplasms"
57 + },
58 + "scope": {
59 + "type": "literal",
60 + "xml:lang": "en",
61 + "value": "Tumors or cancer of the human BREAST."
62 + },
63 + "introduced": {
64 + "type": "literal",
65 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
66 + "value": "1966-01-01"
67 + },
68 + "trees": {
69 + "type": "literal",
70 + "value": "C04.588.180|C17.800.090.500"
71 + },
72 + "terms": {
73 + "type": "literal",
74 + "value": "Breast Neoplasm|Breast Neoplasms|Breast Tumor|Breast Tumors|Neoplasm, Breast|Neoplasms, Breast|Tumor, Breast|Tumors, Breast"
75 + },
76 + "related": {
77 + "type": "literal",
78 + "value": "Breast Cancer|Breast Carcinoma|Breast Carcinomas|Breast Malignant Neoplasm|Breast Malignant Neoplasms|Breast Malignant Tumor|Breast Malignant Tumors|Cancer of Breast|Cancer of the Breast|Cancer, Breast|Cancer, Mammary|Cancers, Mammary|Carcinoma, Breast|Carcinoma, Human Mammary|Carcinomas, Breast|Carcinomas, Human Mammary|Human Mammary Carcinoma|Human Mammary Carcinomas|Human Mammary Neoplasm|Human Mammary Neoplasms|Malignant Neoplasm of Breast|Malignant Tumor of Breast|Mammary Cancer|Mammary Cancers|Mammary Carcinoma, Human|Mammary Carcinomas, Human|Mammary Neoplasm, Human|Mammary Neoplasms, Human|Neoplasm, Human Mammary|Neoplasms, Human Mammary"
79 + }
80 + },
81 + {
82 + "d": {
83 + "type": "uri",
84 + "value": "http://id.nlm.nih.gov/mesh/D002289"
85 + },
86 + "label": {
87 + "type": "literal",
88 + "xml:lang": "en",
89 + "value": "Carcinoma, Non-Small-Cell Lung"
90 + },
91 + "scope": {
92 + "type": "literal",
93 + "xml:lang": "en",
94 + "value": "A heterogeneous aggregate of at least three distinct histological types of lung cancer, including SQUAMOUS CELL CARCINOMA; ADENOCARCINOMA; and LARGE CELL CARCINOMA. They are dealt with collectively because of their shared treatment strategy."
95 + },
96 + "introduced": {
97 + "type": "literal",
98 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
99 + "value": "1987-01-01"
100 + },
101 + "trees": {
102 + "type": "literal",
103 + "value": "C04.588.894.797.520.109.220.249|C08.381.540.140.500|C08.785.520.100.220.500"
104 + },
105 + "terms": {
106 + "type": "literal",
107 + "value": "Carcinoma, Non Small Cell Lung|Carcinoma, Non-Small Cell Lung|Carcinoma, Non-Small-Cell Lung|Carcinomas, Non-Small-Cell Lung|Lung Carcinoma, Non-Small-Cell|Lung Carcinomas, Non-Small-Cell|Non Small Cell Lung Carcinoma|Non-Small Cell Lung Cancer|Non-Small Cell Lung Carcinoma|Non-Small-Cell Lung Carcinoma|Non-Small-Cell Lung Carcinomas|Nonsmall Cell Lung Cancer"
108 + },
109 + "related": {
110 + "type": "literal",
111 + "value": ""
112 + }
113 + },
114 + {
115 + "d": {
116 + "type": "uri",
117 + "value": "http://id.nlm.nih.gov/mesh/D008175"
118 + },
119 + "label": {
120 + "type": "literal",
121 + "xml:lang": "en",
122 + "value": "Lung Neoplasms"
123 + },
124 + "scope": {
125 + "type": "literal",
126 + "xml:lang": "en",
127 + "value": "Tumors or cancer of the LUNG."
128 + },
129 + "introduced": {
130 + "type": "literal",
131 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
132 + "value": "1966-01-01"
133 + },
134 + "trees": {
135 + "type": "literal",
136 + "value": "C04.588.894.797.520|C08.381.540|C08.785.520"
137 + },
138 + "terms": {
139 + "type": "literal",
140 + "value": "Lung Neoplasm|Lung Neoplasms|Neoplasm, Lung|Neoplasm, Pulmonary|Neoplasms, Lung|Neoplasms, Pulmonary|Pulmonary Neoplasm|Pulmonary Neoplasms"
141 + },
142 + "related": {
143 + "type": "literal",
144 + "value": "Cancer of Lung|Cancer of the Lung|Cancer, Lung|Cancer, Pulmonary|Cancers, Lung|Cancers, Pulmonary|Lung Cancer|Lung Cancers|Pulmonary Cancer|Pulmonary Cancers"
145 + }
146 + },
147 + {
148 + "d": {
149 + "type": "uri",
150 + "value": "http://id.nlm.nih.gov/mesh/D009369"
151 + },
152 + "label": {
153 + "type": "literal",
154 + "xml:lang": "en",
155 + "value": "Neoplasms"
156 + },
157 + "scope": {
158 + "type": "literal",
159 + "xml:lang": "en",
160 + "value": "New abnormal growth of tissue. Malignant neoplasms show a greater degree of anaplasia and have the properties of invasion and metastasis, compared to benign neoplasms."
161 + },
162 + "introduced": {
163 + "type": "literal",
164 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
165 + "value": "1966-01-01"
166 + },
167 + "trees": {
168 + "type": "literal",
169 + "value": "C04"
170 + },
171 + "terms": {
172 + "type": "literal",
173 + "value": "Neoplasia|Neoplasias|Neoplasm|Neoplasms|Tumor|Tumors"
174 + },
175 + "related": {
176 + "type": "literal",
177 + "value": "Benign Neoplasm|Benign Neoplasms|Cancer|Cancers|Malignancies|Malignancy|Malignant Neoplasm|Malignant Neoplasms|Neoplasm, Benign|Neoplasm, Malignant|Neoplasms, Benign|Neoplasms, Malignant"
178 + }
179 + },
180 + {
181 + "d": {
182 + "type": "uri",
183 + "value": "http://id.nlm.nih.gov/mesh/D013964"
184 + },
185 + "label": {
186 + "type": "literal",
187 + "xml:lang": "en",
188 + "value": "Thyroid Neoplasms"
189 + },
190 + "scope": {
191 + "type": "literal",
192 + "xml:lang": "en",
193 + "value": "Tumors or cancer of the THYROID GLAND."
194 + },
195 + "introduced": {
196 + "type": "literal",
197 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
198 + "value": "1966-01-01"
199 + },
200 + "trees": {
201 + "type": "literal",
202 + "value": "C04.588.322.894|C04.588.443.915|C19.344.894|C19.874.788"
203 + },
204 + "terms": {
205 + "type": "literal",
206 + "value": "Neoplasm, Thyroid|Neoplasms, Thyroid|Thyroid Neoplasm|Thyroid Neoplasms"
207 + },
208 + "related": {
209 + "type": "literal",
210 + "value": "Adenoma, Thyroid|Adenomas, Thyroid|Cancer of Thyroid|Cancer of the Thyroid|Cancer, Thyroid|Cancers, Thyroid|Carcinoma, Thyroid|Carcinomas, Thyroid|Thyroid Adenoma|Thyroid Adenomas|Thyroid Cancer|Thyroid Cancers|Thyroid Carcinoma|Thyroid Carcinomas"
211 + }
212 + },
213 + {
214 + "d": {
215 + "type": "uri",
216 + "value": "http://id.nlm.nih.gov/mesh/D015464"
217 + },
218 + "label": {
219 + "type": "literal",
220 + "xml:lang": "en",
221 + "value": "Leukemia, Myelogenous, Chronic, BCR-ABL Positive"
222 + },
223 + "scope": {
224 + "type": "literal",
225 + "xml:lang": "en",
226 + "value": "Clonal hematopoetic disorder caused by an acquired genetic defect in PLURIPOTENT STEM CELLS. It starts in MYELOID CELLS of the bone marrow, invades the blood and then other organs. The condition progresses from a stable, more indolent, chronic phase (LEUKEMIA, MYELOID, CHRONIC PHASE) lasting up to 7 years, to an advanced phase composed of an accelerated phase (LEUKEMIA, MYELOID, ACCELERATED PHASE) and BLAST CRISIS."
227 + },
228 + "introduced": {
229 + "type": "literal",
230 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
231 + "value": "1989-01-01"
232 + },
233 + "trees": {
234 + "type": "literal",
235 + "value": "C04.557.337.539.250|C15.378.190.636.370|C15.378.508.539.250|C23.550.291.500.485"
236 + },
237 + "terms": {
238 + "type": "literal",
239 + "value": "Chronic Granulocytic Leukemia|Chronic Granulocytic Leukemias|Chronic Myelocytic Leukemia|Chronic Myelocytic Leukemias|Chronic Myelogenous Leukemia|Chronic Myelogenous Leukemias|Chronic Myeloid Leukemia|Chronic Myeloid Leukemias|Granulocytic Leukemia, Chronic|Granulocytic Leukemias, Chronic|Leukemia, Chronic Granulocytic|Leukemia, Chronic Myelocytic|Leukemia, Chronic Myelogenous|Leukemia, Chronic Myeloid|Leukemia, Granulocytic, Chronic|Leukemia, Myelocytic, Chronic|Leukemia, Myelogenous, Chronic|Leukemia, Myelogenous, Chronic, BCR-ABL Positive|Leukemia, Myelogenous, Ph1 Positive|Leukemia, Myelogenous, Ph1-Positive|Leukemia, Myeloid, Chronic|Leukemia, Myeloid, Ph1 Positive|Leukemia, Myeloid, Ph1-Positive|Leukemia, Myeloid, Philadelphia Positive|Leukemia, Myeloid, Philadelphia-Positive|Leukemia, Ph1-Positive Myelogenous|Leukemia, Ph1-Positive Myeloid|Leukemia, Philadelphia-Positive Myeloid|Leukemias, Chronic Granulocytic|Leukemias, Chronic Myelocytic|Leukemias, Chronic Myelogenous|Leukemias, Chronic Myeloid|Leukemias, Ph1-Positive Myelogenous|Leukemias, Ph1-Positive Myeloid|Leukemias, Philadelphia-Positive Myeloid|Myelocytic Leukemia, Chronic|Myelocytic Leukemias, Chronic|Myelogenous Leukemia, Chronic|Myelogenous Leukemia, Ph1 Positive|Myelogenous Leukemia, Ph1-Positive|Myelogenous Leukemias, Chronic|Myelogenous Leukemias, Ph1-Positive|Myeloid Leukemia, Chronic|Myeloid Leukemia, Ph1 Positive|Myeloid Leukemia, Ph1-Positive|Myeloid Leukemia, Philadelphia Positive|Myeloid Leukemia, Philadelphia-Positive|Myeloid Leukemias, Chronic|Myeloid Leukemias, Ph1-Positive|Myeloid Leukemias, Philadelphia-Positive|Ph1-Positive Myelogenous Leukemia|Ph1-Positive Myelogenous Leukemias|Ph1-Positive Myeloid Leukemia|Ph1-Positive Myeloid Leukemias|Philadelphia-Positive Myeloid Leukemia|Philadelphia-Positive Myeloid Leukemias"
240 + },
241 + "related": {
242 + "type": "literal",
243 + "value": ""
244 + }
245 + },
246 + {
247 + "d": {
248 + "type": "uri",
249 + "value": "http://id.nlm.nih.gov/mesh/D016393"
250 + },
251 + "label": {
252 + "type": "literal",
253 + "xml:lang": "en",
254 + "value": "Lymphoma, B-Cell"
255 + },
256 + "scope": {
257 + "type": "literal",
258 + "xml:lang": "en",
259 + "value": "A group of heterogeneous lymphoid tumors generally expressing one or more B-cell antigens or representing malignant transformations of B-lymphocytes."
260 + },
261 + "introduced": {
262 + "type": "literal",
263 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
264 + "value": "1991-01-01"
265 + },
266 + "trees": {
267 + "type": "literal",
268 + "value": "C04.557.386.480.150|C15.604.515.569.480.150|C20.683.515.761.480.150"
269 + },
270 + "terms": {
271 + "type": "literal",
272 + "value": "B Cell Lymphoma|B-Cell Lymphoma|B-Cell Lymphomas|Lymphoma, B Cell|Lymphoma, B-Cell|Lymphomas, B-Cell"
273 + },
274 + "related": {
275 + "type": "literal",
276 + "value": ""
277 + }
278 + },
279 + {
280 + "d": {
281 + "type": "uri",
282 + "value": "http://id.nlm.nih.gov/mesh/D019337"
283 + },
284 + "label": {
285 + "type": "literal",
286 + "xml:lang": "en",
287 + "value": "Hematologic Neoplasms"
288 + },
289 + "scope": {
290 + "type": "literal",
291 + "xml:lang": "en",
292 + "value": "Neoplasms located in the blood and blood-forming tissue (the bone marrow and lymphatic tissue). The commonest forms are the various types of LEUKEMIA, of LYMPHOMA, and of the progressive, life-threatening forms of the MYELODYSPLASTIC SYNDROMES."
293 + },
294 + "introduced": {
295 + "type": "literal",
296 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
297 + "value": "1997-01-01"
298 + },
299 + "trees": {
300 + "type": "literal",
301 + "value": "C04.588.448|C15.378.400"
302 + },
303 + "terms": {
304 + "type": "literal",
305 + "value": "Blood Cancer|Blood Cancers|Cancer, Blood|Hematologic Malignancies|Hematologic Malignancy|Hematologic Neoplasm|Hematologic Neoplasms|Hematological Malignancies|Hematological Malignancy|Hematological Neoplasm|Hematological Neoplasms|Malignancies, Hematologic|Malignancy, Hematologic|Malignancy, Hematological|Neoplasm, Hematologic|Neoplasm, Hematological|Neoplasms, Hematologic"
306 + },
307 + "related": {
308 + "type": "literal",
309 + "value": "Hematopoietic Malignancies|Hematopoietic Malignancy|Hematopoietic Neoplasm|Hematopoietic Neoplasms|Malignancy, Hematopoietic|Neoplasm, Hematopoietic|Neoplasms, Hematopoietic"
310 + }
311 + },
312 + {
313 + "d": {
314 + "type": "uri",
315 + "value": "http://id.nlm.nih.gov/mesh/D054218"
316 + },
317 + "label": {
318 + "type": "literal",
319 + "xml:lang": "en",
320 + "value": "Precursor T-Cell Lymphoblastic Leukemia-Lymphoma"
321 + },
322 + "scope": {
323 + "type": "literal",
324 + "xml:lang": "en",
325 + "value": "A leukemia/lymphoma found predominately in children and young adults and characterized LYMPHADENOPATHY and THYMUS GLAND involvement. It most frequently presents as a lymphoma, but a leukemic progression in the bone marrow is common."
326 + },
327 + "introduced": {
328 + "type": "literal",
329 + "datatype": "http://www.w3.org/2001/XMLSchema#date",
330 + "value": "2008-01-01"
331 + },
332 + "trees": {
333 + "type": "literal",
334 + "value": "C04.557.337.428.600.620|C15.378.508.428.600.620|C15.604.515.560.600.620|C20.683.515.528.600.620"
335 + },
336 + "terms": {
337 + "type": "literal",
338 + "value": "Acute T-Cell Leukemia|Acute T-Cell Leukemias|Acute T-Lymphocytic Leukemia|Acute T-Lymphocytic Leukemias|Leukemia, Acute T-Cell|Leukemia, Acute T-Lymphocytic|Leukemia, Lymphoblastic, Acute, T Cell|Leukemia, Lymphoblastic, Acute, T-Cell|Leukemia, Lymphocytic, Acute T Cell|Leukemia, Lymphocytic, Acute, T-Cell|Leukemia, T-Cell, Acute|Leukemias, Acute T-Cell|Leukemias, Acute T-Lymphocytic|Lymphoblastic Leukemia, Acute, T Cell|Lymphoblastic Leukemia, Acute, T-Cell|Lymphocytic Leukemia, T Cell, Acute|Lymphocytic Leukemia, T-Cell, Acute|Precursor T Cell Lymphoblastic Leukemia|Precursor T Cell Lymphoblastic Leukemia Lymphoma|Precursor T Cell Lymphoblastic Lymphoma|Precursor T-Cell Lymphoblastic Leukemia|Precursor T-Cell Lymphoblastic Leukemia-Lymphoma|Precursor T-Cell Lymphoblastic Lymphoma|T Cell Leukemia, Acute|T Lymphocytic Leukemia, Acute|T-ALL|T-Cell Acute Lymphocytic Leukemia|T-Cell Leukemia, Acute|T-Cell Leukemias, Acute|T-Lymphocytic Leukemia, Acute|T-Lymphocytic Leukemias, Acute"
339 + },
340 + "related": {
341 + "type": "literal",
342 + "value": ""
343 + }
344 + },
345 + {
346 + "d": {
347 + "type": "uri",
348 + "value": "http://id.nlm.nih.gov/mesh/X123"
349 + },
350 + "label": {
351 + "type": "literal",
352 + "value": "Broken Row"
353 + },
354 + "trees": {
355 + "type": "literal",
356 + "value": ""
357 + },
358 + "terms": {
359 + "type": "literal",
360 + "value": "Broken Row"
361 + }
362 + }
363 + ]
364 + }
365 +}
\ No newline at end of file
added packages/connectors/src/connectors/mesh/index.ts +199 −0
@@ -0,0 +1,199 @@
1 +import { sql } from 'drizzle-orm';
2 +import { normalizeLabel } from '@cancerindex/shared';
3 +import { cancerAliases, cancerCodes } from '@cancerindex/database';
4 +import { CancerResolver } from '@cancerindex/ontology';
5 +import { Connector, type ConnectorHealth, type RunContext } from '../../sdk/run.js';
6 +import { manifest } from './manifest.js';
7 +import { Descriptor, HEALTH_QUERY, PAGE_SIZE, SparqlAsk, aliasWorthyTerms, descriptorsQuery, inNeoplasmsTree, mapDescriptor, parseDescriptorPage, sparqlUrl, yearGraphQuery } from './sparql.js';
8 +import { DESC_XML_URL, descriptorRecords, parseDescriptorRecord } from './xml.js';
9 +
10 +const ENDPOINT = manifest.access.baseUrl!;
11 +/** The endpoint ignores `format=JSON` when Accept lists application/json (it answers SPARQL-XML as text/plain); `*\/*` makes it honour the parameter. */
12 +const SPARQL_INIT: RequestInit = { headers: { accept: '*/*' } };
13 +/** Below this many descriptors the Neoplasms tree cannot be complete (703 under C04 alone in MeSH 2026). */
14 +const MIN_DESCRIPTORS = 500;
15 +
16 +interface MeshStats {
17 + descriptors: number;
18 + inC04: number;
19 + mapped: Record<string, number>;
20 + mappedC04: number;
21 + headingsAdded: number;
22 + aliasesAdded: number;
23 + unresolved: number;
24 + skippedOutsideC04: number;
25 + malformed: number;
26 +}
27 +
28 +/**
29 + * MeSH connector: maps the descriptors of the Neoplasms tree (and malignant hematologic subtrees)
30 + * onto canonical cancers. Writes cancer_codes (system "mesh", code = descriptor UI, match_type from
31 + * the resolver) and, for exact-grade matches, the heading (alias_type "mesh_heading") and entry terms
32 + * (alias_type "synonym") with source_terminology "MeSH" — the PubMed connector turns those into
33 + * `"<heading>"[mh]` queries and resolves article MeSH headings by UI. Never mints cancers (§340).
34 + */
35 +export class MeshConnector extends Connector {
36 + readonly manifest = manifest;
37 +
38 + async healthCheck(ctx: RunContext): Promise<ConnectorHealth> {
39 + try {
40 + const res = SparqlAsk.parse(await ctx.http.json(sparqlUrl(ENDPOINT, HEALTH_QUERY), SPARQL_INIT));
41 + return res.boolean ? { status: 'healthy', detail: 'SPARQL endpoint answers; D009369 "Neoplasms" present' } : { status: 'degraded', detail: 'root descriptor D009369 missing from the current graph' };
42 + } catch (e) {
43 + return { status: 'failing', detail: (e as Error).message };
44 + }
45 + }
46 +
47 + async sync(ctx: RunContext): Promise<void> {
48 + const year = await this.detectYear(ctx);
49 + ctx.datasetVersion = `MeSH ${year}`;
50 + ctx.info(`MeSH current graph → dataset version ${ctx.datasetVersion}`);
51 +
52 + const stats: MeshStats = { descriptors: 0, inC04: 0, mapped: {}, mappedC04: 0, headingsAdded: 0, aliasesAdded: 0, unresolved: 0, skippedOutsideC04: 0, malformed: 0 };
53 + let descriptors: Descriptor[];
54 + let route: 'sparql' | 'xml' = 'sparql';
55 + try {
56 + descriptors = await this.fetchViaSparql(ctx, stats);
57 + } catch (e) {
58 + if (ctx.mode === 'dry_run') throw e;
59 + ctx.warn(`SPARQL route failed (${(e as Error).message}) — falling back to the descriptor XML desc${year}.xml`);
60 + route = 'xml';
61 + descriptors = await this.fetchViaXml(ctx, year, stats);
62 + }
63 + stats.descriptors = descriptors.length;
64 + stats.inC04 = descriptors.filter(inNeoplasmsTree).length;
65 + ctx.info(`fetched ${descriptors.length} descriptors via ${route} (${stats.inC04} in C04, ${stats.malformed} malformed rows)`);
66 + if (ctx.mode === 'dry_run') {
67 + ctx.counters.fetched = descriptors.length;
68 + return;
69 + }
70 + if (descriptors.length < MIN_DESCRIPTORS) throw new Error(`anomaly: only ${descriptors.length} MeSH descriptors fetched (expected ≥ ${MIN_DESCRIPTORS}) — refusing to persist (CLAUDE.md §171)`);
71 +
72 + const resolver = new CancerResolver(ctx.db);
73 + await resolver.warm();
74 + // Existing (cancer, normalized alias) pairs: MeSH entry terms already known under any type are not re-added.
75 + const existingAliases = new Set<string>();
76 + for (const a of await ctx.db.select({ cancerId: cancerAliases.cancerId, normalized: cancerAliases.normalized }).from(cancerAliases)) existingAliases.add(`${a.cancerId}|${a.normalized}`);
77 +
78 + await ctx.addProvenance({
79 + dataset: 'MeSH descriptors — Neoplasms tree (C04) + hematologic/immunoproliferative subtrees',
80 + datasetVersion: ctx.datasetVersion,
81 + sourceUrl: route === 'sparql' ? ENDPOINT : DESC_XML_URL(year),
82 + evidenceType: 'expert_curation',
83 + accessLevel: 'open',
84 + methodology: 'Descriptor heading and entry terms reconciled against canonical cancer aliases (identifiers → curated aliases → normalized strings); descriptor UI stored as cancer_codes system "mesh" with the resolver match type',
85 + });
86 +
87 + for (const d of descriptors) {
88 + const mapping = mapDescriptor(resolver, d);
89 + await ctx.upsertSourceRecord('descriptor', d.ui, { ...d, meshYear: year, inC04: inNeoplasmsTree(d) }, mapping ? { canonicalType: 'cancer', canonicalId: mapping.cancerId } : {});
90 + if (!mapping) {
91 + if (inNeoplasmsTree(d)) {
92 + stats.unresolved++;
93 + const suggestion = await resolver.suggest(d.label);
94 + await ctx.recordUnresolved('cancer', d.label, normalizeLabel(d.label), { meshUi: d.ui, treeNumbers: d.treeNumbers, entryTerms: d.terms.slice(0, 12), scopeNote: d.scopeNote }, suggestion ? { id: suggestion.cancerId, matchType: 'FUZZY', score: suggestion.score } : undefined);
95 + } else stats.skippedOutsideC04++;
96 + continue;
97 + }
98 + stats.mapped[mapping.matchType] = (stats.mapped[mapping.matchType] ?? 0) + 1;
99 + if (inNeoplasmsTree(d)) stats.mappedC04++;
100 + await ctx.db.insert(cancerCodes).values({ cancerId: mapping.cancerId, system: 'mesh', code: d.ui, matchType: mapping.matchType, sourceId: ctx.sourceId, validFrom: d.dateIntroduced ?? undefined }).onConflictDoNothing();
101 + if (!mapping.addAliases) continue;
102 + // Heading: its own alias type so it never collides with an identical NCIt synonym (the PubMed
103 + // connector needs the exact descriptor name, tagged MeSH, to build "<heading>"[mh]).
104 + const heading = await ctx.db
105 + .insert(cancerAliases)
106 + .values({ cancerId: mapping.cancerId, alias: d.label, normalized: normalizeLabel(d.label), aliasType: 'mesh_heading', sourceId: ctx.sourceId, sourceTerminology: 'MeSH' })
107 + .onConflictDoNothing()
108 + .returning({ id: cancerAliases.id });
109 + if (heading.length) stats.headingsAdded++;
110 + for (const term of aliasWorthyTerms(d)) {
111 + const norm = normalizeLabel(term);
112 + const key = `${mapping.cancerId}|${norm}`;
113 + if (existingAliases.has(key)) continue;
114 + const ins = await ctx.db.insert(cancerAliases).values({ cancerId: mapping.cancerId, alias: term, normalized: norm, aliasType: 'synonym', sourceId: ctx.sourceId, sourceTerminology: 'MeSH' }).onConflictDoNothing().returning({ id: cancerAliases.id });
115 + existingAliases.add(key);
116 + if (ins.length) stats.aliasesAdded++;
117 + }
118 + }
119 +
120 + const mappedTotal = Object.values(stats.mapped).reduce((a, b) => a + b, 0);
121 + const top = await ctx.db.execute<{ source_text: string; count: number; context: { meshUi?: string } }>(sql`SELECT source_text, count, context FROM unresolved_labels WHERE source_id = ${ctx.sourceId} AND entity_kind = 'cancer' AND status = 'open' ORDER BY count DESC, source_text LIMIT 20`);
122 + ctx.info(
123 + `MeSH pass complete: ${stats.descriptors} descriptors (${stats.inC04} in C04) — mapped ${mappedTotal} (${Object.entries(stats.mapped)
124 + .map(([k, v]) => `${k} ${v}`)
125 + .join(', ')}); C04 resolution ${stats.mappedC04}/${stats.inC04} (${stats.inC04 ? Math.round((1000 * stats.mappedC04) / stats.inC04) / 10 : 0} %); headings added ${stats.headingsAdded}, entry-term aliases added ${stats.aliasesAdded}; unresolved ${stats.unresolved}; outside-C04 skipped ${stats.skippedOutsideC04}`,
126 + { ...stats, topUnresolved: top.map((u) => `${u.source_text} [${u.context?.meshUi ?? '?'}] (${u.count})`) },
127 + );
128 + ctx.cursor = { year, route, descriptors: stats.descriptors, syncedAt: new Date().toISOString() };
129 + }
130 +
131 + /* ---------------------------------------------------------------------------------------------- */
132 +
133 + /** Latest year graph present on the endpoint (next year's graph appears in the autumn). */
134 + private async detectYear(ctx: RunContext): Promise<number> {
135 + const thisYear = new Date().getUTCFullYear();
136 + for (const y of [thisYear + 1, thisYear]) {
137 + try {
138 + const res = SparqlAsk.parse(await ctx.http.json(sparqlUrl(ENDPOINT, yearGraphQuery(y)), SPARQL_INIT));
139 + if (res.boolean) return y;
140 + } catch (e) {
141 + ctx.warn(`year graph probe ${y} failed: ${(e as Error).message}`);
142 + }
143 + }
144 + return thisYear;
145 + }
146 +
147 + private async fetchViaSparql(ctx: RunContext, stats: MeshStats): Promise<Descriptor[]> {
148 + const out: Descriptor[] = [];
149 + const seen = new Set<string>();
150 + for (let offset = 0; ; offset += PAGE_SIZE) {
151 + const json = await ctx.http.json(sparqlUrl(ENDPOINT, descriptorsQuery(offset, PAGE_SIZE)), SPARQL_INIT);
152 + const page = parseDescriptorPage(json);
153 + for (const m of page.malformed) {
154 + stats.malformed++;
155 + ctx.counters.validationFailures++;
156 + ctx.counters.rejected++;
157 + ctx.warn(`malformed descriptor row: ${m.error}`);
158 + }
159 + let distinct = 0;
160 + for (const d of page.descriptors) {
161 + ctx.observe('descriptor', d);
162 + if (seen.has(d.ui)) continue; // a descriptor with two scope notes would produce two grouped rows
163 + seen.add(d.ui);
164 + out.push(d);
165 + distinct++;
166 + }
167 + ctx.info(`SPARQL page offset ${offset}: ${page.descriptors.length} rows, ${distinct} new descriptors`);
168 + if (page.descriptors.length + page.malformed.length < PAGE_SIZE || ctx.mode === 'dry_run') break;
169 + if (ctx.shouldStop()) break;
170 + }
171 + return out;
172 + }
173 +
174 + private async fetchViaXml(ctx: RunContext, year: number, stats: MeshStats): Promise<Descriptor[]> {
175 + const res = await ctx.http.request(DESC_XML_URL(year));
176 + if (!res.body) throw new Error('empty XML body');
177 + const out: Descriptor[] = [];
178 + let records = 0;
179 + for await (const xml of descriptorRecords(res.body as unknown as AsyncIterable<Uint8Array>)) {
180 + records++;
181 + try {
182 + const d = parseDescriptorRecord(xml);
183 + if (d) {
184 + ctx.observe('descriptor', d);
185 + out.push(d);
186 + }
187 + } catch (e) {
188 + stats.malformed++;
189 + ctx.counters.validationFailures++;
190 + ctx.counters.rejected++;
191 + ctx.warn(`malformed DescriptorRecord: ${(e as Error).message}`);
192 + }
193 + }
194 + ctx.info(`XML fallback: ${records} DescriptorRecords scanned, ${out.length} in the fetched subtrees`);
195 + return out;
196 + }
197 +}
198 +
199 +export const connector = new MeshConnector();
added packages/connectors/src/connectors/mesh/manifest.ts +55 −0
@@ -0,0 +1,55 @@
1 +import { defineManifest } from '../../sdk/manifest.js';
2 +
3 +/**
4 + * Docs verified 2026-09-08:
5 + * - MeSH RDF SPARQL endpoint https://id.nlm.nih.gov/mesh/sparql (GET ?query=&format=JSON&inference=true),
6 + * docs https://hhs.github.io/meshrdf/ (Descriptor / Concept / Term model, tree numbers as
7 + * resources with rdfs:label "C04.588.180", year graphs <http://id.nlm.nih.gov/mesh/YYYY> and the
8 + * current graph <http://id.nlm.nih.gov/mesh>). Predicates probed live on D001943 "Breast Neoplasms":
9 + * meshv:treeNumber, meshv:concept / meshv:preferredConcept, meshv:term / meshv:preferredTerm,
10 + * meshv:prefLabel / meshv:altLabel on Terms, meshv:scopeNote on Concepts, meshv:dateIntroduced,
11 + * meshv:broaderDescriptor. Current graph: 703 descriptors under C04 (Neoplasms), 30,512 topical
12 + * descriptors in <http://id.nlm.nih.gov/mesh/2026>.
13 + * - Descriptor XML https://nlmpubs.nlm.nih.gov/projects/mesh/MESH_FILES/xmlmesh/desc2026.xml
14 + * (Last-Modified 2026-08-12, ~313 MB) — fallback only, chunk-parsed record by record.
15 + * - Terms (https://www.nlm.nih.gov/databases/download/terms_and_conditions.html, fetched
16 + * 2026-09-08): users agree to "acknowledge NLM as the source of the data by including the phrase
17 + * 'Courtesy of the U.S. National Library of Medicine'", to "not indicate or imply that NLM has
18 + * endorsed its products/services/applications", and, when redistributing, to "maintain the most
19 + * current version of all distributed data" or make known that the product does not reflect NLM's
20 + * latest data. "No charges, usage fees or royalties are paid to NLM." NLM web policies
21 + * (https://www.nlm.nih.gov/web_policies.html): "Works produced by the U.S. government are not
22 + * subject to copyright protection in the United States."
23 + */
24 +export const manifest = defineManifest({
25 + id: 'mesh',
26 + name: 'MeSH — Medical Subject Headings (NLM)',
27 + organization: 'U.S. National Library of Medicine',
28 + category: 'terminology',
29 + tier: 1,
30 + description:
31 + 'NLM controlled vocabulary used to index PubMed. CancerIndex maps the MeSH descriptors of the Neoplasms tree (C04) and the malignant hematologic / immunoproliferative subtrees (C15.378.190.6*, C20.683) onto canonical cancers: descriptor UIs become cancer_codes (system "mesh"), headings and entry terms become aliases, so literature queries can be MeSH-anchored and PubMed→cancer edges resolve by UI. MeSH never creates canonical cancers (it is an indexing vocabulary, not a disease ontology).',
32 + homepage: 'https://www.nlm.nih.gov/mesh/',
33 + docsUrl: 'https://hhs.github.io/meshrdf/',
34 + termsUrl: 'https://www.nlm.nih.gov/databases/download/terms_and_conditions.html',
35 + access: { type: 'api', auth: 'none', baseUrl: 'https://id.nlm.nih.gov/mesh/sparql' },
36 + license: 'Public domain (U.S. Government work) — NLM Terms and Conditions (acknowledgment requested)',
37 + licenseStatus: 'approved',
38 + commercialUse: 'allowed',
39 + redistribution: 'attribution',
40 + attribution: 'Courtesy of the U.S. National Library of Medicine. MeSH (Medical Subject Headings), https://www.nlm.nih.gov/mesh/.',
41 + termsReviewedAt: '2026-09-08',
42 + termsNotes:
43 + 'NLM terms (2026-09-08): acknowledge NLM ("Courtesy of the U.S. National Library of Medicine"), no implied endorsement, redistributors keep data current or disclose staleness — the /source page shows the MeSH year and retrieval date. U.S. Government works are not subject to copyright in the United States.',
44 + updateFrequency: 'Annual release (year graph, e.g. 2026) with in-year updates',
45 + expectedLatency: 'Weeks after the annual release',
46 + supportsIncrementalSync: false,
47 + entities: ['cancer_codes', 'cancer_aliases'],
48 + rateLimits: { requestsPerSecond: 2, maxConcurrency: 1, notes: 'A handful of SPARQL pages (1,000 descriptors each) per run; be polite with the public endpoint.' },
49 + retryPolicy: { maxRetries: 4, baseDelayMs: 1000, maxDelayMs: 30_000 },
50 + rawRetention: 'full',
51 + documentationVerifiedAt: '2026-09-08',
52 + status: 'active',
53 + schedule: '0 4 2 * *',
54 + anomalyGuard: { minRatioOfPrevious: 0.8 },
55 +});
added packages/connectors/src/connectors/mesh/mesh.test.ts +136 −0
@@ -0,0 +1,136 @@
1 +import { readFileSync } from 'node:fs';
2 +import path from 'node:path';
3 +import { describe, expect, it } from 'vitest';
4 +import type { CancerMatch } from '@cancerindex/ontology';
5 +import { manifest } from './manifest.js';
6 +import { Descriptor, HEALTH_QUERY, MESH_GRAPH, aliasWorthyTerms, descriptorsQuery, inNeoplasmsTree, mapDescriptor, parseDescriptorPage, sparqlUrl, type LabelResolver } from './sparql.js';
7 +import { descriptorRecords, parseDescriptorRecord } from './xml.js';
8 +
9 +const fx = (name: string) => readFileSync(path.join(import.meta.dirname, 'fixtures', name), 'utf8');
10 +const page = () => JSON.parse(fx('descriptors-page.json')) as unknown;
11 +
12 +describe('mesh manifest', () => {
13 + it('is approved public-domain data with NLM acknowledgment and verified docs', () => {
14 + expect(manifest.id).toBe('mesh');
15 + expect(manifest.licenseStatus).toBe('approved');
16 + expect(manifest.redistribution).toBe('attribution');
17 + expect(manifest.attribution).toMatch(/Courtesy of the U\.S\. National Library of Medicine/);
18 + expect(manifest.documentationVerifiedAt).toBe('2026-09-08');
19 + expect(manifest.entities).toEqual(['cancer_codes', 'cancer_aliases']);
20 + });
21 +});
22 +
23 +describe('SPARQL query builder', () => {
24 + it('restricts to the current graph, the neoplasm subtrees and paginates', () => {
25 + const q = descriptorsQuery(2000, 1000);
26 + expect(q).toContain(`FROM <${MESH_GRAPH}>`);
27 + expect(q).toContain("STRSTARTS(STR(?tnl0), 'C04')");
28 + expect(q).toContain("STRSTARTS(STR(?tnl0), 'C20.683')");
29 + expect(q).toContain('LIMIT 1000 OFFSET 2000');
30 + expect(q).toContain('meshv:prefLabel|meshv:altLabel');
31 + expect(q).toContain('FILTER(?oc != ?pc)');
32 + const url = new URL(sparqlUrl('https://id.nlm.nih.gov/mesh/sparql', HEALTH_QUERY));
33 + expect(url.searchParams.get('format')).toBe('JSON');
34 + expect(url.searchParams.get('inference')).toBe('true');
35 + expect(url.searchParams.get('query')).toContain('ASK');
36 + });
37 +});
38 +
39 +describe('descriptor page parsing', () => {
40 + it('parses real rows and isolates the malformed one', () => {
41 + const { descriptors, malformed } = parseDescriptorPage(page());
42 + expect(descriptors).toHaveLength(10);
43 + expect(malformed).toHaveLength(1);
44 + expect(malformed[0]!.error).toMatch(/ui|treeNumbers/);
45 + const breast = descriptors.find((d) => d.ui === 'D001943')!;
46 + expect(breast.label).toBe('Breast Neoplasms');
47 + expect(breast.treeNumbers).toEqual(expect.arrayContaining(['C04.588.180', 'C17.800.090.500']));
48 + expect(breast.terms).toEqual(expect.arrayContaining(['Breast Neoplasms', 'Neoplasm, Breast']));
49 + expect(breast.relatedTerms).toEqual(expect.arrayContaining(['Breast Cancer'])); // narrower concept, never used for mapping
50 + const adeno = descriptors.find((d) => d.ui === 'D000230')!;
51 + expect(adeno.terms).not.toContain('Adenocarcinoma, Basal Cell');
52 + expect(adeno.relatedTerms).toContain('Adenocarcinoma, Basal Cell');
53 + expect(breast.scopeNote).toMatch(/BREAST/);
54 + expect(inNeoplasmsTree(breast)).toBe(true);
55 + const cml = descriptors.find((d) => d.ui === 'D015464')!;
56 + expect(cml.label).toBe('Leukemia, Myelogenous, Chronic, BCR-ABL Positive');
57 + });
58 + it('handles an empty page', () => {
59 + const { descriptors, malformed } = parseDescriptorPage({ head: { vars: [] }, results: { bindings: [] } });
60 + expect(descriptors).toEqual([]);
61 + expect(malformed).toEqual([]);
62 + });
63 + it('rejects a non-SPARQL body (HTML error page parsed as JSON, server error)', () => {
64 + expect(() => parseDescriptorPage({ error: 'Service Unavailable' })).toThrow(/malformed SPARQL response/);
65 + });
66 +});
67 +
68 +describe('alias-worthy entry terms', () => {
69 + it('drops the heading, short terms and inverted/plural permutations', () => {
70 + const d = Descriptor.parse({ ui: 'D001943', label: 'Breast Neoplasms', treeNumbers: ['C04.588.180'], terms: ['Breast Neoplasms', 'Breast Neoplasm', 'Neoplasm, Breast', 'Neoplasms, Breast', 'Breast Cancer', 'Cancer, Breast', 'Breast Tumors', 'Tumors, Breast', 'Cancer of Breast', 'BC'], scopeNote: null, dateIntroduced: null });
71 + expect(aliasWorthyTerms(d)).toEqual(['Breast Cancer', 'Breast Tumors', 'Cancer of Breast']);
72 + });
73 +});
74 +
75 +function fakeResolver(table: Record<string, CancerMatch | null>): LabelResolver & { calls: string[] } {
76 + const calls: string[] = [];
77 + return {
78 + calls,
79 + byLabel(label: string) {
80 + calls.push(label);
81 + return table[label] ?? null;
82 + },
83 + };
84 +}
85 +
86 +describe('descriptor → cancer mapping', () => {
87 + const base = { treeNumbers: ['C04.588.180'], scopeNote: null, dateIntroduced: null };
88 + it('exact heading match → code + aliases with the resolver match type', () => {
89 + const r = fakeResolver({ 'Breast Neoplasms': { cancerId: 'CI-CAN-1', matchType: 'ONTOLOGY_EXACT', confidence: 0.98, via: 'alias' } });
90 + const m = mapDescriptor(r, Descriptor.parse({ ui: 'D001943', label: 'Breast Neoplasms', terms: ['Breast Neoplasms', 'Breast Cancer'], ...base }));
91 + expect(m).toMatchObject({ cancerId: 'CI-CAN-1', matchType: 'ONTOLOGY_EXACT', addAliases: true });
92 + expect(r.calls).toEqual(['Breast Neoplasms']);
93 + });
94 + it('CURATED_BROADER heading → code without aliases, unless an entry term is exact', () => {
95 + const r = fakeResolver({ 'Neoplasm Metastasis': { cancerId: 'CI-CAN-ROOT', matchType: 'CURATED_BROADER', confidence: 0.78, via: 'base' } });
96 + const m = mapDescriptor(r, Descriptor.parse({ ui: 'D009362', label: 'Neoplasm Metastasis', terms: ['Neoplasm Metastasis', 'Metastases'], ...base }));
97 + expect(m).toMatchObject({ cancerId: 'CI-CAN-ROOT', matchType: 'CURATED_BROADER', addAliases: false });
98 + const r2 = fakeResolver({ 'Neoplasm Metastasis': { cancerId: 'CI-CAN-ROOT', matchType: 'CURATED_BROADER', confidence: 0.78, via: 'base' }, 'Metastatic Neoplasm': { cancerId: 'CI-CAN-2', matchType: 'ONTOLOGY_EXACT', confidence: 0.98, via: 'alias' } });
99 + const m2 = mapDescriptor(r2, Descriptor.parse({ ui: 'D009362', label: 'Neoplasm Metastasis', terms: ['Neoplasm Metastasis', 'Metastatic Neoplasm'], ...base }));
100 + expect(m2).toMatchObject({ cancerId: 'CI-CAN-2', matchType: 'ALIAS', confidence: 0.9, addAliases: true });
101 + });
102 + it('entry-term hit is recorded as ALIAS; low-confidence hits are ignored; miss → null', () => {
103 + const r = fakeResolver({ 'Chronic Myeloid Leukemia': { cancerId: 'CI-CAN-3', matchType: 'ALIAS', confidence: 0.9, via: 'alias' } });
104 + const m = mapDescriptor(r, Descriptor.parse({ ui: 'D015464', label: 'Leukemia, Myelogenous, Chronic, BCR-ABL Positive', terms: ['Leukemia, Myelogenous, Chronic, BCR-ABL Positive', 'Chronic Myeloid Leukemia'], ...base }));
105 + expect(m).toMatchObject({ cancerId: 'CI-CAN-3', matchType: 'ALIAS', addAliases: true });
106 + const weak = fakeResolver({ 'Foo Neoplasms': { cancerId: 'CI-CAN-9', matchType: 'ALIAS', confidence: 0.5, via: 'weak' } });
107 + expect(mapDescriptor(weak, Descriptor.parse({ ui: 'D000001', label: 'Foo Neoplasms', terms: ['Foo Neoplasms'], ...base }))).toBeNull();
108 + expect(mapDescriptor(fakeResolver({}), Descriptor.parse({ ui: 'D000002', label: 'Bar Neoplasms', terms: ['Bar Neoplasms'], ...base }))).toBeNull();
109 + });
110 +});
111 +
112 +describe('XML fallback', () => {
113 + it('splits a streamed file into DescriptorRecords and keeps only the neoplasm subtrees', async () => {
114 + const xml = fx('desc-records.xml');
115 + // Feed the file in small chunks to exercise boundary handling.
116 + const chunks = (async function* () {
117 + for (let i = 0; i < xml.length; i += 700) yield xml.slice(i, i + 700);
118 + })();
119 + const records: string[] = [];
120 + for await (const r of descriptorRecords(chunks)) records.push(r);
121 + expect(records).toHaveLength(3);
122 + const parsed = records.map((r) => parseDescriptorRecord(r));
123 + expect(parsed[1]).toBeNull(); // Calcimycin (D03) is outside the fetched subtrees
124 + const breast = parsed[0]!;
125 + expect(breast).toMatchObject({ ui: 'D001943', label: 'Breast Neoplasms', treeNumbers: ['C04.588.180', 'C17.800.090.500'], scopeNote: 'Tumors or cancer of the human BREAST.', dateIntroduced: '1966-01-01' });
126 + expect(breast.terms).toEqual(['Breast Neoplasms', 'Neoplasms, Breast', 'Breast Tumors']);
127 + expect(breast.relatedTerms).toEqual(['Breast Cancer', 'Cancer, Breast']);
128 + const mds = parsed[2]!;
129 + expect(mds.ui).toBe('D009190');
130 + expect(inNeoplasmsTree(mds)).toBe(false);
131 + expect(mds.dateIntroduced).toBeNull();
132 + });
133 + it('rejects a chunk that is not a DescriptorRecord', () => {
134 + expect(() => parseDescriptorRecord('<Foo/>')).toThrow(/not a DescriptorRecord/);
135 + });
136 +});
added packages/connectors/src/connectors/mesh/sparql.ts +167 −0
@@ -0,0 +1,167 @@
1 +import { z } from 'zod';
2 +import { normalizeLabel, uninvertMeshTerm } from '@cancerindex/shared';
3 +import type { CancerMatch } from '@cancerindex/ontology';
4 +
5 +export const MESH_GRAPH = 'http://id.nlm.nih.gov/mesh';
6 +export const PAGE_SIZE = 1000;
7 +
8 +/**
9 + * Tree-number prefixes fetched. C04 = Neoplasms (the whole tree). Outside C04, MeSH files several
10 + * malignant entities under hematologic / immunoproliferative disorders: C15.378.190.6* (Bone Marrow
11 + * Diseases → Myelodysplastic Syndromes .625, Myeloproliferative Disorders .636, MDS/MPN .615) and
12 + * C20.683 (Immunoproliferative Disorders: lymphoproliferative disorders, paraproteinemias). Those
13 + * subtrees also contain non-neoplastic descriptors (Infectious Mononucleosis, Hypergammaglobulinemia…):
14 + * they are mapped when the resolver knows them and otherwise skipped silently (only C04 misses go to
15 + * the curation queue, CLAUDE.md §222).
16 + */
17 +export const TREE_PREFIXES = ['C04', 'C15.378.190.6', 'C20.683'] as const;
18 +
19 +const PREFIXES = `PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX meshv: <http://id.nlm.nih.gov/mesh/vocab#> `;
20 +
21 +function treeFilter(varName: string): string {
22 + return TREE_PREFIXES.map((p) => `STRSTARTS(STR(${varName}), '${p}')`).join(' || ');
23 +}
24 +
25 +/** Paged descriptor query: one row per descriptor with '|'-joined tree numbers and entry terms. */
26 +export function descriptorsQuery(offset: number, limit = PAGE_SIZE): string {
27 + return (
28 + PREFIXES +
29 + `SELECT ?d ?label ?scope ?introduced (GROUP_CONCAT(DISTINCT ?tnl; separator='|') AS ?trees) (GROUP_CONCAT(DISTINCT ?t; separator='|') AS ?terms) (GROUP_CONCAT(DISTINCT ?o; separator='|') AS ?related) ` +
30 + `FROM <${MESH_GRAPH}> WHERE { ` +
31 + `{ SELECT DISTINCT ?d WHERE { ?d a meshv:TopicalDescriptor ; meshv:treeNumber ?tn0 . ?tn0 rdfs:label ?tnl0 . FILTER(${treeFilter('?tnl0')}) } ORDER BY ?d LIMIT ${limit} OFFSET ${offset} } ` +
32 + `?d rdfs:label ?label ; meshv:treeNumber ?tn . ?tn rdfs:label ?tnl . ` +
33 + // Synonyms proper = terms of the preferred concept. Terms of the other (narrower/related) concepts
34 + // are fetched separately: "Adenocarcinoma, Basal Cell" is an entry term of Adenocarcinoma but names
35 + // a narrower disease — it must never map or alias the descriptor.
36 + `?d meshv:preferredConcept ?pc . ?pc meshv:term|meshv:preferredTerm ?pt . ?pt meshv:prefLabel|meshv:altLabel ?t . ` +
37 + `OPTIONAL { ?d meshv:concept ?oc . FILTER(?oc != ?pc) ?oc meshv:term|meshv:preferredTerm ?ot . ?ot meshv:prefLabel|meshv:altLabel ?o } ` +
38 + `OPTIONAL { ?pc meshv:scopeNote ?scope } OPTIONAL { ?d meshv:dateIntroduced ?introduced } } ` +
39 + `GROUP BY ?d ?label ?scope ?introduced ORDER BY ?d`
40 + );
41 +}
42 +
43 +/** Cheap liveness probe: the root descriptor "Neoplasms" must exist in the current graph. */
44 +export const HEALTH_QUERY = PREFIXES + `ASK FROM <${MESH_GRAPH}> { <${MESH_GRAPH}/D009369> a meshv:TopicalDescriptor }`;
45 +
46 +/** Is the year graph present (used to label the dataset version, e.g. "MeSH 2026")? */
47 +export function yearGraphQuery(year: number): string {
48 + return PREFIXES + `ASK { GRAPH <${MESH_GRAPH}/${year}> { <${MESH_GRAPH}/${year}/D009369> a meshv:TopicalDescriptor } }`;
49 +}
50 +
51 +export function sparqlUrl(endpoint: string, query: string): string {
52 + const u = new URL(endpoint);
53 + u.searchParams.set('query', query);
54 + u.searchParams.set('format', 'JSON');
55 + u.searchParams.set('inference', 'true');
56 + return u.toString();
57 +}
58 +
59 +const Binding = z.object({ type: z.string(), value: z.string() });
60 +const SparqlJson = z.object({ head: z.object({ vars: z.array(z.string()).optional() }).passthrough(), results: z.object({ bindings: z.array(z.record(z.string(), Binding)) }) });
61 +export const SparqlAsk = z.object({ boolean: z.boolean() });
62 +
63 +export const MESH_UI_RE = /^[CD]\d{6,9}$/;
64 +
65 +export const Descriptor = z.object({
66 + ui: z.string().regex(MESH_UI_RE),
67 + label: z.string().min(1),
68 + treeNumbers: z.array(z.string().regex(/^[A-Z]\d{2}(\.\d{3})*$/)).min(1),
69 + /** Entry terms of the preferred concept (true synonyms of the heading). */
70 + terms: z.array(z.string().min(1)).min(1),
71 + /** Entry terms of the descriptor's other concepts (narrower / related diseases) — raw record only. */
72 + relatedTerms: z.array(z.string().min(1)).default([]),
73 + scopeNote: z.string().nullable(),
74 + dateIntroduced: z.string().nullable(),
75 +});
76 +export type Descriptor = z.infer<typeof Descriptor>;
77 +
78 +/** Parse one SPARQL JSON page into descriptors; malformed rows are returned separately (never dropped silently). */
79 +export function parseDescriptorPage(json: unknown): { descriptors: Descriptor[]; malformed: Array<{ row: unknown; error: string }> } {
80 + const page = SparqlJson.safeParse(json);
81 + if (!page.success) throw new Error(`malformed SPARQL response: ${page.error.issues[0]?.message ?? 'unexpected shape'}`);
82 + const descriptors: Descriptor[] = [];
83 + const malformed: Array<{ row: unknown; error: string }> = [];
84 + for (const b of page.data.results.bindings) {
85 + const candidate = {
86 + ui: b.d?.value.split('/').pop() ?? '',
87 + label: b.label?.value ?? '',
88 + treeNumbers: splitJoined(b.trees?.value),
89 + terms: splitJoined(b.terms?.value),
90 + relatedTerms: splitJoined(b.related?.value).filter((t) => !splitJoined(b.terms?.value).includes(t)),
91 + scopeNote: b.scope?.value ?? null,
92 + dateIntroduced: b.introduced?.value ?? null,
93 + };
94 + const parsed = Descriptor.safeParse(candidate);
95 + if (parsed.success) descriptors.push(parsed.data);
96 + else malformed.push({ row: b, error: `${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}` });
97 + }
98 + return { descriptors, malformed };
99 +}
100 +
101 +function splitJoined(v: string | undefined): string[] {
102 + return [...new Set((v ?? '').split('|').map((s) => s.trim()).filter(Boolean))];
103 +}
104 +
105 +export function inNeoplasmsTree(d: Pick<Descriptor, 'treeNumbers'>): boolean {
106 + return d.treeNumbers.some((t) => t.startsWith('C04'));
107 +}
108 +
109 +/* ------------------------------------------------------------------------------------------------ */
110 +
111 +export type MeshMatchType = 'ONTOLOGY_EXACT' | 'CURATED_EXACT' | 'ALIAS' | 'CURATED_BROADER';
112 +
113 +export interface MeshMapping {
114 + cancerId: string;
115 + matchType: MeshMatchType;
116 + confidence: number;
117 + via: string;
118 + /** Aliases (heading + entry terms) may only be attached to exact-grade matches. */
119 + addAliases: boolean;
120 +}
121 +
122 +export interface LabelResolver {
123 + byLabel(label: string, opts?: { allowMeshInversion?: boolean }): CancerMatch | null;
124 +}
125 +
126 +const EXACT_GRADES = new Set(['ONTOLOGY_EXACT', 'CURATED_EXACT', 'ALIAS']);
127 +
128 +/**
129 + * Map one descriptor onto a canonical cancer (CLAUDE.md §69, §221):
130 + * 1. heading (with MeSH inversion "Carcinoma, Non-Small-Cell Lung" → "Non-Small-Cell Lung Carcinoma");
131 + * exact grades (ONTOLOGY_EXACT / CURATED_EXACT / ALIAS, confidence ≥ 0.9) → code + aliases;
132 + * CURATED_BROADER (qualified state / lineage) → code only, no aliases;
133 + * 2. entry terms, exact grades only → recorded as ALIAS (the heading itself is not a known name);
134 + * 3. null → unresolved queue (caller).
135 + */
136 +export function mapDescriptor(resolver: LabelResolver, d: Descriptor): MeshMapping | null {
137 + const heading = resolver.byLabel(d.label, { allowMeshInversion: true });
138 + if (heading && EXACT_GRADES.has(heading.matchType) && heading.confidence >= 0.9) return { cancerId: heading.cancerId, matchType: heading.matchType as MeshMatchType, confidence: heading.confidence, via: `heading: ${heading.via}`, addAliases: true };
139 + let broader: MeshMapping | null = null;
140 + if (heading && heading.matchType === 'CURATED_BROADER') broader = { cancerId: heading.cancerId, matchType: 'CURATED_BROADER', confidence: heading.confidence, via: `heading (broader): ${heading.via}`, addAliases: false };
141 + for (const term of d.terms) {
142 + if (term === d.label) continue;
143 + const hit = resolver.byLabel(term, { allowMeshInversion: true });
144 + if (hit && EXACT_GRADES.has(hit.matchType) && hit.confidence >= 0.9) return { cancerId: hit.cancerId, matchType: 'ALIAS', confidence: Math.min(hit.confidence, 0.9), via: `entry term "${term}": ${hit.via}`, addAliases: true };
145 + }
146 + return broader;
147 +}
148 +
149 +/**
150 + * Entry terms worth storing as aliases: ≥ 3 chars, not the heading, and not a mere permutation of a
151 + * term already kept (MeSH lists "Neoplasm, Breast", "Neoplasms, Breast", "Breast Neoplasm" for
152 + * "Breast Neoplasms" — they normalize to the same string once un-inverted).
153 + */
154 +export function aliasWorthyTerms(d: Descriptor): string[] {
155 + const seen = new Set<string>([normalizeLabel(d.label)]);
156 + const out: string[] = [];
157 + for (const term of d.terms) {
158 + if (term.length < 3 || term === d.label) continue;
159 + const norm = normalizeLabel(term);
160 + const uninverted = normalizeLabel(uninvertMeshTerm(term));
161 + if (!norm || seen.has(norm) || seen.has(uninverted)) continue;
162 + seen.add(norm);
163 + seen.add(uninverted);
164 + out.push(term);
165 + }
166 + return out;
167 +}
added packages/connectors/src/connectors/mesh/xml.ts +61 −0
@@ -0,0 +1,61 @@
1 +import { XMLParser } from 'fast-xml-parser';
2 +import { Descriptor, TREE_PREFIXES } from './sparql.js';
3 +
4 +/**
5 + * Fallback path when the SPARQL endpoint is unavailable: the annual descriptor XML
6 + * (desc<year>.xml, ~313 MB). The file is never loaded whole — the response body is consumed as a
7 + * stream and split on `</DescriptorRecord>` boundaries; each record (a few KB) is parsed on its own.
8 + */
9 +export const DESC_XML_URL = (year: number) => `https://nlmpubs.nlm.nih.gov/projects/mesh/MESH_FILES/xmlmesh/desc${year}.xml`;
10 +
11 +const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_', isArray: (name) => ['TreeNumber', 'Concept', 'Term', 'ConceptList', 'TermList'].includes(name), trimValues: true, parseTagValue: false });
12 +
13 +const RECORD_END = '</DescriptorRecord>';
14 +
15 +/** Split a text stream into complete `<DescriptorRecord>…</DescriptorRecord>` chunks. */
16 +export async function* descriptorRecords(chunks: AsyncIterable<string | Uint8Array>): AsyncGenerator<string> {
17 + const decoder = new TextDecoder();
18 + let buf = '';
19 + for await (const c of chunks) {
20 + buf += typeof c === 'string' ? c : decoder.decode(c, { stream: true });
21 + let end = buf.indexOf(RECORD_END);
22 + while (end !== -1) {
23 + // "<DescriptorRecord " / "<DescriptorRecord>" only — never the "<DescriptorRecordSet" root.
24 + const start = buf.search(/<DescriptorRecord[\s>]/);
25 + if (start !== -1 && start < end) yield buf.slice(start, end + RECORD_END.length);
26 + buf = buf.slice(end + RECORD_END.length);
27 + end = buf.indexOf(RECORD_END);
28 + }
29 + // Keep memory bounded when a record is pathological
30 + if (buf.length > 50_000_000) throw new Error('descriptor record larger than 50 MB — aborting XML fallback');
31 + }
32 +}
33 +
34 +type Rec = Record<string, unknown>;
35 +const text = (v: unknown): string | null => (v == null ? null : typeof v === 'object' ? text((v as Rec)['#text'] ?? (v as Rec)['String']) : String(v));
36 +
37 +/** Parse one DescriptorRecord; returns null when the record is outside the fetched subtrees. */
38 +export function parseDescriptorRecord(xml: string): Descriptor | null {
39 + const doc = parser.parse(xml) as Rec;
40 + const rec = doc['DescriptorRecord'] as Rec | undefined;
41 + if (!rec) throw new Error('not a DescriptorRecord');
42 + const ui = text(rec['DescriptorUI']) ?? '';
43 + const label = text((rec['DescriptorName'] as Rec | undefined)?.['String']) ?? '';
44 + const treeNumbers = (((rec['TreeNumberList'] as Rec | undefined)?.['TreeNumber'] as unknown[] | undefined) ?? []).map((t) => text(t) ?? '').filter(Boolean);
45 + if (!treeNumbers.some((t) => TREE_PREFIXES.some((p) => t.startsWith(p)))) return null;
46 + const terms: string[] = [];
47 + const relatedTerms: string[] = [];
48 + let scopeNote: string | null = null;
49 + const concepts = (((rec['ConceptList'] as unknown[] | undefined)?.[0] as Rec | undefined)?.['Concept'] as Rec[] | undefined) ?? [];
50 + for (const c of concepts) {
51 + const preferred = c['@_PreferredConceptYN'] === 'Y';
52 + if (preferred && scopeNote === null) scopeNote = text(c['ScopeNote']);
53 + const termList = ((c['TermList'] as unknown[] | undefined)?.[0] as Rec | undefined)?.['Term'] as Rec[] | undefined;
54 + for (const t of termList ?? []) {
55 + const s = text(t['String']);
56 + if (s) (preferred ? terms : relatedTerms).push(s);
57 + }
58 + }
59 + const dateIntroduced = rec['DateEstablished'] ? `${text((rec['DateEstablished'] as Rec)['Year'])}-${text((rec['DateEstablished'] as Rec)['Month'])}-${text((rec['DateEstablished'] as Rec)['Day'])}` : null;
60 + return Descriptor.parse({ ui, label, treeNumbers, terms: [...new Set(terms)], relatedTerms: [...new Set(relatedTerms)].filter((t) => !terms.includes(t)), scopeNote, dateIntroduced });
61 +}
modified packages/connectors/src/connectors/ncit-evs/index.ts +1 −0
@@ -132,6 +132,7 @@ export class NcitEvsConnector extends Connector {
132 132 if (prevKept > 0 && kept.size < prevKept * manifest.anomalyGuard.minRatioOfPrevious) {
133 133 throw new Error(`anomaly: kept ${kept.size} concepts < ${manifest.anomalyGuard.minRatioOfPrevious} × previous ${prevKept} — refusing to persist (CLAUDE.md §171)`);
134 134 }
135 + await ctx.guardCount('concept+qualified_state+retired', kept.size + qualified.length + retired.length); // = records this run will upsert, vs previous run's records_fetched
135 136
136 137 await ctx.addProvenance({ dataset: 'NCI Thesaurus', datasetVersion: version, sourceUrl: FLAT_URL, evidenceType: 'expert_curation', accessLevel: 'open', methodology: 'NCIt FLAT release: stated parent-child relations, synonyms and definitions as edited by NCI EVS' });
137 138
modified packages/connectors/src/connectors/oncotree/index.ts +1 −0
@@ -70,6 +70,7 @@ export class OncotreeConnector extends Connector {
70 70 }
71 71 ctx.info(`fetched ${nodes.length} tumor types`);
72 72 if (nodes.length < 500) throw new Error(`anomaly: only ${nodes.length} OncoTree nodes returned (expected ~900) — refusing to persist (CLAUDE.md §171)`);
73 + await ctx.guardCount('tumor_type', nodes.length); // vs previous successful run (CLAUDE.md §171)
73 74 if (ctx.mode === 'dry_run') {
74 75 ctx.counters.fetched = nodes.length;
75 76 return;
added packages/connectors/src/connectors/openfda/fixtures/drugsfda-imatinib.json +758 −0
@@ -0,0 +1,758 @@
1 +{
2 + "meta": {
3 + "disclaimer": "Do not rely on openFDA to make decisions regarding medical care. While we make every effort to ensure that data is accurate, you should assume all results are unvalidated. We may limit or otherwise restrict your access to the API in line with our Terms of Service.",
4 + "terms": "https://open.fda.gov/terms/",
5 + "license": "https://open.fda.gov/license/",
6 + "last_updated": "2026-09-04",
7 + "results": {
8 + "skip": 0,
9 + "limit": 99,
10 + "total": 12
11 + }
12 + },
13 + "results": [
14 + {
15 + "application_number": "ANDA078340",
16 + "sponsor_name": "SUN PHARM",
17 + "openfda": {
18 + "application_number": [
19 + "ANDA078340"
20 + ],
21 + "brand_name": [
22 + "IMATINIB MESYLATE"
23 + ],
24 + "generic_name": [
25 + "IMATINIB MESYLATE"
26 + ],
27 + "manufacturer_name": [
28 + "BluePoint Laboratories",
29 + "Sun Pharmaceutical Industries, Inc."
30 + ],
31 + "substance_name": [
32 + "IMATINIB MESYLATE"
33 + ],
34 + "unii": [
35 + "8A1O1M485B"
36 + ],
37 + "product_type": [
38 + "HUMAN PRESCRIPTION DRUG"
39 + ],
40 + "route": [
41 + "ORAL"
42 + ]
43 + },
44 + "products": [
45 + {
46 + "product_number": "001",
47 + "reference_drug": "No",
48 + "brand_name": "IMATINIB MESYLATE",
49 + "active_ingredients": [
50 + {
51 + "name": "IMATINIB MESYLATE",
52 + "strength": "EQ 100MG BASE"
53 + }
54 + ],
55 + "dosage_form": "TABLET",
56 + "route": "ORAL",
57 + "marketing_status": "Discontinued"
58 + },
59 + {
60 + "product_number": "002",
61 + "reference_drug": "No",
62 + "brand_name": "IMATINIB MESYLATE",
63 + "active_ingredients": [
64 + {
65 + "name": "IMATINIB MESYLATE",
66 + "strength": "EQ 400MG BASE"
67 + }
68 + ],
69 + "dosage_form": "TABLET",
70 + "route": "ORAL",
71 + "marketing_status": "Discontinued"
72 + }
73 + ],
74 + "submissions": [
75 + {
76 + "submission_type": "ORIG",
77 + "submission_number": "1",
78 + "submission_status": "AP",
79 + "submission_status_date": "20151203",
80 + "application_docs": [
81 + {
82 + "id": "31357",
83 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2015/078340Orig1s000ltr.pdf",
84 + "date": "20151208",
85 + "type": "Letter"
86 + }
87 + ]
88 + }
89 + ]
90 + },
91 + {
92 + "application_number": "NDA021588",
93 + "sponsor_name": "NOVARTIS",
94 + "openfda": {
95 + "application_number": [
96 + "NDA021588"
97 + ],
98 + "brand_name": [
99 + "GLEEVEC"
100 + ],
101 + "generic_name": [
102 + "IMATINIB MESYLATE"
103 + ],
104 + "manufacturer_name": [
105 + "Novartis Pharmaceuticals Corporation"
106 + ],
107 + "substance_name": [
108 + "IMATINIB MESYLATE"
109 + ],
110 + "unii": [
111 + "8A1O1M485B"
112 + ],
113 + "product_type": [
114 + "HUMAN PRESCRIPTION DRUG"
115 + ],
116 + "route": [
117 + "ORAL"
118 + ]
119 + },
120 + "products": [
121 + {
122 + "product_number": "001",
123 + "reference_drug": "Yes",
124 + "brand_name": "GLEEVEC",
125 + "active_ingredients": [
126 + {
127 + "name": "IMATINIB MESYLATE",
128 + "strength": "EQ 100MG BASE"
129 + }
130 + ],
131 + "dosage_form": "TABLET",
132 + "route": "ORAL",
133 + "marketing_status": "Prescription"
134 + },
135 + {
136 + "product_number": "002",
137 + "reference_drug": "Yes",
138 + "brand_name": "GLEEVEC",
139 + "active_ingredients": [
140 + {
141 + "name": "IMATINIB MESYLATE",
142 + "strength": "EQ 400MG BASE"
143 + }
144 + ],
145 + "dosage_form": "TABLET",
146 + "route": "ORAL",
147 + "marketing_status": "Prescription"
148 + }
149 + ],
150 + "submissions": [
151 + {
152 + "submission_type": "SUPPL",
153 + "submission_number": "45",
154 + "submission_status": "AP",
155 + "submission_status_date": "20160825",
156 + "review_priority": "STANDARD",
157 + "submission_class_code": "EFFICACY",
158 + "submission_class_code_description": "Efficacy",
159 + "submission_property_type": [
160 + {
161 + "code": "Orphan"
162 + }
163 + ],
164 + "application_docs": [
165 + {
166 + "id": "45063",
167 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2016/021588Orig1s045,s046ltr.pdf",
168 + "date": "20160825",
169 + "type": "Letter"
170 + },
171 + {
172 + "id": "45169",
173 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2016/021588s045s046lbl.pdf",
174 + "date": "20160825",
175 + "type": "Label"
176 + }
177 + ]
178 + },
179 + {
180 + "submission_type": "ORIG",
181 + "submission_number": "1",
182 + "submission_status": "AP",
183 + "submission_status_date": "20030418",
184 + "review_priority": "PRIORITY",
185 + "submission_class_code": "TYPE 3",
186 + "submission_class_code_description": "Type 3 - New Dosage Form",
187 + "submission_property_type": [
188 + {
189 + "code": "Orphan"
190 + }
191 + ],
192 + "application_docs": [
193 + {
194 + "id": "42219",
195 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2003/021588s000_GleevecTOC.html",
196 + "date": "20060607",
197 + "type": "Review"
198 + },
199 + {
200 + "id": "44136",
201 + "url": "http://www.fda.gov/Drugs/DrugSafety/PostmarketDrugSafetyInformationforPatientsandProviders/ucm110502.htm",
202 + "date": "20031209",
203 + "type": "Other Important Information from FDA"
204 + }
205 + ]
206 + },
207 + {
208 + "submission_type": "SUPPL",
209 + "submission_number": "37",
210 + "submission_status": "AP",
211 + "submission_status_date": "20130125",
212 + "review_priority": "PRIORITY",
213 + "submission_class_code": "EFFICACY",
214 + "submission_class_code_description": "Efficacy",
215 + "application_docs": [
216 + {
217 + "id": "37290",
218 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2013/021588s037lbl.pdf",
219 + "date": "20130130",
220 + "type": "Label"
221 + },
222 + {
223 + "id": "5801",
224 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2013/021588Orig1s037ltr.pdf",
225 + "date": "20130130",
226 + "type": "Letter"
227 + }
228 + ]
229 + },
230 + {
231 + "submission_type": "SUPPL",
232 + "submission_number": "35",
233 + "submission_status": "AP",
234 + "submission_status_date": "20120131",
235 + "review_priority": "PRIORITY",
236 + "submission_class_code": "EFFICACY",
237 + "submission_class_code_description": "Efficacy",
238 + "application_docs": [
239 + {
240 + "id": "5800",
241 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2012/021588s035ltr_cor.pdf",
242 + "date": "20120203",
243 + "type": "Letter"
244 + },
245 + {
246 + "id": "15837",
247 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2012/021588s035lbl.pdf",
248 + "date": "20120131",
249 + "type": "Label"
250 + }
251 + ]
252 + },
253 + {
254 + "submission_type": "SUPPL",
255 + "submission_number": "8",
256 + "submission_status": "AP",
257 + "submission_status_date": "20051020",
258 + "review_priority": "UNKNOWN",
259 + "submission_class_code": "EFFICACY",
260 + "submission_class_code_description": "Efficacy",
261 + "application_docs": [
262 + {
263 + "id": "5789",
264 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2005/021588s008ltr.pdf",
265 + "date": "20051025",
266 + "type": "Letter"
267 + },
268 + {
269 + "id": "15831",
270 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2005/021588s008lbl.pdf",
271 + "date": "20051025",
272 + "type": "Label"
273 + }
274 + ]
275 + },
276 + {
277 + "submission_type": "SUPPL",
278 + "submission_number": "25",
279 + "submission_status": "AP",
280 + "submission_status_date": "20081219",
281 + "review_priority": "PRIORITY",
282 + "submission_class_code": "EFFICACY",
283 + "submission_class_code_description": "Efficacy",
284 + "application_docs": [
285 + {
286 + "id": "5797",
287 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2008/021588s025ltr.pdf",
288 + "date": "20081229",
289 + "type": "Letter"
290 + }
291 + ]
292 + },
293 + {
294 + "submission_type": "SUPPL",
295 + "submission_number": "17",
296 + "submission_status": "AP",
297 + "submission_status_date": "20061019",
298 + "review_priority": "UNKNOWN",
299 + "submission_class_code": "EFFICACY",
300 + "submission_class_code_description": "Efficacy",
301 + "application_docs": [
302 + {
303 + "id": "21749",
304 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2006/021588_s017_gleevec.pdf",
305 + "date": "20061019",
306 + "type": "Review"
307 + },
308 + {
309 + "id": "28191",
310 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2006/021588s011, s012, s013, s014, s017ltr.pdf",
311 + "date": "20061023",
312 + "type": "Letter"
313 + }
314 + ]
315 + },
316 + {
317 + "submission_type": "SUPPL",
318 + "submission_number": "2",
319 + "submission_status": "AP",
320 + "submission_status_date": "20031208",
321 + "review_priority": "STANDARD",
322 + "submission_class_code": "EFFICACY",
323 + "submission_class_code_description": "Efficacy",
324 + "application_docs": [
325 + {
326 + "id": "42221",
327 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2003/21-588s002_Gleevec.html",
328 + "date": "20050408",
329 + "type": "Review"
330 + },
331 + {
332 + "id": "5787",
333 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2003/21588se7-002ltr.pdf",
334 + "date": "20031222",
335 + "type": "Letter"
336 + }
337 + ]
338 + },
339 + {
340 + "submission_type": "SUPPL",
341 + "submission_number": "13",
342 + "submission_status": "AP",
343 + "submission_status_date": "20061019",
344 + "review_priority": "STANDARD",
345 + "submission_class_code": "EFFICACY",
346 + "submission_class_code_description": "Efficacy",
347 + "application_docs": [
348 + {
349 + "id": "42223",
350 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2006/021588_s013_gleevic.pdf",
351 + "date": "20061019",
352 + "type": "Review"
353 + },
354 + {
355 + "id": "5793",
356 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2006/021588s011, s012, s013, s014, s017ltr.pdf",
357 + "date": "20061023",
358 + "type": "Letter"
359 + }
360 + ]
361 + },
362 + {
363 + "submission_type": "SUPPL",
364 + "submission_number": "11",
365 + "submission_status": "AP",
366 + "submission_status_date": "20061019",
367 + "review_priority": "STANDARD",
368 + "submission_class_code": "EFFICACY",
369 + "submission_class_code_description": "Efficacy",
370 + "application_docs": [
371 + {
372 + "id": "5792",
373 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2006/021588s011, s012, s013, s014, s017ltr.pdf",
374 + "date": "20061023",
375 + "type": "Letter"
376 + },
377 + {
378 + "id": "15832",
379 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2006/021588s011s012s013s014s017lbl.pdf",
380 + "date": "20061023",
381 + "type": "Label"
382 + }
383 + ]
384 + },
385 + {
386 + "submission_type": "SUPPL",
387 + "submission_number": "42",
388 + "submission_status": "AP",
389 + "submission_status_date": "20150130",
390 + "review_priority": "STANDARD",
391 + "submission_class_code": "EFFICACY",
392 + "submission_class_code_description": "Efficacy",
393 + "submission_property_type": [
394 + {
395 + "code": "Orphan"
396 + }
397 + ],
398 + "application_docs": [
399 + {
400 + "id": "15838",
401 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2015/021588s042lbl.pdf",
402 + "date": "20150203",
403 + "type": "Label"
404 + },
405 + {
406 + "id": "28198",
407 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2015/021588Orig1s042ltr.pdf",
408 + "date": "20150202",
409 + "type": "Letter"
410 + }
411 + ]
412 + },
413 + {
414 + "submission_type": "SUPPL",
415 + "submission_number": "26",
416 + "submission_status": "AP",
417 + "submission_status_date": "20090527",
418 + "review_priority": "STANDARD",
419 + "submission_class_code": "EFFICACY",
420 + "submission_class_code_description": "Efficacy",
421 + "application_docs": [
422 + {
423 + "id": "28195",
424 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2009/021588s026,021588s028ltr.pdf",
425 + "date": "20090602",
426 + "type": "Letter"
427 + }
428 + ]
429 + },
430 + {
431 + "submission_type": "SUPPL",
432 + "submission_number": "5",
433 + "submission_status": "AP",
434 + "submission_status_date": "20050314",
435 + "review_priority": "UNKNOWN",
436 + "submission_class_code": "EFFICACY",
437 + "submission_class_code_description": "Efficacy",
438 + "application_docs": [
439 + {
440 + "id": "15830",
441 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2005/21588s005lbl.pdf",
442 + "date": "20050318",
443 + "type": "Label"
444 + },
445 + {
446 + "id": "21747",
447 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2005/021588_s005_GLEVAC.pdf",
448 + "date": "20150526",
449 + "type": "Review"
450 + }
451 + ]
452 + },
453 + {
454 + "submission_type": "SUPPL",
455 + "submission_number": "24",
456 + "submission_status": "AP",
457 + "submission_status_date": "20080926",
458 + "review_priority": "STANDARD",
459 + "submission_class_code": "EFFICACY",
460 + "submission_class_code_description": "Efficacy",
461 + "application_docs": [
462 + {
463 + "id": "28194",
464 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2008/021588s024ltr.pdf",
465 + "date": "20081001",
466 + "type": "Letter"
467 + },
468 + {
469 + "id": "37286",
470 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2008/021588s024lbl.pdf",
471 + "date": "20080926",
472 + "type": "Label"
473 + }
474 + ]
475 + },
476 + {
477 + "submission_type": "SUPPL",
478 + "submission_number": "16",
479 + "submission_status": "AP",
480 + "submission_status_date": "20060927",
481 + "review_priority": "PRIORITY",
482 + "submission_class_code": "EFFICACY",
483 + "submission_class_code_description": "Efficacy",
484 + "application_docs": [
485 + {
486 + "id": "51975",
487 + "url": "https://www.accessdata.fda.gov/drugsatfda_docs/pediatric/021588s016_imatinib_Gleevec-Med&StatsRev.pdf",
488 + "date": "19000101",
489 + "type": "Pediatric Medical Review"
490 + },
491 + {
492 + "id": "51976",
493 + "url": "https://www.accessdata.fda.gov/drugsatfda_docs/pediatric/021588s016_imatinib_Orig_WR.pdf",
494 + "date": "19000101",
495 + "type": "Pediatric Written Request"
496 + }
497 + ]
498 + },
499 + {
500 + "submission_type": "SUPPL",
501 + "submission_number": "30",
502 + "submission_status": "AP",
503 + "submission_status_date": "20110401",
504 + "review_priority": "STANDARD",
505 + "submission_class_code": "EFFICACY",
506 + "submission_class_code_description": "Efficacy",
507 + "application_docs": [
508 + {
509 + "id": "28197",
510 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2011/021588s030,s031ltr.pdf",
511 + "date": "20110406",
512 + "type": "Letter"
513 + },
514 + {
515 + "id": "37288",
516 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2011/021588s030s031lbl.pdf",
517 + "date": "20110401",
518 + "type": "Label"
519 + }
520 + ]
521 + },
522 + {
523 + "submission_type": "SUPPL",
524 + "submission_number": "1",
525 + "submission_status": "AP",
526 + "submission_status_date": "20030520",
527 + "review_priority": "PRIORITY",
528 + "submission_class_code": "EFFICACY",
529 + "submission_class_code_description": "Efficacy",
530 + "application_docs": [
531 + {
532 + "id": "42220",
533 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2003/21-335s003_21-588s001_Gleevec.html",
534 + "date": "20040809",
535 + "type": "Review"
536 + },
537 + {
538 + "id": "5786",
539 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2003/21588se5-001ltr.pdf",
540 + "date": "20030608",
541 + "type": "Letter"
542 + }
543 + ]
544 + },
545 + {
546 + "submission_type": "SUPPL",
547 + "submission_number": "14",
548 + "submission_status": "AP",
549 + "submission_status_date": "20061019",
550 + "review_priority": "UNKNOWN",
551 + "submission_class_code": "EFFICACY",
552 + "submission_class_code_description": "Efficacy",
553 + "application_docs": [
554 + {
555 + "id": "42224",
556 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2006/021588_s014_gleevec.pdf",
557 + "date": "20061019",
558 + "type": "Review"
559 + },
560 + {
561 + "id": "5794",
562 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2006/021588s011, s012, s013, s014, s017ltr.pdf",
563 + "date": "20061023",
564 + "type": "Letter"
565 + }
566 + ]
567 + },
568 + {
569 + "submission_type": "SUPPL",
570 + "submission_number": "20",
571 + "submission_status": "AP",
572 + "submission_status_date": "20070913",
573 + "review_priority": "UNKNOWN",
574 + "submission_class_code": "EFFICACY",
575 + "submission_class_code_description": "Efficacy",
576 + "application_docs": [
577 + {
578 + "id": "28192",
579 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2007/021588s020ltr.pdf",
580 + "date": "20070925",
581 + "type": "Letter"
582 + },
583 + {
584 + "id": "37283",
585 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2007/021588s020lbl.pdf",
586 + "date": "20070919",
587 + "type": "Label"
588 + }
589 + ]
590 + },
591 + {
592 + "submission_type": "SUPPL",
593 + "submission_number": "12",
594 + "submission_status": "AP",
595 + "submission_status_date": "20061019",
596 + "review_priority": "STANDARD",
597 + "submission_class_code": "EFFICACY",
598 + "submission_class_code_description": "Efficacy",
599 + "application_docs": [
600 + {
601 + "id": "37281",
602 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2006/021588s011s012s013s014s017lbl.pdf",
603 + "date": "20061023",
604 + "type": "Label"
605 + },
606 + {
607 + "id": "42222",
608 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2006/021588_s012_gleevic.pdf",
609 + "date": "20061019",
610 + "type": "Review"
611 + }
612 + ]
613 + }
614 + ]
615 + },
616 + {
617 + "application_number": "ANDA204285",
618 + "sponsor_name": "TEVA PHARMS USA",
619 + "openfda": {
620 + "application_number": [
621 + "ANDA204285"
622 + ],
623 + "brand_name": [
624 + "IMATINIB MESYLATE"
625 + ],
626 + "generic_name": [
627 + "IMATINIB MESYLATE"
628 + ],
629 + "manufacturer_name": [
630 + "Teva Pharmaceuticals USA, Inc."
631 + ],
632 + "substance_name": [
633 + "IMATINIB MESYLATE"
634 + ],
635 + "unii": [
636 + "8A1O1M485B"
637 + ],
638 + "product_type": [
639 + "HUMAN PRESCRIPTION DRUG"
640 + ],
641 + "route": [
642 + "ORAL"
643 + ]
644 + },
645 + "products": [
646 + {
647 + "product_number": "002",
648 + "reference_drug": "No",
649 + "brand_name": "IMATINIB MESYLATE",
650 + "active_ingredients": [
651 + {
652 + "name": "IMATINIB MESYLATE",
653 + "strength": "EQ 400MG BASE"
654 + }
655 + ],
656 + "dosage_form": "TABLET",
657 + "route": "ORAL",
658 + "marketing_status": "Prescription"
659 + },
660 + {
661 + "product_number": "001",
662 + "reference_drug": "No",
663 + "brand_name": "IMATINIB MESYLATE",
664 + "active_ingredients": [
665 + {
666 + "name": "IMATINIB MESYLATE",
667 + "strength": "EQ 100MG BASE"
668 + }
669 + ],
670 + "dosage_form": "TABLET",
671 + "route": "ORAL",
672 + "marketing_status": "Prescription"
673 + }
674 + ],
675 + "submissions": [
676 + {
677 + "submission_type": "ORIG",
678 + "submission_number": "1",
679 + "submission_status": "AP",
680 + "submission_status_date": "20160804",
681 + "submission_class_code": "UNKNOWN",
682 + "application_docs": []
683 + }
684 + ]
685 + },
686 + {
687 + "application_number": "NDA219097",
688 + "sponsor_name": "SHORLA ONCOLOGY",
689 + "openfda": {
690 + "application_number": [
691 + "NDA219097"
692 + ],
693 + "brand_name": [
694 + "IMKELDI"
695 + ],
696 + "generic_name": [
697 + "IMATINIB ORAL"
698 + ],
699 + "manufacturer_name": [
700 + "Shorla Oncology Inc.,"
701 + ],
702 + "substance_name": [
703 + "IMATINIB MESYLATE"
704 + ],
705 + "unii": [
706 + "8A1O1M485B"
707 + ],
708 + "product_type": [
709 + "HUMAN PRESCRIPTION DRUG"
710 + ],
711 + "route": [
712 + "ORAL"
713 + ]
714 + },
715 + "products": [
716 + {
717 + "product_number": "001",
718 + "reference_drug": "Yes",
719 + "brand_name": "IMKELDI",
720 + "active_ingredients": [
721 + {
722 + "name": "IMATINIB MESYLATE",
723 + "strength": "EQ 80MG BASE/ML"
724 + }
725 + ],
726 + "dosage_form": "SOLUTION",
727 + "route": "ORAL",
728 + "marketing_status": "Prescription"
729 + }
730 + ],
731 + "submissions": [
732 + {
733 + "submission_type": "ORIG",
734 + "submission_number": "1",
735 + "submission_status": "AP",
736 + "submission_status_date": "20241122",
737 + "review_priority": "STANDARD",
738 + "submission_class_code": "TYPE 3",
739 + "submission_class_code_description": "Type 3 - New Dosage Form",
740 + "application_docs": [
741 + {
742 + "id": "80358",
743 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2024/219097s000lbl.pdf",
744 + "date": "20241125",
745 + "type": "Label"
746 + },
747 + {
748 + "id": "80427",
749 + "url": "https://www.accessdata.fda.gov/drugsatfda_docs/appletter/2024/219097Orig1s000ltr.pdf",
750 + "date": "20241126",
751 + "type": "Letter"
752 + }
753 + ]
754 + }
755 + ]
756 + }
757 + ]
758 +}
\ No newline at end of file
added packages/connectors/src/connectors/openfda/fixtures/drugsfda-osimertinib.json +225 −0
@@ -0,0 +1,225 @@
1 +{
2 + "meta": {
3 + "disclaimer": "Do not rely on openFDA to make decisions regarding medical care. While we make every effort to ensure that data is accurate, you should assume all results are unvalidated. We may limit or otherwise restrict your access to the API in line with our Terms of Service.",
4 + "terms": "https://open.fda.gov/terms/",
5 + "license": "https://open.fda.gov/license/",
6 + "last_updated": "2026-09-04",
7 + "results": {
8 + "skip": 0,
9 + "limit": 1,
10 + "total": 1
11 + }
12 + },
13 + "results": [
14 + {
15 + "application_number": "NDA208065",
16 + "sponsor_name": "ASTRAZENECA",
17 + "openfda": {
18 + "application_number": [
19 + "NDA208065"
20 + ],
21 + "brand_name": [
22 + "TAGRISSO"
23 + ],
24 + "generic_name": [
25 + "OSIMERTINIB"
26 + ],
27 + "manufacturer_name": [
28 + "AstraZeneca Pharmaceuticals LP"
29 + ],
30 + "substance_name": [
31 + "OSIMERTINIB"
32 + ],
33 + "unii": [
34 + "3C06JJ0Z2O"
35 + ],
36 + "pharm_class_epc": [
37 + "Kinase Inhibitor [EPC]"
38 + ],
39 + "product_type": [
40 + "HUMAN PRESCRIPTION DRUG"
41 + ],
42 + "route": [
43 + "ORAL"
44 + ]
45 + },
46 + "products": [
47 + {
48 + "product_number": "001",
49 + "reference_drug": "Yes",
50 + "brand_name": "TAGRISSO",
51 + "active_ingredients": [
52 + {
53 + "name": "OSIMERTINIB MESYLATE",
54 + "strength": "EQ 40MG BASE"
55 + }
56 + ],
57 + "dosage_form": "TABLET",
58 + "route": "ORAL",
59 + "marketing_status": "Prescription"
60 + },
61 + {
62 + "product_number": "002",
63 + "reference_drug": "Yes",
64 + "brand_name": "TAGRISSO",
65 + "active_ingredients": [
66 + {
67 + "name": "OSIMERTINIB MESYLATE",
68 + "strength": "EQ 80MG BASE"
69 + }
70 + ],
71 + "dosage_form": "TABLET",
72 + "route": "ORAL",
73 + "marketing_status": "Prescription"
74 + }
75 + ],
76 + "submissions": [
77 + {
78 + "submission_type": "SUPPL",
79 + "submission_number": "8",
80 + "submission_status": "AP",
81 + "submission_status_date": "20180418",
82 + "review_priority": "PRIORITY",
83 + "submission_class_code": "EFFICACY",
84 + "submission_class_code_description": "Efficacy",
85 + "submission_property_type": [
86 + {
87 + "code": "Orphan"
88 + }
89 + ],
90 + "application_docs": [
91 + {
92 + "id": "53732",
93 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2018/208065s008lbl.pdf",
94 + "date": "20180419",
95 + "type": "Label"
96 + },
97 + {
98 + "id": "53772",
99 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2018/208065Orig1s008ltr.pdf",
100 + "date": "20180423",
101 + "type": "Letter"
102 + }
103 + ]
104 + },
105 + {
106 + "submission_type": "SUPPL",
107 + "submission_number": "6",
108 + "submission_status": "AP",
109 + "submission_status_date": "20170330",
110 + "review_priority": "PRIORITY",
111 + "submission_class_code": "EFFICACY",
112 + "submission_class_code_description": "Efficacy",
113 + "submission_property_type": [
114 + {
115 + "code": "Orphan"
116 + }
117 + ],
118 + "application_docs": [
119 + {
120 + "id": "47794",
121 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2017/208065s006lbl.pdf",
122 + "date": "20170330",
123 + "type": "Label"
124 + },
125 + {
126 + "id": "47821",
127 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2017/208065Orig1s006ltr.pdf",
128 + "date": "20170403",
129 + "type": "Letter"
130 + }
131 + ]
132 + },
133 + {
134 + "submission_type": "SUPPL",
135 + "submission_number": "2",
136 + "submission_status": "AP",
137 + "submission_status_date": "20160812",
138 + "review_priority": "STANDARD",
139 + "submission_class_code": "LABELING",
140 + "submission_class_code_description": "Labeling",
141 + "application_docs": [
142 + {
143 + "id": "45077",
144 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2016/208065s002lbl.pdf",
145 + "date": "20160817",
146 + "type": "Label"
147 + },
148 + {
149 + "id": "45128",
150 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2016/208065Orig1s002ltr.pdf",
151 + "date": "20160812",
152 + "type": "Letter"
153 + }
154 + ]
155 + },
156 + {
157 + "submission_type": "SUPPL",
158 + "submission_number": "18",
159 + "submission_status": "AP",
160 + "submission_status_date": "20201209",
161 + "review_priority": "STANDARD",
162 + "submission_class_code": "EFFICACY",
163 + "submission_class_code_description": "Efficacy",
164 + "submission_property_type": [
165 + {
166 + "code": "Orphan"
167 + }
168 + ],
169 + "application_docs": [
170 + {
171 + "id": "65555",
172 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/label/2020/208065s018lbl.pdf",
173 + "date": "20201210",
174 + "type": "Label"
175 + },
176 + {
177 + "id": "65559",
178 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2020/208065Orig1s018ltr.pdf",
179 + "date": "20201210",
180 + "type": "Letter"
181 + }
182 + ]
183 + },
184 + {
185 + "submission_type": "ORIG",
186 + "submission_number": "1",
187 + "submission_status": "AP",
188 + "submission_status_date": "20151113",
189 + "review_priority": "PRIORITY",
190 + "submission_class_code": "TYPE 1",
191 + "submission_class_code_description": "Type 1 - New Molecular Entity",
192 + "submission_property_type": [
193 + {
194 + "code": "Orphan"
195 + }
196 + ],
197 + "application_docs": [
198 + {
199 + "id": "44971",
200 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/nda/2015/208065Orig1s000SumR.pdf",
201 + "date": "20151211",
202 + "type": "Summary Review"
203 + },
204 + {
205 + "id": "10641",
206 + "url": "http://www.accessdata.fda.gov/drugsatfda_docs/appletter/2015/208065Orig1s000Ltr.pdf",
207 + "date": "20151113",
208 + "type": "Letter"
209 + }
210 + ]
211 + },
212 + {
213 + "submission_type": "SUPPL",
214 + "submission_number": "1",
215 + "submission_status": "AP",
216 + "submission_status_date": "20160210",
217 + "review_priority": "PRIORITY",
218 + "submission_class_code": "MANUF (CMC)",
219 + "submission_class_code_description": "Manufacturing (CMC)",
220 + "application_docs": []
221 + }
222 + ]
223 + }
224 + ]
225 +}
\ No newline at end of file
added packages/connectors/src/connectors/openfda/fixtures/drugsfda-trastuzumab.json +308 −0
@@ -0,0 +1,308 @@
1 +{
2 + "meta": {
3 + "disclaimer": "Do not rely on openFDA to make decisions regarding medical care. While we make every effort to ensure that data is accurate, you should assume all results are unvalidated. We may limit or otherwise restrict your access to the API in line with our Terms of Service.",
4 + "terms": "https://open.fda.gov/terms/",
5 + "license": "https://open.fda.gov/license/",
6 + "last_updated": "2026-09-04",
7 + "results": {
8 + "skip": 0,
9 + "limit": 100,
10 + "total": 10
11 + }
12 + },
13 + "results": [
14 + {
15 + "application_number": "BLA761073",
16 + "sponsor_name": "AMGEN INC",
17 + "openfda": {
18 + "application_number": [
19 + "BLA761073"
20 + ],
21 + "brand_name": [
22 + "KANJINTI"
23 + ],
24 + "generic_name": [
25 + "TRASTUZUMAB-ANNS"
26 + ],
27 + "manufacturer_name": [
28 + "Amgen, Inc"
29 + ],
30 + "substance_name": [
31 + "TRASTUZUMAB"
32 + ],
33 + "unii": [
34 + "P188ANX8CK"
35 + ],
36 + "product_type": [
37 + "HUMAN PRESCRIPTION DRUG"
38 + ],
39 + "route": [
40 + "INTRAVENOUS"
41 + ]
42 + },
43 + "products": [
44 + {
45 + "product_number": "002",
46 + "brand_name": "KANJINTI",
47 + "active_ingredients": [
48 + {
49 + "name": "TRASTUZUMAB-ANNS",
50 + "strength": "150MG"
51 + }
52 + ],
53 + "dosage_form": "VIAL",
54 + "route": "SINGLE-DOSE",
55 + "marketing_status": "Prescription"
56 + }
57 + ],
58 + "submissions": [
59 + {
60 + "submission_type": "ORIG",
61 + "submission_number": "1",
62 + "submission_status": "AP",
63 + "submission_status_date": "20190613",
64 + "review_priority": "STANDARD",
65 + "submission_class_code": "UNKNOWN",
66 + "application_docs": []
67 + }
68 + ]
69 + },
70 + {
71 + "application_number": "BLA103792",
72 + "sponsor_name": "GENENTECH",
73 + "openfda": {
74 + "application_number": [
75 + "BLA103792"
76 + ],
77 + "brand_name": [
78 + "HERCEPTIN"
79 + ],
80 + "generic_name": [
81 + "TRASTUZUMAB"
82 + ],
83 + "manufacturer_name": [
84 + "Genentech, Inc."
85 + ],
86 + "substance_name": [
87 + "TRASTUZUMAB"
88 + ],
89 + "unii": [
90 + "P188ANX8CK"
91 + ],
92 + "product_type": [
93 + "HUMAN PRESCRIPTION DRUG"
94 + ],
95 + "route": [
96 + "INTRAVENOUS"
97 + ]
98 + },
99 + "products": [
100 + {
101 + "product_number": "001",
102 + "brand_name": "HERCEPTIN",
103 + "active_ingredients": [
104 + {
105 + "name": "TRASTUZUMAB",
106 + "strength": "21MG/ML"
107 + }
108 + ],
109 + "dosage_form": "VIAL",
110 + "route": "INTRAVENOUS",
111 + "marketing_status": "Prescription"
112 + }
113 + ],
114 + "submissions": [
115 + {
116 + "submission_type": "ORIG",
117 + "submission_number": "1",
118 + "submission_status": "AP",
119 + "submission_status_date": "19980925",
120 + "review_priority": "PRIORITY",
121 + "submission_class_code": "TYPE 1",
122 + "submission_class_code_description": "Type 1 - New Molecular Entity",
123 + "application_docs": []
124 + }
125 + ]
126 + },
127 + {
128 + "application_number": "BLA761139",
129 + "sponsor_name": "DAIICHI SANKYO",
130 + "openfda": {
131 + "application_number": [
132 + "BLA761139"
133 + ],
134 + "brand_name": [
135 + "ENHERTU"
136 + ],
137 + "generic_name": [
138 + "FAM-TRASTUZUMAB DERUXTECAN-NXKI"
139 + ],
140 + "manufacturer_name": [
141 + "Daiichi Sankyo Inc."
142 + ],
143 + "substance_name": [
144 + "TRASTUZUMAB DERUXTECAN"
145 + ],
146 + "unii": [
147 + "5384HK7574"
148 + ],
149 + "product_type": [
150 + "HUMAN PRESCRIPTION DRUG"
151 + ],
152 + "route": [
153 + "INTRAVENOUS"
154 + ]
155 + },
156 + "products": [
157 + {
158 + "product_number": "001",
159 + "brand_name": "ENHERTU",
160 + "active_ingredients": [
161 + {
162 + "name": "FAM-TRASTUZUMAB DERUXTECAN-NXKI",
163 + "strength": "100MG"
164 + }
165 + ],
166 + "dosage_form": "INJECTABLE",
167 + "route": "INJECTION",
168 + "marketing_status": "Prescription"
169 + }
170 + ],
171 + "submissions": [
172 + {
173 + "submission_type": "ORIG",
174 + "submission_number": "1",
175 + "submission_status": "AP",
176 + "submission_status_date": "20191220",
177 + "review_priority": "PRIORITY",
178 + "submission_class_code": "TYPE 1",
179 + "submission_class_code_description": "Type 1 - New Molecular Entity",
180 + "submission_property_type": [
181 + {
182 + "code": "Orphan"
183 + }
184 + ],
185 + "application_docs": []
186 + }
187 + ]
188 + },
189 + {
190 + "application_number": "BLA761170",
191 + "sponsor_name": "GENENTECH INC",
192 + "openfda": {
193 + "application_number": [
194 + "BLA761170"
195 + ],
196 + "brand_name": [
197 + "PHESGO"
198 + ],
199 + "generic_name": [
200 + "PERTUZUMAB, TRASTUZUMAB, AND HYALURONIDASE-ZZXF"
201 + ],
202 + "manufacturer_name": [
203 + "Genentech, Inc."
204 + ],
205 + "substance_name": [
206 + "PERTUZUMAB",
207 + "TRASTUZUMAB",
208 + "HYALURONIDASE (HUMAN RECOMBINANT)"
209 + ],
210 + "unii": [
211 + "K16AIQ8CTM",
212 + "P188ANX8CK",
213 + "743QUY4VD8"
214 + ],
215 + "product_type": [
216 + "HUMAN PRESCRIPTION DRUG"
217 + ],
218 + "route": [
219 + "SUBCUTANEOUS"
220 + ]
221 + },
222 + "products": [
223 + {
224 + "product_number": "001",
225 + "brand_name": "PHESGO",
226 + "active_ingredients": [
227 + {
228 + "name": "PERTUZUMAB, TRASTUZUMAB, AND HYALURONIDASE-ZZXF",
229 + "strength": "1,200MG"
230 + }
231 + ],
232 + "dosage_form": "INJECTABLE",
233 + "route": "SUBCUTANEOUS",
234 + "marketing_status": "Prescription"
235 + }
236 + ],
237 + "submissions": [
238 + {
239 + "submission_type": "ORIG",
240 + "submission_number": "1",
241 + "submission_status": "AP",
242 + "submission_status_date": "20200629",
243 + "review_priority": "STANDARD",
244 + "submission_class_code": "TYPE 4",
245 + "submission_class_code_description": "Type 4 - New Combination",
246 + "application_docs": []
247 + }
248 + ]
249 + },
250 + {
251 + "application_number": "BLA125427",
252 + "sponsor_name": "GENENTECH",
253 + "openfda": {
254 + "application_number": [
255 + "BLA125427"
256 + ],
257 + "brand_name": [
258 + "KADCYLA"
259 + ],
260 + "generic_name": [
261 + "ADO-TRASTUZUMAB EMTANSINE"
262 + ],
263 + "manufacturer_name": [
264 + "Genentech, Inc."
265 + ],
266 + "substance_name": [
267 + "TRASTUZUMAB EMTANSINE"
268 + ],
269 + "unii": [
270 + "SE2KH7T06F"
271 + ],
272 + "product_type": [
273 + "HUMAN PRESCRIPTION DRUG"
274 + ],
275 + "route": [
276 + "INTRAVENOUS"
277 + ]
278 + },
279 + "products": [
280 + {
281 + "product_number": "001",
282 + "brand_name": "KADCYLA",
283 + "active_ingredients": [
284 + {
285 + "name": "ADO-TRASTUZUMAB EMTANSINE",
286 + "strength": "100MG"
287 + }
288 + ],
289 + "dosage_form": "VIAL",
290 + "route": "SINGLE-USE",
291 + "marketing_status": "Prescription"
292 + }
293 + ],
294 + "submissions": [
295 + {
296 + "submission_type": "ORIG",
297 + "submission_number": "1",
298 + "submission_status": "AP",
299 + "submission_status_date": "20130222",
300 + "review_priority": "PRIORITY",
301 + "submission_class_code": "TYPE 1",
302 + "submission_class_code_description": "Type 1 - New Molecular Entity",
303 + "application_docs": []
304 + }
305 + ]
306 + }
307 + ]
308 +}
\ No newline at end of file
added packages/connectors/src/connectors/openfda/fixtures/label-gleevec.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "meta": {
3 + "disclaimer": "Do not rely on openFDA to make decisions regarding medical care. While we make every effort to ensure that data is accurate, you should assume all results are unvalidated. We may limit or otherwise restrict your access to the API in line with our Terms of Service.",
4 + "terms": "https://open.fda.gov/terms/",
5 + "license": "https://open.fda.gov/license/",
6 + "last_updated": "2026-09-07",
7 + "results": {
8 + "skip": 0,
9 + "limit": 1,
10 + "total": 1
11 + }
12 + },
13 + "results": [
14 + {
15 + "set_id": "211ef2da-2868-4a77-8055-1cb2cd78e24b",
16 + "id": "2b30c65b-bf6e-4a30-9754-585d500d59f3",
17 + "version": "41",
18 + "effective_time": "20260713",
19 + "indications_and_usage": [
20 + "1 INDICATIONS AND USAGE Gleevec is a kinase inhibitor indicated for the treatment of: Newly diagnosed adult and pediatric patients with Philadelphia chromosome positive chronic myeloid leukemia (Ph+ CML) in chronic phase. ( 1.1 ) Patients with Philadelphia chromosome positive chronic myeloid leukemia (Ph+ CML) in blast crisis (BC), accelerated phase (AP), or in chronic phase (CP) after failure of interferon-alpha therapy. ( 1.2 ) Adult patients with relapsed or refractory Philadelphia chromosome positive acute lymphoblastic leukemia (Ph+ ALL). ( 1.3 ) Pediatric patients with newly diagnosed Philadelphia chromosome positive acute lymphoblastic leukemia (Ph+ ALL) in combination with chemotherapy. ( 1.4 ) Adult patients with myelodysplastic/myeloproliferative diseases (MDS/MPD) associated with platelet-derived growth factor receptor (PDGFR) gene re-arrangements. ( 1.5 ) Adult patients with aggressive systemic mastocytosis (ASM) without the D816V c-Kit mutation or with c-Kit mutational status unknown. ( 1.6 ) Adult patients with hypereosinophilic syndrome (HES) and/or chronic eosinophilic leukemia (CEL) who have the FIP1L1-PDGFR\u03b1 fusion kinase (mutational analysis or fluorescence in situ hybridization [FISH] demonstration of CHIC2 allele deletion) and for patients with HES and/or CEL who are FIP1L1-PDGFR\u03b1 fusion kinase negative or unknown. ( 1.7 ) Adult patients with unresectable, recurrent and/or metastatic dermatofibrosarcoma protuberans (DFSP). ( 1.8 ) Patients with Kit (CD117) positive unresectable and/or metastatic malignant gastrointestinal stromal tumors (GIST). ( 1.9 ) Adjuvant treatment of adult patients following resection of Kit (CD117) positive GIST. ( 1.10 ) 1.1 Newly Diagnosed Philadelphia Positive Chronic Myeloid Leukemia (Ph+ CML) Newly diagnosed adult and pediatric patients with Philadelphia chromosome positive chronic myeloid leukemia (Ph+ CML) in chronic phase. 1.2 Ph+ CML in Blast Crisis (BC), Accelerated Phase (AP) or Chronic Phase (CP) After Interferon-alpha (IFN) Therapy Patients with Philadelphia chromosome positive chronic myeloid leukemia in blast crisis, accelerated phase, or in chronic phase after failure of interferon-alpha therapy. 1.3 Adult Patients With Ph+ Acute Lymphoblastic Leukemia (ALL) Adult patients with relapsed or refractory Philadelphia chromosome positive acute lymphoblastic leukemia (Ph+ ALL). 1.4 Pediatric Patients With Ph+ Acute Lymphoblastic Leukemia (ALL) Pediatric patients with newly diagnosed Philadelphia chromosome positive acute lymphoblastic leukemia (Ph+ ALL) in combination with chemotherapy. 1.5 Myelodysplastic/Myeloproliferative Diseases (MDS/MPD) Adult patients with myelodysplastic/myeloproliferative diseases associated with platelet-derived growth factor receptor (PDGFR) gene re-arrangements. 1.6 Aggressive Systemic Mastocytosis (ASM) Adult patients with aggressive systemic mastocytosis without the D816V c-Kit mutation or with c-Kit mutational status unknown. 1.7 Hypereosinophilic Syndrome (HES) and/or Chronic Eosinophilic Leukemia (CEL) Adult patients with hypereosinophilic syndrome and/or chronic eosinophilic leukemia who have the FIP1L1-PDGFR\u03b1 fusion kinase (mutational analysis or fluorescence in situ hybridization [FISH] demonstration of CHIC2 allele deletion) and for patients with HES and/or CEL who are FIP1L1-PDGFR\u03b1 fusion kinase negative or unknown. 1.8 Dermatofibrosarcoma Protuberans (DFSP) Adult patients with unresectable, recurrent and/or metastatic dermatofibrosarcoma protuberans. 1.9 Kit+ Gastrointestinal Stromal Tumors (GIST) Patients with Kit (CD117) positive unresectable and/or metastatic malignant gastrointestinal stromal tumors. 1.10 Adjuvant Treatment of GIST Adjuvant treatment of adult patients following complete gross resection of Kit (CD117) positive GIST."
21 + ],
22 + "openfda": {
23 + "application_number": [
24 + "NDA021588"
25 + ],
26 + "brand_name": [
27 + "Gleevec"
28 + ],
29 + "generic_name": [
30 + "IMATINIB MESYLATE"
31 + ],
32 + "manufacturer_name": [
33 + "Novartis Pharmaceuticals Corporation"
34 + ],
35 + "substance_name": [
36 + "IMATINIB MESYLATE"
37 + ],
38 + "unii": [
39 + "8A1O1M485B"
40 + ],
41 + "product_type": [
42 + "HUMAN PRESCRIPTION DRUG"
43 + ],
44 + "route": [
45 + "ORAL"
46 + ]
47 + }
48 + }
49 + ]
50 +}
\ No newline at end of file
added packages/connectors/src/connectors/openfda/fixtures/label-osimertinib.json +53 −0
@@ -0,0 +1,53 @@
1 +{
2 + "meta": {
3 + "disclaimer": "Do not rely on openFDA to make decisions regarding medical care. While we make every effort to ensure that data is accurate, you should assume all results are unvalidated. We may limit or otherwise restrict your access to the API in line with our Terms of Service.",
4 + "terms": "https://open.fda.gov/terms/",
5 + "license": "https://open.fda.gov/license/",
6 + "last_updated": "2026-09-07",
7 + "results": {
8 + "skip": 0,
9 + "limit": 1,
10 + "total": 1
11 + }
12 + },
13 + "results": [
14 + {
15 + "set_id": "5e81b4a7-b971-45e1-9c31-29cea8c87ce7",
16 + "id": "d703b5ed-4b1c-44fe-8005-ebe3d8f563e6",
17 + "version": "36",
18 + "effective_time": "20240925",
19 + "indications_and_usage": [
20 + "1 INDICATIONS AND USAGE TAGRISSO is a kinase inhibitor indicated for: \u2022 adjuvant therapy after tumor resection in adult patients with non-small cell lung cancer (NSCLC) whose tumors have epidermal growth factor receptor (EGFR) exon 19 deletions or exon 21 L858R mutations, as detected by an FDA-approved test. ( 1.1 , 2.2 ) \u2022 the treatment of adult patients with locally advanced, unresectable (stage III) NSCLC whose disease has not progressed during or following concurrent or sequential platinum-based chemoradiation therapy and whose tumors have EGFR exon 19 deletions or exon 21 L858R mutations, as detected by an FDA-approved test. ( 1.2 , 2.2 ) \u2022 the first-line treatment of adult patients with metastatic NSCLC whose tumors have EGFR exon 19 deletions or exon 21 L858R mutations, as detected by an FDA-approved test. ( 1.3 , 2.2 ) \u2022 in combination with pemetrexed and platinum-based chemotherapy, the first-line treatment of adult patients with locally advanced or metastatic NSCLC whose tumors have EGFR exon 19 deletions or exon 21 L858R mutations, as detected by an FDA-approved test. ( 1.4 , 2.2 ) \u2022 the treatment of adult patients with metastatic EGFR T790M mutation-positive NSCLC, as detected by an FDA-approved test, whose disease has progressed on or after EGFR TKI therapy. ( 1.5 , 2.2 ) 1.1 Adjuvant Treatment of EGFR Mutation-Positive Non-Small Cell Lung Cancer (NSCLC) TAGRISSO is indicated as adjuvant therapy after tumor resection in adult patients with non-small cell lung cancer (NSCLC) whose tumors have epidermal growth factor receptor (EGFR) exon 19 deletions or exon 21 L858R mutations, as detected by an FDA-approved test [see Dosage and Administration (2.2) ] . 1.2 Locally Advanced, Unresectable (Stage III) EGFR Mutation-Positive NSCLC TAGRISSO is indicated for the treatment of adult patients with locally advanced, unresectable (stage III) NSCLC whose disease has not progressed during or following concurrent or sequential platinum-based chemoradiation therapy and whose tumors have EGFR exon 19 deletions or exon 21 L858R mutations, as detected by an FDA-approved test [see Dosage and Administration (2.2)]. 1.3 First-line Treatment of EGFR Mutation-Positive Metastatic NSCLC TAGRISSO is indicated for the first-line treatment of adult patients with metastatic NSCLC whose tumors have EGFR exon 19 deletions or exon 21 L858R mutations, as detected by an FDA-approved test [see Dosage and Administration (2.2) ]. 1.4 First-line Treatment of EGFR Mutation-Positive Locally Advanced or Metastatic NSCLC TAGRISSO in combination with pemetrexed and platinum-based chemotherapy is indicated for the first-line treatment of adult patients with locally advanced or metastatic NSCLC whose tumors have EGFR exon 19 deletions or exon 21 L858R mutations, as detected by an FDA-approved test [see Dosage and Administration (2.2) ]. 1.5 Previously Treated EGFR T790M Mutation-Positive Metastatic NSCLC TAGRISSO is indicated for the treatment of adult patients with metastatic EGFR T790M mutation-positive NSCLC, as detected by an FDA-approved test, whose disease has progressed on or after EGFR tyrosine kinase inhibitor (TKI) therapy [see Dosage and Administration (2.2) ] ."
21 + ],
22 + "openfda": {
23 + "application_number": [
24 + "NDA208065"
25 + ],
26 + "brand_name": [
27 + "TAGRISSO"
28 + ],
29 + "generic_name": [
30 + "OSIMERTINIB"
31 + ],
32 + "manufacturer_name": [
33 + "AstraZeneca Pharmaceuticals LP"
34 + ],
35 + "substance_name": [
36 + "OSIMERTINIB"
37 + ],
38 + "unii": [
39 + "3C06JJ0Z2O"
40 + ],
41 + "pharm_class_epc": [
42 + "Kinase Inhibitor [EPC]"
43 + ],
44 + "product_type": [
45 + "HUMAN PRESCRIPTION DRUG"
46 + ],
47 + "route": [
48 + "ORAL"
49 + ]
50 + }
51 + }
52 + ]
53 +}
\ No newline at end of file
added packages/connectors/src/connectors/openfda/fixtures/not-found.json +6 −0
@@ -0,0 +1,6 @@
1 +{
2 + "error": {
3 + "code": "NOT_FOUND",
4 + "message": "No matches found!"
5 + }
6 +}
\ No newline at end of file
added packages/connectors/src/connectors/openfda/index.ts +499 −0
@@ -0,0 +1,499 @@
1 +import { and, asc, eq, gt, sql } from 'drizzle-orm';
2 +import { normalizeLabel } from '@cancerindex/shared';
3 +import { cancerAliases, cancerHierarchy, cancers, drugAliases, drugApprovals, drugs, knowledgeEdges } from '@cancerindex/database';
4 +import { CancerResolver } from '@cancerindex/ontology';
5 +import { HttpError } from '../../sdk/http.js';
6 +import { RawLake } from '../../sdk/lake.js';
7 +import { Connector, type ConnectorHealth, type RunContext } from '../../sdk/run.js';
8 +import { INDICATION_MAX_CHARS, OPENFDA_API_KEY_ENV, OPENFDA_DAILY_BUDGET_NO_KEY, DRUGSFDA_MAX_PAGES, DRUGSFDA_PAGE_SIZE, applicationProvenanceUrl, drugsfdaBrandOrSubstanceUrl, drugsfdaSearchUrl, healthUrl, labelByApplicationUrl, manifest } from './manifest.js';
9 +import { Application, CancerDictionary, CancerReconciler, DrugsFdaResponse, LabelResponse, LabelResult, OpenFdaMeta, approvalEvents, classifyApplication, isNoMatch, resolveBullet, splitIndications, truncateIndication, type ApplicationKind, type ApprovalEvent } from './normalize.js';
10 +
11 +interface OpenFdaCursor {
12 + pass?: string;
13 + startedAt?: string;
14 + completedAt?: string;
15 + lastDrugId?: string;
16 + stats?: Stats;
17 +}
18 +
19 +interface Stats {
20 + drugsSeen: number;
21 + skippedClassNames: number;
22 + matched: number;
23 + unresolved: number;
24 + applications: number;
25 + referenceApplications: number;
26 + genericApplications: number;
27 + approvalRows: number;
28 + rowsWithCancer: number;
29 + rowsWithoutCancer: number;
30 + tumorAgnosticRows: number;
31 + ambiguousBullets: number;
32 + edges: number;
33 + labelsMissing: number;
34 +}
35 +
36 +interface ApprovalRowDraft {
37 + key: string;
38 + cancerId: string | null;
39 + tumorAgnostic: boolean;
40 + approvalType: 'ORIG' | 'SUPPL';
41 + approvalDate: string | null;
42 + accelerated: boolean | null;
43 + indication: string;
44 + raw: Record<string, unknown>;
45 +}
46 +
47 +/** CIViC therapy concepts that are drug classes / regimens, never a Drugs@FDA generic name. */
48 +const CLASS_NAME_RE = /\b(inhibitors?|regimen|therapy|chemotherapy|radiation|radiotherapy|sirna|shrna|antisense|agonists?|antagonists?|vaccine|analogs?|combination|placebo|unspecified|other|blocker|modulator|antibod(y|ies)|conjugate|transplant(ation)?|surgery|resection|treatment|supportive care|best supportive care|observation)\b/i;
49 +const BRAND_LIKE_RE = /^[A-Z][a-z]{3,}$/;
50 +
51 +/**
52 + * openFDA connector — US regulatory layer (CLAUDE.md §13). For each drug in `drugs`:
53 + * 1. Drugs@FDA applications by generic name (strict post-filter: same molecule, salt or biosimilar).
54 + * 2. Source record per application; UNII / brand aliases enrich the drug.
55 + * 3. Approved originals + approved efficacy supplements → drug_approvals (US / FDA, dated, per
56 + * application). ORIG rows are split per label indication bullet; a bullet naming exactly one
57 + * cancer gets that cancerId (PROBABILISTIC, label-text dictionary), otherwise cancerId stays null.
58 + * 4. knowledge_edges drug APPROVED_FOR cancer (evidenceCategory regulatory_status) for mapped rows.
59 + * Restartable: ctx.cursor.lastDrugId; daily quota guard without OPENFDA_API_KEY.
60 + */
61 +export class OpenFdaConnector extends Connector {
62 + readonly manifest = manifest;
63 + private reconciler: CancerReconciler | undefined;
64 +
65 + async healthCheck(ctx: RunContext): Promise<ConnectorHealth> {
66 + try {
67 + const res = DrugsFdaResponse.safeParse(await ctx.http.json(healthUrl()));
68 + if (!res.success) return { status: 'degraded', detail: 'unexpected drugsfda payload' };
69 + return { status: 'healthy', detail: `Drugs@FDA last_updated ${res.data.meta.last_updated ?? '?'} (${res.data.meta.results?.total ?? '?'} applications)` };
70 + } catch (e) {
71 + return { status: 'failing', detail: (e as Error).message };
72 + }
73 + }
74 +
75 + async sync(ctx: RunContext): Promise<void> {
76 + if (ctx.mode === 'dry_run') return this.dryRun(ctx);
77 + if (ctx.mode === 'backfill') return this.backfill(ctx);
78 + const hasKey = !!process.env[OPENFDA_API_KEY_ENV]?.trim();
79 + const state = ctx.cursor as OpenFdaCursor;
80 + if (!state.pass || state.completedAt) {
81 + for (const k of Object.keys(state)) delete (state as Record<string, unknown>)[k];
82 + state.pass = new Date().toISOString().slice(0, 10);
83 + state.startedAt = new Date().toISOString();
84 + ctx.info(`starting a fresh openFDA pass (api key: ${hasKey ? 'yes' : 'no — 1,000 req/day budget'})`);
85 + } else ctx.info(`resuming openFDA pass ${state.pass} after drug ${state.lastDrugId ?? '(start)'}`);
86 + const stats: Stats = state.stats ?? { drugsSeen: 0, skippedClassNames: 0, matched: 0, unresolved: 0, applications: 0, referenceApplications: 0, genericApplications: 0, approvalRows: 0, rowsWithCancer: 0, rowsWithoutCancer: 0, tumorAgnosticRows: 0, ambiguousBullets: 0, edges: 0, labelsMissing: 0 };
87 + state.stats = stats;
88 +
89 + const resolver = new CancerResolver(ctx.db);
90 + await resolver.warm();
91 + const { dict, reconciler } = await this.buildDictionary(ctx, resolver);
92 + this.reconciler = reconciler;
93 + ctx.info(`cancer dictionary: ${dict.size()} aliases (${dict.ambiguousDropped} ambiguous dropped), longest ${dict.maxTokens} tokens`);
94 +
95 + const [{ n: total }] = (await ctx.db.execute<{ n: string }>(sql`SELECT count(*)::text AS n FROM drugs`)) as unknown as [{ n: string }];
96 + ctx.info(`${total} drugs to check against Drugs@FDA`);
97 +
98 + for (;;) {
99 + const batch = await ctx.db
100 + .select({ id: drugs.id, name: drugs.name })
101 + .from(drugs)
102 + .where(state.lastDrugId ? gt(drugs.id, state.lastDrugId) : sql`true`)
103 + .orderBy(asc(drugs.id))
104 + .limit(100);
105 + if (batch.length === 0) break;
106 + for (const drug of batch) {
107 + if (ctx.shouldStop()) {
108 + ctx.info(`stopping after drug ${state.lastDrugId ?? '(start)'} — ${stats.drugsSeen} drugs seen this pass; cursor saved`);
109 + return;
110 + }
111 + if (!hasKey && ctx.http.stats.requests >= OPENFDA_DAILY_BUDGET_NO_KEY) {
112 + ctx.warn(`openFDA daily budget without API key reached (${ctx.http.stats.requests} requests) — stopping; set ${OPENFDA_API_KEY_ENV} or rerun tomorrow (cursor saved)`);
113 + return;
114 + }
115 + await this.processDrug(ctx, drug, dict, stats);
116 + state.lastDrugId = drug.id;
117 + stats.drugsSeen++;
118 + }
119 + }
120 + state.completedAt = new Date().toISOString();
121 + ctx.info(`openFDA pass complete: drugs ${stats.drugsSeen} (class/regimen names skipped ${stats.skippedClassNames}), matched ${stats.matched}, unresolved ${stats.unresolved}, applications ${stats.applications} (reference ${stats.referenceApplications}, generic/biosimilar ${stats.genericApplications}), approval rows ${stats.approvalRows} (with cancer ${stats.rowsWithCancer}, without ${stats.rowsWithoutCancer}, tumor-agnostic ${stats.tumorAgnosticRows}), ambiguous bullets ${stats.ambiguousBullets}, edges ${stats.edges}, labels missing ${stats.labelsMissing}`, { ...stats });
122 + }
123 +
124 + /* ---------------------------------------------------------------------------------------------- */
125 +
126 + /**
127 + * `--mode backfill`: re-derive drug_approvals / edges from the payloads already in the raw lake
128 + * (source_records.raw_path) — no HTTP, no cursor change. Used after a mapping-rule change
129 + * (RAW → NORMALIZED stays replayable, CLAUDE.md §2) or when the daily quota is exhausted.
130 + */
131 + private async backfill(ctx: RunContext): Promise<void> {
132 + const resolver = new CancerResolver(ctx.db);
133 + await resolver.warm();
134 + const { dict, reconciler } = await this.buildDictionary(ctx, resolver);
135 + this.reconciler = reconciler;
136 + const recs = await ctx.db.execute<{ entity_kind: string; source_record_id: string; canonical_id: string | null; raw_path: string | null }>(sql`SELECT entity_kind, source_record_id, canonical_id, raw_path FROM source_records WHERE source_id = ${ctx.sourceId} AND entity_kind IN ('application','label') AND raw_path IS NOT NULL ORDER BY entity_kind, id`);
137 + const appsByDrug = new Map<string, Array<{ app: Application; kind: Exclude<ApplicationKind, 'unmatched'> }>>();
138 + const labels = new Map<string, { label: LabelResult; meta: OpenFdaMeta | null }>();
139 + const drugNames = new Map((await ctx.db.select({ id: drugs.id, name: drugs.name }).from(drugs)).map((d) => [d.id, d.name]));
140 + // One application can serve several drug rows (e.g. "Abiraterone" and "Abiraterone Acetate");
141 + // the source record keeps only the last canonical id, so previous approval rows are consulted too.
142 + const drugsByApp = new Map<string, Set<string>>();
143 + for (const a of await ctx.db.selectDistinct({ drugId: drugApprovals.drugId, appNo: drugApprovals.applicationNumber }).from(drugApprovals).where(eq(drugApprovals.sourceId, ctx.sourceId))) {
144 + if (a.appNo) drugsByApp.set(a.appNo, new Set([...(drugsByApp.get(a.appNo) ?? []), a.drugId]));
145 + }
146 + let unreadable = 0;
147 + for (const r of recs) {
148 + const payload = await RawLake.read(r.raw_path!);
149 + if (!payload) {
150 + unreadable++;
151 + continue;
152 + }
153 + if (r.entity_kind === 'label') {
154 + const parsed = LabelResult.safeParse(payload);
155 + if (parsed.success) for (const appNo of parsed.data.openfda?.application_number ?? []) labels.set(appNo, { label: parsed.data, meta: null });
156 + continue;
157 + }
158 + const parsed = Application.safeParse(payload);
159 + if (!parsed.success) continue;
160 + const drugIds = new Set<string>([...(r.canonical_id ? [r.canonical_id] : []), ...(drugsByApp.get(r.source_record_id) ?? [])]);
161 + for (const drugId of drugIds) {
162 + const name = drugNames.get(drugId);
163 + if (!name) continue;
164 + const kind = classifyApplication(parsed.data, name);
165 + if (kind === 'unmatched') continue;
166 + appsByDrug.set(drugId, [...(appsByDrug.get(drugId) ?? []), { app: parsed.data, kind }]);
167 + }
168 + }
169 + ctx.info(`backfill: ${recs.length} lake records (${unreadable} unreadable), ${appsByDrug.size} drugs with applications, ${labels.size} labels; dictionary ${dict.size()} aliases`);
170 + const stats: Stats = { drugsSeen: 0, skippedClassNames: 0, matched: 0, unresolved: 0, applications: 0, referenceApplications: 0, genericApplications: 0, approvalRows: 0, rowsWithCancer: 0, rowsWithoutCancer: 0, tumorAgnosticRows: 0, ambiguousBullets: 0, edges: 0, labelsMissing: 0 };
171 + for (const [drugId, apps] of appsByDrug) {
172 + if (ctx.shouldStop()) break;
173 + await this.persistDrugApplications(ctx, { id: drugId, name: drugNames.get(drugId)! }, apps, dict, stats, labels);
174 + stats.drugsSeen++;
175 + }
176 + ctx.info(`backfill complete: drugs ${stats.drugsSeen}, applications ${stats.applications}, approval rows ${stats.approvalRows} (with cancer ${stats.rowsWithCancer}, without ${stats.rowsWithoutCancer}, tumor-agnostic ${stats.tumorAgnosticRows}), ambiguous bullets ${stats.ambiguousBullets}, edges ${stats.edges}, labels missing ${stats.labelsMissing}`, { ...stats });
177 + }
178 +
179 + private async dryRun(ctx: RunContext): Promise<void> {
180 + for (const name of ['osimertinib', 'pembrolizumab', 'imatinib']) {
181 + const apps = await this.searchApplications(ctx, name, []);
182 + ctx.counters.fetched += apps.length;
183 + for (const a of apps) ctx.observe('application', a.app);
184 + const summary = apps.map((a) => `${a.app.application_number}[${a.kind}] events=${approvalEvents(a.app).length}`).join(', ');
185 + ctx.info(`[dry_run] ${name}: ${apps.length} application(s) — ${summary || 'none'}`);
186 + }
187 + }
188 +
189 + /**
190 + * Dictionary from cancer_aliases of active malignant concepts (all NCIt-derived entity types except
191 + * precursors/other) plus a reconciler (canonical names, aliases, hierarchy) that collapses duplicate
192 + * concepts minted by two terminologies and one-lineage candidates.
193 + */
194 + private async buildDictionary(ctx: RunContext, resolver: CancerResolver): Promise<{ dict: CancerDictionary; reconciler: CancerReconciler }> {
195 + const rows = await ctx.db
196 + .select({ alias: cancerAliases.alias, cancerId: cancerAliases.cancerId, aliasType: cancerAliases.aliasType })
197 + .from(cancerAliases)
198 + .innerJoin(cancers, eq(cancers.id, cancerAliases.cancerId))
199 + .where(and(eq(cancers.status, 'active'), eq(cancers.malignant, true), sql`(${cancers.topLevel} OR ${cancers.entityType} IN ('cancer','cancer_family','hematologic_malignancy','subtype','histology','molecular_subtype'))`, sql`length(${cancerAliases.alias}) >= 4`));
200 + const canon = await ctx.db.select({ id: cancers.id, name: cancers.canonicalName, ncit: cancers.primaryNcitCode }).from(cancers).where(eq(cancers.status, 'active'));
201 + const edges = await ctx.db.select({ parentId: cancerHierarchy.parentId, childId: cancerHierarchy.childId }).from(cancerHierarchy);
202 + const canonical = new Map(canon.map((c) => [c.id, normalizeLabel(c.name)]));
203 + const ncitCoded = new Set(canon.filter((c) => !!c.ncit).map((c) => c.id));
204 + const aliasSets = new Map<string, Set<string>>();
205 + for (const r of rows) {
206 + const s = aliasSets.get(r.cancerId) ?? new Set<string>();
207 + s.add(normalizeLabel(r.alias));
208 + aliasSets.set(r.cancerId, s);
209 + }
210 + const parents = new Map<string, string[]>();
211 + for (const e of edges) parents.set(e.childId, [...(parents.get(e.childId) ?? []), e.parentId]);
212 + const reconciler = new CancerReconciler(canonical, aliasSets, parents, ncitCoded);
213 + return { dict: CancerDictionary.build(rows, (alias) => resolver.byLabel(alias)?.cancerId ?? null, reconciler), reconciler };
214 + }
215 +
216 + /** GET that treats openFDA's 404 NOT_FOUND as an empty page. */
217 + private async getOrEmpty<T>(ctx: RunContext, url: string, schema: { parse(v: unknown): T }): Promise<T | null> {
218 + try {
219 + return schema.parse(await ctx.http.json(url));
220 + } catch (e) {
221 + if (e instanceof HttpError && isNoMatch(e.status, e.bodySnippet)) return null;
222 + throw e;
223 + }
224 + }
225 +
226 + private async searchApplications(ctx: RunContext, name: string, aliases: string[]): Promise<Array<{ app: Application; kind: Exclude<ApplicationKind, 'unmatched'> }>> {
227 + const phrase = name.replace(/[^\w\s\-(),.'+]/g, ' ').replace(/\s+/g, ' ').trim();
228 + if (!phrase) return [];
229 + const seen = new Set<string>();
230 + const accepted: Array<{ app: Application; kind: Exclude<ApplicationKind, 'unmatched'> }> = [];
231 + const consume = (results: unknown[]) => {
232 + for (const raw of results) {
233 + const parsed = Application.safeParse(raw);
234 + if (!parsed.success) {
235 + ctx.counters.validationFailures++;
236 + ctx.counters.rejected++;
237 + ctx.warn(`invalid application record: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}`);
238 + continue;
239 + }
240 + if (seen.has(parsed.data.application_number)) continue;
241 + seen.add(parsed.data.application_number);
242 + const kind = classifyApplication(parsed.data, name);
243 + if (kind !== 'unmatched') accepted.push({ app: parsed.data, kind });
244 + }
245 + };
246 + // 1. generic name (paged, bounded)
247 + for (let page = 0; page < DRUGSFDA_MAX_PAGES; page++) {
248 + const res = await this.getOrEmpty(ctx, drugsfdaSearchUrl('openfda.generic_name', phrase, page * DRUGSFDA_PAGE_SIZE), DrugsFdaResponse);
249 + if (!res) break;
250 + this.noteMeta(ctx, res.meta);
251 + consume(res.results);
252 + const total = res.meta.results?.total ?? res.results.length;
253 + if ((page + 1) * DRUGSFDA_PAGE_SIZE >= total) break;
254 + }
255 + if (accepted.length) return accepted;
256 + // 2. brand / substance name for single-token molecule names
257 + if (!/\s/.test(phrase) && phrase.length >= 5) {
258 + const res = await this.getOrEmpty(ctx, drugsfdaBrandOrSubstanceUrl(phrase), DrugsFdaResponse);
259 + if (res) {
260 + this.noteMeta(ctx, res.meta);
261 + consume(res.results);
262 + }
263 + if (accepted.length) return accepted;
264 + }
265 + // 3. brand-like aliases (max 2)
266 + for (const alias of aliases.filter((a) => BRAND_LIKE_RE.test(a) && normalizeLabel(a) !== normalizeLabel(name)).slice(0, 2)) {
267 + const res = await this.getOrEmpty(ctx, drugsfdaSearchUrl('openfda.brand_name', alias, 0, DRUGSFDA_PAGE_SIZE), DrugsFdaResponse);
268 + if (!res) continue;
269 + this.noteMeta(ctx, res.meta);
270 + consume(res.results);
271 + if (accepted.length) return accepted;
272 + }
273 + return accepted;
274 + }
275 +
276 + private noteMeta(ctx: RunContext, meta: OpenFdaMeta): void {
277 + if (meta.last_updated && !ctx.datasetVersion) ctx.datasetVersion = `drugsfda-${meta.last_updated}`;
278 + }
279 +
280 + private async processDrug(ctx: RunContext, drug: { id: string; name: string }, dict: CancerDictionary, stats: Stats): Promise<void> {
281 + const name = drug.name.trim();
282 + if (CLASS_NAME_RE.test(name) && /\s/.test(name)) {
283 + stats.skippedClassNames++;
284 + await ctx.recordUnresolved('drug', name, normalizeLabel(name), { source: 'openfda', reason: 'class_or_regimen_name', drugId: drug.id });
285 + return;
286 + }
287 + const aliasRows = await ctx.db.select({ alias: drugAliases.alias }).from(drugAliases).where(eq(drugAliases.drugId, drug.id));
288 + const apps = await this.searchApplications(
289 + ctx,
290 + name,
291 + aliasRows.map((a) => a.alias),
292 + );
293 + if (apps.length === 0) {
294 + stats.unresolved++;
295 + await ctx.recordUnresolved('drug', name, normalizeLabel(name), { source: 'openfda', reason: 'no_drugsfda_match', drugId: drug.id });
296 + return;
297 + }
298 + await this.persistDrugApplications(ctx, drug, apps, dict, stats);
299 + }
300 +
301 + /** Source records, identifier/alias enrichment and approval rows for the accepted applications of one drug. */
302 + private async persistDrugApplications(ctx: RunContext, drug: { id: string; name: string }, apps: Array<{ app: Application; kind: Exclude<ApplicationKind, 'unmatched'> }>, dict: CancerDictionary, stats: Stats, preloadedLabels?: Map<string, { label: LabelResult; meta: OpenFdaMeta | null }>): Promise<void> {
303 + const name = drug.name.trim();
304 + stats.matched++;
305 + const reference = apps.filter((a) => a.kind === 'reference');
306 + const generics = apps.filter((a) => a.kind === 'generic_or_biosimilar');
307 + stats.applications += apps.length;
308 + stats.referenceApplications += reference.length;
309 + stats.genericApplications += generics.length;
310 +
311 + // Source records + identifier / alias enrichment for every accepted application.
312 + const brandNames = new Set<string>();
313 + const appStatus = new Map<string, 'created' | 'updated' | 'unchanged'>();
314 + let unii: string | null = null;
315 + for (const { app } of apps) {
316 + const r = await ctx.upsertSourceRecord('application', app.application_number, app, { canonicalType: 'drug', canonicalId: drug.id });
317 + appStatus.set(app.application_number, r.status);
318 + for (const b of app.openfda?.brand_name ?? []) brandNames.add(b);
319 + unii ??= app.openfda?.unii?.[0] ?? null;
320 + }
321 + if (unii) await ctx.db.execute(sql`UPDATE drugs SET unii = COALESCE(unii, ${unii}), updated_at = now() WHERE id = ${drug.id}`);
322 + for (const b of brandNames) {
323 + const norm = normalizeLabel(b);
324 + if (!norm || norm === normalizeLabel(name)) continue;
325 + await ctx.db.insert(drugAliases).values({ drugId: drug.id, alias: b.trim(), normalized: norm, aliasType: 'brand', sourceId: ctx.sourceId }).onConflictDoNothing();
326 + }
327 +
328 + // Approval rows: reference applications; when the molecule only exists as generics /
329 + // biosimilars, the earliest-approved one stands for US availability (others = source records).
330 + let bearing = reference.map((a) => a.app);
331 + if (bearing.length === 0 && generics.length) {
332 + const sorted = generics.map((g) => g.app).sort((a, b) => (approvalEvents(a)[0]?.approvalDate ?? '9999').localeCompare(approvalEvents(b)[0]?.approvalDate ?? '9999'));
333 + bearing = sorted.slice(0, 1);
334 + }
335 + for (const app of bearing) await this.persistApplication(ctx, drug, app, appStatus.get(app.application_number) ?? 'created', dict, stats, preloadedLabels);
336 + }
337 +
338 + private async persistApplication(ctx: RunContext, drug: { id: string; name: string }, app: Application, appStatus: 'created' | 'updated' | 'unchanged', dict: CancerDictionary, stats: Stats, preloadedLabels?: Map<string, { label: LabelResult; meta: OpenFdaMeta | null }>): Promise<void> {
339 + const events = approvalEvents(app);
340 + if (events.length === 0) return;
341 + const appNo = app.application_number;
342 +
343 + // Current label (SPL) of the application — from the lake in backfill mode, else fetched.
344 + let label: LabelResult | null = null;
345 + let labelMeta: OpenFdaMeta | null = null;
346 + let labelStatus: 'created' | 'updated' | 'unchanged' = 'unchanged';
347 + if (preloadedLabels) {
348 + const pre = preloadedLabels.get(appNo);
349 + if (pre) {
350 + label = pre.label;
351 + labelMeta = pre.meta;
352 + }
353 + } else {
354 + const labelRes = await this.getOrEmpty(ctx, labelByApplicationUrl(appNo), LabelResponse);
355 + if (labelRes && labelRes.results.length) {
356 + const parsed = LabelResult.safeParse(labelRes.results[0]);
357 + if (parsed.success) {
358 + label = parsed.data;
359 + labelMeta = labelRes.meta;
360 + } else {
361 + ctx.counters.validationFailures++;
362 + ctx.warn(`invalid label record for ${appNo}: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}`);
363 + }
364 + }
365 + }
366 + if (label) {
367 + const r = await ctx.upsertSourceRecord('label', label.set_id ?? `${appNo}-label`, label, { canonicalType: 'drug', canonicalId: drug.id, sourceUpdatedAt: label.effective_time ? new Date(`${label.effective_time.slice(0, 4)}-${label.effective_time.slice(4, 6)}-${label.effective_time.slice(6, 8)}`) : null });
368 + labelStatus = r.status;
369 + }
370 + if (!label) stats.labelsMissing++;
371 + const indicationText = label?.indications_and_usage?.join(' ') ?? '';
372 +
373 + // Draft rows
374 + const drafts: ApprovalRowDraft[] = [];
375 + const common = { applicationNumber: appNo, sponsor: app.sponsor_name ?? null, labelSetId: label?.set_id ?? null, labelVersion: label?.version ?? null, labelEffectiveTime: label?.effective_time ?? null };
376 + for (const ev of events) {
377 + const base = { ...common, submissionNumber: ev.submissionNumber, reviewPriority: ev.reviewPriority, submissionClassCode: ev.classCode, labelDocUrl: ev.labelDocUrl, letterDocUrl: ev.letterDocUrl, acceleratedNote: 'review_priority PRIORITY is a review track, not accelerated approval; accelerated is set only from explicit label wording' };
378 + if (ev.approvalType === 'SUPPL') {
379 + drafts.push({ key: `SUPPL|${ev.submissionNumber ?? ev.approvalDate ?? 'na'}`, cancerId: null, tumorAgnostic: false, approvalType: 'SUPPL', approvalDate: ev.approvalDate, accelerated: null, indication: `Efficacy supplement ${ev.approvalDate ?? '(date unknown)'} (see label)`, raw: { ...base, kind: 'efficacy_supplement' } });
380 + continue;
381 + }
382 + if (!indicationText) {
383 + drafts.push({ key: `ORIG|${ev.submissionNumber ?? 'na'}|nolabel`, cancerId: null, tumorAgnostic: false, approvalType: 'ORIG', approvalDate: ev.approvalDate, accelerated: null, indication: `Original approval ${ev.approvalDate ?? '(date unknown)'} — label text not available on openFDA`, raw: { ...base, kind: 'original', label: 'missing' } });
384 + continue;
385 + }
386 + const bullets = splitIndications(indicationText).map((b) => resolveBullet(b, dict, this.reconciler));
387 + const unresolved = bullets.filter((b) => !b.cancerId);
388 + bullets.forEach((b, i) => {
389 + if (!b.cancerId) return;
390 + drafts.push({
391 + key: `ORIG|${ev.submissionNumber ?? 'na'}|b${i}`,
392 + cancerId: b.cancerId,
393 + tumorAgnostic: false,
394 + approvalType: 'ORIG',
395 + approvalDate: ev.approvalDate,
396 + accelerated: b.accelerated,
397 + indication: truncateIndication(b.text, INDICATION_MAX_CHARS),
398 + raw: { ...base, kind: 'original', bulletIndex: i, cancer_match: { matchType: 'PROBABILISTIC', via: `label text dictionary${b.via && b.via !== 'single' ? ` (${b.via})` : ''}`, alias: b.alias, distinctMentions: b.distinctCancers } },
399 + });
400 + });
401 + for (const b of unresolved) if (b.distinctCancers >= 2) stats.ambiguousBullets++;
402 + if (unresolved.length) {
403 + const joined = unresolved.map((b) => b.text).join(' • ');
404 + drafts.push({
405 + key: `ORIG|${ev.submissionNumber ?? 'na'}|rest`,
406 + cancerId: null,
407 + tumorAgnostic: unresolved.some((b) => b.tumorAgnostic),
408 + approvalType: 'ORIG',
409 + approvalDate: ev.approvalDate,
410 + accelerated: unresolved.some((b) => b.accelerated) ? true : null,
411 + indication: truncateIndication(joined, INDICATION_MAX_CHARS),
412 + raw: { ...base, kind: 'original', bullets: unresolved.map((b) => ({ distinctCancers: b.distinctCancers, tumorAgnostic: b.tumorAgnostic })), cancer_match: { matchType: 'UNRESOLVED', via: 'label text dictionary: 0 or ≥2 cancers per bullet, or tumor-agnostic' } },
413 + });
414 + }
415 + }
416 +
417 + // Idempotent write: rows of this application keyed by raw->>'key'.
418 + const existing = await ctx.db
419 + .select({ id: drugApprovals.id, raw: drugApprovals.raw, provenanceId: drugApprovals.provenanceId })
420 + .from(drugApprovals)
421 + .where(and(eq(drugApprovals.sourceId, ctx.sourceId), eq(drugApprovals.drugId, drug.id), eq(drugApprovals.applicationNumber, appNo)));
422 + const byKey = new Map(existing.map((r) => [String((r.raw as Record<string, unknown> | null)?.key ?? ''), r]));
423 + // Nothing changed upstream and every draft already exists → keep the previous provenance row
424 + // (a new provenance per weekly run would only duplicate identical evidence).
425 + const unchanged = appStatus === 'unchanged' && labelStatus === 'unchanged' && existing.length > 0 && drafts.every((d) => byKey.has(d.key));
426 + const provenanceId = unchanged
427 + ? existing[0]!.provenanceId
428 + : await ctx.addProvenance({
429 + sourceRecordId: appNo,
430 + sourceUrl: applicationProvenanceUrl(appNo),
431 + dataset: 'Drugs@FDA via openFDA',
432 + datasetVersion: ctx.datasetVersion ?? undefined,
433 + evidenceType: 'regulatory',
434 + accessLevel: 'open',
435 + publishedAt: events[0]?.approvalDate ?? undefined,
436 + updatedAt: labelMeta?.last_updated ?? undefined,
437 + methodology: `Drugs@FDA submissions (ORIG and EFFICACY supplements with status AP) for ${appNo}; indications from the current SPL label${label?.effective_time ? ` (effective ${label.effective_time}, version ${label.version ?? '?'})` : ' (label unavailable)'}; cancer mapping = conservative dictionary over indication bullets (PROBABILISTIC)`,
438 + confidence: 1,
439 + });
440 + const keep = new Set<string>();
441 + for (const d of drafts) {
442 + keep.add(d.key);
443 + const values = {
444 + drugId: drug.id,
445 + cancerId: d.cancerId,
446 + tumorAgnostic: d.tumorAgnostic,
447 + jurisdiction: 'US',
448 + authority: 'FDA',
449 + indication: d.indication,
450 + approvalType: d.approvalType,
451 + accelerated: d.accelerated,
452 + approvalDate: d.approvalDate,
453 + status: 'approved',
454 + applicationNumber: appNo,
455 + sourceId: ctx.sourceId,
456 + provenanceId,
457 + raw: { key: d.key, ...d.raw },
458 + updatedAt: new Date(),
459 + };
460 + const prev = byKey.get(d.key);
461 + if (prev) await ctx.db.update(drugApprovals).set(values).where(eq(drugApprovals.id, prev.id));
462 + else await ctx.db.insert(drugApprovals).values(values);
463 + stats.approvalRows++;
464 + if (d.cancerId) stats.rowsWithCancer++;
465 + else stats.rowsWithoutCancer++;
466 + if (d.tumorAgnostic) stats.tumorAgnosticRows++;
467 + if (d.cancerId) {
468 + await this.upsertEdge(ctx, drug.id, d.cancerId, appNo, d.approvalType, provenanceId);
469 + stats.edges++;
470 + }
471 + }
472 + // Rows of this application that no longer derive from its (successfully fetched) payload.
473 + const stale = existing.filter((r) => !keep.has(String((r.raw as Record<string, unknown> | null)?.key ?? '')));
474 + for (const r of stale) await ctx.db.delete(drugApprovals).where(eq(drugApprovals.id, r.id));
475 + if (stale.length) ctx.info(`${appNo}: removed ${stale.length} stale approval row(s) superseded by the current label/submissions`);
476 + }
477 +
478 + private async upsertEdge(ctx: RunContext, drugId: string, cancerId: string, appNo: string, approvalType: 'ORIG' | 'SUPPL', provenanceId: number): Promise<void> {
479 + const set = {
480 + cancerContextIds: [cancerId],
481 + direction: 'supports',
482 + evidenceLevel: `FDA ${approvalType}`,
483 + evidenceCategory: 'regulatory_status',
484 + status: 'active',
485 + provenanceIds: [provenanceId],
486 + lastSeenAt: new Date(),
487 + };
488 + await ctx.db
489 + .insert(knowledgeEdges)
490 + .values({ sourceEntityType: 'drug', sourceEntityId: drugId, targetEntityType: 'cancer', targetEntityId: cancerId, relationshipType: 'APPROVED_FOR', sourceId: ctx.sourceId, sourceRecordId: appNo, supportCount: 1, ...set })
491 + .onConflictDoUpdate({
492 + target: [knowledgeEdges.sourceEntityType, knowledgeEdges.sourceEntityId, knowledgeEdges.targetEntityType, knowledgeEdges.targetEntityId, knowledgeEdges.relationshipType, knowledgeEdges.sourceId, knowledgeEdges.sourceRecordId],
493 + set,
494 + });
495 + }
496 +}
497 +
498 +export type { ApprovalEvent };
499 +export const connector = new OpenFdaConnector();
added packages/connectors/src/connectors/openfda/manifest.ts +116 −0
@@ -0,0 +1,116 @@
1 +import { defineManifest } from '../../sdk/manifest.js';
2 +
3 +/**
4 + * Docs verified 2026-09-08 — https://open.fda.gov/apis/ (Getting started, Query syntax),
5 + * https://open.fda.gov/apis/authentication/ (rate limits), https://open.fda.gov/apis/drug/drugsfda/
6 + * and https://open.fda.gov/apis/drug/label/ (field references), plus the live API:
7 + * - GET /drug/drugsfda.json?search=openfda.generic_name:"osimertinib"&limit=1 → results[] {
8 + * application_number "NDA208065", sponsor_name, openfda{generic_name[], brand_name[],
9 + * manufacturer_name[], substance_name[], unii[], pharm_class_epc[], pharm_class_moa[], rxcui[]…},
10 + * products[] {product_number, brand_name, active_ingredients[{name,strength}], dosage_form, route,
11 + * marketing_status, reference_drug}, submissions[] {submission_type ORIG|SUPPL, submission_number,
12 + * submission_status AP|TA|…, submission_status_date YYYYMMDD, submission_class_code (TYPE 1…,
13 + * EFFICACY, LABELING, MANUF (CMC)…), review_priority PRIORITY|STANDARD, application_docs[]} }
14 + * and meta{disclaimer, terms, license, last_updated "2026-09-04", results{skip,limit,total}}.
15 + * - GET /drug/label.json?search=openfda.application_number:"NDA208065"&limit=1 → the current SPL
16 + * (effective_time YYYYMMDD, version, set_id, indications_and_usage[] text).
17 + * - No match → HTTP 404 {"error":{"code":"NOT_FOUND","message":"No matches found!"}} — treated as
18 + * an empty result, never as a failure.
19 + * - `limit` ≤ 1000 (drugsfda accepts 100 and 1000), `skip` for pagination; phrase search with
20 + * quotes matches tokens (generic_name:"trastuzumab" also returns ADO-TRASTUZUMAB EMTANSINE and
21 + * FAM-TRASTUZUMAB DERUXTECAN), hence the strict post-filter in normalize.ts.
22 + *
23 + * Rate limits (https://open.fda.gov/apis/authentication/, verified 2026-09-08): "With no API key:
24 + * 240 requests per minute, per IP address. 1,000 requests per day, per IP address. With an API key:
25 + * 240 requests per minute, per key. 120,000 requests per day, per key." → OPENFDA_API_KEY optional;
26 + * without it the connector stops itself at ~950 requests and resumes the next day from its cursor.
27 + *
28 + * Terms (https://open.fda.gov/terms/, "Data Rights and Usage"): "Unless otherwise noted, the
29 + * content, data, documentation, code, and related materials on openFDA is public domain and made
30 + * available with a Creative Commons CC0 1.0 Universal dedication. In short, FDA waives all rights to
31 + * the work worldwide under copyright law, including all related and neighboring rights, to the
32 + * extent allowed by law. You can copy, modify, distribute, and perform the work, even for commercial
33 + * purposes, all without asking permission. FDA makes no warranties about the work, and disclaims
34 + * liability for all uses of the work, to the fullest extent permitted by applicable law."
35 + * License page (https://open.fda.gov/license/): same CC0 1.0 dedication; "When using or citing the
36 + * work, you should not imply endorsement by the author or the affirmer."
37 + * Disclaimer (every API response `meta.disclaimer` and https://open.fda.gov/apis/drug/drugsfda/
38 + * "Responsible use of the data"): see `termsNotes` below — stored verbatim.
39 + */
40 +export const OPENFDA_DISCLAIMER =
41 + 'Do not rely on openFDA to make decisions regarding medical care. While we make every effort to ensure that data is accurate, you should assume all results are unvalidated. We may limit or otherwise restrict your access to the API in line with our Terms of Service.';
42 +
43 +export const manifest = defineManifest({
44 + id: 'openfda',
45 + name: 'openFDA — Drugs@FDA applications and drug labels (SPL)',
46 + organization: 'U.S. Food and Drug Administration (openFDA)',
47 + category: 'regulatory',
48 + tier: 3,
49 + description:
50 + 'US regulatory layer (CLAUDE.md §13, jurisdiction US / authority FDA): for every drug already known to CancerIndex, the Drugs@FDA application(s) (NDA/BLA/ANDA) with their submission history (original approvals and efficacy supplements, approval dates, review priority) and the current prescribing-information "Indications and Usage" section. Approval rows are country-aware, dated, per application; cancer mapping from label text is conservative and flagged PROBABILISTIC.',
51 + homepage: 'https://open.fda.gov',
52 + docsUrl: 'https://open.fda.gov/apis/drug/drugsfda/',
53 + termsUrl: 'https://open.fda.gov/terms/',
54 + access: { type: 'rest', auth: 'none', baseUrl: 'https://api.fda.gov' },
55 + license: 'CC0 1.0 Universal (public domain; https://open.fda.gov/license/)',
56 + licenseStatus: 'approved',
57 + commercialUse: 'allowed',
58 + redistribution: 'allowed',
59 + attribution: 'Data from openFDA (U.S. Food and Drug Administration), Drugs@FDA and FDA drug labeling (SPL); public domain, CC0 1.0. No FDA endorsement implied.',
60 + termsReviewedAt: '2026-09-08',
61 + termsNotes: `openFDA disclaimer (meta.disclaimer, quoted verbatim): "${OPENFDA_DISCLAIMER}" Drugs@FDA page adds: "Always speak to your health provider about the risks and benefits of FDA-regulated products." CancerIndex mirrors this: regulatory status is displayed as a dated, sourced fact per application — never as a treatment recommendation. Terms of service: public domain / CC0 1.0 ("You can copy, modify, distribute, and perform the work, even for commercial purposes, all without asking permission"); some copyrighted third-party material on openFDA may not be public domain (not applicable to Drugs@FDA/SPL data used here).`,
62 + updateFrequency: 'Drugs@FDA weekly; drug labels weekly',
63 + expectedLatency: 'Days',
64 + // Every run walks the local `drugs` table (ordered by id) and persists the last processed id in
65 + // ctx.cursor so an interrupted or daily-quota-limited run resumes (CLAUDE.md §90).
66 + supportsIncrementalSync: true,
67 + entities: ['drug_approvals', 'drug_aliases', 'knowledge_edges', 'source_records'],
68 + metrics: [],
69 + rateLimits: {
70 + requestsPerSecond: 3,
71 + maxConcurrency: 1,
72 + notes: 'openFDA: 240 req/min and 1,000 req/day per IP without key; 240 req/min and 120,000 req/day with OPENFDA_API_KEY (api_key query parameter). 3 req/s = 180/min keeps a margin.',
73 + },
74 + rawRetention: 'full',
75 + documentationVerifiedAt: '2026-09-08',
76 + status: 'active',
77 + schedule: '0 4 * * 1',
78 +});
79 +
80 +export const OPENFDA_API = manifest.access.baseUrl!;
81 +export const OPENFDA_API_KEY_ENV = 'OPENFDA_API_KEY';
82 +/** Daily budget without an API key (1,000/day per IP) minus a safety margin for health checks. */
83 +export const OPENFDA_DAILY_BUDGET_NO_KEY = 950;
84 +export const DRUGSFDA_PAGE_SIZE = 100;
85 +export const DRUGSFDA_MAX_PAGES = 3;
86 +export const INDICATION_MAX_CHARS = 4000;
87 +
88 +function apiKeyParam(): string {
89 + const key = process.env[OPENFDA_API_KEY_ENV]?.trim();
90 + return key ? `api_key=${encodeURIComponent(key)}&` : '';
91 +}
92 +
93 +/** Drugs@FDA applications whose openfda.generic_name contains the phrase. */
94 +export function drugsfdaSearchUrl(field: 'openfda.generic_name' | 'openfda.brand_name' | 'openfda.substance_name' | 'application_number', phrase: string, skip = 0, limit = DRUGSFDA_PAGE_SIZE): string {
95 + return `${OPENFDA_API}/drug/drugsfda.json?${apiKeyParam()}search=${encodeURIComponent(`${field}:"${phrase.replace(/"/g, '')}"`)}&limit=${limit}&skip=${skip}`;
96 +}
97 +
98 +/** Brand or substance name (OR search: a space between terms is an OR in openFDA syntax). */
99 +export function drugsfdaBrandOrSubstanceUrl(phrase: string, limit = DRUGSFDA_PAGE_SIZE): string {
100 + const p = phrase.replace(/"/g, '');
101 + return `${OPENFDA_API}/drug/drugsfda.json?${apiKeyParam()}search=${encodeURIComponent(`openfda.brand_name:"${p}"`)}+${encodeURIComponent(`openfda.substance_name:"${p}"`)}&limit=${limit}`;
102 +}
103 +
104 +/** Current label (SPL) of an application. */
105 +export function labelByApplicationUrl(applicationNumber: string): string {
106 + return `${OPENFDA_API}/drug/label.json?${apiKeyParam()}search=${encodeURIComponent(`openfda.application_number:"${applicationNumber}"`)}&limit=1`;
107 +}
108 +
109 +/** Public, key-free provenance URL of an application (never store the api_key in provenance). */
110 +export function applicationProvenanceUrl(applicationNumber: string): string {
111 + return `${OPENFDA_API}/drug/drugsfda.json?search=${encodeURIComponent(`application_number:"${applicationNumber}"`)}`;
112 +}
113 +
114 +export function healthUrl(): string {
115 + return `${OPENFDA_API}/drug/drugsfda.json?${apiKeyParam()}limit=1`;
116 +}
added packages/connectors/src/connectors/openfda/normalize.ts +442 −0
@@ -0,0 +1,442 @@
1 +import { z } from 'zod';
2 +import { normalizeLabel } from '@cancerindex/shared';
3 +
4 +/* ------------------------------------------------------------------------------------------------
5 + * Response shapes (verified against the live API on 2026-09-08; unknown fields kept for the lake).
6 + * ---------------------------------------------------------------------------------------------- */
7 +
8 +export const OpenFdaMeta = z.object({
9 + disclaimer: z.string().optional(),
10 + terms: z.string().optional(),
11 + license: z.string().optional(),
12 + last_updated: z.string().optional(),
13 + results: z.object({ skip: z.number(), limit: z.number(), total: z.number() }).optional(),
14 +});
15 +
16 +export const Submission = z.object({
17 + submission_type: z.string(), // ORIG | SUPPL
18 + submission_number: z.string().nullish(),
19 + submission_status: z.string().nullish(), // AP | TA | …
20 + submission_status_date: z.string().nullish(), // YYYYMMDD
21 + submission_class_code: z.string().nullish(), // TYPE 1…, EFFICACY, LABELING, MANUF (CMC)…
22 + submission_class_code_description: z.string().nullish(),
23 + review_priority: z.string().nullish(), // PRIORITY | STANDARD
24 + submission_property_type: z.array(z.object({ code: z.string().nullish() }).passthrough()).nullish(),
25 + application_docs: z.array(z.object({ id: z.string().nullish(), url: z.string().nullish(), date: z.string().nullish(), type: z.string().nullish() }).passthrough()).nullish(),
26 +});
27 +export type Submission = z.infer<typeof Submission>;
28 +
29 +export const OpenFdaBlock = z
30 + .object({
31 + application_number: z.array(z.string()).optional(),
32 + brand_name: z.array(z.string()).optional(),
33 + generic_name: z.array(z.string()).optional(),
34 + manufacturer_name: z.array(z.string()).optional(),
35 + substance_name: z.array(z.string()).optional(),
36 + unii: z.array(z.string()).optional(),
37 + rxcui: z.array(z.string()).optional(),
38 + pharm_class_epc: z.array(z.string()).optional(),
39 + pharm_class_moa: z.array(z.string()).optional(),
40 + product_type: z.array(z.string()).optional(),
41 + route: z.array(z.string()).optional(),
42 + })
43 + .passthrough();
44 +
45 +export const Application = z.object({
46 + application_number: z.string(),
47 + sponsor_name: z.string().optional(),
48 + openfda: OpenFdaBlock.optional(),
49 + products: z
50 + .array(
51 + z
52 + .object({
53 + product_number: z.string().nullish(),
54 + brand_name: z.string().nullish(),
55 + active_ingredients: z.array(z.object({ name: z.string().nullish(), strength: z.string().nullish() })).nullish(),
56 + dosage_form: z.string().nullish(),
57 + route: z.string().nullish(), // null observed live (2026-09-08) for one product
58 + marketing_status: z.string().nullish(),
59 + reference_drug: z.string().nullish(),
60 + })
61 + .passthrough(),
62 + )
63 + .nullish(),
64 + submissions: z.array(Submission).nullish(),
65 +});
66 +export type Application = z.infer<typeof Application>;
67 +
68 +export type OpenFdaMeta = z.infer<typeof OpenFdaMeta>;
69 +
70 +export const DrugsFdaResponse = z.object({ meta: OpenFdaMeta, results: z.array(z.unknown()) });
71 +
72 +export const LabelResult = z
73 + .object({
74 + id: z.string().optional(),
75 + set_id: z.string().optional(),
76 + version: z.string().optional(),
77 + effective_time: z.string().optional(), // YYYYMMDD
78 + indications_and_usage: z.array(z.string()).optional(),
79 + openfda: OpenFdaBlock.optional(),
80 + })
81 + .passthrough();
82 +export type LabelResult = z.infer<typeof LabelResult>;
83 +
84 +export const LabelResponse = z.object({ meta: OpenFdaMeta, results: z.array(z.unknown()) });
85 +
86 +export const NotFoundBody = z.object({ error: z.object({ code: z.string(), message: z.string().optional() }) });
87 +
88 +/** openFDA answers HTTP 404 with {"error":{"code":"NOT_FOUND"}} when a search has no match. */
89 +export function isNoMatch(status: number, body: string): boolean {
90 + if (status !== 404) return false;
91 + try {
92 + const parsed = NotFoundBody.safeParse(JSON.parse(body));
93 + return parsed.success && parsed.data.error.code === 'NOT_FOUND';
94 + } catch {
95 + return true; // a bare 404 on a search URL is still "no results" for openFDA
96 + }
97 +}
98 +
99 +/** "20151113" → "2015-11-13"; anything else → null (never fabricate a date). */
100 +export function fdaDate(s: string | undefined | null): string | null {
101 + if (!s) return null;
102 + const m = /^(\d{4})(\d{2})(\d{2})$/.exec(s.trim());
103 + if (!m) return null;
104 + const [, y, mo, d] = m;
105 + const month = Number(mo);
106 + const day = Number(d);
107 + if (month < 1 || month > 12 || day < 1 || day > 31) return null;
108 + return `${y}-${mo}-${d}`;
109 +}
110 +
111 +/* ------------------------------------------------------------------------------------------------
112 + * Drug ↔ application matching. A phrase search on openfda.generic_name is token-based, so
113 + * "trastuzumab" also returns antibody-drug conjugates (FAM-TRASTUZUMAB DERUXTECAN) and fixed
114 + * combinations (PERTUZUMAB, TRASTUZUMAB, AND HYALURONIDASE). Only applications whose generic /
115 + * substance / active-ingredient name is the drug itself (optionally a salt form or a biosimilar
116 + * four-letter suffix) are accepted (CLAUDE.md §7: brand names and salts are aliases of one molecule).
117 + * ---------------------------------------------------------------------------------------------- */
118 +
119 +const SALT_WORDS = new Set([
120 + 'mesylate', 'mesilate', 'hydrochloride', 'hcl', 'dihydrochloride', 'sodium', 'disodium', 'potassium', 'calcium', 'acetate', 'citrate', 'malate', 'maleate', 'dimaleate', 'sulfate', 'sulphate', 'tartrate', 'bitartrate', 'phosphate', 'diphosphate', 'tosylate', 'fumarate', 'succinate', 'bromide', 'tromethamine', 'besylate', 'camsylate', 'ethanolate', 'monohydrate', 'dihydrate', 'trihydrate', 'hydrate', 'anhydrous', 'lactate', 'hydrobromide', 'pamoate', 'decanoate', 'palmitate', 'propionate', 'valerate', 'benzoate', 'gluconate', 'glucuronate', 'isethionate', 'xinafoate', 'napsylate', 'oxalate', 'nitrate', 'hemihydrate', 'sesquihydrate', 'trifluoroacetate', 'dimesylate', 'hyclate', 'ditosylate', 'aspartate', 'ethylsuccinate',
121 +]);
122 +
123 +/** Compare a candidate ingredient name with the drug name: exact, salt form, or biosimilar suffix. */
124 +export function ingredientMatchesDrug(candidate: string, drugName: string): boolean {
125 + const drug = normalizeLabel(drugName);
126 + if (!drug) return false;
127 + let cand = normalizeLabel(candidate.replace(/-[a-z]{4}\b/gi, '')); // biologic suffix (trastuzumab-anns, deruxtecan-nxki)
128 + if (cand === drug) return true;
129 + // FDA proper-name prefixes ("fam-trastuzumab deruxtecan", "ado-trastuzumab emtansine") when the drug name omits them.
130 + const prefixed = /^[a-z]{2,4} (.+)$/.exec(cand);
131 + if (prefixed && !drug.startsWith(`${cand.split(' ')[0]} `) && prefixed[1] === drug) return true;
132 + if (prefixed && !drug.startsWith(`${cand.split(' ')[0]} `) && prefixed[1]!.startsWith(`${drug} `)) cand = prefixed[1]!;
133 + if (!cand.startsWith(`${drug} `)) return false;
134 + const rest = cand.slice(drug.length).trim().split(' ').filter(Boolean);
135 + return rest.length > 0 && rest.every((w) => SALT_WORDS.has(w) || /^eq$/.test(w));
136 +}
137 +
138 +export type ApplicationKind = 'reference' | 'generic_or_biosimilar' | 'unmatched';
139 +
140 +/**
141 + * reference = NDA/BLA for the molecule itself; generic_or_biosimilar = ANDA, or a BLA whose name
142 + * carries a biosimilar suffix; unmatched = the phrase search matched another molecule.
143 + */
144 +export function classifyApplication(app: Application, drugName: string): ApplicationKind {
145 + // Product-level names (generic name of the application, active ingredients of its products).
146 + const productNames = new Set<string>();
147 + for (const n of app.openfda?.generic_name ?? []) productNames.add(n);
148 + for (const p of app.products ?? []) for (const ai of p.active_ingredients ?? []) if (ai.name) productNames.add(ai.name);
149 + const substances = [...new Set((app.openfda?.substance_name ?? []).map((s) => normalizeLabel(s)).filter(Boolean))];
150 + const combinationRe = /,|\band\b|\bwith\b|\//i;
151 + const drugIsCombination = combinationRe.test(drugName);
152 + // A fixed-dose combination (several substances, or a "X, Y, and Z" generic name) is not an approval of the molecule alone.
153 + if (!drugIsCombination) {
154 + if (substances.length > 1 && substances.some((s) => !ingredientMatchesDrug(s, drugName))) return 'unmatched';
155 + if ([...productNames].some((n) => combinationRe.test(n))) return 'unmatched';
156 + }
157 + const matchedProducts = [...productNames].filter((n) => ingredientMatchesDrug(n, drugName));
158 + const matchedSubstances = substances.filter((s) => ingredientMatchesDrug(s, drugName));
159 + if (matchedProducts.length === 0 && !(productNames.size === 0 && matchedSubstances.length > 0)) return 'unmatched';
160 + if (/^ANDA/i.test(app.application_number)) return 'generic_or_biosimilar';
161 + if (matchedProducts.length > 0 && matchedProducts.every((n) => /-[a-z]{4}\b/i.test(n))) return 'generic_or_biosimilar';
162 + return 'reference';
163 +}
164 +
165 +/* ------------------------------------------------------------------------------------------------
166 + * Submissions → approval events. Only approved (AP) originals and approved EFFICACY supplements
167 + * are regulatory approval events; labeling / manufacturing supplements are not. `review_priority`
168 + * PRIORITY is a review track, NOT accelerated approval (21 CFR 314 Subpart H) — `accelerated`
169 + * stays null unless the label text itself says "accelerated approval".
170 + * ---------------------------------------------------------------------------------------------- */
171 +
172 +export interface ApprovalEvent {
173 + approvalType: 'ORIG' | 'SUPPL';
174 + submissionNumber: string | null;
175 + approvalDate: string | null;
176 + reviewPriority: string | null;
177 + classCode: string | null;
178 + labelDocUrl: string | null;
179 + letterDocUrl: string | null;
180 +}
181 +
182 +export function approvalEvents(app: Application): ApprovalEvent[] {
183 + const out: ApprovalEvent[] = [];
184 + for (const s of app.submissions ?? []) {
185 + if (s.submission_status !== 'AP') continue;
186 + const type = s.submission_type.toUpperCase();
187 + const cls = (s.submission_class_code ?? '').toUpperCase();
188 + if (type === 'ORIG' || (type === 'SUPPL' && cls === 'EFFICACY')) {
189 + out.push({
190 + approvalType: type === 'ORIG' ? 'ORIG' : 'SUPPL',
191 + submissionNumber: s.submission_number ?? null,
192 + approvalDate: fdaDate(s.submission_status_date),
193 + reviewPriority: s.review_priority ?? null,
194 + classCode: s.submission_class_code ?? null,
195 + labelDocUrl: s.application_docs?.find((d) => /label/i.test(d.type ?? ''))?.url ?? null,
196 + letterDocUrl: s.application_docs?.find((d) => /letter/i.test(d.type ?? ''))?.url ?? null,
197 + });
198 + }
199 + }
200 + // Oldest first, ORIG before supplements of the same day.
201 + return out.sort((a, b) => (a.approvalDate ?? '').localeCompare(b.approvalDate ?? '') || (a.approvalType === 'ORIG' ? -1 : 1));
202 +}
203 +
204 +/* ------------------------------------------------------------------------------------------------
205 + * "Indications and Usage" → bullets. Highlights enumerate one indication per "( 1.N )" reference:
206 + * "… indicated for: • adjuvant therapy … NSCLC … ( 1.1 , 2.2 ) • the treatment of … ( 1.2 , 2.2 ) 1.1 Adjuvant …"
207 + * The full-text subsections that follow ("1.1 Title …") restate the same indications, so only the
208 + * highlights are split. Fallbacks: "•" bullets, then numbered subsections, then the whole text.
209 + * ---------------------------------------------------------------------------------------------- */
210 +
211 +const SECTION_START = /(?:^|\s)1\.1\s+[A-Z][A-Za-z]/;
212 +const REF_DELIMITER = /\(\s*1\.\d{1,2}(?:\s*,\s*\d+(?:\.\d+)?)*\s*\)/g;
213 +
214 +export function splitIndications(text: string): string[] {
215 + let t = text.replace(/[•●▪◦‣⁃·∙]/g, '•').replace(/\s+/g, ' ').trim();
216 + t = t.replace(/^1\s+INDICATIONS AND USAGE\s*/i, '').replace(/^INDICATIONS AND USAGE\s*/i, '');
217 + const sectionAt = SECTION_START.exec(t)?.index;
218 + const highlights = sectionAt !== undefined && sectionAt > 40 ? t.slice(0, sectionAt) : t;
219 + let bullets: string[] = [];
220 + if (REF_DELIMITER.test(highlights)) {
221 + REF_DELIMITER.lastIndex = 0;
222 + bullets = highlights.split(REF_DELIMITER);
223 + } else if (highlights.includes('•')) {
224 + bullets = highlights.split('•');
225 + } else if (sectionAt !== undefined && sectionAt > 40) {
226 + bullets = t.slice(sectionAt).split(/\s(?=1\.\d{1,2}\s+[A-Z])/);
227 + } else bullets = [highlights];
228 + const clean = (b: string) => b.replace(/^[\s•]+/, '').replace(/^1\.\d{1,2}\s+/, '').trim();
229 + let fragments = bullets.map((b) => b.trim()).filter((b) => b.length > 0);
230 + // The first fragment usually starts with the intro ("X is a kinase inhibitor indicated for:"):
231 + // drop a bare intro, or cut it off when the first indication follows on the same fragment.
232 + if (fragments.length > 1) {
233 + const first = fragments[0]!;
234 + if (/(indicated (for|as|in)[^.]*:|indicated:)\s*$/i.test(first)) fragments = fragments.slice(1);
235 + else {
236 + const m = /^(.*?\b(?:indicated (?:for|as|in)|indicated)\b[^:]*:)\s*(.+)$/i.exec(first);
237 + if (m && clean(m[2]!).length >= 12) fragments[0] = m[2]!;
238 + }
239 + }
240 + const cleaned = fragments.map(clean).filter((b) => b.length >= 12);
241 + return cleaned.length ? cleaned : [t].filter((s) => s.length > 0);
242 +}
243 +
244 +/* ------------------------------------------------------------------------------------------------
245 + * Conservative cancer dictionary over a bullet: whole-word n-gram lookup of normalized cancer
246 + * aliases (≥ 6 chars, generic words excluded), longest match wins on overlaps, exactly one distinct
247 + * cancer → PROBABILISTIC mapping; 0 or ≥ 2 → null (tumor-agnostic / uncertain, kept for curation).
248 + * ---------------------------------------------------------------------------------------------- */
249 +
250 +export const GENERIC_ALIAS_STOPLIST = new Set([
251 + 'cancer', 'tumor', 'neoplasm', 'carcinoma', 'malignancy', 'malignant neoplasm', 'malignant tumor', 'solid tumor', 'solid neoplasm', 'malignant solid neoplasm', 'malignant solid tumor', 'advanced solid tumor', 'metastatic solid tumor', 'adult solid neoplasm', 'childhood solid neoplasm', 'advanced cancer', 'metastatic cancer', 'recurrent cancer', 'primary cancer', 'secondary cancer', 'metastatic neoplasm', 'metastatic tumor', 'metastatic malignant neoplasm', 'advanced malignant neoplasm', 'recurrent malignant neoplasm', 'refractory malignant neoplasm', 'leukemia', 'lymphoma', 'sarcoma', 'adenocarcinoma', 'squamous cell carcinoma', 'carcinoma in situ', 'disease', 'malignant disease', 'neoplastic disease', 'hematologic malignancy', 'hematologic cancer', 'hematopoietic and lymphoid cell neoplasm', 'blood cancer', 'primary tumor', 'secondary tumor', 'metastatic disease', 'advanced disease', 'benign neoplasm', 'tumor cells', 'cancer cells', 'neoplasm by site', 'carcinoma by site', 'mixed tumor', 'unresectable cancer', 'locally advanced cancer', 'childhood cancer', 'adult cancer', 'rare cancer', 'invasive carcinoma', 'invasive cancer', 'in situ neoplasm', 'in situ carcinoma', 'malignant cell', 'metastasis', 'metastases', 'brain metastases', 'liver metastases', 'bone metastases', 'lung metastases', 'micrometastasis', 'stage iv cancer', 'stage iii cancer',
252 +]);
253 +
254 +export interface DictionaryEntry {
255 + cancerId: string;
256 + alias: string;
257 + /** cancer_aliases.alias_type; curated abbreviations (NSCLC, GIST, HNSCC…) are allowed from 4 chars. */
258 + aliasType?: string | null;
259 +}
260 +
261 +/** Minimum alias length: 6 characters, or 4 for curated upper-case abbreviations. */
262 +export function aliasMinLength(alias: string, aliasType?: string | null): number {
263 + return aliasType === 'abbreviation' && /^[A-Z][A-Za-z0-9+-]{3,}$/.test(alias.trim()) ? 4 : 6;
264 +}
265 +
266 +/**
267 + * Collapses several candidate cancers into one when they are the same concept or one lineage:
268 + * - equivalence: the canonical name of A is an alias of B (or vice versa) — happens when two
269 + * terminologies minted separate rows for one disease (OncoTree "Non-Small Cell Lung Cancer" vs
270 + * NCIt "Lung Non-Small Cell Carcinoma"); the NCIt-coded row wins, else the lowest id;
271 + * - lineage: one candidate is a descendant (any hierarchy) of every other → the narrowest concept:
272 + * an indication sentence names the specific disease and uses broader words as context
273 + * ("Head and Neck Squamous Cell Cancer (HNSCC) … squamous cell cancer" → HNSCC, not SCC).
274 + * Returns null when the candidates are genuinely different diseases.
275 + */
276 +export class CancerReconciler {
277 + constructor(
278 + private readonly canonical: Map<string, string>, // id → normalized canonical name
279 + private readonly aliases: Map<string, Set<string>>, // id → normalized aliases
280 + private readonly parents: Map<string, string[]>, // child id → parent ids
281 + private readonly ncitCoded: Set<string>,
282 + ) {}
283 +
284 + private isAncestor(ancestor: string, id: string): boolean {
285 + const seen = new Set<string>();
286 + let frontier = [id];
287 + for (let depth = 0; depth < 14 && frontier.length; depth++) {
288 + const next: string[] = [];
289 + for (const n of frontier) {
290 + for (const p of this.parents.get(n) ?? []) {
291 + if (p === ancestor) return true;
292 + if (!seen.has(p)) {
293 + seen.add(p);
294 + next.push(p);
295 + }
296 + }
297 + }
298 + frontier = next;
299 + }
300 + return false;
301 + }
302 +
303 + equivalent(a: string, b: string): boolean {
304 + if (a === b) return true;
305 + const ca = this.canonical.get(a);
306 + const cb = this.canonical.get(b);
307 + return (!!ca && (this.aliases.get(b)?.has(ca) ?? false)) || (!!cb && (this.aliases.get(a)?.has(cb) ?? false));
308 + }
309 +
310 + private preferred(ids: string[]): string {
311 + return [...ids].sort((x, y) => Number(this.ncitCoded.has(y)) - Number(this.ncitCoded.has(x)) || x.localeCompare(y))[0]!;
312 + }
313 +
314 + /** One id for the set, or null. */
315 + collapse(ids: string[]): { cancerId: string; via: 'single' | 'equivalent' | 'narrowest_of_lineage' } | null {
316 + const distinct = [...new Set(ids)];
317 + if (distinct.length === 0) return null;
318 + if (distinct.length === 1) return { cancerId: distinct[0]!, via: 'single' };
319 + // Merge equivalence classes first.
320 + const groups: string[][] = [];
321 + for (const id of distinct) {
322 + const g = groups.find((grp) => grp.some((m) => this.equivalent(m, id)));
323 + if (g) g.push(id);
324 + else groups.push([id]);
325 + }
326 + const reps = groups.map((g) => this.preferred(g));
327 + if (reps.length === 1) return { cancerId: reps[0]!, via: 'equivalent' };
328 + // `cand` is the narrowest when every other group contains an ancestor of cand (or of an equivalent of cand).
329 + for (let gi = 0; gi < reps.length; gi++) {
330 + const candGroup = groups[gi]!;
331 + const others = groups.filter((_, j) => j !== gi);
332 + if (others.every((g) => g.some((o) => candGroup.some((c) => this.isAncestor(o, c))))) return { cancerId: reps[gi]!, via: 'narrowest_of_lineage' };
333 + }
334 + return null;
335 + }
336 +}
337 +
338 +export interface CancerMention {
339 + cancerId: string;
340 + alias: string;
341 + start: number; // token offset
342 + length: number; // tokens
343 +}
344 +
345 +/**
346 + * Dictionary keyed by normalized alias; built by the connector from cancer_aliases (see index.ts).
347 + * An alias shared by several concepts is only kept when `disambiguate` (the CancerResolver's
348 + * preferred-name / display-name / broadest-lineage logic) picks one; otherwise it is dropped.
349 + */
350 +export class CancerDictionary {
351 + private readonly map = new Map<string, string>();
352 + maxTokens = 1;
353 + ambiguousDropped = 0;
354 + size(): number {
355 + return this.map.size;
356 + }
357 + static build(entries: Iterable<DictionaryEntry>, disambiguate: (alias: string) => string | null, reconciler?: CancerReconciler): CancerDictionary {
358 + const groups = new Map<string, { alias: string; ids: Set<string> }>();
359 + for (const e of entries) {
360 + const norm = normalizeLabel(e.alias);
361 + if (norm.length < aliasMinLength(e.alias, e.aliasType) || GENERIC_ALIAS_STOPLIST.has(norm)) continue;
362 + const g = groups.get(norm) ?? { alias: e.alias, ids: new Set<string>() };
363 + g.ids.add(e.cancerId);
364 + groups.set(norm, g);
365 + }
366 + const dict = new CancerDictionary();
367 + for (const [norm, g] of groups) {
368 + let id: string | null = g.ids.size === 1 ? [...g.ids][0]! : (reconciler?.collapse([...g.ids])?.cancerId ?? disambiguate(g.alias));
369 + if (id && !g.ids.has(id)) id = null;
370 + if (!id) {
371 + dict.ambiguousDropped++;
372 + continue;
373 + }
374 + dict.set(norm, id);
375 + }
376 + return dict;
377 + }
378 + /** Add one normalized alias (tests / curated additions). */
379 + set(normalizedAlias: string, cancerId: string): void {
380 + const norm = normalizeLabel(normalizedAlias);
381 + if (norm.length < 4 || GENERIC_ALIAS_STOPLIST.has(norm)) return;
382 + this.map.set(norm, cancerId);
383 + this.maxTokens = Math.max(this.maxTokens, norm.split(' ').length);
384 + }
385 + /** Non-overlapping mentions, longest span first. */
386 + mentions(text: string): CancerMention[] {
387 + const tokens = normalizeLabel(text).split(' ').filter(Boolean);
388 + const found: CancerMention[] = [];
389 + for (let i = 0; i < tokens.length; i++) {
390 + for (let n = Math.min(this.maxTokens, tokens.length - i); n >= 1; n--) {
391 + const key = tokens.slice(i, i + n).join(' ');
392 + if (key.length < 4) continue;
393 + const id = this.map.get(key);
394 + if (id) found.push({ cancerId: id, alias: key, start: i, length: n });
395 + }
396 + }
397 + found.sort((a, b) => b.length - a.length || a.start - b.start);
398 + const taken: boolean[] = new Array<boolean>(tokens.length).fill(false);
399 + const out: CancerMention[] = [];
400 + for (const m of found) {
401 + let free = true;
402 + for (let k = m.start; k < m.start + m.length; k++) if (taken[k]) free = false;
403 + if (!free) continue;
404 + for (let k = m.start; k < m.start + m.length; k++) taken[k] = true;
405 + out.push(m);
406 + }
407 + return out.sort((a, b) => a.start - b.start);
408 + }
409 +}
410 +
411 +const SOLID_TUMORS_RE = /\bsolid tumou?rs?\b/i;
412 +const AGNOSTIC_BIOMARKER_RE = /\b(NTRK|MSI-H|microsatellite instability|dMMR|mismatch repair|TMB-H|tumor mutational burden|BRAF V600E|RET(?: gene)? fusion|HER2(?:-positive| \(ERBB2\))|KRAS G12C|FGFR|IDH1)\b/i;
413 +
414 +export interface BulletResolution {
415 + text: string;
416 + cancerId: string | null;
417 + alias: string | null;
418 + distinctCancers: number;
419 + via: 'single' | 'equivalent' | 'narrowest_of_lineage' | null;
420 + tumorAgnostic: boolean;
421 + accelerated: boolean | null;
422 +}
423 +
424 +/** One bullet → (cancerId | null), tumor-agnostic flag, accelerated-approval flag (label wording). */
425 +export function resolveBullet(bullet: string, dict: CancerDictionary, reconciler?: CancerReconciler): BulletResolution {
426 + const accelerated = /accelerated approval/i.test(bullet) ? true : null;
427 + const tumorAgnostic = SOLID_TUMORS_RE.test(bullet) && AGNOSTIC_BIOMARKER_RE.test(bullet);
428 + if (tumorAgnostic) return { text: bullet, cancerId: null, alias: null, distinctCancers: 0, via: null, tumorAgnostic: true, accelerated };
429 + const mentions = dict.mentions(bullet);
430 + const distinct = [...new Set(mentions.map((m) => m.cancerId))];
431 + if (distinct.length === 1) return { text: bullet, cancerId: distinct[0]!, alias: mentions[0]!.alias, distinctCancers: 1, via: 'single', tumorAgnostic: false, accelerated };
432 + if (distinct.length >= 2 && reconciler) {
433 + const merged = reconciler.collapse(distinct);
434 + if (merged) return { text: bullet, cancerId: merged.cancerId, alias: mentions.map((m) => m.alias).join(' | '), distinctCancers: distinct.length, via: merged.via, tumorAgnostic: false, accelerated };
435 + }
436 + return { text: bullet, cancerId: null, alias: null, distinctCancers: distinct.length, via: null, tumorAgnostic: false, accelerated };
437 +}
438 +
439 +export function truncateIndication(text: string, max: number): string {
440 + const t = text.replace(/\s+/g, ' ').trim();
441 + return t.length <= max ? t : `${t.slice(0, max - 1)}…`;
442 +}
added packages/connectors/src/connectors/openfda/openfda.test.ts +229 −0
@@ -0,0 +1,229 @@
1 +import { readFileSync } from 'node:fs';
2 +import path from 'node:path';
3 +import { afterEach, describe, expect, it } from 'vitest';
4 +import { OPENFDA_API_KEY_ENV, OPENFDA_DISCLAIMER, applicationProvenanceUrl, drugsfdaBrandOrSubstanceUrl, drugsfdaSearchUrl, labelByApplicationUrl, manifest } from './manifest.js';
5 +import { Application, CancerDictionary, CancerReconciler, DrugsFdaResponse, LabelResponse, LabelResult, approvalEvents, classifyApplication, fdaDate, ingredientMatchesDrug, isNoMatch, resolveBullet, splitIndications, truncateIndication } from './normalize.js';
6 +
7 +const fx = (name: string) => JSON.parse(readFileSync(path.join(import.meta.dirname, 'fixtures', name), 'utf8')) as unknown;
8 +const fxText = (name: string) => readFileSync(path.join(import.meta.dirname, 'fixtures', name), 'utf8');
9 +
10 +describe('openfda manifest', () => {
11 + const original = process.env[OPENFDA_API_KEY_ENV];
12 + afterEach(() => {
13 + if (original === undefined) delete process.env[OPENFDA_API_KEY_ENV];
14 + else process.env[OPENFDA_API_KEY_ENV] = original;
15 + });
16 + it('is a reviewed public-domain regulatory source with the disclaimer stored verbatim', () => {
17 + expect(manifest.id).toBe('openfda');
18 + expect(manifest.category).toBe('regulatory');
19 + expect(manifest.licenseStatus).toBe('approved');
20 + expect(manifest.license).toMatch(/CC0 1\.0/);
21 + expect(manifest.termsNotes).toContain(OPENFDA_DISCLAIMER);
22 + expect(manifest.documentationVerifiedAt).toBe('2026-09-08');
23 + expect(manifest.rateLimits.notes).toMatch(/1,000 req\/day/);
24 + });
25 + it('builds search URLs with phrase syntax and never leaks the api key into provenance URLs', () => {
26 + delete process.env[OPENFDA_API_KEY_ENV];
27 + expect(drugsfdaSearchUrl('openfda.generic_name', 'osimertinib')).toBe('https://api.fda.gov/drug/drugsfda.json?search=openfda.generic_name%3A%22osimertinib%22&limit=100&skip=0');
28 + expect(decodeURIComponent(drugsfdaBrandOrSubstanceUrl('tagrisso'))).toContain('openfda.brand_name:"tagrisso"+openfda.substance_name:"tagrisso"');
29 + expect(decodeURIComponent(labelByApplicationUrl('NDA208065'))).toContain('openfda.application_number:"NDA208065"');
30 + process.env[OPENFDA_API_KEY_ENV] = 'secret';
31 + expect(drugsfdaSearchUrl('openfda.generic_name', 'osimertinib')).toContain('api_key=secret&search=');
32 + expect(applicationProvenanceUrl('NDA208065')).not.toContain('secret');
33 + expect(decodeURIComponent(applicationProvenanceUrl('NDA208065'))).toBe('https://api.fda.gov/drug/drugsfda.json?search=application_number:"NDA208065"');
34 + });
35 +});
36 +
37 +describe('no-match handling and dates', () => {
38 + it('treats the openFDA 404 NOT_FOUND body as "no results", not other errors', () => {
39 + expect(isNoMatch(404, fxText('not-found.json'))).toBe(true);
40 + expect(isNoMatch(404, 'gateway 404')).toBe(true);
41 + expect(isNoMatch(500, fxText('not-found.json'))).toBe(false);
42 + expect(isNoMatch(429, '')).toBe(false);
43 + });
44 + it('converts YYYYMMDD to ISO dates and refuses garbage', () => {
45 + expect(fdaDate('20151113')).toBe('2015-11-13');
46 + expect(fdaDate('2015-11-13')).toBeNull();
47 + expect(fdaDate('20151340')).toBeNull();
48 + expect(fdaDate(undefined)).toBeNull();
49 + });
50 +});
51 +
52 +describe('Drugs@FDA applications', () => {
53 + it('parses the osimertinib application and derives ORIG + efficacy supplement events only', () => {
54 + const res = DrugsFdaResponse.parse(fx('drugsfda-osimertinib.json'));
55 + expect(res.meta.last_updated).toBe('2026-09-04');
56 + expect(res.meta.disclaimer).toBe(OPENFDA_DISCLAIMER);
57 + const app = Application.parse(res.results[0]);
58 + expect(app.application_number).toBe('NDA208065');
59 + expect(app.openfda?.unii).toEqual(['3C06JJ0Z2O']);
60 + expect(classifyApplication(app, 'Osimertinib')).toBe('reference');
61 + const events = approvalEvents(app);
62 + const types = app.submissions!.map((s) => `${s.submission_type}/${s.submission_class_code}`);
63 + expect(types).toContain('SUPPL/LABELING'); // present upstream…
64 + expect(events.every((e) => e.approvalType === 'ORIG' || e.approvalType === 'SUPPL')).toBe(true);
65 + expect(events.filter((e) => e.approvalType === 'ORIG')).toHaveLength(1);
66 + expect(events.filter((e) => e.approvalType === 'SUPPL')).toHaveLength(3); // …but only EFFICACY supplements become events
67 + expect(events[0]).toMatchObject({ approvalType: 'ORIG', approvalDate: '2015-11-13', reviewPriority: 'PRIORITY' });
68 + expect(events[0]!.letterDocUrl).toMatch(/208065Orig1s000Ltr\.pdf$/);
69 + // PRIORITY review is not accelerated approval: no event carries an accelerated flag.
70 + expect(Object.keys(events[0]!)).not.toContain('accelerated');
71 + });
72 + it('a drug with several supplements: efficacy supplements are chronological and dated', () => {
73 + const res = DrugsFdaResponse.parse(fx('drugsfda-imatinib.json'));
74 + const gleevec = res.results.map((r) => Application.parse(r)).find((a) => a.application_number === 'NDA021588')!;
75 + const events = approvalEvents(gleevec);
76 + expect(events[0]).toMatchObject({ approvalType: 'ORIG', approvalDate: '2003-04-18' }); // NDA021588 = Gleevec tablets (capsules were NDA021335, 2001)
77 + const suppl = events.filter((e) => e.approvalType === 'SUPPL');
78 + expect(suppl.length).toBeGreaterThanOrEqual(10);
79 + for (let i = 1; i < suppl.length; i++) expect(suppl[i]!.approvalDate! >= suppl[i - 1]!.approvalDate!).toBe(true);
80 + expect(suppl.every((e) => e.classCode === 'EFFICACY')).toBe(true);
81 + });
82 + it('classifies reference NDA/BLA vs ANDA generics vs biosimilars vs other molecules (phrase-search noise)', () => {
83 + const imatinib = DrugsFdaResponse.parse(fx('drugsfda-imatinib.json')).results.map((r) => Application.parse(r));
84 + const kinds = Object.fromEntries(imatinib.map((a) => [a.application_number, classifyApplication(a, 'Imatinib')]));
85 + expect(kinds).toMatchObject({ NDA021588: 'reference', ANDA078340: 'generic_or_biosimilar', ANDA204285: 'generic_or_biosimilar', NDA219097: 'reference' });
86 + const trastuzumab = DrugsFdaResponse.parse(fx('drugsfda-trastuzumab.json')).results.map((r) => Application.parse(r));
87 + const t = Object.fromEntries(trastuzumab.map((a) => [a.application_number, classifyApplication(a, 'Trastuzumab')]));
88 + expect(t).toEqual({ BLA103792: 'reference', BLA761073: 'generic_or_biosimilar', BLA761139: 'unmatched', BLA761170: 'unmatched', BLA125427: 'unmatched' });
89 + // The ADC is its own molecule: it matches only under its own name (the -nxki suffix makes the
90 + // kind ambiguous with a biosimilar; either way it is accepted, and being the only application it
91 + // still bears the approval rows — see processDrug).
92 + expect(classifyApplication(trastuzumab.find((a) => a.application_number === 'BLA761139')!, 'Fam-trastuzumab deruxtecan')).not.toBe('unmatched');
93 + expect(classifyApplication(trastuzumab.find((a) => a.application_number === 'BLA761139')!, 'Trastuzumab deruxtecan')).not.toBe('unmatched');
94 + });
95 + it('salt forms and biosimilar suffixes match the molecule; other molecules do not', () => {
96 + expect(ingredientMatchesDrug('IMATINIB MESYLATE', 'Imatinib')).toBe(true);
97 + expect(ingredientMatchesDrug('OSIMERTINIB MESYLATE', 'osimertinib')).toBe(true);
98 + expect(ingredientMatchesDrug('TRASTUZUMAB-ANNS', 'Trastuzumab')).toBe(true);
99 + expect(ingredientMatchesDrug('ADO-TRASTUZUMAB EMTANSINE', 'Trastuzumab')).toBe(false);
100 + expect(ingredientMatchesDrug('FAM-TRASTUZUMAB DERUXTECAN-NXKI', 'Trastuzumab')).toBe(false);
101 + expect(ingredientMatchesDrug('ABIRATERONE ACETATE', 'Abiraterone Acetate')).toBe(true);
102 + expect(ingredientMatchesDrug('ABIRATERONE ACETATE', 'Abiraterone')).toBe(true);
103 + });
104 + it('rejects a malformed application and accepts an empty result set', () => {
105 + expect(Application.safeParse({ sponsor_name: 'X' }).success).toBe(false);
106 + expect(Application.safeParse({ application_number: 'NDA1', submissions: [{ submission_status: 'AP' }] }).success).toBe(false);
107 + const empty = DrugsFdaResponse.parse({ meta: { results: { skip: 0, limit: 1, total: 0 } }, results: [] });
108 + expect(empty.results).toHaveLength(0);
109 + });
110 +});
111 +
112 +/** Tiny dictionary standing in for cancer_aliases (ids are arbitrary). */
113 +function dict(): CancerDictionary {
114 + const d = CancerDictionary.build(
115 + [
116 + { alias: 'Non-Small Cell Lung Cancer', cancerId: 'CI-CAN-NSCLC' },
117 + { alias: 'NSCLC', cancerId: 'CI-CAN-NSCLC', aliasType: 'abbreviation' },
118 + { alias: 'Chronic Myeloid Leukemia', cancerId: 'CI-CAN-CML' },
119 + { alias: 'Acute Lymphoblastic Leukemia', cancerId: 'CI-CAN-ALL' },
120 + { alias: 'Gastrointestinal Stromal Tumor', cancerId: 'CI-CAN-GIST' },
121 + { alias: 'GIST', cancerId: 'CI-CAN-GIST', aliasType: 'abbreviation' },
122 + { alias: 'AML', cancerId: 'CI-CAN-AML', aliasType: 'abbreviation' }, // 3 chars → excluded
123 + { alias: 'gist', cancerId: 'CI-CAN-OTHER', aliasType: 'synonym' }, // non-abbreviation short alias → excluded
124 + { alias: 'Hypereosinophilic Syndrome', cancerId: 'CI-CAN-HES' },
125 + { alias: 'Chronic Eosinophilic Leukemia', cancerId: 'CI-CAN-CEL' },
126 + { alias: 'Dermatofibrosarcoma Protuberans', cancerId: 'CI-CAN-DFSP' },
127 + { alias: 'Systemic Mastocytosis', cancerId: 'CI-CAN-SM' },
128 + { alias: 'Melanoma', cancerId: 'CI-CAN-MEL' },
129 + { alias: 'Breast Cancer', cancerId: 'CI-CAN-BRCA' },
130 + { alias: 'Breast Cancer', cancerId: 'CI-CAN-BRCA-CHILD' }, // ambiguous → resolver decides
131 + { alias: 'Metastatic Breast Cancer', cancerId: 'CI-CAN-MBC' },
132 + { alias: 'Solid Tumor', cancerId: 'CI-CAN-SOLID' }, // stoplisted
133 + { alias: 'Cancer', cancerId: 'CI-CAN-ANY' }, // stoplisted / too short
134 + { alias: 'Colon Cancer', cancerId: 'CI-CAN-COLON-A' },
135 + { alias: 'Colon Cancer', cancerId: 'CI-CAN-COLON-B' }, // ambiguous, resolver returns null → dropped
136 + ],
137 + (alias) => (alias === 'Breast Cancer' ? 'CI-CAN-BRCA' : null),
138 + );
139 + return d;
140 +}
141 +
142 +describe('indication text → bullets → conservative cancer mapping', () => {
143 + it('splits the osimertinib highlights into one bullet per "( 1.N )" reference, all NSCLC', () => {
144 + const label = LabelResult.parse(LabelResponse.parse(fx('label-osimertinib.json')).results[0]);
145 + expect(label.effective_time).toBe('20240925');
146 + const bullets = splitIndications(label.indications_and_usage!.join(' '));
147 + expect(bullets).toHaveLength(5);
148 + expect(bullets[0]).toMatch(/^adjuvant therapy after tumor resection/);
149 + expect(bullets[4]).toMatch(/EGFR T790M/);
150 + const d = dict();
151 + const res = bullets.map((b) => resolveBullet(b, d));
152 + expect(res.every((r) => r.cancerId === 'CI-CAN-NSCLC')).toBe(true);
153 + expect(res.every((r) => r.accelerated === null)).toBe(true);
154 + });
155 + it('Gleevec: one cancer per bullet is mapped; the HES/CEL bullet names two cancers → null', () => {
156 + const label = LabelResult.parse(LabelResponse.parse(fx('label-gleevec.json')).results[0]);
157 + const bullets = splitIndications(label.indications_and_usage!.join(' '));
158 + expect(bullets.length).toBe(10);
159 + const d = dict();
160 + const res = bullets.map((b) => resolveBullet(b, d));
161 + expect(res[0]!.cancerId).toBe('CI-CAN-CML'); // newly diagnosed Ph+ CML
162 + expect(res[2]!.cancerId).toBe('CI-CAN-ALL');
163 + const hes = res.find((r) => /hypereosinophilic/i.test(r.text))!;
164 + expect(hes.cancerId).toBeNull();
165 + expect(hes.distinctCancers).toBe(2);
166 + expect(res.find((r) => /GIST/.test(r.text) && /adjuvant/i.test(r.text))!.cancerId).toBe('CI-CAN-GIST');
167 + expect(res.find((r) => /dermatofibrosarcoma/i.test(r.text))!.cancerId).toBe('CI-CAN-DFSP');
168 + });
169 + it('an indication naming two cancers stays unmapped; longest alias wins on overlaps', () => {
170 + const d = dict();
171 + expect(resolveBullet('for the treatment of adult patients with melanoma or breast cancer', d)).toMatchObject({ cancerId: null, distinctCancers: 2 });
172 + expect(resolveBullet('for the treatment of adult patients with metastatic breast cancer after prior therapy', d)).toMatchObject({ cancerId: 'CI-CAN-MBC', alias: 'metastatic breast cancer' });
173 + expect(resolveBullet('for the treatment of adult patients with breast cancer', d)).toMatchObject({ cancerId: 'CI-CAN-BRCA' });
174 + expect(resolveBullet('for the treatment of colon cancer', d)).toMatchObject({ cancerId: null, distinctCancers: 0 }); // ambiguous alias dropped at build time
175 + expect(d.ambiguousDropped).toBe(1);
176 + });
177 + it('tumor-agnostic wording (solid tumors + biomarker) is flagged and never mapped; accelerated approval only from label wording', () => {
178 + const d = dict();
179 + const r = resolveBullet('for the treatment of adult and pediatric patients with unresectable or metastatic microsatellite instability-high (MSI-H) or mismatch repair deficient (dMMR) solid tumors. This indication is approved under accelerated approval based on tumor response rate and durability of response.', d);
180 + expect(r).toMatchObject({ cancerId: null, tumorAgnostic: true, accelerated: true });
181 + expect(resolveBullet('for the treatment of adult patients with NSCLC', d).accelerated).toBeNull();
182 + expect(resolveBullet('solid tumors of the pancreas', d).tumorAgnostic).toBe(false); // no biomarker → not agnostic wording
183 + });
184 + it('reconciler collapses duplicate concepts (two terminologies) and one-lineage candidates, never distinct diseases', () => {
185 + const canonical = new Map([
186 + ['CI-ONC-NSCLC', 'non small cell lung cancer'], // OncoTree-minted row
187 + ['CI-NCIT-NSCLC', 'lung non small cell carcinoma'], // NCIt row
188 + ['CI-MEL', 'melanoma'],
189 + ['CI-UVEAL', 'uveal melanoma'],
190 + ['CI-BRCA', 'breast carcinoma'],
191 + ]);
192 + const aliases = new Map([
193 + ['CI-ONC-NSCLC', new Set(['non small cell lung cancer', 'nsclc'])],
194 + ['CI-NCIT-NSCLC', new Set(['lung non small cell carcinoma', 'non small cell lung cancer', 'nsclc'])],
195 + ['CI-MEL', new Set(['melanoma'])],
196 + ['CI-UVEAL', new Set(['uveal melanoma'])],
197 + ['CI-BRCA', new Set(['breast carcinoma', 'breast cancer'])],
198 + ]);
199 + const parents = new Map([['CI-UVEAL', ['CI-MEL']]]);
200 + const rec = new CancerReconciler(canonical, aliases, parents, new Set(['CI-NCIT-NSCLC', 'CI-MEL', 'CI-UVEAL', 'CI-BRCA']));
201 + expect(rec.collapse(['CI-ONC-NSCLC', 'CI-NCIT-NSCLC'])).toEqual({ cancerId: 'CI-NCIT-NSCLC', via: 'equivalent' }); // NCIt-coded row wins
202 + expect(rec.collapse(['CI-UVEAL', 'CI-MEL'])).toEqual({ cancerId: 'CI-UVEAL', via: 'narrowest_of_lineage' }); // the specific disease named in the sentence
203 + expect(rec.collapse(['CI-MEL', 'CI-BRCA'])).toBeNull();
204 + expect(rec.collapse(['CI-MEL'])).toEqual({ cancerId: 'CI-MEL', via: 'single' });
205 + // Dictionary build keeps an alias shared by equivalent rows; bullets with both spellings map to one cancer.
206 + const d = CancerDictionary.build(
207 + [
208 + { alias: 'Non-Small Cell Lung Cancer', cancerId: 'CI-ONC-NSCLC' },
209 + { alias: 'Non-Small Cell Lung Cancer', cancerId: 'CI-NCIT-NSCLC' },
210 + { alias: 'NSCLC', cancerId: 'CI-ONC-NSCLC', aliasType: 'abbreviation' },
211 + { alias: 'Melanoma', cancerId: 'CI-MEL' },
212 + { alias: 'Breast Cancer', cancerId: 'CI-BRCA' },
213 + ],
214 + () => null,
215 + rec,
216 + );
217 + expect(d.ambiguousDropped).toBe(0);
218 + expect(resolveBullet('adjuvant therapy in adult patients with non-small cell lung cancer (NSCLC) after resection', d, rec)).toMatchObject({ cancerId: 'CI-NCIT-NSCLC', via: 'equivalent', distinctCancers: 2 });
219 + expect(resolveBullet('patients with melanoma or breast cancer', d, rec)).toMatchObject({ cancerId: null, distinctCancers: 2 });
220 + expect(resolveBullet('patients with melanoma or breast cancer', d)).toMatchObject({ cancerId: null }); // without a reconciler: unchanged behaviour
221 + });
222 + it('stoplisted and short aliases are excluded from the dictionary; text without a hit is null', () => {
223 + const d = dict();
224 + expect(resolveBullet('for the treatment of advanced solid tumor and cancer', d)).toMatchObject({ cancerId: null, distinctCancers: 0 });
225 + expect(splitIndications('X is indicated for the treatment of adult patients with melanoma.')).toEqual(['X is indicated for the treatment of adult patients with melanoma.']);
226 + expect(splitIndications('DRUG is a kinase inhibitor indicated for: • the treatment of adult patients with NSCLC. • the treatment of adult patients with melanoma.')).toEqual(['the treatment of adult patients with NSCLC.', 'the treatment of adult patients with melanoma.']);
227 + expect(truncateIndication('a'.repeat(5000), 4000)).toHaveLength(4000);
228 + });
229 +});
modified packages/connectors/src/index.ts +1 −0
@@ -2,3 +2,4 @@ export * from './sdk/index.js';
2 2 export * from './registry.js';
3 3 export * from './sources-sync.js';
4 4 export * from './planned.js';
5 +export * from './ops/index.js';
added packages/connectors/src/ops/doctor.ts +334 −0
@@ -0,0 +1,334 @@
1 +import { accessSync, constants, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync, unlinkSync } from 'node:fs';
2 +import { statfs } from 'node:fs/promises';
3 +import path from 'node:path';
4 +import { fileURLToPath } from 'node:url';
5 +import { sql } from 'drizzle-orm';
6 +import { dataDir } from '@cancerindex/shared';
7 +import { listAlerts, type Database, type SystemAlert } from '@cancerindex/database';
8 +import { CONNECTORS } from '../registry.js';
9 +import { humanDuration, isStale } from './schedule.js';
10 +
11 +export type CheckLevel = 'ok' | 'info' | 'warn' | 'fail';
12 +
13 +export interface DoctorCheck {
14 + section: string;
15 + name: string;
16 + level: CheckLevel;
17 + detail: string;
18 +}
19 +
20 +export interface ConnectorReport {
21 + id: string;
22 + status: string;
23 + licenseStatus: string;
24 + health: string;
25 + paused: boolean;
26 + lastSuccessAt: Date | null;
27 + lastSuccessAge: string;
28 + schedule: string | null;
29 + stale: boolean;
30 + lastRun: { id: string; status: string; startedAt: Date; recordsFetched: number; anomaly: string | null; drift: number; error: string | null } | null;
31 + cursorSummary: string;
32 +}
33 +
34 +export interface DoctorReport {
35 + generatedAt: Date;
36 + checks: DoctorCheck[];
37 + connectors: ConnectorReport[];
38 + tables: Array<{ table: string; rows: number }>;
39 + unresolvedTop: Array<{ source: string; entityKind: string; sourceText: string; count: number }>;
40 + alerts: SystemAlert[];
41 + disk: { rawDir: string; bytes: number; files: number; freeBytes: number | null } | null;
42 + rankings: { snapshots: number; current: number; latest: Date | null } | null;
43 + hardFailures: number;
44 +}
45 +
46 +export interface DoctorOptions {
47 + /** Folder holding drizzle migrations (defaults to packages/database/migrations resolved from this file, then cwd). */
48 + migrationsDir?: string;
49 + /** Skip the data/raw walk (large lakes). */
50 + skipDisk?: boolean;
51 + now?: Date;
52 +}
53 +
54 +const COUNTED_TABLES = ['sources', 'ingest_runs', 'source_records', 'provenance', 'cancers', 'cancer_aliases', 'cancer_hierarchy', 'cancer_codes', 'genes', 'variants', 'drugs', 'clinical_trials', 'trial_conditions', 'publications', 'literature_counts', 'civic_evidence_items', 'genomic_cohorts', 'cancer_gene_frequencies', 'epidemiology_observations', 'survival_observations', 'knowledge_edges', 'unresolved_labels', 'entity_counters', 'rankings', 'ranking_snapshots', 'system_alerts'];
55 +const OPTIONAL_KEYS = ['NCBI_API_KEY', 'SEER_API_KEY', 'ADMIN_TOKEN', 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY'];
56 +
57 +/**
58 + * Readiness report for operators (`pnpm cix doctor`): environment, database + extensions +
59 + * migrations, table sizes, per-connector state (health, freshness, last run, anomaly, drift,
60 + * cursor), curation backlog, data-lake disk usage, ranking freshness and open alerts.
61 + * `hardFailures > 0` ⇒ the CLI exits 1.
62 + */
63 +export async function runDoctor(db: Database, opts: DoctorOptions = {}): Promise<DoctorReport> {
64 + const now = opts.now ?? new Date();
65 + const checks: DoctorCheck[] = [];
66 + const add = (section: string, name: string, level: CheckLevel, detail: string) => checks.push({ section, name, level, detail });
67 + const report: DoctorReport = { generatedAt: now, checks, connectors: [], tables: [], unresolvedTop: [], alerts: [], disk: null, rankings: null, hardFailures: 0 };
68 +
69 + /* ---------------------------------------------------------------- env */
70 + const dbUrl = process.env.DATABASE_URL;
71 + if (dbUrl) add('env', 'DATABASE_URL', 'ok', redactUrl(dbUrl));
72 + else add('env', 'DATABASE_URL', 'warn', 'not set — using default postgres://localhost:5432/cancerindex');
73 + const raw = path.join(dataDir(), 'raw');
74 + try {
75 + mkdirSync(raw, { recursive: true });
76 + accessSync(raw, constants.W_OK);
77 + const probe = path.join(raw, `.doctor-${process.pid}`);
78 + writeFileSync(probe, 'ok');
79 + unlinkSync(probe);
80 + add('env', 'CI_DATA_DIR', 'ok', `${dataDir()} (raw lake writable)`);
81 + } catch (e) {
82 + add('env', 'CI_DATA_DIR', 'fail', `${dataDir()} not writable: ${(e as Error).message}`);
83 + }
84 + if (process.env.NCBI_EMAIL) add('env', 'NCBI_EMAIL', 'ok', process.env.NCBI_EMAIL);
85 + else add('env', 'NCBI_EMAIL', 'warn', 'absent — NCBI E-utilities (PubMed) require tool + email');
86 + for (const k of OPTIONAL_KEYS) {
87 + const v = process.env[k];
88 + if (!v) add('env', k, 'info', 'absent');
89 + else if (k === 'ADMIN_TOKEN' && (v === 'change-me' || v.length < 16)) add('env', k, 'warn', 'placeholder / too short — admin endpoints disabled or weak');
90 + else add('env', k, 'ok', `present (${v.length} chars)`);
91 + }
92 +
93 + /* ---------------------------------------------------------------- database */
94 + let reachable = false;
95 + try {
96 + const [v] = await db.execute<{ v: string; db: string }>(sql`SELECT version() AS v, current_database() AS db`);
97 + reachable = true;
98 + add('database', 'reachable', 'ok', `${v?.db} — ${(v?.v ?? '').split(' on ')[0]}`);
99 + } catch (e) {
100 + add('database', 'reachable', 'fail', (e as Error).message);
101 + }
102 + if (reachable) {
103 + const ext = await db.execute<{ name: string; installed: string | null }>(sql`SELECT name, installed_version AS installed FROM pg_available_extensions WHERE name IN ('vector','pg_trgm','unaccent')`);
104 + const byName = new Map(ext.map((r) => [r.name, r.installed]));
105 + for (const name of ['pg_trgm', 'unaccent', 'vector']) {
106 + const installed = byName.get(name);
107 + if (installed) add('database', `extension ${name}`, 'ok', `v${installed}`);
108 + else if (byName.has(name)) add('database', `extension ${name}`, name === 'vector' ? 'warn' : 'fail', 'available but not created (pnpm db:migrate creates it)');
109 + else add('database', `extension ${name}`, name === 'vector' ? 'warn' : 'fail', 'not available on this server');
110 + }
111 +
112 + // Pending migrations: journal entries newer than the last applied drizzle migration.
113 + const dir = opts.migrationsDir ?? findMigrationsDir();
114 + const journalPath = dir ? path.join(dir, 'meta', '_journal.json') : null;
115 + if (!journalPath || !existsSync(journalPath)) add('database', 'migrations', 'warn', `migrations journal not found (${dir ?? 'no folder'})`);
116 + else {
117 + const journal = JSON.parse(readFileSync(journalPath, 'utf8')) as { entries: Array<{ tag: string; when: number }> };
118 + const applied = await db
119 + .execute<{ n: string; last: string | null }>(sql`SELECT count(*)::text AS n, max(created_at)::text AS last FROM drizzle.__drizzle_migrations`)
120 + .catch(() => [] as Array<{ n: string; last: string | null }>);
121 + const row = applied[0];
122 + if (!row) add('database', 'migrations', 'fail', 'drizzle.__drizzle_migrations missing — run pnpm db:migrate');
123 + else {
124 + const last = row.last ? Number(row.last) : 0;
125 + const pending = journal.entries.filter((e) => e.when > last);
126 + if (pending.length) add('database', 'migrations', 'fail', `${pending.length} pending: ${pending.map((p) => p.tag).join(', ')} — run pnpm db:migrate`);
127 + else add('database', 'migrations', 'ok', `${row.n} applied, ${journal.entries.length} in folder, none pending`);
128 + }
129 + }
130 + const [ops] = await db.execute<{ ok: string | null }>(sql`SELECT to_regclass('public.system_alerts')::text AS ok`);
131 + if (ops?.ok) add('database', 'ops schema', 'ok', 'system_alerts present');
132 + else add('database', 'ops schema', 'fail', 'system_alerts table missing — apply the ext-ops migration (docs/schema-changes-ops.md)');
133 +
134 + /* ------------------------------------------------------------ tables */
135 + const counts = await db.execute<{ relname: string; n: string }>(sql`SELECT relname, n_live_tup::text AS n FROM pg_stat_user_tables WHERE schemaname = 'public'`);
136 + const byTable = new Map(counts.map((r) => [r.relname, Number(r.n)]));
137 + report.tables = COUNTED_TABLES.map((t) => ({ table: t, rows: byTable.get(t) ?? -1 }));
138 + for (const t of ['cancers', 'sources']) if ((byTable.get(t) ?? 0) === 0) add('data', t, 'warn', 'empty — run db:seed / cix sources:sync / ncit-evs');
139 +
140 + /* ------------------------------------------------------------ connectors */
141 + const cursors = await db.execute<{ connector_id: string; health: string; paused: boolean; last_success_at: Date | null; cursor: Record<string, unknown> }>(sql`SELECT connector_id, health, paused, last_success_at, cursor FROM connector_cursors`);
142 + const curById = new Map(cursors.map((c) => [c.connector_id, c]));
143 + const lastRuns = await db.execute<{ id: string; connector_id: string; status: string; started_at: Date; records_fetched: number; anomaly: string | null; drift: number; error: string | null }>(sql`
144 + SELECT DISTINCT ON (connector_id) id, connector_id, status, started_at, records_fetched, anomaly, jsonb_array_length(schema_drift) AS drift, error
145 + FROM ingest_runs WHERE mode NOT IN ('probe') ORDER BY connector_id, started_at DESC`);
146 + const runById = new Map(lastRuns.map((r) => [r.connector_id, r]));
147 + const srcRows = await db.execute<{ slug: string; license_status: string; status: string }>(sql`SELECT slug, license_status, status FROM sources`);
148 + const srcById = new Map(srcRows.map((s) => [s.slug, s]));
149 + for (const c of CONNECTORS) {
150 + const m = c.manifest;
151 + const cur = curById.get(m.id);
152 + const run = runById.get(m.id);
153 + const src = srcById.get(m.id);
154 + const lastSuccessAt = cur?.last_success_at ? new Date(cur.last_success_at) : null;
155 + const st = isStale(lastSuccessAt, m.schedule ?? null, now.getTime());
156 + const active = m.status === 'active' && !cur?.paused;
157 + const stale = active && !!m.schedule && st.stale;
158 + const cursorSummary = summarizeCursor(cur?.cursor ?? {});
159 + report.connectors.push({
160 + id: m.id,
161 + status: m.status,
162 + licenseStatus: m.licenseStatus,
163 + health: cur?.health ?? 'never-run',
164 + paused: !!cur?.paused,
165 + lastSuccessAt,
166 + lastSuccessAge: lastSuccessAt ? humanDuration(now.getTime() - lastSuccessAt.getTime()) : 'never',
167 + schedule: m.schedule ?? null,
168 + stale,
169 + lastRun: run ? { id: run.id, status: run.status, startedAt: new Date(run.started_at), recordsFetched: Number(run.records_fetched), anomaly: run.anomaly, drift: Number(run.drift), error: run.error } : null,
170 + cursorSummary,
171 + });
172 + if (!src) add('connectors', m.id, 'warn', 'not in sources table — run cix sources:sync');
173 + else if (src.license_status !== m.licenseStatus) add('connectors', m.id, 'warn', `sources.license_status=${src.license_status} ≠ manifest ${m.licenseStatus} — run cix sources:sync`);
174 + if (stale) add('connectors', m.id, 'warn', lastSuccessAt ? `stale: last success ${humanDuration(st.ageMs)} ago > 2× schedule (${humanDuration(st.limitMs)})` : `never succeeded (schedule ${m.schedule})`);
175 + if (run?.anomaly) add('connectors', m.id, 'warn', `last run ${run.id} flagged an anomaly: ${run.anomaly.slice(0, 160)}`);
176 + else if (run?.status === 'failed' && active) add('connectors', m.id, 'warn', `last run ${run.id} failed: ${(run.error ?? '').split('\n')[0]?.slice(0, 160)}`);
177 + if (cur?.health === 'failing' && active) add('connectors', m.id, 'warn', `health failing`);
178 + }
179 +
180 + /* ------------------------------------------------------------ curation backlog */
181 + const unresolved = await db.execute<{ slug: string; entity_kind: string; source_text: string; count: number }>(sql`
182 + SELECT s.slug, u.entity_kind, u.source_text, u.count FROM unresolved_labels u JOIN sources s ON s.id = u.source_id
183 + WHERE u.status = 'open' ORDER BY u.count DESC, u.id LIMIT 10`);
184 + report.unresolvedTop = unresolved.map((u) => ({ source: u.slug, entityKind: u.entity_kind, sourceText: u.source_text, count: Number(u.count) }));
185 + const [openTotal] = await db.execute<{ n: string }>(sql`SELECT count(*)::text AS n FROM unresolved_labels WHERE status = 'open'`);
186 + add('curation', 'unresolved labels (open)', Number(openTotal?.n ?? 0) > 5000 ? 'warn' : 'info', `${openTotal?.n ?? 0}`);
187 +
188 + /* ------------------------------------------------------------ rankings */
189 + const [rk] = await db.execute<{ snapshots: string; current: string; latest: Date | null }>(sql`SELECT count(*)::text AS snapshots, count(*) FILTER (WHERE is_current)::text AS current, max(generated_at) AS latest FROM ranking_snapshots`);
190 + const latest = rk?.latest ? new Date(rk.latest) : null;
191 + report.rankings = { snapshots: Number(rk?.snapshots ?? 0), current: Number(rk?.current ?? 0), latest };
192 + if (!latest) add('rankings', 'snapshots', 'warn', 'none — run cix counters && cix rank');
193 + else if (now.getTime() - latest.getTime() > 3 * 24 * 3600_000) add('rankings', 'snapshots', 'warn', `latest ${humanDuration(now.getTime() - latest.getTime())} ago (${report.rankings.current} current)`);
194 + else add('rankings', 'snapshots', 'ok', `latest ${humanDuration(now.getTime() - latest.getTime())} ago, ${report.rankings.current} current / ${report.rankings.snapshots} total`);
195 +
196 + /* ------------------------------------------------------------ alerts */
197 + if (ops?.ok) {
198 + report.alerts = await listAlerts(db, { status: 'active', limit: 100 });
199 + const critical = report.alerts.filter((a) => a.severity === 'critical').length;
200 + add('alerts', 'open', report.alerts.length ? (critical ? 'fail' : 'warn') : 'ok', report.alerts.length ? `${report.alerts.length} open (${critical} critical)` : 'none');
201 + }
202 + }
203 +
204 + /* ---------------------------------------------------------------- disk */
205 + if (!opts.skipDisk) {
206 + try {
207 + const usage = walkSize(raw);
208 + let freeBytes: number | null = null;
209 + try {
210 + const fsStat = await statfs(raw);
211 + freeBytes = Number(fsStat.bavail) * Number(fsStat.bsize);
212 + } catch {
213 + /* statfs unsupported */
214 + }
215 + report.disk = { rawDir: raw, bytes: usage.bytes, files: usage.files, freeBytes };
216 + const low = freeBytes !== null && freeBytes < 20 * 1024 ** 3;
217 + add('disk', 'data/raw', low ? 'warn' : 'ok', `${humanBytes(usage.bytes)} in ${usage.files} files${freeBytes !== null ? `, ${humanBytes(freeBytes)} free${low ? ' (< 20 GB)' : ''}` : ''}`);
218 + } catch (e) {
219 + add('disk', 'data/raw', 'warn', (e as Error).message);
220 + }
221 + }
222 +
223 + report.hardFailures = checks.filter((c) => c.level === 'fail').length;
224 + return report;
225 +}
226 +
227 +/** Plain-text rendering for the terminal. */
228 +export function formatDoctorReport(r: DoctorReport): string {
229 + const out: string[] = [];
230 + const tag = (l: CheckLevel) => ({ ok: '[ OK ]', info: '[INFO]', warn: '[WARN]', fail: '[FAIL]' })[l];
231 + out.push(`CancerIndex doctor — ${r.generatedAt.toISOString()}`);
232 + for (const section of ['env', 'database', 'data', 'disk', 'rankings', 'curation', 'alerts', 'connectors']) {
233 + const rows = r.checks.filter((c) => c.section === section);
234 + if (!rows.length) continue;
235 + out.push('', `## ${section}`);
236 + for (const c of rows) out.push(`${tag(c.level)} ${c.name.padEnd(28)} ${c.detail}`);
237 + }
238 + if (r.tables.length) {
239 + out.push('', '## tables (≈ live rows)');
240 + const line: string[] = [];
241 + for (const t of r.tables) line.push(`${t.table}=${t.rows < 0 ? '?' : t.rows}`);
242 + for (let i = 0; i < line.length; i += 4) out.push(' ' + line.slice(i, i + 4).map((s) => s.padEnd(34)).join('').trimEnd());
243 + }
244 + if (r.connectors.length) {
245 + out.push('', '## connectors');
246 + out.push(` ${'id'.padEnd(16)} ${'status'.padEnd(20)} ${'license'.padEnd(10)} ${'health'.padEnd(20)} ${'last ok'.padEnd(12)} ${'last run'.padEnd(34)} cursor`);
247 + for (const c of r.connectors) {
248 + const lr = c.lastRun ? `${c.lastRun.status}${c.lastRun.anomaly ? '!anomaly' : ''}${c.lastRun.drift ? ` drift=${c.lastRun.drift}` : ''} n=${c.lastRun.recordsFetched}` : '-';
249 + out.push(` ${c.id.padEnd(16)} ${(c.status + (c.paused ? ' (paused)' : '')).padEnd(20)} ${c.licenseStatus.padEnd(10)} ${(c.health + (c.stale ? ' STALE' : '')).padEnd(20)} ${c.lastSuccessAge.padEnd(12)} ${lr.padEnd(34)} ${c.cursorSummary}`);
250 + }
251 + }
252 + if (r.unresolvedTop.length) {
253 + out.push('', '## unresolved labels — top 10 (open)');
254 + for (const u of r.unresolvedTop) out.push(` ${String(u.count).padStart(6)} ${u.source.padEnd(16)} ${u.entityKind.padEnd(8)} ${u.sourceText.slice(0, 80)}`);
255 + }
256 + if (r.alerts.length) {
257 + out.push('', '## open alerts');
258 + for (const a of r.alerts) out.push(` ${a.severity.padEnd(8)} ${a.kind.padEnd(18)} ${(a.connectorId ?? '-').padEnd(16)} ×${String(a.count).padEnd(4)} ${a.lastSeenAt.toISOString().slice(0, 16)} ${a.message.slice(0, 100)}`);
259 + }
260 + out.push('', r.hardFailures ? `RESULT: ${r.hardFailures} hard failure(s)` : 'RESULT: ready');
261 + return out.join('\n');
262 +}
263 +
264 +export function formatAlerts(alerts: SystemAlert[]): string {
265 + if (!alerts.length) return 'no open alerts';
266 + const out = [`${'id'.padStart(5)} ${'severity'.padEnd(8)} ${'status'.padEnd(12)} ${'kind'.padEnd(18)} ${'connector'.padEnd(16)} ${'count'.padStart(5)} ${'first seen'.padEnd(16)} ${'last seen'.padEnd(16)} message`];
267 + for (const a of alerts) out.push(`${String(a.id).padStart(5)} ${a.severity.padEnd(8)} ${a.status.padEnd(12)} ${a.kind.padEnd(18)} ${(a.connectorId ?? '-').padEnd(16)} ${String(a.count).padStart(5)} ${a.firstSeenAt.toISOString().slice(0, 16)} ${a.lastSeenAt.toISOString().slice(0, 16)} ${a.message}`);
268 + return out.join('\n');
269 +}
270 +
271 +/* -------------------------------------------------------------------------------------------- */
272 +
273 +function findMigrationsDir(): string | null {
274 + const here = path.dirname(fileURLToPath(import.meta.url));
275 + for (const candidate of [path.resolve(here, '../../../database/migrations'), path.resolve(process.cwd(), 'packages/database/migrations')]) if (existsSync(candidate)) return candidate;
276 + return null;
277 +}
278 +
279 +function summarizeCursor(cursor: Record<string, unknown>): string {
280 + const keys = Object.keys(cursor);
281 + if (!keys.length) return '{}';
282 + const parts = keys.slice(0, 6).map((k) => {
283 + const v = cursor[k];
284 + const s = typeof v === 'string' ? (v.length > 24 ? v.slice(0, 21) + '…' : v) : typeof v === 'object' && v !== null ? '{…}' : String(v);
285 + return `${k}=${s}`;
286 + });
287 + return parts.join(' ') + (keys.length > 6 ? ` +${keys.length - 6}` : '');
288 +}
289 +
290 +function walkSize(dir: string): { bytes: number; files: number } {
291 + let bytes = 0;
292 + let files = 0;
293 + const stack = [dir];
294 + while (stack.length) {
295 + const d = stack.pop()!;
296 + let entries: string[] = [];
297 + try {
298 + entries = readdirSync(d);
299 + } catch {
300 + continue;
301 + }
302 + for (const name of entries) {
303 + const p = path.join(d, name);
304 + let st;
305 + try {
306 + st = statSync(p);
307 + } catch {
308 + continue;
309 + }
310 + if (st.isDirectory()) stack.push(p);
311 + else {
312 + bytes += st.size;
313 + files++;
314 + }
315 + }
316 + }
317 + return { bytes, files };
318 +}
319 +
320 +export function humanBytes(n: number): string {
321 + if (n < 1024) return `${n} B`;
322 + const units = ['KB', 'MB', 'GB', 'TB'];
323 + let v = n / 1024;
324 + let i = 0;
325 + while (v >= 1024 && i < units.length - 1) {
326 + v /= 1024;
327 + i++;
328 + }
329 + return `${v.toFixed(v >= 100 ? 0 : 1)} ${units[i]}`;
330 +}
331 +
332 +function redactUrl(u: string): string {
333 + return u.replace(/\/\/([^:@/]+):([^@/]+)@/, '//$1:***@');
334 +}
added packages/connectors/src/ops/index.ts +2 −0
@@ -0,0 +1,2 @@
1 +export * from './schedule.js';
2 +export * from './doctor.js';
added packages/connectors/src/ops/schedule.test.ts +50 −0
@@ -0,0 +1,50 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { humanDuration, isStale, scheduleIntervalMs } from './schedule.js';
3 +
4 +const H = 3600_000;
5 +const D = 24 * H;
6 +
7 +describe('scheduleIntervalMs', () => {
8 + it('classifies the manifest schedules', () => {
9 + expect(scheduleIntervalMs('0 2 * * *')).toBe(D); // civic daily
10 + expect(scheduleIntervalMs('30 2 * * *')).toBe(D); // clinicaltrials daily
11 + expect(scheduleIntervalMs('0 5 * * 2')).toBe(7 * D); // clinvar / hgnc weekly
12 + expect(scheduleIntervalMs('0 3 1 * *')).toBe(31 * D); // ncit-evs monthly
13 + expect(scheduleIntervalMs('0 7 1 7 *')).toBe(31 * D); // cdc-uscs yearly → treated as monthly floor
14 + expect(scheduleIntervalMs('15 * * * *')).toBe(H); // health probe hourly
15 + expect(scheduleIntervalMs('*/10 * * * *')).toBe(10 * 60_000);
16 + expect(scheduleIntervalMs('0 */6 * * *')).toBe(6 * H);
17 + expect(scheduleIntervalMs('0 2,14 * * *')).toBe(12 * H);
18 + expect(scheduleIntervalMs('0 4 * * 1,4')).toBe(3.5 * D);
19 + });
20 + it('returns null for malformed / missing expressions', () => {
21 + expect(scheduleIntervalMs(undefined)).toBeNull();
22 + expect(scheduleIntervalMs('')).toBeNull();
23 + expect(scheduleIntervalMs('0 6 15 1,7 * *')).toBeNull(); // 6 fields
24 + });
25 +});
26 +
27 +describe('isStale', () => {
28 + const now = Date.parse('2026-09-08T12:00:00Z');
29 + it('flags no success within 2× the interval', () => {
30 + expect(isStale(new Date(now - 1.5 * D), '0 2 * * *', now).stale).toBe(false);
31 + expect(isStale(new Date(now - 2.5 * D), '0 2 * * *', now).stale).toBe(true);
32 + expect(isStale(new Date(now - 10 * D), '0 5 * * 2', now).stale).toBe(false);
33 + expect(isStale(new Date(now - 15 * D), '0 5 * * 2', now).stale).toBe(true);
34 + });
35 + it('never-succeeded scheduled connectors are stale; unscheduled ones never are', () => {
36 + expect(isStale(null, '0 2 * * *', now)).toMatchObject({ stale: true, ageMs: null, limitMs: 2 * D });
37 + expect(isStale(null, undefined, now).stale).toBe(false);
38 + expect(isStale(new Date(now - 100 * D), null, now).stale).toBe(false);
39 + });
40 +});
41 +
42 +describe('humanDuration', () => {
43 + it('formats', () => {
44 + expect(humanDuration(30_000)).toBe('30 s');
45 + expect(humanDuration(5 * 60_000)).toBe('5 min');
46 + expect(humanDuration(3 * H + 5 * 60_000)).toBe('3 h 5 min');
47 + expect(humanDuration(3 * D + 2 * H)).toBe('3 d 2 h');
48 + expect(humanDuration(null)).toBe('-');
49 + });
50 +});
added packages/connectors/src/ops/schedule.ts +63 −0
@@ -0,0 +1,63 @@
1 +const MINUTE = 60_000;
2 +const HOUR = 60 * MINUTE;
3 +const DAY = 24 * HOUR;
4 +
5 +/**
6 + * Approximate period of a 5-field cron expression (minute hour day-of-month month day-of-week),
7 + * used for staleness checks ("no success within 2× the schedule interval"). Exact evaluation is
8 + * not needed: monthly → 31 d, weekly → 7 d, daily → 1 d (÷ number of listed hours), hourly → 1 h
9 + * (or N minutes for step expressions). Returns null for malformed expressions.
10 + */
11 +export function scheduleIntervalMs(cron: string | undefined | null): number | null {
12 + if (!cron) return null;
13 + const f = cron.trim().split(/\s+/);
14 + if (f.length !== 5) return null;
15 + const [minute, hour, dom, month, dow] = f as [string, string, string, string, string];
16 + if (month !== '*' || dom !== '*') return 31 * DAY;
17 + if (dow !== '*') return (7 * DAY) / Math.max(1, listCount(dow));
18 + if (hour !== '*') {
19 + const step = stepOf(hour);
20 + if (step) return step * HOUR;
21 + return DAY / Math.max(1, listCount(hour));
22 + }
23 + if (minute !== '*') {
24 + const step = stepOf(minute);
25 + if (step) return step * MINUTE;
26 + return HOUR / Math.max(1, listCount(minute));
27 + }
28 + return MINUTE;
29 +}
30 +
31 +function listCount(field: string): number {
32 + return field.split(',').reduce((n, part) => {
33 + const m = /^(\d+)-(\d+)$/.exec(part);
34 + return n + (m ? Math.max(1, Number(m[2]) - Number(m[1]) + 1) : 1);
35 + }, 0);
36 +}
37 +
38 +function stepOf(field: string): number | null {
39 + const m = /^\*\/(\d+)$/.exec(field);
40 + return m ? Math.max(1, Number(m[1])) : null;
41 +}
42 +
43 +/** Stale = no success for more than `factor` × the schedule interval (default 2×). */
44 +export function isStale(lastSuccessAt: Date | string | null | undefined, cron: string | undefined | null, now = Date.now(), factor = 2): { stale: boolean; ageMs: number | null; limitMs: number | null } {
45 + const limitMs = scheduleIntervalMs(cron);
46 + const ageMs = lastSuccessAt ? now - new Date(lastSuccessAt).getTime() : null;
47 + if (limitMs === null) return { stale: false, ageMs, limitMs: null };
48 + if (ageMs === null) return { stale: true, ageMs: null, limitMs: limitMs * factor };
49 + return { stale: ageMs > limitMs * factor, ageMs, limitMs: limitMs * factor };
50 +}
51 +
52 +/** Human duration ("3 d 4 h", "12 min"). */
53 +export function humanDuration(ms: number | null | undefined): string {
54 + if (ms === null || ms === undefined || !Number.isFinite(ms)) return '-';
55 + const s = Math.round(ms / 1000);
56 + if (s < 60) return `${s} s`;
57 + const m = Math.floor(s / 60);
58 + if (m < 60) return `${m} min`;
59 + const h = Math.floor(m / 60);
60 + if (h < 48) return `${h} h ${m % 60} min`;
61 + const d = Math.floor(h / 24);
62 + return `${d} d ${h % 24} h`;
63 +}
modified packages/connectors/src/registry.ts +5 −1
@@ -12,6 +12,10 @@ import { connector as cdcUscs } from './connectors/cdc-uscs/index.js';
12 12 import { connector as seer } from './connectors/seer/index.js';
13 13 import { connector as seerExplorer } from './connectors/seer/explorer.js';
14 14 import { connector as iarcGlobocan } from './connectors/iarc-globocan/index.js';
15 +import { connector as openfda } from './connectors/openfda/index.js';
16 +import { connector as cbioportal } from './connectors/cbioportal/index.js';
17 +import { connector as mesh } from './connectors/mesh/index.js';
18 +import { connector as chembl } from './connectors/chembl/index.js';
15 19
16 20 /**
17 21 * Connector registry. Order = recommended first-run order (CLAUDE.md §351): terminology first
@@ -19,7 +23,7 @@ import { connector as iarcGlobocan } from './connectors/iarc-globocan/index.js';
19 23 * genomics/trials/literature/variants/evidence, then epidemiology.
20 24 * Add new connectors here; `pnpm cix sources:sync` seeds their manifests into `sources`.
21 25 */
22 −export const CONNECTORS: Connector[] = [ncitEvs, oncotree, hgnc, clinicaltrials, pubmed, civic, clinvar, gdc, cdcWonder, cdcUscs, seer, seerExplorer, iarcGlobocan];
26 +export const CONNECTORS: Connector[] = [ncitEvs, oncotree, hgnc, clinicaltrials, pubmed, civic, clinvar, gdc, cdcWonder, cdcUscs, seer, seerExplorer, iarcGlobocan, mesh, chembl, openfda, cbioportal];
23 27
24 28 export function getConnector(id: string): Connector | undefined {
25 29 return CONNECTORS.find((c) => c.manifest.id === id);
added packages/connectors/src/sdk/lake.test.ts +61 −0
@@ -0,0 +1,61 @@
1 +import { mkdtempSync, rmSync } from 'node:fs';
2 +import os from 'node:os';
3 +import path from 'node:path';
4 +import { afterAll, describe, expect, it } from 'vitest';
5 +import { RawLake } from './lake.js';
6 +
7 +const root = mkdtempSync(path.join(os.tmpdir(), 'ci-lake-'));
8 +afterAll(() => rmSync(root, { recursive: true, force: true }));
9 +
10 +describe('RawLake', () => {
11 + it('serialises 5,000 concurrent writes without piling up drain listeners and reads any line back', async () => {
12 + const warnings: string[] = [];
13 + const onWarning = (w: Error) => warnings.push(`${w.name}: ${w.message}`);
14 + process.on('warning', onWarning);
15 + try {
16 + const lake = new RawLake('fake-source', 'ING-FAKE-20260908-000001', new Date('2026-09-08T00:00:00Z'), root);
17 + // Large-ish payloads so the gzip stream hits back-pressure (highWaterMark 16 KB) many times.
18 + const refs = await Promise.all(Array.from({ length: 5000 }, (_, i) => lake.put('record', { i, id: `REC-${i}`, text: 'x'.repeat(400), nested: { odd: i % 2 === 1 } })));
19 + await lake.close();
20 + expect(refs).toHaveLength(5000);
21 + expect(refs[0]).toMatch(/fake-source\/2026-09-08\/record\/ING-FAKE-20260908-000001-001\.jsonl\.gz#0$/);
22 + expect(refs[4999]!.endsWith('#4999')).toBe(true);
23 + // Same file for all refs (well below the 64 MB part limit), line numbers unique and in call order.
24 + expect(new Set(refs.map((r) => r.split('#')[0])).size).toBe(1);
25 + expect(new Set(refs).size).toBe(5000);
26 +
27 + const one = (await RawLake.read(refs[4242]!)) as { i: number; id: string };
28 + expect(one.i).toBe(4242);
29 + expect(one.id).toBe('REC-4242');
30 + const last = (await RawLake.read(refs[4999]!)) as { i: number };
31 + expect(last.i).toBe(4999);
32 + expect(await RawLake.read(`${refs[0]!.split('#')[0]}#5000`)).toBeNull();
33 + } finally {
34 + // Let any pending 'warning' events flush before asserting.
35 + await new Promise((r) => setImmediate(r));
36 + process.off('warning', onWarning);
37 + }
38 + expect(warnings.filter((w) => w.includes('MaxListenersExceededWarning'))).toEqual([]);
39 + });
40 +
41 + it('flush() makes already-written lines readable before close()', async () => {
42 + const lake = new RawLake('fake-source', 'ING-FAKE-20260908-000002', new Date('2026-09-08T00:00:00Z'), root);
43 + const refs = await Promise.all(Array.from({ length: 50 }, (_, i) => lake.put('entity', { i })));
44 + await lake.flush();
45 + // The trailer is missing until close(), but the flushed block decodes: the target line is found.
46 + const row = (await RawLake.read(refs[10]!).catch(() => null)) as { i: number } | null;
47 + expect(row?.i).toBe(10);
48 + await lake.close();
49 + expect(((await RawLake.read(refs[49]!)) as { i: number }).i).toBe(49);
50 + });
51 +
52 + it('keeps separate writers per entity', async () => {
53 + const lake = new RawLake('fake-source', 'ING-FAKE-20260908-000003', new Date('2026-09-08T00:00:00Z'), root);
54 + const [a, b] = await Promise.all([lake.put('alpha', { a: 1 }), lake.put('beta', { b: 2 })]);
55 + await lake.close();
56 + expect(a).toContain('/alpha/');
57 + expect(b).toContain('/beta/');
58 + expect(await RawLake.read(a)).toEqual({ a: 1 });
59 + expect(await RawLake.read(b)).toEqual({ b: 2 });
60 + });
61 +});
modified packages/connectors/src/sdk/lake.ts +86 −18
@@ -6,14 +6,33 @@ import { pipeline } from 'node:stream/promises';
6 6 import { createInterface } from 'node:readline';
7 7 import { dataDir } from '@cancerindex/shared';
8 8
9 +interface Writer {
10 + path: string;
11 + stream: ReturnType<typeof createWriteStream>;
12 + gz: ReturnType<typeof createGzip>;
13 + lines: number;
14 + part: number;
15 + bytes: number;
16 + /** Single pending drain promise: every writer waiting for back-pressure shares it (no listener pile-up). */
17 + drain: Promise<void> | null;
18 +}
19 +
20 +const PART_BYTES = 64 * 1024 * 1024;
21 +
9 22 /**
10 23 * Raw data lake (CLAUDE.md §26): every source payload retained when licensing allows.
11 24 * Layout: {CI_DATA_DIR}/raw/{source}/{YYYY-MM-DD}/{entity}/{runId}-{part}.jsonl.gz
12 25 * A record's `rawPath` is "<file>#<line>" so any canonical value can be traced to its raw payload.
26 + *
27 + * Writes are serialised through one promise chain: concurrent `put()` calls (e.g. `Promise.all`
28 + * over a batch) are appended in call order and back-pressure is awaited once, through a single
29 + * shared drain promise — never one `drain` listener per caller (that produced
30 + * `MaxListenersExceededWarning: 11 drain listeners added to [Gzip]`).
13 31 */
14 32 export class RawLake {
15 33 private readonly root: string;
16 − private writers = new Map<string, { path: string; stream: ReturnType<typeof createWriteStream>; gz: ReturnType<typeof createGzip>; lines: number; part: number; bytes: number }>();
34 + private writers = new Map<string, Writer>();
35 + private chain: Promise<unknown> = Promise.resolve();
17 36 constructor(
18 37 private readonly source: string,
19 38 private readonly runId: string,
@@ -28,11 +47,22 @@ export class RawLake {
28 47 return path.join(this.root, this.source, day, entity);
29 48 }
30 49
50 + /** Run `fn` after every previously queued lake operation (FIFO). */
51 + private serialize<T>(fn: () => Promise<T>): Promise<T> {
52 + const next = this.chain.then(fn, fn);
53 + this.chain = next.catch(() => undefined);
54 + return next;
55 + }
56 +
31 57 /** Append one raw JSON payload; returns its rawPath reference. */
32 − async put(entity: string, payload: unknown): Promise<string> {
58 + put(entity: string, payload: unknown): Promise<string> {
59 + return this.serialize(() => this.write(entity, payload));
60 + }
61 +
62 + private async write(entity: string, payload: unknown): Promise<string> {
33 63 let w = this.writers.get(entity);
34 − if (!w || w.bytes > 64 * 1024 * 1024) {
35 − if (w) await this.closeWriter(entity);
64 + if (!w || w.bytes > PART_BYTES) {
65 + if (w) await this.closeWriterNow(entity);
36 66 const part = (w?.part ?? 0) + 1;
37 67 const dir = this.dir(entity);
38 68 if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
@@ -40,18 +70,30 @@ export class RawLake {
40 70 const stream = createWriteStream(file);
41 71 const gz = createGzip({ level: 6 });
42 72 gz.pipe(stream);
43 − w = { path: file, stream, gz, lines: 0, part, bytes: 0 };
73 + w = { path: file, stream, gz, lines: 0, part, bytes: 0, drain: null };
44 74 this.writers.set(entity, w);
45 75 }
46 76 const line = JSON.stringify(payload) + '\n';
47 77 w.bytes += line.length;
48 78 const ref = `${w.path}#${w.lines}`;
49 79 w.lines++;
50 − if (!w.gz.write(line)) await new Promise<void>((r) => w!.gz.once('drain', () => r()));
80 + if (!w.gz.write(line)) await this.awaitDrain(w);
51 81 return ref;
52 82 }
53 83
54 − private async closeWriter(entity: string): Promise<void> {
84 + private awaitDrain(w: Writer): Promise<void> {
85 + if (!w.drain) {
86 + w.drain = new Promise<void>((resolve) => {
87 + w.gz.once('drain', () => {
88 + w.drain = null;
89 + resolve();
90 + });
91 + });
92 + }
93 + return w.drain;
94 + }
95 +
96 + private async closeWriterNow(entity: string): Promise<void> {
55 97 const w = this.writers.get(entity);
56 98 if (!w) return;
57 99 await new Promise<void>((resolve, reject) => {
@@ -62,25 +104,51 @@ export class RawLake {
62 104 this.writers.delete(entity);
63 105 }
64 106
65 − async close(): Promise<void> {
66 − for (const entity of [...this.writers.keys()]) await this.closeWriter(entity);
107 + /**
108 + * Flush compressed data written so far to disk (Z_SYNC_FLUSH) without closing the files — used on
109 + * abort so the lines already referenced by `source_records.raw_path` are readable even if the
110 + * process is killed before `close()`.
111 + */
112 + flush(): Promise<void> {
113 + return this.serialize(async () => {
114 + for (const w of this.writers.values()) await new Promise<void>((resolve) => w.gz.flush(() => resolve()));
115 + });
116 + }
117 +
118 + close(): Promise<void> {
119 + return this.serialize(async () => {
120 + for (const entity of [...this.writers.keys()]) await this.closeWriterNow(entity);
121 + });
67 122 }
68 123
69 − /** Read a raw payload back by reference (admin TRACE, reprocessing). */
124 + /**
125 + * Read a raw payload back by reference (admin TRACE, reprocessing). Tolerates a file whose gzip
126 + * trailer is missing (run killed before `close()`): lines flushed before the cut are still
127 + * returned, later ones yield null instead of an unhandled zlib error.
128 + */
70 129 static async read(ref: string): Promise<unknown | null> {
71 130 const [file, lineStr] = ref.split('#');
72 131 if (!file || lineStr === undefined) return null;
73 132 const target = Number(lineStr);
74 − const rl = createInterface({ input: createReadStream(file).pipe(createGunzip()) });
75 − let i = 0;
76 − for await (const line of rl) {
77 − if (i === target) {
78 − rl.close();
79 − return JSON.parse(line);
133 + const src = createReadStream(file);
134 + const gunzip = createGunzip();
135 + const swallow = () => gunzip.destroy();
136 + src.on('error', swallow);
137 + gunzip.on('error', swallow);
138 + src.pipe(gunzip);
139 + const rl = createInterface({ input: gunzip, crlfDelay: Infinity });
140 + try {
141 + let i = 0;
142 + for await (const line of rl) {
143 + if (i === target) return JSON.parse(line);
144 + i++;
80 145 }
81 − i++;
146 + return null;
147 + } finally {
148 + rl.close();
149 + gunzip.destroy();
150 + src.destroy();
82 151 }
83 − return null;
84 152 }
85 153 }
86 154
modified packages/connectors/src/sdk/manifest.ts +6 −0
@@ -53,6 +53,12 @@ export const ConnectorManifest = z.object({
53 53 minRatioOfPrevious: z.number().min(0).max(1).default(0.5),
54 54 })
55 55 .default({ minRatioOfPrevious: 0.5 }),
56 + /**
57 + * Mid-run cursor checkpoint (CLAUDE.md §90): `connector_cursors.cursor` is persisted after this many
58 + * upserted records (or every 60 s) whenever `ctx.cursor` changed, so a killed run resumes from the
59 + * last completed page instead of from the start of the run. See docs/connectors/README.md.
60 + */
61 + checkpointEvery: z.number().int().positive().default(2000),
56 62 });
57 63 export type ConnectorManifest = z.infer<typeof ConnectorManifest>;
58 64
added packages/connectors/src/sdk/run.test.ts +278 −0
@@ -0,0 +1,278 @@
1 +/**
2 + * Restartability / alerting tests against a real database (the SDK bookkeeping is SQL).
3 + * Gated: they run only when CI_TEST_DATABASE_URL (or DATABASE_URL) is set, so `pnpm -r test`
4 + * without Postgres still passes. Point it at a development database that has the schema pushed
5 + * (system_alerts included) — never at production:
6 + *
7 + * CI_TEST_DATABASE_URL=postgres://localhost:5432/cancerindex_a pnpm --filter @cancerindex/connectors test
8 + */
9 +import { mkdtempSync, rmSync } from 'node:fs';
10 +import os from 'node:os';
11 +import path from 'node:path';
12 +import { sql } from 'drizzle-orm';
13 +import { afterAll, beforeAll, describe, expect, it } from 'vitest';
14 +import { defineManifest } from './manifest.js';
15 +import { AnomalyError, Connector, RunContext, runConnector, type ConnectorHealth } from './run.js';
16 +
17 +const TEST_URL = process.env.CI_TEST_DATABASE_URL ?? process.env.DATABASE_URL;
18 +const ID = 'zz-fake-paged';
19 +const SOURCE_ID = 'CI-SOURCE-99999901';
20 +const tmp = mkdtempSync(path.join(os.tmpdir(), 'ci-run-'));
21 +
22 +type Db = import('@cancerindex/database').Database;
23 +let db: Db | null = null;
24 +let closeDb: (() => Promise<void>) | null = null;
25 +
26 +const manifest = defineManifest({
27 + id: ID,
28 + name: 'Fake paged source (tests)',
29 + organization: 'CancerIndex tests',
30 + category: 'terminology',
31 + tier: 11,
32 + description: 'In-memory paged source used to test cursor checkpoints, abort and anomaly guard.',
33 + homepage: 'https://example.invalid/',
34 + access: { type: 'api', auth: 'none' },
35 + license: 'CC0',
36 + licenseStatus: 'approved',
37 + commercialUse: 'allowed',
38 + redistribution: 'allowed',
39 + updateFrequency: 'test',
40 + supportsIncrementalSync: true,
41 + entities: ['record'],
42 + rateLimits: { requestsPerSecond: 1000 },
43 + documentationVerifiedAt: '2026-09-08',
44 + status: 'active',
45 + checkpointEvery: 5,
46 + anomalyGuard: { minRatioOfPrevious: 0.5 },
47 +});
48 +
49 +interface Hooks {
50 + pages: number;
51 + pageSize: number;
52 + /** Called after each page is persisted and the cursor advanced (before the next page). */
53 + afterPage?: (ctx: RunContext, page: number) => Promise<void> | void;
54 + /** Declared total handed to ctx.guardCount before the first page (undefined = no guard). */
55 + declaredTotal?: number;
56 +}
57 +
58 +/** ClinVar/ClinicalTrials-like connector: one cursor advance per page, checks shouldStop() between pages. */
59 +class FakePagedConnector extends Connector {
60 + readonly manifest = manifest;
61 + constructor(private readonly hooks: Hooks) {
62 + super();
63 + }
64 + async healthCheck(): Promise<ConnectorHealth> {
65 + return { status: 'healthy' };
66 + }
67 + async sync(ctx: RunContext): Promise<void> {
68 + const cursor = ctx.cursor as { page?: number; done?: boolean };
69 + if (this.hooks.declaredTotal !== undefined) await ctx.guardCount('record', this.hooks.declaredTotal);
70 + let page = cursor.page ?? 0;
71 + while (page < this.hooks.pages) {
72 + if (ctx.shouldStop()) return;
73 + for (let i = 0; i < this.hooks.pageSize; i++) {
74 + const n = page * this.hooks.pageSize + i;
75 + await ctx.upsertSourceRecord('record', `REC-${n}`, { n, page, v: 1 });
76 + }
77 + page++;
78 + cursor.page = page; // safe point: everything before is persisted
79 + await this.hooks.afterPage?.(ctx, page);
80 + }
81 + cursor.done = true;
82 + }
83 +}
84 +
85 +async function cursorInDb(): Promise<Record<string, unknown> | null> {
86 + const [row] = await db!.execute<{ cursor: Record<string, unknown> }>(sql`SELECT cursor FROM connector_cursors WHERE connector_id = ${ID}`);
87 + return row?.cursor ?? null;
88 +}
89 +async function run(id: string) {
90 + const [row] = await db!.execute<{ status: string; anomaly: string | null; error: string | null; cursor_before: unknown; cursor_after: unknown; records_fetched: number }>(sql`SELECT status, anomaly, error, cursor_before, cursor_after, records_fetched FROM ingest_runs WHERE id = ${id}`);
91 + return row!;
92 +}
93 +async function alerts(kind?: string) {
94 + return db!.execute<{ kind: string; severity: string; status: string; count: number; message: string }>(sql`SELECT kind, severity, status, count, message FROM system_alerts WHERE connector_id = ${ID} ${kind ? sql`AND kind = ${kind}` : sql``} ORDER BY id`);
95 +}
96 +async function cleanup() {
97 + if (!db) return;
98 + await db.execute(sql`DELETE FROM ingest_runs WHERE connector_id = ${ID}`);
99 + await db.execute(sql`DELETE FROM connector_cursors WHERE connector_id = ${ID}`);
100 + await db.execute(sql`DELETE FROM connector_field_stats WHERE connector_id = ${ID}`);
101 + await db.execute(sql`DELETE FROM system_alerts WHERE connector_id = ${ID}`);
102 + await db.execute(sql`DELETE FROM source_records WHERE source_id = ${SOURCE_ID}`);
103 + await db.execute(sql`DELETE FROM provenance WHERE source_id = ${SOURCE_ID}`);
104 +}
105 +
106 +/** Emit a real signal event on `process`, delivered only to listeners added since `baseline` (i.e. the run's handler). */
107 +function emitSignal(signal: NodeJS.Signals, baseline: Set<unknown>) {
108 + const others = process.rawListeners(signal).filter((l) => baseline.has(l));
109 + for (const l of others) process.off(signal, l as never);
110 + try {
111 + process.emit(signal, signal);
112 + } finally {
113 + for (const l of others) process.on(signal, l as never);
114 + }
115 +}
116 +
117 +beforeAll(async () => {
118 + if (!TEST_URL) return;
119 + process.env.CI_DATA_DIR = tmp;
120 + const mod = await import('@cancerindex/database');
121 + db = mod.getDb({ url: TEST_URL, max: 4 });
122 + closeDb = mod.closeDb;
123 + await db.execute(sql`SELECT 1`);
124 + await db.execute(sql`SELECT 1 FROM system_alerts LIMIT 1`); // ops schema must be pushed
125 + await db.execute(sql`INSERT INTO sources (id, slug, name, category, access_type, access_auth, license_status, status, manifest)
126 + VALUES (${SOURCE_ID}, ${ID}, 'Fake paged source (tests)', 'terminology', 'api', 'none', 'approved', 'active', '{}'::jsonb)
127 + ON CONFLICT (id) DO UPDATE SET slug = EXCLUDED.slug`);
128 + await cleanup();
129 +}, 30_000);
130 +
131 +afterAll(async () => {
132 + await cleanup();
133 + if (db) await db.execute(sql`DELETE FROM sources WHERE id = ${SOURCE_ID}`);
134 + await closeDb?.();
135 + rmSync(tmp, { recursive: true, force: true });
136 +});
137 +
138 +describe.skipIf(!TEST_URL)('runConnector restartability (database-backed)', () => {
139 + it('checkpoints the cursor mid-run every checkpointEvery records when it changed', async () => {
140 + const seen: Array<{ page: number; saved: unknown }> = [];
141 + const c = new FakePagedConnector({
142 + pages: 4,
143 + pageSize: 5, // = checkpointEvery → after each page's 5th record the cursor of the *previous* page is saved
144 + afterPage: async (_ctx, page) => {
145 + seen.push({ page, saved: (await cursorInDb())?.page ?? null });
146 + },
147 + });
148 + const r = await runConnector(db!, c, { maxMinutes: 5, handleSignals: false });
149 + expect(r.status).toBe('succeeded');
150 + expect(r.counters.fetched).toBe(20);
151 + // Page k+1's checkpoint (fires on its 5th record) persisted cursor {page: k}; page 1 has nothing changed yet.
152 + expect(seen.map((s) => s.saved)).toEqual([null, 1, 2, 3]);
153 + expect(await cursorInDb()).toMatchObject({ page: 4, done: true });
154 + const row = await run(r.runId);
155 + expect(row.status).toBe('succeeded');
156 + expect(row.cursor_after).toMatchObject({ page: 4, done: true });
157 + expect((await alerts()).filter((a) => a.status !== 'resolved')).toEqual([]);
158 + });
159 +
160 + it('saveCursor() persists immediately and dry runs never touch the cursor', async () => {
161 + let inside: unknown = null;
162 + const c = new FakePagedConnector({
163 + pages: 2,
164 + pageSize: 2,
165 + afterPage: async (ctx, page) => {
166 + if (page === 1) {
167 + await ctx.saveCursor();
168 + inside = (await cursorInDb())?.page;
169 + }
170 + },
171 + });
172 + const before = await cursorInDb();
173 + expect(before).toMatchObject({ page: 4 }); // left by the previous test
174 + const dry = await runConnector(db!, c, { maxMinutes: 5, handleSignals: false, mode: 'dry_run', resetCursor: true });
175 + expect(dry.status).toBe('succeeded');
176 + expect(inside).toBe(4); // dry_run: saveCursor is a no-op, the stored cursor is still the previous run's
177 + expect(await cursorInDb()).toEqual(before);
178 + const real = await runConnector(db!, c, { maxMinutes: 5, handleSignals: false, resetCursor: true });
179 + expect(real.status).toBe('succeeded');
180 + expect(inside).toBe(1);
181 + });
182 +
183 + it('SIGTERM mid-run: run marked aborted, cursor kept at the last completed page, next run resumes there', async () => {
184 + let listenersDuring = 0;
185 + const baselineListeners = new Set(process.rawListeners('SIGTERM'));
186 + const baseline = baselineListeners.size;
187 + const c = new FakePagedConnector({
188 + pages: 6,
189 + pageSize: 3,
190 + afterPage: (ctx, page) => {
191 + if (page === 3) {
192 + listenersDuring = process.listenerCount('SIGTERM');
193 + emitSignal('SIGTERM', baselineListeners); // deliver only to the run's handler
194 + expect(ctx.abortReason).toBe('SIGTERM received');
195 + expect(ctx.shouldStop()).toBe(true);
196 + }
197 + },
198 + });
199 + const r = await runConnector(db!, c, { maxMinutes: 5, resetCursor: true });
200 + expect(r.status).toBe('aborted');
201 + expect(r.counters.fetched).toBe(9); // 3 pages × 3 records, nothing after the signal
202 + expect(listenersDuring).toBe(baseline + 1);
203 + expect(process.listenerCount('SIGTERM')).toBe(baseline); // handlers removed at the end of the run
204 + const row = await run(r.runId);
205 + expect(row.status).toBe('aborted');
206 + expect(row.error).toContain('SIGTERM');
207 + expect(row.cursor_after).toMatchObject({ page: 3 });
208 + expect(await cursorInDb()).toMatchObject({ page: 3 });
209 + const aborted = await alerts('connector_aborted');
210 + expect(aborted).toHaveLength(1);
211 + expect(aborted[0]).toMatchObject({ severity: 'info', status: 'open' });
212 +
213 + // Resume: starts from page 3, finishes the remaining 3 pages, success resolves the alert.
214 + const c2 = new FakePagedConnector({ pages: 6, pageSize: 3 });
215 + const r2 = await runConnector(db!, c2, { maxMinutes: 5, handleSignals: false });
216 + expect(r2.status).toBe('succeeded');
217 + expect(r2.counters.fetched).toBe(9);
218 + const row2 = await run(r2.runId);
219 + expect(row2.cursor_before).toMatchObject({ page: 3 });
220 + expect(row2.cursor_after).toMatchObject({ page: 6, done: true });
221 + expect((await alerts('connector_aborted'))[0]!.status).toBe('resolved');
222 + // Idempotent hand-over: the 9 records of the aborted run were not re-fetched, the 9 of the resume are new to it, no duplicates.
223 + const touched = await db!.execute<{ run: string; n: string; distinct_ids: string }>(sql`SELECT last_seen_run AS run, count(*)::text AS n, count(DISTINCT source_record_id)::text AS distinct_ids FROM source_records WHERE source_id = ${SOURCE_ID} AND last_seen_run IN (${r.runId}, ${r2.runId}) GROUP BY 1 ORDER BY 1`);
224 + expect(touched.map((t) => [t.run, Number(t.n), Number(t.distinct_ids)])).toEqual([
225 + [r.runId, 9, 9],
226 + [r2.runId, 9, 9],
227 + ]);
228 + });
229 +
230 + it('guardCount() refuses a shrunken total: run failed with anomaly, critical alert, previous data untouched', async () => {
231 + // Baseline: a successful run that fetched 20 records.
232 + const ok = await runConnector(db!, new FakePagedConnector({ pages: 4, pageSize: 5 }), { maxMinutes: 5, handleSignals: false, resetCursor: true });
233 + expect(ok.status).toBe('succeeded');
234 + expect((await run(ok.runId)).records_fetched).toBe(20);
235 +
236 + const bad = await runConnector(db!, new FakePagedConnector({ pages: 1, pageSize: 5, declaredTotal: 5 }), { maxMinutes: 5, handleSignals: false, resetCursor: true });
237 + expect(bad.status).toBe('failed');
238 + expect(bad.anomaly).toMatch(/^anomaly: record count 5 is 25\.0% of the previous successful run/);
239 + expect(bad.counters.fetched).toBe(0); // refused before fetching anything
240 + const row = await run(bad.runId);
241 + expect(row.status).toBe('failed');
242 + expect(row.anomaly).toContain('CLAUDE.md §171');
243 + const crit = await alerts('anomaly');
244 + expect(crit).toHaveLength(1);
245 + expect(crit[0]).toMatchObject({ severity: 'critical', status: 'open', count: 1 });
246 + const active = await db!.execute<{ n: string }>(sql`SELECT count(*)::text AS n FROM source_records WHERE source_id = ${SOURCE_ID} AND status = 'active'`);
247 + expect(Number(active[0]!.n)).toBe(20);
248 +
249 + // Same anomaly again → deduplicated (count 2); an acceptable total passes and resolves it.
250 + const again = await runConnector(db!, new FakePagedConnector({ pages: 1, pageSize: 5, declaredTotal: 5 }), { maxMinutes: 5, handleSignals: false, resetCursor: true });
251 + expect(again.status).toBe('failed');
252 + expect((await alerts('anomaly'))[0]!.count).toBe(2);
253 + const fine = await runConnector(db!, new FakePagedConnector({ pages: 4, pageSize: 5, declaredTotal: 18 }), { maxMinutes: 5, handleSignals: false, resetCursor: true });
254 + expect(fine.status).toBe('succeeded');
255 + expect((await alerts('anomaly'))[0]!.status).toBe('resolved');
256 + });
257 +
258 + it('guardCount() throws AnomalyError directly when called on a bare context', async () => {
259 + const ctx = new RunContext(db!, manifest, SOURCE_ID, 'incremental', { maxMinutes: 1 }, 99);
260 + await expect(ctx.guardCount('record', 1)).rejects.toBeInstanceOf(AnomalyError);
261 + expect(ctx.anomaly).toContain('anomaly: record count 1');
262 + const okRes = await ctx.guardCount('record', 15, { minRatio: 0.7 });
263 + expect(okRes.previous).toBe(20);
264 + expect(okRes.ratio).toBeCloseTo(0.75);
265 + });
266 +
267 + it('cix doctor report lists the connectors and reports no hard failure on a healthy dev database', async () => {
268 + const { runDoctor, formatDoctorReport } = await import('../ops/doctor.js');
269 + const report = await runDoctor(db!, { skipDisk: true });
270 + expect(report.checks.find((c) => c.name === 'reachable')?.level).toBe('ok');
271 + expect(report.checks.find((c) => c.name === 'ops schema')?.level).toBe('ok');
272 + expect(report.connectors.length).toBeGreaterThan(5);
273 + expect(report.tables.find((t) => t.table === 'system_alerts')).toBeTruthy();
274 + const text = formatDoctorReport(report);
275 + expect(text).toContain('## connectors');
276 + expect(text).toMatch(/RESULT: (ready|\d+ hard failure)/);
277 + });
278 +});
modified packages/connectors/src/sdk/run.ts +0 −0

Binary file not shown.

added packages/database/migrations/0001_jazzy_arclight.sql +16 −0
@@ -0,0 +1,16 @@
1 +CREATE TABLE "system_alerts" (
2 + "id" bigserial PRIMARY KEY NOT NULL,
3 + "kind" text NOT NULL,
4 + "severity" text DEFAULT 'warn' NOT NULL,
5 + "connector_id" text,
6 + "message" text NOT NULL,
7 + "detail" jsonb DEFAULT '{}'::jsonb NOT NULL,
8 + "first_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
9 + "last_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
10 + "count" integer DEFAULT 1 NOT NULL,
11 + "status" text DEFAULT 'open' NOT NULL,
12 + "resolved_at" timestamp with time zone
13 +);
14 +--> statement-breakpoint
15 +CREATE INDEX "system_alerts_status_idx" ON "system_alerts" USING btree ("status","kind","connector_id");--> statement-breakpoint
16 +CREATE INDEX "system_alerts_seen_idx" ON "system_alerts" USING btree ("last_seen_at");
\ No newline at end of file
added packages/database/migrations/meta/0001_snapshot.json +2149 −0
@@ -0,0 +1,7046 @@
1 +{
2 + "id": "dc7abeda-b455-4b0f-86ae-e01f476fb4f0",
3 + "prevId": "7cee03f3-3bb2-47c2-b164-ea15161ffedd",
4 + "version": "7",
5 + "dialect": "postgresql",
6 + "tables": {
7 + "public.audit_log": {
8 + "name": "audit_log",
9 + "schema": "",
10 + "columns": {
11 + "id": {
12 + "name": "id",
13 + "type": "bigserial",
14 + "primaryKey": true,
15 + "notNull": true
16 + },
17 + "actor": {
18 + "name": "actor",
19 + "type": "text",
20 + "primaryKey": false,
21 + "notNull": true
22 + },
23 + "action": {
24 + "name": "action",
25 + "type": "text",
26 + "primaryKey": false,
27 + "notNull": true
28 + },
29 + "entity_type": {
30 + "name": "entity_type",
31 + "type": "text",
32 + "primaryKey": false,
33 + "notNull": false
34 + },
35 + "entity_id": {
36 + "name": "entity_id",
37 + "type": "text",
38 + "primaryKey": false,
39 + "notNull": false
40 + },
41 + "before": {
42 + "name": "before",
43 + "type": "jsonb",
44 + "primaryKey": false,
45 + "notNull": false
46 + },
47 + "after": {
48 + "name": "after",
49 + "type": "jsonb",
50 + "primaryKey": false,
51 + "notNull": false
52 + },
53 + "reason": {
54 + "name": "reason",
55 + "type": "text",
56 + "primaryKey": false,
57 + "notNull": false
58 + },
59 + "created_at": {
60 + "name": "created_at",
61 + "type": "timestamp with time zone",
62 + "primaryKey": false,
63 + "notNull": true,
64 + "default": "now()"
65 + }
66 + },
67 + "indexes": {},
68 + "foreignKeys": {},
69 + "compositePrimaryKeys": {},
70 + "uniqueConstraints": {},
71 + "policies": {},
72 + "checkConstraints": {},
73 + "isRLSEnabled": false
74 + },
75 + "public.change_events": {
76 + "name": "change_events",
77 + "schema": "",
78 + "columns": {
79 + "id": {
80 + "name": "id",
81 + "type": "bigserial",
82 + "primaryKey": true,
83 + "notNull": true
84 + },
85 + "entity_type": {
86 + "name": "entity_type",
87 + "type": "text",
88 + "primaryKey": false,
89 + "notNull": true
90 + },
91 + "entity_id": {
92 + "name": "entity_id",
93 + "type": "text",
94 + "primaryKey": false,
95 + "notNull": true
96 + },
97 + "kind": {
98 + "name": "kind",
99 + "type": "text",
100 + "primaryKey": false,
101 + "notNull": true
102 + },
103 + "summary": {
104 + "name": "summary",
105 + "type": "text",
106 + "primaryKey": false,
107 + "notNull": true
108 + },
109 + "before": {
110 + "name": "before",
111 + "type": "jsonb",
112 + "primaryKey": false,
113 + "notNull": false
114 + },
115 + "after": {
116 + "name": "after",
117 + "type": "jsonb",
118 + "primaryKey": false,
119 + "notNull": false
120 + },
121 + "ingest_run_id": {
122 + "name": "ingest_run_id",
123 + "type": "text",
124 + "primaryKey": false,
125 + "notNull": false
126 + },
127 + "created_at": {
128 + "name": "created_at",
129 + "type": "timestamp with time zone",
130 + "primaryKey": false,
131 + "notNull": true,
132 + "default": "now()"
133 + }
134 + },
135 + "indexes": {
136 + "change_events_entity_idx": {
137 + "name": "change_events_entity_idx",
138 + "columns": [
139 + {
140 + "expression": "entity_type",
141 + "isExpression": false,
142 + "asc": true,
143 + "nulls": "last"
144 + },
145 + {
146 + "expression": "entity_id",
147 + "isExpression": false,
148 + "asc": true,
149 + "nulls": "last"
150 + },
151 + {
152 + "expression": "created_at",
153 + "isExpression": false,
154 + "asc": true,
155 + "nulls": "last"
156 + }
157 + ],
158 + "isUnique": false,
159 + "concurrently": false,
160 + "method": "btree",
161 + "with": {}
162 + }
163 + },
164 + "foreignKeys": {},
165 + "compositePrimaryKeys": {},
166 + "uniqueConstraints": {},
167 + "policies": {},
168 + "checkConstraints": {},
169 + "isRLSEnabled": false
170 + },
171 + "public.connector_cursors": {
172 + "name": "connector_cursors",
173 + "schema": "",
174 + "columns": {
175 + "connector_id": {
176 + "name": "connector_id",
177 + "type": "text",
178 + "primaryKey": true,
179 + "notNull": true
180 + },
181 + "cursor": {
182 + "name": "cursor",
183 + "type": "jsonb",
184 + "primaryKey": false,
185 + "notNull": true,
186 + "default": "'{}'::jsonb"
187 + },
188 + "last_success_at": {
189 + "name": "last_success_at",
190 + "type": "timestamp with time zone",
191 + "primaryKey": false,
192 + "notNull": false
193 + },
194 + "last_attempt_at": {
195 + "name": "last_attempt_at",
196 + "type": "timestamp with time zone",
197 + "primaryKey": false,
198 + "notNull": false
199 + },
200 + "paused": {
201 + "name": "paused",
202 + "type": "boolean",
203 + "primaryKey": false,
204 + "notNull": true,
205 + "default": false
206 + },
207 + "health": {
208 + "name": "health",
209 + "type": "text",
210 + "primaryKey": false,
211 + "notNull": true,
212 + "default": "'unknown'"
213 + },
214 + "health_detail": {
215 + "name": "health_detail",
216 + "type": "text",
217 + "primaryKey": false,
218 + "notNull": false
219 + },
220 + "updated_at": {
221 + "name": "updated_at",
222 + "type": "timestamp with time zone",
223 + "primaryKey": false,
224 + "notNull": true,
225 + "default": "now()"
226 + }
227 + },
228 + "indexes": {},
229 + "foreignKeys": {},
230 + "compositePrimaryKeys": {},
231 + "uniqueConstraints": {},
232 + "policies": {},
233 + "checkConstraints": {},
234 + "isRLSEnabled": false
235 + },
236 + "public.connector_field_stats": {
237 + "name": "connector_field_stats",
238 + "schema": "",
239 + "columns": {
240 + "id": {
241 + "name": "id",
242 + "type": "bigserial",
243 + "primaryKey": true,
244 + "notNull": true
245 + },
246 + "connector_id": {
247 + "name": "connector_id",
248 + "type": "text",
249 + "primaryKey": false,
250 + "notNull": true
251 + },
252 + "entity": {
253 + "name": "entity",
254 + "type": "text",
255 + "primaryKey": false,
256 + "notNull": true
257 + },
258 + "field": {
259 + "name": "field",
260 + "type": "text",
261 + "primaryKey": false,
262 + "notNull": true
263 + },
264 + "types": {
265 + "name": "types",
266 + "type": "text[]",
267 + "primaryKey": false,
268 + "notNull": true,
269 + "default": "'{}'"
270 + },
271 + "seen_count": {
272 + "name": "seen_count",
273 + "type": "integer",
274 + "primaryKey": false,
275 + "notNull": true,
276 + "default": 0
277 + },
278 + "null_count": {
279 + "name": "null_count",
280 + "type": "integer",
281 + "primaryKey": false,
282 + "notNull": true,
283 + "default": 0
284 + },
285 + "first_seen_run": {
286 + "name": "first_seen_run",
287 + "type": "text",
288 + "primaryKey": false,
289 + "notNull": false
290 + },
291 + "last_seen_run": {
292 + "name": "last_seen_run",
293 + "type": "text",
294 + "primaryKey": false,
295 + "notNull": false
296 + },
297 + "updated_at": {
298 + "name": "updated_at",
299 + "type": "timestamp with time zone",
300 + "primaryKey": false,
301 + "notNull": true,
302 + "default": "now()"
303 + }
304 + },
305 + "indexes": {
306 + "connector_field_stats_uq": {
307 + "name": "connector_field_stats_uq",
308 + "columns": [
309 + {
310 + "expression": "connector_id",
311 + "isExpression": false,
312 + "asc": true,
313 + "nulls": "last"
314 + },
315 + {
316 + "expression": "entity",
317 + "isExpression": false,
318 + "asc": true,
319 + "nulls": "last"
320 + },
321 + {
322 + "expression": "field",
323 + "isExpression": false,
324 + "asc": true,
325 + "nulls": "last"
326 + }
327 + ],
328 + "isUnique": true,
329 + "concurrently": false,
330 + "method": "btree",
331 + "with": {}
332 + }
333 + },
334 + "foreignKeys": {},
335 + "compositePrimaryKeys": {},
336 + "uniqueConstraints": {},
337 + "policies": {},
338 + "checkConstraints": {},
339 + "isRLSEnabled": false
340 + },
341 + "public.entity_merges": {
342 + "name": "entity_merges",
343 + "schema": "",
344 + "columns": {
345 + "id": {
346 + "name": "id",
347 + "type": "bigserial",
348 + "primaryKey": true,
349 + "notNull": true
350 + },
351 + "entity_type": {
352 + "name": "entity_type",
353 + "type": "text",
354 + "primaryKey": false,
355 + "notNull": true
356 + },
357 + "keep_id": {
358 + "name": "keep_id",
359 + "type": "text",
360 + "primaryKey": false,
361 + "notNull": true
362 + },
363 + "merge_id": {
364 + "name": "merge_id",
365 + "type": "text",
366 + "primaryKey": false,
367 + "notNull": true
368 + },
369 + "evidence": {
370 + "name": "evidence",
371 + "type": "jsonb",
372 + "primaryKey": false,
373 + "notNull": true,
374 + "default": "'{}'::jsonb"
375 + },
376 + "status": {
377 + "name": "status",
378 + "type": "text",
379 + "primaryKey": false,
380 + "notNull": true,
381 + "default": "'proposed'"
382 + },
383 + "decided_by": {
384 + "name": "decided_by",
385 + "type": "text",
386 + "primaryKey": false,
387 + "notNull": false
388 + },
389 + "decided_at": {
390 + "name": "decided_at",
391 + "type": "timestamp with time zone",
392 + "primaryKey": false,
393 + "notNull": false
394 + },
395 + "created_at": {
396 + "name": "created_at",
397 + "type": "timestamp with time zone",
398 + "primaryKey": false,
399 + "notNull": true,
400 + "default": "now()"
401 + }
402 + },
403 + "indexes": {},
404 + "foreignKeys": {},
405 + "compositePrimaryKeys": {},
406 + "uniqueConstraints": {},
407 + "policies": {},
408 + "checkConstraints": {},
409 + "isRLSEnabled": false
410 + },
411 + "public.id_sequences": {
412 + "name": "id_sequences",
413 + "schema": "",
414 + "columns": {
415 + "namespace": {
416 + "name": "namespace",
417 + "type": "varchar(16)",
418 + "primaryKey": true,
419 + "notNull": true
420 + },
421 + "next": {
422 + "name": "next",
423 + "type": "bigint",
424 + "primaryKey": false,
425 + "notNull": true,
426 + "default": 1
427 + }
428 + },
429 + "indexes": {},
430 + "foreignKeys": {},
431 + "compositePrimaryKeys": {},
432 + "uniqueConstraints": {},
433 + "policies": {},
434 + "checkConstraints": {},
435 + "isRLSEnabled": false
436 + },
437 + "public.ingest_runs": {
438 + "name": "ingest_runs",
439 + "schema": "",
440 + "columns": {
441 + "id": {
442 + "name": "id",
443 + "type": "text",
444 + "primaryKey": true,
445 + "notNull": true
446 + },
447 + "connector_id": {
448 + "name": "connector_id",
449 + "type": "text",
450 + "primaryKey": false,
451 + "notNull": true
452 + },
453 + "source_id": {
454 + "name": "source_id",
455 + "type": "varchar(32)",
456 + "primaryKey": false,
457 + "notNull": true
458 + },
459 + "mode": {
460 + "name": "mode",
461 + "type": "text",
462 + "primaryKey": false,
463 + "notNull": true,
464 + "default": "'incremental'"
465 + },
466 + "status": {
467 + "name": "status",
468 + "type": "text",
469 + "primaryKey": false,
470 + "notNull": true,
471 + "default": "'running'"
472 + },
473 + "started_at": {
474 + "name": "started_at",
475 + "type": "timestamp with time zone",
476 + "primaryKey": false,
477 + "notNull": true,
478 + "default": "now()"
479 + },
480 + "finished_at": {
481 + "name": "finished_at",
482 + "type": "timestamp with time zone",
483 + "primaryKey": false,
484 + "notNull": false
485 + },
486 + "duration_ms": {
487 + "name": "duration_ms",
488 + "type": "integer",
489 + "primaryKey": false,
490 + "notNull": false
491 + },
492 + "records_fetched": {
493 + "name": "records_fetched",
494 + "type": "integer",
495 + "primaryKey": false,
496 + "notNull": true,
497 + "default": 0
498 + },
499 + "records_created": {
500 + "name": "records_created",
501 + "type": "integer",
502 + "primaryKey": false,
503 + "notNull": true,
504 + "default": 0
505 + },
506 + "records_updated": {
507 + "name": "records_updated",
508 + "type": "integer",
509 + "primaryKey": false,
510 + "notNull": true,
511 + "default": 0
512 + },
513 + "records_unchanged": {
514 + "name": "records_unchanged",
515 + "type": "integer",
516 + "primaryKey": false,
517 + "notNull": true,
518 + "default": 0
519 + },
520 + "records_rejected": {
521 + "name": "records_rejected",
522 + "type": "integer",
523 + "primaryKey": false,
524 + "notNull": true,
525 + "default": 0
526 + },
527 + "http_requests": {
528 + "name": "http_requests",
529 + "type": "integer",
530 + "primaryKey": false,
531 + "notNull": true,
532 + "default": 0
533 + },
534 + "http_failures": {
535 + "name": "http_failures",
536 + "type": "integer",
537 + "primaryKey": false,
538 + "notNull": true,
539 + "default": 0
540 + },
541 + "rate_limit_events": {
542 + "name": "rate_limit_events",
543 + "type": "integer",
544 + "primaryKey": false,
545 + "notNull": true,
546 + "default": 0
547 + },
548 + "validation_failures": {
549 + "name": "validation_failures",
550 + "type": "integer",
551 + "primaryKey": false,
552 + "notNull": true,
553 + "default": 0
554 + },
555 + "schema_drift": {
556 + "name": "schema_drift",
557 + "type": "jsonb",
558 + "primaryKey": false,
559 + "notNull": true,
560 + "default": "'[]'::jsonb"
561 + },
562 + "cursor_before": {
563 + "name": "cursor_before",
564 + "type": "jsonb",
565 + "primaryKey": false,
566 + "notNull": false
567 + },
568 + "cursor_after": {
569 + "name": "cursor_after",
570 + "type": "jsonb",
571 + "primaryKey": false,
572 + "notNull": false
573 + },
574 + "error": {
575 + "name": "error",
576 + "type": "text",
577 + "primaryKey": false,
578 + "notNull": false
579 + },
580 + "log": {
581 + "name": "log",
582 + "type": "jsonb",
583 + "primaryKey": false,
584 + "notNull": true,
585 + "default": "'[]'::jsonb"
586 + },
587 + "dataset_version": {
588 + "name": "dataset_version",
589 + "type": "text",
590 + "primaryKey": false,
591 + "notNull": false
592 + },
593 + "anomaly": {
594 + "name": "anomaly",
595 + "type": "text",
596 + "primaryKey": false,
597 + "notNull": false
598 + }
599 + },
600 + "indexes": {
601 + "ingest_runs_connector_idx": {
602 + "name": "ingest_runs_connector_idx",
603 + "columns": [
604 + {
605 + "expression": "connector_id",
606 + "isExpression": false,
607 + "asc": true,
608 + "nulls": "last"
609 + },
610 + {
611 + "expression": "started_at",
612 + "isExpression": false,
613 + "asc": true,
614 + "nulls": "last"
615 + }
616 + ],
617 + "isUnique": false,
618 + "concurrently": false,
619 + "method": "btree",
620 + "with": {}
621 + }
622 + },
623 + "foreignKeys": {},
624 + "compositePrimaryKeys": {},
625 + "uniqueConstraints": {},
626 + "policies": {},
627 + "checkConstraints": {},
628 + "isRLSEnabled": false
629 + },
630 + "public.provenance": {
631 + "name": "provenance",
632 + "schema": "",
633 + "columns": {
634 + "id": {
635 + "name": "id",
636 + "type": "bigserial",
637 + "primaryKey": true,
638 + "notNull": true
639 + },
640 + "public_id": {
641 + "name": "public_id",
642 + "type": "varchar(32)",
643 + "primaryKey": false,
644 + "notNull": false
645 + },
646 + "source_id": {
647 + "name": "source_id",
648 + "type": "varchar(32)",
649 + "primaryKey": false,
650 + "notNull": true
651 + },
652 + "source_record_id": {
653 + "name": "source_record_id",
654 + "type": "text",
655 + "primaryKey": false,
656 + "notNull": false
657 + },
658 + "source_url": {
659 + "name": "source_url",
660 + "type": "text",
661 + "primaryKey": false,
662 + "notNull": false
663 + },
664 + "dataset": {
665 + "name": "dataset",
666 + "type": "text",
667 + "primaryKey": false,
668 + "notNull": false
669 + },
670 + "dataset_version": {
671 + "name": "dataset_version",
672 + "type": "text",
673 + "primaryKey": false,
674 + "notNull": false
675 + },
676 + "publication_id": {
677 + "name": "publication_id",
678 + "type": "varchar(32)",
679 + "primaryKey": false,
680 + "notNull": false
681 + },
682 + "pmid": {
683 + "name": "pmid",
684 + "type": "text",
685 + "primaryKey": false,
686 + "notNull": false
687 + },
688 + "doi": {
689 + "name": "doi",
690 + "type": "text",
691 + "primaryKey": false,
692 + "notNull": false
693 + },
694 + "retrieved_at": {
695 + "name": "retrieved_at",
696 + "type": "timestamp with time zone",
697 + "primaryKey": false,
698 + "notNull": true
699 + },
700 + "published_at": {
701 + "name": "published_at",
702 + "type": "text",
703 + "primaryKey": false,
704 + "notNull": false
705 + },
706 + "updated_at_source": {
707 + "name": "updated_at_source",
708 + "type": "text",
709 + "primaryKey": false,
710 + "notNull": false
711 + },
712 + "geography": {
713 + "name": "geography",
714 + "type": "text",
715 + "primaryKey": false,
716 + "notNull": false
717 + },
718 + "population": {
719 + "name": "population",
720 + "type": "text",
721 + "primaryKey": false,
722 + "notNull": false
723 + },
724 + "cohort_size": {
725 + "name": "cohort_size",
726 + "type": "integer",
727 + "primaryKey": false,
728 + "notNull": false
729 + },
730 + "methodology": {
731 + "name": "methodology",
732 + "type": "text",
733 + "primaryKey": false,
734 + "notNull": false
735 + },
736 + "evidence_type": {
737 + "name": "evidence_type",
738 + "type": "text",
739 + "primaryKey": false,
740 + "notNull": true
741 + },
742 + "access_level": {
743 + "name": "access_level",
744 + "type": "text",
745 + "primaryKey": false,
746 + "notNull": true,
747 + "default": "'open'"
748 + },
749 + "confidence": {
750 + "name": "confidence",
751 + "type": "real",
752 + "primaryKey": false,
753 + "notNull": false
754 + },
755 + "license": {
756 + "name": "license",
757 + "type": "text",
758 + "primaryKey": false,
759 + "notNull": false
760 + },
761 + "ingest_run_id": {
762 + "name": "ingest_run_id",
763 + "type": "text",
764 + "primaryKey": false,
765 + "notNull": false
766 + },
767 + "created_at": {
768 + "name": "created_at",
769 + "type": "timestamp with time zone",
770 + "primaryKey": false,
771 + "notNull": true,
772 + "default": "now()"
773 + }
774 + },
775 + "indexes": {
776 + "provenance_source_idx": {
777 + "name": "provenance_source_idx",
778 + "columns": [
779 + {
780 + "expression": "source_id",
781 + "isExpression": false,
782 + "asc": true,
783 + "nulls": "last"
784 + },
785 + {
786 + "expression": "source_record_id",
787 + "isExpression": false,
788 + "asc": true,
789 + "nulls": "last"
790 + }
791 + ],
792 + "isUnique": false,
793 + "concurrently": false,
794 + "method": "btree",
795 + "with": {}
796 + },
797 + "provenance_pmid_idx": {
798 + "name": "provenance_pmid_idx",
799 + "columns": [
800 + {
801 + "expression": "pmid",
802 + "isExpression": false,
803 + "asc": true,
804 + "nulls": "last"
805 + }
806 + ],
807 + "isUnique": false,
808 + "concurrently": false,
809 + "method": "btree",
810 + "with": {}
811 + }
812 + },
813 + "foreignKeys": {},
814 + "compositePrimaryKeys": {},
815 + "uniqueConstraints": {},
816 + "policies": {},
817 + "checkConstraints": {},
818 + "isRLSEnabled": false
819 + },
820 + "public.source_records": {
821 + "name": "source_records",
822 + "schema": "",
823 + "columns": {
824 + "id": {
825 + "name": "id",
826 + "type": "bigserial",
827 + "primaryKey": true,
828 + "notNull": true
829 + },
830 + "source_id": {
831 + "name": "source_id",
832 + "type": "varchar(32)",
833 + "primaryKey": false,
834 + "notNull": true
835 + },
836 + "entity_kind": {
837 + "name": "entity_kind",
838 + "type": "text",
839 + "primaryKey": false,
840 + "notNull": true
841 + },
842 + "source_record_id": {
843 + "name": "source_record_id",
844 + "type": "text",
845 + "primaryKey": false,
846 + "notNull": true
847 + },
848 + "payload_hash": {
849 + "name": "payload_hash",
850 + "type": "text",
851 + "primaryKey": false,
852 + "notNull": true
853 + },
854 + "raw_path": {
855 + "name": "raw_path",
856 + "type": "text",
857 + "primaryKey": false,
858 + "notNull": false
859 + },
860 + "status": {
861 + "name": "status",
862 + "type": "text",
863 + "primaryKey": false,
864 + "notNull": true,
865 + "default": "'active'"
866 + },
867 + "first_seen_run": {
868 + "name": "first_seen_run",
869 + "type": "text",
870 + "primaryKey": false,
871 + "notNull": false
872 + },
873 + "last_seen_run": {
874 + "name": "last_seen_run",
875 + "type": "text",
876 + "primaryKey": false,
877 + "notNull": false
878 + },
879 + "retrieved_at": {
880 + "name": "retrieved_at",
881 + "type": "timestamp with time zone",
882 + "primaryKey": false,
883 + "notNull": true,
884 + "default": "now()"
885 + },
886 + "source_updated_at": {
887 + "name": "source_updated_at",
888 + "type": "timestamp with time zone",
889 + "primaryKey": false,
890 + "notNull": false
891 + },
892 + "canonical_type": {
893 + "name": "canonical_type",
894 + "type": "text",
895 + "primaryKey": false,
896 + "notNull": false
897 + },
898 + "canonical_id": {
899 + "name": "canonical_id",
900 + "type": "text",
901 + "primaryKey": false,
902 + "notNull": false
903 + },
904 + "created_at": {
905 + "name": "created_at",
906 + "type": "timestamp with time zone",
907 + "primaryKey": false,
908 + "notNull": true,
909 + "default": "now()"
910 + },
911 + "updated_at": {
912 + "name": "updated_at",
913 + "type": "timestamp with time zone",
914 + "primaryKey": false,
915 + "notNull": true,
916 + "default": "now()"
917 + }
918 + },
919 + "indexes": {
920 + "source_records_uq": {
921 + "name": "source_records_uq",
922 + "columns": [
923 + {
924 + "expression": "source_id",
925 + "isExpression": false,
926 + "asc": true,
927 + "nulls": "last"
928 + },
929 + {
930 + "expression": "entity_kind",
931 + "isExpression": false,
932 + "asc": true,
933 + "nulls": "last"
934 + },
935 + {
936 + "expression": "source_record_id",
937 + "isExpression": false,
938 + "asc": true,
939 + "nulls": "last"
940 + }
941 + ],
942 + "isUnique": true,
943 + "concurrently": false,
944 + "method": "btree",
945 + "with": {}
946 + },
947 + "source_records_canonical_idx": {
948 + "name": "source_records_canonical_idx",
949 + "columns": [
950 + {
951 + "expression": "canonical_type",
952 + "isExpression": false,
953 + "asc": true,
954 + "nulls": "last"
955 + },
956 + {
957 + "expression": "canonical_id",
958 + "isExpression": false,
959 + "asc": true,
960 + "nulls": "last"
961 + }
962 + ],
963 + "isUnique": false,
964 + "concurrently": false,
965 + "method": "btree",
966 + "with": {}
967 + }
968 + },
969 + "foreignKeys": {},
970 + "compositePrimaryKeys": {},
971 + "uniqueConstraints": {},
972 + "policies": {},
973 + "checkConstraints": {},
974 + "isRLSEnabled": false
975 + },
976 + "public.sources": {
977 + "name": "sources",
978 + "schema": "",
979 + "columns": {
980 + "id": {
981 + "name": "id",
982 + "type": "varchar(32)",
983 + "primaryKey": true,
984 + "notNull": true
985 + },
986 + "slug": {
987 + "name": "slug",
988 + "type": "text",
989 + "primaryKey": false,
990 + "notNull": true
991 + },
992 + "name": {
993 + "name": "name",
994 + "type": "text",
995 + "primaryKey": false,
996 + "notNull": true
997 + },
998 + "organization": {
999 + "name": "organization",
1000 + "type": "text",
1001 + "primaryKey": false,
1002 + "notNull": false
1003 + },
1004 + "category": {
1005 + "name": "category",
1006 + "type": "text",
1007 + "primaryKey": false,
1008 + "notNull": true
1009 + },
1010 + "description": {
1011 + "name": "description",
1012 + "type": "text",
1013 + "primaryKey": false,
1014 + "notNull": false
1015 + },
1016 + "homepage": {
1017 + "name": "homepage",
1018 + "type": "text",
1019 + "primaryKey": false,
1020 + "notNull": false
1021 + },
1022 + "docs_url": {
1023 + "name": "docs_url",
1024 + "type": "text",
1025 + "primaryKey": false,
1026 + "notNull": false
1027 + },
1028 + "terms_url": {
1029 + "name": "terms_url",
1030 + "type": "text",
1031 + "primaryKey": false,
1032 + "notNull": false
1033 + },
1034 + "access_type": {
1035 + "name": "access_type",
1036 + "type": "text",
1037 + "primaryKey": false,
1038 + "notNull": true
1039 + },
1040 + "access_auth": {
1041 + "name": "access_auth",
1042 + "type": "text",
1043 + "primaryKey": false,
1044 + "notNull": true
1045 + },
1046 + "license": {
1047 + "name": "license",
1048 + "type": "text",
1049 + "primaryKey": false,
1050 + "notNull": false
1051 + },
1052 + "license_status": {
1053 + "name": "license_status",
1054 + "type": "text",
1055 + "primaryKey": false,
1056 + "notNull": true,
1057 + "default": "'review'"
1058 + },
1059 + "commercial_use": {
1060 + "name": "commercial_use",
1061 + "type": "text",
1062 + "primaryKey": false,
1063 + "notNull": true,
1064 + "default": "'unknown'"
1065 + },
1066 + "redistribution": {
1067 + "name": "redistribution",
1068 + "type": "text",
1069 + "primaryKey": false,
1070 + "notNull": true,
1071 + "default": "'unknown'"
1072 + },
1073 + "attribution": {
1074 + "name": "attribution",
1075 + "type": "text",
1076 + "primaryKey": false,
1077 + "notNull": false
1078 + },
1079 + "license_reviewed_at": {
1080 + "name": "license_reviewed_at",
1081 + "type": "timestamp with time zone",
1082 + "primaryKey": false,
1083 + "notNull": false
1084 + },
1085 + "approved_for_production": {
1086 + "name": "approved_for_production",
1087 + "type": "boolean",
1088 + "primaryKey": false,
1089 + "notNull": true,
1090 + "default": false
1091 + },
1092 + "update_frequency": {
1093 + "name": "update_frequency",
1094 + "type": "text",
1095 + "primaryKey": false,
1096 + "notNull": false
1097 + },
1098 + "supports_incremental": {
1099 + "name": "supports_incremental",
1100 + "type": "boolean",
1101 + "primaryKey": false,
1102 + "notNull": true,
1103 + "default": false
1104 + },
1105 + "entities": {
1106 + "name": "entities",
1107 + "type": "text[]",
1108 + "primaryKey": false,
1109 + "notNull": true,
1110 + "default": "'{}'"
1111 + },
1112 + "metrics": {
1113 + "name": "metrics",
1114 + "type": "text[]",
1115 + "primaryKey": false,
1116 + "notNull": true,
1117 + "default": "'{}'"
1118 + },
1119 + "rate_limit": {
1120 + "name": "rate_limit",
1121 + "type": "text",
1122 + "primaryKey": false,
1123 + "notNull": false
1124 + },
1125 + "status": {
1126 + "name": "status",
1127 + "type": "text",
1128 + "primaryKey": false,
1129 + "notNull": true,
1130 + "default": "'planned'"
1131 + },
1132 + "tier": {
1133 + "name": "tier",
1134 + "type": "integer",
1135 + "primaryKey": false,
1136 + "notNull": true,
1137 + "default": 0
1138 + },
1139 + "manifest": {
1140 + "name": "manifest",
1141 + "type": "jsonb",
1142 + "primaryKey": false,
1143 + "notNull": true,
1144 + "default": "'{}'::jsonb"
1145 + },
1146 + "created_at": {
1147 + "name": "created_at",
1148 + "type": "timestamp with time zone",
1149 + "primaryKey": false,
1150 + "notNull": true,
1151 + "default": "now()"
1152 + },
1153 + "updated_at": {
1154 + "name": "updated_at",
1155 + "type": "timestamp with time zone",
1156 + "primaryKey": false,
1157 + "notNull": true,
1158 + "default": "now()"
1159 + }
1160 + },
1161 + "indexes": {
1162 + "sources_slug_uq": {
1163 + "name": "sources_slug_uq",
1164 + "columns": [
1165 + {
1166 + "expression": "slug",
1167 + "isExpression": false,
1168 + "asc": true,
1169 + "nulls": "last"
1170 + }
1171 + ],
1172 + "isUnique": true,
1173 + "concurrently": false,
1174 + "method": "btree",
1175 + "with": {}
1176 + }
1177 + },
1178 + "foreignKeys": {},
1179 + "compositePrimaryKeys": {},
1180 + "uniqueConstraints": {},
1181 + "policies": {},
1182 + "checkConstraints": {},
1183 + "isRLSEnabled": false
1184 + },
1185 + "public.unresolved_labels": {
1186 + "name": "unresolved_labels",
1187 + "schema": "",
1188 + "columns": {
1189 + "id": {
1190 + "name": "id",
1191 + "type": "bigserial",
1192 + "primaryKey": true,
1193 + "notNull": true
1194 + },
1195 + "source_id": {
1196 + "name": "source_id",
1197 + "type": "varchar(32)",
1198 + "primaryKey": false,
1199 + "notNull": true
1200 + },
1201 + "entity_kind": {
1202 + "name": "entity_kind",
1203 + "type": "text",
1204 + "primaryKey": false,
1205 + "notNull": true
1206 + },
1207 + "source_text": {
1208 + "name": "source_text",
1209 + "type": "text",
1210 + "primaryKey": false,
1211 + "notNull": true
1212 + },
1213 + "normalized": {
1214 + "name": "normalized",
1215 + "type": "text",
1216 + "primaryKey": false,
1217 + "notNull": true
1218 + },
1219 + "context": {
1220 + "name": "context",
1221 + "type": "jsonb",
1222 + "primaryKey": false,
1223 + "notNull": true,
1224 + "default": "'{}'::jsonb"
1225 + },
1226 + "count": {
1227 + "name": "count",
1228 + "type": "integer",
1229 + "primaryKey": false,
1230 + "notNull": true,
1231 + "default": 1
1232 + },
1233 + "status": {
1234 + "name": "status",
1235 + "type": "text",
1236 + "primaryKey": false,
1237 + "notNull": true,
1238 + "default": "'open'"
1239 + },
1240 + "suggested_id": {
1241 + "name": "suggested_id",
1242 + "type": "text",
1243 + "primaryKey": false,
1244 + "notNull": false
1245 + },
1246 + "suggested_match_type": {
1247 + "name": "suggested_match_type",
1248 + "type": "text",
1249 + "primaryKey": false,
1250 + "notNull": false
1251 + },
1252 + "suggested_score": {
1253 + "name": "suggested_score",
1254 + "type": "real",
1255 + "primaryKey": false,
1256 + "notNull": false
1257 + },
1258 + "resolved_id": {
1259 + "name": "resolved_id",
1260 + "type": "text",
1261 + "primaryKey": false,
1262 + "notNull": false
1263 + },
1264 + "resolved_by": {
1265 + "name": "resolved_by",
1266 + "type": "text",
1267 + "primaryKey": false,
1268 + "notNull": false
1269 + },
1270 + "created_at": {
1271 + "name": "created_at",
1272 + "type": "timestamp with time zone",
1273 + "primaryKey": false,
1274 + "notNull": true,
1275 + "default": "now()"
1276 + },
1277 + "updated_at": {
1278 + "name": "updated_at",
1279 + "type": "timestamp with time zone",
1280 + "primaryKey": false,
1281 + "notNull": true,
1282 + "default": "now()"
1283 + }
1284 + },
1285 + "indexes": {
1286 + "unresolved_labels_uq": {
1287 + "name": "unresolved_labels_uq",
1288 + "columns": [
1289 + {
1290 + "expression": "source_id",
1291 + "isExpression": false,
1292 + "asc": true,
1293 + "nulls": "last"
1294 + },
1295 + {
1296 + "expression": "entity_kind",
1297 + "isExpression": false,
1298 + "asc": true,
1299 + "nulls": "last"
1300 + },
1301 + {
1302 + "expression": "normalized",
1303 + "isExpression": false,
1304 + "asc": true,
1305 + "nulls": "last"
1306 + }
1307 + ],
1308 + "isUnique": true,
1309 + "concurrently": false,
1310 + "method": "btree",
1311 + "with": {}
1312 + },
1313 + "unresolved_labels_count_idx": {
1314 + "name": "unresolved_labels_count_idx",
1315 + "columns": [
1316 + {
1317 + "expression": "status",
1318 + "isExpression": false,
1319 + "asc": true,
1320 + "nulls": "last"
1321 + },
1322 + {
1323 + "expression": "count",
1324 + "isExpression": false,
1325 + "asc": true,
1326 + "nulls": "last"
1327 + }
1328 + ],
1329 + "isUnique": false,
1330 + "concurrently": false,
1331 + "method": "btree",
1332 + "with": {}
1333 + }
1334 + },
1335 + "foreignKeys": {},
1336 + "compositePrimaryKeys": {},
1337 + "uniqueConstraints": {},
1338 + "policies": {},
1339 + "checkConstraints": {},
1340 + "isRLSEnabled": false
1341 + },
1342 + "public.anatomical_sites": {
1343 + "name": "anatomical_sites",
1344 + "schema": "",
1345 + "columns": {
1346 + "id": {
1347 + "name": "id",
1348 + "type": "varchar(32)",
1349 + "primaryKey": true,
1350 + "notNull": true
1351 + },
1352 + "name": {
1353 + "name": "name",
1354 + "type": "text",
1355 + "primaryKey": false,
1356 + "notNull": true
1357 + },
1358 + "slug": {
1359 + "name": "slug",
1360 + "type": "text",
1361 + "primaryKey": false,
1362 + "notNull": true
1363 + },
1364 + "ncit_code": {
1365 + "name": "ncit_code",
1366 + "type": "text",
1367 + "primaryKey": false,
1368 + "notNull": false
1369 + },
1370 + "uberon_id": {
1371 + "name": "uberon_id",
1372 + "type": "text",
1373 + "primaryKey": false,
1374 + "notNull": false
1375 + },
1376 + "parent_id": {
1377 + "name": "parent_id",
1378 + "type": "varchar(32)",
1379 + "primaryKey": false,
1380 + "notNull": false
1381 + },
1382 + "system": {
1383 + "name": "system",
1384 + "type": "text",
1385 + "primaryKey": false,
1386 + "notNull": false
1387 + }
1388 + },
1389 + "indexes": {
1390 + "anatomical_sites_slug_uq": {
1391 + "name": "anatomical_sites_slug_uq",
1392 + "columns": [
1393 + {
1394 + "expression": "slug",
1395 + "isExpression": false,
1396 + "asc": true,
1397 + "nulls": "last"
1398 + }
1399 + ],
1400 + "isUnique": true,
1401 + "concurrently": false,
1402 + "method": "btree",
1403 + "with": {}
1404 + }
1405 + },
1406 + "foreignKeys": {},
1407 + "compositePrimaryKeys": {},
1408 + "uniqueConstraints": {},
1409 + "policies": {},
1410 + "checkConstraints": {},
1411 + "isRLSEnabled": false
1412 + },
1413 + "public.cancer_aliases": {
1414 + "name": "cancer_aliases",
1415 + "schema": "",
1416 + "columns": {
1417 + "id": {
1418 + "name": "id",
1419 + "type": "bigserial",
1420 + "primaryKey": true,
1421 + "notNull": true
1422 + },
1423 + "cancer_id": {
1424 + "name": "cancer_id",
1425 + "type": "varchar(32)",
1426 + "primaryKey": false,
1427 + "notNull": true
1428 + },
1429 + "alias": {
1430 + "name": "alias",
1431 + "type": "text",
1432 + "primaryKey": false,
1433 + "notNull": true
1434 + },
1435 + "normalized": {
1436 + "name": "normalized",
1437 + "type": "text",
1438 + "primaryKey": false,
1439 + "notNull": true
1440 + },
1441 + "alias_type": {
1442 + "name": "alias_type",
1443 + "type": "text",
1444 + "primaryKey": false,
1445 + "notNull": true,
1446 + "default": "'synonym'"
1447 + },
1448 + "source_id": {
1449 + "name": "source_id",
1450 + "type": "varchar(32)",
1451 + "primaryKey": false,
1452 + "notNull": false
1453 + },
1454 + "source_terminology": {
1455 + "name": "source_terminology",
1456 + "type": "text",
1457 + "primaryKey": false,
1458 + "notNull": false
1459 + },
1460 + "language": {
1461 + "name": "language",
1462 + "type": "text",
1463 + "primaryKey": false,
1464 + "notNull": true,
1465 + "default": "'en'"
1466 + }
1467 + },
1468 + "indexes": {
1469 + "cancer_aliases_uq": {
1470 + "name": "cancer_aliases_uq",
1471 + "columns": [
1472 + {
1473 + "expression": "cancer_id",
1474 + "isExpression": false,
1475 + "asc": true,
1476 + "nulls": "last"
1477 + },
1478 + {
1479 + "expression": "normalized",
1480 + "isExpression": false,
1481 + "asc": true,
1482 + "nulls": "last"
1483 + },
1484 + {
1485 + "expression": "alias_type",
1486 + "isExpression": false,
1487 + "asc": true,
1488 + "nulls": "last"
1489 + }
1490 + ],
1491 + "isUnique": true,
1492 + "concurrently": false,
1493 + "method": "btree",
1494 + "with": {}
1495 + },
1496 + "cancer_aliases_norm_idx": {
1497 + "name": "cancer_aliases_norm_idx",
1498 + "columns": [
1499 + {
1500 + "expression": "normalized",
1501 + "isExpression": false,
1502 + "asc": true,
1503 + "nulls": "last"
1504 + }
1505 + ],
1506 + "isUnique": false,
1507 + "concurrently": false,
1508 + "method": "btree",
1509 + "with": {}
1510 + }
1511 + },
1512 + "foreignKeys": {},
1513 + "compositePrimaryKeys": {},
1514 + "uniqueConstraints": {},
1515 + "policies": {},
1516 + "checkConstraints": {},
1517 + "isRLSEnabled": false
1518 + },
1519 + "public.cancer_anatomy": {
1520 + "name": "cancer_anatomy",
1521 + "schema": "",
1522 + "columns": {
1523 + "id": {
1524 + "name": "id",
1525 + "type": "bigserial",
1526 + "primaryKey": true,
1527 + "notNull": true
1528 + },
1529 + "cancer_id": {
1530 + "name": "cancer_id",
1531 + "type": "varchar(32)",
1532 + "primaryKey": false,
1533 + "notNull": true
1534 + },
1535 + "site_id": {
1536 + "name": "site_id",
1537 + "type": "varchar(32)",
1538 + "primaryKey": false,
1539 + "notNull": true
1540 + },
1541 + "relation": {
1542 + "name": "relation",
1543 + "type": "text",
1544 + "primaryKey": false,
1545 + "notNull": true,
1546 + "default": "'primary'"
1547 + },
1548 + "source_id": {
1549 + "name": "source_id",
1550 + "type": "varchar(32)",
1551 + "primaryKey": false,
1552 + "notNull": false
1553 + }
1554 + },
1555 + "indexes": {
1556 + "cancer_anatomy_uq": {
1557 + "name": "cancer_anatomy_uq",
1558 + "columns": [
1559 + {
1560 + "expression": "cancer_id",
1561 + "isExpression": false,
1562 + "asc": true,
1563 + "nulls": "last"
1564 + },
1565 + {
1566 + "expression": "site_id",
1567 + "isExpression": false,
1568 + "asc": true,
1569 + "nulls": "last"
1570 + },
1571 + {
1572 + "expression": "relation",
1573 + "isExpression": false,
1574 + "asc": true,
1575 + "nulls": "last"
1576 + }
1577 + ],
1578 + "isUnique": true,
1579 + "concurrently": false,
1580 + "method": "btree",
1581 + "with": {}
1582 + }
1583 + },
1584 + "foreignKeys": {},
1585 + "compositePrimaryKeys": {},
1586 + "uniqueConstraints": {},
1587 + "policies": {},
1588 + "checkConstraints": {},
1589 + "isRLSEnabled": false
1590 + },
1591 + "public.cancer_codes": {
1592 + "name": "cancer_codes",
1593 + "schema": "",
1594 + "columns": {
1595 + "id": {
1596 + "name": "id",
1597 + "type": "bigserial",
1598 + "primaryKey": true,
1599 + "notNull": true
1600 + },
1601 + "cancer_id": {
1602 + "name": "cancer_id",
1603 + "type": "varchar(32)",
1604 + "primaryKey": false,
1605 + "notNull": true
1606 + },
1607 + "system": {
1608 + "name": "system",
1609 + "type": "text",
1610 + "primaryKey": false,
1611 + "notNull": true
1612 + },
1613 + "code": {
1614 + "name": "code",
1615 + "type": "text",
1616 + "primaryKey": false,
1617 + "notNull": true
1618 + },
1619 + "match_type": {
1620 + "name": "match_type",
1621 + "type": "text",
1622 + "primaryKey": false,
1623 + "notNull": true,
1624 + "default": "'EXACT_IDENTIFIER'"
1625 + },
1626 + "source_id": {
1627 + "name": "source_id",
1628 + "type": "varchar(32)",
1629 + "primaryKey": false,
1630 + "notNull": false
1631 + },
1632 + "valid_from": {
1633 + "name": "valid_from",
1634 + "type": "text",
1635 + "primaryKey": false,
1636 + "notNull": false
1637 + },
1638 + "valid_to": {
1639 + "name": "valid_to",
1640 + "type": "text",
1641 + "primaryKey": false,
1642 + "notNull": false
1643 + }
1644 + },
1645 + "indexes": {
1646 + "cancer_codes_uq": {
1647 + "name": "cancer_codes_uq",
1648 + "columns": [
1649 + {
1650 + "expression": "cancer_id",
1651 + "isExpression": false,
1652 + "asc": true,
1653 + "nulls": "last"
1654 + },
1655 + {
1656 + "expression": "system",
1657 + "isExpression": false,
1658 + "asc": true,
1659 + "nulls": "last"
1660 + },
1661 + {
1662 + "expression": "code",
1663 + "isExpression": false,
1664 + "asc": true,
1665 + "nulls": "last"
1666 + }
1667 + ],
1668 + "isUnique": true,
1669 + "concurrently": false,
1670 + "method": "btree",
1671 + "with": {}
1672 + },
1673 + "cancer_codes_lookup_idx": {
1674 + "name": "cancer_codes_lookup_idx",
1675 + "columns": [
1676 + {
1677 + "expression": "system",
1678 + "isExpression": false,
1679 + "asc": true,
1680 + "nulls": "last"
1681 + },
1682 + {
1683 + "expression": "code",
1684 + "isExpression": false,
1685 + "asc": true,
1686 + "nulls": "last"
1687 + }
1688 + ],
1689 + "isUnique": false,
1690 + "concurrently": false,
1691 + "method": "btree",
1692 + "with": {}
1693 + }
1694 + },
1695 + "foreignKeys": {},
1696 + "compositePrimaryKeys": {},
1697 + "uniqueConstraints": {},
1698 + "policies": {},
1699 + "checkConstraints": {},
1700 + "isRLSEnabled": false
1701 + },
1702 + "public.cancer_hierarchy": {
1703 + "name": "cancer_hierarchy",
1704 + "schema": "",
1705 + "columns": {
1706 + "id": {
1707 + "name": "id",
1708 + "type": "bigserial",
1709 + "primaryKey": true,
1710 + "notNull": true
1711 + },
1712 + "parent_id": {
1713 + "name": "parent_id",
1714 + "type": "varchar(32)",
1715 + "primaryKey": false,
1716 + "notNull": true
1717 + },
1718 + "child_id": {
1719 + "name": "child_id",
1720 + "type": "varchar(32)",
1721 + "primaryKey": false,
1722 + "notNull": true
1723 + },
1724 + "hierarchy_type": {
1725 + "name": "hierarchy_type",
1726 + "type": "text",
1727 + "primaryKey": false,
1728 + "notNull": true
1729 + },
1730 + "source_id": {
1731 + "name": "source_id",
1732 + "type": "varchar(32)",
1733 + "primaryKey": false,
1734 + "notNull": false
1735 + }
1736 + },
1737 + "indexes": {
1738 + "cancer_hierarchy_uq": {
1739 + "name": "cancer_hierarchy_uq",
1740 + "columns": [
1741 + {
1742 + "expression": "parent_id",
1743 + "isExpression": false,
1744 + "asc": true,
1745 + "nulls": "last"
1746 + },
1747 + {
1748 + "expression": "child_id",
1749 + "isExpression": false,
1750 + "asc": true,
1751 + "nulls": "last"
1752 + },
1753 + {
1754 + "expression": "hierarchy_type",
1755 + "isExpression": false,
1756 + "asc": true,
1757 + "nulls": "last"
1758 + }
1759 + ],
1760 + "isUnique": true,
1761 + "concurrently": false,
1762 + "method": "btree",
1763 + "with": {}
1764 + },
1765 + "cancer_hierarchy_child_idx": {
1766 + "name": "cancer_hierarchy_child_idx",
1767 + "columns": [
1768 + {
1769 + "expression": "child_id",
1770 + "isExpression": false,
1771 + "asc": true,
1772 + "nulls": "last"
1773 + }
1774 + ],
1775 + "isUnique": false,
1776 + "concurrently": false,
1777 + "method": "btree",
1778 + "with": {}
1779 + }
1780 + },
1781 + "foreignKeys": {},
1782 + "compositePrimaryKeys": {},
1783 + "uniqueConstraints": {},
1784 + "policies": {},
1785 + "checkConstraints": {},
1786 + "isRLSEnabled": false
1787 + },
1788 + "public.cancers": {
1789 + "name": "cancers",
1790 + "schema": "",
1791 + "columns": {
1792 + "id": {
1793 + "name": "id",
1794 + "type": "varchar(32)",
1795 + "primaryKey": true,
1796 + "notNull": true
1797 + },
1798 + "slug": {
1799 + "name": "slug",
1800 + "type": "text",
1801 + "primaryKey": false,
1802 + "notNull": true
1803 + },
1804 + "canonical_name": {
1805 + "name": "canonical_name",
1806 + "type": "text",
1807 + "primaryKey": false,
1808 + "notNull": true
1809 + },
1810 + "short_name": {
1811 + "name": "short_name",
1812 + "type": "text",
1813 + "primaryKey": false,
1814 + "notNull": false
1815 + },
1816 + "entity_type": {
1817 + "name": "entity_type",
1818 + "type": "text",
1819 + "primaryKey": false,
1820 + "notNull": true,
1821 + "default": "'cancer'"
1822 + },
1823 + "malignant": {
1824 + "name": "malignant",
1825 + "type": "boolean",
1826 + "primaryKey": false,
1827 + "notNull": true,
1828 + "default": true
1829 + },
1830 + "solid_tumor": {
1831 + "name": "solid_tumor",
1832 + "type": "boolean",
1833 + "primaryKey": false,
1834 + "notNull": true,
1835 + "default": true
1836 + },
1837 + "hematologic": {
1838 + "name": "hematologic",
1839 + "type": "boolean",
1840 + "primaryKey": false,
1841 + "notNull": true,
1842 + "default": false
1843 + },
1844 + "pediatric_relevant": {
1845 + "name": "pediatric_relevant",
1846 + "type": "boolean",
1847 + "primaryKey": false,
1848 + "notNull": true,
1849 + "default": false
1850 + },
1851 + "rare_cancer": {
1852 + "name": "rare_cancer",
1853 + "type": "boolean",
1854 + "primaryKey": false,
1855 + "notNull": false
1856 + },
1857 + "top_level": {
1858 + "name": "top_level",
1859 + "type": "boolean",
1860 + "primaryKey": false,
1861 + "notNull": true,
1862 + "default": false
1863 + },
1864 + "description": {
1865 + "name": "description",
1866 + "type": "text",
1867 + "primaryKey": false,
1868 + "notNull": false
1869 + },
1870 + "description_provenance_id": {
1871 + "name": "description_provenance_id",
1872 + "type": "integer",
1873 + "primaryKey": false,
1874 + "notNull": false
1875 + },
1876 + "primary_ncit_code": {
1877 + "name": "primary_ncit_code",
1878 + "type": "text",
1879 + "primaryKey": false,
1880 + "notNull": false
1881 + },
1882 + "primary_oncotree_code": {
1883 + "name": "primary_oncotree_code",
1884 + "type": "text",
1885 + "primaryKey": false,
1886 + "notNull": false
1887 + },
1888 + "depth": {
1889 + "name": "depth",
1890 + "type": "integer",
1891 + "primaryKey": false,
1892 + "notNull": true,
1893 + "default": 0
1894 + },
1895 + "status": {
1896 + "name": "status",
1897 + "type": "text",
1898 + "primaryKey": false,
1899 + "notNull": true,
1900 + "default": "'active'"
1901 + },
1902 + "merged_into": {
1903 + "name": "merged_into",
1904 + "type": "varchar(32)",
1905 + "primaryKey": false,
1906 + "notNull": false
1907 + },
1908 + "deprecated_reason": {
1909 + "name": "deprecated_reason",
1910 + "type": "text",
1911 + "primaryKey": false,
1912 + "notNull": false
1913 + },
1914 + "classification_version": {
1915 + "name": "classification_version",
1916 + "type": "text",
1917 + "primaryKey": false,
1918 + "notNull": false
1919 + },
1920 + "semantic_types": {
1921 + "name": "semantic_types",
1922 + "type": "text[]",
1923 + "primaryKey": false,
1924 + "notNull": true,
1925 + "default": "'{}'"
1926 + },
1927 + "created_at": {
1928 + "name": "created_at",
1929 + "type": "timestamp with time zone",
1930 + "primaryKey": false,
1931 + "notNull": true,
1932 + "default": "now()"
1933 + },
1934 + "updated_at": {
1935 + "name": "updated_at",
1936 + "type": "timestamp with time zone",
1937 + "primaryKey": false,
1938 + "notNull": true,
1939 + "default": "now()"
1940 + }
1941 + },
1942 + "indexes": {
1943 + "cancers_slug_uq": {
1944 + "name": "cancers_slug_uq",
1945 + "columns": [
1946 + {
1947 + "expression": "slug",
1948 + "isExpression": false,
1949 + "asc": true,
1950 + "nulls": "last"
1951 + }
1952 + ],
1953 + "isUnique": true,
1954 + "concurrently": false,
1955 + "method": "btree",
1956 + "with": {}
1957 + },
1958 + "cancers_ncit_uq": {
1959 + "name": "cancers_ncit_uq",
1960 + "columns": [
1961 + {
1962 + "expression": "primary_ncit_code",
1963 + "isExpression": false,
1964 + "asc": true,
1965 + "nulls": "last"
1966 + }
1967 + ],
1968 + "isUnique": true,
1969 + "concurrently": false,
1970 + "method": "btree",
1971 + "with": {}
1972 + },
1973 + "cancers_name_idx": {
1974 + "name": "cancers_name_idx",
1975 + "columns": [
1976 + {
1977 + "expression": "canonical_name",
1978 + "isExpression": false,
1979 + "asc": true,
1980 + "nulls": "last"
1981 + }
1982 + ],
1983 + "isUnique": false,
1984 + "concurrently": false,
1985 + "method": "btree",
1986 + "with": {}
1987 + },
1988 + "cancers_type_idx": {
1989 + "name": "cancers_type_idx",
1990 + "columns": [
1991 + {
1992 + "expression": "entity_type",
1993 + "isExpression": false,
1994 + "asc": true,
1995 + "nulls": "last"
1996 + },
1997 + {
1998 + "expression": "malignant",
1999 + "isExpression": false,
2000 + "asc": true,
2001 + "nulls": "last"
2002 + },
2003 + {
2004 + "expression": "top_level",
2005 + "isExpression": false,
2006 + "asc": true,
2007 + "nulls": "last"
2008 + }
2009 + ],
2010 + "isUnique": false,
2011 + "concurrently": false,
2012 + "method": "btree",
2013 + "with": {}
2014 + }
2015 + },
2016 + "foreignKeys": {},
2017 + "compositePrimaryKeys": {},
2018 + "uniqueConstraints": {},
2019 + "policies": {},
2020 + "checkConstraints": {},
2021 + "isRLSEnabled": false
2022 + },
2023 + "public.cohort_definitions": {
2024 + "name": "cohort_definitions",
2025 + "schema": "",
2026 + "columns": {
2027 + "id": {
2028 + "name": "id",
2029 + "type": "bigserial",
2030 + "primaryKey": true,
2031 + "notNull": true
2032 + },
2033 + "name": {
2034 + "name": "name",
2035 + "type": "text",
2036 + "primaryKey": false,
2037 + "notNull": true
2038 + },
2039 + "cancer_id": {
2040 + "name": "cancer_id",
2041 + "type": "varchar(32)",
2042 + "primaryKey": false,
2043 + "notNull": true
2044 + },
2045 + "biomarker_ids": {
2046 + "name": "biomarker_ids",
2047 + "type": "text[]",
2048 + "primaryKey": false,
2049 + "notNull": true,
2050 + "default": "'{}'"
2051 + },
2052 + "variant_ids": {
2053 + "name": "variant_ids",
2054 + "type": "text[]",
2055 + "primaryKey": false,
2056 + "notNull": true,
2057 + "default": "'{}'"
2058 + },
2059 + "stage": {
2060 + "name": "stage",
2061 + "type": "text",
2062 + "primaryKey": false,
2063 + "notNull": false
2064 + },
2065 + "attributes": {
2066 + "name": "attributes",
2067 + "type": "jsonb",
2068 + "primaryKey": false,
2069 + "notNull": true,
2070 + "default": "'{}'::jsonb"
2071 + },
2072 + "confidence": {
2073 + "name": "confidence",
2074 + "type": "real",
2075 + "primaryKey": false,
2076 + "notNull": false
2077 + },
2078 + "created_at": {
2079 + "name": "created_at",
2080 + "type": "timestamp with time zone",
2081 + "primaryKey": false,
2082 + "notNull": true,
2083 + "default": "now()"
2084 + }
2085 + },
2086 + "indexes": {},
2087 + "foreignKeys": {},
2088 + "compositePrimaryKeys": {},
2089 + "uniqueConstraints": {},
2090 + "policies": {},
2091 + "checkConstraints": {},
2092 + "isRLSEnabled": false
2093 + },
2094 + "public.geographies": {
2095 + "name": "geographies",
2096 + "schema": "",
2097 + "columns": {
2098 + "id": {
2099 + "name": "id",
2100 + "type": "varchar(32)",
2101 + "primaryKey": true,
2102 + "notNull": true
2103 + },
2104 + "slug": {
2105 + "name": "slug",
2106 + "type": "text",
2107 + "primaryKey": false,
2108 + "notNull": true
2109 + },
2110 + "name": {
2111 + "name": "name",
2112 + "type": "text",
2113 + "primaryKey": false,
2114 + "notNull": true
2115 + },
2116 + "kind": {
2117 + "name": "kind",
2118 + "type": "text",
2119 + "primaryKey": false,
2120 + "notNull": true
2121 + },
2122 + "iso2": {
2123 + "name": "iso2",
2124 + "type": "text",
2125 + "primaryKey": false,
2126 + "notNull": false
2127 + },
2128 + "iso3": {
2129 + "name": "iso3",
2130 + "type": "text",
2131 + "primaryKey": false,
2132 + "notNull": false
2133 + },
2134 + "parent_id": {
2135 + "name": "parent_id",
2136 + "type": "varchar(32)",
2137 + "primaryKey": false,
2138 + "notNull": false
2139 + },
2140 + "who_region": {
2141 + "name": "who_region",
2142 + "type": "text",
2143 + "primaryKey": false,
2144 + "notNull": false
2145 + },
2146 + "population": {
2147 + "name": "population",
2148 + "type": "integer",
2149 + "primaryKey": false,

Diff truncated — file too large.