SPB Git
spboucher.ai

spb/cancerindex

Public

Foundation for wave 3: intelligence schema (trial_intelligence, trial_site_country_counts, drug_pipeline, drug_codes, research_gap_components; migration 0002), trial intervention→drug reconciliation (alias/salt-stripped/probabilistic), new metric definitions (trial growth, termination share, sponsor HHI, per-1000-deaths, gap ratios), maintenance.intel job, API route stubs (epidemiology/approvals/graph/intelligence), navigation restructure

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 11, 2026) parent 718f988 Browse files

27 changed files +8,980 −12

27 changed files
Load all 27 files Unified Split
modified apps/api/src/app.ts +12 −0
@@ -26,6 +26,10 @@ import { sourceRoutes } from './routes/sources.js';
26 import { statsRoutes } from './routes/stats.js'; 26 import { statsRoutes } from './routes/stats.js';
27 import { changeRoutes } from './routes/changes.js'; 27 import { changeRoutes } from './routes/changes.js';
28 import { adminRoutes } from './routes/admin.js'; 28 import { adminRoutes } from './routes/admin.js';
29+import { graphRoutes } from './routes/graph.js';
30+import { epidemiologyRoutes } from './routes/epidemiology.js';
31+import { approvalRoutes } from './routes/approvals.js';
32+import { intelligenceRoutes } from './routes/intelligence.js';
29 33
30 export interface BuildOptions { 34 export interface BuildOptions {
31 db?: Database; 35 db?: Database;
@@ -103,6 +107,10 @@ export async function buildApp(opts: BuildOptions = {}): Promise<FastifyInstance
103 { name: 'sources', description: 'Source registry, licenses, connector health' }, 107 { name: 'sources', description: 'Source registry, licenses, connector health' },
104 { name: 'stats' }, 108 { name: 'stats' },
105 { name: 'changes' }, 109 { name: 'changes' },
110+ { name: 'epidemiology', description: 'Time-aware incidence/mortality/prevalence observations with provenance (Data Explorer)' },
111+ { name: 'approvals', description: 'Jurisdiction-aware regulatory approvals and the drug development pipeline' },
112+ { name: 'graph', description: 'Cancer–gene–variant–drug–trial knowledge graph (contextual neighbourhoods)' },
113+ { name: 'intelligence', description: 'Derived clinical-trial intelligence and research-gap components (computed metrics with formula versions)' },
106 { name: 'admin', description: 'Operator endpoints (x-admin-token)' }, 114 { name: 'admin', description: 'Operator endpoints (x-admin-token)' },
107 { name: 'system' }, 115 { name: 'system' },
108 ], 116 ],
@@ -152,6 +160,10 @@ export async function buildApp(opts: BuildOptions = {}): Promise<FastifyInstance
152 await v1.register(sourceRoutes); 160 await v1.register(sourceRoutes);
153 await v1.register(statsRoutes); 161 await v1.register(statsRoutes);
154 await v1.register(changeRoutes); 162 await v1.register(changeRoutes);
163+ await v1.register(epidemiologyRoutes);
164+ await v1.register(approvalRoutes);
165+ await v1.register(graphRoutes);
166+ await v1.register(intelligenceRoutes);
155 await v1.register(adminRoutes, { prefix: '/admin' }); 167 await v1.register(adminRoutes, { prefix: '/admin' });
156 }, 168 },
157 { prefix: '/v1' }, 169 { prefix: '/v1' },
modified apps/api/src/lib/queue.ts +1 −0
@@ -8,6 +8,7 @@ export const JOBS = {
8 connectorRun: 'connector.run', 8 connectorRun: 'connector.run',
9 counters: 'maintenance.counters', 9 counters: 'maintenance.counters',
10 rank: 'maintenance.rank', 10 rank: 'maintenance.rank',
11+ intel: 'maintenance.intel',
11 healthProbe: 'health.probe', 12 healthProbe: 'health.probe',
12 } as const; 13 } as const;
13 export type JobName = (typeof JOBS)[keyof typeof JOBS]; 14 export type JobName = (typeof JOBS)[keyof typeof JOBS];
added apps/api/src/routes/approvals.ts +10 −0
@@ -0,0 +1,10 @@
1+import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
2+
3+/**
4+ * Regulatory routes (SPEC §13-15, §63): `GET /approvals` (jurisdiction-aware, dated, filterable by
5+ * authority, cancer, drug, date range), `GET /approvals/recent`, `GET /pipeline` (drug development
6+ * stages). Filled by the Approvals & Pipeline work package.
7+ */
8+export const approvalRoutes: FastifyPluginAsyncZod = async (_app) => {
9+ /* routes added by the approvals work package */
10+};
added apps/api/src/routes/epidemiology.ts +10 −0
@@ -0,0 +1,10 @@
1+import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
2+
3+/**
4+ * Epidemiology routes (SPEC §20, §63, §110): `GET /epidemiology` → observations filtered by metric,
5+ * cancer, geography, sex, age group and year range; `GET /epidemiology/coverage` → which
6+ * metric × geography × year combinations exist. Filled by the Data Explorer work package.
7+ */
8+export const epidemiologyRoutes: FastifyPluginAsyncZod = async (_app) => {
9+ /* routes added by the data-explorer work package */
10+};
added apps/api/src/routes/graph.ts +10 −0
@@ -0,0 +1,10 @@
1+import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
2+
3+/**
4+ * Knowledge-graph routes (SPEC §18, §80): `GET /graph/:type/:id` → contextual neighbours of one
5+ * entity (cancer | gene | variant | drug | biomarker | trial) with relationship type, cancer context,
6+ * evidence level and provenance on every edge. Filled by the Knowledge Graph work package.
7+ */
8+export const graphRoutes: FastifyPluginAsyncZod = async (_app) => {
9+ /* routes added by the knowledge-graph work package */
10+};
added apps/api/src/routes/intelligence.ts +18 −0
@@ -0,0 +1,18 @@
1+import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
2+
3+/**
4+ * Intelligence routes (SPEC §10-11, §34): `GET /trials/intelligence` (per-cancer trial metrics),
5+ * `GET /trials/sites` (country/city aggregates for the map), `GET /research-gap` (components and
6+ * ratios per scope). Filled by the Trial Intelligence and Research Gap work packages
7+ * (each adds its own routes to this file; keep route groups in separate functions).
8+ */
9+export const intelligenceRoutes: FastifyPluginAsyncZod = async (app) => {
10+ await app.register(trialIntelligenceRoutes);
11+ await app.register(researchGapRoutes);
12+};
13+
14+/** Filled by the Trial Intelligence work package. */
15+export const trialIntelligenceRoutes: FastifyPluginAsyncZod = async (_app) => {};
16+
17+/** Filled by the Research Gap work package. */
18+export const researchGapRoutes: FastifyPluginAsyncZod = async (_app) => {};
modified apps/web/src/components/layout/mobile-nav.tsx +2 −2
@@ -1,6 +1,6 @@
1 import Link from 'next/link'; 1 import Link from 'next/link';
2 import { Menu } from 'lucide-react'; 2 import { Menu } from 'lucide-react';
3−import { NAV, FOOTER_NAV } from '@/lib/site'; 3+import { NAV, MORE_NAV, FOOTER_NAV } from '@/lib/site';
4 4
5 /** CSS-only disclosure menu for small screens (no client JS). */ 5 /** CSS-only disclosure menu for small screens (no client JS). */
6 export function MobileNav() { 6 export function MobileNav() {
@@ -16,7 +16,7 @@ export function MobileNav() {
16 </Link> 16 </Link>
17 ))} 17 ))}
18 <div className="my-1 border-t border-rule" /> 18 <div className="my-1 border-t border-rule" />
19− {FOOTER_NAV.filter((f) => !NAV.some((n) => n.href === f.href)).map((n) => ( 19+ {[...MORE_NAV, ...FOOTER_NAV.filter((f) => !NAV.some((n) => n.href === f.href) && !MORE_NAV.some((m) => m.href === f.href))].map((n) => (
20 <Link key={n.href} href={n.href} className="block px-3 py-1.5 text-[13px] text-ink-2 no-underline hover:bg-paper-2"> 20 <Link key={n.href} href={n.href} className="block px-3 py-1.5 text-[13px] text-ink-2 no-underline hover:bg-paper-2">
21 {n.label} 21 {n.label}
22 </Link> 22 </Link>
modified apps/web/src/lib/site.ts +20 −5
@@ -7,13 +7,24 @@ export const CONTACT_EMAIL = 'corrections@cancerindex.io';
7 7
8 export const NAV = [ 8 export const NAV = [
9 { href: '/cancers', label: 'Cancers' }, 9 { href: '/cancers', label: 'Cancers' },
10− { href: '/taxonomy', label: 'Taxonomy' }, 10+ { href: '/explore', label: 'Data' },
11+ { href: '/trials', label: 'Trials' },
12+ { href: '/drugs', label: 'Drugs' },
13+ { href: '/genes', label: 'Genes' },
14+ { href: '/approvals', label: 'Approvals' },
11 { href: '/rankings', label: 'Rankings' }, 15 { href: '/rankings', label: 'Rankings' },
16+ { href: '/research-gap', label: 'Research gap' },
17+ { href: '/graph', label: 'Graph' },
18+] as const;
19+
20+/** Secondary navigation (mobile menu + footer): everything not in the primary bar. */
21+export const MORE_NAV = [
22+ { href: '/taxonomy', label: 'Taxonomy' },
12 { href: '/countries', label: 'Countries' }, 23 { href: '/countries', label: 'Countries' },
13 { href: '/compare', label: 'Compare' }, 24 { href: '/compare', label: 'Compare' },
14− { href: '/genes', label: 'Genes' }, 25+ { href: '/trials/intelligence', label: 'Trial intelligence' },
15− { href: '/drugs', label: 'Drugs' }, 26+ { href: '/trials/map', label: 'Trial map' },
16− { href: '/trials', label: 'Trials' }, 27+ { href: '/pipeline', label: 'Drug pipeline' },
17 { href: '/sources', label: 'Sources' }, 28 { href: '/sources', label: 'Sources' },
18 { href: '/methodology', label: 'Methodology' }, 29 { href: '/methodology', label: 'Methodology' },
19 ] as const; 30 ] as const;
@@ -21,10 +32,14 @@ export const NAV = [
21 export const FOOTER_NAV = [ 32 export const FOOTER_NAV = [
22 { href: '/about', label: 'About' }, 33 { href: '/about', label: 'About' },
23 { href: '/trust', label: 'Trust & policies' }, 34 { href: '/trust', label: 'Trust & policies' },
35+ { href: '/explore', label: 'Data explorer' },
24 { href: '/data', label: 'Data downloads' }, 36 { href: '/data', label: 'Data downloads' },
25− { href: '/developers', label: 'Developers' }, 37+ { href: '/developers', label: 'Developers (API)' },
26 { href: '/sources', label: 'Sources' }, 38 { href: '/sources', label: 'Sources' },
27 { href: '/methodology', label: 'Methodology' }, 39 { href: '/methodology', label: 'Methodology' },
40+ { href: '/taxonomy', label: 'Taxonomy' },
41+ { href: '/countries', label: 'Countries' },
42+ { href: '/compare', label: 'Compare' },
28 ] as const; 43 ] as const;
29 44
30 /** External identifier browsers (chips on cancer pages link out to the authority). */ 45 /** External identifier browsers (chips on cancer pages link out to the authority). */
added packages/connectors/src/connectors/clinicaltrials/drugs.test.ts +39 −0
@@ -0,0 +1,39 @@
1+import { describe, expect, it } from 'vitest';
2+import { stripToMolecule, NON_DRUG_RE } from './drugs.js';
3+
4+describe('stripToMolecule (trial intervention → molecule)', () => {
5+ it('strips leading registry labels', () => {
6+ expect(stripToMolecule('Drug: Cisplatin')).toBe('cisplatin');
7+ expect(stripToMolecule('Biological: Pembrolizumab')).toBe('pembrolizumab');
8+ });
9+ it('strips salts and esters', () => {
10+ expect(stripToMolecule('Doxorubicin Hydrochloride')).toBe('doxorubicin');
11+ expect(stripToMolecule('Gemcitabine Hydrochloride Injection')).toBe('gemcitabine');
12+ expect(stripToMolecule('Imatinib Mesylate')).toBe('imatinib');
13+ expect(stripToMolecule('Vincristine Sulfate')).toBe('vincristine');
14+ });
15+ it('strips doses, routes and parentheticals', () => {
16+ expect(stripToMolecule('Pembrolizumab 200 mg IV')).toBe('pembrolizumab');
17+ expect(stripToMolecule('Pembrolizumab (MK-3475)')).toBe('pembrolizumab');
18+ expect(stripToMolecule('Oral capecitabine 1000 mg/m2')).toBe('capecitabine');
19+ });
20+ it('returns null when nothing changes (pass 1 already covered it)', () => {
21+ expect(stripToMolecule('Pembrolizumab')).toBeNull();
22+ expect(stripToMolecule('nab-paclitaxel')).toBeNull();
23+ });
24+ it('never collapses a distinct product to its parent molecule', () => {
25+ // "protein bound" is not a salt/formulation token: the qualifier is kept, so the label is either
26+ // unchanged (null → stays in pass 1 territory) or only loses its dose.
27+ expect(stripToMolecule('Paclitaxel protein-bound')).toBeNull();
28+ expect(stripToMolecule('Paclitaxel protein-bound 100 mg/m2')).toBe('paclitaxel protein bound');
29+ });
30+});
31+
32+describe('NON_DRUG_RE', () => {
33+ it('flags placebo / procedures / care as non-drugs', () => {
34+ for (const s of ['Placebo', 'placebo matching pembrolizumab', 'Standard of care', 'Best Supportive Care', 'Surgery', 'Radiation therapy']) expect(NON_DRUG_RE.test(s)).toBe(true);
35+ });
36+ it('lets molecule names through', () => {
37+ for (const s of ['Cisplatin', 'Dexamethasone', 'Placebo-controlled nivolumab']) expect(NON_DRUG_RE.test(s)).toBe(false);
38+ });
39+});
added packages/connectors/src/connectors/clinicaltrials/drugs.ts +265 −0
@@ -0,0 +1,265 @@
1+import { sql } from 'drizzle-orm';
2+import type { Database } from '@cancerindex/database';
3+import { normalizeLabel } from '@cancerindex/shared';
4+
5+/**
6+ * Trial intervention → canonical drug reconciliation (CLAUDE.md §69, §221; SPEC §9, §97).
7+ *
8+ * Registrants type intervention names freely ("Pembrolizumab", "MK-3475", "Keytruda 200 mg",
9+ * "Drug: cisplatin", "Doxorubicin Hydrochloride"). We reconcile in three deterministic passes, each
10+ * recorded as its own `match_type` so the UI can show how a link was made:
11+ *
12+ * 1. ALIAS — normalized name equals a `drug_aliases.normalized` (generic, brand,
13+ * development code or synonym) that maps to exactly one drug.
14+ * 2. ALIAS — the same after stripping registry noise: a leading "drug:"/"biological:"
15+ * label, dose/route suffixes ("200 mg", "iv", "tablets"), and a trailing salt
16+ * or ester ("hydrochloride", "sulfate", "acetate"…). Stored with `via` in the
17+ * normalized form so the strip is auditable.
18+ * 3. PROBABILISTIC — the first word of a multi-word name is itself an unambiguous generic alias
19+ * (e.g. "paclitaxel protein bound" → paclitaxel is *not* accepted here because
20+ * nab-paclitaxel is a distinct product; the pass is limited to names whose
21+ * remaining words are dose/route/formulation tokens only).
22+ *
23+ * Aliases that point to several drugs are skipped (ambiguous) — never guessed. Placebo, "standard
24+ * of care", procedures and classes are left UNRESOLVED on purpose; they are not drugs.
25+ *
26+ * Everything is set-based SQL (fast on ~250 k rows) and idempotent: rows already mapped are left
27+ * untouched unless `remap` is requested (after an alias-set change).
28+ */
29+
30+/** Salts / esters / counter-ions that name the same molecule (trailing tokens). */
31+export const SALT_TOKENS = [
32+ 'hydrochloride',
33+ 'dihydrochloride',
34+ 'hcl',
35+ 'sulfate',
36+ 'sulphate',
37+ 'acetate',
38+ 'sodium',
39+ 'disodium',
40+ 'potassium',
41+ 'calcium',
42+ 'magnesium',
43+ 'mesylate',
44+ 'mesilate',
45+ 'dimesylate',
46+ 'citrate',
47+ 'tartrate',
48+ 'bitartrate',
49+ 'maleate',
50+ 'malate',
51+ 'fumarate',
52+ 'phosphate',
53+ 'diphosphate',
54+ 'succinate',
55+ 'tosylate',
56+ 'besylate',
57+ 'bromide',
58+ 'chloride',
59+ 'lactate',
60+ 'gluconate',
61+ 'pamoate',
62+ 'trihydrate',
63+ 'dihydrate',
64+ 'monohydrate',
65+ 'hydrate',
66+ 'anhydrous',
67+ 'ditosylate',
68+ 'camsylate',
69+ 'hemihydrate',
70+ 'hydrobromide',
71+ 'nitrate',
72+ 'oxalate',
73+ 'propionate',
74+ 'valerate',
75+ 'decanoate',
76+ 'enanthate',
77+ 'undecanoate',
78+ 'cypionate',
79+ 'pivalate',
80+ 'isethionate',
81+ 'trifluoroacetate',
82+];
83+
84+/** Formulation / dose / route tokens that never change the molecule. */
85+export const FORMULATION_TOKENS = [
86+ 'mg',
87+ 'mcg',
88+ 'ug',
89+ 'g',
90+ 'ml',
91+ 'iu',
92+ 'units',
93+ 'unit',
94+ 'tablet',
95+ 'tablets',
96+ 'capsule',
97+ 'capsules',
98+ 'injection',
99+ 'injections',
100+ 'infusion',
101+ 'solution',
102+ 'suspension',
103+ 'oral',
104+ 'iv',
105+ 'intravenous',
106+ 'subcutaneous',
107+ 'sc',
108+ 'im',
109+ 'intramuscular',
110+ 'topical',
111+ 'cream',
112+ 'gel',
113+ 'patch',
114+ 'kg',
115+ 'm2',
116+ 'daily',
117+ 'weekly',
118+ 'bid',
119+ 'qd',
120+ 'dose',
121+ 'doses',
122+ 'low',
123+ 'high',
124+ 'standard',
125+ 'extended',
126+ 'release',
127+ 'er',
128+ 'xr',
129+ 'sr',
130+ 'depot',
131+ 'vial',
132+ 'for',
133+ 'per',
134+ 'x',
135+];
136+
137+const LEADING_LABEL = /^(drug|biological|biologic|medication|agent|intervention|treatment|arm [a-z0-9]+|experimental|active comparator|placebo comparator)\s*:\s*/i;
138+
139+/**
140+ * Pure normalizer for pass 2: label → salt/formulation-stripped normalized string, or null when
141+ * nothing changes (so pass 2 never repeats pass 1) or when the result is empty.
142+ */
143+export function stripToMolecule(label: string): string | null {
144+ let s = label.replace(LEADING_LABEL, '');
145+ s = s.replace(/\([^)]*\)/g, ' '); // parentheticals: "(MK-3475)", "(200 mg)"
146+ s = s.replace(/\[[^\]]*\]/g, ' ');
147+ s = s.replace(/\b\d+(\.\d+)?\s*(mg|mcg|ug|g|ml|iu|units?)(\s*\/\s*(kg|m2|ml|day|d|week|wk|dose))?\b/gi, ' '); // doses
148+ s = s.replace(/\b\d+(\.\d+)?\s*%/g, ' ');
149+ let norm = normalizeLabel(s);
150+ if (!norm) return null;
151+ const tokens = norm.split(' ');
152+ // strip trailing salt / formulation tokens (repeat: "doxorubicin hydrochloride injection")
153+ while (tokens.length > 1) {
154+ const last = tokens[tokens.length - 1]!;
155+ if (SALT_TOKENS.includes(last) || FORMULATION_TOKENS.includes(last) || /^\d+(\.\d+)?$/.test(last)) tokens.pop();
156+ else break;
157+ }
158+ // strip leading formulation tokens ("oral capecitabine", "iv cisplatin")
159+ while (tokens.length > 1) {
160+ const first = tokens[0]!;
161+ if (FORMULATION_TOKENS.includes(first)) tokens.shift();
162+ else break;
163+ }
164+ norm = tokens.join(' ').trim();
165+ if (!norm || norm === normalizeLabel(label)) return null;
166+ return norm;
167+}
168+
169+/** Names that are never a drug entity (kept UNRESOLVED without being queued as drug labels). */
170+export const NON_DRUG_RE = /^(placebo|placebos|saline|normal saline|sham|vehicle|standard of care|standard care|best supportive care|supportive care|observation|no intervention|no treatment|control|usual care|chemotherapy|radiotherapy|radiation|radiation therapy|surgery|questionnaire|survey|blood draw|blood sample|biopsy|imaging|mri|ct scan|pet scan|exercise|diet|counseling|education|physical therapy|acupuncture|massage|yoga|music therapy|hypnosis|dexamethasone placebo)( .*)?$/i;
171+
172+export interface DrugReconcileResult {
173+ pass1: number;
174+ pass2: number;
175+ pass3: number;
176+ ambiguousSkipped: number;
177+ remainingUnresolved: number;
178+ distinctUnresolved: number;
179+}
180+
181+/**
182+ * Set-based reconciliation of `trial_interventions` (DRUG/BIOLOGICAL rows) to `drugs`.
183+ * `remap = true` resets ALIAS/PROBABILISTIC rows first (after alias changes); default only fills
184+ * UNRESOLVED rows.
185+ */
186+export async function reconcileInterventionDrugs(db: Database, opts: { remap?: boolean } = {}): Promise<DrugReconcileResult> {
187+ if (opts.remap) {
188+ await db.execute(sql`UPDATE trial_interventions SET drug_id = NULL, match_type = 'UNRESOLVED' WHERE intervention_type IN ('DRUG','BIOLOGICAL') AND match_type IN ('ALIAS','PROBABILISTIC')`);
189+ }
190+ // Unambiguous alias map: normalized → drug_id (aliases shared by several drugs are excluded).
191+ await db.execute(sql`CREATE TEMP TABLE IF NOT EXISTS _alias_map (normalized text PRIMARY KEY, drug_id varchar(32) NOT NULL)`);
192+ await db.execute(sql`TRUNCATE _alias_map`);
193+ await db.execute(sql`
194+ INSERT INTO _alias_map (normalized, drug_id)
195+ SELECT normalized, min(drug_id) FROM drug_aliases WHERE length(normalized) >= 3
196+ GROUP BY normalized HAVING count(DISTINCT drug_id) = 1`);
197+ const [amb] = await db.execute<{ n: string }>(sql`SELECT count(*)::text AS n FROM (SELECT normalized FROM drug_aliases GROUP BY normalized HAVING count(DISTINCT drug_id) > 1) x`);
198+
199+ // Pass 1: exact normalized alias.
200+ const p1 = await db.execute<{ n: string }>(sql`
201+ WITH u AS (
202+ UPDATE trial_interventions ti SET drug_id = m.drug_id, match_type = 'ALIAS'
203+ FROM _alias_map m
204+ WHERE ti.intervention_type IN ('DRUG','BIOLOGICAL') AND ti.match_type = 'UNRESOLVED' AND ti.normalized = m.normalized
205+ RETURNING 1)
206+ SELECT count(*)::text AS n FROM u`);
207+
208+ // Pass 2: salt/dose/label-stripped form. Distinct labels are stripped in JS (pure function) then
209+ // matched in one statement.
210+ const remaining = await db.execute<{ normalized: string; name: string }>(sql`
211+ SELECT normalized, min(name) AS name FROM trial_interventions
212+ WHERE intervention_type IN ('DRUG','BIOLOGICAL') AND match_type = 'UNRESOLVED' GROUP BY normalized`);
213+ const stripped: Array<[string, string]> = [];
214+ for (const r of remaining) {
215+ if (NON_DRUG_RE.test(r.name)) continue;
216+ const m = stripToMolecule(r.name);
217+ if (m) stripped.push([r.normalized, m]);
218+ }
219+ let p2 = 0;
220+ for (let i = 0; i < stripped.length; i += 2000) {
221+ const chunk = stripped.slice(i, i + 2000);
222+ const res = await db.execute<{ n: string }>(sql`
223+ WITH s(normalized, molecule) AS (SELECT * FROM unnest(${sql.param(chunk.map((c) => c[0]))}::text[], ${sql.param(chunk.map((c) => c[1]))}::text[])),
224+ u AS (
225+ UPDATE trial_interventions ti SET drug_id = m.drug_id, match_type = 'ALIAS'
226+ FROM s JOIN _alias_map m ON m.normalized = s.molecule
227+ WHERE ti.intervention_type IN ('DRUG','BIOLOGICAL') AND ti.match_type = 'UNRESOLVED' AND ti.normalized = s.normalized
228+ RETURNING 1)
229+ SELECT count(*)::text AS n FROM u`);
230+ p2 += Number(res[0]?.n ?? 0);
231+ }
232+
233+ // Pass 3 (PROBABILISTIC): "<generic> <formulation words only>" where the generic is a *generic*
234+ // alias (not brand/synonym) — e.g. "cisplatin 75 mg/m2 iv d1" already handled by pass 2; this
235+ // catches "pembrolizumab injection solution" style remainders whose extra tokens are all
236+ // formulation tokens. Names with any non-formulation extra token stay UNRESOLVED.
237+ const rem2 = await db.execute<{ normalized: string; name: string }>(sql`
238+ SELECT normalized, min(name) AS name FROM trial_interventions
239+ WHERE intervention_type IN ('DRUG','BIOLOGICAL') AND match_type = 'UNRESOLVED' GROUP BY normalized`);
240+ const heads: Array<[string, string]> = [];
241+ for (const r of rem2) {
242+ if (NON_DRUG_RE.test(r.name)) continue;
243+ const tokens = r.normalized.split(' ');
244+ if (tokens.length < 2) continue;
245+ const rest = tokens.slice(1);
246+ if (rest.every((t) => FORMULATION_TOKENS.includes(t) || SALT_TOKENS.includes(t) || /^\d+(\.\d+)?$/.test(t))) heads.push([r.normalized, tokens[0]!]);
247+ }
248+ let p3 = 0;
249+ for (let i = 0; i < heads.length; i += 2000) {
250+ const chunk = heads.slice(i, i + 2000);
251+ const res = await db.execute<{ n: string }>(sql`
252+ WITH s(normalized, head) AS (SELECT * FROM unnest(${sql.param(chunk.map((c) => c[0]))}::text[], ${sql.param(chunk.map((c) => c[1]))}::text[])),
253+ g AS (SELECT a.normalized, min(a.drug_id) AS drug_id FROM drug_aliases a WHERE a.alias_type = 'generic' GROUP BY a.normalized HAVING count(DISTINCT a.drug_id) = 1),
254+ u AS (
255+ UPDATE trial_interventions ti SET drug_id = g.drug_id, match_type = 'PROBABILISTIC'
256+ FROM s JOIN g ON g.normalized = s.head
257+ WHERE ti.intervention_type IN ('DRUG','BIOLOGICAL') AND ti.match_type = 'UNRESOLVED' AND ti.normalized = s.normalized
258+ RETURNING 1)
259+ SELECT count(*)::text AS n FROM u`);
260+ p3 += Number(res[0]?.n ?? 0);
261+ }
262+
263+ const [left] = await db.execute<{ n: string; d: string }>(sql`SELECT count(*)::text AS n, count(DISTINCT normalized)::text AS d FROM trial_interventions WHERE intervention_type IN ('DRUG','BIOLOGICAL') AND match_type = 'UNRESOLVED'`);
264+ return { pass1: Number(p1[0]?.n ?? 0), pass2: p2, pass3: p3, ambiguousSkipped: Number(amb?.n ?? 0), remainingUnresolved: Number(left?.n ?? 0), distinctUnresolved: Number(left?.d ?? 0) };
265+}
modified packages/connectors/src/connectors/clinicaltrials/index.ts +4 −0
@@ -8,10 +8,12 @@ import { manifest } from './manifest.js';
8 import { ONCOLOGY_EXPRESSION, STUDY_FIELDS, normalizeStudy, trimStudyPayload, type ApiPage, type ApiStudy, type ApiVersion, type NormalizedTrial } from './normalize.js'; 8 import { ONCOLOGY_EXPRESSION, STUDY_FIELDS, normalizeStudy, trimStudyPayload, type ApiPage, type ApiStudy, type ApiVersion, type NormalizedTrial } from './normalize.js';
9 import { reconcileConditions, type ConditionMapping, type ResolverLike } from './reconcile.js'; 9 import { reconcileConditions, type ConditionMapping, type ResolverLike } from './reconcile.js';
10 import { refreshTrialPulse } from './pulse.js'; 10 import { refreshTrialPulse } from './pulse.js';
11+import { reconcileInterventionDrugs } from './drugs.js';
11 12
12 export { ONCOLOGY_EXPRESSION, STUDY_FIELDS, normalizeStudy, trimStudyPayload } from './normalize.js'; 13 export { ONCOLOGY_EXPRESSION, STUDY_FIELDS, normalizeStudy, trimStudyPayload } from './normalize.js';
13 export { reconcileConditions, conditionCandidates } from './reconcile.js'; 14 export { reconcileConditions, conditionCandidates } from './reconcile.js';
14 export { refreshTrialPulse } from './pulse.js'; 15 export { refreshTrialPulse } from './pulse.js';
16+export { reconcileInterventionDrugs, stripToMolecule, NON_DRUG_RE } from './drugs.js';
15 17
16 export const PAGE_SIZE = 1000; 18 export const PAGE_SIZE = 1000;
17 /** Trigram suggestions are computed for at most this many unresolved labels per run (curation aid only). */ 19 /** Trigram suggestions are computed for at most this many unresolved labels per run (curation aid only). */
@@ -200,6 +202,8 @@ export class ClinicalTrialsConnector extends Connector {
200 const re = await this.rereconcileUnresolved(ctx, resolver); 202 const re = await this.rereconcileUnresolved(ctx, resolver);
201 const pulse = await refreshTrialPulse(ctx.db); 203 const pulse = await refreshTrialPulse(ctx.db);
202 ctx.info(`re-reconciled ${re.labels} unresolved labels → ${re.rows} trial_conditions rows updated; trial_pulse rows ${pulse}`); 204 ctx.info(`re-reconciled ${re.labels} unresolved labels → ${re.rows} trial_conditions rows updated; trial_pulse rows ${pulse}`);
205+ const dr = await reconcileInterventionDrugs(ctx.db);
206+ ctx.info(`interventions → drugs: alias ${dr.pass1}, stripped ${dr.pass2}, probabilistic ${dr.pass3}; ambiguous aliases skipped ${dr.ambiguousSkipped}; still unresolved ${dr.remainingUnresolved} rows (${dr.distinctUnresolved} distinct labels)`);
203 } 207 }
204 const pct = agg.conditionsTotal ? Math.round((agg.conditionsResolved / agg.conditionsTotal) * 1000) / 10 : 0; 208 const pct = agg.conditionsTotal ? Math.round((agg.conditionsResolved / agg.conditionsTotal) * 1000) / 10 : 0;
205 const top = [...agg.unresolved.values()].sort((a, b) => b.count - a.count).slice(0, 15).map((u) => `${u.sourceText} (${u.count})`); 209 const top = [...agg.unresolved.values()].sort((a, b) => b.count - a.count).slice(0, 15).map((u) => `${u.sourceText} (${u.count})`);
modified packages/connectors/src/index.ts +1 −0
@@ -3,3 +3,4 @@ export * from './registry.js';
3 export * from './sources-sync.js'; 3 export * from './sources-sync.js';
4 export * from './planned.js'; 4 export * from './planned.js';
5 export * from './ops/index.js'; 5 export * from './ops/index.js';
6+export { reconcileInterventionDrugs, stripToMolecule, NON_DRUG_RE, SALT_TOKENS, FORMULATION_TOKENS } from './connectors/clinicaltrials/drugs.js';
added packages/database/migrations/0002_dizzy_pete_wisdom.sql +126 −0
@@ -0,0 +1,126 @@
1+CREATE TABLE "drug_codes" (
2+ "id" bigserial PRIMARY KEY NOT NULL,
3+ "drug_id" varchar(32) NOT NULL,
4+ "system" text NOT NULL,
5+ "code" text NOT NULL,
6+ "label" text,
7+ "match_type" text DEFAULT 'EXACT_IDENTIFIER' NOT NULL,
8+ "source_id" varchar(32)
9+);
10+--> statement-breakpoint
11+CREATE TABLE "drug_pipeline" (
12+ "id" bigserial PRIMARY KEY NOT NULL,
13+ "drug_id" varchar(32) NOT NULL,
14+ "cancer_id" varchar(32),
15+ "stage" text NOT NULL,
16+ "max_phase" text,
17+ "active_trials" integer DEFAULT 0 NOT NULL,
18+ "recruiting_trials" integer DEFAULT 0 NOT NULL,
19+ "phase3_trials" integer DEFAULT 0 NOT NULL,
20+ "total_trials" integer DEFAULT 0 NOT NULL,
21+ "approvals" integer DEFAULT 0 NOT NULL,
22+ "jurisdictions" text[] DEFAULT '{}' NOT NULL,
23+ "first_approval_date" text,
24+ "latest_approval_date" text,
25+ "first_trial_date" text,
26+ "formula_version" text NOT NULL,
27+ "inputs" jsonb DEFAULT '{}'::jsonb NOT NULL,
28+ "updated_at" timestamp with time zone DEFAULT now() NOT NULL
29+);
30+--> statement-breakpoint
31+CREATE TABLE "research_gap_components" (
32+ "id" bigserial PRIMARY KEY NOT NULL,
33+ "cancer_id" varchar(32) NOT NULL,
34+ "geography" text NOT NULL,
35+ "year" integer NOT NULL,
36+ "sex" text DEFAULT 'all' NOT NULL,
37+ "burden_source_id" varchar(32) NOT NULL,
38+ "deaths" double precision,
39+ "incidence" double precision,
40+ "active_trials" integer DEFAULT 0 NOT NULL,
41+ "phase3_trials" integer DEFAULT 0 NOT NULL,
42+ "publications_5y" integer DEFAULT 0 NOT NULL,
43+ "approved_drugs" integer DEFAULT 0 NOT NULL,
44+ "death_share" real,
45+ "trial_share" real,
46+ "publication_share" real,
47+ "trial_gap_ratio" real,
48+ "research_gap_ratio" real,
49+ "trials_per_1000_deaths" real,
50+ "publications_per_1000_deaths" real,
51+ "eligible" boolean DEFAULT true NOT NULL,
52+ "ineligible_reason" text,
53+ "formula_version" text NOT NULL,
54+ "inputs" jsonb DEFAULT '{}'::jsonb NOT NULL,
55+ "updated_at" timestamp with time zone DEFAULT now() NOT NULL
56+);
57+--> statement-breakpoint
58+CREATE TABLE "trial_intelligence" (
59+ "id" bigserial PRIMARY KEY NOT NULL,
60+ "cancer_id" varchar(32) NOT NULL,
61+ "entity_level" text DEFAULT 'all' NOT NULL,
62+ "total_trials" integer DEFAULT 0 NOT NULL,
63+ "active_trials" integer DEFAULT 0 NOT NULL,
64+ "recruiting_trials" integer DEFAULT 0 NOT NULL,
65+ "phase1_active" integer DEFAULT 0 NOT NULL,
66+ "phase2_active" integer DEFAULT 0 NOT NULL,
67+ "phase3_active" integer DEFAULT 0 NOT NULL,
68+ "phase3_recruiting" integer DEFAULT 0 NOT NULL,
69+ "phase4_active" integer DEFAULT 0 NOT NULL,
70+ "completed_trials" integer DEFAULT 0 NOT NULL,
71+ "terminated_trials" integer DEFAULT 0 NOT NULL,
72+ "withdrawn_trials" integer DEFAULT 0 NOT NULL,
73+ "suspended_trials" integer DEFAULT 0 NOT NULL,
74+ "with_results" integer DEFAULT 0 NOT NULL,
75+ "new_trials_12m" integer DEFAULT 0 NOT NULL,
76+ "new_trials_prior_12m" integer DEFAULT 0 NOT NULL,
77+ "trial_growth_yoy" real,
78+ "avg_enrollment" real,
79+ "median_enrollment" real,
80+ "total_enrollment_active" integer,
81+ "distinct_sponsors" integer DEFAULT 0 NOT NULL,
82+ "industry_share" real,
83+ "sponsor_hhi" real,
84+ "top_sponsor" text,
85+ "top_sponsor_share" real,
86+ "distinct_countries" integer DEFAULT 0 NOT NULL,
87+ "us_share" real,
88+ "top_country" text,
89+ "top_country_share" real,
90+ "country_hhi" real,
91+ "termination_share" real,
92+ "why_stopped_breakdown" jsonb DEFAULT '{}'::jsonb NOT NULL,
93+ "trials_per_1000_deaths" real,
94+ "trials_per_100k_cases" real,
95+ "burden_geography" text,
96+ "burden_year" integer,
97+ "burden_source_id" varchar(32),
98+ "formula_version" text NOT NULL,
99+ "inputs" jsonb DEFAULT '{}'::jsonb NOT NULL,
100+ "updated_at" timestamp with time zone DEFAULT now() NOT NULL
101+);
102+--> statement-breakpoint
103+CREATE TABLE "trial_site_country_counts" (
104+ "id" bigserial PRIMARY KEY NOT NULL,
105+ "cancer_id" varchar(32),
106+ "phase" text,
107+ "recruiting_only" boolean DEFAULT false NOT NULL,
108+ "country" text NOT NULL,
109+ "iso3" text,
110+ "sites" integer NOT NULL,
111+ "trials" integer NOT NULL,
112+ "formula_version" text NOT NULL,
113+ "updated_at" timestamp with time zone DEFAULT now() NOT NULL
114+);
115+--> statement-breakpoint
116+CREATE UNIQUE INDEX "drug_codes_uq" ON "drug_codes" USING btree ("drug_id","system","code");--> statement-breakpoint
117+CREATE INDEX "drug_codes_lookup_idx" ON "drug_codes" USING btree ("system","code");--> statement-breakpoint
118+CREATE UNIQUE INDEX "drug_pipeline_uq" ON "drug_pipeline" USING btree ("drug_id","cancer_id");--> statement-breakpoint
119+CREATE INDEX "drug_pipeline_stage_idx" ON "drug_pipeline" USING btree ("stage");--> statement-breakpoint
120+CREATE INDEX "drug_pipeline_cancer_idx" ON "drug_pipeline" USING btree ("cancer_id","stage");--> statement-breakpoint
121+CREATE UNIQUE INDEX "research_gap_components_uq" ON "research_gap_components" USING btree ("cancer_id","geography","year","sex","burden_source_id");--> statement-breakpoint
122+CREATE INDEX "research_gap_scope_idx" ON "research_gap_components" USING btree ("geography","year","sex");--> statement-breakpoint
123+CREATE UNIQUE INDEX "trial_intelligence_uq" ON "trial_intelligence" USING btree ("cancer_id","entity_level");--> statement-breakpoint
124+CREATE INDEX "trial_intelligence_active_idx" ON "trial_intelligence" USING btree ("entity_level","active_trials");--> statement-breakpoint
125+CREATE UNIQUE INDEX "trial_site_country_uq" ON "trial_site_country_counts" USING btree ("cancer_id","phase","recruiting_only","country");--> statement-breakpoint
126+CREATE INDEX "trial_site_country_lookup_idx" ON "trial_site_country_counts" USING btree ("cancer_id","phase","recruiting_only");
\ No newline at end of file
added packages/database/migrations/meta/0002_snapshot.json +8029 −0
Large diff (8,030 lines) is not loaded by default to keep the page fast.Load diff
modified packages/database/migrations/meta/_journal.json +7 −0
Large diff (14 lines) is not loaded by default to keep the page fast.Load diff
modified packages/database/src/schema/index.ts +1 −0
Large diff (4 lines) is not loaded by default to keep the page fast.Load diff
added packages/database/src/schema/intelligence.ts +164 −0
Large diff (164 lines) is not loaded by default to keep the page fast.Load diff
modified packages/database/src/seed-data/metrics.ts +122 −0
Large diff (126 lines) is not loaded by default to keep the page fast.Load diff
added packages/ranking/src/drug-pipeline.ts +16 −0
Large diff (16 lines) is not loaded by default to keep the page fast.Load diff
modified packages/ranking/src/engine.ts +1 −1
Large diff (8 lines) is not loaded by default to keep the page fast.Load diff
modified packages/ranking/src/index.ts +4 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added packages/ranking/src/intelligence.ts +48 −0
Large diff (48 lines) is not loaded by default to keep the page fast.Load diff
added packages/ranking/src/research-gap.ts +18 −0
Large diff (18 lines) is not loaded by default to keep the page fast.Load diff
added packages/ranking/src/trial-intelligence.ts +17 −0
Large diff (17 lines) is not loaded by default to keep the page fast.Load diff
modified scripts/ci.ts +13 −1
Large diff (26 lines) is not loaded by default to keep the page fast.Load diff
modified workers/lib/env.ts +2 −0
Large diff (14 lines) is not loaded by default to keep the page fast.Load diff
modified workers/main.ts +20 −3
Large diff (62 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere