SPB Git
spboucher.ai

spb/cancerindex

Public

Foundation for wave 3: intelligence schema (trial_intelligence, trial_site_country_counts, drug_pipeline, drug_codes, research_gap_components; migration 0002), trial intervention→drug reconciliation (alias/salt-stripped/probabilistic), new metric definitions (trial growth, termination share, sponsor HHI, per-1000-deaths, gap ratios), maintenance.intel job, API route stubs (epidemiology/approvals/graph/intelligence), navigation restructure

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 11, 2026) parent 718f988 Browse files

27 changed files +8,980 −12

27 changed files
Load all 27 files Unified Split
modified apps/api/src/app.ts +12 −0
@@ -26,6 +26,10 @@ import { sourceRoutes } from './routes/sources.js';
26 26 import { statsRoutes } from './routes/stats.js';
27 27 import { changeRoutes } from './routes/changes.js';
28 28 import { adminRoutes } from './routes/admin.js';
29 +import { graphRoutes } from './routes/graph.js';
30 +import { epidemiologyRoutes } from './routes/epidemiology.js';
31 +import { approvalRoutes } from './routes/approvals.js';
32 +import { intelligenceRoutes } from './routes/intelligence.js';
29 33
30 34 export interface BuildOptions {
31 35 db?: Database;
@@ -103,6 +107,10 @@ export async function buildApp(opts: BuildOptions = {}): Promise<FastifyInstance
103 107 { name: 'sources', description: 'Source registry, licenses, connector health' },
104 108 { name: 'stats' },
105 109 { name: 'changes' },
110 + { name: 'epidemiology', description: 'Time-aware incidence/mortality/prevalence observations with provenance (Data Explorer)' },
111 + { name: 'approvals', description: 'Jurisdiction-aware regulatory approvals and the drug development pipeline' },
112 + { name: 'graph', description: 'Cancer–gene–variant–drug–trial knowledge graph (contextual neighbourhoods)' },
113 + { name: 'intelligence', description: 'Derived clinical-trial intelligence and research-gap components (computed metrics with formula versions)' },
106 114 { name: 'admin', description: 'Operator endpoints (x-admin-token)' },
107 115 { name: 'system' },
108 116 ],
@@ -152,6 +160,10 @@ export async function buildApp(opts: BuildOptions = {}): Promise<FastifyInstance
152 160 await v1.register(sourceRoutes);
153 161 await v1.register(statsRoutes);
154 162 await v1.register(changeRoutes);
163 + await v1.register(epidemiologyRoutes);
164 + await v1.register(approvalRoutes);
165 + await v1.register(graphRoutes);
166 + await v1.register(intelligenceRoutes);
155 167 await v1.register(adminRoutes, { prefix: '/admin' });
156 168 },
157 169 { prefix: '/v1' },
modified apps/api/src/lib/queue.ts +1 −0
@@ -8,6 +8,7 @@ export const JOBS = {
8 8 connectorRun: 'connector.run',
9 9 counters: 'maintenance.counters',
10 10 rank: 'maintenance.rank',
11 + intel: 'maintenance.intel',
11 12 healthProbe: 'health.probe',
12 13 } as const;
13 14 export type JobName = (typeof JOBS)[keyof typeof JOBS];
added apps/api/src/routes/approvals.ts +10 −0
@@ -0,0 +1,10 @@
1 +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
2 +
3 +/**
4 + * Regulatory routes (SPEC §13-15, §63): `GET /approvals` (jurisdiction-aware, dated, filterable by
5 + * authority, cancer, drug, date range), `GET /approvals/recent`, `GET /pipeline` (drug development
6 + * stages). Filled by the Approvals & Pipeline work package.
7 + */
8 +export const approvalRoutes: FastifyPluginAsyncZod = async (_app) => {
9 + /* routes added by the approvals work package */
10 +};
added apps/api/src/routes/epidemiology.ts +10 −0
@@ -0,0 +1,10 @@
1 +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
2 +
3 +/**
4 + * Epidemiology routes (SPEC §20, §63, §110): `GET /epidemiology` → observations filtered by metric,
5 + * cancer, geography, sex, age group and year range; `GET /epidemiology/coverage` → which
6 + * metric × geography × year combinations exist. Filled by the Data Explorer work package.
7 + */
8 +export const epidemiologyRoutes: FastifyPluginAsyncZod = async (_app) => {
9 + /* routes added by the data-explorer work package */
10 +};
added apps/api/src/routes/graph.ts +10 −0
@@ -0,0 +1,10 @@
1 +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
2 +
3 +/**
4 + * Knowledge-graph routes (SPEC §18, §80): `GET /graph/:type/:id` → contextual neighbours of one
5 + * entity (cancer | gene | variant | drug | biomarker | trial) with relationship type, cancer context,
6 + * evidence level and provenance on every edge. Filled by the Knowledge Graph work package.
7 + */
8 +export const graphRoutes: FastifyPluginAsyncZod = async (_app) => {
9 + /* routes added by the knowledge-graph work package */
10 +};
added apps/api/src/routes/intelligence.ts +18 −0
@@ -0,0 +1,18 @@
1 +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
2 +
3 +/**
4 + * Intelligence routes (SPEC §10-11, §34): `GET /trials/intelligence` (per-cancer trial metrics),
5 + * `GET /trials/sites` (country/city aggregates for the map), `GET /research-gap` (components and
6 + * ratios per scope). Filled by the Trial Intelligence and Research Gap work packages
7 + * (each adds its own routes to this file; keep route groups in separate functions).
8 + */
9 +export const intelligenceRoutes: FastifyPluginAsyncZod = async (app) => {
10 + await app.register(trialIntelligenceRoutes);
11 + await app.register(researchGapRoutes);
12 +};
13 +
14 +/** Filled by the Trial Intelligence work package. */
15 +export const trialIntelligenceRoutes: FastifyPluginAsyncZod = async (_app) => {};
16 +
17 +/** Filled by the Research Gap work package. */
18 +export const researchGapRoutes: FastifyPluginAsyncZod = async (_app) => {};
modified apps/web/src/components/layout/mobile-nav.tsx +2 −2
@@ -1,6 +1,6 @@
1 1 import Link from 'next/link';
2 2 import { Menu } from 'lucide-react';
3 −import { NAV, FOOTER_NAV } from '@/lib/site';
3 +import { NAV, MORE_NAV, FOOTER_NAV } from '@/lib/site';
4 4
5 5 /** CSS-only disclosure menu for small screens (no client JS). */
6 6 export function MobileNav() {
@@ -16,7 +16,7 @@ export function MobileNav() {
16 16 </Link>
17 17 ))}
18 18 <div className="my-1 border-t border-rule" />
19 − {FOOTER_NAV.filter((f) => !NAV.some((n) => n.href === f.href)).map((n) => (
19 + {[...MORE_NAV, ...FOOTER_NAV.filter((f) => !NAV.some((n) => n.href === f.href) && !MORE_NAV.some((m) => m.href === f.href))].map((n) => (
20 20 <Link key={n.href} href={n.href} className="block px-3 py-1.5 text-[13px] text-ink-2 no-underline hover:bg-paper-2">
21 21 {n.label}
22 22 </Link>
modified apps/web/src/lib/site.ts +20 −5
@@ -7,13 +7,24 @@ export const CONTACT_EMAIL = 'corrections@cancerindex.io';
7 7
8 8 export const NAV = [
9 9 { href: '/cancers', label: 'Cancers' },
10 − { href: '/taxonomy', label: 'Taxonomy' },
10 + { href: '/explore', label: 'Data' },
11 + { href: '/trials', label: 'Trials' },
12 + { href: '/drugs', label: 'Drugs' },
13 + { href: '/genes', label: 'Genes' },
14 + { href: '/approvals', label: 'Approvals' },
11 15 { href: '/rankings', label: 'Rankings' },
16 + { href: '/research-gap', label: 'Research gap' },
17 + { href: '/graph', label: 'Graph' },
18 +] as const;
19 +
20 +/** Secondary navigation (mobile menu + footer): everything not in the primary bar. */
21 +export const MORE_NAV = [
22 + { href: '/taxonomy', label: 'Taxonomy' },
12 23 { href: '/countries', label: 'Countries' },
13 24 { href: '/compare', label: 'Compare' },
14 − { href: '/genes', label: 'Genes' },
15 − { href: '/drugs', label: 'Drugs' },
16 − { href: '/trials', label: 'Trials' },
25 + { href: '/trials/intelligence', label: 'Trial intelligence' },
26 + { href: '/trials/map', label: 'Trial map' },
27 + { href: '/pipeline', label: 'Drug pipeline' },
17 28 { href: '/sources', label: 'Sources' },
18 29 { href: '/methodology', label: 'Methodology' },
19 30 ] as const;
@@ -21,10 +32,14 @@ export const NAV = [
21 32 export const FOOTER_NAV = [
22 33 { href: '/about', label: 'About' },
23 34 { href: '/trust', label: 'Trust & policies' },
35 + { href: '/explore', label: 'Data explorer' },
24 36 { href: '/data', label: 'Data downloads' },
25 − { href: '/developers', label: 'Developers' },
37 + { href: '/developers', label: 'Developers (API)' },
26 38 { href: '/sources', label: 'Sources' },
27 39 { href: '/methodology', label: 'Methodology' },
40 + { href: '/taxonomy', label: 'Taxonomy' },
41 + { href: '/countries', label: 'Countries' },
42 + { href: '/compare', label: 'Compare' },
28 43 ] as const;
29 44
30 45 /** External identifier browsers (chips on cancer pages link out to the authority). */
added packages/connectors/src/connectors/clinicaltrials/drugs.test.ts +39 −0
@@ -0,0 +1,39 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { stripToMolecule, NON_DRUG_RE } from './drugs.js';
3 +
4 +describe('stripToMolecule (trial intervention → molecule)', () => {
5 + it('strips leading registry labels', () => {
6 + expect(stripToMolecule('Drug: Cisplatin')).toBe('cisplatin');
7 + expect(stripToMolecule('Biological: Pembrolizumab')).toBe('pembrolizumab');
8 + });
9 + it('strips salts and esters', () => {
10 + expect(stripToMolecule('Doxorubicin Hydrochloride')).toBe('doxorubicin');
11 + expect(stripToMolecule('Gemcitabine Hydrochloride Injection')).toBe('gemcitabine');
12 + expect(stripToMolecule('Imatinib Mesylate')).toBe('imatinib');
13 + expect(stripToMolecule('Vincristine Sulfate')).toBe('vincristine');
14 + });
15 + it('strips doses, routes and parentheticals', () => {
16 + expect(stripToMolecule('Pembrolizumab 200 mg IV')).toBe('pembrolizumab');
17 + expect(stripToMolecule('Pembrolizumab (MK-3475)')).toBe('pembrolizumab');
18 + expect(stripToMolecule('Oral capecitabine 1000 mg/m2')).toBe('capecitabine');
19 + });
20 + it('returns null when nothing changes (pass 1 already covered it)', () => {
21 + expect(stripToMolecule('Pembrolizumab')).toBeNull();
22 + expect(stripToMolecule('nab-paclitaxel')).toBeNull();
23 + });
24 + it('never collapses a distinct product to its parent molecule', () => {
25 + // "protein bound" is not a salt/formulation token: the qualifier is kept, so the label is either
26 + // unchanged (null → stays in pass 1 territory) or only loses its dose.
27 + expect(stripToMolecule('Paclitaxel protein-bound')).toBeNull();
28 + expect(stripToMolecule('Paclitaxel protein-bound 100 mg/m2')).toBe('paclitaxel protein bound');
29 + });
30 +});
31 +
32 +describe('NON_DRUG_RE', () => {
33 + it('flags placebo / procedures / care as non-drugs', () => {
34 + for (const s of ['Placebo', 'placebo matching pembrolizumab', 'Standard of care', 'Best Supportive Care', 'Surgery', 'Radiation therapy']) expect(NON_DRUG_RE.test(s)).toBe(true);
35 + });
36 + it('lets molecule names through', () => {
37 + for (const s of ['Cisplatin', 'Dexamethasone', 'Placebo-controlled nivolumab']) expect(NON_DRUG_RE.test(s)).toBe(false);
38 + });
39 +});
added packages/connectors/src/connectors/clinicaltrials/drugs.ts +265 −0
@@ -0,0 +1,265 @@
1 +import { sql } from 'drizzle-orm';
2 +import type { Database } from '@cancerindex/database';
3 +import { normalizeLabel } from '@cancerindex/shared';
4 +
5 +/**
6 + * Trial intervention → canonical drug reconciliation (CLAUDE.md §69, §221; SPEC §9, §97).
7 + *
8 + * Registrants type intervention names freely ("Pembrolizumab", "MK-3475", "Keytruda 200 mg",
9 + * "Drug: cisplatin", "Doxorubicin Hydrochloride"). We reconcile in three deterministic passes, each
10 + * recorded as its own `match_type` so the UI can show how a link was made:
11 + *
12 + * 1. ALIAS — normalized name equals a `drug_aliases.normalized` (generic, brand,
13 + * development code or synonym) that maps to exactly one drug.
14 + * 2. ALIAS — the same after stripping registry noise: a leading "drug:"/"biological:"
15 + * label, dose/route suffixes ("200 mg", "iv", "tablets"), and a trailing salt
16 + * or ester ("hydrochloride", "sulfate", "acetate"…). Stored with `via` in the
17 + * normalized form so the strip is auditable.
18 + * 3. PROBABILISTIC — the first word of a multi-word name is itself an unambiguous generic alias
19 + * (e.g. "paclitaxel protein bound" → paclitaxel is *not* accepted here because
20 + * nab-paclitaxel is a distinct product; the pass is limited to names whose
21 + * remaining words are dose/route/formulation tokens only).
22 + *
23 + * Aliases that point to several drugs are skipped (ambiguous) — never guessed. Placebo, "standard
24 + * of care", procedures and classes are left UNRESOLVED on purpose; they are not drugs.
25 + *
26 + * Everything is set-based SQL (fast on ~250 k rows) and idempotent: rows already mapped are left
27 + * untouched unless `remap` is requested (after an alias-set change).
28 + */
29 +
30 +/** Salts / esters / counter-ions that name the same molecule (trailing tokens). */
31 +export const SALT_TOKENS = [
32 + 'hydrochloride',
33 + 'dihydrochloride',
34 + 'hcl',
35 + 'sulfate',
36 + 'sulphate',
37 + 'acetate',
38 + 'sodium',
39 + 'disodium',
40 + 'potassium',
41 + 'calcium',
42 + 'magnesium',
43 + 'mesylate',
44 + 'mesilate',
45 + 'dimesylate',
46 + 'citrate',
47 + 'tartrate',
48 + 'bitartrate',
49 + 'maleate',
50 + 'malate',
51 + 'fumarate',
52 + 'phosphate',
53 + 'diphosphate',
54 + 'succinate',
55 + 'tosylate',
56 + 'besylate',
57 + 'bromide',
58 + 'chloride',
59 + 'lactate',
60 + 'gluconate',
61 + 'pamoate',
62 + 'trihydrate',
63 + 'dihydrate',
64 + 'monohydrate',
65 + 'hydrate',
66 + 'anhydrous',
67 + 'ditosylate',
68 + 'camsylate',
69 + 'hemihydrate',
70 + 'hydrobromide',
71 + 'nitrate',
72 + 'oxalate',
73 + 'propionate',
74 + 'valerate',
75 + 'decanoate',
76 + 'enanthate',
77 + 'undecanoate',
78 + 'cypionate',
79 + 'pivalate',
80 + 'isethionate',
81 + 'trifluoroacetate',
82 +];
83 +
84 +/** Formulation / dose / route tokens that never change the molecule. */
85 +export const FORMULATION_TOKENS = [
86 + 'mg',
87 + 'mcg',
88 + 'ug',
89 + 'g',
90 + 'ml',
91 + 'iu',
92 + 'units',
93 + 'unit',
94 + 'tablet',
95 + 'tablets',
96 + 'capsule',
97 + 'capsules',
98 + 'injection',
99 + 'injections',
100 + 'infusion',
101 + 'solution',
102 + 'suspension',
103 + 'oral',
104 + 'iv',
105 + 'intravenous',
106 + 'subcutaneous',
107 + 'sc',
108 + 'im',
109 + 'intramuscular',
110 + 'topical',
111 + 'cream',
112 + 'gel',
113 + 'patch',
114 + 'kg',
115 + 'm2',
116 + 'daily',
117 + 'weekly',
118 + 'bid',
119 + 'qd',
120 + 'dose',
121 + 'doses',
122 + 'low',
123 + 'high',
124 + 'standard',
125 + 'extended',
126 + 'release',
127 + 'er',
128 + 'xr',
129 + 'sr',
130 + 'depot',
131 + 'vial',
132 + 'for',
133 + 'per',
134 + 'x',
135 +];
136 +
137 +const LEADING_LABEL = /^(drug|biological|biologic|medication|agent|intervention|treatment|arm [a-z0-9]+|experimental|active comparator|placebo comparator)\s*:\s*/i;
138 +
139 +/**
140 + * Pure normalizer for pass 2: label → salt/formulation-stripped normalized string, or null when
141 + * nothing changes (so pass 2 never repeats pass 1) or when the result is empty.
142 + */
143 +export function stripToMolecule(label: string): string | null {
144 + let s = label.replace(LEADING_LABEL, '');
145 + s = s.replace(/\([^)]*\)/g, ' '); // parentheticals: "(MK-3475)", "(200 mg)"
146 + s = s.replace(/\[[^\]]*\]/g, ' ');
147 + s = s.replace(/\b\d+(\.\d+)?\s*(mg|mcg|ug|g|ml|iu|units?)(\s*\/\s*(kg|m2|ml|day|d|week|wk|dose))?\b/gi, ' '); // doses
148 + s = s.replace(/\b\d+(\.\d+)?\s*%/g, ' ');
149 + let norm = normalizeLabel(s);
150 + if (!norm) return null;
151 + const tokens = norm.split(' ');
152 + // strip trailing salt / formulation tokens (repeat: "doxorubicin hydrochloride injection")
153 + while (tokens.length > 1) {
154 + const last = tokens[tokens.length - 1]!;
155 + if (SALT_TOKENS.includes(last) || FORMULATION_TOKENS.includes(last) || /^\d+(\.\d+)?$/.test(last)) tokens.pop();
156 + else break;
157 + }
158 + // strip leading formulation tokens ("oral capecitabine", "iv cisplatin")
159 + while (tokens.length > 1) {
160 + const first = tokens[0]!;
161 + if (FORMULATION_TOKENS.includes(first)) tokens.shift();
162 + else break;
163 + }
164 + norm = tokens.join(' ').trim();
165 + if (!norm || norm === normalizeLabel(label)) return null;
166 + return norm;
167 +}
168 +
169 +/** Names that are never a drug entity (kept UNRESOLVED without being queued as drug labels). */
170 +export const NON_DRUG_RE = /^(placebo|placebos|saline|normal saline|sham|vehicle|standard of care|standard care|best supportive care|supportive care|observation|no intervention|no treatment|control|usual care|chemotherapy|radiotherapy|radiation|radiation therapy|surgery|questionnaire|survey|blood draw|blood sample|biopsy|imaging|mri|ct scan|pet scan|exercise|diet|counseling|education|physical therapy|acupuncture|massage|yoga|music therapy|hypnosis|dexamethasone placebo)( .*)?$/i;
171 +
172 +export interface DrugReconcileResult {
173 + pass1: number;
174 + pass2: number;
175 + pass3: number;
176 + ambiguousSkipped: number;
177 + remainingUnresolved: number;
178 + distinctUnresolved: number;
179 +}
180 +
181 +/**
182 + * Set-based reconciliation of `trial_interventions` (DRUG/BIOLOGICAL rows) to `drugs`.
183 + * `remap = true` resets ALIAS/PROBABILISTIC rows first (after alias changes); default only fills
184 + * UNRESOLVED rows.
185 + */
186 +export async function reconcileInterventionDrugs(db: Database, opts: { remap?: boolean } = {}): Promise<DrugReconcileResult> {
187 + if (opts.remap) {
188 + await db.execute(sql`UPDATE trial_interventions SET drug_id = NULL, match_type = 'UNRESOLVED' WHERE intervention_type IN ('DRUG','BIOLOGICAL') AND match_type IN ('ALIAS','PROBABILISTIC')`);
189 + }
190 + // Unambiguous alias map: normalized → drug_id (aliases shared by several drugs are excluded).
191 + await db.execute(sql`CREATE TEMP TABLE IF NOT EXISTS _alias_map (normalized text PRIMARY KEY, drug_id varchar(32) NOT NULL)`);
192 + await db.execute(sql`TRUNCATE _alias_map`);
193 + await db.execute(sql`
194 + INSERT INTO _alias_map (normalized, drug_id)
195 + SELECT normalized, min(drug_id) FROM drug_aliases WHERE length(normalized) >= 3
196 + GROUP BY normalized HAVING count(DISTINCT drug_id) = 1`);
197 + const [amb] = await db.execute<{ n: string }>(sql`SELECT count(*)::text AS n FROM (SELECT normalized FROM drug_aliases GROUP BY normalized HAVING count(DISTINCT drug_id) > 1) x`);
198 +
199 + // Pass 1: exact normalized alias.
200 + const p1 = await db.execute<{ n: string }>(sql`
201 + WITH u AS (
202 + UPDATE trial_interventions ti SET drug_id = m.drug_id, match_type = 'ALIAS'
203 + FROM _alias_map m
204 + WHERE ti.intervention_type IN ('DRUG','BIOLOGICAL') AND ti.match_type = 'UNRESOLVED' AND ti.normalized = m.normalized
205 + RETURNING 1)
206 + SELECT count(*)::text AS n FROM u`);
207 +
208 + // Pass 2: salt/dose/label-stripped form. Distinct labels are stripped in JS (pure function) then
209 + // matched in one statement.
210 + const remaining = await db.execute<{ normalized: string; name: string }>(sql`
211 + SELECT normalized, min(name) AS name FROM trial_interventions
212 + WHERE intervention_type IN ('DRUG','BIOLOGICAL') AND match_type = 'UNRESOLVED' GROUP BY normalized`);
213 + const stripped: Array<[string, string]> = [];
214 + for (const r of remaining) {
215 + if (NON_DRUG_RE.test(r.name)) continue;
216 + const m = stripToMolecule(r.name);
217 + if (m) stripped.push([r.normalized, m]);
218 + }
219 + let p2 = 0;
220 + for (let i = 0; i < stripped.length; i += 2000) {
221 + const chunk = stripped.slice(i, i + 2000);
222 + const res = await db.execute<{ n: string }>(sql`
223 + WITH s(normalized, molecule) AS (SELECT * FROM unnest(${sql.param(chunk.map((c) => c[0]))}::text[], ${sql.param(chunk.map((c) => c[1]))}::text[])),
224 + u AS (
225 + UPDATE trial_interventions ti SET drug_id = m.drug_id, match_type = 'ALIAS'
226 + FROM s JOIN _alias_map m ON m.normalized = s.molecule
227 + WHERE ti.intervention_type IN ('DRUG','BIOLOGICAL') AND ti.match_type = 'UNRESOLVED' AND ti.normalized = s.normalized
228 + RETURNING 1)
229 + SELECT count(*)::text AS n FROM u`);
230 + p2 += Number(res[0]?.n ?? 0);
231 + }
232 +
233 + // Pass 3 (PROBABILISTIC): "<generic> <formulation words only>" where the generic is a *generic*
234 + // alias (not brand/synonym) — e.g. "cisplatin 75 mg/m2 iv d1" already handled by pass 2; this
235 + // catches "pembrolizumab injection solution" style remainders whose extra tokens are all
236 + // formulation tokens. Names with any non-formulation extra token stay UNRESOLVED.
237 + const rem2 = await db.execute<{ normalized: string; name: string }>(sql`
238 + SELECT normalized, min(name) AS name FROM trial_interventions
239 + WHERE intervention_type IN ('DRUG','BIOLOGICAL') AND match_type = 'UNRESOLVED' GROUP BY normalized`);
240 + const heads: Array<[string, string]> = [];
241 + for (const r of rem2) {
242 + if (NON_DRUG_RE.test(r.name)) continue;
243 + const tokens = r.normalized.split(' ');
244 + if (tokens.length < 2) continue;
245 + const rest = tokens.slice(1);
246 + if (rest.every((t) => FORMULATION_TOKENS.includes(t) || SALT_TOKENS.includes(t) || /^\d+(\.\d+)?$/.test(t))) heads.push([r.normalized, tokens[0]!]);
247 + }
248 + let p3 = 0;
249 + for (let i = 0; i < heads.length; i += 2000) {
250 + const chunk = heads.slice(i, i + 2000);
251 + const res = await db.execute<{ n: string }>(sql`
252 + WITH s(normalized, head) AS (SELECT * FROM unnest(${sql.param(chunk.map((c) => c[0]))}::text[], ${sql.param(chunk.map((c) => c[1]))}::text[])),
253 + g AS (SELECT a.normalized, min(a.drug_id) AS drug_id FROM drug_aliases a WHERE a.alias_type = 'generic' GROUP BY a.normalized HAVING count(DISTINCT a.drug_id) = 1),
254 + u AS (
255 + UPDATE trial_interventions ti SET drug_id = g.drug_id, match_type = 'PROBABILISTIC'
256 + FROM s JOIN g ON g.normalized = s.head
257 + WHERE ti.intervention_type IN ('DRUG','BIOLOGICAL') AND ti.match_type = 'UNRESOLVED' AND ti.normalized = s.normalized
258 + RETURNING 1)
259 + SELECT count(*)::text AS n FROM u`);
260 + p3 += Number(res[0]?.n ?? 0);
261 + }
262 +
263 + const [left] = await db.execute<{ n: string; d: string }>(sql`SELECT count(*)::text AS n, count(DISTINCT normalized)::text AS d FROM trial_interventions WHERE intervention_type IN ('DRUG','BIOLOGICAL') AND match_type = 'UNRESOLVED'`);
264 + return { pass1: Number(p1[0]?.n ?? 0), pass2: p2, pass3: p3, ambiguousSkipped: Number(amb?.n ?? 0), remainingUnresolved: Number(left?.n ?? 0), distinctUnresolved: Number(left?.d ?? 0) };
265 +}
modified packages/connectors/src/connectors/clinicaltrials/index.ts +4 −0
@@ -8,10 +8,12 @@ import { manifest } from './manifest.js';
8 8 import { ONCOLOGY_EXPRESSION, STUDY_FIELDS, normalizeStudy, trimStudyPayload, type ApiPage, type ApiStudy, type ApiVersion, type NormalizedTrial } from './normalize.js';
9 9 import { reconcileConditions, type ConditionMapping, type ResolverLike } from './reconcile.js';
10 10 import { refreshTrialPulse } from './pulse.js';
11 +import { reconcileInterventionDrugs } from './drugs.js';
11 12
12 13 export { ONCOLOGY_EXPRESSION, STUDY_FIELDS, normalizeStudy, trimStudyPayload } from './normalize.js';
13 14 export { reconcileConditions, conditionCandidates } from './reconcile.js';
14 15 export { refreshTrialPulse } from './pulse.js';
16 +export { reconcileInterventionDrugs, stripToMolecule, NON_DRUG_RE } from './drugs.js';
15 17
16 18 export const PAGE_SIZE = 1000;
17 19 /** Trigram suggestions are computed for at most this many unresolved labels per run (curation aid only). */
@@ -200,6 +202,8 @@ export class ClinicalTrialsConnector extends Connector {
200 202 const re = await this.rereconcileUnresolved(ctx, resolver);
201 203 const pulse = await refreshTrialPulse(ctx.db);
202 204 ctx.info(`re-reconciled ${re.labels} unresolved labels → ${re.rows} trial_conditions rows updated; trial_pulse rows ${pulse}`);
205 + const dr = await reconcileInterventionDrugs(ctx.db);
206 + ctx.info(`interventions → drugs: alias ${dr.pass1}, stripped ${dr.pass2}, probabilistic ${dr.pass3}; ambiguous aliases skipped ${dr.ambiguousSkipped}; still unresolved ${dr.remainingUnresolved} rows (${dr.distinctUnresolved} distinct labels)`);
203 207 }
204 208 const pct = agg.conditionsTotal ? Math.round((agg.conditionsResolved / agg.conditionsTotal) * 1000) / 10 : 0;
205 209 const top = [...agg.unresolved.values()].sort((a, b) => b.count - a.count).slice(0, 15).map((u) => `${u.sourceText} (${u.count})`);
modified packages/connectors/src/index.ts +1 −0
@@ -3,3 +3,4 @@ export * from './registry.js';
3 3 export * from './sources-sync.js';
4 4 export * from './planned.js';
5 5 export * from './ops/index.js';
6 +export { reconcileInterventionDrugs, stripToMolecule, NON_DRUG_RE, SALT_TOKENS, FORMULATION_TOKENS } from './connectors/clinicaltrials/drugs.js';
added packages/database/migrations/0002_dizzy_pete_wisdom.sql +126 −0
@@ -0,0 +1,126 @@
1 +CREATE TABLE "drug_codes" (
2 + "id" bigserial PRIMARY KEY NOT NULL,
3 + "drug_id" varchar(32) NOT NULL,
4 + "system" text NOT NULL,
5 + "code" text NOT NULL,
6 + "label" text,
7 + "match_type" text DEFAULT 'EXACT_IDENTIFIER' NOT NULL,
8 + "source_id" varchar(32)
9 +);
10 +--> statement-breakpoint
11 +CREATE TABLE "drug_pipeline" (
12 + "id" bigserial PRIMARY KEY NOT NULL,
13 + "drug_id" varchar(32) NOT NULL,
14 + "cancer_id" varchar(32),
15 + "stage" text NOT NULL,
16 + "max_phase" text,
17 + "active_trials" integer DEFAULT 0 NOT NULL,
18 + "recruiting_trials" integer DEFAULT 0 NOT NULL,
19 + "phase3_trials" integer DEFAULT 0 NOT NULL,
20 + "total_trials" integer DEFAULT 0 NOT NULL,
21 + "approvals" integer DEFAULT 0 NOT NULL,
22 + "jurisdictions" text[] DEFAULT '{}' NOT NULL,
23 + "first_approval_date" text,
24 + "latest_approval_date" text,
25 + "first_trial_date" text,
26 + "formula_version" text NOT NULL,
27 + "inputs" jsonb DEFAULT '{}'::jsonb NOT NULL,
28 + "updated_at" timestamp with time zone DEFAULT now() NOT NULL
29 +);
30 +--> statement-breakpoint
31 +CREATE TABLE "research_gap_components" (
32 + "id" bigserial PRIMARY KEY NOT NULL,
33 + "cancer_id" varchar(32) NOT NULL,
34 + "geography" text NOT NULL,
35 + "year" integer NOT NULL,
36 + "sex" text DEFAULT 'all' NOT NULL,
37 + "burden_source_id" varchar(32) NOT NULL,
38 + "deaths" double precision,
39 + "incidence" double precision,
40 + "active_trials" integer DEFAULT 0 NOT NULL,
41 + "phase3_trials" integer DEFAULT 0 NOT NULL,
42 + "publications_5y" integer DEFAULT 0 NOT NULL,
43 + "approved_drugs" integer DEFAULT 0 NOT NULL,
44 + "death_share" real,
45 + "trial_share" real,
46 + "publication_share" real,
47 + "trial_gap_ratio" real,
48 + "research_gap_ratio" real,
49 + "trials_per_1000_deaths" real,
50 + "publications_per_1000_deaths" real,
51 + "eligible" boolean DEFAULT true NOT NULL,
52 + "ineligible_reason" text,
53 + "formula_version" text NOT NULL,
54 + "inputs" jsonb DEFAULT '{}'::jsonb NOT NULL,
55 + "updated_at" timestamp with time zone DEFAULT now() NOT NULL
56 +);
57 +--> statement-breakpoint
58 +CREATE TABLE "trial_intelligence" (
59 + "id" bigserial PRIMARY KEY NOT NULL,
60 + "cancer_id" varchar(32) NOT NULL,
61 + "entity_level" text DEFAULT 'all' NOT NULL,
62 + "total_trials" integer DEFAULT 0 NOT NULL,
63 + "active_trials" integer DEFAULT 0 NOT NULL,
64 + "recruiting_trials" integer DEFAULT 0 NOT NULL,
65 + "phase1_active" integer DEFAULT 0 NOT NULL,
66 + "phase2_active" integer DEFAULT 0 NOT NULL,
67 + "phase3_active" integer DEFAULT 0 NOT NULL,
68 + "phase3_recruiting" integer DEFAULT 0 NOT NULL,
69 + "phase4_active" integer DEFAULT 0 NOT NULL,
70 + "completed_trials" integer DEFAULT 0 NOT NULL,
71 + "terminated_trials" integer DEFAULT 0 NOT NULL,
72 + "withdrawn_trials" integer DEFAULT 0 NOT NULL,
73 + "suspended_trials" integer DEFAULT 0 NOT NULL,
74 + "with_results" integer DEFAULT 0 NOT NULL,
75 + "new_trials_12m" integer DEFAULT 0 NOT NULL,
76 + "new_trials_prior_12m" integer DEFAULT 0 NOT NULL,
77 + "trial_growth_yoy" real,
78 + "avg_enrollment" real,
79 + "median_enrollment" real,
80 + "total_enrollment_active" integer,
81 + "distinct_sponsors" integer DEFAULT 0 NOT NULL,
82 + "industry_share" real,
83 + "sponsor_hhi" real,
84 + "top_sponsor" text,
85 + "top_sponsor_share" real,
86 + "distinct_countries" integer DEFAULT 0 NOT NULL,
87 + "us_share" real,
88 + "top_country" text,
89 + "top_country_share" real,
90 + "country_hhi" real,
91 + "termination_share" real,
92 + "why_stopped_breakdown" jsonb DEFAULT '{}'::jsonb NOT NULL,
93 + "trials_per_1000_deaths" real,
94 + "trials_per_100k_cases" real,
95 + "burden_geography" text,
96 + "burden_year" integer,
97 + "burden_source_id" varchar(32),
98 + "formula_version" text NOT NULL,
99 + "inputs" jsonb DEFAULT '{}'::jsonb NOT NULL,
100 + "updated_at" timestamp with time zone DEFAULT now() NOT NULL
101 +);
102 +--> statement-breakpoint
103 +CREATE TABLE "trial_site_country_counts" (
104 + "id" bigserial PRIMARY KEY NOT NULL,
105 + "cancer_id" varchar(32),
106 + "phase" text,
107 + "recruiting_only" boolean DEFAULT false NOT NULL,
108 + "country" text NOT NULL,
109 + "iso3" text,
110 + "sites" integer NOT NULL,
111 + "trials" integer NOT NULL,
112 + "formula_version" text NOT NULL,
113 + "updated_at" timestamp with time zone DEFAULT now() NOT NULL
114 +);
115 +--> statement-breakpoint
116 +CREATE UNIQUE INDEX "drug_codes_uq" ON "drug_codes" USING btree ("drug_id","system","code");--> statement-breakpoint
117 +CREATE INDEX "drug_codes_lookup_idx" ON "drug_codes" USING btree ("system","code");--> statement-breakpoint
118 +CREATE UNIQUE INDEX "drug_pipeline_uq" ON "drug_pipeline" USING btree ("drug_id","cancer_id");--> statement-breakpoint
119 +CREATE INDEX "drug_pipeline_stage_idx" ON "drug_pipeline" USING btree ("stage");--> statement-breakpoint
120 +CREATE INDEX "drug_pipeline_cancer_idx" ON "drug_pipeline" USING btree ("cancer_id","stage");--> statement-breakpoint
121 +CREATE UNIQUE INDEX "research_gap_components_uq" ON "research_gap_components" USING btree ("cancer_id","geography","year","sex","burden_source_id");--> statement-breakpoint
122 +CREATE INDEX "research_gap_scope_idx" ON "research_gap_components" USING btree ("geography","year","sex");--> statement-breakpoint
123 +CREATE UNIQUE INDEX "trial_intelligence_uq" ON "trial_intelligence" USING btree ("cancer_id","entity_level");--> statement-breakpoint
124 +CREATE INDEX "trial_intelligence_active_idx" ON "trial_intelligence" USING btree ("entity_level","active_trials");--> statement-breakpoint
125 +CREATE UNIQUE INDEX "trial_site_country_uq" ON "trial_site_country_counts" USING btree ("cancer_id","phase","recruiting_only","country");--> statement-breakpoint
126 +CREATE INDEX "trial_site_country_lookup_idx" ON "trial_site_country_counts" USING btree ("cancer_id","phase","recruiting_only");
\ No newline at end of file
added packages/database/migrations/meta/0002_snapshot.json +8029 −0
Large diff (8,030 lines) is not loaded by default to keep the page fast.Load diff
modified packages/database/migrations/meta/_journal.json +7 −0
Large diff (14 lines) is not loaded by default to keep the page fast.Load diff
modified packages/database/src/schema/index.ts +1 −0
Large diff (4 lines) is not loaded by default to keep the page fast.Load diff
added packages/database/src/schema/intelligence.ts +164 −0
Large diff (164 lines) is not loaded by default to keep the page fast.Load diff
modified packages/database/src/seed-data/metrics.ts +122 −0
Large diff (126 lines) is not loaded by default to keep the page fast.Load diff
added packages/ranking/src/drug-pipeline.ts +16 −0
Large diff (16 lines) is not loaded by default to keep the page fast.Load diff
modified packages/ranking/src/engine.ts +1 −1
Large diff (8 lines) is not loaded by default to keep the page fast.Load diff
modified packages/ranking/src/index.ts +4 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added packages/ranking/src/intelligence.ts +48 −0
Large diff (48 lines) is not loaded by default to keep the page fast.Load diff
added packages/ranking/src/research-gap.ts +18 −0
Large diff (18 lines) is not loaded by default to keep the page fast.Load diff
added packages/ranking/src/trial-intelligence.ts +17 −0
Large diff (17 lines) is not loaded by default to keep the page fast.Load diff
modified scripts/ci.ts +13 −1
Large diff (26 lines) is not loaded by default to keep the page fast.Load diff
modified workers/lib/env.ts +2 −0
Large diff (14 lines) is not loaded by default to keep the page fast.Load diff
modified workers/main.ts +20 −3
Large diff (62 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere