Web: paginate approvals and gene cohort tables
8 changed files +64 −17
modified
apps/web/src/app/cancer/[slug]/[[...tab]]/page.tsx
+1 −1
@@ -69,7 +69,7 @@ export default async function CancerPage({ params, searchParams }: { params: Pro | ||
| 69 | 69 | } |
| 70 | 70 | case 'drugs': { |
| 71 | 71 | const sp = await searchParams; |
| 72 | − body = <DrugsTab b={b} jurisdiction={str(sp, 'jurisdiction') || null} />; | |
| 72 | + body = <DrugsTab b={b} jurisdiction={str(sp, 'jurisdiction') || null} aPage={int(sp, 'aPage', 1, 1, 100_000)} />; | |
| 73 | 73 | break; |
| 74 | 74 | } |
| 75 | 75 | case 'trials': { |
modified
apps/web/src/app/drug/[slug]/page.tsx
+3 −2
@@ -34,6 +34,7 @@ export default async function DrugPage({ params, searchParams }: { params: Promi | ||
| 34 | 34 | const [evTotal, tTotal] = await Promise.all([evidenceForDrugCount(d.id), trialsForDrugCount(d.id)]); |
| 35 | 35 | const ev = pageInfo(int(sp, 'evPage', 1, 1, 100_000), EVIDENCE_PAGE_SIZE, evTotal); |
| 36 | 36 | const tp = pageInfo(int(sp, 'tPage', 1, 1, 100_000), TRIAL_PAGE_SIZE, tTotal); |
| 37 | + const aPage = int(sp, 'aPage', 1, 1, 100_000); | |
| 37 | 38 | const [approvals, evidence, trials] = await Promise.all([approvalsForDrug(d.id), evTotal ? evidenceForDrug(d.id, { page: ev.page, pageSize: ev.pageSize }) : Promise.resolve([]), tTotal ? trialsForDrug(d.id, { page: tp.page, pageSize: tp.pageSize }) : Promise.resolve([])]); |
| 38 | 39 | const prov = await loadProvenance([...approvals.map((a) => a.provenance_id), ...evidence.map((e) => e.provenance_id)]); |
| 39 | 40 | const jurisdictions = [...new Set(approvals.map((a) => a.jurisdiction))].sort(); |
@@ -41,7 +42,7 @@ export default async function DrugPage({ params, searchParams }: { params: Promi | ||
| 41 | 42 | const selected = jurisdictions.includes(wanted) ? wanted : null; |
| 42 | 43 | const shown = selected ? approvals.filter((a) => a.jurisdiction === selected) : approvals; |
| 43 | 44 | const aliases = d.aliases ?? []; |
| 44 | − const current = { jurisdiction: selected ?? '', evPage: ev.page > 1 ? ev.page : '', tPage: tp.page > 1 ? tp.page : '' }; | |
| 45 | + const current = { jurisdiction: selected ?? '', evPage: ev.page > 1 ? ev.page : '', tPage: tp.page > 1 ? tp.page : '', aPage: aPage > 1 ? aPage : '' }; | |
| 45 | 46 | const href = (o: Record<string, string | number | null | undefined>, hash?: string) => `/drug/${d.slug}${withParams(current, o)}${hash ? `#${hash}` : ''}`; |
| 46 | 47 | |
| 47 | 48 | return ( |
@@ -89,7 +90,7 @@ export default async function DrugPage({ params, searchParams }: { params: Promi | ||
| 89 | 90 | </Link> |
| 90 | 91 | ))} |
| 91 | 92 | </nav> |
| 92 | − <ApprovalsTable rows={shown} prov={prov} showDrug={false} /> | |
| 93 | + <ApprovalsTable rows={shown} prov={prov} showDrug={false} page={aPage} hrefFor={(p) => href({ aPage: p > 1 ? p : '' }, 'approvals')} /> | |
| 93 | 94 | <Freshness dataUpdatedAt={approvals.reduce<Date | string | null>((m, a) => (m == null || String(a.updated_at) > String(m) ? a.updated_at : m), null)} /> |
| 94 | 95 | </> |
| 95 | 96 | ) : ( |
modified
apps/web/src/app/gene/[symbol]/page.tsx
+3 −2
@@ -35,6 +35,7 @@ export default async function GenePage({ params, searchParams }: { params: Promi | ||
| 35 | 35 | const sp = await searchParams; |
| 36 | 36 | const evPageReq = int(sp, 'evPage', 1, 1, 100_000); |
| 37 | 37 | const vPageReq = int(sp, 'vPage', 1, 1, 100_000); |
| 38 | + const cPageReq = int(sp, 'cPage', 1, 1, 100_000); | |
| 38 | 39 | const pPageReq = int(sp, 'pPage', 1, 1, 100_000); |
| 39 | 40 | |
| 40 | 41 | const [evTotal, vTotal, pTotal] = await Promise.all([evidenceForGeneCount(g.id, g.symbol), variantsForGeneCount(g.id), recentPublicationsForCount('gene', [g.id])]); |
@@ -50,7 +51,7 @@ export default async function GenePage({ params, searchParams }: { params: Promi | ||
| 50 | 51 | ]); |
| 51 | 52 | const prov = await loadProvenance([...evidence.map((e) => e.provenance_id), ...freqs.map((f) => f.provenance_id)]); |
| 52 | 53 | |
| 53 | − const current = { evPage: ev.page > 1 ? ev.page : '', vPage: vp.page > 1 ? vp.page : '', pPage: pp.page > 1 ? pp.page : '' }; | |
| 54 | + const current = { evPage: ev.page > 1 ? ev.page : '', vPage: vp.page > 1 ? vp.page : '', pPage: pp.page > 1 ? pp.page : '', cPage: cPageReq > 1 ? cPageReq : '' }; | |
| 54 | 55 | const href = (o: Record<string, string | number | null | undefined>, hash: string) => `/gene/${g.symbol}${withParams(current, o)}#${hash}`; |
| 55 | 56 | |
| 56 | 57 | return ( |
@@ -126,7 +127,7 @@ export default async function GenePage({ params, searchParams }: { params: Promi | ||
| 126 | 127 | </Section> |
| 127 | 128 | |
| 128 | 129 | <Section id="frequencies" kicker="Cohorts" title={`Alteration frequency by cohort (${fmtInt(freqs.length)})`} description="Frequency = cases affected / cases profiled within one cohort. Cohorts are never pooled."> |
| 129 | − {freqs.length ? <GeneFrequencyTable rows={freqs} prov={prov} /> : <EmptyState compact>No cohort frequency recorded for this gene.</EmptyState>} | |
| 130 | + {freqs.length ? <GeneFrequencyTable rows={freqs} prov={prov} page={cPageReq} hrefFor={(p) => href({ cPage: p > 1 ? p : '' }, 'frequencies')} /> : <EmptyState compact>No cohort frequency recorded for this gene.</EmptyState>} | |
| 130 | 131 | </Section> |
| 131 | 132 | |
| 132 | 133 | <Section id="publications" kicker="Literature" title={`Linked publications (${fmtInt(pTotal)})`} description={pTotal ? `${PUBLICATION_PAGE_SIZE} per page, newest first.` : undefined}> |
modified
apps/web/src/components/cancer/tabs/drugs.tsx
+2 −2
@@ -10,7 +10,7 @@ import { loadProvenance } from '@/lib/queries/provenance'; | ||
| 10 | 10 | import { fmtInt } from '@/lib/format'; |
| 11 | 11 | import type { CancerBundle } from '../load'; |
| 12 | 12 | |
| 13 | −export async function DrugsTab({ b, jurisdiction }: { b: CancerBundle; jurisdiction: string | null }) { | |
| 13 | +export async function DrugsTab({ b, jurisdiction, aPage = 1 }: { b: CancerBundle; jurisdiction: string | null; aPage?: number }) { | |
| 14 | 14 | // Therapies mentioned in curated evidence (not approvals) — aggregated in SQL (no evidence rows |
| 15 | 15 | // are shipped to the page), listed separately and never called "approved". |
| 16 | 16 | const [approvals, therapies] = await Promise.all([approvalsForCancer(b.descendants), therapyMentionsForCancer(b.descendants)]); |
@@ -43,7 +43,7 @@ export async function DrugsTab({ b, jurisdiction }: { b: CancerBundle; jurisdict | ||
| 43 | 43 | </Link> |
| 44 | 44 | ))} |
| 45 | 45 | </nav> |
| 46 | − <ApprovalsTable rows={shownApprovals} prov={prov} /> | |
| 46 | + <ApprovalsTable rows={shownApprovals} prov={prov} page={aPage} hrefFor={(p) => `/cancer/${b.cancer.slug}/drugs?${new URLSearchParams({ ...(selected ? { jurisdiction: selected } : {}), ...(p > 1 ? { aPage: String(p) } : {}) }).toString()}#approvals`} /> | |
| 47 | 47 | <Freshness dataUpdatedAt={approvals.reduce((m, a) => (a.updated_at > m ? a.updated_at : m), approvals[0]!.updated_at)} /> |
| 48 | 48 | </> |
| 49 | 49 | ) : ( |
modified
apps/web/src/components/data/approvals-table.tsx
+7 −1
@@ -5,13 +5,18 @@ import type { ApprovalRow } from '@/lib/queries/drugs'; | ||
| 5 | 5 | import type { ProvRow } from '@/lib/queries/provenance'; |
| 6 | 6 | import { toInfo } from '@/lib/queries/provenance'; |
| 7 | 7 | import { fmtDate, fmtInt } from '@/lib/format'; |
| 8 | +import { Pager } from '@/components/ui/pager'; | |
| 9 | +import { pageInfo } from '@/lib/pagination'; | |
| 8 | 10 | |
| 9 | 11 | /** |
| 10 | 12 | * Jurisdiction-aware regulatory status (§13). Never a bare "approved". Approvals may come from |
| 11 | 13 | * different authorities' datasets, so rows keep a compact badge with a dataset · version title; |
| 12 | 14 | * the caption carries one full popover (first row's provenance) and the claim label. |
| 13 | 15 | */ |
| 14 | −export function ApprovalsTable({ rows, prov, showDrug = true, showCancer = true }: { rows: ApprovalRow[]; prov: Map<number, ProvRow>; showDrug?: boolean; showCancer?: boolean }) { | |
| 16 | +export function ApprovalsTable({ rows: allRows, prov, showDrug = true, showCancer = true, page = 1, pageSize = 50, hrefFor }: { rows: ApprovalRow[]; prov: Map<number, ProvRow>; showDrug?: boolean; showCancer?: boolean; page?: number; pageSize?: number; hrefFor?: (page: number) => string }) { | |
| 17 | + // Optional server-side pagination (URL state); without hrefFor the first page only is shown with a count. | |
| 18 | + const info = pageInfo(page, pageSize, allRows.length); | |
| 19 | + const rows = allRows.slice(info.offset, info.offset + pageSize); | |
| 15 | 20 | const first = rows[0]; |
| 16 | 21 | const caption = first ? (toInfo(prov.get(first.provenance_id)) ?? { sourceSlug: first.source_slug, sourceName: first.source_name }) : null; |
| 17 | 22 | const sources = new Set(rows.map((r) => r.source_slug)); |
@@ -84,6 +89,7 @@ export function ApprovalsTable({ rows, prov, showDrug = true, showCancer = true | ||
| 84 | 89 | </tbody> |
| 85 | 90 | </table> |
| 86 | 91 | </div> |
| 92 | + {hrefFor ? <Pager total={allRows.length} pageSize={pageSize} page={info.page} hrefFor={hrefFor} label="Approval pages" noun="approval records" /> : allRows.length > rows.length ? <p className="mt-1 text-[12px] text-ink-3">Showing {rows.length} of {fmtInt(allRows.length)} approval records.</p> : null} | |
| 87 | 93 | </div> |
| 88 | 94 | ); |
| 89 | 95 | } |
modified
apps/web/src/components/data/frequency-table.tsx
+8 −3
@@ -6,6 +6,8 @@ import type { FreqRow } from '@/lib/queries/genomics'; | ||
| 6 | 6 | import type { ProvRow } from '@/lib/queries/provenance'; |
| 7 | 7 | import { toInfo } from '@/lib/queries/provenance'; |
| 8 | 8 | import { fmtInt, fmtPct, humanize } from '@/lib/format'; |
| 9 | +import { Pager } from '@/components/ui/pager'; | |
| 10 | +import { pageInfo } from '@/lib/pagination'; | |
| 9 | 11 | |
| 10 | 12 | /** |
| 11 | 13 | * Gene alteration frequencies with mandatory denominators (§260-261), one table per cohort. |
@@ -116,15 +118,17 @@ export function FrequencyTables({ rows, prov, cohortFilter, showCancer = false } | ||
| 116 | 118 | * Rows come from different cohorts (different releases), so each compact badge carries a `title` |
| 117 | 119 | * with dataset · version · retrieved; the caption shows one popover for the source. |
| 118 | 120 | */ |
| 119 | −export function GeneFrequencyTable({ rows, prov }: { rows: FreqRow[]; prov: Map<number, ProvRow> }) { | |
| 120 | − const sorted = [...rows].sort((a, b) => b.frequency - a.frequency); | |
| 121 | +export function GeneFrequencyTable({ rows, prov, page = 1, pageSize = 50, hrefFor }: { rows: FreqRow[]; prov: Map<number, ProvRow>; page?: number; pageSize?: number; hrefFor?: (page: number) => string }) { | |
| 122 | + const all = [...rows].sort((a, b) => b.frequency - a.frequency); | |
| 123 | + const info = pageInfo(page, pageSize, all.length); | |
| 124 | + const sorted = all.slice(info.offset, info.offset + pageSize); | |
| 121 | 125 | const first = sorted[0]; |
| 122 | 126 | const caption = first ? (toInfo(prov.get(first.provenance_id)) ?? { sourceSlug: first.source_slug, sourceName: first.source_name }) : null; |
| 123 | 127 | return ( |
| 124 | 128 | <div> |
| 125 | 129 | {caption ? ( |
| 126 | 130 | <TableProvenance p={caption} claim={<ClaimBadge kind="observed" />}> |
| 127 | − {sorted.length} cohort row{sorted.length === 1 ? '' : 's'} · frequency = affected / profiled, as published by each cohort; cohorts are not pooled. Hover a row badge for its release. | |
| 131 | + {all.length} cohort row{all.length === 1 ? '' : 's'}{all.length > sorted.length ? ` (showing ${sorted.length}, sorted by frequency)` : ''} · frequency = affected / profiled, as published by each cohort; cohorts are not pooled. Hover a row badge for its release. | |
| 128 | 132 | </TableProvenance> |
| 129 | 133 | ) : null} |
| 130 | 134 | <div className="ci-table-wrap"> |
@@ -182,6 +186,7 @@ export function GeneFrequencyTable({ rows, prov }: { rows: FreqRow[]; prov: Map< | ||
| 182 | 186 | </tbody> |
| 183 | 187 | </table> |
| 184 | 188 | </div> |
| 189 | + {hrefFor ? <Pager total={all.length} pageSize={pageSize} page={info.page} hrefFor={hrefFor} label="Cohort pages" noun="cohorts" className="mt-2 text-[12px]" /> : null} | |
| 185 | 190 | </div> |
| 186 | 191 | ); |
| 187 | 192 | } |
modified
docs/connectors/cbioportal.md
+16 −2
@@ -50,9 +50,23 @@ TCGA cohorts also present in GDC are kept as **separate cohorts** (different pro | ||
| 50 | 50 | |
| 51 | 51 | `dry_run` fetches studies + cancer types and the top-5 mutated genes of `acc_tcga` without writing. |
| 52 | 52 | |
| 53 | −## Observed run (cancerindex_b, 2026-09-08) | |
| 53 | +## Observed run (cancerindex_b, 2026-09-08 — OncoTree + NCIt + HGNC loaded) | |
| 54 | 54 | |
| 55 | −See the run summary appended below (10-record smoke, then 20-minute run). | |
| 55 | +| | | | |
| 56 | +|---|---| | |
| 57 | +| Smoke (`--max-records 10`) | 5 studies (acbc_mskcc_2015 … acc_tcga_pan_can_atlas_2018), 12 requests, 5 s, all `EXACT_IDENTIFIER` | | |
| 58 | +| Full pass (`--max-minutes 20`) | **545 studies / 545 cohorts** in 6–8 min, 1,080 requests, 0 HTTP failures, 0 studies rejected | | |
| 59 | +| Cancer mapping | **489 / 545 (89.7 %)**: 432 `EXACT_IDENTIFIER` (OncoTree code), 57 `CURATED_BROADER` (tissue-level rewrite); 48 `mixed` pan-cancer cohorts unmappable by design (MSK-IMPACT 2017, GENIE-derived pan-cancer sets, mixed_allen_2018…); **98.4 % of non-mixed studies mapped**; 8 unresolved (tissue-level sites without a site-level malignancy alias in the ontology yet) | | |
| 60 | +| Programs | 139 distinct — MSK 171, TCGA 88, TCGA GDC 33, Broad 15, DFCI 10, CPTAC GDC 10, CPTAC 6…; 17 studies without a parenthetical source (`program null`) | | |
| 61 | +| Frequencies | **100,444 rows** in 539 cohorts (9,972 distinct genes, all resolved to a `genes` row — 0 minimal genes created after HGNC), 90,521 rows with a `cancerId`; denominators 1 – 54,331; **7 rejected** by `validateFrequency` (bowel_colitis_msk_2022: FAT3 11/1, SMOX 4/1… — portal gene-panel matrix inconsistency, never stored) | | |
| 62 | +| Publications | 378 PMIDs → publication stubs + `publication_entity_edges` to the cohort's cancer | | |
| 63 | +| Counters | `genomic_cohorts` ranking eligible 109 cancers (all scopes) / 9 top-level; e.g. Malignant Breast Neoplasm 7 cohorts, Malignant Lung Neoplasm 5 | | |
| 64 | + | |
| 65 | +`msk_impact_2017` (10,945 samples, `mixed`, program MSK, PMID 28481359) top 5 — `cases_affected / cases_profiled`: TP53 4,538/10,945 (0.415), KRAS 1,643/10,945 (0.150), TERT 1,460/10,945 (0.133), PIK3CA 1,355/10,945 (0.124), APC 1,121/10,945 (0.102) — identical to the portal's study-summary table; genes only on the 410-gene panel carry the panel-aware denominator 8,136. | |
| 66 | + | |
| 67 | +`acc_tcga` (90 sequenced) top 3: ZFPM1 47/90, MUC5B 35/90, GARS1 34/90 (Firehose Legacy calls; the PanCancer Atlas cohort `acc_tcga_pan_can_atlas_2018` of the same tumours gives TP53 18/91, CTNNB1 14/91 — kept as separate cohorts, as is GDC's TCGA-ACC). | |
| 68 | + | |
| 69 | +Idempotent: a second full pass (run twice today after rule changes, `--reset-cursor`) updated cohorts and frequencies in place, reused cohort ids and provenance rows for unchanged studies, and created no duplicate genes or publications. | |
| 56 | 70 | |
| 57 | 71 | ## Limitations / notes |
| 58 | 72 | |
modified
packages/connectors/src/connectors/chembl/index.ts
+24 −4
@@ -123,6 +123,7 @@ export class ChemblConnector extends Connector { | ||
| 123 | 123 | const geneCache = new GeneCache(ctx.db); |
| 124 | 124 | const targetCache = new Map<string, TargetInfo>(); |
| 125 | 125 | const neoplasmMesh = await this.loadNeoplasmMeshSet(ctx); |
| 126 | + const meshCodes = await this.loadMeshCodes(ctx); | |
| 126 | 127 | const stats: Stats = { considered: 0, skippedNonMolecule: 0, matched: 0, matchedByPrefName: 0, matchedBySynonym: 0, unresolved: 0, aliases: 0, mechanisms: 0, targetEdges: 0, genesLinked: 0, indications: 0, indicationEdges: 0, indicationsUnresolvedCancer: 0, indicationsNonCancer: 0, malformed: 0 }; |
| 127 | 128 | |
| 128 | 129 | let sinceFlush = 0; |
@@ -134,7 +135,7 @@ export class ChemblConnector extends Connector { | ||
| 134 | 135 | } |
| 135 | 136 | stats.considered++; |
| 136 | 137 | try { |
| 137 | − await this.processDrug(ctx, drug, { resolver, geneCache, targetCache, neoplasmMesh, stats }); | |
| 138 | + await this.processDrug(ctx, drug, { resolver, geneCache, targetCache, neoplasmMesh, meshCodes, stats }); | |
| 138 | 139 | } catch (e) { |
| 139 | 140 | // One bad drug must not kill the pass; the error is logged and the drug is retried next pass. |
| 140 | 141 | ctx.counters.rejected++; |
@@ -171,6 +172,24 @@ export class ChemblConnector extends Connector { | ||
| 171 | 172 | return set; |
| 172 | 173 | } |
| 173 | 174 | |
| 175 | + /** | |
| 176 | + * MeSH UI → canonical cancer for every match type. `CancerResolver.byCode` only caches | |
| 177 | + * EXACT_IDENTIFIER / CURATED_EXACT / ONTOLOGY_EXACT codes, but the mesh connector also records | |
| 178 | + * deliberate ALIAS (entry-term) and CURATED_BROADER mappings — an indication coded D015464 (CML, | |
| 179 | + * mapped as ALIAS) must still resolve. Ambiguous UIs (two cancers) are dropped. | |
| 180 | + */ | |
| 181 | + private async loadMeshCodes(ctx: RunContext): Promise<Map<string, { cancerId: string; matchType: string }>> { | |
| 182 | + const map = new Map<string, { cancerId: string; matchType: string }>(); | |
| 183 | + const ambiguous = new Set<string>(); | |
| 184 | + for (const r of await ctx.db.select({ cancerId: cancerCodes.cancerId, code: cancerCodes.code, matchType: cancerCodes.matchType }).from(cancerCodes).where(eq(cancerCodes.system, 'mesh'))) { | |
| 185 | + const prev = map.get(r.code); | |
| 186 | + if (prev && prev.cancerId !== r.cancerId) ambiguous.add(r.code); | |
| 187 | + else map.set(r.code, { cancerId: r.cancerId, matchType: r.matchType }); | |
| 188 | + } | |
| 189 | + for (const c of ambiguous) map.delete(c); | |
| 190 | + return map; | |
| 191 | + } | |
| 192 | + | |
| 174 | 193 | private async persistCursor(ctx: RunContext): Promise<void> { |
| 175 | 194 | await ctx.db.update(connectorCursors).set({ cursor: ctx.cursor, updatedAt: new Date() }).where(eq(connectorCursors.connectorId, manifest.id)); |
| 176 | 195 | } |
@@ -188,8 +207,8 @@ export class ChemblConnector extends Connector { | ||
| 188 | 207 | } |
| 189 | 208 | } |
| 190 | 209 | |
| 191 | − private async processDrug(ctx: RunContext, drug: DrugRow, env: { resolver: CancerResolver; geneCache: GeneCache; targetCache: Map<string, TargetInfo>; neoplasmMesh: Set<string>; stats: Stats }): Promise<void> { | |
| 192 | − const { resolver, geneCache, targetCache, neoplasmMesh, stats } = env; | |
| 210 | + private async processDrug(ctx: RunContext, drug: DrugRow, env: { resolver: CancerResolver; geneCache: GeneCache; targetCache: Map<string, TargetInfo>; neoplasmMesh: Set<string>; meshCodes: Map<string, { cancerId: string; matchType: string }>; stats: Stats }): Promise<void> { | |
| 211 | + const { resolver, geneCache, targetCache, neoplasmMesh, meshCodes, stats } = env; | |
| 193 | 212 | if (isNonMoleculeName(drug.name)) { |
| 194 | 213 | stats.skippedNonMolecule++; |
| 195 | 214 | return; |
@@ -285,7 +304,8 @@ export class ChemblConnector extends Connector { | ||
| 285 | 304 | |
| 286 | 305 | // 7. drug INVESTIGATED_FOR cancer — MeSH-coded indications resolved by UI. |
| 287 | 306 | for (const ind of indications) { |
| 288 | − const cancer = resolver.byCode('mesh', ind.meshId); | |
| 307 | + const byCode = resolver.byCode('mesh', ind.meshId); | |
| 308 | + const cancer = byCode ? { cancerId: byCode.cancerId } : meshCodes.get(ind.meshId); | |
| 289 | 309 | if (!cancer) { |
| 290 | 310 | if (neoplasmMesh.has(ind.meshId) || NEOPLASM_HEADING_RE.test(ind.meshHeading)) { |
| 291 | 311 | stats.indicationsUnresolvedCancer++; |
| 292 | 312 | |