SPB Git forge
spboucher.ai ↗

spb/cancerindex

Public
37commits 1branches 0releases
2.9 MBsize
maindefault branch
10 days agolast push
TypeScript 97.2% SQL 1.5% CSS 0.6% JavaScript 0.5%

Data explorer: /explore (metric × cancers × geography × sex × age × years → comparable chart groups + table + CSV/JSON/permalink), coverage matrix, /v1/epidemiology API, methodology

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 13 days ago (Sep 11, 2026) parent 5a6f024

18 changed files +2,321 −5

modified apps/api/src/routes/epidemiology.ts +207 −5
@@ -1,10 +1,212 @@
1 +import { sql } from 'drizzle-orm';
1 2 import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
3 +import { z } from 'zod';
4 +import { paginate } from '../lib/envelope.js';
5 +import { BadRequest, NotFound } from '../lib/errors.js';
6 +import { pageQuery } from '../lib/pagination.js';
7 +import { resolveCancer } from '../lib/resolve.js';
8 +import { AnyList, num, ok, respond } from '../lib/respond.js';
9 +
10 +const MAX_CANCERS = 8;
11 +const METRIC_RE = /^[a-z][a-z0-9_]{1,63}$/;
12 +
13 +/** Labels shared with the web app (apps/web/src/lib/queries/epidemiology.ts EPI_METRIC_LABEL). */
14 +const METRIC_LABEL: Record<string, string> = {
15 + incidence_count: 'New cases',
16 + incidence_rate: 'Incidence rate (crude)',
17 + as_incidence_rate: 'Incidence rate (age-standardized)',
18 + mortality_count: 'Deaths',
19 + mortality_rate: 'Mortality rate (crude)',
20 + as_mortality_rate: 'Mortality rate (age-standardized)',
21 + prevalence: 'Prevalence',
22 + prevalence_5y: '5-year prevalence',
23 +};
24 +
25 +/** `cancer=a&cancer=b` or `cancer=a,b` (or both) → distinct trimmed refs, max MAX_CANCERS. */
26 +export function splitRefs(v: string | string[] | undefined, max = MAX_CANCERS): string[] {
27 + const raw = (Array.isArray(v) ? v : v == null ? [] : [v]).flatMap((s) => String(s).split(','));
28 + const out: string[] = [];
29 + for (const t of raw.map((s) => s.trim()).filter(Boolean)) if (!out.includes(t)) out.push(t);
30 + if (out.length > max) throw new BadRequest(`at most ${max} cancers per request (got ${out.length})`);
31 + return out;
32 +}
33 +
34 +const cancerParam = z.union([z.string(), z.array(z.string())]).optional().describe('Cancer id or slug — repeatable or comma-separated, max 8');
35 +const geographyParam = z.string().trim().min(1).max(100).optional().describe('Geography slug or ISO3 code (e.g. united-states, USA)');
36 +
37 +type GeoRow = Record<string, unknown> & { id: string; slug: string; name: string; iso3: string | null };
2 38
3 39 /**
4 − * Epidemiology routes (SPEC §20, §63, §110): `GET /epidemiology` → observations filtered by metric,
5 − * cancer, geography, sex, age group and year range; `GET /epidemiology/coverage` → which
6 − * metric × geography × year combinations exist. Filled by the Data Explorer work package.
40 + * Epidemiology routes (SPEC §20, §63, §110). Observations are returned exactly as stored — one row per
41 + * (cancer, geography, year, sex, age group, metric, source, site definition) — with the standard population,
42 + * estimate type and full provenance so consumers can apply the comparability rules
43 + * (docs/methodology/data-explorer.md): never overlay different standard populations, sources or age groups.
7 44 */
8 −export const epidemiologyRoutes: FastifyPluginAsyncZod = async (_app) => {
9 − /* routes added by the data-explorer work package */
45 +export const epidemiologyRoutes: FastifyPluginAsyncZod = async (app) => {
46 + async function resolveGeography(ref: string): Promise<GeoRow> {
47 + const rows = await app.db.execute<GeoRow>(sql`SELECT id, slug, name, iso3 FROM geographies WHERE slug = ${ref.toLowerCase()} OR upper(iso3) = ${ref.toUpperCase()} OR id = ${ref} ORDER BY (slug = ${ref.toLowerCase()}) DESC LIMIT 1`);
48 + const g = rows[0];
49 + if (!g) throw new NotFound('geography', ref);
50 + return g;
51 + }
52 +
53 + app.get(
54 + '/epidemiology',
55 + {
56 + schema: {
57 + tags: ['epidemiology'],
58 + summary: 'Epidemiology observations filtered by metric, cancer(s), geography, sex, age group, years, source and estimate type',
59 + querystring: z.object({
60 + metric: z.string().regex(METRIC_RE).optional().describe('incidence_count | as_incidence_rate | mortality_count | mortality_rate | as_mortality_rate | …'),
61 + cancer: cancerParam,
62 + geography: geographyParam,
63 + sex: z.enum(['all', 'male', 'female']).optional(),
64 + age: z.string().trim().max(24).optional().describe('Age group label as stored (default: any; "all" = all ages)'),
65 + from: z.coerce.number().int().min(1900).max(2100).optional().describe('First year (inclusive)'),
66 + to: z.coerce.number().int().min(1900).max(2100).optional().describe('Last year (inclusive)'),
67 + source: z.string().trim().max(64).optional().describe('Source slug or CI-SOURCE id'),
68 + estimateType: z.enum(['observed', 'estimated', 'projected']).optional(),
69 + ...pageQuery,
70 + }),
71 + response: ok(AnyList, true),
72 + },
73 + },
74 + async (req) => {
75 + const q = req.query;
76 + const conds = [sql`true`];
77 + if (q.metric) conds.push(sql`o.metric = ${q.metric}`);
78 + const refs = splitRefs(q.cancer);
79 + if (refs.length > 0) {
80 + const ids = await Promise.all(refs.map((r) => resolveCancer(app.db, r).then((c) => c.id)));
81 + conds.push(sql`o.cancer_id = ANY(${sql.param(ids)}::text[])`);
82 + }
83 + if (q.geography) {
84 + const g = await resolveGeography(q.geography);
85 + conds.push(sql`o.geography_id = ${g.id}`);
86 + }
87 + if (q.sex) conds.push(sql`o.sex = ${q.sex}`);
88 + if (q.age) conds.push(sql`o.age_group = ${q.age}`);
89 + if (q.from != null) conds.push(sql`coalesce(o.year_end, o.year) >= ${q.from}`);
90 + if (q.to != null) conds.push(sql`o.year <= ${q.to}`);
91 + if (q.source) conds.push(q.source.startsWith('CI-SOURCE-') ? sql`o.source_id = ${q.source}` : sql`s.slug = ${q.source.toLowerCase()}`);
92 + if (q.estimateType) conds.push(sql`o.estimate_type = ${q.estimateType}`);
93 + if (q.from != null && q.to != null && q.from > q.to) throw new BadRequest('`from` must not exceed `to`');
94 +
95 + const rows = await app.db.execute<Record<string, unknown> & { total: string }>(sql`
96 + SELECT o.id, o.cancer_id, c.slug AS cancer_slug, c.canonical_name AS cancer_name,
97 + o.geography_id, g.slug AS geography_slug, g.name AS geography_name, g.iso3,
98 + o.year, o.year_end, o.sex, o.age_group, o.metric, o.value, o.unit, o.lower_ci, o.upper_ci, o.standard_population, o.estimate_type, o.site_definition,
99 + o.source_id, s.slug AS source_slug, o.provenance_id, p.dataset, p.dataset_version, p.source_url, p.retrieved_at, o.updated_at,
100 + count(*) OVER() AS total
101 + FROM epidemiology_observations o
102 + JOIN cancers c ON c.id = o.cancer_id
103 + JOIN geographies g ON g.id = o.geography_id
104 + JOIN sources s ON s.id = o.source_id
105 + LEFT JOIN provenance p ON p.id = o.provenance_id
106 + WHERE ${sql.join(conds, sql` AND `)}
107 + ORDER BY c.canonical_name, c.id, g.name, o.sex, o.year, s.slug, o.site_definition
108 + LIMIT ${q.limit} OFFSET ${q.offset}`);
109 + const total = rows.length ? num(rows[0]!.total) : 0;
110 + const data = rows.map((r) => ({
111 + id: num(r.id),
112 + cancer: { id: r.cancer_id, slug: r.cancer_slug, name: r.cancer_name },
113 + geography: { id: r.geography_id, slug: r.geography_slug, name: r.geography_name, iso3: r.iso3 },
114 + year: num(r.year),
115 + yearEnd: r.year_end == null ? null : num(r.year_end),
116 + sex: r.sex,
117 + ageGroup: r.age_group,
118 + metric: r.metric,
119 + value: num(r.value),
120 + unit: r.unit,
121 + lowerCi: r.lower_ci == null ? null : num(r.lower_ci),
122 + upperCi: r.upper_ci == null ? null : num(r.upper_ci),
123 + standardPopulation: r.standard_population,
124 + estimateType: r.estimate_type,
125 + siteDefinition: r.site_definition,
126 + source: { id: r.source_id, slug: r.source_slug },
127 + provenance: { id: num(r.provenance_id), dataset: r.dataset, datasetVersion: r.dataset_version, sourceUrl: r.source_url, retrievedAt: r.retrieved_at },
128 + updatedAt: r.updated_at,
129 + }));
130 + return respond(app, data, data.map((d) => d.source.id as string), paginate(total, q.limit, q.offset));
131 + },
132 + );
133 +
134 + app.get(
135 + '/epidemiology/coverage',
136 + {
137 + schema: {
138 + tags: ['epidemiology'],
139 + summary: 'Coverage matrix: metric × geography × sex × age group × source × standard population with year span and counts',
140 + querystring: z.object({ cancer: cancerParam, geography: geographyParam, metric: z.string().regex(METRIC_RE).optional() }),
141 + response: ok(AnyList),
142 + },
143 + },
144 + async (req) => {
145 + const q = req.query;
146 + const conds = [sql`true`];
147 + const refs = splitRefs(q.cancer);
148 + if (refs.length > 0) {
149 + const ids = await Promise.all(refs.map((r) => resolveCancer(app.db, r).then((c) => c.id)));
150 + conds.push(sql`o.cancer_id = ANY(${sql.param(ids)}::text[])`);
151 + }
152 + if (q.geography) {
153 + const g = await resolveGeography(q.geography);
154 + conds.push(sql`o.geography_id = ${g.id}`);
155 + }
156 + if (q.metric) conds.push(sql`o.metric = ${q.metric}`);
157 + const rows = await app.db.execute<Record<string, unknown>>(sql`
158 + SELECT o.metric, min(o.unit) AS unit, g.id AS geography_id, g.slug AS geography_slug, g.name AS geography_name, g.iso3, o.sex, o.age_group,
159 + o.source_id, s.slug AS source_slug, o.standard_population, array_agg(DISTINCT o.estimate_type) AS estimate_types,
160 + min(o.year) AS year_from, max(coalesce(o.year_end, o.year)) AS year_to, count(DISTINCT o.year) AS years, count(*) AS observations, count(DISTINCT o.cancer_id) AS cancers, max(o.updated_at) AS last_updated
161 + FROM epidemiology_observations o JOIN geographies g ON g.id = o.geography_id JOIN sources s ON s.id = o.source_id
162 + WHERE ${sql.join(conds, sql` AND `)}
163 + GROUP BY o.metric, g.id, g.slug, g.name, g.iso3, o.sex, o.age_group, o.source_id, s.slug, o.standard_population
164 + ORDER BY o.metric, g.name, (o.sex = 'all') DESC, o.sex, (o.age_group = 'all') DESC, o.age_group, s.slug, o.standard_population`);
165 + const data = rows.map((r) => ({
166 + metric: r.metric,
167 + unit: r.unit,
168 + geography: { id: r.geography_id, slug: r.geography_slug, name: r.geography_name, iso3: r.iso3 },
169 + sex: r.sex,
170 + ageGroup: r.age_group,
171 + source: { id: r.source_id, slug: r.source_slug },
172 + standardPopulation: r.standard_population,
173 + estimateTypes: r.estimate_types,
174 + yearFrom: num(r.year_from),
175 + yearTo: num(r.year_to),
176 + years: num(r.years),
177 + observations: num(r.observations),
178 + cancers: num(r.cancers),
179 + lastUpdated: r.last_updated,
180 + }));
181 + return respond(
182 + app,
183 + data,
184 + data.map((d) => d.source.id as string),
185 + );
186 + },
187 + );
188 +
189 + app.get('/epidemiology/metrics', { schema: { tags: ['epidemiology'], summary: 'Distinct epidemiology metrics present, with unit, label, year span and counts', response: ok(AnyList) } }, async () => {
190 + const rows = await app.db.execute<Record<string, unknown>>(sql`
191 + SELECT o.metric, min(o.unit) AS unit, count(*) AS n, count(DISTINCT o.cancer_id) AS cancers, count(DISTINCT o.geography_id) AS geographies, min(o.year) AS year_from, max(coalesce(o.year_end, o.year)) AS year_to,
192 + array_agg(DISTINCT s.slug ORDER BY s.slug) AS sources, array_remove(array_agg(DISTINCT o.standard_population), NULL) AS standard_populations
193 + FROM epidemiology_observations o JOIN sources s ON s.id = o.source_id GROUP BY o.metric ORDER BY o.metric`);
194 + const data = rows.map((r) => ({
195 + metric: r.metric,
196 + label: METRIC_LABEL[r.metric as string] ?? String(r.metric).replace(/_/g, ' '),
197 + unit: r.unit,
198 + n: num(r.n),
199 + cancers: num(r.cancers),
200 + geographies: num(r.geographies),
201 + yearFrom: num(r.year_from),
202 + yearTo: num(r.year_to),
203 + sources: r.sources,
204 + standardPopulations: r.standard_populations,
205 + }));
206 + return respond(
207 + app,
208 + data,
209 + data.flatMap((d) => d.sources as string[]),
210 + );
211 + });
10 212 };
added apps/web/src/app/api/export/epidemiology.csv/route.ts +88 −0
@@ -0,0 +1,88 @@
1 +import { explorerObservations, resolveCancerRefs, resolveGeographyRef, explorerOptions } from '@/lib/queries/explorer';
2 +import { parseExplorerParams, yearRangeLabel, sexLabel, ageLabel, SEX_ANY } from '@/lib/explorer-params';
3 +import { csvComment, csvFileName, csvLine, EPI_CSV_COLUMNS } from '@/lib/explorer-csv';
4 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
5 +import { SITE_URL } from '@/lib/site';
6 +import { isoDate, toDate } from '@/lib/format';
7 +import type { SP } from '@/lib/search-params';
8 +
9 +export const dynamic = 'force-dynamic';
10 +
11 +const MAX_ROWS = 50_000;
12 +
13 +/**
14 + * GET /api/export/epidemiology.csv?metric=&cancers=a,b&geography=&sex=&age=&from=&to=
15 + * Same filters as /explore. Attribution header rows (`#`) precede the column header: CancerIndex,
16 + * each underlying source with its license, dataset version and retrieval date, methodology URL.
17 + * Every row carries its provenance id and source URL. Capped at 50 000 rows (stated in the header).
18 + */
19 +export async function GET(req: Request) {
20 + const url = new URL(req.url);
21 + const sp: SP = {};
22 + for (const [k, v] of url.searchParams) sp[k] = sp[k] == null ? v : [...(Array.isArray(sp[k]) ? (sp[k] as string[]) : [sp[k] as string]), v];
23 + const state = parseExplorerParams(sp, { metric: '', geography: '', cancers: [] });
24 + if (!state.metric) return new Response('metric is required', { status: 400 });
25 + if (!state.geography) return new Response('geography is required (slug or ISO3)', { status: 400 });
26 + if (state.cancers.length === 0) return new Response('cancers is required (comma-separated slugs or CI-CAN ids, max 6)', { status: 400 });
27 +
28 + const [geo, cancers, options] = await Promise.all([resolveGeographyRef(state.geography), resolveCancerRefs(state.cancers), explorerOptions()]);
29 + if (!geo) return new Response('unknown geography', { status: 404 });
30 + if (cancers.length === 0) return new Response('no known cancer in the selection', { status: 404 });
31 +
32 + const rows = await explorerObservations({ metric: state.metric, cancerIds: cancers.map((c) => c.id), geographyId: geo.id, sex: state.sex, age: state.age, from: state.from, to: state.to, limit: MAX_ROWS });
33 + const usedSlugs = new Set(rows.map((r) => r.source_slug));
34 + const sources = options.sources.filter((s) => usedSlugs.has(s.slug));
35 + const retrievedBySource = new Map<string, { retrieved: Date | null; dataset: string | null; version: string | null }>();
36 + for (const r of rows) {
37 + const d = toDate(r.retrieved_at);
38 + const cur = retrievedBySource.get(r.source_slug);
39 + if (!cur || (d && (!cur.retrieved || d > cur.retrieved))) retrievedBySource.set(r.source_slug, { retrieved: d, dataset: r.dataset, version: r.dataset_version });
40 + }
41 + const metricLabel = EPI_METRIC_LABEL[state.metric] ?? state.metric;
42 + const now = new Date();
43 +
44 + const header = [
45 + csvComment(`CancerIndex epidemiology export — ${metricLabel} · ${geo.name} · ${sexLabel(state.sex)} · ${ageLabel(state.age)} · ${yearRangeLabel(state.from, state.to)} · cancers: ${cancers.map((c) => c.slug).join(', ')}`),
46 + csvComment(`Source: CancerIndex (${SITE_URL}) — harmonized observations, CC BY 4.0 for the harmonization. Values are exactly as published by the providers below and remain under their licenses.`),
47 + ...sources.map((s) => {
48 + const r = retrievedBySource.get(s.slug);
49 + return csvComment(`Underlying source: ${s.name} [${s.slug}] · license: ${s.license ?? 'see source page'} · dataset: ${r?.dataset ?? '—'}${r?.version ? ` (${r.version})` : ''} · retrieved: ${r?.retrieved ? r.retrieved.toISOString() : 'unknown'} · ${s.homepage ?? `${SITE_URL}/source/${s.slug}`}`);
50 + }),
51 + ...sources.filter((s) => s.attribution).map((s) => csvComment(`Attribution (${s.slug}): ${s.attribution}`)),
52 + csvComment(`Methodology: ${SITE_URL}/methodology#data-explorer · comparability: never compare rows with different standard_population, source_slug or age_group on one axis · population statistics do not predict individual outcomes.`),
53 + csvComment(`Generated: ${now.toISOString()} · rows: ${rows.length}${rows.length >= MAX_ROWS ? ` (capped at ${MAX_ROWS}; narrow the selection or use the API)` : ''} · API: ${SITE_URL}/api/v1/epidemiology?metric=${encodeURIComponent(state.metric)}&geography=${encodeURIComponent(geo.slug)}${cancers.map((c) => `&cancer=${encodeURIComponent(c.slug)}`).join('')}${state.sex !== SEX_ANY ? `&sex=${state.sex}` : ''}&age=${encodeURIComponent(state.age)}`),
54 + ];
55 + const lines = rows.map((r) =>
56 + csvLine([
57 + r.cancer_id,
58 + r.cancer_slug,
59 + r.cancer_name,
60 + r.geography_id,
61 + r.geography_slug,
62 + r.geography_name,
63 + r.iso3,
64 + r.year,
65 + r.year_end,
66 + r.sex,
67 + r.age_group,
68 + r.metric,
69 + r.value,
70 + r.unit,
71 + r.lower_ci,
72 + r.upper_ci,
73 + r.standard_population,
74 + r.estimate_type,
75 + r.site_definition,
76 + r.source_slug,
77 + r.source_name,
78 + r.provenance_id,
79 + r.dataset,
80 + r.dataset_version,
81 + r.source_url,
82 + toDate(r.retrieved_at)?.toISOString() ?? '',
83 + ]),
84 + );
85 + const body = [...header, EPI_CSV_COLUMNS.join(','), ...lines].join('\n') + '\n';
86 + const fname = csvFileName([state.metric, geo.slug, state.sex, state.age, state.from ?? rows[0]?.year, state.to ?? rows[rows.length - 1]?.year, isoDate(now)]);
87 + return new Response(body, { headers: { 'Content-Type': 'text/csv; charset=utf-8', 'Content-Disposition': `attachment; filename="${fname}"`, 'Cache-Control': 'public, max-age=300' } });
88 +}
added apps/web/src/app/explore/loading.tsx +5 −0
@@ -0,0 +1,5 @@
1 +import { PageSkeleton } from '@/components/ui/skeleton';
2 +
3 +export default function Loading() {
4 + return <PageSkeleton title="Loading the data explorer" />;
5 +}
added apps/web/src/app/explore/page.tsx +258 −0
@@ -0,0 +1,258 @@
1 +import type { Metadata } from 'next';
2 +import Link from 'next/link';
3 +import { PageHeader, Section, Note } from '@/components/ui/section';
4 +import { EmptyState } from '@/components/ui/empty-state';
5 +import { Freshness } from '@/components/ui/freshness';
6 +import { Badge } from '@/components/ui/badge';
7 +import { ExplorerFilters } from '@/components/explorer/filters';
8 +import { ChartGroup, assignSlots, groupProvenance } from '@/components/explorer/chart-group';
9 +import { ObservationsTable } from '@/components/explorer/observations-table';
10 +import { CoverageTable } from '@/components/explorer/coverage-table';
11 +import { explorerOptions, resolveGeographyRef, resolveCancerRefs, topLevelCancerChoices, topCancersByLatest, yearRangeFor, explorerObservations, coverageMatrix, pendingEpidemiologySources, type ExplorerObsRow } from '@/lib/queries/explorer';
12 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
13 +import { parseExplorerParams, serializeExplorerParams, apiQueryFor, yearRangeLabel, sexLabel, ageLabel, SEX_ANY, MAX_CANCERS, type ExplorerState } from '@/lib/explorer-params';
14 +import { groupComparable, explainSplit } from '@/lib/explorer-series';
15 +import { SITE_URL, SITE_NAME } from '@/lib/site';
16 +import type { SP } from '@/lib/search-params';
17 +import { fmtDate, fmtInt, humanize, toDate, unitLabel } from '@/lib/format';
18 +
19 +export const revalidate = 600;
20 +
21 +export const metadata: Metadata = {
22 + title: 'Data explorer — cancer statistics by metric, cancer, geography, sex and year',
23 + description: 'Explore cancer incidence and mortality observations: choose a metric, up to six cancers, a geography, sex, age group and years; chart, table, sources, CSV and API links, permalink.',
24 + alternates: { canonical: '/explore' },
25 +};
26 +
27 +const DEFAULT_METRIC = 'mortality_count';
28 +const DEFAULT_GEOGRAPHY = 'united-states';
29 +
30 +/**
31 + * /explore — "Our World in Data"-style explorer over epidemiology_observations. URL = state = permalink.
32 + * Comparable observations (same metric, unit, geography, source, standard population, age group) share
33 + * one chart; anything else is a separate chart with the reason stated. Every number is shown with its
34 + * unit, geography, years, sex, age group, standard population, source and retrieval date.
35 + */
36 +export default async function ExplorePage({ searchParams }: { searchParams: Promise<SP> }) {
37 + const sp = await searchParams;
38 + const [options, pending] = await Promise.all([explorerOptions(), pendingEpidemiologySources()]);
39 + const metricFallback = options.metrics.some((m) => m.metric === DEFAULT_METRIC) ? DEFAULT_METRIC : (options.metrics[0]?.metric ?? DEFAULT_METRIC);
40 + const geoFallback = options.geographies.some((g) => g.slug === DEFAULT_GEOGRAPHY) ? DEFAULT_GEOGRAPHY : (options.geographies[0]?.slug ?? DEFAULT_GEOGRAPHY);
41 +
42 + // Pass 1: metric / geography / sex / age from the URL so the computed defaults (top cancers, years) match them.
43 + const prelim = parseExplorerParams(sp, { metric: metricFallback, geography: geoFallback, cancers: [] });
44 + const geo = await resolveGeographyRef(prelim.geography);
45 + const [top, range] = geo ? await Promise.all([topCancersByLatest(prelim.metric, geo.id, prelim.sex === SEX_ANY ? 'all' : prelim.sex, prelim.age, 5), yearRangeFor(prelim.metric, geo.id)]) : [null, null];
46 + const state = parseExplorerParams(sp, { metric: metricFallback, geography: geoFallback, cancers: top?.cancers.map((c) => c.slug) ?? [], from: range?.min ?? null, to: range?.max ?? null });
47 + const usingDefaultCancers = !sp.cancers || (Array.isArray(sp.cancers) ? sp.cancers.every((s) => !s.trim()) : !sp.cancers.trim());
48 +
49 + const [cancers, choices] = await Promise.all([resolveCancerRefs(state.cancers), topLevelCancerChoices(state.metric, geo?.id ?? null)]);
50 + const unresolved = state.cancers.filter((c) => !cancers.some((k) => k.slug === c || k.id === c));
51 + const obs = geo && cancers.length > 0 ? await explorerObservations({ metric: state.metric, cancerIds: cancers.map((c) => c.id), geographyId: geo.id, sex: state.sex, age: state.age, from: state.from, to: state.to }) : [];
52 + const groups = groupComparable(obs);
53 + const slots = assignSlots(groups);
54 + const split = explainSplit(groups);
55 +
56 + const metricLabel = EPI_METRIC_LABEL[state.metric] ?? humanize(state.metric);
57 + const metricOpt = options.metrics.find((m) => m.metric === state.metric);
58 + const permalink = `${SITE_URL}/explore${serializeExplorerParams(state, { page: 1 })}`;
59 + const csvHref = `/api/export/epidemiology.csv${serializeExplorerParams(state, { page: 1 })}`;
60 + const apiHref = `/api/v1/epidemiology${apiQueryFor(state)}`;
61 + const hrefFor = (page: number) => `/explore${serializeExplorerParams(state, { page })}`;
62 +
63 + // Sources actually behind the result, with their latest retrieval date.
64 + const sourcesUsed = sourcesIn(obs);
65 + const freshest = obs.map((o) => toDate(o.updated_at)).filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? null;
66 + const coverage = obs.length === 0 ? await coverageMatrix({ cancerIds: cancers.map((c) => c.id), geographyId: geo?.id ?? null }) : [];
67 + const ingestedNames = options.sources.map((s) => s.name.split(' — ')[0]!);
68 +
69 + return (
70 + <div>
71 + <PageHeader kicker="Data" title="Data explorer" lede="Chart and download registry observations by metric, cancer, geography, sex, age group and year. Values are exactly as published by the source; CancerIndex harmonizes units and labels, never the numbers. The URL is the permalink.">
72 + <p className="mt-2 text-[12.5px] text-ink-3">
73 + {fmtInt(options.n_obs)} observations · {options.metrics.length} metrics · {options.geographies.length} {options.geographies.length === 1 ? 'geography' : 'geographies'} · {options.year_min}–{options.year_max} · sources: {options.sources.map((s) => s.slug).join(', ')} ·{' '}
74 + <Link className="ci-link" href="/explore/coverage">
75 + full coverage matrix
76 + </Link>{' '}
77 + ·{' '}
78 + <Link className="ci-link" href="/methodology#data-explorer">
79 + comparability rules
80 + </Link>
81 + </p>
82 + </PageHeader>
83 +
84 + <ExplorerFilters options={options} state={state} choices={choices} />
85 +
86 + {unresolved.length > 0 ? (
87 + <Note tone="warn">
88 + Unknown cancer reference{unresolved.length === 1 ? '' : 's'} ignored: <span className="ci-mono">{unresolved.join(', ')}</span>. Use a taxonomy slug (as in /cancer/&lt;slug&gt;) or a CI-CAN id.
89 + </Note>
90 + ) : null}
91 +
92 + <Section
93 + id="results"
94 + kicker="Selection"
95 + title={<SelectionSentence state={state} metricLabel={metricLabel} unit={metricOpt?.unit ?? obs[0]?.unit ?? null} geographyName={geo?.name ?? state.geography} />}
96 + description={
97 + <>
98 + {sourcesUsed.length > 0 ? (
99 + <>
100 + Source{sourcesUsed.length === 1 ? '' : 's'}: {sourcesUsed.map((s) => `${s.name}${s.retrieved ? ` (retrieved ${fmtDate(s.retrieved)})` : ''}`).join('; ')}.{' '}
101 + </>
102 + ) : null}
103 + {usingDefaultCancers && top?.cancers.length ? (
104 + <>
105 + Default selection: the {top.cancers.length} top-level cancers with the highest {metricLabel.toLowerCase()} in {top.year} ({geo?.name}), computed from the observations — not a curated list.
106 + </>
107 + ) : null}
108 + </>
109 + }
110 + actions={<Downloads csvHref={csvHref} apiHref={apiHref} disabled={obs.length === 0} />}
111 + >
112 + {obs.length === 0 ? (
113 + <div className="space-y-4">
114 + <EmptyState
115 + title="Data not yet available"
116 + knows={[
117 + ...(geo ? [{ label: `${geo.name} country page`, href: `/country/${geo.slug}` }] : []),
118 + ...cancers.slice(0, 3).map((c) => ({ label: c.canonical_name, href: `/cancer/${c.slug}/statistics` })),
119 + { label: 'Full coverage matrix', href: '/explore/coverage' },
120 + { label: 'Sources and license status', href: '/sources' },
121 + ]}
122 + >
123 + No observation matches {metricLabel.toLowerCase()} · {geo?.name ?? `geography "${state.geography}"`} · {sexLabel(state.sex)} · {ageLabel(state.age)} · {yearRangeLabel(state.from, state.to)}
124 + {cancers.length > 0 ? ` for ${cancers.map((c) => c.canonical_name).join(', ')}` : cancers.length === 0 && state.cancers.length === 0 ? ' — no cancer selected' : ''}.
125 + {!geo ? ` "${state.geography}" is not a known geography slug or ISO3 code.` : ''} Nothing is estimated or extrapolated. {coverage.length > 0 ? 'What does exist for this selection is listed below.' : ''}
126 + </EmptyState>
127 + {coverage.length > 0 ? (
128 + <div>
129 + <p className="ci-kicker mb-1">What exists for {cancers.length > 0 ? `${cancers.length === 1 ? cancers[0]!.canonical_name : `these ${cancers.length} cancers`}` : 'every cancer'}{geo ? ` in ${geo.name}` : ''}</p>
130 + <CoverageTable rows={coverage} cancers={cancers.map((c) => c.slug)} compact />
131 + </div>
132 + ) : null}
133 + </div>
134 + ) : (
135 + <div className="space-y-8">
136 + {split ? <Note tone="warn">{split}</Note> : null}
137 + {groups.map((g, i) => (
138 + <ChartGroup key={g.key} group={g} view={state.view} slots={slots} provenance={groupProvenance(g, provenanceFor(g.source_slug, g.provenance_ids, obs))} index={i} total={groups.length} />
139 + ))}
140 + <Freshness dataUpdatedAt={freshest} sourceVersion={sourcesUsed.map((s) => s.version).filter(Boolean).join(' · ') || null} extra={`${fmtInt(obs.length)} observations · ${groups.length} comparable ${groups.length === 1 ? 'group' : 'groups'}`} />
141 + </div>
142 + )}
143 + </Section>
144 +
145 + {obs.length > 0 ? (
146 + <Section id="observations" kicker="Table" title="Observations" description="Every observation behind the charts. Sorted by cancer, geography, sex and year. Counts are per site group as published — they are not summed into an all-sites total.">
147 + <ObservationsTable rows={obs} page={state.page} hrefFor={hrefFor} />
148 + </Section>
149 + ) : null}
150 +
151 + <Section id="share" kicker="Share and cite" title="Permalink, downloads and citation">
152 + <div className="grid gap-4 md:grid-cols-2">
153 + <div className="min-w-0">
154 + <label className="flex flex-col gap-1 text-[13px]">
155 + <span className="ci-kicker">Permalink (this selection)</span>
156 + <input readOnly value={permalink} className="ci-mono w-full border border-rule-strong bg-paper-2 px-2 py-1.5 text-[12px] text-ink-2" aria-label="Permalink for this selection" />
157 + </label>
158 + <ul className="mt-2 flex flex-wrap gap-x-4 gap-y-1 text-[13px]">
159 + <li>
160 + <a className="ci-link" href={csvHref}>
161 + Download CSV
162 + </a>{' '}
163 + <span className="text-ink-3">(attribution header, provenance id and source URL per row, ≤ 50 000 rows)</span>
164 + </li>
165 + <li>
166 + <a className="ci-link" href={apiHref}>
167 + JSON (API)
168 + </a>{' '}
169 + <span className="text-ink-3">
170 + · <Link className="ci-link" href="/developers">API docs</Link>
171 + </span>
172 + </li>
173 + </ul>
174 + </div>
175 + <div className="min-w-0 text-[13px] leading-relaxed">
176 + <p className="ci-kicker mb-1">Cite</p>
177 + <p className="text-ink-2">
178 + {SITE_NAME} ({new Date().getUTCFullYear()}). Data explorer: {metricLabel}, {geo?.name ?? state.geography}, {sexLabel(state.sex)}, {ageLabel(state.age)}, {yearRangeLabel(state.from, state.to)}. {SITE_URL}/explore (accessed {fmtDate(new Date())}).{' '}
179 + {sourcesUsed.length > 0 ? (
180 + <>
181 + Underlying observations: {sourcesUsed.map((s) => `${s.name}${s.dataset ? `, ${s.dataset}` : ''}${s.retrieved ? `, retrieved ${fmtDate(s.retrieved)}` : ''}`).join('; ')}.
182 + </>
183 + ) : null}
184 + </p>
185 + <p className="mt-1 text-[12px] text-ink-3">Underlying observations remain under their providers' licenses (see each source page); CancerIndex's harmonization is CC BY 4.0.</p>
186 + </div>
187 + </div>
188 + </Section>
189 +
190 + <div className="mt-6 space-y-2">
191 + <Note tone="warn">Population statistics describe groups defined by geography, period, sex and age. They do not predict any individual's risk or outcome. Values labelled "estimated" or "projected" are model outputs of the source, not registry counts.</Note>
192 + <Note>
193 + Geographies are limited to the sources currently ingested: {ingestedNames.join('; ')} ({options.geographies.map((g) => g.name).join(', ')}).
194 + {pending.length > 0 ? <> Registered but not yet ingested: {pending.map((p) => `${p.name.split(' — ')[0]} (${pendingReason(p)})`).join('; ')} — their geographies appear only once the licensing gate is passed and a sync has succeeded; nothing is shown from them until then.</> : null} Up to {MAX_CANCERS} cancers per chart; different standard populations, sources or age groups are never overlaid (
195 + <Link className="ci-link" href="/methodology#data-explorer">
196 + rules
197 + </Link>
198 + ).
199 + </Note>
200 + </div>
201 + </div>
202 + );
203 +}
204 +
205 +/** Why a registered epidemiology source has no observation yet: its connector status when not active, else its license status. */
206 +function pendingReason(p: { status: string; license_status: string }): string {
207 + const s = p.status !== 'active' ? p.status : p.license_status;
208 + return s.replace(/_/g, ' ');
209 +}
210 +
211 +function SelectionSentence({ state, metricLabel, unit, geographyName }: { state: ExplorerState; metricLabel: string; unit: string | null; geographyName: string }) {
212 + return (
213 + <span className="text-lg sm:text-xl">
214 + {metricLabel}
215 + {unit ? <span className="text-ink-3"> ({unitLabel(unit)})</span> : null} · {geographyName} · {sexLabel(state.sex)} · {ageLabel(state.age)} · {yearRangeLabel(state.from, state.to)}
216 + {state.view === 'multiples' ? <Badge tone="outline" className="ml-2 align-middle">small multiples</Badge> : null}
217 + </span>
218 + );
219 +}
220 +
221 +function Downloads({ csvHref, apiHref, disabled }: { csvHref: string; apiHref: string; disabled: boolean }) {
222 + if (disabled) return <span className="text-ink-4">no rows to download</span>;
223 + return (
224 + <>
225 + <a className="ci-link" href={csvHref}>
226 + CSV
227 + </a>
228 + <span aria-hidden className="text-ink-4">
229 + ·
230 + </span>
231 + <a className="ci-link" href={apiHref}>
232 + JSON (API)
233 + </a>
234 + </>
235 + );
236 +}
237 +
238 +function sourcesIn(obs: ExplorerObsRow[]): Array<{ slug: string; name: string; retrieved: Date | null; dataset: string | null; version: string | null }> {
239 + const m = new Map<string, { slug: string; name: string; retrieved: Date | null; dataset: string | null; version: string | null }>();
240 + for (const o of obs) {
241 + const r = toDate(o.retrieved_at);
242 + const cur = m.get(o.source_slug);
243 + if (!cur) m.set(o.source_slug, { slug: o.source_slug, name: o.source_name, retrieved: r, dataset: o.dataset, version: o.dataset_version });
244 + else if (r && (!cur.retrieved || r > cur.retrieved)) m.set(o.source_slug, { ...cur, retrieved: r, dataset: o.dataset, version: o.dataset_version });
245 + }
246 + return [...m.values()].sort((a, b) => a.slug.localeCompare(b.slug));
247 +}
248 +
249 +/** Most recently retrieved provenance row of a group (its observations may span several datasets/runs). */
250 +function provenanceFor(sourceSlug: string, provenanceIds: number[], obs: ExplorerObsRow[]) {
251 + const ids = new Set(provenanceIds);
252 + let best: ExplorerObsRow | undefined;
253 + for (const o of obs) {
254 + if (o.source_slug !== sourceSlug || !ids.has(o.provenance_id)) continue;
255 + if (!best || String(o.retrieved_at ?? '') > String(best.retrieved_at ?? '')) best = o;
256 + }
257 + return best ? { dataset: best.dataset, dataset_version: best.dataset_version, retrieved_at: best.retrieved_at, source_url: best.source_url, license: best.source_license } : undefined;
258 +}
added apps/web/src/components/charts/explorer-chart.tsx +113 −0
@@ -0,0 +1,113 @@
1 +import { fmtValue } from '@/lib/format';
2 +
3 +export interface ExplorerChartSeries {
4 + name: string;
5 + points: Array<{ x: number; y: number; lo?: number | null; hi?: number | null }>;
6 + dashed?: boolean; // estimate_type ≠ observed
7 + /** Fixed colour slot (0-based) so the same cancer keeps its hue across panels and after filtering. */
8 + slot?: number;
9 +}
10 +
11 +/**
12 + * Categorical palette of the Data explorer: eight hues assigned in fixed order (never cycled — beyond
13 + * eight series the page switches to small multiples). Light and dark values are separate steps of the
14 + * same hues, both validated for colour-vision-deficiency separation and contrast against the paper
15 + * surfaces (`--color-paper` #fafaf7 / #151514). Colour is never the only carrier: every series has a text
16 + * legend entry, the tooltip names the series, and the observations table lists every value.
17 + */
18 +export const EXPLORER_PALETTE_LIGHT = ['#2a78d6', '#eb6834', '#1baf7a', '#eda100', '#e87ba4', '#008300', '#4a3aa7', '#e34948'] as const;
19 +export const EXPLORER_PALETTE_DARK = ['#3987e5', '#d95926', '#199e70', '#c98500', '#d55181', '#008300', '#9085e9', '#e66767'] as const;
20 +export const MAX_PALETTE_SLOTS = EXPLORER_PALETTE_LIGHT.length;
21 +
22 +const PALETTE_CSS = `.ci-explorer-chart{${EXPLORER_PALETTE_LIGHT.map((c, i) => `--ex-${i}:${c};`).join('')}}
23 +:root[data-theme='dark'] .ci-explorer-chart,:root:not([data-theme='light']):not([data-theme='dark']).ci-system-dark .ci-explorer-chart{${EXPLORER_PALETTE_DARK.map((c, i) => `--ex-${i}:${c};`).join('')}}
24 +@media (prefers-color-scheme: dark){:root:not([data-theme='light']) .ci-explorer-chart{${EXPLORER_PALETTE_DARK.map((c, i) => `--ex-${i}:${c};`).join('')}}}`;
25 +
26 +export function slotColor(slot: number): string {
27 + return `var(--ex-${Math.max(0, Math.min(MAX_PALETTE_SLOTS - 1, slot))})`;
28 +}
29 +
30 +/**
31 + * Time-series chart in pure SVG (server-rendered): years on x, values on y from zero, optional 95 % CI
32 + * band, dashed lines for estimated values, 8 px hit targets with a native tooltip per point.
33 + * `yMax` lets several panels share one axis (small multiples); the axis range is always stated by the caller.
34 + */
35 +export function ExplorerChart({ series, unit, ariaLabel, height = 240, yMax, compact = false }: { series: ExplorerChartSeries[]; unit?: string | null; ariaLabel: string; height?: number; yMax?: number; compact?: boolean }) {
36 + const all = series.flatMap((s) => s.points);
37 + if (all.length === 0) return null;
38 + const xs = all.map((p) => p.x);
39 + const ys = all.flatMap((p) => [p.y, p.lo ?? p.y, p.hi ?? p.y]);
40 + const xMin = Math.min(...xs);
41 + const xMax = Math.max(...xs);
42 + const top = Math.max(yMax ?? 0, ...ys, Number.EPSILON) * 1.08;
43 + const width = 640;
44 + const pad = { l: 60, r: 14, t: 12, b: 28 };
45 + const iw = width - pad.l - pad.r;
46 + const ih = height - pad.t - pad.b;
47 + const sx = (x: number) => pad.l + (xMax === xMin ? iw / 2 : ((x - xMin) / (xMax - xMin)) * iw);
48 + const sy = (y: number) => pad.t + ih - (y / top) * ih;
49 + const yTicks = 4;
50 + const xTickCount = Math.min(compact ? 5 : 8, xMax - xMin + 1);
51 + const xTicks = Array.from({ length: xTickCount }, (_, i) => Math.round(xMin + ((xMax - xMin) * i) / Math.max(1, xTickCount - 1)));
52 + const fmt = (v: number) => fmtValue(v, unit);
53 + return (
54 + <figure className="ci-explorer-chart w-full">
55 + <style href="ci-explorer-chart-palette" precedence="default">
56 + {PALETTE_CSS}
57 + </style>
58 + <svg viewBox={`0 0 ${width} ${height}`} width="100%" role="img" aria-label={ariaLabel} className="block">
59 + <title>{ariaLabel}</title>
60 + {Array.from({ length: yTicks + 1 }, (_, i) => {
61 + const v = (top / yTicks) * i;
62 + const y = sy(v);
63 + return (
64 + <g key={i}>
65 + <line x1={pad.l} x2={width - pad.r} y1={y} y2={y} stroke="var(--color-rule)" strokeWidth="1" />
66 + <text x={pad.l - 6} y={y + 4} textAnchor="end" fontSize="11" fill="var(--color-ink-3)" style={{ fontVariantNumeric: 'tabular-nums' }}>
67 + {fmt(v)}
68 + </text>
69 + </g>
70 + );
71 + })}
72 + <line x1={pad.l} x2={width - pad.r} y1={sy(0)} y2={sy(0)} stroke="var(--color-rule-strong)" strokeWidth="1" />
73 + {xTicks.map((x) => (
74 + <text key={x} x={sx(x)} y={height - 8} textAnchor="middle" fontSize="11" fill="var(--color-ink-3)">
75 + {x}
76 + </text>
77 + ))}
78 + {series.map((s, si) => {
79 + const pts = [...s.points].sort((a, b) => a.x - b.x);
80 + const color = slotColor(s.slot ?? si);
81 + const d = pts.map((p, i) => `${i === 0 ? 'M' : 'L'}${sx(p.x).toFixed(1)},${sy(p.y).toFixed(1)}`).join(' ');
82 + const band = pts.filter((p) => p.lo != null && p.hi != null);
83 + const bandPath = band.length > 1 ? `${band.map((p, i) => `${i === 0 ? 'M' : 'L'}${sx(p.x).toFixed(1)},${sy(p.hi!).toFixed(1)}`).join(' ')} ${[...band].reverse().map((p) => `L${sx(p.x).toFixed(1)},${sy(p.lo!).toFixed(1)}`).join(' ')} Z` : null;
84 + return (
85 + <g key={s.name}>
86 + {bandPath ? <path d={bandPath} fill={color} opacity="0.12" /> : null}
87 + <path d={d} fill="none" stroke={color} strokeWidth="2" strokeLinejoin="round" strokeDasharray={s.dashed ? '5 3' : undefined} />
88 + {pts.map((p) => (
89 + <g key={p.x}>
90 + <circle cx={sx(p.x)} cy={sy(p.y)} r="2" fill={color} stroke="var(--color-paper)" strokeWidth="1" />
91 + <circle cx={sx(p.x)} cy={sy(p.y)} r="5" fill="transparent">
92 + <title>{`${s.name} — ${p.x}: ${fmt(p.y)}${p.lo != null && p.hi != null ? ` (95% CI ${fmt(p.lo)}–${fmt(p.hi)})` : ''}${s.dashed ? ' · estimated' : ''}`}</title>
93 + </circle>
94 + </g>
95 + ))}
96 + </g>
97 + );
98 + })}
99 + </svg>
100 + {series.length > 1 || !compact ? (
101 + <figcaption className="mt-1 flex flex-wrap gap-x-4 gap-y-1 text-[12px] text-ink-2">
102 + {series.map((s, si) => (
103 + <span key={s.name} className="inline-flex items-center gap-1.5">
104 + <span className="inline-block h-0 w-4 border-t-2" style={{ borderColor: slotColor(s.slot ?? si), borderTopStyle: s.dashed ? 'dashed' : 'solid' }} aria-hidden />
105 + {s.name}
106 + {s.dashed ? ' (estimated)' : ''}
107 + </span>
108 + ))}
109 + </figcaption>
110 + ) : null}
111 + </figure>
112 + );
113 +}
added apps/web/src/components/explorer/chart-group.tsx +120 −0
@@ -0,0 +1,120 @@
1 +import { ClaimBadge, Badge } from '@/components/ui/badge';
2 +import { SourceBadge, type ProvenanceInfo } from '@/components/ui/source-badge';
3 +import { ExplorerChart, MAX_PALETTE_SLOTS } from '@/components/charts/explorer-chart';
4 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
5 +import { fmtDate, fmtInt, fmtValue, humanize, unitLabel } from '@/lib/format';
6 +import { splitIntoMultiples, shouldUseMultiples, type ComparableGroup, type ExplorerSeries } from '@/lib/explorer-series';
7 +import type { ExplorerView } from '@/lib/explorer-params';
8 +
9 +/**
10 + * Colour slots follow the entity, never the rank: one slot per (cancer × sex) identity in order of first
11 + * appearance across every group of the page. Beyond the palette size the panels are per cancer and the
12 + * slot follows the sex inside each panel (the panel title carries the cancer).
13 + */
14 +export function assignSlots(groups: readonly ComparableGroup[]): { bySeries: Map<string, number> | null; bySex: Map<string, number> } {
15 + const ids: string[] = [];
16 + const sexes: string[] = [];
17 + for (const g of groups)
18 + for (const s of g.series) {
19 + const id = `${s.cancer_slug}|${s.sex}`;
20 + if (!ids.includes(id)) ids.push(id);
21 + if (!sexes.includes(s.sex)) sexes.push(s.sex);
22 + }
23 + const bySex = new Map(sexes.map((s, i) => [s, i % MAX_PALETTE_SLOTS]));
24 + if (ids.length > MAX_PALETTE_SLOTS) return { bySeries: null, bySex };
25 + return { bySeries: new Map(ids.map((id, i) => [id, i])), bySex };
26 +}
27 +
28 +function slotFor(s: ExplorerSeries, slots: ReturnType<typeof assignSlots>): number {
29 + return slots.bySeries?.get(`${s.cancer_slug}|${s.sex}`) ?? slots.bySex.get(s.sex) ?? 0;
30 +}
31 +
32 +export function groupProvenance(g: ComparableGroup, prov: { dataset?: string | null; dataset_version?: string | null; retrieved_at?: Date | string | null; source_url?: string | null; license?: string | null } | undefined): ProvenanceInfo {
33 + return {
34 + sourceSlug: g.source_slug,
35 + sourceName: g.source_name,
36 + dataset: prov?.dataset ?? null,
37 + datasetVersion: prov?.dataset_version ?? null,
38 + retrievedAt: prov?.retrieved_at ?? null,
39 + sourceUrl: prov?.source_url ?? null,
40 + license: prov?.license ?? null,
41 + layer: 'normalized',
42 + evidenceType: 'observed_data',
43 + };
44 +}
45 +
46 +/**
47 + * One comparable group → one chart (or small multiples with a shared axis). The caption states every
48 + * dimension that defines the group: metric, unit, geography, sex, age group, years, standard population,
49 + * source and retrieval date — nothing on the axis is left to be inferred.
50 + */
51 +export function ChartGroup({
52 + group,
53 + view,
54 + slots,
55 + provenance,
56 + index,
57 + total,
58 +}: {
59 + group: ComparableGroup;
60 + view: ExplorerView;
61 + slots: ReturnType<typeof assignSlots>;
62 + provenance: ProvenanceInfo;
63 + index: number;
64 + total: number;
65 +}) {
66 + const g = group;
67 + const metricLabel = EPI_METRIC_LABEL[g.metric] ?? humanize(g.metric);
68 + const multiples = shouldUseMultiples(g, view);
69 + const panels = multiples ? splitIntoMultiples(g) : [g];
70 + const sexes = [...new Set(g.series.map((s) => s.sex))];
71 + const hasEstimates = g.series.some((s) => s.dashed);
72 + const hasCi = g.series.some((s) => s.points.some((p) => p.lo != null && p.hi != null));
73 + const years = g.year_min === g.year_max ? String(g.year_min) : `${g.year_min}–${g.year_max}`;
74 + const toChart = (s: ExplorerSeries) => ({ name: s.name, points: s.points, dashed: s.dashed, slot: slotFor(s, slots) });
75 + return (
76 + <section aria-label={`${metricLabel}, ${g.geography_name}, source ${g.source_slug}${total > 1 ? ` (chart ${index + 1} of ${total})` : ''}`} className="min-w-0">
77 + <div className="mb-1.5 flex flex-wrap items-baseline justify-between gap-x-3 gap-y-1">
78 + <h3 className="text-base">
79 + {total > 1 ? <span className="ci-kicker mr-2">Chart {index + 1} of {total}</span> : null}
80 + {metricLabel} · {g.geography_name}
81 + </h3>
82 + <span className="text-[12px] text-ink-3">
83 + {unitLabel(g.unit)} · {sexes.length === 1 ? (sexes[0] === 'all' ? 'both sexes' : humanize(sexes[0]!).toLowerCase()) : 'by sex'} · {g.age_group === 'all' ? 'all ages' : `ages ${g.age_group}`} · {years}
84 + {g.standard_population ? ` · standard population: ${g.standard_population}` : ''}
85 + </span>
86 + </div>
87 + {multiples ? (
88 + <>
89 + <p className="mb-2 text-[12px] text-ink-3">
90 + Small multiples: one panel per cancer, <strong className="font-medium text-ink-2">shared y-axis 0–{fmtValue(g.y_max * 1.08, g.unit)} {unitLabel(g.unit)}</strong> across the {panels.length} panels so heights are comparable.
91 + </p>
92 + <div className="grid gap-x-6 gap-y-4 sm:grid-cols-2 lg:grid-cols-3">
93 + {panels.map((p) => (
94 + <div key={p.key} className="min-w-0">
95 + <p className="mb-0.5 truncate text-[13px] font-medium text-ink" title={p.series[0]?.cancer_name}>
96 + {p.series[0]?.cancer_name}
97 + </p>
98 + <ExplorerChart series={p.series.map(toChart)} unit={p.unit} yMax={g.y_max} height={170} compact ariaLabel={`${metricLabel}, ${p.series[0]?.cancer_name ?? ''}, ${g.geography_name}, ${years}, source ${g.source_slug}`} />
99 + </div>
100 + ))}
101 + </div>
102 + </>
103 + ) : (
104 + <ExplorerChart series={g.series.map(toChart)} unit={g.unit} ariaLabel={`${metricLabel} in ${g.geography_name}, ${years}, ${g.series.length} series, source ${g.source_slug}`} />
105 + )}
106 + {/* div, not p: the SourceBadge popover contains a <dl>. */}
107 + <div className="mt-1.5 flex flex-wrap items-center gap-x-2 gap-y-1 text-[12px] text-ink-3">
108 + <SourceBadge p={provenance} />
109 + <ClaimBadge kind="observed" />
110 + <span>
111 + {g.source_name ?? g.source_slug}
112 + {provenance.retrievedAt ? ` (retrieved ${fmtDate(provenance.retrievedAt)})` : ''} · {fmtInt(g.n_obs)} observations · {g.series.length} series
113 + </span>
114 + {hasCi ? <span>· shaded band = 95% CI as published</span> : null}
115 + {hasEstimates ? <Badge tone="warn">dashed = estimated by the source</Badge> : null}
116 + {g.unit === 'count' ? <span>· counts are per site group, not summed</span> : null}
117 + </div>
118 + </section>
119 + );
120 +}
added apps/web/src/components/explorer/coverage-table.tsx +92 −0
@@ -0,0 +1,92 @@
1 +import Link from 'next/link';
2 +import { Badge } from '@/components/ui/badge';
3 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
4 +import type { CoverageMatrixRow } from '@/lib/queries/explorer';
5 +import { fmtDate, fmtInt, humanize, unitLabel } from '@/lib/format';
6 +
7 +/**
8 + * Coverage matrix: what exists, per metric × geography × sex × age group × source × standard population,
9 + * with the year span, the number of distinct years, observations and cancers. Each row links to the
10 + * explorer with the same dimensions so a gap can be checked in one click.
11 + */
12 +export function CoverageTable({ rows, cancers, compact = false }: { rows: CoverageMatrixRow[]; cancers?: string[]; compact?: boolean }) {
13 + if (rows.length === 0) return null;
14 + const hrefFor = (r: CoverageMatrixRow) => {
15 + const qs = new URLSearchParams({ metric: r.metric, geography: r.geography_slug, sex: r.sex, age: r.age_group, from: String(r.year_from), to: String(r.year_to) });
16 + if (cancers && cancers.length > 0) qs.set('cancers', cancers.join(','));
17 + return `/explore?${qs.toString()}`;
18 + };
19 + return (
20 + <div className="ci-table-wrap">
21 + <table className="ci-table">
22 + <thead>
23 + <tr>
24 + <th scope="col">Metric</th>
25 + <th scope="col">Geography</th>
26 + <th scope="col">Sex</th>
27 + <th scope="col">Age</th>
28 + <th scope="col">Source</th>
29 + <th scope="col">Standard population</th>
30 + <th scope="col">Years</th>
31 + <th scope="col" className="num">
32 + Distinct years
33 + </th>
34 + <th scope="col" className="num">
35 + Observations
36 + </th>
37 + {compact ? null : (
38 + <th scope="col" className="num">
39 + Cancers
40 + </th>
41 + )}
42 + <th scope="col">Type</th>
43 + {compact ? null : <th scope="col">Updated</th>}
44 + <th scope="col">
45 + <span className="sr-only">Open</span>
46 + </th>
47 + </tr>
48 + </thead>
49 + <tbody>
50 + {rows.map((r, i) => (
51 + <tr key={`${r.metric}|${r.geography_id}|${r.sex}|${r.age_group}|${r.source_slug}|${r.standard_population ?? ''}|${i}`}>
52 + <td className="whitespace-nowrap">
53 + {EPI_METRIC_LABEL[r.metric] ?? humanize(r.metric)} <span className="text-[11.5px] text-ink-3">({unitLabel(r.unit)})</span>
54 + </td>
55 + <td>
56 + <Link className="ci-link" href={`/country/${r.geography_slug}`}>
57 + {r.geography_name}
58 + </Link>
59 + {r.iso3 ? <span className="ci-mono ml-1 text-[11px] text-ink-3">{r.iso3}</span> : null}
60 + </td>
61 + <td>{r.sex === 'all' ? 'Both' : humanize(r.sex)}</td>
62 + <td>{r.age_group === 'all' ? 'All ages' : r.age_group}</td>
63 + <td>
64 + <Link className="ci-src" href={`/source/${r.source_slug}`} title={r.source_name}>
65 + {r.source_slug}
66 + </Link>
67 + </td>
68 + <td className="max-w-[220px] text-[12px] text-ink-3">{r.standard_population ?? '—'}</td>
69 + <td className="ci-num whitespace-nowrap">{r.year_from === r.year_to ? r.year_from : `${r.year_from}–${r.year_to}`}</td>
70 + <td className="num">{fmtInt(r.years)}</td>
71 + <td className="num">{fmtInt(r.observations)}</td>
72 + {compact ? null : <td className="num">{fmtInt(r.n_cancers)}</td>}
73 + <td>
74 + {r.estimate_types.map((t) => (
75 + <Badge key={t} tone={t === 'observed' ? 'ok' : 'warn'} className="mr-1">
76 + {t}
77 + </Badge>
78 + ))}
79 + </td>
80 + {compact ? null : <td className="whitespace-nowrap text-[12px] text-ink-3">{fmtDate(r.last_updated)}</td>}
81 + <td>
82 + <Link className="ci-link whitespace-nowrap text-[12px]" href={hrefFor(r)}>
83 + Open →
84 + </Link>
85 + </td>
86 + </tr>
87 + ))}
88 + </tbody>
89 + </table>
90 + </div>
91 + );
92 +}
added apps/web/src/components/explorer/filters.tsx +115 −0
@@ -0,0 +1,115 @@
1 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
2 +import type { ExplorerOptions, CancerChoice } from '@/lib/queries/explorer';
3 +import { MAX_CANCERS, SEX_ANY, VIEWS, type ExplorerState } from '@/lib/explorer-params';
4 +import { fmtInt, humanize, unitLabel } from '@/lib/format';
5 +
6 +const INPUT = 'border border-rule-strong bg-white px-2 py-1.5 text-[13.5px] text-ink outline-none focus:border-accent dark:bg-paper-2';
7 +
8 +/**
9 + * GET form: every control is a plain HTML input so the page works without JavaScript and the resulting
10 + * URL is the permalink. Checkboxes and the free slug input share the name `cancers`; the parser merges
11 + * both (comma or repeated) and keeps at most MAX_CANCERS.
12 + */
13 +export function ExplorerFilters({ options, state, choices, action = '/explore' }: { options: ExplorerOptions; state: ExplorerState; choices: CancerChoice[]; action?: string }) {
14 + const selected = new Set(state.cancers);
15 + const known = new Set(choices.map((c) => c.slug));
16 + const free = state.cancers.filter((c) => !known.has(c));
17 + const metric = options.metrics.find((m) => m.metric === state.metric);
18 + return (
19 + <form method="get" action={action} className="border-y border-rule py-3" aria-label="Data explorer filters">
20 + <div className="grid gap-3 text-[13.5px] sm:grid-cols-2 lg:grid-cols-[2fr_1.6fr_1fr_1fr_0.8fr_0.8fr]">
21 + <label className="flex min-w-0 flex-col gap-1">
22 + <span className="ci-kicker">Metric</span>
23 + <select name="metric" defaultValue={state.metric} className={INPUT}>
24 + {options.metrics.map((m) => (
25 + <option key={m.metric} value={m.metric}>
26 + {EPI_METRIC_LABEL[m.metric] ?? humanize(m.metric)} ({unitLabel(m.unit)}, {m.year_min}–{m.year_max})
27 + </option>
28 + ))}
29 + </select>
30 + </label>
31 + <label className="flex min-w-0 flex-col gap-1">
32 + <span className="ci-kicker">Geography</span>
33 + <select name="geography" defaultValue={state.geography} className={INPUT}>
34 + {options.geographies.map((g) => (
35 + <option key={g.id} value={g.slug}>
36 + {g.name} ({humanize(g.kind).toLowerCase()}, {fmtInt(g.n)} obs.)
37 + </option>
38 + ))}
39 + {!options.geographies.some((g) => g.slug === state.geography || g.iso3 === state.geography) ? <option value={state.geography}>{state.geography} (no data)</option> : null}
40 + </select>
41 + </label>
42 + <label className="flex flex-col gap-1">
43 + <span className="ci-kicker">Sex</span>
44 + <select name="sex" defaultValue={state.sex} className={INPUT}>
45 + {options.sexes.map((s) => (
46 + <option key={s} value={s}>
47 + {s === 'all' ? 'Both sexes (as published)' : humanize(s)}
48 + </option>
49 + ))}
50 + <option value={SEX_ANY}>Every sex (one series each)</option>
51 + </select>
52 + </label>
53 + <label className="flex flex-col gap-1">
54 + <span className="ci-kicker">Age group</span>
55 + <select name="age" defaultValue={state.age} className={INPUT}>
56 + {options.age_groups.map((a) => (
57 + <option key={a} value={a}>
58 + {a === 'all' ? 'All ages' : a}
59 + </option>
60 + ))}
61 + {!options.age_groups.includes(state.age) ? <option value={state.age}>{state.age} (no data)</option> : null}
62 + </select>
63 + </label>
64 + <label className="flex flex-col gap-1">
65 + <span className="ci-kicker">From</span>
66 + <input name="from" type="number" inputMode="numeric" min={options.year_min ?? 1900} max={options.year_max ?? 2100} defaultValue={state.from ?? ''} placeholder={String(metric?.year_min ?? options.year_min ?? '')} className={INPUT} />
67 + </label>
68 + <label className="flex flex-col gap-1">
69 + <span className="ci-kicker">To</span>
70 + <input name="to" type="number" inputMode="numeric" min={options.year_min ?? 1900} max={options.year_max ?? 2100} defaultValue={state.to ?? ''} placeholder={String(metric?.year_max ?? options.year_max ?? '')} className={INPUT} />
71 + </label>
72 + </div>
73 +
74 + <fieldset className="mt-3 min-w-0">
75 + <legend className="ci-kicker mb-1">
76 + Cancers (top-level site groups, up to {MAX_CANCERS}) — {selected.size} selected
77 + </legend>
78 + <div className="grid grid-cols-2 gap-x-3 gap-y-1 text-[13px] sm:grid-cols-3 lg:grid-cols-4">
79 + {choices.map((c) => (
80 + <label key={c.id} className={`flex items-center gap-1.5 ${c.n_obs === 0 ? 'text-ink-4' : 'text-ink-2'}`} title={c.n_obs === 0 ? 'No observation for the selected metric and geography' : `${fmtInt(c.n_obs)} observations for the selected metric and geography`}>
81 + <input type="checkbox" name="cancers" value={c.slug} defaultChecked={selected.has(c.slug)} className="accent-accent" />
82 + <span className="truncate">{c.canonical_name}</span>
83 + {c.n_obs === 0 ? <span className="shrink-0 text-[11px]">(no data)</span> : null}
84 + </label>
85 + ))}
86 + </div>
87 + <label className="mt-2 flex flex-col gap-1 text-[13px] sm:max-w-xl">
88 + <span className="text-ink-3">
89 + Other cancer slugs or CI-CAN ids, comma-separated (any entity of the taxonomy; shown only when it carries observations)
90 + </span>
91 + <input name="cancers" defaultValue={free.join(', ')} placeholder="e.g. malignant-thyroid-gland-neoplasm" className={`${INPUT} ci-mono`} />
92 + </label>
93 + </fieldset>
94 +
95 + <div className="mt-3 flex flex-wrap items-center gap-x-5 gap-y-2 text-[13px]">
96 + <fieldset className="flex items-center gap-3">
97 + <legend className="sr-only">Chart view</legend>
98 + <span className="ci-kicker">View</span>
99 + {VIEWS.map((v) => (
100 + <label key={v} className="inline-flex items-center gap-1.5 text-ink-2">
101 + <input type="radio" name="view" value={v} defaultChecked={state.view === v} className="accent-accent" />
102 + {v === 'lines' ? 'Overlaid lines' : 'Small multiples'}
103 + </label>
104 + ))}
105 + </fieldset>
106 + <button type="submit" className="border border-accent bg-accent px-3 py-1.5 text-[13px] font-medium text-white hover:bg-accent-2">
107 + Update
108 + </button>
109 + <a className="ci-link" href={action}>
110 + Reset to defaults
111 + </a>
112 + </div>
113 + </form>
114 + );
115 +}
added apps/web/src/components/explorer/observations-table.tsx +113 −0
@@ -0,0 +1,113 @@
1 +import Link from 'next/link';
2 +import { Badge, ClaimBadge } from '@/components/ui/badge';
3 +import { SourceBadge, provenanceTitle, type ProvenanceInfo } from '@/components/ui/source-badge';
4 +import { Pager } from '@/components/ui/pager';
5 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
6 +import type { ExplorerObsRow } from '@/lib/queries/explorer';
7 +import { TABLE_PAGE_SIZE } from '@/lib/explorer-params';
8 +import { fmtInt, fmtValue, humanize, unitLabel } from '@/lib/format';
9 +
10 +export function obsProvenance(r: ExplorerObsRow): ProvenanceInfo {
11 + return {
12 + sourceSlug: r.source_slug,
13 + sourceName: r.source_name,
14 + dataset: r.dataset,
15 + datasetVersion: r.dataset_version,
16 + retrievedAt: r.retrieved_at,
17 + sourceUrl: r.source_url,
18 + license: r.source_license,
19 + layer: 'normalized',
20 + evidenceType: 'observed_data',
21 + };
22 +}
23 +
24 +/**
25 + * Every observation behind the charts, 50 per page, with the unit, CI, estimate type, standard
26 + * population, site definition and a per-row provenance popover (datasets differ between rows).
27 + */
28 +export function ObservationsTable({ rows, page, hrefFor }: { rows: ExplorerObsRow[]; page: number; hrefFor: (page: number) => string }) {
29 + const total = rows.length;
30 + const start = (page - 1) * TABLE_PAGE_SIZE;
31 + const shown = rows.slice(start, start + TABLE_PAGE_SIZE);
32 + const units = [...new Set(rows.map((r) => r.unit))];
33 + const observedOnly = rows.every((r) => r.estimate_type === 'observed');
34 + // One provenance popover per distinct (source, dataset) in the caption; rows carry compact badges whose
35 + // title repeats dataset · version · retrieved (dense tables: the popover markup is not repeated 50 times).
36 + const captionProv = new Map<string, ProvenanceInfo>();
37 + for (const r of rows) {
38 + const k = `${r.source_slug}|${r.dataset ?? ''}`;
39 + if (!captionProv.has(k)) captionProv.set(k, obsProvenance(r));
40 + }
41 + return (
42 + <div>
43 + {/* div, not p: popovers contain a <dl>. */}
44 + <div className="mb-2 flex flex-wrap items-center gap-2 text-[12px] text-ink-3">
45 + {[...captionProv.values()].map((p) => (
46 + <SourceBadge key={`${p.sourceSlug}|${p.dataset ?? ''}`} p={p} />
47 + ))}
48 + <ClaimBadge kind="observed" />
49 + <span>
50 + {fmtInt(total)} observation{total === 1 ? '' : 's'} · values exactly as published by the source, units harmonized ({units.map(unitLabel).join(', ')}) · hover a row's source badge for its dataset, version and retrieval date
51 + {observedOnly ? '' : ' · rows labelled estimated/projected are model outputs of the source'}
52 + </span>
53 + </div>
54 + <div className="ci-table-wrap">
55 + <table className="ci-table">
56 + <thead>
57 + <tr>
58 + <th scope="col">Year</th>
59 + <th scope="col">Cancer</th>
60 + <th scope="col">Geography</th>
61 + <th scope="col">Sex</th>
62 + <th scope="col">Age</th>
63 + <th scope="col">Metric</th>
64 + <th scope="col" className="num">
65 + Value
66 + </th>
67 + <th scope="col">Unit</th>
68 + <th scope="col" className="num">
69 + 95% CI
70 + </th>
71 + <th scope="col">Type</th>
72 + <th scope="col">Standard population</th>
73 + <th scope="col">Site definition</th>
74 + <th scope="col">Source</th>
75 + </tr>
76 + </thead>
77 + <tbody>
78 + {shown.map((r) => (
79 + <tr key={r.id}>
80 + <td className="ci-num">{r.year_end && r.year_end !== r.year ? `${r.year}–${r.year_end}` : r.year}</td>
81 + <td>
82 + <Link className="ci-link" href={`/cancer/${r.cancer_slug}/statistics`}>
83 + {r.cancer_name}
84 + </Link>
85 + </td>
86 + <td>
87 + <Link className="ci-link" href={`/country/${r.geography_slug}`}>
88 + {r.geography_name}
89 + </Link>
90 + </td>
91 + <td>{r.sex === 'all' ? 'Both' : humanize(r.sex)}</td>
92 + <td>{r.age_group === 'all' ? 'All ages' : r.age_group}</td>
93 + <td className="whitespace-nowrap text-[12.5px]">{EPI_METRIC_LABEL[r.metric] ?? humanize(r.metric)}</td>
94 + <td className="num font-medium">{fmtValue(r.value, r.unit)}</td>
95 + <td className="text-[12px] text-ink-3">{unitLabel(r.unit)}</td>
96 + <td className="num text-ink-3">{r.lower_ci != null && r.upper_ci != null ? `${fmtValue(r.lower_ci, r.unit)}–${fmtValue(r.upper_ci, r.unit)}` : '—'}</td>
97 + <td>
98 + <Badge tone={r.estimate_type === 'observed' ? 'ok' : 'warn'}>{r.estimate_type}</Badge>
99 + </td>
100 + <td className="max-w-[200px] text-[12px] text-ink-3">{r.standard_population ?? '—'}</td>
101 + <td className="max-w-[260px] text-[12px] text-ink-3">{r.site_definition ?? '—'}</td>
102 + <td>
103 + <SourceBadge compact p={obsProvenance(r)} title={provenanceTitle(obsProvenance(r))} />
104 + </td>
105 + </tr>
106 + ))}
107 + </tbody>
108 + </table>
109 + </div>
110 + <Pager total={total} pageSize={TABLE_PAGE_SIZE} page={page} hrefFor={hrefFor} noun="observations" label="Observations pagination" />
111 + </div>
112 + );
113 +}
added apps/web/src/components/home/explorer-module.tsx +78 −0
@@ -0,0 +1,78 @@
1 +import Link from 'next/link';
2 +import { Section } from '@/components/ui/section';
3 +import { EmptyState } from '@/components/ui/empty-state';
4 +import { Freshness } from '@/components/ui/freshness';
5 +import { ClaimBadge } from '@/components/ui/badge';
6 +import { SourceBadge } from '@/components/ui/source-badge';
7 +import { ExplorerChart } from '@/components/charts/explorer-chart';
8 +import { resolveGeographyRef, topCancersByLatest, yearRangeFor, explorerObservations, explorerOptions } from '@/lib/queries/explorer';
9 +import { EPI_METRIC_LABEL } from '@/lib/queries/epidemiology';
10 +import { groupComparable } from '@/lib/explorer-series';
11 +import { serializeExplorerParams, type ExplorerState } from '@/lib/explorer-params';
12 +import { fmtDate, fmtInt, humanize, toDate, unitLabel } from '@/lib/format';
13 +
14 +/**
15 + * Home module: the explorer's default chart — the five top-level cancers with the highest annual deaths
16 + * in the latest year (computed, not curated), one comparable group only (a single source and standard),
17 + * with the source line and a link that opens the same selection in /explore.
18 + */
19 +export async function ExplorerModule({ metric = 'mortality_count', geographySlug = 'united-states', n = 5 }: { metric?: string; geographySlug?: string; n?: number }) {
20 + const [geo, options] = await Promise.all([resolveGeographyRef(geographySlug), explorerOptions()]);
21 + const hasMetric = options.metrics.some((m) => m.metric === metric);
22 + const top = geo && hasMetric ? await topCancersByLatest(metric, geo.id, 'all', 'all', n) : null;
23 + const range = geo && hasMetric ? await yearRangeFor(metric, geo.id) : null;
24 + const obs = geo && top && top.cancers.length > 0 ? await explorerObservations({ metric, cancerIds: top.cancers.map((c) => c.id), geographyId: geo.id, sex: 'all', age: 'all', from: range?.min ?? null, to: range?.max ?? null }) : [];
25 + const groups = groupComparable(obs);
26 + const g = groups[0];
27 + const metricLabel = EPI_METRIC_LABEL[metric] ?? humanize(metric);
28 + const state: ExplorerState | null =
29 + geo && top
30 + ? { metric, cancers: top.cancers.map((c) => c.slug), geography: geo.slug, sex: 'all', age: 'all', from: range?.min ?? null, to: range?.max ?? null, view: 'lines', normalize: 'none', page: 1 }
31 + : null;
32 + const href = state ? `/explore${serializeExplorerParams(state)}` : '/explore';
33 + const prov = g ? obs.filter((o) => o.source_slug === g.source_slug).sort((a, b) => String(b.retrieved_at ?? '') .localeCompare(String(a.retrieved_at ?? '')))[0] : undefined;
34 + const freshest = obs.map((o) => toDate(o.updated_at)).filter((d): d is Date => !!d).sort((a, b) => b.getTime() - a.getTime())[0] ?? null;
35 +
36 + return (
37 + <Section
38 + id="explorer"
39 + kicker="Data explorer"
40 + title={g ? `${metricLabel} · ${g.geography_name} · ${g.year_min}–${g.year_max}` : 'Data explorer'}
41 + description={g && top ? `The ${top.cancers.length} top-level cancers with the highest ${metricLabel.toLowerCase()} in ${top.year}, both sexes, all ages, ${unitLabel(g.unit)} per site group as published by ${g.source_name ?? g.source_slug}. Selection computed from the observations; every metric, cancer, geography, sex, age group and year is one click away.` : 'Chart and download registry observations by metric, cancer, geography, sex, age group and year.'}
42 + actions={
43 + <Link href={href} className="ci-link">
44 + Open in the Data explorer →
45 + </Link>
46 + }
47 + >
48 + {!g ? (
49 + <EmptyState title="Data not yet available">
50 + The explorer chart appears once a licensed registry connector has ingested {metricLabel.toLowerCase()} observations{geo ? ` for ${geo.name}` : ''}.{' '}
51 + <Link className="ci-link" href="/explore/coverage">
52 + Coverage matrix
53 + </Link>
54 + </EmptyState>
55 + ) : (
56 + <>
57 + <ExplorerChart series={g.series.map((s, i) => ({ name: s.name, points: s.points, dashed: s.dashed, slot: i }))} unit={g.unit} ariaLabel={`${metricLabel} in ${g.geography_name}, ${g.year_min}–${g.year_max}, ${g.series.length} cancers, source ${g.source_slug}`} />
58 + {/* div, not p: the SourceBadge popover contains a <dl>. */}
59 + <div className="mt-1.5 flex flex-wrap items-center gap-x-2 gap-y-1 text-[12px] text-ink-3">
60 + <SourceBadge p={{ sourceSlug: g.source_slug, sourceName: g.source_name, dataset: prov?.dataset ?? null, datasetVersion: prov?.dataset_version ?? null, retrievedAt: prov?.retrieved_at ?? null, sourceUrl: prov?.source_url ?? null, license: prov?.source_license ?? null, layer: 'normalized', evidenceType: 'observed_data' }} />
61 + <ClaimBadge kind="observed" />
62 + <span>
63 + {g.source_name ?? g.source_slug}
64 + {prov?.retrieved_at ? ` (retrieved ${fmtDate(prov.retrieved_at)})` : ''} · {fmtInt(g.n_obs)} observations · {unitLabel(g.unit)} · both sexes · all ages
65 + {g.standard_population ? ` · standard: ${g.standard_population}` : ''}
66 + </span>
67 + {groups.length > 1 ? (
68 + <span>
69 + · {groups.length - 1} other source{groups.length > 2 ? 's' : ''} publish{groups.length > 2 ? '' : 'es'} this metric — shown separately in the explorer, never overlaid
70 + </span>
71 + ) : null}
72 + </div>
73 + <Freshness dataUpdatedAt={freshest} sourceVersion={prov?.dataset_version ?? null} />
74 + </>
75 + )}
76 + </Section>
77 + );
78 +}
added apps/web/src/lib/explorer-csv.ts +60 −0
@@ -0,0 +1,60 @@
1 +/**
2 + * CSV helpers for the Data explorer export (pure, unit-tested). RFC 4180: fields containing a comma,
3 + * a double quote, CR or LF are quoted and inner quotes doubled. Attribution rows are prefixed with `#`
4 + * so spreadsheet users see them and parsers with a comment option can skip them.
5 + */
6 +
7 +export function csvCell(v: unknown): string {
8 + if (v == null) return '';
9 + const s = v instanceof Date ? v.toISOString() : String(v);
10 + return /[",\r\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
11 +}
12 +
13 +export function csvLine(values: readonly unknown[]): string {
14 + return values.map(csvCell).join(',');
15 +}
16 +
17 +/** Comment row: one line, newlines collapsed so the header stays line-oriented. */
18 +export function csvComment(text: string): string {
19 + return `# ${text.replace(/[\r\n]+/g, ' ').trim()}`;
20 +}
21 +
22 +export const EPI_CSV_COLUMNS = [
23 + 'cancer_id',
24 + 'cancer_slug',
25 + 'cancer_name',
26 + 'geography_id',
27 + 'geography_slug',
28 + 'geography_name',
29 + 'iso3',
30 + 'year',
31 + 'year_end',
32 + 'sex',
33 + 'age_group',
34 + 'metric',
35 + 'value',
36 + 'unit',
37 + 'lower_ci',
38 + 'upper_ci',
39 + 'standard_population',
40 + 'estimate_type',
41 + 'site_definition',
42 + 'source_slug',
43 + 'source_name',
44 + 'provenance_id',
45 + 'dataset',
46 + 'dataset_version',
47 + 'source_url',
48 + 'retrieved_at',
49 +] as const;
50 +
51 +export type EpiCsvColumn = (typeof EPI_CSV_COLUMNS)[number];
52 +
53 +/** File name: cancerindex-epidemiology-<metric>-<geography>-<from>-<to>.csv, safe characters only. */
54 +export function csvFileName(parts: Array<string | number | null | undefined>): string {
55 + const clean = parts
56 + .filter((p) => p != null && p !== '')
57 + .map((p) => String(p).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''))
58 + .filter(Boolean);
59 + return `cancerindex-epidemiology-${clean.join('-') || 'export'}.csv`;
60 +}
added apps/web/src/lib/explorer-params.ts +168 −0
@@ -0,0 +1,168 @@
1 +/**
2 + * Data explorer URL state (§ "Our World in Data"-style explorer). Pure: shared by the /explore page,
3 + * the CSV route and unit tests. Defaults that depend on the database (top cancers, year range,
4 + * available metric/geography) are injected by the caller — nothing is hardcoded here.
5 + */
6 +import type { SP } from '@/lib/search-params';
7 +
8 +export const MAX_CANCERS = 6;
9 +export const TABLE_PAGE_SIZE = 50;
10 +export const VIEWS = ['lines', 'multiples'] as const;
11 +export type ExplorerView = (typeof VIEWS)[number];
12 +export const NORMALIZE = ['none'] as const;
13 +export type ExplorerNormalize = (typeof NORMALIZE)[number];
14 +/** "any" = no sex filter: series become cancer × sex inside a comparable group. */
15 +export const SEX_ANY = 'any';
16 +
17 +export interface ExplorerState {
18 + metric: string;
19 + cancers: string[]; // cancer slugs (or CI-CAN ids), ≤ MAX_CANCERS, deduplicated, order kept
20 + geography: string; // geography slug or ISO3
21 + sex: string; // all | male | female | any
22 + age: string; // age group label, 'all' by default
23 + from: number | null;
24 + to: number | null;
25 + view: ExplorerView;
26 + normalize: ExplorerNormalize;
27 + page: number; // observations table page (1-based)
28 +}
29 +
30 +export interface ExplorerDefaults {
31 + metric: string;
32 + geography: string;
33 + cancers: string[];
34 + sex?: string;
35 + age?: string;
36 + from?: number | null;
37 + to?: number | null;
38 +}
39 +
40 +const SLUG_RE = /^[a-z0-9][a-z0-9-]{0,99}$/;
41 +const CI_ID_RE = /^CI-CAN-\d{8}$/i;
42 +const METRIC_RE = /^[a-z][a-z0-9_]{1,63}$/;
43 +const GEO_RE = /^[a-z0-9][a-z0-9-]{0,99}$/i;
44 +const AGE_RE = /^[A-Za-z0-9+_\- ]{1,24}$/; // e.g. "all", "0-14", "65+", "85 and over"
45 +const SEX_RE = /^[a-z_]{1,16}$/;
46 +const YEAR_MIN = 1900;
47 +const YEAR_MAX = 2100;
48 +
49 +function first(v: string | string[] | undefined): string {
50 + const s = Array.isArray(v) ? v[0] : v;
51 + return (s ?? '').toString().trim();
52 +}
53 +
54 +/**
55 + * Cancer references from `cancers=a,b,c`, repeated `cancers=a&cancers=b`, or a mix (checkboxes + a free
56 + * comma-separated input share the same name). Lower-cases slugs, upper-cases CI ids, drops invalid tokens
57 + * and duplicates, keeps at most `max` in the order given.
58 + */
59 +export function parseCancerList(v: string | string[] | undefined, max = MAX_CANCERS): string[] {
60 + const raw = (Array.isArray(v) ? v : v == null ? [] : [v]).flatMap((s) => String(s).split(/[,\s]+/));
61 + const out: string[] = [];
62 + for (const tok of raw) {
63 + const t = tok.trim();
64 + if (!t) continue;
65 + const norm = CI_ID_RE.test(t) ? t.toUpperCase() : t.toLowerCase();
66 + if (!CI_ID_RE.test(norm) && !SLUG_RE.test(norm)) continue;
67 + if (out.includes(norm)) continue;
68 + out.push(norm);
69 + if (out.length >= max) break;
70 + }
71 + return out;
72 +}
73 +
74 +function parseYear(v: string, fallback: number | null): number | null {
75 + if (v === '') return fallback;
76 + const n = Number.parseInt(v, 10);
77 + if (!Number.isFinite(n) || n < YEAR_MIN || n > YEAR_MAX) return fallback;
78 + return n;
79 +}
80 +
81 +/** Parse the search params into a complete state, filling gaps from the (database-computed) defaults. */
82 +export function parseExplorerParams(sp: SP, d: ExplorerDefaults): ExplorerState {
83 + const metricRaw = first(sp.metric).toLowerCase();
84 + const geoRaw = first(sp.geography);
85 + const sexRaw = first(sp.sex).toLowerCase();
86 + const ageRaw = first(sp.age);
87 + const viewRaw = first(sp.view).toLowerCase();
88 + const normRaw = first(sp.normalize).toLowerCase();
89 + const cancers = parseCancerList(sp.cancers);
90 + let from = parseYear(first(sp.from), d.from ?? null);
91 + let to = parseYear(first(sp.to), d.to ?? null);
92 + if (from != null && to != null && from > to) [from, to] = [to, from];
93 + const pageN = Number.parseInt(first(sp.page), 10);
94 + return {
95 + metric: METRIC_RE.test(metricRaw) ? metricRaw : d.metric,
96 + cancers: cancers.length > 0 ? cancers : d.cancers.slice(0, MAX_CANCERS),
97 + geography: GEO_RE.test(geoRaw) ? (geoRaw.length === 3 ? geoRaw.toUpperCase() : geoRaw.toLowerCase()) : d.geography,
98 + sex: SEX_RE.test(sexRaw) ? sexRaw : (d.sex ?? 'all'),
99 + age: AGE_RE.test(ageRaw) ? ageRaw : (d.age ?? 'all'),
100 + from,
101 + to,
102 + view: (VIEWS as readonly string[]).includes(viewRaw) ? (viewRaw as ExplorerView) : 'lines',
103 + normalize: (NORMALIZE as readonly string[]).includes(normRaw) ? (normRaw as ExplorerNormalize) : 'none',
104 + page: Number.isFinite(pageN) && pageN > 1 ? Math.min(pageN, 100_000) : 1,
105 + };
106 +}
107 +
108 +/**
109 + * Serialize a state to a query string. Every dimension is written explicitly so a permalink is
110 + * self-describing and stable when the computed defaults change; `view`, `normalize` and `page` are
111 + * omitted at their default values. `cancers` is comma-separated.
112 + */
113 +export function serializeExplorerParams(s: ExplorerState, overrides: Partial<ExplorerState> = {}): string {
114 + const m = { ...s, ...overrides };
115 + const qs = new URLSearchParams();
116 + qs.set('metric', m.metric);
117 + if (m.cancers.length > 0) qs.set('cancers', m.cancers.join(','));
118 + qs.set('geography', m.geography);
119 + qs.set('sex', m.sex);
120 + qs.set('age', m.age);
121 + if (m.from != null) qs.set('from', String(m.from));
122 + if (m.to != null) qs.set('to', String(m.to));
123 + if (m.view !== 'lines') qs.set('view', m.view);
124 + if (m.normalize !== 'none') qs.set('normalize', m.normalize);
125 + if (m.page > 1) qs.set('page', String(m.page));
126 + return `?${qs.toString()}`;
127 +}
128 +
129 +/** Same filters expressed for the public API (`/api/v1/epidemiology`) — repeated `cancer=` params, `from`/`to`. */
130 +export function apiQueryFor(s: ExplorerState, limit = 200): string {
131 + const qs = new URLSearchParams();
132 + qs.set('metric', s.metric);
133 + for (const c of s.cancers) qs.append('cancer', c);
134 + qs.set('geography', s.geography);
135 + if (s.sex !== SEX_ANY) qs.set('sex', s.sex);
136 + qs.set('age', s.age);
137 + if (s.from != null) qs.set('from', String(s.from));
138 + if (s.to != null) qs.set('to', String(s.to));
139 + qs.set('limit', String(limit));
140 + return `?${qs.toString()}`;
141 +}
142 +
143 +/** Year label for headers and captions: "1999–2024", "2024" or "all years". */
144 +export function yearRangeLabel(from: number | null, to: number | null): string {
145 + if (from != null && to != null) return from === to ? String(from) : `${from}–${to}`;
146 + if (from != null) return `from ${from}`;
147 + if (to != null) return `to ${to}`;
148 + return 'all years';
149 +}
150 +
151 +export function sexLabel(sex: string): string {
152 + switch (sex) {
153 + case 'all':
154 + return 'both sexes';
155 + case 'male':
156 + return 'male';
157 + case 'female':
158 + return 'female';
159 + case SEX_ANY:
160 + return 'by sex';
161 + default:
162 + return sex.replace(/_/g, ' ');
163 + }
164 +}
165 +
166 +export function ageLabel(age: string): string {
167 + return age === 'all' ? 'all ages' : `ages ${age}`;
168 +}
added apps/web/src/lib/explorer-series.ts +241 −0
@@ -0,0 +1,241 @@
1 +/**
2 + * Comparability rules of the Data explorer (docs/methodology/data-explorer.md). Pure, unit-tested.
3 + *
4 + * Observations are only overlaid on one chart when they share metric, unit, geography, source,
5 + * standard population and age group. Anything else (a rate standardized to the US 2000 standard next to
6 + * one standardized to the World standard, CDC WONDER next to USCS, ages 65+ next to all ages) becomes a
7 + * separate chart with a caption naming the dimension that differs — never a silent overlay.
8 + */
9 +
10 +export interface ComparableObs {
11 + cancer_slug: string;
12 + cancer_name: string;
13 + geography_slug: string;
14 + geography_name: string;
15 + year: number;
16 + year_end?: number | null;
17 + sex: string;
18 + age_group: string;
19 + metric: string;
20 + value: number;
21 + unit: string;
22 + lower_ci?: number | null;
23 + upper_ci?: number | null;
24 + standard_population: string | null;
25 + estimate_type: string;
26 + site_definition?: string | null;
27 + source_slug: string;
28 + source_name?: string | null;
29 + provenance_id?: number | null;
30 +}
31 +
32 +export interface ExplorerSeries {
33 + key: string;
34 + name: string;
35 + cancer_slug: string;
36 + cancer_name: string;
37 + sex: string;
38 + estimate_type: string;
39 + site_definition: string | null;
40 + dashed: boolean;
41 + points: Array<{ x: number; y: number; lo?: number | null; hi?: number | null }>;
42 +}
43 +
44 +export interface ComparableGroup {
45 + key: string;
46 + metric: string;
47 + unit: string;
48 + geography_slug: string;
49 + geography_name: string;
50 + source_slug: string;
51 + source_name: string | null;
52 + standard_population: string | null;
53 + age_group: string;
54 + series: ExplorerSeries[];
55 + n_obs: number;
56 + year_min: number;
57 + year_max: number;
58 + provenance_ids: number[];
59 + y_max: number; // max of value / upper CI across the group (for a shared axis)
60 +}
61 +
62 +/** Dimensions that decide whether two observations may share one chart (order = caption order). */
63 +export const COMPARABILITY_DIMENSIONS = ['metric', 'unit', 'geography_slug', 'source_slug', 'standard_population', 'age_group'] as const;
64 +export type ComparabilityDimension = (typeof COMPARABILITY_DIMENSIONS)[number];
65 +
66 +export function groupKey(o: Pick<ComparableObs, ComparabilityDimension>): string {
67 + return COMPARABILITY_DIMENSIONS.map((d) => o[d] ?? '').join('|');
68 +}
69 +
70 +function seriesKey(o: ComparableObs): string {
71 + return `${o.cancer_slug}|${o.sex}|${o.estimate_type}|${o.site_definition ?? ''}`;
72 +}
73 +
74 +function sexSuffix(sex: string): string {
75 + return sex === 'all' ? 'both sexes' : sex.replace(/_/g, ' ');
76 +}
77 +
78 +/**
79 + * Group observations into comparable chart groups; inside a group one series per (cancer × sex ×
80 + * estimate type × site definition). Series named "<cancer> · <sex>" (sex omitted when every series in
81 + * the group shares it); dashed when the source labels the value estimated/projected.
82 + * Groups are ordered by observation count (largest first), series by cancer name then sex.
83 + */
84 +export function groupComparable(obs: readonly ComparableObs[]): ComparableGroup[] {
85 + const groups = new Map<string, { meta: ComparableObs; series: Map<string, ExplorerSeries & { _rows: ComparableObs[] }>; prov: Set<number>; n: number; ymin: number; ymax: number; vmax: number }>();
86 + for (const o of obs) {
87 + if (!Number.isFinite(o.value) || !Number.isFinite(o.year)) continue;
88 + const gk = groupKey(o);
89 + let g = groups.get(gk);
90 + if (!g) {
91 + g = { meta: o, series: new Map(), prov: new Set(), n: 0, ymin: o.year, ymax: o.year, vmax: 0 };
92 + groups.set(gk, g);
93 + }
94 + const sk = seriesKey(o);
95 + let s = g.series.get(sk);
96 + if (!s) {
97 + s = { key: sk, name: '', cancer_slug: o.cancer_slug, cancer_name: o.cancer_name, sex: o.sex, estimate_type: o.estimate_type, site_definition: o.site_definition ?? null, dashed: o.estimate_type !== 'observed', points: [], _rows: [] };
98 + g.series.set(sk, s);
99 + }
100 + s.points.push({ x: o.year, y: o.value, lo: o.lower_ci ?? null, hi: o.upper_ci ?? null });
101 + s._rows.push(o);
102 + g.n += 1;
103 + g.ymin = Math.min(g.ymin, o.year);
104 + g.ymax = Math.max(g.ymax, o.year_end ?? o.year);
105 + g.vmax = Math.max(g.vmax, o.value, o.upper_ci ?? 0);
106 + if (o.provenance_id != null) g.prov.add(o.provenance_id);
107 + }
108 +
109 + const out: ComparableGroup[] = [];
110 + for (const [key, g] of groups) {
111 + const list = [...g.series.values()];
112 + const sexes = new Set(list.map((s) => s.sex));
113 + const siteDefsByCancer = new Map<string, Set<string>>();
114 + for (const s of list) {
115 + if (!siteDefsByCancer.has(s.cancer_slug)) siteDefsByCancer.set(s.cancer_slug, new Set());
116 + siteDefsByCancer.get(s.cancer_slug)!.add(s.site_definition ?? '');
117 + }
118 + for (const s of list) {
119 + const bits = [s.cancer_name];
120 + if (sexes.size > 1) bits.push(sexSuffix(s.sex));
121 + if ((siteDefsByCancer.get(s.cancer_slug)?.size ?? 0) > 1 && s.site_definition) bits.push(s.site_definition);
122 + if (s.estimate_type !== 'observed') bits.push(s.estimate_type);
123 + s.name = bits.join(' · ');
124 + s.points.sort((a, b) => a.x - b.x);
125 + }
126 + list.sort((a, b) => a.cancer_name.localeCompare(b.cancer_name) || sexOrder(a.sex) - sexOrder(b.sex) || Number(a.estimate_type !== 'observed') - Number(b.estimate_type !== 'observed') || a.estimate_type.localeCompare(b.estimate_type));
127 + out.push({
128 + key,
129 + metric: g.meta.metric,
130 + unit: g.meta.unit,
131 + geography_slug: g.meta.geography_slug,
132 + geography_name: g.meta.geography_name,
133 + source_slug: g.meta.source_slug,
134 + source_name: g.meta.source_name ?? null,
135 + standard_population: g.meta.standard_population,
136 + age_group: g.meta.age_group,
137 + series: list.map(({ _rows, ...s }) => s),
138 + n_obs: g.n,
139 + year_min: g.ymin,
140 + year_max: g.ymax,
141 + provenance_ids: [...g.prov].sort((a, b) => a - b),
142 + y_max: g.vmax,
143 + });
144 + }
145 + out.sort((a, b) => b.n_obs - a.n_obs || a.source_slug.localeCompare(b.source_slug) || (a.standard_population ?? '').localeCompare(b.standard_population ?? ''));
146 + return out;
147 +}
148 +
149 +function sexOrder(sex: string): number {
150 + return sex === 'all' ? 0 : sex === 'male' ? 1 : sex === 'female' ? 2 : 3;
151 +}
152 +
153 +/** Human labels for the comparability dimensions in captions. */
154 +export function dimensionLabel(d: ComparabilityDimension): string {
155 + switch (d) {
156 + case 'metric':
157 + return 'metric';
158 + case 'unit':
159 + return 'unit';
160 + case 'geography_slug':
161 + return 'geography';
162 + case 'source_slug':
163 + return 'source';
164 + case 'standard_population':
165 + return 'standard population';
166 + case 'age_group':
167 + return 'age group';
168 + }
169 +}
170 +
171 +function dimValue(g: ComparableGroup, d: ComparabilityDimension): string {
172 + switch (d) {
173 + case 'geography_slug':
174 + return g.geography_name;
175 + case 'source_slug':
176 + return g.source_slug;
177 + case 'standard_population':
178 + return g.standard_population ?? 'no standard population (crude or count)';
179 + case 'age_group':
180 + return g.age_group === 'all' ? 'all ages' : g.age_group;
181 + default:
182 + return String(g[d]);
183 + }
184 +}
185 +
186 +/** Dimensions on which at least two groups differ — the reason they are drawn as separate charts. */
187 +export function differingDimensions(groups: readonly ComparableGroup[]): ComparabilityDimension[] {
188 + if (groups.length < 2) return [];
189 + return COMPARABILITY_DIMENSIONS.filter((d) => new Set(groups.map((g) => dimValue(g, d))).size > 1);
190 +}
191 +
192 +/**
193 + * Caption explaining why a set of groups is not overlaid, e.g.
194 + * "Shown as 2 separate charts: the observations differ by standard population (2000 U.S. standard population
195 + * (19 age groups) vs 2000 U.S. Std. Population) and source (cdc-uscs vs cdc-wonder). Values standardized to
196 + * different populations or published by different sources are never overlaid."
197 + */
198 +export function explainSplit(groups: readonly ComparableGroup[]): string | null {
199 + const dims = differingDimensions(groups);
200 + if (dims.length === 0) return null;
201 + const parts = dims.map((d) => `${dimensionLabel(d)} (${[...new Set(groups.map((g) => dimValue(g, d)))].join(' vs ')})`);
202 + return `Shown as ${groups.length} separate charts: the observations differ by ${parts.join(' and ')}. Values that differ on any of these dimensions are never overlaid on one axis.`;
203 +}
204 +
205 +/** The values of the dimensions that identify one group, for its own caption ("what exactly is on this chart"). */
206 +export function groupDescriptor(g: ComparableGroup, dims: readonly ComparabilityDimension[] = COMPARABILITY_DIMENSIONS): Array<{ label: string; value: string }> {
207 + return dims.map((d) => ({ label: dimensionLabel(d), value: dimValue(g, d) }));
208 +}
209 +
210 +export const MAX_OVERLAID_SERIES = 4;
211 +
212 +/**
213 + * Split a group into small multiples, one per cancer, keeping the group's y_max so panels share one
214 + * axis. Used when the user asks for `view=multiples` or when a group would overlay more than
215 + * MAX_OVERLAID_SERIES series.
216 + */
217 +export function splitIntoMultiples(g: ComparableGroup): ComparableGroup[] {
218 + const byCancer = new Map<string, ExplorerSeries[]>();
219 + for (const s of g.series) {
220 + if (!byCancer.has(s.cancer_slug)) byCancer.set(s.cancer_slug, []);
221 + byCancer.get(s.cancer_slug)!.push(s);
222 + }
223 + return [...byCancer.entries()].map(([slug, series]) => ({
224 + ...g,
225 + key: `${g.key}|${slug}`,
226 + series,
227 + n_obs: series.reduce((n, s) => n + s.points.length, 0),
228 + // y_max intentionally inherited: shared axis across panels
229 + }));
230 +}
231 +
232 +export function shouldUseMultiples(g: ComparableGroup, view: 'lines' | 'multiples'): boolean {
233 + return view === 'multiples' ? new Set(g.series.map((s) => s.cancer_slug)).size > 1 : g.series.length > MAX_OVERLAID_SERIES;
234 +}
235 +
236 +/** Latest observed point of each series (for "latest value" captions). */
237 +export function latestPoint(s: ExplorerSeries): { x: number; y: number } | null {
238 + if (s.points.length === 0) return null;
239 + const p = s.points[s.points.length - 1]!;
240 + return { x: p.x, y: p.y };
241 +}
added apps/web/src/lib/queries/explorer.ts +321 −0
@@ -0,0 +1,321 @@
1 +import 'server-only';
2 +import { run, sql, safe } from '@/lib/db';
3 +import type { ComparableObs } from '@/lib/explorer-series';
4 +import { SEX_ANY } from '@/lib/explorer-params';
5 +
6 +/**
7 + * Read helpers of the Data explorer (/explore, CSV export, home module). Everything is read from
8 + * `epidemiology_observations` joined to its cancer, geography, source and provenance rows; nothing is
9 + * estimated, summed across sites or extrapolated. Options (metrics, geographies, sexes, age groups,
10 + * years, sources) are the distinct values actually present, so the UI never advertises data it lacks.
11 + */
12 +
13 +export interface MetricOption {
14 + metric: string;
15 + unit: string;
16 + n: number;
17 + n_cancers: number;
18 + year_min: number;
19 + year_max: number;
20 +}
21 +
22 +export interface GeographyOption {
23 + id: string;
24 + slug: string;
25 + name: string;
26 + iso3: string | null;
27 + kind: string;
28 + n: number;
29 + year_min: number;
30 + year_max: number;
31 +}
32 +
33 +export interface SourceOption {
34 + id: string;
35 + slug: string;
36 + name: string;
37 + license: string | null;
38 + license_status: string | null;
39 + homepage: string | null;
40 + attribution: string | null;
41 + n: number;
42 + retrieved_at: Date | string | null; // latest provenance retrieval behind its observations
43 +}
44 +
45 +export interface ExplorerOptions {
46 + metrics: MetricOption[];
47 + geographies: GeographyOption[];
48 + sexes: string[];
49 + age_groups: string[];
50 + year_min: number | null;
51 + year_max: number | null;
52 + sources: SourceOption[];
53 + n_obs: number;
54 +}
55 +
56 +export async function explorerOptions(): Promise<ExplorerOptions> {
57 + const [metrics, geographies, sexes, ages, years, sources] = await Promise.all([
58 + safe(
59 + () =>
60 + run<MetricOption>(sql`
61 + SELECT metric, min(unit) AS unit, count(*)::int AS n, count(DISTINCT cancer_id)::int AS n_cancers, min(year)::int AS year_min, max(coalesce(year_end, year))::int AS year_max
62 + FROM epidemiology_observations GROUP BY metric ORDER BY metric`),
63 + [] as MetricOption[],
64 + ),
65 + safe(
66 + () =>
67 + run<GeographyOption>(sql`
68 + SELECT g.id, g.slug, g.name, g.iso3, g.kind, a.n::int AS n, a.year_min::int AS year_min, a.year_max::int AS year_max
69 + FROM geographies g JOIN (SELECT geography_id, count(*) AS n, min(year) AS year_min, max(coalesce(year_end, year)) AS year_max FROM epidemiology_observations GROUP BY geography_id) a ON a.geography_id = g.id
70 + ORDER BY (g.kind = 'world') DESC, (g.kind = 'country') DESC, g.name`),
71 + [] as GeographyOption[],
72 + ),
73 + safe(() => run<{ sex: string }>(sql`SELECT sex FROM epidemiology_observations GROUP BY sex ORDER BY (sex = 'all') DESC, sex`), [] as Array<{ sex: string }>),
74 + safe(() => run<{ age_group: string }>(sql`SELECT age_group FROM epidemiology_observations GROUP BY age_group ORDER BY (age_group = 'all') DESC, age_group`), [] as Array<{ age_group: string }>),
75 + safe(() => run<{ y0: number | null; y1: number | null }>(sql`SELECT min(year)::int AS y0, max(coalesce(year_end, year))::int AS y1 FROM epidemiology_observations`), [] as Array<{ y0: number | null; y1: number | null }>),
76 + safe(
77 + () =>
78 + run<SourceOption>(sql`
79 + SELECT s.id, s.slug, s.name, s.license, s.license_status, s.homepage, s.attribution, a.n::int AS n,
80 + (SELECT max(p.retrieved_at) FROM provenance p WHERE p.id IN (SELECT DISTINCT o2.provenance_id FROM epidemiology_observations o2 WHERE o2.source_id = s.id)) AS retrieved_at
81 + FROM sources s JOIN (SELECT source_id, count(*) AS n FROM epidemiology_observations GROUP BY source_id) a ON a.source_id = s.id
82 + ORDER BY s.slug`),
83 + [] as SourceOption[],
84 + ),
85 + ]);
86 + return {
87 + metrics,
88 + geographies,
89 + sexes: sexes.map((r) => r.sex),
90 + age_groups: ages.map((r) => r.age_group),
91 + year_min: years[0]?.y0 ?? null,
92 + year_max: years[0]?.y1 ?? null,
93 + sources,
94 + n_obs: metrics.reduce((n, m) => n + m.n, 0),
95 + };
96 +}
97 +
98 +export interface GeographyRef {
99 + id: string;
100 + slug: string;
101 + name: string;
102 + iso3: string | null;
103 + kind: string;
104 +}
105 +
106 +/** Geography by slug or ISO3 (case-insensitive). */
107 +export async function resolveGeographyRef(ref: string): Promise<GeographyRef | null> {
108 + const r = ref.trim();
109 + if (!r) return null;
110 + const rows = await safe(
111 + () => run<GeographyRef>(sql`SELECT id, slug, name, iso3, kind FROM geographies WHERE slug = ${r.toLowerCase()} OR upper(iso3) = ${r.toUpperCase()} OR id = ${r} ORDER BY (slug = ${r.toLowerCase()}) DESC LIMIT 1`),
112 + [] as GeographyRef[],
113 + );
114 + return rows[0] ?? null;
115 +}
116 +
117 +export interface CancerRef {
118 + id: string;
119 + slug: string;
120 + canonical_name: string;
121 + top_level: boolean;
122 + status: string;
123 +}
124 +
125 +/** Cancers by slug or CI-CAN id, in the order requested; unknown references are dropped (reported by the caller). */
126 +export async function resolveCancerRefs(refs: readonly string[]): Promise<CancerRef[]> {
127 + if (refs.length === 0) return [];
128 + const rows = await safe(
129 + () =>
130 + run<CancerRef>(sql`
131 + SELECT c.id, c.slug, c.canonical_name, c.top_level, c.status FROM cancers c
132 + WHERE c.slug = ANY(${sql.param(refs.map((r) => r.toLowerCase()))}::text[]) OR c.id = ANY(${sql.param(refs)}::text[])`),
133 + [] as CancerRef[],
134 + );
135 + const bySlug = new Map(rows.map((r) => [r.slug, r]));
136 + const byId = new Map(rows.map((r) => [r.id, r]));
137 + const out: CancerRef[] = [];
138 + for (const r of refs) {
139 + const hit = bySlug.get(r.toLowerCase()) ?? byId.get(r);
140 + if (hit && !out.some((o) => o.id === hit.id)) out.push(hit);
141 + }
142 + return out;
143 +}
144 +
145 +export interface CancerChoice {
146 + id: string;
147 + slug: string;
148 + canonical_name: string;
149 + n_obs: number; // observations for the selected metric × geography (0 = no data for this selection)
150 +}
151 +
152 +/** Top-level cancers (checkbox list), with their observation count for a metric × geography so the form can flag gaps. */
153 +export async function topLevelCancerChoices(metric: string, geographyId: string | null): Promise<CancerChoice[]> {
154 + return safe(
155 + () =>
156 + run<CancerChoice>(sql`
157 + SELECT c.id, c.slug, c.canonical_name,
158 + (SELECT count(*) FROM epidemiology_observations o WHERE o.cancer_id = c.id AND o.metric = ${metric} AND (${geographyId}::text IS NULL OR o.geography_id = ${geographyId}))::int AS n_obs
159 + FROM cancers c WHERE c.top_level AND c.status = 'active' ORDER BY c.canonical_name`),
160 + [] as CancerChoice[],
161 + );
162 +}
163 +
164 +export interface TopByLatest {
165 + year: number | null;
166 + cancers: Array<{ id: string; slug: string; canonical_name: string; value: number; unit: string; source_slug: string }>;
167 +}
168 +
169 +/**
170 + * Default cancer selection: the N top-level cancers with the highest value in the latest year of the
171 + * metric for this geography/sex/age. One row per cancer (when two sources publish the same year the
172 + * larger observation is kept — the selection is a convenience, every value is then shown per source).
173 + */
174 +export async function topCancersByLatest(metric: string, geographyId: string, sex: string, age: string, n = 5): Promise<TopByLatest> {
175 + const sexCond = sex === SEX_ANY ? sql`true` : sql`o.sex = ${sex}`;
176 + const rows = await safe(
177 + () =>
178 + run<{ id: string; slug: string; canonical_name: string; value: number; unit: string; source_slug: string; year: number }>(sql`
179 + WITH latest AS (
180 + SELECT max(o.year) AS year FROM epidemiology_observations o JOIN cancers c ON c.id = o.cancer_id
181 + WHERE o.metric = ${metric} AND o.geography_id = ${geographyId} AND ${sexCond} AND o.age_group = ${age} AND c.top_level AND c.status = 'active'
182 + )
183 + SELECT DISTINCT ON (c.id) c.id, c.slug, c.canonical_name, o.value, o.unit, s.slug AS source_slug, o.year
184 + FROM epidemiology_observations o JOIN cancers c ON c.id = o.cancer_id JOIN sources s ON s.id = o.source_id, latest
185 + WHERE o.metric = ${metric} AND o.geography_id = ${geographyId} AND ${sexCond} AND o.age_group = ${age} AND o.year = latest.year AND c.top_level AND c.status = 'active'
186 + ORDER BY c.id, o.value DESC`),
187 + [] as Array<{ id: string; slug: string; canonical_name: string; value: number; unit: string; source_slug: string; year: number }>,
188 + );
189 + const top = rows.sort((a, b) => Number(b.value) - Number(a.value)).slice(0, n);
190 + return { year: top[0]?.year ?? null, cancers: top.map(({ year: _y, ...r }) => ({ ...r, value: Number(r.value) })) };
191 +}
192 +
193 +/** Year span of a metric for a geography (null when nothing exists). */
194 +export async function yearRangeFor(metric: string, geographyId: string | null): Promise<{ min: number; max: number } | null> {
195 + const rows = await safe(
196 + () => run<{ y0: number | null; y1: number | null }>(sql`SELECT min(year)::int AS y0, max(coalesce(year_end, year))::int AS y1 FROM epidemiology_observations WHERE metric = ${metric} AND (${geographyId}::text IS NULL OR geography_id = ${geographyId})`),
197 + [] as Array<{ y0: number | null; y1: number | null }>,
198 + );
199 + const r = rows[0];
200 + return r && r.y0 != null && r.y1 != null ? { min: r.y0, max: r.y1 } : null;
201 +}
202 +
203 +export interface ExplorerObsRow extends ComparableObs {
204 + id: number;
205 + cancer_id: string;
206 + geography_id: string;
207 + iso3: string | null;
208 + year_end: number | null;
209 + lower_ci: number | null;
210 + upper_ci: number | null;
211 + site_definition: string | null;
212 + source_id: string;
213 + source_name: string;
214 + source_license: string | null;
215 + provenance_id: number;
216 + dataset: string | null;
217 + dataset_version: string | null;
218 + source_url: string | null;
219 + retrieved_at: Date | string | null;
220 + updated_at: Date | string;
221 +}
222 +
223 +export interface ExplorerSelection {
224 + metric: string;
225 + cancerIds: readonly string[];
226 + geographyId: string;
227 + sex: string; // 'any' = no filter
228 + age: string;
229 + from: number | null;
230 + to: number | null;
231 + limit?: number;
232 +}
233 +
234 +/** Observations for a selection, sorted by cancer, geography, sex, year (then source) — the order the table and CSV use. */
235 +export async function explorerObservations(sel: ExplorerSelection): Promise<ExplorerObsRow[]> {
236 + if (sel.cancerIds.length === 0) return [];
237 + const limit = Math.max(1, Math.min(sel.limit ?? 20_000, 50_000));
238 + const conds = [sql`o.metric = ${sel.metric}`, sql`o.geography_id = ${sel.geographyId}`, sql`o.cancer_id = ANY(${sql.param([...sel.cancerIds])}::text[])`, sql`o.age_group = ${sel.age}`];
239 + if (sel.sex !== SEX_ANY) conds.push(sql`o.sex = ${sel.sex}`);
240 + if (sel.from != null) conds.push(sql`coalesce(o.year_end, o.year) >= ${sel.from}`);
241 + if (sel.to != null) conds.push(sql`o.year <= ${sel.to}`);
242 + const rows = await safe(
243 + () =>
244 + run<ExplorerObsRow>(sql`
245 + SELECT o.id, o.cancer_id, c.slug AS cancer_slug, c.canonical_name AS cancer_name,
246 + o.geography_id, g.slug AS geography_slug, g.name AS geography_name, g.iso3,
247 + o.year, o.year_end, o.sex, o.age_group, o.metric, o.value, o.unit, o.lower_ci, o.upper_ci, o.standard_population, o.estimate_type, o.site_definition,
248 + o.source_id, s.slug AS source_slug, s.name AS source_name, s.license AS source_license,
249 + o.provenance_id, p.dataset, p.dataset_version, p.source_url, p.retrieved_at, o.updated_at
250 + FROM epidemiology_observations o
251 + JOIN cancers c ON c.id = o.cancer_id
252 + JOIN geographies g ON g.id = o.geography_id
253 + JOIN sources s ON s.id = o.source_id
254 + LEFT JOIN provenance p ON p.id = o.provenance_id
255 + WHERE ${sql.join(conds, sql` AND `)}
256 + ORDER BY c.canonical_name, g.name, o.sex, o.year, s.slug, o.site_definition LIMIT ${limit}`),
257 + [] as ExplorerObsRow[],
258 + );
259 + return rows.map((r) => ({ ...r, value: Number(r.value), lower_ci: r.lower_ci == null ? null : Number(r.lower_ci), upper_ci: r.upper_ci == null ? null : Number(r.upper_ci), year: Number(r.year), year_end: r.year_end == null ? null : Number(r.year_end), provenance_id: Number(r.provenance_id) }));
260 +}
261 +
262 +export interface PendingSource {
263 + slug: string;
264 + name: string;
265 + license_status: string;
266 + status: string;
267 +}
268 +
269 +/** Epidemiology sources registered in the catalogue that have not contributed a single observation yet (license review, credentials…). */
270 +export async function pendingEpidemiologySources(): Promise<PendingSource[]> {
271 + return safe(
272 + () =>
273 + run<PendingSource>(sql`
274 + SELECT s.slug, s.name, s.license_status, s.status FROM sources s
275 + WHERE s.category = 'epidemiology' AND NOT EXISTS (SELECT 1 FROM epidemiology_observations o WHERE o.source_id = s.id)
276 + ORDER BY s.slug`),
277 + [] as PendingSource[],
278 + );
279 +}
280 +
281 +export interface CoverageMatrixRow {
282 + metric: string;
283 + unit: string;
284 + geography_id: string;
285 + geography_slug: string;
286 + geography_name: string;
287 + iso3: string | null;
288 + sex: string;
289 + age_group: string;
290 + source_slug: string;
291 + source_name: string;
292 + standard_population: string | null;
293 + estimate_types: string[];
294 + year_from: number;
295 + year_to: number;
296 + years: number;
297 + observations: number;
298 + n_cancers: number;
299 + last_updated: Date | string;
300 +}
301 +
302 +/** Coverage matrix: what exists, per metric × geography × sex × age × source × standard population. */
303 +export async function coverageMatrix(f: { cancerIds?: readonly string[]; geographyId?: string | null; metric?: string | null } = {}): Promise<CoverageMatrixRow[]> {
304 + const conds = [sql`true`];
305 + if (f.cancerIds && f.cancerIds.length > 0) conds.push(sql`o.cancer_id = ANY(${sql.param([...f.cancerIds])}::text[])`);
306 + if (f.geographyId) conds.push(sql`o.geography_id = ${f.geographyId}`);
307 + if (f.metric) conds.push(sql`o.metric = ${f.metric}`);
308 + return safe(
309 + () =>
310 + run<CoverageMatrixRow>(sql`
311 + SELECT o.metric, min(o.unit) AS unit, g.id AS geography_id, g.slug AS geography_slug, g.name AS geography_name, g.iso3, o.sex, o.age_group,
312 + s.slug AS source_slug, s.name AS source_name, o.standard_population, array_agg(DISTINCT o.estimate_type) AS estimate_types,
313 + min(o.year)::int AS year_from, max(coalesce(o.year_end, o.year))::int AS year_to, count(DISTINCT o.year)::int AS years, count(*)::int AS observations,
314 + count(DISTINCT o.cancer_id)::int AS n_cancers, max(o.updated_at) AS last_updated
315 + FROM epidemiology_observations o JOIN geographies g ON g.id = o.geography_id JOIN sources s ON s.id = o.source_id
316 + WHERE ${sql.join(conds, sql` AND `)}
317 + GROUP BY o.metric, g.id, g.slug, g.name, g.iso3, o.sex, o.age_group, s.slug, s.name, o.standard_population
318 + ORDER BY o.metric, g.name, (o.sex = 'all') DESC, o.sex, (o.age_group = 'all') DESC, o.age_group, s.slug, o.standard_population`),
319 + [] as CoverageMatrixRow[],
320 + );
321 +}
added apps/web/test/explorer-csv.test.ts +43 −0
@@ -0,0 +1,43 @@
1 +import { describe, it, expect } from 'vitest';
2 +import { csvCell, csvLine, csvComment, csvFileName, EPI_CSV_COLUMNS } from '@/lib/explorer-csv';
3 +
4 +describe('csvCell', () => {
5 + it('leaves plain values untouched and renders null/undefined as empty', () => {
6 + expect(csvCell('lung')).toBe('lung');
7 + expect(csvCell(12.5)).toBe('12.5');
8 + expect(csvCell(null)).toBe('');
9 + expect(csvCell(undefined)).toBe('');
10 + });
11 + it('quotes commas, quotes, CR and LF and doubles inner quotes', () => {
12 + expect(csvCell('USCS "Pancreas" (ICD-10 C25; ICD-O-3 C250-C259)')).toBe('"USCS ""Pancreas"" (ICD-10 C25; ICD-O-3 C250-C259)"');
13 + expect(csvCell('a,b')).toBe('"a,b"');
14 + expect(csvCell('line1\nline2')).toBe('"line1\nline2"');
15 + expect(csvCell('x\r\ny')).toBe('"x\r\ny"');
16 + });
17 + it('serializes dates as ISO-8601', () => {
18 + expect(csvCell(new Date('2026-09-08T12:22:48.904Z'))).toBe('2026-09-08T12:22:48.904Z');
19 + });
20 +});
21 +
22 +describe('csvLine / csvComment', () => {
23 + it('joins cells with commas', () => {
24 + expect(csvLine(['a', 1, null, 'b,c'])).toBe('a,1,,"b,c"');
25 + });
26 + it('keeps comments on one line', () => {
27 + expect(csvComment('Source: X\nlicense: Y')).toBe('# Source: X license: Y');
28 + });
29 +});
30 +
31 +describe('columns and file name', () => {
32 + it('exports provenance id and source URL columns', () => {
33 + expect(EPI_CSV_COLUMNS).toContain('provenance_id');
34 + expect(EPI_CSV_COLUMNS).toContain('source_url');
35 + expect(EPI_CSV_COLUMNS).toContain('standard_population');
36 + expect(new Set(EPI_CSV_COLUMNS).size).toBe(EPI_CSV_COLUMNS.length);
37 + });
38 + it('builds a safe file name', () => {
39 + expect(csvFileName(['mortality_count', 'united-states', 'all', 'all', 1999, 2024, '2026-09-11'])).toBe('cancerindex-epidemiology-mortality-count-united-states-all-all-1999-2024-2026-09-11.csv');
40 + expect(csvFileName([null, '', undefined])).toBe('cancerindex-epidemiology-export.csv');
41 + expect(csvFileName(['../etc/passwd'])).toBe('cancerindex-epidemiology-etc-passwd.csv');
42 + });
43 +});
added apps/web/test/explorer-params.test.ts +85 −0
@@ -0,0 +1,85 @@
1 +import { describe, it, expect } from 'vitest';
2 +import { parseCancerList, parseExplorerParams, serializeExplorerParams, apiQueryFor, yearRangeLabel, sexLabel, ageLabel, MAX_CANCERS } from '@/lib/explorer-params';
3 +
4 +const D = { metric: 'mortality_count', geography: 'united-states', cancers: ['a', 'b', 'c', 'd', 'e'], from: 1999, to: 2024 };
5 +
6 +describe('parseCancerList', () => {
7 + it('accepts comma-separated, repeated and mixed forms', () => {
8 + expect(parseCancerList('lung,breast')).toEqual(['lung', 'breast']);
9 + expect(parseCancerList(['lung', 'breast'])).toEqual(['lung', 'breast']);
10 + expect(parseCancerList(['lung,breast', 'colon'])).toEqual(['lung', 'breast', 'colon']);
11 + });
12 + it('normalizes case, drops duplicates, invalid tokens and blanks', () => {
13 + expect(parseCancerList(' Lung , lung, ,CI-CAN-00000042, ../etc, ok_slug ')).toEqual(['lung', 'CI-CAN-00000042']);
14 + });
15 + it('caps at MAX_CANCERS keeping order', () => {
16 + const many = Array.from({ length: 10 }, (_, i) => `c${i}`).join(',');
17 + expect(parseCancerList(many)).toEqual(['c0', 'c1', 'c2', 'c3', 'c4', 'c5']);
18 + expect(parseCancerList(many).length).toBe(MAX_CANCERS);
19 + });
20 + it('returns [] for undefined', () => {
21 + expect(parseCancerList(undefined)).toEqual([]);
22 + });
23 +});
24 +
25 +describe('parseExplorerParams', () => {
26 + it('fills every gap from the computed defaults', () => {
27 + const s = parseExplorerParams({}, D);
28 + expect(s).toEqual({ metric: 'mortality_count', cancers: ['a', 'b', 'c', 'd', 'e'], geography: 'united-states', sex: 'all', age: 'all', from: 1999, to: 2024, view: 'lines', normalize: 'none', page: 1 });
29 + });
30 + it('reads valid values and rejects malformed ones', () => {
31 + const s = parseExplorerParams({ metric: 'AS_Mortality_Rate', cancers: 'lung', geography: 'usa', sex: 'Female', age: '65+', from: '2010', to: '2005', view: 'multiples', page: '3' }, D);
32 + expect(s.metric).toBe('as_mortality_rate');
33 + expect(s.cancers).toEqual(['lung']);
34 + expect(s.geography).toBe('USA'); // 3-letter → ISO3 upper-case
35 + expect(s.sex).toBe('female');
36 + expect(s.age).toBe('65+');
37 + expect([s.from, s.to]).toEqual([2005, 2010]); // swapped when inverted
38 + expect(s.view).toBe('multiples');
39 + expect(s.page).toBe(3);
40 + const bad = parseExplorerParams({ metric: 'DROP TABLE', geography: 'a b', sex: '1;2', age: '<script>', from: 'x', to: '99999', view: 'pie', page: '-2' }, D);
41 + expect(bad.metric).toBe('mortality_count');
42 + expect(bad.geography).toBe('united-states');
43 + expect(bad.sex).toBe('all');
44 + expect(bad.age).toBe('all');
45 + expect(bad.from).toBe(1999);
46 + expect(bad.to).toBe(2024);
47 + expect(bad.view).toBe('lines');
48 + expect(bad.page).toBe(1);
49 + });
50 + it('accepts "any" for sex (series become cancer × sex)', () => {
51 + expect(parseExplorerParams({ sex: 'any' }, D).sex).toBe('any');
52 + });
53 +});
54 +
55 +describe('serializeExplorerParams / apiQueryFor', () => {
56 + const s = parseExplorerParams({ cancers: 'lung,breast', view: 'multiples', page: '2' }, D);
57 + it('writes every dimension explicitly, comma-joins cancers, omits defaults for view/page', () => {
58 + expect(serializeExplorerParams(s, { page: 1, view: 'lines' })).toBe('?metric=mortality_count&cancers=lung%2Cbreast&geography=united-states&sex=all&age=all&from=1999&to=2024');
59 + expect(serializeExplorerParams(s)).toContain('view=multiples');
60 + expect(serializeExplorerParams(s)).toContain('page=2');
61 + });
62 + it('round-trips through the parser', () => {
63 + const qs = new URLSearchParams(serializeExplorerParams(s).slice(1));
64 + const sp: Record<string, string> = {};
65 + for (const [k, v] of qs) sp[k] = v;
66 + expect(parseExplorerParams(sp, { metric: 'x', geography: 'y', cancers: [] })).toEqual(s);
67 + });
68 + it('builds the public API query with repeated cancer= params and no sex when "any"', () => {
69 + expect(apiQueryFor(s)).toBe('?metric=mortality_count&cancer=lung&cancer=breast&geography=united-states&sex=all&age=all&from=1999&to=2024&limit=200');
70 + expect(apiQueryFor({ ...s, sex: 'any' })).not.toContain('sex=');
71 + });
72 +});
73 +
74 +describe('labels', () => {
75 + it('year range, sex and age labels', () => {
76 + expect(yearRangeLabel(1999, 2024)).toBe('1999–2024');
77 + expect(yearRangeLabel(2024, 2024)).toBe('2024');
78 + expect(yearRangeLabel(null, null)).toBe('all years');
79 + expect(yearRangeLabel(2000, null)).toBe('from 2000');
80 + expect(sexLabel('all')).toBe('both sexes');
81 + expect(sexLabel('any')).toBe('by sex');
82 + expect(ageLabel('all')).toBe('all ages');
83 + expect(ageLabel('65+')).toBe('ages 65+');
84 + });
85 +});
added apps/web/test/explorer-series.test.ts +99 −0
@@ -0,0 +1,99 @@
1 +import { describe, it, expect } from 'vitest';
2 +import { groupComparable, explainSplit, differingDimensions, splitIntoMultiples, shouldUseMultiples, groupKey, latestPoint, type ComparableObs } from '@/lib/explorer-series';
3 +
4 +function obs(over: Partial<ComparableObs> & { year: number; value: number }): ComparableObs {
5 + return {
6 + cancer_slug: 'lung',
7 + cancer_name: 'Lung',
8 + geography_slug: 'united-states',
9 + geography_name: 'United States',
10 + sex: 'all',
11 + age_group: 'all',
12 + metric: 'as_mortality_rate',
13 + unit: 'per_100k',
14 + standard_population: 'US 2000 standard',
15 + estimate_type: 'observed',
16 + source_slug: 'cdc-uscs',
17 + source_name: 'USCS',
18 + provenance_id: 1,
19 + ...over,
20 + };
21 +}
22 +
23 +describe('groupComparable', () => {
24 + it('overlays observations that share metric, unit, geography, source, standard population and age group', () => {
25 + const rows = [obs({ year: 2000, value: 50 }), obs({ year: 2001, value: 48 }), obs({ cancer_slug: 'breast', cancer_name: 'Breast', year: 2000, value: 20 }), obs({ cancer_slug: 'breast', cancer_name: 'Breast', year: 2001, value: 19 })];
26 + const g = groupComparable(rows);
27 + expect(g).toHaveLength(1);
28 + expect(g[0]!.series.map((s) => s.name)).toEqual(['Breast', 'Lung']); // sex omitted when every series shares it
29 + expect(g[0]!.series[1]!.points.map((p) => p.x)).toEqual([2000, 2001]);
30 + expect(g[0]!.n_obs).toBe(4);
31 + expect([g[0]!.year_min, g[0]!.year_max]).toEqual([2000, 2001]);
32 + expect(g[0]!.y_max).toBe(50);
33 + });
34 + it('separates different standard populations and different sources — never overlaid', () => {
35 + const rows = [obs({ year: 2000, value: 50 }), obs({ year: 2000, value: 52, standard_population: 'World (Segi)' }), obs({ year: 2000, value: 51, source_slug: 'cdc-wonder', standard_population: 'US 2000 Std. Population' })];
36 + const g = groupComparable(rows);
37 + expect(g).toHaveLength(3);
38 + const dims = differingDimensions(g);
39 + expect(dims).toEqual(['source_slug', 'standard_population']);
40 + const why = explainSplit(g)!;
41 + expect(why).toMatch(/3 separate charts/);
42 + expect(why).toMatch(/standard population \(.*US 2000 standard.*vs.*World \(Segi\)/);
43 + expect(why).toMatch(/source \(cdc-uscs vs cdc-wonder\)/);
44 + });
45 + it('separates age groups and metrics/units; counts get "no standard population" in the caption', () => {
46 + const rows = [obs({ year: 2000, value: 50 }), obs({ year: 2000, value: 5, age_group: '65+' }), obs({ year: 2000, value: 1000, metric: 'mortality_count', unit: 'count', standard_population: null })];
47 + const g = groupComparable(rows);
48 + expect(g).toHaveLength(3);
49 + expect(explainSplit(g)).toMatch(/no standard population \(crude or count\)/);
50 + expect(explainSplit(g)).toMatch(/age group \(all ages vs 65\+\)/);
51 + });
52 + it('returns null explanation for a single group', () => {
53 + expect(explainSplit(groupComparable([obs({ year: 2000, value: 1 })]))).toBeNull();
54 + });
55 + it('makes cancer × sex series, dashed when estimated, and names them accordingly', () => {
56 + const rows = [obs({ year: 2000, value: 50, sex: 'male' }), obs({ year: 2000, value: 40, sex: 'female' }), obs({ year: 2001, value: 39, sex: 'female', estimate_type: 'estimated' })];
57 + const g = groupComparable(rows);
58 + expect(g).toHaveLength(1);
59 + const names = g[0]!.series.map((s) => `${s.name}${s.dashed ? '*' : ''}`);
60 + expect(names).toEqual(['Lung · male', 'Lung · female', 'Lung · female · estimated*']);
61 + });
62 + it('disambiguates two site definitions of one cancer inside one source', () => {
63 + const rows = [obs({ year: 2000, value: 50, site_definition: 'C33-C34' }), obs({ year: 2000, value: 45, site_definition: 'C34' })];
64 + const g = groupComparable(rows);
65 + expect(g[0]!.series.map((s) => s.name).sort()).toEqual(['Lung · C33-C34', 'Lung · C34']);
66 + });
67 + it('orders groups by size and skips non-finite values', () => {
68 + const rows = [obs({ year: 2000, value: 1, source_slug: 'x' }), obs({ year: 2000, value: 2 }), obs({ year: 2001, value: 3 }), obs({ year: 2002, value: Number.NaN })];
69 + const g = groupComparable(rows);
70 + expect(g.map((x) => x.source_slug)).toEqual(['cdc-uscs', 'x']);
71 + expect(g[0]!.n_obs).toBe(2);
72 + });
73 + it('groupKey is stable and null-safe', () => {
74 + expect(groupKey(obs({ year: 1, value: 1, standard_population: null }))).toBe('as_mortality_rate|per_100k|united-states|cdc-uscs||all');
75 + });
76 +});
77 +
78 +describe('small multiples', () => {
79 + const rows = ['a', 'b', 'c', 'd', 'e'].flatMap((c) => [obs({ cancer_slug: c, cancer_name: c.toUpperCase(), year: 2000, value: 10 }), obs({ cancer_slug: c, cancer_name: c.toUpperCase(), year: 2001, value: c === 'e' ? 99 : 12 })]);
80 + const g = groupComparable(rows)[0]!;
81 + it('forces multiples beyond 4 overlaid series, honours the explicit view otherwise', () => {
82 + expect(shouldUseMultiples(g, 'lines')).toBe(true);
83 + const small = groupComparable(rows.slice(0, 4))[0]!;
84 + expect(shouldUseMultiples(small, 'lines')).toBe(false);
85 + expect(shouldUseMultiples(small, 'multiples')).toBe(true);
86 + const single = groupComparable(rows.slice(0, 2))[0]!;
87 + expect(shouldUseMultiples(single, 'multiples')).toBe(false); // one cancer → nothing to split
88 + });
89 + it('splits per cancer and keeps the shared y_max', () => {
90 + const panels = splitIntoMultiples(g);
91 + expect(panels).toHaveLength(5);
92 + expect(panels.every((p) => p.y_max === 99)).toBe(true);
93 + expect(panels.map((p) => p.series[0]!.cancer_slug)).toEqual(['a', 'b', 'c', 'd', 'e']);
94 + expect(panels[0]!.n_obs).toBe(2);
95 + });
96 + it('latestPoint returns the last chronological point', () => {
97 + expect(latestPoint(g.series[4]!)).toEqual({ x: 2001, y: 99 });
98 + });
99 +});
added docs/methodology/data-explorer.md +115 −0
@@ -0,0 +1,115 @@
1 +# Data explorer — methodology
2 +
3 +`/explore` charts and exports registry observations stored in `epidemiology_observations` (see
4 +`docs/DATA-MODEL.md` and `docs/METHODOLOGY.md` §1–2). It shows values **exactly as published by the
5 +source**; CancerIndex harmonizes units (`count`, `per_100k`) and labels, never the numbers. Nothing
6 +is estimated, interpolated, summed across site groups or extrapolated beyond the years a source
7 +publishes. Every chart and table states unit, geography, sex, age group, years, standard population,
8 +source and retrieval date, and each row keeps its `provenance_id`.
9 +
10 +Population statistics describe groups defined by geography, period, sex and age; they never predict
11 +an individual outcome (METHODOLOGY §325).
12 +
13 +## 1. Comparability rules
14 +
15 +Two observations are drawn on **one axis** only when they share all of:
16 +
17 +| Dimension | Why it matters |
18 +|---|---|
19 +| `metric` | deaths, new cases, crude rates and age-standardized rates are different quantities |
20 +| `unit` | `count` vs `per_100k` |
21 +| `geography` | a country and one of its subdivisions are not comparable series |
22 +| `source` | registries differ in coverage, site definitions (e.g. USCS "Pancreas" = ICD-10 C25 vs another grouping), vintage and revisions; the same year from CDC WONDER and USCS is two observations, not one |
23 +| `standard_population` | rates standardized to the "2000 U.S. standard population" and to the "World (Segi)" standard are not on the same scale; crude rates and counts have no standard population |
24 +| `age_group` | all ages vs 65+ etc. |
25 +
26 +Observations that differ on any of these dimensions are rendered as **separate charts**, and a caption
27 +names the dimension(s) that differ, e.g. *"Shown as 2 separate charts: the observations differ by
28 +source (cdc-uscs vs cdc-wonder) and standard population (2000 U.S. standard population (19 age groups)
29 +vs 2000 U.S. Std. Population)."* The groups are never overlaid, indexed or rescaled to appear
30 +comparable. Inside one group the series are **cancer × sex** (× estimate type × site definition when a
31 +source publishes two definitions for one cancer). Series whose `estimate_type` is not `observed`
32 +(estimated, projected) are dashed and labelled.
33 +
34 +Implementation: `apps/web/src/lib/explorer-series.ts` (`groupComparable`, `explainSplit`), unit-tested
35 +in `apps/web/test/explorer-series.test.ts`.
36 +
37 +### Small multiples
38 +
39 +When more than 4 series would be overlaid, or when the user chooses `view=multiples`, a group is
40 +split into one panel per cancer. Panels **share one y-axis** (0 to the group maximum, stated above the
41 +panels) so heights remain comparable. Colour follows the entity (cancer × sex) across panels and
42 +groups and is never cycled; beyond eight identities, colour follows the sex inside each panel and the
43 +panel title carries the cancer. Colour is never the only carrier: every series has a text legend
44 +entry, a tooltip with the series name, and the observations table lists every value.
45 +
46 +### Sex
47 +
48 +`sex=all` shows the source's "both sexes" observation (as published — not the sum of male and
49 +female). `sex=any` shows every sex the source publishes as separate series inside the same group.
50 +
51 +## 2. Defaults (computed, not curated)
52 +
53 +When the URL carries no parameter:
54 +
55 +- `metric` = `mortality_count` when present in the database, else the first metric present;
56 +- `geography` = `united-states` when it carries observations, else the first geography that does;
57 +- `sex` = `all`, `age` = `all`;
58 +- `cancers` = the **5 top-level cancers with the highest value in the latest year** of the selected
59 + metric, geography, sex and age group (`topCancersByLatest`, one row per cancer — when two sources
60 + publish the same year, the larger value decides membership only; both sources are then charted
61 + separately). The results header says so and names the year;
62 +- `from`/`to` = the full year span of the metric for the geography.
63 +
64 +The list of metrics, geographies, sexes, age groups, years and sources offered by the form is read from
65 +the observations (`explorerOptions`), so the explorer never advertises data it does not hold. Sources
66 +registered for epidemiology but not yet ingested (IARC/GLOBOCAN under license review, SEER awaiting
67 +credentials) are listed as such.
68 +
69 +## 3. Permalink parameters
70 +
71 +The URL is the state. `serializeExplorerParams` writes every dimension explicitly so a link stays
72 +stable when the computed defaults change.
73 +
74 +| Parameter | Values | Notes |
75 +|---|---|---|
76 +| `metric` | `mortality_count`, `as_mortality_rate`, `mortality_rate`, `incidence_count`, `as_incidence_rate`, … | one metric per view |
77 +| `cancers` | comma-separated slugs or `CI-CAN-…` ids, max 6 | repeated `cancers=` also accepted; unknown refs are ignored and reported |
78 +| `geography` | slug or ISO3 (`united-states`, `USA`) | |
79 +| `sex` | `all` (default), `male`, `female`, `any` | `any` = every sex, one series each |
80 +| `age` | age group label as stored (`all` default) | |
81 +| `from`, `to` | years (inclusive); swapped when inverted | multi-year observations match when the span overlaps |
82 +| `view` | `lines` (default), `multiples` | |
83 +| `normalize` | `none` (only value in Phase 1) | reserved for indexed views |
84 +| `page` | observations table page (50 rows/page) | omitted from downloads |
85 +
86 +Examples:
87 +
88 +- `/explore` — defaults;
89 +- `/explore?metric=as_mortality_rate&cancers=malignant-lung-neoplasm,malignant-breast-neoplasm&geography=united-states&sex=any&age=all&from=1999&to=2024`;
90 +- `/explore?metric=mortality_count&cancers=malignant-pancreatic-neoplasm&geography=united-states&sex=all&age=all&from=2018&to=2024&view=multiples`.
91 +
92 +## 4. Downloads and API
93 +
94 +- **CSV** — `/api/export/epidemiology.csv?<same parameters>` (metric, cancers, geography required).
95 + Header rows prefixed `#`: CancerIndex attribution and licence of the harmonization (CC BY 4.0),
96 + each underlying source with its licence text, dataset, version and retrieval timestamp, source
97 + attribution sentences, methodology URL and comparability reminder, generation time and row count.
98 + Columns: `cancer_id, cancer_slug, cancer_name, geography_id, geography_slug, geography_name, iso3,
99 + year, year_end, sex, age_group, metric, value, unit, lower_ci, upper_ci, standard_population,
100 + estimate_type, site_definition, source_slug, source_name, provenance_id, dataset, dataset_version,
101 + source_url, retrieved_at`. RFC 4180 quoting. Capped at 50 000 rows (stated in the header).
102 +- **JSON** — `GET /api/v1/epidemiology?metric=&cancer=<repeatable, max 8>&geography=&sex=&age=&from=&to=&source=&estimateType=&limit&offset`
103 + (paginated, max 200 per page, envelope with `sources`), `GET /api/v1/epidemiology/coverage?cancer=&geography=&metric=`
104 + and `GET /api/v1/epidemiology/metrics`. Each observation carries `standardPopulation`,
105 + `estimateType`, `siteDefinition`, `source` and `provenance {id, dataset, datasetVersion, sourceUrl, retrievedAt}`
106 + so consumers can apply the same comparability rules.
107 +- **Coverage matrix** — `/explore/coverage`: metric × geography × sex × age × source × standard
108 + population with year span, distinct years, observations and cancers. The empty state of `/explore`
109 + shows the same matrix restricted to the selected cancers/geography so a user sees what exists.
110 +
111 +## 5. Citation
112 +
113 +The "Cite" line combines CancerIndex (URL, access date) with every underlying source, its dataset and
114 +retrieval date. Underlying observations remain under their providers' licences (CDC WONDER and USCS:
115 +US Government work, statistical reporting only); CancerIndex's harmonization is CC BY 4.0.
116