SPB Git forge
spboucher.ai ↗

spb/cancerindex

Public
37commits 1branches 0releases
2.9 MBsize
maindefault branch
10 days agolast push
TypeScript 97.2% SQL 1.5% CSS 0.6% JavaScript 0.5%

openFDA: shared parent/child aliases resolve to the broadest concept (fixes 67 approvals mapped to childhood entities), backfill supersedes edges no longer asserted; RawLake.read returns null on a missing file instead of hanging the process

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 13 days ago (Sep 11, 2026) parent 52613d9

5 changed files +121 −5

modified docs/connectors/openfda.md +11 −0
@@ -107,3 +107,14 @@ Idempotent: re-running updates rows in place (`raw.key`), reuses provenance when
107 107 - ANDA generics / biosimilars of a molecule that already has a reference NDA/BLA are kept as source records only.
108 108 - Line of therapy, disease stage and biomarker ids (`lineOfTherapy`, `diseaseStage`, `biomarkerIds`) are not extracted (Phase 2).
109 109 - Only US / FDA. Health Canada, EMA, MHRA, PMDA are separate connectors (spec §13).
110 +
111 +## 2026-09-11 — shared-alias mapping fix
112 +
113 +A label alias shared by a concept and one of its descendants ("breast cancer" on *Breast Carcinoma*
114 +and on *Childhood Breast Carcinoma*) used to resolve to the **narrowest** concept when the dictionary
115 +was built (the per-sentence collapse rule leaked into dictionary construction): 67 approval rows and
116 +their `APPROVED_FOR` edges pointed at childhood entities. `CancerDictionary.build` now resolves a
117 +shared alias with the resolver's preferred/display/broadest rule, then equivalence merging, then the
118 +**broadest** of one lineage (`CancerReconciler.broadest`), and never the narrowest. `--mode backfill`
119 +replays the lake with the corrected dictionary and marks edges the replay no longer asserts as
120 +`superseded` (99 on the 2026-09-10 lake). Regression test: `dictionary-shared-alias.test.ts`.
added packages/connectors/src/connectors/openfda/dictionary-shared-alias.test.ts +60 −0
@@ -0,0 +1,60 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { CancerDictionary, CancerReconciler } from './normalize.js';
3 +
4 +/**
5 + * Regression (2026-09-11): an alias shared by a parent and its child ("breast cancer" on Breast
6 + * Carcinoma and on Childhood Breast Carcinoma) must resolve to the BROADEST concept in the
7 + * dictionary, never to the child. Per-sentence collapsing (narrowest of lineage) is unchanged.
8 + */
9 +const canonical = new Map([
10 + ['BREAST', 'breast carcinoma'],
11 + ['CHILD_BREAST', 'childhood breast carcinoma'],
12 + ['HNSCC', 'head and neck squamous cell carcinoma'],
13 + ['SCC', 'squamous cell carcinoma'],
14 +]);
15 +const aliases = new Map([
16 + ['BREAST', new Set(['breast cancer'])],
17 + ['CHILD_BREAST', new Set(['breast cancer', 'pediatric breast cancer'])],
18 + ['HNSCC', new Set(['hnscc'])],
19 + ['SCC', new Set(['scc'])],
20 +]);
21 +const parents = new Map([
22 + ['CHILD_BREAST', ['BREAST']],
23 + ['HNSCC', ['SCC']],
24 +]);
25 +const reconciler = new CancerReconciler(canonical, aliases, parents, new Set(['BREAST', 'CHILD_BREAST', 'HNSCC', 'SCC']));
26 +
27 +describe('CancerDictionary.build with a parent/child shared alias', () => {
28 + const entries = [
29 + { cancerId: 'BREAST', alias: 'breast cancer' },
30 + { cancerId: 'CHILD_BREAST', alias: 'breast cancer' },
31 + { cancerId: 'CHILD_BREAST', alias: 'pediatric breast cancer' },
32 + ];
33 + it('keeps the broadest concept when the resolver cannot disambiguate', () => {
34 + const d = CancerDictionary.build(entries, () => null, reconciler);
35 + expect(d.mentions('indicated for breast cancer')[0]?.cancerId).toBe('BREAST');
36 + expect(d.mentions('pediatric breast cancer')[0]?.cancerId).toBe('CHILD_BREAST');
37 + });
38 + it('prefers the resolver disambiguation when it exists', () => {
39 + const d = CancerDictionary.build(entries, (a) => (a === 'breast cancer' ? 'BREAST' : null), reconciler);
40 + expect(d.mentions('breast cancer')[0]?.cancerId).toBe('BREAST');
41 + });
42 + it('never returns an id outside the alias group', () => {
43 + const d = CancerDictionary.build(entries, () => 'SCC', reconciler);
44 + expect(d.mentions('breast cancer')).toEqual([]);
45 + });
46 +});
47 +
48 +describe('CancerReconciler.collapse / broadest', () => {
49 + it('collapse still picks the narrowest of one lineage for sentence mentions', () => {
50 + expect(reconciler.collapse(['HNSCC', 'SCC'])?.cancerId).toBe('HNSCC');
51 + });
52 + it('collapse without lineage returns null for parent/child', () => {
53 + expect(reconciler.collapse(['HNSCC', 'SCC'], { allowLineage: false })).toBeNull();
54 + });
55 + it('broadest picks the ancestor', () => {
56 + expect(reconciler.broadest(['HNSCC', 'SCC'])).toBe('SCC');
57 + expect(reconciler.broadest(['CHILD_BREAST', 'BREAST'])).toBe('BREAST');
58 + expect(reconciler.broadest(['BREAST', 'SCC'])).toBeNull();
59 + });
60 +});
modified packages/connectors/src/connectors/openfda/index.ts +8 −0
@@ -173,6 +173,14 @@ export class OpenFdaConnector extends Connector {
173 173 await this.persistDrugApplications(ctx, { id: drugId, name: drugNames.get(drugId)! }, apps, dict, stats, labels);
174 174 stats.drugsSeen++;
175 175 }
176 + // A backfill replays every application: edges this source asserted before but did not re-assert
177 + // now (mapping rule changed, e.g. the 2026-09-11 shared-alias fix) are superseded, never deleted.
178 + if (!ctx.stoppedEarly) {
179 + const stale = await ctx.db.execute<{ n: string }>(sql`
180 + WITH u AS (UPDATE knowledge_edges SET status = 'superseded' WHERE source_id = ${ctx.sourceId} AND relationship_type = 'APPROVED_FOR' AND status = 'active' AND last_seen_at < ${ctx.startedAt.toISOString()}::timestamptz RETURNING 1)
181 + SELECT count(*)::text AS n FROM u`);
182 + if (Number(stale[0]?.n ?? 0) > 0) ctx.info(`backfill: ${stale[0]!.n} APPROVED_FOR edges no longer asserted by the replay → superseded`);
183 + }
176 184 ctx.info(`backfill complete: drugs ${stats.drugsSeen}, applications ${stats.applications}, approval rows ${stats.approvalRows} (with cancer ${stats.rowsWithCancer}, without ${stats.rowsWithoutCancer}, tumor-agnostic ${stats.tumorAgnosticRows}), ambiguous bullets ${stats.ambiguousBullets}, edges ${stats.edges}, labels missing ${stats.labelsMissing}`, { ...stats });
177 185 }
178 186
modified packages/connectors/src/connectors/openfda/normalize.ts +32 −3
@@ -311,8 +311,32 @@ export class CancerReconciler {
311 311 return [...ids].sort((x, y) => Number(this.ncitCoded.has(y)) - Number(this.ncitCoded.has(x)) || x.localeCompare(y))[0]!;
312 312 }
313 313
314 − /** One id for the set, or null. */
315 − collapse(ids: string[]): { cancerId: string; via: 'single' | 'equivalent' | 'narrowest_of_lineage' } | null {
314 + /**
315 + * The broadest candidate when the set is one lineage (one candidate is an ancestor of every other,
316 + * equivalence classes merged), else null. Used for *dictionary aliases*: the bare text
317 + * "breast cancer" shared by "Breast Carcinoma" and "Childhood Breast Carcinoma" denotes the
318 + * general concept, never the narrower one (that was a mapping bug found 2026-09-11).
319 + */
320 + broadest(ids: string[]): string | null {
321 + const distinct = [...new Set(ids)];
322 + if (distinct.length <= 1) return distinct[0] ?? null;
323 + const groups: string[][] = [];
324 + for (const id of distinct) {
325 + const g = groups.find((grp) => grp.some((m) => this.equivalent(m, id)));
326 + if (g) g.push(id);
327 + else groups.push([id]);
328 + }
329 + if (groups.length === 1) return this.preferred(groups[0]!);
330 + for (let gi = 0; gi < groups.length; gi++) {
331 + const candGroup = groups[gi]!;
332 + const others = groups.filter((_, j) => j !== gi);
333 + if (others.every((g) => g.some((o) => candGroup.some((c) => this.isAncestor(c, o))))) return this.preferred(candGroup);
334 + }
335 + return null;
336 + }
337 +
338 + /** One id for the set, or null. `allowLineage: false` keeps only the equivalence merge (dictionary use). */
339 + collapse(ids: string[], opts: { allowLineage?: boolean } = {}): { cancerId: string; via: 'single' | 'equivalent' | 'narrowest_of_lineage' } | null {
316 340 const distinct = [...new Set(ids)];
317 341 if (distinct.length === 0) return null;
318 342 if (distinct.length === 1) return { cancerId: distinct[0]!, via: 'single' };
@@ -325,6 +349,7 @@ export class CancerReconciler {
325 349 }
326 350 const reps = groups.map((g) => this.preferred(g));
327 351 if (reps.length === 1) return { cancerId: reps[0]!, via: 'equivalent' };
352 + if (opts.allowLineage === false) return null;
328 353 // `cand` is the narrowest when every other group contains an ancestor of cand (or of an equivalent of cand).
329 354 for (let gi = 0; gi < reps.length; gi++) {
330 355 const candGroup = groups[gi]!;
@@ -365,7 +390,11 @@ export class CancerDictionary {
365 390 }
366 391 const dict = new CancerDictionary();
367 392 for (const [norm, g] of groups) {
368 − let id: string | null = g.ids.size === 1 ? [...g.ids][0]! : (reconciler?.collapse([...g.ids])?.cancerId ?? disambiguate(g.alias));
393 + // Shared alias: the resolver's preferred-name / display-name / broadest-lineage rule first, then
394 + // equivalence merging, then the broadest of one lineage. Never the narrowest: a bare alias names
395 + // the general concept ("breast cancer" ≠ "Childhood Breast Carcinoma").
396 + const ids = [...g.ids];
397 + let id: string | null = ids.length === 1 ? ids[0]! : (disambiguate(g.alias) ?? reconciler?.collapse(ids, { allowLineage: false })?.cancerId ?? reconciler?.broadest(ids) ?? null);
369 398 if (id && !g.ids.has(id)) id = null;
370 399 if (!id) {
371 400 dict.ambiguousDropped++;
modified packages/connectors/src/sdk/lake.ts +10 −2
@@ -130,13 +130,21 @@ export class RawLake {
130 130 const [file, lineStr] = ref.split('#');
131 131 if (!file || lineStr === undefined) return null;
132 132 const target = Number(lineStr);
133 + // A missing or unreadable file must resolve to null: a stream error that only destroys the
134 + // gunzip leaves the readline iterator pending forever, the event loop drains and the process
135 + // exits 0 mid-run without finishing the ingest run (observed 2026-09-11 on a prod copy whose
136 + // lake lives on another host).
137 + if (!existsSync(file)) return null;
133 138 const src = createReadStream(file);
134 139 const gunzip = createGunzip();
135 − const swallow = () => gunzip.destroy();
140 + const rl = createInterface({ input: gunzip, crlfDelay: Infinity });
141 + const swallow = () => {
142 + rl.close();
143 + gunzip.destroy();
144 + };
136 145 src.on('error', swallow);
137 146 gunzip.on('error', swallow);
138 147 src.pipe(gunzip);
139 − const rl = createInterface({ input: gunzip, crlfDelay: Infinity });
140 148 try {
141 149 let i = 0;
142 150 for await (const line of rl) {
143 151