openFDA: shared parent/child aliases resolve to the broadest concept (fixes 67 approvals mapped to childhood entities), backfill supersedes edges no longer asserted; RawLake.read returns null on a missing file instead of hanging the process
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
5 changed files +121 −5
modified
docs/connectors/openfda.md
+11 −0
@@ -107,3 +107,14 @@ Idempotent: re-running updates rows in place (`raw.key`), reuses provenance when | |||||
| 107 | - ANDA generics / biosimilars of a molecule that already has a reference NDA/BLA are kept as source records only. | 107 | - ANDA generics / biosimilars of a molecule that already has a reference NDA/BLA are kept as source records only. | ||
| 108 | - Line of therapy, disease stage and biomarker ids (`lineOfTherapy`, `diseaseStage`, `biomarkerIds`) are not extracted (Phase 2). | 108 | - Line of therapy, disease stage and biomarker ids (`lineOfTherapy`, `diseaseStage`, `biomarkerIds`) are not extracted (Phase 2). | ||
| 109 | - Only US / FDA. Health Canada, EMA, MHRA, PMDA are separate connectors (spec §13). | 109 | - Only US / FDA. Health Canada, EMA, MHRA, PMDA are separate connectors (spec §13). | ||
| 110 | + | ||||
| 111 | +## 2026-09-11 — shared-alias mapping fix | ||||
| 112 | + | ||||
| 113 | +A label alias shared by a concept and one of its descendants ("breast cancer" on *Breast Carcinoma* | ||||
| 114 | +and on *Childhood Breast Carcinoma*) used to resolve to the **narrowest** concept when the dictionary | ||||
| 115 | +was built (the per-sentence collapse rule leaked into dictionary construction): 67 approval rows and | ||||
| 116 | +their `APPROVED_FOR` edges pointed at childhood entities. `CancerDictionary.build` now resolves a | ||||
| 117 | +shared alias with the resolver's preferred/display/broadest rule, then equivalence merging, then the | ||||
| 118 | +**broadest** of one lineage (`CancerReconciler.broadest`), and never the narrowest. `--mode backfill` | ||||
| 119 | +replays the lake with the corrected dictionary and marks edges the replay no longer asserts as | ||||
| 120 | +`superseded` (99 on the 2026-09-10 lake). Regression test: `dictionary-shared-alias.test.ts`. | ||||
added
packages/connectors/src/connectors/openfda/dictionary-shared-alias.test.ts
+60 −0
@@ -0,0 +1,60 @@ | |||||
| 1 | +import { describe, expect, it } from 'vitest'; | ||||
| 2 | +import { CancerDictionary, CancerReconciler } from './normalize.js'; | ||||
| 3 | + | ||||
| 4 | +/** | ||||
| 5 | + * Regression (2026-09-11): an alias shared by a parent and its child ("breast cancer" on Breast | ||||
| 6 | + * Carcinoma and on Childhood Breast Carcinoma) must resolve to the BROADEST concept in the | ||||
| 7 | + * dictionary, never to the child. Per-sentence collapsing (narrowest of lineage) is unchanged. | ||||
| 8 | + */ | ||||
| 9 | +const canonical = new Map([ | ||||
| 10 | + ['BREAST', 'breast carcinoma'], | ||||
| 11 | + ['CHILD_BREAST', 'childhood breast carcinoma'], | ||||
| 12 | + ['HNSCC', 'head and neck squamous cell carcinoma'], | ||||
| 13 | + ['SCC', 'squamous cell carcinoma'], | ||||
| 14 | +]); | ||||
| 15 | +const aliases = new Map([ | ||||
| 16 | + ['BREAST', new Set(['breast cancer'])], | ||||
| 17 | + ['CHILD_BREAST', new Set(['breast cancer', 'pediatric breast cancer'])], | ||||
| 18 | + ['HNSCC', new Set(['hnscc'])], | ||||
| 19 | + ['SCC', new Set(['scc'])], | ||||
| 20 | +]); | ||||
| 21 | +const parents = new Map([ | ||||
| 22 | + ['CHILD_BREAST', ['BREAST']], | ||||
| 23 | + ['HNSCC', ['SCC']], | ||||
| 24 | +]); | ||||
| 25 | +const reconciler = new CancerReconciler(canonical, aliases, parents, new Set(['BREAST', 'CHILD_BREAST', 'HNSCC', 'SCC'])); | ||||
| 26 | + | ||||
| 27 | +describe('CancerDictionary.build with a parent/child shared alias', () => { | ||||
| 28 | + const entries = [ | ||||
| 29 | + { cancerId: 'BREAST', alias: 'breast cancer' }, | ||||
| 30 | + { cancerId: 'CHILD_BREAST', alias: 'breast cancer' }, | ||||
| 31 | + { cancerId: 'CHILD_BREAST', alias: 'pediatric breast cancer' }, | ||||
| 32 | + ]; | ||||
| 33 | + it('keeps the broadest concept when the resolver cannot disambiguate', () => { | ||||
| 34 | + const d = CancerDictionary.build(entries, () => null, reconciler); | ||||
| 35 | + expect(d.mentions('indicated for breast cancer')[0]?.cancerId).toBe('BREAST'); | ||||
| 36 | + expect(d.mentions('pediatric breast cancer')[0]?.cancerId).toBe('CHILD_BREAST'); | ||||
| 37 | + }); | ||||
| 38 | + it('prefers the resolver disambiguation when it exists', () => { | ||||
| 39 | + const d = CancerDictionary.build(entries, (a) => (a === 'breast cancer' ? 'BREAST' : null), reconciler); | ||||
| 40 | + expect(d.mentions('breast cancer')[0]?.cancerId).toBe('BREAST'); | ||||
| 41 | + }); | ||||
| 42 | + it('never returns an id outside the alias group', () => { | ||||
| 43 | + const d = CancerDictionary.build(entries, () => 'SCC', reconciler); | ||||
| 44 | + expect(d.mentions('breast cancer')).toEqual([]); | ||||
| 45 | + }); | ||||
| 46 | +}); | ||||
| 47 | + | ||||
| 48 | +describe('CancerReconciler.collapse / broadest', () => { | ||||
| 49 | + it('collapse still picks the narrowest of one lineage for sentence mentions', () => { | ||||
| 50 | + expect(reconciler.collapse(['HNSCC', 'SCC'])?.cancerId).toBe('HNSCC'); | ||||
| 51 | + }); | ||||
| 52 | + it('collapse without lineage returns null for parent/child', () => { | ||||
| 53 | + expect(reconciler.collapse(['HNSCC', 'SCC'], { allowLineage: false })).toBeNull(); | ||||
| 54 | + }); | ||||
| 55 | + it('broadest picks the ancestor', () => { | ||||
| 56 | + expect(reconciler.broadest(['HNSCC', 'SCC'])).toBe('SCC'); | ||||
| 57 | + expect(reconciler.broadest(['CHILD_BREAST', 'BREAST'])).toBe('BREAST'); | ||||
| 58 | + expect(reconciler.broadest(['BREAST', 'SCC'])).toBeNull(); | ||||
| 59 | + }); | ||||
| 60 | +}); | ||||
modified
packages/connectors/src/connectors/openfda/index.ts
+8 −0
@@ -173,6 +173,14 @@ export class OpenFdaConnector extends Connector { | |||||
| 173 | await this.persistDrugApplications(ctx, { id: drugId, name: drugNames.get(drugId)! }, apps, dict, stats, labels); | 173 | await this.persistDrugApplications(ctx, { id: drugId, name: drugNames.get(drugId)! }, apps, dict, stats, labels); | ||
| 174 | stats.drugsSeen++; | 174 | stats.drugsSeen++; | ||
| 175 | } | 175 | } | ||
| 176 | + // A backfill replays every application: edges this source asserted before but did not re-assert | ||||
| 177 | + // now (mapping rule changed, e.g. the 2026-09-11 shared-alias fix) are superseded, never deleted. | ||||
| 178 | + if (!ctx.stoppedEarly) { | ||||
| 179 | + const stale = await ctx.db.execute<{ n: string }>(sql` | ||||
| 180 | + WITH u AS (UPDATE knowledge_edges SET status = 'superseded' WHERE source_id = ${ctx.sourceId} AND relationship_type = 'APPROVED_FOR' AND status = 'active' AND last_seen_at < ${ctx.startedAt.toISOString()}::timestamptz RETURNING 1) | ||||
| 181 | + SELECT count(*)::text AS n FROM u`); | ||||
| 182 | + if (Number(stale[0]?.n ?? 0) > 0) ctx.info(`backfill: ${stale[0]!.n} APPROVED_FOR edges no longer asserted by the replay → superseded`); | ||||
| 183 | + } | ||||
| 176 | ctx.info(`backfill complete: drugs ${stats.drugsSeen}, applications ${stats.applications}, approval rows ${stats.approvalRows} (with cancer ${stats.rowsWithCancer}, without ${stats.rowsWithoutCancer}, tumor-agnostic ${stats.tumorAgnosticRows}), ambiguous bullets ${stats.ambiguousBullets}, edges ${stats.edges}, labels missing ${stats.labelsMissing}`, { ...stats }); | 184 | ctx.info(`backfill complete: drugs ${stats.drugsSeen}, applications ${stats.applications}, approval rows ${stats.approvalRows} (with cancer ${stats.rowsWithCancer}, without ${stats.rowsWithoutCancer}, tumor-agnostic ${stats.tumorAgnosticRows}), ambiguous bullets ${stats.ambiguousBullets}, edges ${stats.edges}, labels missing ${stats.labelsMissing}`, { ...stats }); | ||
| 177 | } | 185 | } | ||
| 178 | 186 | ||||
modified
packages/connectors/src/connectors/openfda/normalize.ts
+32 −3
@@ -311,8 +311,32 @@ export class CancerReconciler { | |||||
| 311 | return [...ids].sort((x, y) => Number(this.ncitCoded.has(y)) - Number(this.ncitCoded.has(x)) || x.localeCompare(y))[0]!; | 311 | return [...ids].sort((x, y) => Number(this.ncitCoded.has(y)) - Number(this.ncitCoded.has(x)) || x.localeCompare(y))[0]!; | ||
| 312 | } | 312 | } | ||
| 313 | 313 | ||||
| 314 | − /** One id for the set, or null. */ | 314 | + /** | ||
| 315 | − collapse(ids: string[]): { cancerId: string; via: 'single' | 'equivalent' | 'narrowest_of_lineage' } | null { | 315 | + * The broadest candidate when the set is one lineage (one candidate is an ancestor of every other, | ||
| 316 | + * equivalence classes merged), else null. Used for *dictionary aliases*: the bare text | ||||
| 317 | + * "breast cancer" shared by "Breast Carcinoma" and "Childhood Breast Carcinoma" denotes the | ||||
| 318 | + * general concept, never the narrower one (that was a mapping bug found 2026-09-11). | ||||
| 319 | + */ | ||||
| 320 | + broadest(ids: string[]): string | null { | ||||
| 321 | + const distinct = [...new Set(ids)]; | ||||
| 322 | + if (distinct.length <= 1) return distinct[0] ?? null; | ||||
| 323 | + const groups: string[][] = []; | ||||
| 324 | + for (const id of distinct) { | ||||
| 325 | + const g = groups.find((grp) => grp.some((m) => this.equivalent(m, id))); | ||||
| 326 | + if (g) g.push(id); | ||||
| 327 | + else groups.push([id]); | ||||
| 328 | + } | ||||
| 329 | + if (groups.length === 1) return this.preferred(groups[0]!); | ||||
| 330 | + for (let gi = 0; gi < groups.length; gi++) { | ||||
| 331 | + const candGroup = groups[gi]!; | ||||
| 332 | + const others = groups.filter((_, j) => j !== gi); | ||||
| 333 | + if (others.every((g) => g.some((o) => candGroup.some((c) => this.isAncestor(c, o))))) return this.preferred(candGroup); | ||||
| 334 | + } | ||||
| 335 | + return null; | ||||
| 336 | + } | ||||
| 337 | + | ||||
| 338 | + /** One id for the set, or null. `allowLineage: false` keeps only the equivalence merge (dictionary use). */ | ||||
| 339 | + collapse(ids: string[], opts: { allowLineage?: boolean } = {}): { cancerId: string; via: 'single' | 'equivalent' | 'narrowest_of_lineage' } | null { | ||||
| 316 | const distinct = [...new Set(ids)]; | 340 | const distinct = [...new Set(ids)]; | ||
| 317 | if (distinct.length === 0) return null; | 341 | if (distinct.length === 0) return null; | ||
| 318 | if (distinct.length === 1) return { cancerId: distinct[0]!, via: 'single' }; | 342 | if (distinct.length === 1) return { cancerId: distinct[0]!, via: 'single' }; | ||
@@ -325,6 +349,7 @@ export class CancerReconciler { | |||||
| 325 | } | 349 | } | ||
| 326 | const reps = groups.map((g) => this.preferred(g)); | 350 | const reps = groups.map((g) => this.preferred(g)); | ||
| 327 | if (reps.length === 1) return { cancerId: reps[0]!, via: 'equivalent' }; | 351 | if (reps.length === 1) return { cancerId: reps[0]!, via: 'equivalent' }; | ||
| 352 | + if (opts.allowLineage === false) return null; | ||||
| 328 | // `cand` is the narrowest when every other group contains an ancestor of cand (or of an equivalent of cand). | 353 | // `cand` is the narrowest when every other group contains an ancestor of cand (or of an equivalent of cand). | ||
| 329 | for (let gi = 0; gi < reps.length; gi++) { | 354 | for (let gi = 0; gi < reps.length; gi++) { | ||
| 330 | const candGroup = groups[gi]!; | 355 | const candGroup = groups[gi]!; | ||
@@ -365,7 +390,11 @@ export class CancerDictionary { | |||||
| 365 | } | 390 | } | ||
| 366 | const dict = new CancerDictionary(); | 391 | const dict = new CancerDictionary(); | ||
| 367 | for (const [norm, g] of groups) { | 392 | for (const [norm, g] of groups) { | ||
| 368 | − let id: string | null = g.ids.size === 1 ? [...g.ids][0]! : (reconciler?.collapse([...g.ids])?.cancerId ?? disambiguate(g.alias)); | 393 | + // Shared alias: the resolver's preferred-name / display-name / broadest-lineage rule first, then | ||
| 394 | + // equivalence merging, then the broadest of one lineage. Never the narrowest: a bare alias names | ||||
| 395 | + // the general concept ("breast cancer" ≠ "Childhood Breast Carcinoma"). | ||||
| 396 | + const ids = [...g.ids]; | ||||
| 397 | + let id: string | null = ids.length === 1 ? ids[0]! : (disambiguate(g.alias) ?? reconciler?.collapse(ids, { allowLineage: false })?.cancerId ?? reconciler?.broadest(ids) ?? null); | ||||
| 369 | if (id && !g.ids.has(id)) id = null; | 398 | if (id && !g.ids.has(id)) id = null; | ||
| 370 | if (!id) { | 399 | if (!id) { | ||
| 371 | dict.ambiguousDropped++; | 400 | dict.ambiguousDropped++; | ||
modified
packages/connectors/src/sdk/lake.ts
+10 −2
@@ -130,13 +130,21 @@ export class RawLake { | |||||
| 130 | const [file, lineStr] = ref.split('#'); | 130 | const [file, lineStr] = ref.split('#'); | ||
| 131 | if (!file || lineStr === undefined) return null; | 131 | if (!file || lineStr === undefined) return null; | ||
| 132 | const target = Number(lineStr); | 132 | const target = Number(lineStr); | ||
| 133 | + // A missing or unreadable file must resolve to null: a stream error that only destroys the | ||||
| 134 | + // gunzip leaves the readline iterator pending forever, the event loop drains and the process | ||||
| 135 | + // exits 0 mid-run without finishing the ingest run (observed 2026-09-11 on a prod copy whose | ||||
| 136 | + // lake lives on another host). | ||||
| 137 | + if (!existsSync(file)) return null; | ||||
| 133 | const src = createReadStream(file); | 138 | const src = createReadStream(file); | ||
| 134 | const gunzip = createGunzip(); | 139 | const gunzip = createGunzip(); | ||
| 135 | − const swallow = () => gunzip.destroy(); | 140 | + const rl = createInterface({ input: gunzip, crlfDelay: Infinity }); | ||
| 141 | + const swallow = () => { | ||||
| 142 | + rl.close(); | ||||
| 143 | + gunzip.destroy(); | ||||
| 144 | + }; | ||||
| 136 | src.on('error', swallow); | 145 | src.on('error', swallow); | ||
| 137 | gunzip.on('error', swallow); | 146 | gunzip.on('error', swallow); | ||
| 138 | src.pipe(gunzip); | 147 | src.pipe(gunzip); | ||
| 139 | − const rl = createInterface({ input: gunzip, crlfDelay: Infinity }); | ||||
| 140 | try { | 148 | try { | ||
| 141 | let i = 0; | 149 | let i = 0; | ||
| 142 | for await (const line of rl) { | 150 | for await (const line of rl) { | ||
| 143 | 151 | ||||