import { sql } from 'drizzle-orm'; import type { Database } from '@cancerindex/database'; import { normalizeLabel } from '@cancerindex/shared'; export const DRUG_DUPLICATE_RULES_VERSION = 'ci-drug-duplicates-v1'; /** * Salt / ester / hydrate tokens that never change the molecule (CLAUDE.md §7). This is a copy of * `SALT_TOKENS` exported by `@cancerindex/connectors` (connectors/clinicaltrials/drugs.ts): the * ranking package must not depend on the connectors package, so the list is duplicated here on * purpose — keep both in sync when adding a token. */ export const DRUG_SALT_TOKENS = [ 'hydrochloride', 'dihydrochloride', 'hcl', 'sulfate', 'sulphate', 'acetate', 'sodium', 'disodium', 'potassium', 'calcium', 'magnesium', 'mesylate', 'mesilate', 'dimesylate', 'citrate', 'tartrate', 'bitartrate', 'maleate', 'malate', 'fumarate', 'phosphate', 'diphosphate', 'succinate', 'tosylate', 'besylate', 'bromide', 'chloride', 'lactate', 'gluconate', 'pamoate', 'trihydrate', 'dihydrate', 'monohydrate', 'hydrate', 'anhydrous', 'ditosylate', 'camsylate', 'hemihydrate', 'hydrobromide', 'nitrate', 'oxalate', 'propionate', 'valerate', 'decanoate', 'enanthate', 'undecanoate', 'cypionate', 'pivalate', 'isethionate', 'trifluoroacetate', ] as const; const SALTS = new Set(DRUG_SALT_TOKENS); /** Alias types that count as evidence of identity (synonyms are too noisy). */ export const DUPLICATE_ALIAS_TYPES = ['generic', 'brand', 'development_code'] as const; /** Two drugs sharing at least this many qualifying aliases are duplicate candidates. */ export const MIN_SHARED_ALIASES = 2; export interface DrugLite { id: string; name: string; aliases: Array<{ normalized: string; aliasType: string }>; } export type DuplicateReason = 'same_name' | 'salt_form' | 'shared_aliases'; export interface MergeCandidate { keepId: string; mergeId: string; reason: DuplicateReason; evidence: Record; } /** * Molecule key of a drug name: normalized label without parentheticals and without trailing salt * tokens ("Imatinib Mesylate" → "imatinib", "Sorafenib Tosylate" → "sorafenib", "JQ1" → "jq1"). * Deterministic; never touches the first token. */ export function moleculeKey(name: string): string { const tokens = normalizeLabel(name.replace(/\([^)]*\)/g, ' ')).split(' ').filter(Boolean); while (tokens.length > 1 && SALTS.has(tokens[tokens.length - 1]!)) tokens.pop(); return tokens.join(' '); } /** Development codes (TAK-788, AZD9291, RDEA 119, BAY 43-9006) carry digits; INNs do not. */ export function looksLikeDevelopmentCode(name: string): boolean { return /\d/.test(name); } /** * Keep = the base-molecule drug: an INN-like name over a development code, then fewer name * tokens, then the shorter name, then the smaller id. */ export function pickKeep(a: DrugLite, b: DrugLite): { keep: DrugLite; merge: DrugLite } { const ca = looksLikeDevelopmentCode(a.name); const cb = looksLikeDevelopmentCode(b.name); if (ca !== cb) return ca ? { keep: b, merge: a } : { keep: a, merge: b }; const ta = a.name.trim().split(/\s+/).length; const tb = b.name.trim().split(/\s+/).length; if (ta !== tb) return ta < tb ? { keep: a, merge: b } : { keep: b, merge: a }; if (a.name.length !== b.name.length) return a.name.length < b.name.length ? { keep: a, merge: b } : { keep: b, merge: a }; return a.id < b.id ? { keep: a, merge: b } : { keep: b, merge: a }; } /** * Duplicate candidates (SPEC §97): (1) equal molecule keys — identical names or salt-form variants — * and (2) ≥ MIN_SHARED_ALIASES shared generic/brand/development_code aliases. Pure; one candidate * per unordered pair, the strongest reason first (same_name > salt_form > shared_aliases). */ export function detectDuplicateCandidates(drugs: DrugLite[]): MergeCandidate[] { const byId = new Map(drugs.map((d) => [d.id, d])); const out = new Map(); const pairKey = (a: string, b: string) => (a < b ? `${a}|${b}` : `${b}|${a}`); // 1. Molecule key groups const groups = new Map(); for (const d of drugs) { const key = moleculeKey(d.name); if (!key) continue; groups.set(key, [...(groups.get(key) ?? []), d]); } for (const [key, members] of groups) { if (members.length < 2) continue; for (let i = 0; i < members.length; i++) { for (let j = i + 1; j < members.length; j++) { const a = members[i]!; const b = members[j]!; const { keep, merge } = pickKeep(a, b); const sameName = normalizeLabel(a.name) === normalizeLabel(b.name); out.set(pairKey(a.id, b.id), { keepId: keep.id, mergeId: merge.id, reason: sameName ? 'same_name' : 'salt_form', evidence: { rule: sameName ? 'identical normalized names' : 'equal names after stripping salt tokens', version: DRUG_DUPLICATE_RULES_VERSION, moleculeKey: key, keepName: keep.name, mergeName: merge.name, saltTokens: DRUG_SALT_TOKENS.length }, }); } } } // 2. Shared aliases const aliasOwners = new Map>(); for (const d of drugs) { for (const a of d.aliases) { if (!(DUPLICATE_ALIAS_TYPES as readonly string[]).includes(a.aliasType) || !a.normalized) continue; aliasOwners.set(a.normalized, (aliasOwners.get(a.normalized) ?? new Set()).add(d.id)); } } const shared = new Map>(); for (const [alias, owners] of aliasOwners) { if (owners.size < 2 || owners.size > 6) continue; // very common aliases (class words) are not identity evidence const ids = [...owners].sort(); for (let i = 0; i < ids.length; i++) for (let j = i + 1; j < ids.length; j++) shared.set(pairKey(ids[i]!, ids[j]!), (shared.get(pairKey(ids[i]!, ids[j]!)) ?? new Set()).add(alias)); } for (const [key, aliases] of shared) { if (aliases.size < MIN_SHARED_ALIASES || out.has(key)) continue; const [ia, ib] = key.split('|') as [string, string]; const a = byId.get(ia)!; const b = byId.get(ib)!; const { keep, merge } = pickKeep(a, b); out.set(key, { keepId: keep.id, mergeId: merge.id, reason: 'shared_aliases', evidence: { rule: `≥ ${MIN_SHARED_ALIASES} shared aliases of type ${DUPLICATE_ALIAS_TYPES.join('/')}`, version: DRUG_DUPLICATE_RULES_VERSION, sharedAliases: [...aliases].sort().slice(0, 20), keepName: keep.name, mergeName: merge.name }, }); } return [...out.values()].sort((x, y) => x.keepId.localeCompare(y.keepId) || x.mergeId.localeCompare(y.mergeId)); } export interface ProposeMergesResult { candidates: number; inserted: number; alreadyQueued: number; } /** * Insert `entity_merges` proposals (entity_type 'drug', status 'proposed') for every candidate pair * not already present in any status. Never merges anything (CLAUDE.md §70: reversible, curated). */ export async function proposeDrugMerges(db: Database): Promise { const rows = await db.execute<{ id: string; name: string; aliases: Array<{ normalized: string; aliasType: string }> | null }>(sql` SELECT d.id, d.name, (SELECT json_agg(json_build_object('normalized', a.normalized, 'aliasType', a.alias_type)) FROM drug_aliases a WHERE a.drug_id = d.id) AS aliases FROM drugs d`); const drugs: DrugLite[] = rows.map((r) => ({ id: r.id, name: r.name, aliases: r.aliases ?? [] })); const candidates = detectDuplicateCandidates(drugs); const existing = await db.execute<{ keep_id: string; merge_id: string }>(sql`SELECT keep_id, merge_id FROM entity_merges WHERE entity_type = 'drug'`); const seen = new Set(existing.map((e) => (e.keep_id < e.merge_id ? `${e.keep_id}|${e.merge_id}` : `${e.merge_id}|${e.keep_id}`))); let inserted = 0; let alreadyQueued = 0; for (const c of candidates) { const key = c.keepId < c.mergeId ? `${c.keepId}|${c.mergeId}` : `${c.mergeId}|${c.keepId}`; if (seen.has(key)) { alreadyQueued++; continue; } await db.execute(sql`INSERT INTO entity_merges (entity_type, keep_id, merge_id, evidence, status) VALUES ('drug', ${c.keepId}, ${c.mergeId}, ${JSON.stringify({ reason: c.reason, ...c.evidence })}::jsonb, 'proposed')`); seen.add(key); inserted++; } return { candidates: candidates.length, inserted, alreadyQueued }; }