spb/cancerindex

1import { sql } from 'drizzle-orm';2import type { Database } from '@cancerindex/database';3import { normalizeLabel } from '@cancerindex/shared';45export const DRUG_DUPLICATE_RULES_VERSION = 'ci-drug-duplicates-v1';67/**8 * Salt / ester / hydrate tokens that never change the molecule (CLAUDE.md §7). This is a copy of9 * `SALT_TOKENS` exported by `@cancerindex/connectors` (connectors/clinicaltrials/drugs.ts): the10 * ranking package must not depend on the connectors package, so the list is duplicated here on11 * purpose — keep both in sync when adding a token.12 */13export const DRUG_SALT_TOKENS = [14  'hydrochloride', 'dihydrochloride', 'hcl', 'sulfate', 'sulphate', 'acetate', 'sodium', 'disodium', 'potassium', 'calcium', 'magnesium', 'mesylate', 'mesilate', 'dimesylate', 'citrate', 'tartrate', 'bitartrate', 'maleate', 'malate', 'fumarate', 'phosphate', 'diphosphate', 'succinate', 'tosylate', 'besylate', 'bromide', 'chloride', 'lactate', 'gluconate', 'pamoate', 'trihydrate', 'dihydrate', 'monohydrate', 'hydrate', 'anhydrous', 'ditosylate', 'camsylate', 'hemihydrate', 'hydrobromide', 'nitrate', 'oxalate', 'propionate', 'valerate', 'decanoate', 'enanthate', 'undecanoate', 'cypionate', 'pivalate', 'isethionate', 'trifluoroacetate',15] as const;16const SALTS = new Set<string>(DRUG_SALT_TOKENS);1718/** Alias types that count as evidence of identity (synonyms are too noisy). */19export const DUPLICATE_ALIAS_TYPES = ['generic', 'brand', 'development_code'] as const;20/** Two drugs sharing at least this many qualifying aliases are duplicate candidates. */21export const MIN_SHARED_ALIASES = 2;2223export interface DrugLite {24  id: string;25  name: string;26  aliases: Array<{ normalized: string; aliasType: string }>;27}2829export type DuplicateReason = 'same_name' | 'salt_form' | 'shared_aliases';3031export interface MergeCandidate {32  keepId: string;33  mergeId: string;34  reason: DuplicateReason;35  evidence: Record<string, unknown>;36}3738/**39 * Molecule key of a drug name: normalized label without parentheticals and without trailing salt40 * tokens ("Imatinib Mesylate" → "imatinib", "Sorafenib Tosylate" → "sorafenib", "JQ1" → "jq1").41 * Deterministic; never touches the first token.42 */43export function moleculeKey(name: string): string {44  const tokens = normalizeLabel(name.replace(/\([^)]*\)/g, ' ')).split(' ').filter(Boolean);45  while (tokens.length > 1 && SALTS.has(tokens[tokens.length - 1]!)) tokens.pop();46  return tokens.join(' ');47}4849/** Development codes (TAK-788, AZD9291, RDEA 119, BAY 43-9006) carry digits; INNs do not. */50export function looksLikeDevelopmentCode(name: string): boolean {51  return /\d/.test(name);52}5354/**55 * Keep = the base-molecule drug: an INN-like name over a development code, then fewer name56 * tokens, then the shorter name, then the smaller id.57 */58export function pickKeep(a: DrugLite, b: DrugLite): { keep: DrugLite; merge: DrugLite } {59  const ca = looksLikeDevelopmentCode(a.name);60  const cb = looksLikeDevelopmentCode(b.name);61  if (ca !== cb) return ca ? { keep: b, merge: a } : { keep: a, merge: b };62  const ta = a.name.trim().split(/\s+/).length;63  const tb = b.name.trim().split(/\s+/).length;64  if (ta !== tb) return ta < tb ? { keep: a, merge: b } : { keep: b, merge: a };65  if (a.name.length !== b.name.length) return a.name.length < b.name.length ? { keep: a, merge: b } : { keep: b, merge: a };66  return a.id < b.id ? { keep: a, merge: b } : { keep: b, merge: a };67}6869/**70 * Duplicate candidates (SPEC §97): (1) equal molecule keys — identical names or salt-form variants —71 * and (2) ≥ MIN_SHARED_ALIASES shared generic/brand/development_code aliases. Pure; one candidate72 * per unordered pair, the strongest reason first (same_name > salt_form > shared_aliases).73 */74export function detectDuplicateCandidates(drugs: DrugLite[]): MergeCandidate[] {75  const byId = new Map(drugs.map((d) => [d.id, d]));76  const out = new Map<string, MergeCandidate>();77  const pairKey = (a: string, b: string) => (a < b ? `${a}|${b}` : `${b}|${a}`);7879  // 1. Molecule key groups80  const groups = new Map<string, DrugLite[]>();81  for (const d of drugs) {82    const key = moleculeKey(d.name);83    if (!key) continue;84    groups.set(key, [...(groups.get(key) ?? []), d]);85  }86  for (const [key, members] of groups) {87    if (members.length < 2) continue;88    for (let i = 0; i < members.length; i++) {89      for (let j = i + 1; j < members.length; j++) {90        const a = members[i]!;91        const b = members[j]!;92        const { keep, merge } = pickKeep(a, b);93        const sameName = normalizeLabel(a.name) === normalizeLabel(b.name);94        out.set(pairKey(a.id, b.id), {95          keepId: keep.id,96          mergeId: merge.id,97          reason: sameName ? 'same_name' : 'salt_form',98          evidence: { rule: sameName ? 'identical normalized names' : 'equal names after stripping salt tokens', version: DRUG_DUPLICATE_RULES_VERSION, moleculeKey: key, keepName: keep.name, mergeName: merge.name, saltTokens: DRUG_SALT_TOKENS.length },99        });100      }101    }102  }103104  // 2. Shared aliases105  const aliasOwners = new Map<string, Set<string>>();106  for (const d of drugs) {107    for (const a of d.aliases) {108      if (!(DUPLICATE_ALIAS_TYPES as readonly string[]).includes(a.aliasType) || !a.normalized) continue;109      aliasOwners.set(a.normalized, (aliasOwners.get(a.normalized) ?? new Set()).add(d.id));110    }111  }112  const shared = new Map<string, Set<string>>();113  for (const [alias, owners] of aliasOwners) {114    if (owners.size < 2 || owners.size > 6) continue; // very common aliases (class words) are not identity evidence115    const ids = [...owners].sort();116    for (let i = 0; i < ids.length; i++) for (let j = i + 1; j < ids.length; j++) shared.set(pairKey(ids[i]!, ids[j]!), (shared.get(pairKey(ids[i]!, ids[j]!)) ?? new Set()).add(alias));117  }118  for (const [key, aliases] of shared) {119    if (aliases.size < MIN_SHARED_ALIASES || out.has(key)) continue;120    const [ia, ib] = key.split('|') as [string, string];121    const a = byId.get(ia)!;122    const b = byId.get(ib)!;123    const { keep, merge } = pickKeep(a, b);124    out.set(key, {125      keepId: keep.id,126      mergeId: merge.id,127      reason: 'shared_aliases',128      evidence: { rule: `≥ ${MIN_SHARED_ALIASES} shared aliases of type ${DUPLICATE_ALIAS_TYPES.join('/')}`, version: DRUG_DUPLICATE_RULES_VERSION, sharedAliases: [...aliases].sort().slice(0, 20), keepName: keep.name, mergeName: merge.name },129    });130  }131  return [...out.values()].sort((x, y) => x.keepId.localeCompare(y.keepId) || x.mergeId.localeCompare(y.mergeId));132}133134export interface ProposeMergesResult {135  candidates: number;136  inserted: number;137  alreadyQueued: number;138}139140/**141 * Insert `entity_merges` proposals (entity_type 'drug', status 'proposed') for every candidate pair142 * not already present in any status. Never merges anything (CLAUDE.md §70: reversible, curated).143 */144export async function proposeDrugMerges(db: Database): Promise<ProposeMergesResult> {145  const rows = await db.execute<{ id: string; name: string; aliases: Array<{ normalized: string; aliasType: string }> | null }>(sql`146    SELECT d.id, d.name, (SELECT json_agg(json_build_object('normalized', a.normalized, 'aliasType', a.alias_type)) FROM drug_aliases a WHERE a.drug_id = d.id) AS aliases147    FROM drugs d`);148  const drugs: DrugLite[] = rows.map((r) => ({ id: r.id, name: r.name, aliases: r.aliases ?? [] }));149  const candidates = detectDuplicateCandidates(drugs);150  const existing = await db.execute<{ keep_id: string; merge_id: string }>(sql`SELECT keep_id, merge_id FROM entity_merges WHERE entity_type = 'drug'`);151  const seen = new Set(existing.map((e) => (e.keep_id < e.merge_id ? `${e.keep_id}|${e.merge_id}` : `${e.merge_id}|${e.keep_id}`)));152  let inserted = 0;153  let alreadyQueued = 0;154  for (const c of candidates) {155    const key = c.keepId < c.mergeId ? `${c.keepId}|${c.mergeId}` : `${c.mergeId}|${c.keepId}`;156    if (seen.has(key)) {157      alreadyQueued++;158      continue;159    }160    await db.execute(sql`INSERT INTO entity_merges (entity_type, keep_id, merge_id, evidence, status) VALUES ('drug', ${c.keepId}, ${c.mergeId}, ${JSON.stringify({ reason: c.reason, ...c.evidence })}::jsonb, 'proposed')`);161    seen.add(key);162    inserted++;163  }164  return { candidates: candidates.length, inserted, alreadyQueued };165}166