import { db, entityAliases, entities, eq, sourceEntities, sql, textArray } from "@websensor/db"; import { log } from "./config"; /** * Entity resolution. The source's own entities are always attached (subject). Then aliases * of all known entities are matched against the event text (word-boundary, case-insensitive, * longest alias first) so products/models/regulators named in the change are linked too. */ interface AliasIndex { loadedAt: number; byLen: { alias: string; re: RegExp; entityId: string; ambiguous: boolean }[]; importance: Map; } let index: AliasIndex | null = null; /** * Aliases that are ordinary words. Matching them in free text would attach the entity to almost every * event ("FIRST" → Forum of Incident Response, "HAS" → Haute Autorité de santé, "make", "who", "cell"…). * They only count when written exactly as an upper-case acronym in the original text (WHO, NASA, FIRST), * or when the entity already belongs to the source (subject). */ export const AMBIGUOUS_ALIASES = new Set( "first has who make cell nature science shell orange apple bell block square meta oracle mint target gap next box slack zoom stripe uber lyft ring nest arm hp ge box tesla sky sun star time life people vice wired verge edge chrome safari brave signal telegram discord slack notion linear figma canva medium substack ghost dash zapier make render fly neon turso tigris planet scale cockroach confluent elastic redis mongo mongodb vercel netlify heroku render railway supabase firebase play store market cloud one plus max pro air mini studio watch music tv news post times globe mail star sun herald standard journal press daily weekly review register wire record hill point line frontier alliance alpha beta gamma delta omega origin echo nova atlas titan vector prism pulse radar forge anchor arc arrow beam bolt bond bridge canvas circle core crest crown drift ember flare flow fuse glow grid halo haven horizon iris jet key lift loop lumen mesh nexus node oasis orbit peak pillar pivot quest realm relay ridge rise river rock root sage scope shift spark sphere spire stone stream summit surge swift tide torch trace trail unity vault venture verge vista wave zenith bank trust fund capital global national international federal united american canadian european royal general standard central pacific atlantic western eastern northern southern".split(/\s+/), ); async function loadIndex(): Promise { if (index && Date.now() - index.loadedAt < 5 * 60_000) return index; const rows = await db.select({ alias: entityAliases.alias, entityId: entityAliases.entityId }).from(entityAliases); const ents = await db.select({ id: entities.id, importance: entities.importance }).from(entities); const byLen = rows .filter((r) => r.alias.length >= 3 && !/^\d+$/.test(r.alias)) .map((r) => { const ambiguous = AMBIGUOUS_ALIASES.has(r.alias) || (r.alias.length <= 3 && !/[.-]/.test(r.alias)); // Ambiguous / very short aliases: case-sensitive upper-case acronym match only (WHO, FIRST, HAS as acronym). const re = ambiguous ? new RegExp(`(^|[^\\p{L}\\p{N}])${escapeRe(r.alias.toUpperCase())}(?=$|[^\\p{L}\\p{N}])`, "u") : new RegExp(`(^|[^\\p{L}\\p{N}])${escapeRe(r.alias)}(?=$|[^\\p{L}\\p{N}])`, "iu"); return { alias: r.alias, entityId: r.entityId, re, ambiguous }; }) .sort((a, b) => b.alias.length - a.alias.length); index = { loadedAt: Date.now(), byLen, importance: new Map(ents.map((e) => [e.id, e.importance])) }; log.debug({ aliases: byLen.length }, "alias index loaded"); return index; } function escapeRe(s: string): string { return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } export function invalidateEntityIndex(): void { index = null; } export async function resolveEntities(input: { sourceId: string; text: string; hints: string[] }): Promise<{ subject: string[]; mentioned: string[]; importance: number }> { const idx = await loadIndex(); const subject = (await db.select({ entityId: sourceEntities.entityId }).from(sourceEntities).where(eq(sourceEntities.sourceId, input.sourceId))).map((r) => r.entityId); // Only the organization (not every product) is a default subject; products are attached when named. const orgSubjects = subject.filter((s) => s.startsWith("org_")); const mentioned = new Set(); const hay = `${input.text}\n${input.hints.join("\n")}`; const hayLower = hay.toLowerCase(); for (const a of idx.byLen) { if (mentioned.size >= 12) break; if (orgSubjects.includes(a.entityId)) continue; if (!hayLower.includes(a.alias)) continue; if (a.ambiguous && !subject.includes(a.entityId) && !a.re.test(hay)) continue; // needs the exact acronym form if (a.re.test(hay)) mentioned.add(a.entityId); } // Products of the source named in the text are subjects too. for (const s of subject) if (!s.startsWith("org_") && mentioned.has(s)) mentioned.delete(s), orgSubjects.push(s); const all = [...orgSubjects, ...mentioned]; const importance = all.length ? Math.max(...all.map((e) => idx.importance.get(e) ?? 40)) : 40; return { subject: orgSubjects, mentioned: [...mentioned], importance }; } /** * Maintenance: re-resolve "mentioned" entity links for recent events with the current alias rules * (drops links created by ambiguous aliases). Subjects (source entities) are never touched. */ export async function relinkMentionedEntities(days = 7): Promise<{ events: number; removed: number; added: number }> { invalidateEntityIndex(); const idx = await loadIndex(); const rows = await db.execute<{ id: string; source_id: string; title: string; summary: string; keywords: string[] | null }>(sql`select id, source_id, title, summary, keywords from events where detected_at >= now() - make_interval(days => ${days}) order by detected_at desc`); let removed = 0; let added = 0; const subjectsBySource = new Map(); for (const ev of rows.rows) { let subs = subjectsBySource.get(ev.source_id); if (!subs) { subs = (await db.select({ entityId: sourceEntities.entityId }).from(sourceEntities).where(eq(sourceEntities.sourceId, ev.source_id))).map((r) => r.entityId); subjectsBySource.set(ev.source_id, subs); } const hay = `${ev.title}\n${ev.summary}\n${(ev.keywords ?? []).join("\n")}`; const hayLower = hay.toLowerCase(); const want = new Set(); for (const a of idx.byLen) { if (want.size >= 12) break; if (subs.some((s) => s.startsWith("org_") && s === a.entityId)) continue; if (!hayLower.includes(a.alias)) continue; if (a.ambiguous && !subs.includes(a.entityId) && !a.re.test(hay)) continue; if (a.re.test(hay)) want.add(a.entityId); } const current = (await db.execute<{ entity_id: string; role: string }>(sql`select entity_id, role from event_entities where event_id = ${ev.id}`)).rows; for (const c of current) { if (c.role === "mentioned" && !want.has(c.entity_id) && !subs.includes(c.entity_id)) { await db.execute(sql`delete from event_entities where event_id = ${ev.id} and entity_id = ${c.entity_id}`); removed++; } } for (const w of want) { if (!current.some((c) => c.entity_id === w)) { await db.execute(sql`insert into event_entities (event_id, entity_id, role) values (${ev.id}, ${w}, 'mentioned') on conflict do nothing`); added++; } } } // Recount entity totals + daily table from the corrected links. await db.execute(sql`update entities en set event_count = (select count(*) from event_entities ee where ee.entity_id = en.id), last_event_at = (select max(e.detected_at) from events e join event_entities ee on ee.event_id = e.id where ee.entity_id = en.id)`); await db.execute(sql`delete from entity_daily where day >= (now() at time zone 'UTC')::date - ${sql.raw(String(Math.max(1, Math.floor(days))))}`); await db.execute(sql`insert into entity_daily (entity_id, day, events, silent, breaking, max_importance) select ee.entity_id, (e.detected_at at time zone 'UTC')::date, count(*), sum(case when e.silent_change then 1 else 0 end), sum(case when e.importance >= 80 then 1 else 0 end), max(e.importance) from events e join event_entities ee on ee.event_id = e.id where e.detected_at >= now() - make_interval(days => ${days}) group by 1, 2 on conflict (entity_id, day) do update set events = excluded.events, silent = excluded.silent, breaking = excluded.breaking, max_importance = excluded.max_importance`); log.info({ events: rows.rows.length, removed, added }, "mentioned entity links re-resolved"); return { events: rows.rows.length, removed, added }; } export async function bumpEntityCounters(entityIds: string[], at: Date, opts: { silent?: boolean; importance?: number } = {}): Promise { if (!entityIds.length) return; await db.execute(sql`update entities set event_count = event_count + 1, last_event_at = ${at} where id = any(${textArray(entityIds)})`); const imp = Number(opts.importance ?? 0); await db .execute( sql`insert into entity_daily (entity_id, day, events, silent, breaking, max_importance) select unnest(${textArray(entityIds)}), (${at} at time zone 'UTC')::date, 1, ${opts.silent ? 1 : 0}, ${imp >= 80 ? 1 : 0}, ${imp} on conflict (entity_id, day) do update set events = entity_daily.events + 1, silent = entity_daily.silent + ${opts.silent ? 1 : 0}, breaking = entity_daily.breaking + ${imp >= 80 ? 1 : 0}, max_importance = greatest(entity_daily.max_importance, ${imp})`, ) .catch((e) => log.warn({ err: (e as Error).message }, "entity_daily update failed")); }