TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { db, entityAliases, entities, eq, sourceEntities, sql, textArray } from "@websensor/db";2import { log } from "./config";34/**5 * Entity resolution. The source's own entities are always attached (subject). Then aliases6 * of all known entities are matched against the event text (word-boundary, case-insensitive,7 * longest alias first) so products/models/regulators named in the change are linked too.8 */9interface AliasIndex {10 loadedAt: number;11 byLen: { alias: string; re: RegExp; entityId: string; ambiguous: boolean }[];12 importance: Map<string, number>;13}1415let index: AliasIndex | null = null;1617/**18 * Aliases that are ordinary words. Matching them in free text would attach the entity to almost every19 * event ("FIRST" → Forum of Incident Response, "HAS" → Haute Autorité de santé, "make", "who", "cell"…).20 * They only count when written exactly as an upper-case acronym in the original text (WHO, NASA, FIRST),21 * or when the entity already belongs to the source (subject).22 */23export const AMBIGUOUS_ALIASES = new Set(24 "first has who make cell nature science shell orange apple bell block square meta oracle mint target gap next box slack zoom stripe uber lyft ring nest arm hp ge box tesla sky sun star time life people vice wired verge edge chrome safari brave signal telegram discord slack notion linear figma canva medium substack ghost dash zapier make render fly neon turso tigris planet scale cockroach confluent elastic redis mongo mongodb vercel netlify heroku render railway supabase firebase play store market cloud one plus max pro air mini studio watch music tv news post times globe mail star sun herald standard journal press daily weekly review register wire record hill point line frontier alliance alpha beta gamma delta omega origin echo nova atlas titan vector prism pulse radar forge anchor arc arrow beam bolt bond bridge canvas circle core crest crown drift ember flare flow fuse glow grid halo haven horizon iris jet key lift loop lumen mesh nexus node oasis orbit peak pillar pivot quest realm relay ridge rise river rock root sage scope shift spark sphere spire stone stream summit surge swift tide torch trace trail unity vault venture verge vista wave zenith bank trust fund capital global national international federal united american canadian european royal general standard central pacific atlantic western eastern northern southern".split(/\s+/),25);2627async function loadIndex(): Promise<AliasIndex> {28 if (index && Date.now() - index.loadedAt < 5 * 60_000) return index;29 const rows = await db.select({ alias: entityAliases.alias, entityId: entityAliases.entityId }).from(entityAliases);30 const ents = await db.select({ id: entities.id, importance: entities.importance }).from(entities);31 const byLen = rows32 .filter((r) => r.alias.length >= 3 && !/^\d+$/.test(r.alias))33 .map((r) => {34 const ambiguous = AMBIGUOUS_ALIASES.has(r.alias) || (r.alias.length <= 3 && !/[.-]/.test(r.alias));35 // Ambiguous / very short aliases: case-sensitive upper-case acronym match only (WHO, FIRST, HAS as acronym).36 const re = ambiguous ? new RegExp(`(^|[^\\p{L}\\p{N}])${escapeRe(r.alias.toUpperCase())}(?=$|[^\\p{L}\\p{N}])`, "u") : new RegExp(`(^|[^\\p{L}\\p{N}])${escapeRe(r.alias)}(?=$|[^\\p{L}\\p{N}])`, "iu");37 return { alias: r.alias, entityId: r.entityId, re, ambiguous };38 })39 .sort((a, b) => b.alias.length - a.alias.length);40 index = { loadedAt: Date.now(), byLen, importance: new Map(ents.map((e) => [e.id, e.importance])) };41 log.debug({ aliases: byLen.length }, "alias index loaded");42 return index;43}4445function escapeRe(s: string): string {46 return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");47}4849export function invalidateEntityIndex(): void {50 index = null;51}5253export async function resolveEntities(input: { sourceId: string; text: string; hints: string[] }): Promise<{ subject: string[]; mentioned: string[]; importance: number }> {54 const idx = await loadIndex();55 const subject = (await db.select({ entityId: sourceEntities.entityId }).from(sourceEntities).where(eq(sourceEntities.sourceId, input.sourceId))).map((r) => r.entityId);56 // Only the organization (not every product) is a default subject; products are attached when named.57 const orgSubjects = subject.filter((s) => s.startsWith("org_"));58 const mentioned = new Set<string>();59 const hay = `${input.text}\n${input.hints.join("\n")}`;60 const hayLower = hay.toLowerCase();61 for (const a of idx.byLen) {62 if (mentioned.size >= 12) break;63 if (orgSubjects.includes(a.entityId)) continue;64 if (!hayLower.includes(a.alias)) continue;65 if (a.ambiguous && !subject.includes(a.entityId) && !a.re.test(hay)) continue; // needs the exact acronym form66 if (a.re.test(hay)) mentioned.add(a.entityId);67 }68 // Products of the source named in the text are subjects too.69 for (const s of subject) if (!s.startsWith("org_") && mentioned.has(s)) mentioned.delete(s), orgSubjects.push(s);70 const all = [...orgSubjects, ...mentioned];71 const importance = all.length ? Math.max(...all.map((e) => idx.importance.get(e) ?? 40)) : 40;72 return { subject: orgSubjects, mentioned: [...mentioned], importance };73}7475/**76 * Maintenance: re-resolve "mentioned" entity links for recent events with the current alias rules77 * (drops links created by ambiguous aliases). Subjects (source entities) are never touched.78 */79export async function relinkMentionedEntities(days = 7): Promise<{ events: number; removed: number; added: number }> {80 invalidateEntityIndex();81 const idx = await loadIndex();82 const rows = await db.execute<{ id: string; source_id: string; title: string; summary: string; keywords: string[] | null }>(sql`select id, source_id, title, summary, keywords from events where detected_at >= now() - make_interval(days => ${days}) order by detected_at desc`);83 let removed = 0;84 let added = 0;85 const subjectsBySource = new Map<string, string[]>();86 for (const ev of rows.rows) {87 let subs = subjectsBySource.get(ev.source_id);88 if (!subs) {89 subs = (await db.select({ entityId: sourceEntities.entityId }).from(sourceEntities).where(eq(sourceEntities.sourceId, ev.source_id))).map((r) => r.entityId);90 subjectsBySource.set(ev.source_id, subs);91 }92 const hay = `${ev.title}\n${ev.summary}\n${(ev.keywords ?? []).join("\n")}`;93 const hayLower = hay.toLowerCase();94 const want = new Set<string>();95 for (const a of idx.byLen) {96 if (want.size >= 12) break;97 if (subs.some((s) => s.startsWith("org_") && s === a.entityId)) continue;98 if (!hayLower.includes(a.alias)) continue;99 if (a.ambiguous && !subs.includes(a.entityId) && !a.re.test(hay)) continue;100 if (a.re.test(hay)) want.add(a.entityId);101 }102 const current = (await db.execute<{ entity_id: string; role: string }>(sql`select entity_id, role from event_entities where event_id = ${ev.id}`)).rows;103 for (const c of current) {104 if (c.role === "mentioned" && !want.has(c.entity_id) && !subs.includes(c.entity_id)) {105 await db.execute(sql`delete from event_entities where event_id = ${ev.id} and entity_id = ${c.entity_id}`);106 removed++;107 }108 }109 for (const w of want) {110 if (!current.some((c) => c.entity_id === w)) {111 await db.execute(sql`insert into event_entities (event_id, entity_id, role) values (${ev.id}, ${w}, 'mentioned') on conflict do nothing`);112 added++;113 }114 }115 }116 // Recount entity totals + daily table from the corrected links.117 await db.execute(sql`update entities en set event_count = (select count(*) from event_entities ee where ee.entity_id = en.id), last_event_at = (select max(e.detected_at) from events e join event_entities ee on ee.event_id = e.id where ee.entity_id = en.id)`);118 await db.execute(sql`delete from entity_daily where day >= (now() at time zone 'UTC')::date - ${sql.raw(String(Math.max(1, Math.floor(days))))}`);119 await db.execute(sql`insert into entity_daily (entity_id, day, events, silent, breaking, max_importance)120 select ee.entity_id, (e.detected_at at time zone 'UTC')::date, count(*), sum(case when e.silent_change then 1 else 0 end), sum(case when e.importance >= 80 then 1 else 0 end), max(e.importance)121 from events e join event_entities ee on ee.event_id = e.id where e.detected_at >= now() - make_interval(days => ${days}) group by 1, 2122 on conflict (entity_id, day) do update set events = excluded.events, silent = excluded.silent, breaking = excluded.breaking, max_importance = excluded.max_importance`);123 log.info({ events: rows.rows.length, removed, added }, "mentioned entity links re-resolved");124 return { events: rows.rows.length, removed, added };125}126127export async function bumpEntityCounters(entityIds: string[], at: Date, opts: { silent?: boolean; importance?: number } = {}): Promise<void> {128 if (!entityIds.length) return;129 await db.execute(sql`update entities set event_count = event_count + 1, last_event_at = ${at} where id = any(${textArray(entityIds)})`);130 const imp = Number(opts.importance ?? 0);131 await db132 .execute(133 sql`insert into entity_daily (entity_id, day, events, silent, breaking, max_importance)134 select unnest(${textArray(entityIds)}), (${at} at time zone 'UTC')::date, 1, ${opts.silent ? 1 : 0}, ${imp >= 80 ? 1 : 0}, ${imp}135 on conflict (entity_id, day) do update set events = entity_daily.events + 1, silent = entity_daily.silent + ${opts.silent ? 1 : 0}, breaking = entity_daily.breaking + ${imp >= 80 ? 1 : 0}, max_importance = greatest(entity_daily.max_importance, ${imp})`,136 )137 .catch((e) => log.warn({ err: (e as Error).message }, "entity_daily update failed"));138}139