"""Event clustering (spec §23): the same corporate event seen on several surfaces (newsroom + feed + homepage, leadership + about…) is folded into one canonical event with aggregated sources. Corroboration raises confidence; later duplicates keep their own row (`status='duplicate'`, `cluster_id`) so provenance is never lost. cluster_key = sha(company_id, event_subtype, normalised entity key, 7-day window bucket) Only `services/events.py` and `services/llm/enrich.py` call `attach_to_cluster`; the API reads `event_clusters` / `events.cluster_id`. """ from __future__ import annotations import re import unicodedata from datetime import UTC, datetime from typing import Any from sqlalchemy.ext.asyncio import AsyncConnection from companyatlas.db import execute, fetch_one, jsonb from companyatlas.ids import new_id, stable_hash CLUSTER_WINDOW_DAYS = 7 CORROBORATION_BONUS = 0.03 # confidence bump per extra corroborating surface CONFIDENCE_CAP = 0.99 _non_alnum = re.compile(r"[^a-z0-9]+") def normalize_entity_key(value: str) -> str: """Lowercase ASCII, punctuation collapsed — makes "Jane Doe" / "jane doe" / "Jane DOE," the same entity across surfaces.""" s = unicodedata.normalize("NFKD", value or "").encode("ascii", "ignore").decode("ascii").lower() s = _non_alnum.sub(" ", s).strip() return re.sub(r"\s+", " ", s)[:160] def window_bucket(at: datetime, *, days: int = CLUSTER_WINDOW_DAYS) -> int: if at.tzinfo is None: at = at.replace(tzinfo=UTC) return int(at.timestamp() // (days * 86400)) def cluster_key_for(company_id: str, event_subtype: str, entity_key: str, detected_at: datetime) -> str: return stable_hash(company_id, event_subtype, normalize_entity_key(entity_key), str(window_bucket(detected_at)), length=32) async def attach_to_cluster(conn: AsyncConnection, event: dict[str, Any], *, entity_key: str) -> tuple[str, bool]: """Attach a freshly inserted event to its cluster. Returns (cluster_id, is_duplicate). First event of a cluster becomes canonical. A later event from another sensor/surface is marked `duplicate`, the canonical event gains a corroboration source and a confidence bump. Re-processing the same change (same dedupe key) never reaches here. """ key = cluster_key_for(event["company_id"], event["event_subtype"], entity_key, event["detected_at"]) existing = await fetch_one(conn, "select * from event_clusters where cluster_key = :k for update", k=key) if existing is None: cluster_id = new_id("cluster") await execute(conn, """ insert into event_clusters (id, company_id, cluster_key, event_type, event_subtype, title, first_detected_at, last_detected_at, source_count, surfaces, confidence, canonical_event_id) values (:id, :company_id, :key, :event_type, :event_subtype, :title, :at, :at, 1, cast(:surfaces as text[]), :confidence, :event_id) on conflict (cluster_key) do nothing""", id=cluster_id, company_id=event["company_id"], key=key, event_type=event["event_type"], event_subtype=event["event_subtype"], title=event["title"], at=event["detected_at"], surfaces=[event.get("surface") or "other"], confidence=float(event["confidence"]), event_id=event["id"]) await execute(conn, "update events set cluster_id = :c where id = :e", c=cluster_id, e=event["id"]) return cluster_id, False cluster_id = existing["id"] canonical_id = existing["canonical_event_id"] if canonical_id == event["id"]: return cluster_id, False surfaces: list[str] = list(existing["surfaces"] or []) surface = event.get("surface") or "other" same_source = surface in surfaces and event.get("sensor_id") is not None and await fetch_one( conn, "select 1 from events where id = :c and sensor_id = :s", c=canonical_id, s=event["sensor_id"]) is not None if surface not in surfaces: surfaces.append(surface) extra_surfaces = max(0, len(surfaces) - 1) canonical = await fetch_one(conn, "select confidence, payload from events where id = :id", id=canonical_id) base_conf = float(canonical["confidence"]) if canonical else float(existing["confidence"]) new_conf = min(CONFIDENCE_CAP, max(base_conf, float(event["confidence"])) + CORROBORATION_BONUS * extra_surfaces) await execute(conn, """ update event_clusters set source_count = source_count + 1, surfaces = cast(:surfaces as text[]), confidence = :conf, last_detected_at = greatest(last_detected_at, cast(:at as timestamptz)) where id = :id""", surfaces=surfaces, conf=new_conf, at=event["detected_at"], id=cluster_id) await execute(conn, "update events set cluster_id = :c, status = 'duplicate' where id = :e", c=cluster_id, e=event["id"]) payload = dict((canonical or {}).get("payload") or {}) sources = list(payload.get("sources") or []) src = {"event_id": event["id"], "sensor_id": event.get("sensor_id"), "surface": surface, "source_url": event.get("source_url"), "detected_at": event["detected_at"].isoformat() if isinstance(event["detected_at"], datetime) else str(event["detected_at"])} if src not in sources: sources.append(src) payload["sources"] = sources[:50] payload["corroborations"] = len(sources) await execute(conn, """ update events set confidence = :conf, confidence_label = :label, payload = cast(:payload as jsonb) where id = :id""", conf=new_conf, label=_label(new_conf), payload=jsonb(payload), id=canonical_id) if event.get("source_url") and not same_source: await execute(conn, """ insert into event_sources (event_id, sensor_id, source_url, snapshot_id, surface, detected_at, kind) values (:event_id, :sensor_id, :url, :snap, :surface, :at, 'corroboration') on conflict do nothing""", event_id=canonical_id, sensor_id=event.get("sensor_id"), url=event["source_url"], snap=event.get("snapshot_after"), surface=surface, at=event["detected_at"]) return cluster_id, True def _label(confidence: float) -> str: from companyatlas.taxonomy import confidence_label return confidence_label(confidence) __all__ = ["CLUSTER_WINDOW_DAYS", "CORROBORATION_BONUS", "attach_to_cluster", "cluster_key_for", "normalize_entity_key", "window_bucket"]