spb/company-atlas

1"""Block-level diff + significance (spec §19–20).23    compare(before_blocks, after_blocks, surface=…, before_text=…, after_text=…, structured_delta=…, history=…) -> BlockDiff45Matching: exact block `key` → simhash near-match (Hamming ≤ `NEAR_HAMMING`) → rapidfuzz ratio ≥ `MODIFIED_MIN_RATIO` for modified6blocks; everything else is added/removed. Moves are matched blocks whose relative order changed (LIS on positions).78Significance ∈ [0, 1] is deterministic and explained by `reasons`: weighted changed share of *content* blocks (nav/footer/header do not9count), surface importance, boosts for typed deltas (prices, people, jobs, locations, products, news), novelty from the sensor's10history, and hard caps for classic noise (footer-only churn, tiny edits, counters). Bands (spec §20): < 0.20 noise · < 0.40 minor11· < 0.65 meaningful · < 0.85 major · ≥ 0.85 critical.12"""13from __future__ import annotations1415from collections.abc import Mapping16from typing import Any1718from rapidfuzz import fuzz19from rapidfuzz.distance import Indel2021from companyatlas.sdk.models import Block, BlockDelta, BlockDiff22from companyatlas.sdk.normalize import LOW_VALUE_KINDS, hamming, normalized_text23from companyatlas.taxonomy import SURFACE_IMPORTANCE2425DIFF_VERSION = "diff-v1"2627NEAR_HAMMING = 6                 # simhash distance considered "same block, edited"28MODIFIED_MIN_RATIO = 0.55        # rapidfuzz ratio (0–1) for a near-match to count as modified rather than add+remove29TINY_EDIT_SIMILARITY = 0.97      # modified blocks at/above this similarity are cosmetic30BLOCK_LEN_CAP = 1500             # a single giant block must not dominate the share31TEXT_COMPARE_CAP = 60_000        # characters compared for text_delta_ratio3233# Typed-delta floors: when the pipeline reconciled real entities, the change is at least this significant (spec §20 "affected entities").34TYPED_FLOORS: dict[str, float] = {35    "plans.price_changed": 0.70, "plans.added": 0.60, "plans.removed": 0.60,36    "people.added_executive": 0.68, "people.removed_executive": 0.68, "people.added": 0.50, "people.removed": 0.50, "people.title_changed": 0.55,37    "jobs.added": 0.45, "jobs.removed": 0.42,38    "locations.new_countries": 0.70, "locations.added": 0.55, "locations.removed": 0.52,39    "products.added": 0.55, "products.removed": 0.55,40    "news.added": 0.45,41    "meta.title_changed": 0.30,42}43JOBS_SCALE_FLOOR = 0.62          # many jobs added/removed at once (≥ JOBS_SCALE_COUNT or ≥ 25 % of open jobs)44JOBS_SCALE_COUNT = 1045NOVELTY_STABLE_RUNS = 20         # consecutive unchanged runs before a change counts as "novel"46NOVELTY_BOOST = 1.1047CHURN_RATE = 0.5                 # changes per observation above which the page is considered volatile48CHURN_PENALTY = 0.8549CONTENT_BASE, CONTENT_SPAN, CONTENT_EXP = 0.12, 0.72, 0.7   # pure content share maps to [0.12, 0.84] — critical needs typed deltas / novelty50NOISE_CAP = 0.1951LOW_VALUE_CAP = 0.15525354def _content_len(b: Block) -> float:55    return float(min(len(b.text), BLOCK_LEN_CAP)) * max(0.05, b.weight)565758def _delta(b: Block, *, before: str | None, after: str | None, similarity: float | None = None) -> BlockDelta:59    return BlockDelta(key=b.key, kind=b.kind, path=b.path, before=before, after=after, weight=b.weight, similarity=similarity)606162def _ratio(a: str, b: str) -> float:63    if not a and not b:64        return 1.065    return fuzz.ratio(normalized_text(a)[:BLOCK_LEN_CAP * 2], normalized_text(b)[:BLOCK_LEN_CAP * 2]) / 100.0666768def _lis_positions(seq: list[int]) -> set[int]:69    """Indices (into seq) of one longest strictly increasing subsequence — items outside it moved."""70    if not seq:71        return set()72    import bisect7374    tails: list[int] = []75    tails_idx: list[int] = []76    prev = [-1] * len(seq)77    for i, v in enumerate(seq):78        pos = bisect.bisect_left(tails, v)79        if pos == len(tails):80            tails.append(v)81            tails_idx.append(i)82        else:83            tails[pos] = v84            tails_idx[pos] = i85        prev[i] = tails_idx[pos - 1] if pos > 0 else -186    out: set[int] = set()87    k = tails_idx[-1]88    while k != -1:89        out.add(k)90        k = prev[k]91    return out929394def match_blocks(before: list[Block], after: list[Block]) -> tuple[list[tuple[Block, Block]], list[Block], list[Block]]:95    """Return (pairs, removed, added). Pairs include exact-key matches and near matches (simhash / fuzzy)."""96    by_key_after: dict[str, Block] = {}97    for b in after:98        by_key_after.setdefault(b.key, b)99    pairs: list[tuple[Block, Block]] = []100    used_after: set[int] = set()101    unmatched_before: list[Block] = []102    for b in before:103        a = by_key_after.get(b.key)104        if a is not None and id(a) not in used_after:105            pairs.append((b, a))106            used_after.add(id(a))107        else:108            unmatched_before.append(b)109    remaining_after = [a for a in after if id(a) not in used_after]110    # near matches: same kind, closest simhash within NEAR_HAMMING, tie-break by same path then fuzzy ratio111    still_before: list[Block] = []112    for b in unmatched_before:113        best: tuple[float, int, Block] | None = None114        for a in remaining_after:115            if a.kind != b.kind or id(a) in used_after:116                continue117            d = hamming(a.simhash, b.simhash) if (a.simhash and b.simhash) else 64118            if d <= NEAR_HAMMING:119                score = (1.0 if a.path == b.path else 0.0, -d)120                if best is None or score > (best[0], -best[1]):121                    best = (score[0], d, a)122        if best is None:123            # fuzzy fallback for short/edited blocks (bounded: only blocks of the same kind and path)124            cands = [a for a in remaining_after if a.kind == b.kind and a.path == b.path and id(a) not in used_after]125            best_r = 0.0126            best_a: Block | None = None127            for a in cands[:60]:128                r = _ratio(b.text, a.text)129                if r > best_r:130                    best_r, best_a = r, a131            if best_a is not None and best_r >= MODIFIED_MIN_RATIO:132                best = (0.0, 64, best_a)133        if best is not None:134            pairs.append((b, best[2]))135            used_after.add(id(best[2]))136        else:137            still_before.append(b)138    added = [a for a in after if id(a) not in used_after]139    return pairs, still_before, added140141142def compare(before_blocks: list[Block], after_blocks: list[Block], *, surface: str, before_text: str = "", after_text: str = "",143            structured_delta: Mapping[str, Any] | None = None, history: Mapping[str, Any] | None = None) -> BlockDiff:144    sd = structured_delta or {}145    hist = history or {}146    diff = BlockDiff()147    pairs, removed, added = match_blocks(before_blocks, after_blocks)148149    modified_pairs: list[tuple[Block, Block, float]] = []150    for b, a in pairs:151        if b.hash == a.hash:152            continue153        sim = _ratio(b.text, a.text)154        modified_pairs.append((b, a, sim))155    # moved: exact matches whose order is not increasing156    order_after = {id(a): a.order for _b, a in pairs}157    seq = [order_after[id(a)] for _b, a in sorted(pairs, key=lambda p: p[0].order)]158    keep = _lis_positions(seq)159    sorted_pairs = sorted(pairs, key=lambda p: p[0].order)160    for i, (b, _a) in enumerate(sorted_pairs):161        if i not in keep:162            diff.moved.append(b.key)163164    diff.added = [_delta(a, before=None, after=a.text) for a in added]165    diff.removed = [_delta(b, before=b.text, after=None) for b in removed]166    diff.modified = [_delta(a, before=b.text, after=a.text, similarity=round(sim, 4)) for b, a, sim in modified_pairs]167168    # ---------------------------------------------------------------- weighted changed share (content blocks only)169    total_before = sum(_content_len(b) for b in before_blocks if b.kind not in LOW_VALUE_KINDS)170    total_after = sum(_content_len(b) for b in after_blocks if b.kind not in LOW_VALUE_KINDS)171    total = max(total_before, total_after, 1.0)172    changed = 0.0173    low_value_changed = 0.0174    for a in added:175        if a.kind in LOW_VALUE_KINDS:176            low_value_changed += _content_len(a)177        else:178            changed += _content_len(a)179    for b in removed:180        if b.kind in LOW_VALUE_KINDS:181            low_value_changed += _content_len(b)182        else:183            changed += _content_len(b)184    for b, a, sim in modified_pairs:185        amount = max(_content_len(a), _content_len(b)) * (1.0 - sim)186        if a.kind in LOW_VALUE_KINDS:187            low_value_changed += amount188        else:189            changed += amount190    changed_share = min(1.0, changed / total)191    diff.similarity = round(max(0.0, 1.0 - changed_share), 4)192193    # ---------------------------------------------------------------- text delta ratio194    if before_text or after_text:195        nb = normalized_text(before_text)[:TEXT_COMPARE_CAP]196        na = normalized_text(after_text)[:TEXT_COMPARE_CAP]197        diff.text_delta_ratio = round(Indel.normalized_distance(nb, na), 4) if (nb or na) else 0.0198    else:199        diff.text_delta_ratio = round(changed_share, 4)200201    # ---------------------------------------------------------------- significance202    reasons: list[str] = []203    content_deltas = [d for d in (diff.added + diff.removed + diff.modified) if d.kind not in LOW_VALUE_KINDS]204    low_value_deltas = [d for d in (diff.added + diff.removed + diff.modified) if d.kind in LOW_VALUE_KINDS]205    typed_keys = _typed_keys(sd)206207    if not content_deltas and not low_value_deltas and not typed_keys and diff.text_delta_ratio < 0.001:208        diff.significance = 0.0209        diff.reasons = ["identical"] if not diff.moved else ["blocks reordered only"]210        return diff211212    importance = float(SURFACE_IMPORTANCE.get(surface, 0.3))213    importance_factor = 0.7 + 0.3 * importance214    if content_deltas:215        base = CONTENT_BASE + CONTENT_SPAN * (changed_share ** CONTENT_EXP)216        reasons.append(f"content changed share {changed_share:.3f} over {len(content_deltas)} block(s)")217    elif typed_keys:218        base = 0.2219        reasons.append("typed delta without block-level content change")220    else:221        base = 0.05222        reasons.append("only nav/header/footer blocks changed")223    sig = base * importance_factor224    reasons.append(f"surface {surface} importance {importance:.2f} (×{importance_factor:.2f})")225226    # typed floors227    floor = 0.0228    floor_reason = ""229    for key, value in typed_keys.items():230        f = TYPED_FLOORS.get(key, 0.0)231        if key in ("jobs.added", "jobs.removed"):232            open_before = int(sd.get("jobs", {}).get("open_before") or 0)233            if value >= JOBS_SCALE_COUNT or (open_before and value / open_before >= 0.25):234                f = max(f, JOBS_SCALE_FLOOR)235        if f > floor:236            floor, floor_reason = f, f"{key}={value}"237    if floor > 0:238        # a typed delta lifts the score to its floor and still rewards larger content changes above it239        sig = max(sig, floor + 0.15 * min(1.0, changed_share))240        reasons.append(f"typed delta {floor_reason} → floor {floor:.2f}")241242    # novelty / churn from sensor history243    unchanged_runs = int(hist.get("consecutive_unchanged") or 0)244    observations = int(hist.get("observation_count") or 0)245    changes = int(hist.get("change_count") or 0)246    if unchanged_runs >= NOVELTY_STABLE_RUNS:247        sig *= NOVELTY_BOOST248        reasons.append(f"novel after {unchanged_runs} unchanged runs (×{NOVELTY_BOOST})")249    elif observations >= 6 and changes / max(1, observations) > CHURN_RATE and not typed_keys:250        sig *= CHURN_PENALTY251        reasons.append(f"volatile page ({changes}/{observations} runs changed, ×{CHURN_PENALTY})")252253    # noise caps (never override typed floors)254    if not typed_keys:255        if not content_deltas:256            sig = min(sig, LOW_VALUE_CAP)257            reasons.append("nav/footer-only churn → noise cap")258        else:259            only_tiny = all(d.similarity is not None and d.similarity >= TINY_EDIT_SIMILARITY for d in content_deltas)260            if only_tiny:261                sig = min(sig, NOISE_CAP)262                reasons.append("only cosmetic edits (similarity ≥ 0.97) → noise cap")263            elif changed_share < 0.01 and len(content_deltas) <= 2 and diff.text_delta_ratio < 0.01:264                sig = min(sig, NOISE_CAP)265                reasons.append("negligible share (< 1 %) in ≤ 2 blocks → noise cap")266            elif _all_noise_tokens(content_deltas):267                sig = min(sig, NOISE_CAP)268                reasons.append("changes limited to dates/counters → noise cap")269    diff.significance = round(max(0.0, min(1.0, sig)), 4)270    diff.reasons = reasons271    return diff272273274def _typed_keys(sd: Mapping[str, Any]) -> dict[str, int]:275    """Flatten a StructuredDelta into {"jobs.added": n, "plans.price_changed": n, "people.added_executive": n, …} (non-zero only)."""276    out: dict[str, int] = {}277    for group in ("jobs", "people", "products", "plans", "locations", "news"):278        g = sd.get(group) or {}279        if not isinstance(g, Mapping):280            continue281        for k in ("added", "removed", "price_changed", "title_changed", "new_countries"):282            v = g.get(k)283            if isinstance(v, list) and v:284                out[f"{group}.{k}"] = len(v)285                if group == "people" and k in ("added", "removed"):286                    execs = sum(1 for p in v if isinstance(p, Mapping) and (p.get("is_executive") or p.get("role_category") in287                                ("ceo", "cfo", "cto", "coo", "founder", "president", "chair")))288                    if execs:289                        out[f"people.{k}_executive"] = execs290    meta = sd.get("meta") or {}291    if isinstance(meta, Mapping) and meta.get("title_changed"):292        out["meta.title_changed"] = 1293    return out294295296def _all_noise_tokens(deltas: list[BlockDelta]) -> bool:297    """True when every modified block is identical after noise normalisation (dates, counters…) — belt and braces: such blocks298    normally share the same hash and never reach the delta list, but fuzzy-matched blocks with different keys can."""299    for d in deltas:300        if d.before is None or d.after is None:301            return False302        if normalized_text(d.before) != normalized_text(d.after):303            return False304    return bool(deltas)305306307__all__ = ["DIFF_VERSION", "NEAR_HAMMING", "TYPED_FLOORS", "compare", "match_blocks"]308