canonicalize: re-home benchmark results onto registry variant benchmarks (config.variant/board match, LiveBench category:<Name> → livebench-<slug>, aider well-formed), recompute keys/current rows and evaluated_on; family slugs use <slug>-family on collision and existing families are re-slugged with the old slug kept as alias

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 1 mo ago (Sep 12, 2026) parent 834f1f1 Browse files

2 changed files +160 −5

modified src/aiatlas/services/canonical.py +100 −5

+     fam_rows = await fetch_all(conn, "select id, slug, canonical_name, organization_id from entities where entity_type = 'model_family' and merged_into is null")
+     families: dict[str, dict[str, Any]] = {f["canonical_name"].lower(): f for f in fam_rows}     # one family per label
+     slugs_taken = {r["slug"] for r in await fetch_all(conn, "select slug from entities")}
++
++    def family_slug(label: str) -> str:
++        """slugify(label); on collision with any other entity (the model `gpt-5.5` itself) → `<slug>-family`, then numbered."""
++        base = slugify(label)
++        for c in [base, f"{base}-family", *(f"{base}-family-{n}" for n in range(2, 20))]:
++            if c not in slugs_taken:
++                return c
++        return f"{base}-family-{new_id('model_family')[-6:].lower()}"
++
++    # re-slug families created under the former rule (organisation prefix on collision: `qwen-qwen3`, `mistral-mistral`)
++    for fam in fam_rows:
++        base = slugify(fam["canonical_name"])
++        if fam["slug"] in (base, f"{base}-family") or not fam["slug"].endswith(base):
++            continue
++        slugs_taken.discard(fam["slug"])
++        new_slug = family_slug(fam["canonical_name"])
++        slugs_taken.add(new_slug)
++        rep.bump("families_reslugged")
++        rep.example(f"family slug {fam['slug']} → {new_slug}")
++        if apply:
++            await execute(conn, "update entities set slug = :s, updated_at = now() where id = :id", s=new_slug, id=fam["id"])
++            await execute(conn, "insert into entity_aliases (entity_id, alias, alias_norm, kind) values (:e, :a, :n, 'former_name') on conflict (entity_id, alias_norm) do nothing",
++                          e=fam["id"], a=fam["slug"], n=normalize_alias(fam["slug"]))
++        fam["slug"] = new_slug
+     for m in models:
+         label = family_release_hint(m["canonical_name"])
+         if not label:
+         org_id = next((org_by_key[k] for k in official if k in org_by_key), None) or m["organization_id"]
+         fam = families.get(label.lower())
+         if fam is None:
++            slug = family_slug(label)
+−            base_slug = slugify(label)
+−            org_slug = next((o["slug"] for o in orgs if o["id"] == org_id), None)
+−            candidates = [base_slug] + ([f"{org_slug}-{base_slug}"] if org_slug else []) + [f"family-{base_slug}"] + [f"family-{base_slug}-{n}" for n in range(2, 20)]
+−            slug = next(c for c in candidates if c not in slugs_taken)
+             rep.bump("families_created")
+             rep.example(f"family '{label}' ({slug}) root={root}")
+             fam = {"id": new_id("model_family"), "slug": slug, "canonical_name": label, "organization_id": org_id, "_new": True}
+ # ---------------------------------------------------------------------------------------------- step: results
++async def _rehome_results(conn: AsyncConnection, rep: StepReport, apply: bool, scope: set[str] | None) -> None:
++    """Move results from a benchmark *family head* to the registry's variant entity when the row says which variant it measured:
++        (a) config.variant / config.board equals (case-insensitively) the variant entity's name, an alias or its `variant` attribute;
++        (b) LiveBench `category:<Name>` metrics → `livebench-<slug(Name)>` with the registry metric (`average score`), variant = Name;
++        (c) aider `percent_cases_well_formed` → `aider-polyglot-well-formed`.
++    Then dedupe/config keys, one-current-row and `evaluated_on` relations are recomputed for every touched model."""
++    variants = await fetch_all(conn, """select v.id, v.slug, v.canonical_name, v.attributes, r.object_id as head_id,
++                                               (select array_agg(alias) from entity_aliases a where a.entity_id = v.id) as aliases
++                                        from entities v join relations r on r.subject_id = v.id and r.predicate = 'variant_of' and r.valid_to is null
++                                        where v.entity_type = 'benchmark' and v.merged_into is null""")
++    if not variants:
++        return
++    by_slug: dict[str, dict[str, Any]] = {v["slug"]: v for v in variants}
++    by_head: dict[str, list[tuple[set[str], dict[str, Any]]]] = defaultdict(list)
++    for v in variants:
++        keys = {v["canonical_name"].lower(), v["slug"], *(a.lower() for a in (v["aliases"] or []))}
++        var_attr = (v["attributes"] or {}).get("variant")
++        if isinstance(var_attr, str):
++            keys.add(var_attr.lower())
++        by_head[v["head_id"]].append((keys, v))
++    heads = list(by_head)
++    rows = await fetch_all(conn, """select r.id, r.model_id, r.benchmark_id, r.metric, r.config from benchmark_results r
++                                    where r.benchmark_id = any(cast(:h as text[]))""", h=heads)
++    moves: list[tuple[dict[str, Any], dict[str, Any], str | None, dict[str, Any]]] = []
++    for r in rows:
++        if scope is not None and r["model_id"] not in scope:
++            continue
++        cfg = dict(r["config"] or {})
++        metric = r["metric"] or ""
++        target: dict[str, Any] | None = None
++        new_metric: str | None = None
++        label = next((str(cfg[k]) for k in ("variant", "board") if isinstance(cfg.get(k), str) and cfg[k].strip()), None)
++        if label:
++            target = next((v for keys, v in by_head[r["benchmark_id"]] if label.lower() in keys), None)
++        if target is None and metric.lower().startswith("category:"):
++            name = metric.split(":", 1)[1].strip()
++            cand = by_slug.get(f"livebench-{slugify(name)}")
++            if cand and cand["head_id"] == r["benchmark_id"]:
++                target = cand
++                new_metric = (cand["attributes"] or {}).get("metric") or "average score"
++                cfg["variant"] = name
++        if target is None and metric == "percent_cases_well_formed":
++            cand = by_slug.get("aider-polyglot-well-formed")
++            if cand and cand["head_id"] == r["benchmark_id"]:
++                target = cand
++        if target is None:
++            continue
++        moves.append((r, target, new_metric, cfg))
++    if not moves:
++        return
++    touched: set[tuple[str, str, str]] = set()
++    for r, target, new_metric, cfg in moves:
++        rep.bump(f"results_rehomed:{target['slug']}")
++        touched.add((r["model_id"], r["benchmark_id"], target["id"]))
++        if apply:
++            await execute(conn, "update benchmark_results set benchmark_id = :b, metric = coalesce(:m, metric), config = cast(:c as jsonb), variant = :v where id = :id",
++                          b=target["id"], m=new_metric, c=jsonb(cfg), v=bench_ontology.variant_from_config(cfg), id=r["id"])
++    if not apply:
++        return
++    source_id = await registry_source_id(conn)
++    for model_id in sorted({m for m, _, _ in touched}):
++        from aiatlas.services.merge import recompute_result_keys
++
++        await recompute_result_keys(conn, model_id)
++        await enforce_current_results(conn, model_id=model_id)
++    for model_id, head_id, target_id in sorted(touched):
++        await upsert_relation(conn, model_id, "evaluated_on", target_id, source_id=source_id)
++        left = await fetch_one(conn, "select 1 from benchmark_results where model_id = :m and benchmark_id = :b and valid_to is null limit 1", m=model_id, b=head_id)
++        if not left:
++            await execute(conn, "update relations set valid_to = now() where subject_id = :m and predicate = 'evaluated_on' and object_id = :b and valid_to is null",
++                          m=model_id, b=head_id)
++            rep.bump("evaluated_on_repointed")
++
++
+ async def step_results(conn: AsyncConnection, rep: StepReport, apply: bool, *, scope: set[str] | None = None) -> None:
++    await _rehome_results(conn, rep, apply, scope)
+     rows = await fetch_all(conn, """select r.id, r.model_id, r.config, r.metric, r.unit, r.score, r.config_key, r.trust_level, r.variant, r.run_group, r.extractor, r.valid_to,
+                                            r.is_current, s.key as source_key from benchmark_results r left join sources s on s.id = r.source_id""")
+     if scope is not None:

modified tests/test_canonical.py +60 −0

+         assert (await fetch_one(conn, "select merged_into from entities where id = :id", id=i))["merged_into"] is None
+     again = await _run(canon.step_duplicates, conn, True, keep.id, dup.id, x.id, y.id)
+     assert again.counts.get("merged", 0) == 0
++
++
++# ---------------------------------------------------------------------------------------------- result re-homing & family slugs
++async def test_results_rehomed_to_variant_benchmarks(conn: AsyncConnection) -> None:
++    t = _tag()
++    f = Facts()
++    m = f.entity("model", f"Zeta {t}")
++    head = f.entity("benchmark", f"GPQA {t}", identifiers={"registry_benchmark": f"gpqa-{t}"}, slug_hint=f"gpqa-{t}")
++    diamond = f.entity("benchmark", f"GPQA {t} Diamond", identifiers={"registry_benchmark": f"gpqa-{t}-diamond"}, slug_hint=f"gpqa-{t}-diamond",
++                       aliases=[f"GPQA-{t}-Diamond"], attributes={"variant": "Diamond", "metric": "accuracy"})
++    f.relate(diamond, "variant_of", head)
++    lb = f.entity("benchmark", f"LiveBench {t}", slug_hint=f"livebench-{t}")
++    lb_reason = f.entity("benchmark", f"LiveBench {t} Reasoning", slug_hint=f"livebench-reasoning-{t}", attributes={"variant": "Reasoning", "metric": "average score"})
++    f.relate(lb_reason, "variant_of", lb)
++    f.result(model=m, benchmark=head, score=71.0, metric="accuracy", unit="%", config={"variant": f"GPQA {t} Diamond", "evaluator": "AA"})
++    f.result(model=m, benchmark=head, score=60.0, metric="accuracy", unit="%", config={"evaluator": "AA"})                    # main set stays
++    f.result(model=m, benchmark=lb, score=55.0, metric="global_average", unit="%", config={"release": "2026-06-25"})
++    await _write(conn, f, source_key="artificialanalysis.ai")
++    # a category metric sitting on the head (pre-upgrade LiveBench rows); the variant slug must be livebench-<slug(Name)>
++    await execute(conn, "update entities set slug = :s where id = :id", s=f"livebench-reasoning-{t}", id=lb_reason.id)
++    await execute(conn, """insert into benchmark_results (id, model_id, benchmark_id, score, metric, unit, config, dedupe_key, tier)
++                           values (:id, :m, :b, 48.0, :metric, '%', '{"release": "2026-06-25"}', :d, 2)""",
++                  id=f"res_cat{t}", m=m.id, b=lb.id, metric="category:Reasoning", d=f"cat{t}")
++    # slug(Name) lookup uses exactly `livebench-reasoning`; emulate it by renaming the head-side entity for this test
++    await execute(conn, "update entities set slug = 'livebench-reasoning' where id = :id and not exists (select 1 from entities where slug = 'livebench-reasoning')", id=lb_reason.id)
++    rep = await _run(canon.step_results, conn, True, m.id)
++    assert rep.counts.get(f"results_rehomed:gpqa-{t}-diamond") == 1
++    rows = {r["score"]: r for r in await fetch_all(conn, "select r.score, b.slug, r.metric, r.config, r.dedupe_key, r.is_current from benchmark_results r join entities b on b.id = r.benchmark_id where r.model_id = :m", m=m.id)}
++    assert rows[71.0]["slug"] == f"gpqa-{t}-diamond" and rows[71.0]["dedupe_key"].startswith(f"{m.id}:{diamond.id}:") and rows[71.0]["is_current"]
++    assert rows[60.0]["slug"] == f"gpqa-{t}" and rows[60.0]["is_current"]
++    lb_slug = (await fetch_one(conn, "select slug from entities where id = :id", id=lb_reason.id))["slug"]
++    if lb_slug == "livebench-reasoning":
++        assert rows[48.0]["slug"] == "livebench-reasoning" and rows[48.0]["metric"] == "average score" and rows[48.0]["config"]["variant"] == "Reasoning"
++    rels = {(r["object_id"], r["valid_to"] is None) for r in await fetch_all(conn, "select object_id, valid_to from relations where subject_id = :m and predicate = 'evaluated_on'", m=m.id)}
++    assert (diamond.id, True) in rels and (head.id, True) in rels          # head keeps a live edge: the main-set row is still there
++    again = await _run(canon.step_results, conn, True, m.id)
++    assert again.changes == 0
++
++
++async def test_family_slug_collision_uses_family_suffix(conn: AsyncConnection) -> None:
++    t = _tag()
++    v = str(int(t, 16) % 900 + 100)
++    f = Facts()
++    org = f.entity("company", f"Mistral test {t}")
++    taken = f.entity("model", f"Gemma {v}", organization=org)                 # occupies the slug `gemma-<v>`
++    a = f.entity("model", f"Gemma {v} 9B Test", organization=org)
++    await _write(conn, f)
++    rep = await _run(canon.step_families, conn, True, a.id, taken.id)
++    assert rep.counts.get("families_created", 0) >= 1
++    fam = await fetch_one(conn, "select slug from entities where entity_type = 'model_family' and canonical_name = :n", n=f"Gemma {v}")
++    assert fam["slug"] == f"gemma-{v}-family"
++    # a family created under the old rule (org prefix) is re-slugged and keeps the old slug as an alias
++    await execute(conn, "update entities set slug = :s where id = (select id from entities where entity_type = 'model_family' and canonical_name = :n)", s=f"mistral-gemma-{v}", n=f"Gemma {v}")
++    rep = await _run(canon.step_families, conn, True, a.id)
++    assert rep.counts.get("families_reslugged") == 1
++    fam = await fetch_one(conn, "select id, slug from entities where entity_type = 'model_family' and canonical_name = :n", n=f"Gemma {v}")
++    assert fam["slug"] == f"gemma-{v}-family"
++    assert await fetch_one(conn, "select 1 from entity_aliases where entity_id = :id and alias = :a and kind = 'former_name'", id=fam["id"], a=f"mistral-gemma-{v}")
++    again = await _run(canon.step_families, conn, True, a.id)
++    assert again.changes == 0

spb/ai-atlas

Theme