canonicalize: re-home benchmark results onto registry variant benchmarks (config.variant/board match, LiveBench category:<Name> → livebench-<slug>, aider well-formed), recompute keys/current rows and evaluated_on; family slugs use <slug>-family on collision and existing families are re-slugged with the old slug kept as alias

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 12 days ago (Sep 12, 2026) parent 834f1f1

2 changed files +160 −5

modified src/aiatlas/services/canonical.py +100 −5

		`@@ -455,6 +455,30 @@ async def step_families(conn: AsyncConnection, rep: StepReport, apply: bool, *,`
455	455	fam_rows = await fetch_all(conn, "select id, slug, canonical_name, organization_id from entities where entity_type = 'model_family' and merged_into is null")
456	456	families: dict[str, dict[str, Any]] = {f["canonical_name"].lower(): f for f in fam_rows} # one family per label
457	457	slugs_taken = {r["slug"] for r in await fetch_all(conn, "select slug from entities")}
	458	+
	459	+ def family_slug(label: str) -> str:
	460	+ """slugify(label); on collision with any other entity (the model `gpt-5.5` itself) → `<slug>-family`, then numbered."""
	461	+ base = slugify(label)
	462	+ for c in [base, f"{base}-family", *(f"{base}-family-{n}" for n in range(2, 20))]:
	463	+ if c not in slugs_taken:
	464	+ return c
	465	+ return f"{base}-family-{new_id('model_family')[-6:].lower()}"
	466	+
	467	+ # re-slug families created under the former rule (organisation prefix on collision: `qwen-qwen3`, `mistral-mistral`)
	468	+ for fam in fam_rows:
	469	+ base = slugify(fam["canonical_name"])
	470	+ if fam["slug"] in (base, f"{base}-family") or not fam["slug"].endswith(base):
	471	+ continue
	472	+ slugs_taken.discard(fam["slug"])
	473	+ new_slug = family_slug(fam["canonical_name"])
	474	+ slugs_taken.add(new_slug)
	475	+ rep.bump("families_reslugged")
	476	+ rep.example(f"family slug {fam['slug']} → {new_slug}")
	477	+ if apply:
	478	+ await execute(conn, "update entities set slug = :s, updated_at = now() where id = :id", s=new_slug, id=fam["id"])
	479	+ await execute(conn, "insert into entity_aliases (entity_id, alias, alias_norm, kind) values (:e, :a, :n, 'former_name') on conflict (entity_id, alias_norm) do nothing",
	480	+ e=fam["id"], a=fam["slug"], n=normalize_alias(fam["slug"]))
	481	+ fam["slug"] = new_slug
458	482	for m in models:
459	483	label = family_release_hint(m["canonical_name"])
460	484	if not label:
		`@@ -465,11 +489,7 @@ async def step_families(conn: AsyncConnection, rep: StepReport, apply: bool, *,`
465	489	org_id = next((org_by_key[k] for k in official if k in org_by_key), None) or m["organization_id"]
466	490	fam = families.get(label.lower())
467	491	if fam is None:
468		− # slug = slugify(label); on collision with any other entity (the model "gpt-5.5" itself) prefix with the organisation slug, then "family-"
469		− base_slug = slugify(label)
470		− org_slug = next((o["slug"] for o in orgs if o["id"] == org_id), None)
471		− candidates = [base_slug] + ([f"{org_slug}-{base_slug}"] if org_slug else []) + [f"family-{base_slug}"] + [f"family-{base_slug}-{n}" for n in range(2, 20)]
472		− slug = next(c for c in candidates if c not in slugs_taken)
	492	+ slug = family_slug(label)
473	493	rep.bump("families_created")
474	494	rep.example(f"family '{label}' ({slug}) root={root}")
475	495	fam = {"id": new_id("model_family"), "slug": slug, "canonical_name": label, "organization_id": org_id, "_new": True}
		`@@ -650,7 +670,82 @@ async def step_taxonomy(conn: AsyncConnection, rep: StepReport, apply: bool, *,`
650	670
651	671
652	672	# ---------------------------------------------------------------------------------------------- step: results
	673	+async def _rehome_results(conn: AsyncConnection, rep: StepReport, apply: bool, scope: set[str] \| None) -> None:
	674	+ """Move results from a benchmark family head to the registry's variant entity when the row says which variant it measured:
	675	+ (a) config.variant / config.board equals (case-insensitively) the variant entity's name, an alias or its `variant` attribute;
	676	+ (b) LiveBench `category:<Name>` metrics → `livebench-<slug(Name)>` with the registry metric (`average score`), variant = Name;
	677	+ (c) aider `percent_cases_well_formed` → `aider-polyglot-well-formed`.
	678	+ Then dedupe/config keys, one-current-row and `evaluated_on` relations are recomputed for every touched model."""
	679	+ variants = await fetch_all(conn, """select v.id, v.slug, v.canonical_name, v.attributes, r.object_id as head_id,
	680	+ (select array_agg(alias) from entity_aliases a where a.entity_id = v.id) as aliases
	681	+ from entities v join relations r on r.subject_id = v.id and r.predicate = 'variant_of' and r.valid_to is null
	682	+ where v.entity_type = 'benchmark' and v.merged_into is null""")
	683	+ if not variants:
	684	+ return
	685	+ by_slug: dict[str, dict[str, Any]] = {v["slug"]: v for v in variants}
	686	+ by_head: dict[str, list[tuple[set[str], dict[str, Any]]]] = defaultdict(list)
	687	+ for v in variants:
	688	+ keys = {v["canonical_name"].lower(), v["slug"], *(a.lower() for a in (v["aliases"] or []))}
	689	+ var_attr = (v["attributes"] or {}).get("variant")
	690	+ if isinstance(var_attr, str):
	691	+ keys.add(var_attr.lower())
	692	+ by_head[v["head_id"]].append((keys, v))
	693	+ heads = list(by_head)
	694	+ rows = await fetch_all(conn, """select r.id, r.model_id, r.benchmark_id, r.metric, r.config from benchmark_results r
	695	+ where r.benchmark_id = any(cast(:h as text[]))""", h=heads)
	696	+ moves: list[tuple[dict[str, Any], dict[str, Any], str \| None, dict[str, Any]]] = []
	697	+ for r in rows:
	698	+ if scope is not None and r["model_id"] not in scope:
	699	+ continue
	700	+ cfg = dict(r["config"] or {})
	701	+ metric = r["metric"] or ""
	702	+ target: dict[str, Any] \| None = None
	703	+ new_metric: str \| None = None
	704	+ label = next((str(cfg[k]) for k in ("variant", "board") if isinstance(cfg.get(k), str) and cfg[k].strip()), None)
	705	+ if label:
	706	+ target = next((v for keys, v in by_head[r["benchmark_id"]] if label.lower() in keys), None)
	707	+ if target is None and metric.lower().startswith("category:"):
	708	+ name = metric.split(":", 1)[1].strip()
	709	+ cand = by_slug.get(f"livebench-{slugify(name)}")
	710	+ if cand and cand["head_id"] == r["benchmark_id"]:
	711	+ target = cand
	712	+ new_metric = (cand["attributes"] or {}).get("metric") or "average score"
	713	+ cfg["variant"] = name
	714	+ if target is None and metric == "percent_cases_well_formed":
	715	+ cand = by_slug.get("aider-polyglot-well-formed")
	716	+ if cand and cand["head_id"] == r["benchmark_id"]:
	717	+ target = cand
	718	+ if target is None:
	719	+ continue
	720	+ moves.append((r, target, new_metric, cfg))
	721	+ if not moves:
	722	+ return
	723	+ touched: set[tuple[str, str, str]] = set()
	724	+ for r, target, new_metric, cfg in moves:
	725	+ rep.bump(f"results_rehomed:{target['slug']}")
	726	+ touched.add((r["model_id"], r["benchmark_id"], target["id"]))
	727	+ if apply:
	728	+ await execute(conn, "update benchmark_results set benchmark_id = :b, metric = coalesce(:m, metric), config = cast(:c as jsonb), variant = :v where id = :id",
	729	+ b=target["id"], m=new_metric, c=jsonb(cfg), v=bench_ontology.variant_from_config(cfg), id=r["id"])
	730	+ if not apply:
	731	+ return
	732	+ source_id = await registry_source_id(conn)
	733	+ for model_id in sorted({m for m, _, _ in touched}):
	734	+ from aiatlas.services.merge import recompute_result_keys
	735	+
	736	+ await recompute_result_keys(conn, model_id)
	737	+ await enforce_current_results(conn, model_id=model_id)
	738	+ for model_id, head_id, target_id in sorted(touched):
	739	+ await upsert_relation(conn, model_id, "evaluated_on", target_id, source_id=source_id)
	740	+ left = await fetch_one(conn, "select 1 from benchmark_results where model_id = :m and benchmark_id = :b and valid_to is null limit 1", m=model_id, b=head_id)
	741	+ if not left:
	742	+ await execute(conn, "update relations set valid_to = now() where subject_id = :m and predicate = 'evaluated_on' and object_id = :b and valid_to is null",
	743	+ m=model_id, b=head_id)
	744	+ rep.bump("evaluated_on_repointed")
	745	+
	746	+
653	747	async def step_results(conn: AsyncConnection, rep: StepReport, apply: bool, *, scope: set[str] \| None = None) -> None:
	748	+ await _rehome_results(conn, rep, apply, scope)
654	749	rows = await fetch_all(conn, """select r.id, r.model_id, r.config, r.metric, r.unit, r.score, r.config_key, r.trust_level, r.variant, r.run_group, r.extractor, r.valid_to,
655	750	r.is_current, s.key as source_key from benchmark_results r left join sources s on s.id = r.source_id""")
656	751	if scope is not None:

modified tests/test_canonical.py +60 −0

		`@@ -284,3 +284,63 @@ async def test_duplicates_step(conn: AsyncConnection) -> None:`
284	284	assert (await fetch_one(conn, "select merged_into from entities where id = :id", id=i))["merged_into"] is None
285	285	again = await _run(canon.step_duplicates, conn, True, keep.id, dup.id, x.id, y.id)
286	286	assert again.counts.get("merged", 0) == 0
	287	+
	288	+
	289	+# ---------------------------------------------------------------------------------------------- result re-homing & family slugs
	290	+async def test_results_rehomed_to_variant_benchmarks(conn: AsyncConnection) -> None:
	291	+ t = _tag()
	292	+ f = Facts()
	293	+ m = f.entity("model", f"Zeta {t}")
	294	+ head = f.entity("benchmark", f"GPQA {t}", identifiers={"registry_benchmark": f"gpqa-{t}"}, slug_hint=f"gpqa-{t}")
	295	+ diamond = f.entity("benchmark", f"GPQA {t} Diamond", identifiers={"registry_benchmark": f"gpqa-{t}-diamond"}, slug_hint=f"gpqa-{t}-diamond",
	296	+ aliases=[f"GPQA-{t}-Diamond"], attributes={"variant": "Diamond", "metric": "accuracy"})
	297	+ f.relate(diamond, "variant_of", head)
	298	+ lb = f.entity("benchmark", f"LiveBench {t}", slug_hint=f"livebench-{t}")
	299	+ lb_reason = f.entity("benchmark", f"LiveBench {t} Reasoning", slug_hint=f"livebench-reasoning-{t}", attributes={"variant": "Reasoning", "metric": "average score"})
	300	+ f.relate(lb_reason, "variant_of", lb)
	301	+ f.result(model=m, benchmark=head, score=71.0, metric="accuracy", unit="%", config={"variant": f"GPQA {t} Diamond", "evaluator": "AA"})
	302	+ f.result(model=m, benchmark=head, score=60.0, metric="accuracy", unit="%", config={"evaluator": "AA"}) # main set stays
	303	+ f.result(model=m, benchmark=lb, score=55.0, metric="global_average", unit="%", config={"release": "2026-06-25"})
	304	+ await _write(conn, f, source_key="artificialanalysis.ai")
	305	+ # a category metric sitting on the head (pre-upgrade LiveBench rows); the variant slug must be livebench-<slug(Name)>
	306	+ await execute(conn, "update entities set slug = :s where id = :id", s=f"livebench-reasoning-{t}", id=lb_reason.id)
	307	+ await execute(conn, """insert into benchmark_results (id, model_id, benchmark_id, score, metric, unit, config, dedupe_key, tier)
	308	+ values (:id, :m, :b, 48.0, :metric, '%', '{"release": "2026-06-25"}', :d, 2)""",
	309	+ id=f"res_cat{t}", m=m.id, b=lb.id, metric="category:Reasoning", d=f"cat{t}")
	310	+ # slug(Name) lookup uses exactly `livebench-reasoning`; emulate it by renaming the head-side entity for this test
	311	+ await execute(conn, "update entities set slug = 'livebench-reasoning' where id = :id and not exists (select 1 from entities where slug = 'livebench-reasoning')", id=lb_reason.id)
	312	+ rep = await _run(canon.step_results, conn, True, m.id)
	313	+ assert rep.counts.get(f"results_rehomed:gpqa-{t}-diamond") == 1
	314	+ rows = {r["score"]: r for r in await fetch_all(conn, "select r.score, b.slug, r.metric, r.config, r.dedupe_key, r.is_current from benchmark_results r join entities b on b.id = r.benchmark_id where r.model_id = :m", m=m.id)}
	315	+ assert rows[71.0]["slug"] == f"gpqa-{t}-diamond" and rows[71.0]["dedupe_key"].startswith(f"{m.id}:{diamond.id}:") and rows[71.0]["is_current"]
	316	+ assert rows[60.0]["slug"] == f"gpqa-{t}" and rows[60.0]["is_current"]
	317	+ lb_slug = (await fetch_one(conn, "select slug from entities where id = :id", id=lb_reason.id))["slug"]
	318	+ if lb_slug == "livebench-reasoning":
	319	+ assert rows[48.0]["slug"] == "livebench-reasoning" and rows[48.0]["metric"] == "average score" and rows[48.0]["config"]["variant"] == "Reasoning"
	320	+ rels = {(r["object_id"], r["valid_to"] is None) for r in await fetch_all(conn, "select object_id, valid_to from relations where subject_id = :m and predicate = 'evaluated_on'", m=m.id)}
	321	+ assert (diamond.id, True) in rels and (head.id, True) in rels # head keeps a live edge: the main-set row is still there
	322	+ again = await _run(canon.step_results, conn, True, m.id)
	323	+ assert again.changes == 0
	324	+
	325	+
	326	+async def test_family_slug_collision_uses_family_suffix(conn: AsyncConnection) -> None:
	327	+ t = _tag()
	328	+ v = str(int(t, 16) % 900 + 100)
	329	+ f = Facts()
	330	+ org = f.entity("company", f"Mistral test {t}")
	331	+ taken = f.entity("model", f"Gemma {v}", organization=org) # occupies the slug `gemma-<v>`
	332	+ a = f.entity("model", f"Gemma {v} 9B Test", organization=org)
	333	+ await _write(conn, f)
	334	+ rep = await _run(canon.step_families, conn, True, a.id, taken.id)
	335	+ assert rep.counts.get("families_created", 0) >= 1
	336	+ fam = await fetch_one(conn, "select slug from entities where entity_type = 'model_family' and canonical_name = :n", n=f"Gemma {v}")
	337	+ assert fam["slug"] == f"gemma-{v}-family"
	338	+ # a family created under the old rule (org prefix) is re-slugged and keeps the old slug as an alias
	339	+ await execute(conn, "update entities set slug = :s where id = (select id from entities where entity_type = 'model_family' and canonical_name = :n)", s=f"mistral-gemma-{v}", n=f"Gemma {v}")
	340	+ rep = await _run(canon.step_families, conn, True, a.id)
	341	+ assert rep.counts.get("families_reslugged") == 1
	342	+ fam = await fetch_one(conn, "select id, slug from entities where entity_type = 'model_family' and canonical_name = :n", n=f"Gemma {v}")
	343	+ assert fam["slug"] == f"gemma-{v}-family"
	344	+ assert await fetch_one(conn, "select 1 from entity_aliases where entity_id = :id and alias = :a and kind = 'former_name'", id=fam["id"], a=f"mistral-gemma-{v}")
	345	+ again = await _run(canon.step_families, conn, True, a.id)
	346	+ assert again.changes == 0
287	347