| 14 |
14 |
|
| 15 |
15 |
import json |
| 16 |
16 |
import logging |
|
17 |
+import re |
| 17 |
18 |
from collections import defaultdict |
| 18 |
19 |
from dataclasses import dataclass, field |
| 19 |
20 |
from datetime import UTC, datetime, timedelta |
| 173 |
174 |
return out |
| 174 |
175 |
|
| 175 |
176 |
|
|
177 |
+_SNAPSHOT_REMAINDER = re.compile(r"^(\d{1,8}|preview\d*|exp\d*|latest|beta|alpha|v\d+)$") |
|
178 |
+_EFFORT_REMAINDER = {"reasoning", "nonreasoning", "thinking", "nonthinking", "high", "low", "medium", "xhigh", "minimal", "instruct", "it", "chat"} |
|
179 |
+_VENDOR_PREFIXES = {"meta", "metallama", "google", "openai", "alibaba", "nvidia", "microsoft", "anthropic", "qwen", "deepseek", "mistral", "mistralai", "zai", "moonshotai"} |
|
180 |
+_VERSION_TOKEN = re.compile(r"^v?\d+(\.\d+)*$") |
|
181 |
+ |
|
182 |
+ |
|
183 |
+def _identifier_alias(a: str, b: str) -> bool: |
|
184 |
+ """Two values of one scheme name the same thing when one is a dated/preview snapshot, an effort/instruct suffix, a vendor-prefixed |
|
185 |
+ or re-versioned spelling of the other (`gemini-2.5-flash-lite` ~ `gemini-2.5-flash-lite-preview-06-17`, `google/gemini-2.5-flash-lite` ~ |
|
186 |
+ `google/gemini-2.5-flash-lite-preview`, `meta-llama/Meta-Llama-3.1-8B-Instruct` ~ `meta-llama/Llama-3.1-8B-Instruct`, |
|
187 |
+ `gemini-omni-flash` ~ `gemini-omni-1.1-flash`).""" |
|
188 |
+ na, nb = normalize_alias(a), normalize_alias(b) |
|
189 |
+ if na == nb: |
|
190 |
+ return True |
|
191 |
+ short, long_ = sorted((na, nb), key=len) |
|
192 |
+ if long_.startswith(short): |
|
193 |
+ rest = long_[len(short):] |
|
194 |
+ if _SNAPSHOT_REMAINDER.match(rest) or rest in _EFFORT_REMAINDER or rest.startswith("preview") or rest.startswith("exp"): |
|
195 |
+ return True |
|
196 |
+ if long_.endswith(short) and long_[: -len(short)] in _VENDOR_PREFIXES: |
|
197 |
+ return True |
|
198 |
+ # re-versioned / vendor-duplicated spellings: same word tokens once version and vendor tokens are removed, and the version tokens of |
|
199 |
+ # one side are a subset of the other's (`gemini-omni-flash` ⊂ `gemini-omni-1.1-flash`; `gpt-4` vs `gpt-5` stay different) |
|
200 |
+ ta = [t for t in re.split(r"[-/_.\s]+", a.lower()) if t] |
|
201 |
+ tb = [t for t in re.split(r"[-/_.\s]+", b.lower()) if t] |
|
202 |
+ words_a = [t for t in ta if not _VERSION_TOKEN.match(t) and t not in _VENDOR_PREFIXES] |
|
203 |
+ words_b = [t for t in tb if not _VERSION_TOKEN.match(t) and t not in _VENDOR_PREFIXES] |
|
204 |
+ va = {t for t in ta if _VERSION_TOKEN.match(t)} |
|
205 |
+ vb = {t for t in tb if _VERSION_TOKEN.match(t)} |
|
206 |
+ return bool(words_a) and words_a == words_b and (va <= vb or vb <= va) |
|
207 |
+ |
|
208 |
+ |
| 176 |
209 |
def _identifiers_conflict(a: dict[str, set[str]], b: dict[str, set[str]]) -> bool: |
| 177 |
|
− return any(a[s] and b[s] and a[s] != b[s] for s in set(a) & set(b)) |
|
210 |
+ """Hard contradiction only: a shared scheme whose values are not aliases of one another.""" |
|
211 |
+ for s in set(a) & set(b): |
|
212 |
+ if not a[s] or not b[s] or a[s] == b[s]: |
|
213 |
+ continue |
|
214 |
+ if not all(_identifier_alias(x, y) for x in a[s] for y in b[s]): |
|
215 |
+ return True |
|
216 |
+ return False |
|
217 |
+ |
|
218 |
+ |
|
219 |
+def _conflict_detail(a: dict[str, set[str]], b: dict[str, set[str]]) -> str: |
|
220 |
+ parts = [] |
|
221 |
+ for s in sorted(set(a) & set(b)): |
|
222 |
+ if a[s] and b[s] and a[s] != b[s] and not all(_identifier_alias(x, y) for x in a[s] for y in b[s]): |
|
223 |
+ parts.append(f"{s}: {'/'.join(sorted(a[s]))} vs {'/'.join(sorted(b[s]))}") |
|
224 |
+ return "; ".join(parts) |
|
225 |
+ |
|
226 |
+ |
|
227 |
+def _same_publisher(model_name: str, org_a: dict[str, Any] | None, org_b: dict[str, Any] | None) -> bool: |
|
228 |
+ """Alibaba/Qwen, Meta/Meta AI…: both organisations publish the model's family officially.""" |
|
229 |
+ if not org_a or not org_b: |
|
230 |
+ return False |
|
231 |
+ official = set(official_orgs(model_name)) |
|
232 |
+ return bool(official) and bool(_org_lookup_keys(org_a) & official) and bool(_org_lookup_keys(org_b) & official) |
| 178 |
233 |
|
| 179 |
234 |
|
| 180 |
235 |
def _org_lookup_keys(row: dict[str, Any]) -> set[str]: |
| 192 |
247 |
return scope is None or entity_id in scope |
| 193 |
248 |
|
| 194 |
249 |
|
|
250 |
+async def _retype_routers(conn: AsyncConnection, rep: StepReport, apply: bool, scope: set[str] | None) -> None: |
|
251 |
+ """OpenRouter's own routers (`openrouter/auto`, `openrouter/pareto-code`…) are products, not models: no weights, no developer.""" |
|
252 |
+ rows = await fetch_all(conn, """select distinct e.id, e.slug, e.canonical_name from entities e left join entity_identifiers ei on ei.entity_id = e.id |
|
253 |
+ where e.entity_type = 'model' and e.merged_into is null |
|
254 |
+ and ((ei.scheme = 'openrouter' and ei.value like 'openrouter/%') or e.attributes->>'openrouter_id' like 'openrouter/%')""") |
|
255 |
+ for r in rows: |
|
256 |
+ if not _in_scope(scope, r["id"]): |
|
257 |
+ continue |
|
258 |
+ rep.bump("routers_retyped_as_product") |
|
259 |
+ rep.example(f"product[router] {r['slug']} ({r['canonical_name']})") |
|
260 |
+ if apply: |
|
261 |
+ await execute(conn, """update entities set entity_type = 'product', attributes = attributes || '{"kind": "router"}'::jsonb, updated_at = now() where id = :id""", id=r["id"]) |
|
262 |
+ await execute(conn, "update relations set valid_to = now() where predicate = 'develops' and valid_to is null and (subject_id = :id or object_id = :id)", id=r["id"]) |
|
263 |
+ |
|
264 |
+ |
| 195 |
265 |
async def step_duplicates(conn: AsyncConnection, rep: StepReport, apply: bool, *, scope: set[str] | None = None) -> None: |
| 196 |
266 |
source_id = await registry_source_id(conn) |
|
267 |
+ await _retype_routers(conn, rep, apply, scope) |
| 197 |
268 |
rows = await fetch_all(conn, """select id, entity_type, canonical_name, slug, organization_id, attributes, first_seen_at from entities |
| 198 |
269 |
where merged_into is null and entity_type = any(cast(:types as text[]))""", types=list(DEDUPE_TYPES) + sorted(ORG_TYPES)) |
| 199 |
270 |
if scope is not None: |
| 200 |
271 |
rows = [r for r in rows if r["id"] in scope] |
|
272 |
+ orgs_by_id = {o["id"]: o for o in await fetch_all(conn, "select id, slug, canonical_name, attributes from entities where entity_type = any(cast(:t as text[]))", t=sorted(ORG_TYPES))} |
| 201 |
273 |
# group: exact normalised name within a type; organisations across the org group (+ shared hf_org/github_org) |
| 202 |
274 |
groups: dict[str, list[dict[str, Any]]] = defaultdict(list) |
| 203 |
275 |
for r in rows: |
| 234 |
306 |
claims = await _claim_counts(conn, all_ids) |
| 235 |
307 |
idents = await _identifiers(conn, all_ids) |
| 236 |
308 |
type_rank = {"company": 0, "lab": 0, "university": 0, "organization": 1} # curated org types beat the generic hub "organization" |
|
309 |
+ |
|
310 |
+ def collision_slug(m: dict[str, Any], cluster: list[dict[str, Any]]) -> int: |
|
311 |
+ """1 when the slug is a collision product — an organisation prefix (`google-gemma-4-31b`, `meta-ai-llama-…`) or a `-2` suffix over |
|
312 |
+ another member's slug (`minimax-m3-2`): the plain slug survives.""" |
|
313 |
+ org = orgs_by_id.get(m["organization_id"] or "") |
|
314 |
+ if org and m["slug"].startswith(org["slug"] + "-") and not normalize_alias(m["canonical_name"]).startswith(normalize_alias(org["slug"])): |
|
315 |
+ return 1 |
|
316 |
+ mm = re.match(r"^(.*)-\d+$", m["slug"]) |
|
317 |
+ return int(bool(mm) and any(o["slug"] == mm.group(1) for o in cluster if o is not m)) |
|
318 |
+ |
| 237 |
319 |
for cluster in dup_clusters: |
| 238 |
|
− cluster.sort(key=lambda m: (type_rank.get(m["entity_type"], 0), -claims.get(m["id"], 0), m["first_seen_at"])) |
|
320 |
+ cluster.sort(key=lambda m: (type_rank.get(m["entity_type"], 0), collision_slug(m, cluster), -claims.get(m["id"], 0), m["first_seen_at"])) |
| 239 |
321 |
survivor = cluster[0] |
| 240 |
322 |
for other in cluster[1:]: |
| 241 |
323 |
pair = sorted([survivor["id"], other["id"]]) |
| 243 |
325 |
rep.bump("_kept_separate") |
| 244 |
326 |
continue |
| 245 |
327 |
if _identifiers_conflict(idents[survivor["id"]], idents[other["id"]]): |
| 246 |
|
− if await _review(conn, "merge_candidate", pair, f"'{other['canonical_name']}' duplicates '{survivor['canonical_name']}' but identifiers conflict", |
| 247 |
|
− {"slugs": [other["slug"], survivor["slug"]], "step": "duplicates"}, apply=apply): |
| 248 |
|
− rep.bump("review_conflicting_identifiers") |
| 249 |
|
− rep.example(f"review: {other['canonical_name']} ({other['slug']}) vs {survivor['canonical_name']} ({survivor['slug']}) — identifiers differ") |
|
328 |
+ detail = _conflict_detail(idents[survivor["id"]], idents[other["id"]]) |
|
329 |
+ rep.bump("keep_separate_recorded") |
|
330 |
+ rep.example(f"keep separate: {other['slug']} vs {survivor['slug']} — {detail}") |
|
331 |
+ if apply: |
|
332 |
+ await record_decision(conn, other["id"], survivor["id"], "keep_separate", actor="canonicalize", |
|
333 |
+ note=f"same name, contradictory identifiers ({detail})", payload={"step": "duplicates", "slugs": [other["slug"], survivor["slug"]]}) |
|
334 |
+ await execute(conn, "update review_queue set status = 'rejected', resolved_at = now() where status = 'pending' and kind = 'merge_candidate' and entity_ids @> :ids and entity_ids <@ :ids", |
|
335 |
+ ids=pair) |
| 250 |
336 |
continue |
| 251 |
|
− if survivor["organization_id"] and other["organization_id"] and survivor["organization_id"] != other["organization_id"] and other["entity_type"] not in ORG_TYPES: |
|
337 |
+ if (survivor["organization_id"] and other["organization_id"] and survivor["organization_id"] != other["organization_id"] and other["entity_type"] not in ORG_TYPES |
|
338 |
+ and not _same_publisher(survivor["canonical_name"], orgs_by_id.get(survivor["organization_id"]), orgs_by_id.get(other["organization_id"]))): |
| 252 |
339 |
if await _review(conn, "merge_candidate", pair, f"'{other['canonical_name']}' and '{survivor['canonical_name']}' share a name but have different organisations", |
| 253 |
340 |
{"slugs": [other["slug"], survivor["slug"]], "step": "duplicates"}, apply=apply): |
| 254 |
341 |
rep.bump("review_different_organizations") |
| 288 |
375 |
where e.merged_into is null and e.entity_type = any(cast(:t as text[])) order by e.first_seen_at, e.id""", t=list(types)) |
| 289 |
376 |
|
| 290 |
377 |
|
| 291 |
|
−def _variant_index(models: list[dict[str, Any]]) -> tuple[dict[str, str], dict[str, list[dict[str, Any]]]]: |
| 292 |
|
− """variant_key → canonical model id (unique keys only) and the full multi-map.""" |
|
378 |
+def _variant_index(models: list[dict[str, Any]], aliases: dict[str, list[str]] | None = None) -> tuple[dict[str, str], dict[str, list[dict[str, Any]]]]: |
|
379 |
+ """variant_key → canonical model id (unique keys only) and the full multi-map; with `aliases` (model id → alias strings) every alias of a |
|
380 |
+ canonical model contributes its own key (`gpt-3.5-turbo-0613`, `o3-mini-2025-01-31`…).""" |
| 293 |
381 |
multi: dict[str, list[dict[str, Any]]] = defaultdict(list) |
| 294 |
382 |
for m in models: |
| 295 |
383 |
a = analyze_model_name(m["canonical_name"]) |
| 296 |
384 |
if a.is_effort_variant or a.is_artifact: |
| 297 |
385 |
continue |
| 298 |
|
− multi[variant_key(m["canonical_name"])].append(m) |
|
386 |
+ keys = {variant_key(m["canonical_name"])} |
|
387 |
+ for al in (aliases or {}).get(m["id"], []): |
|
388 |
+ aa = analyze_model_name(al) |
|
389 |
+ if not aa.is_effort_variant and not aa.is_artifact: |
|
390 |
+ keys.add(variant_key(al)) |
|
391 |
+ for k in keys: |
|
392 |
+ if k and m not in multi[k]: |
|
393 |
+ multi[k].append(m) |
| 299 |
394 |
return {k: v[0]["id"] for k, v in multi.items() if len(v) == 1}, multi |
| 300 |
395 |
|
| 301 |
396 |
|
|
397 |
+async def _model_aliases(conn: AsyncConnection, ids: list[str]) -> dict[str, list[str]]: |
|
398 |
+ out: dict[str, list[str]] = defaultdict(list) |
|
399 |
+ for r in await fetch_all(conn, "select entity_id, alias from entity_aliases where entity_id = any(cast(:ids as text[]))", ids=ids): |
|
400 |
+ out[r["entity_id"]].append(r["alias"]) |
|
401 |
+ return out |
|
402 |
+ |
|
403 |
+ |
| 302 |
404 |
async def step_variants(conn: AsyncConnection, rep: StepReport, apply: bool, *, scope: set[str] | None = None) -> None: |
| 303 |
405 |
models = await _models(conn) |
| 304 |
|
− index, _ = _variant_index(models) |
|
406 |
+ index, _ = _variant_index(models, await _model_aliases(conn, [m["id"] for m in models])) |
| 305 |
407 |
resolver = Resolver(conn, source_tier=2, variant_index=index) |
| 306 |
408 |
candidates = [m for m in models if _in_scope(scope, m["id"]) and analyze_model_name(m["canonical_name"]).is_effort_variant] |
|
409 |
+ # names that used to look like variants (before the ontology learnt they are tiers/official releases) drop their hint and regain full identity |
|
410 |
+ for m in models: |
|
411 |
+ if _in_scope(scope, m["id"]) and (m["attributes"] or {}).get("evaluation_variant_of_hint") and not analyze_model_name(m["canonical_name"]).is_effort_variant: |
|
412 |
+ rep.bump("stale_variant_hint_cleared") |
|
413 |
+ if apply: |
|
414 |
+ await execute(conn, """update entities set attributes = attributes - 'evaluation_variant_of_hint', provenance = provenance - 'evaluation_variant_of_hint', |
|
415 |
+ identity_confidence = 'high', updated_at = now() where id = :id""", id=m["id"]) |
|
416 |
+ await execute(conn, "update review_queue set status = 'rejected', resolved_at = now() where status = 'pending' and kind = 'variant_candidate' and :id = any(entity_ids)", id=m["id"]) |
| 307 |
417 |
idents = await _identifiers(conn, [m["id"] for m in candidates]) |
| 308 |
418 |
official = {r["entity_id"] for r in await fetch_all(conn, """select distinct entity_id from claims where tier = 1 and status = 'current' |
| 309 |
419 |
and entity_id = any(cast(:ids as text[]))""", ids=[m["id"] for m in candidates])} if candidates else set() |
| 320 |
430 |
if folded is None: |
| 321 |
431 |
hint = base_name(m["canonical_name"]) |
| 322 |
432 |
attrs = m["attributes"] or {} |
| 323 |
|
− if attrs.get("evaluation_variant_of_hint") == hint and m["identity_confidence"] == "medium": |
|
433 |
+ if attrs.get("evaluation_variant_of_hint") == hint and m["identity_confidence"] == "low": |
| 324 |
434 |
continue |
| 325 |
435 |
rep.bump("unresolved_flagged") |
| 326 |
|
− rep.example(f"unresolved variant {m['slug']} (base '{hint}' not found) → identity medium + review") |
|
436 |
+ rep.example(f"unresolved variant {m['slug']} (base '{hint}' not found) → identity low + review") |
| 327 |
437 |
if apply: |
| 328 |
438 |
await _set_attributes(conn, m["id"], {"evaluation_variant_of_hint": hint}, source_id=await registry_source_id(conn)) |
| 329 |
|
− await execute(conn, "update entities set identity_confidence = 'medium' where id = :id", id=m["id"]) |
|
439 |
+ await execute(conn, "update entities set identity_confidence = 'low' where id = :id", id=m["id"]) |
| 330 |
440 |
await _review(conn, "variant_candidate", [m["id"]], f"'{m['canonical_name']}' looks like an evaluation-effort variant of '{hint}' but no such model exists", |
| 331 |
441 |
{"slug": m["slug"], "base": hint, "effort": a.effort}, apply=apply) |
| 332 |
442 |
continue |
| 382 |
492 |
vk_candidates = [c for c in multi.get(variant_key(probe), []) if c["id"] != m["id"]] |
| 383 |
493 |
cid: str | None = None |
| 384 |
494 |
if official: |
| 385 |
|
− # the developer's own repo is the model (NVIDIA-Nemotron-3-Ultra-…-BF16, DeepSeek-R1 in native FP8) — it only becomes an |
| 386 |
|
− # artifact when the same organisation also has the plain model entity (tencent/Hy-MT2-7B-GGUF next to Hy-MT2-7B) AND nobody |
| 387 |
|
− # else (evaluators, providers) knows it under its own identity; otherwise the pair is a merge candidate for review |
| 388 |
|
− same_org = [c for c in vk_candidates if c["organization_id"] == m["organization_id"] and not analyze_model_name((c["attributes"] or {}).get("hf_repo") or c["canonical_name"]).is_artifact] |
| 389 |
|
− if len(same_org) != 1 or not has_token: |
|
495 |
+ # A quant/precision token in the repo name makes it an artifact whatever the organisation (an official FP8/GGUF repo is a |
|
496 |
+ # conversion of the model). Canonical = the same organisation's plain model with the same variant_key when it exists, else NULL. |
|
497 |
+ # Two guards, because a model is never invented or erased: (1) a native-dtype attribute without a token in the name |
|
498 |
+ # (DeepSeek-R1 `quant_format=fp8`) is the model; (2) when evaluators/providers know the tagged entity under its own identity |
|
499 |
+ # (Nemotron 3 Ultra whose only hub repo is `…-BF16`) it IS the model → merged into the plain sibling when one exists, kept otherwise. |
|
500 |
+ if not has_token: |
| 390 |
501 |
rep.bump("_official_checkpoint_kept_as_model") |
| 391 |
502 |
continue |
| 392 |
|
− schemes = set(official_idents.get(m["id"], {})) - {"hf_repo"} |
| 393 |
|
− if schemes: |
| 394 |
|
− if await _review(conn, "merge_candidate", sorted([m["id"], same_org[0]["id"]]), |
| 395 |
|
− f"'{m['canonical_name']}' and '{same_org[0]['canonical_name']}' look like one model published as two official checkpoints", |
| 396 |
|
− {"slugs": [m["slug"], same_org[0]["slug"]], "step": "artifacts"}, apply=apply): |
| 397 |
|
− rep.bump("review_official_checkpoint_pair") |
| 398 |
|
− rep.example(f"review: {m['slug']} (official repo with dtype tag, known to {sorted(schemes)}) vs {same_org[0]['slug']}") |
|
503 |
+ same_org = [c for c in vk_candidates if c["organization_id"] == m["organization_id"] and not analyze_model_name((c["attributes"] or {}).get("hf_repo") or c["canonical_name"]).is_artifact] |
|
504 |
+ external = set(official_idents.get(m["id"], {})) - {"hf_repo"} |
|
505 |
+ if external: |
|
506 |
+ if len(same_org) == 1 and not await kept_separate(conn, m["id"], same_org[0]["id"]): |
|
507 |
+ rep.bump("official_checkpoint_merged_into_model") |
|
508 |
+ rep.example(f"merge official checkpoint {m['slug']} → {same_org[0]['slug']} (known to {sorted(external)})") |
|
509 |
+ if apply: |
|
510 |
+ await merge_entities(conn, m["id"], same_org[0]["id"], mode="merge", actor="canonicalize", |
|
511 |
+ note="official checkpoint repo of the same release (dtype tag in the repo name)", payload={"step": "artifacts"}) |
|
512 |
+ else: |
|
513 |
+ rep.bump("_official_tagged_repo_is_the_model") |
| 399 |
514 |
continue |
| 400 |
|
− cid = same_org[0]["id"] |
|
515 |
+ cid = same_org[0]["id"] if len(same_org) == 1 else None |
| 401 |
516 |
if attr_quantized or a.is_quantized or a_name.is_quantized or quant_attr in QUANT_ATTR_FORMATS: |
| 402 |
517 |
kind = "quantization" |
| 403 |
518 |
elif a.is_conversion or a_name.is_conversion or a.precision or quant_attr: |
| 405 |
520 |
else: |
| 406 |
521 |
kind = "packaging" |
| 407 |
522 |
if cid is None: |
| 408 |
|
− cid = base_of.get(m["id"]) |
| 409 |
|
− if cid and (cid not in by_id or cid == m["id"]): |
| 410 |
|
− cid = None |
| 411 |
|
− if cid is None: |
|
523 |
+ # 1) a model with the same variant_key (the name says what was quantised); 2) the hub's `quantized_from` object, but only when it |
|
524 |
+ # names the same model — a fine-tune's GGUF (Hermes-2-Pro-Mistral-7B-GGUF) is not an artifact of the foundation model (Mistral 7B) |
| 412 |
525 |
candidates = vk_candidates |
| 413 |
526 |
if len(candidates) > 1: |
| 414 |
527 |
same_org = [c for c in candidates if c["organization_id"] == m["organization_id"]] |
| 416 |
529 |
candidates = same_org or official_c or candidates |
| 417 |
530 |
if len(candidates) == 1: |
| 418 |
531 |
cid = candidates[0]["id"] |
|
532 |
+ if cid is None: |
|
533 |
+ rel = base_of.get(m["id"]) |
|
534 |
+ if rel and rel in by_id and rel != m["id"]: |
|
535 |
+ base_probe = (by_id[rel]["attributes"] or {}).get("hf_repo") or by_id[rel]["canonical_name"] |
|
536 |
+ if variant_key(base_probe) == variant_key(probe) or analyze_model_name(base_probe).base_key in a.base_key: |
|
537 |
+ cid = rel |
| 419 |
538 |
if m["entity_type"] == "artifact" and m["canonical_id"] == cid: |
| 420 |
539 |
continue |
| 421 |
540 |
rep.bump("artifacts_marked" if cid else "artifacts_unresolved") |