"""FactWriter rules against the local dev database, inside a transaction that is always rolled back (nothing persists).""" from __future__ import annotations import uuid from collections.abc import AsyncIterator from datetime import UTC, datetime, timedelta import pytest from sqlalchemy.ext.asyncio import AsyncConnection from aiatlas import db from aiatlas.db import fetch_all, fetch_one from aiatlas.sdk.facts import Claim, EntityRef, Event, Facts, PriceObs, ResultObs, Target, facts_from_json, facts_to_json from aiatlas.sdk.writer import FactWriter pytestmark = pytest.mark.usefixtures("conn") @pytest.fixture async def conn() -> AsyncIterator[AsyncConnection]: async with db.engine().connect() as c: trans = await c.begin() try: yield c finally: await trans.rollback() await db.dispose() def _tag() -> str: return uuid.uuid4().hex[:8] def _writer(conn: AsyncConnection, *, tier: int = 1, extractor: str = "deterministic", url: str = "https://example.com/doc", run_id: str | None = "run_test", observed_at: datetime | None = None, source_key: str | None = None, is_first_run: bool = False) -> FactWriter: return FactWriter(conn, source_id=None, snapshot_id=None, source_url=url, tier=tier, connector_name="test", extractor=extractor, extractor_version="t", observed_at=observed_at, run_id=run_id, source_key=source_key or "", is_first_run=is_first_run) # ---------------------------------------------------------------------------------------------- pure: Facts JSON round-trip def test_facts_json_round_trip() -> None: f = Facts() org = f.entity("company", "Zorg Labs", identifiers={"domain": "zorg.example"}) m = f.entity("model", "Zeta 9", identifiers={"hf_repo": "zorg/Zeta-9"}, organization=org, attributes={"license": "Apache 2.0", "released": datetime(2026, 1, 2, tzinfo=UTC)}, first_seen_hint=datetime(2025, 12, 31, tzinfo=UTC), family=EntityRef("model_family", "Zeta"), artifact_kind=None) f.claim(m, "context_length", 128000, unit="tokens", observed_at=datetime(2026, 1, 3, tzinfo=UTC)) f.relate(org, "develops", m, attributes={"role": "developer"}) f.event("RELEASE", "model", "Zeta 9 released", entity=m, effective_at=datetime(2026, 1, 2, tzinfo=UTC), importance=3) f.price(model=m, provider=f.entity("provider", "Zorg Cloud"), input_per_mtok=1.5, output_per_mtok=6.0, provider_model_id="zeta-9") f.result(model=m, benchmark=f.entity("benchmark", "GPQA Diamond"), score=71.2, metric="accuracy", config={"variant": "Diamond"}, evaluated_at=datetime(2026, 1, 5, tzinfo=UTC)) f.follow("https://zorg.example/more", doc_type="page", key="more", meta={"x": 1}) f.document_entity = m f.document_title = "Zeta 9" data = facts_to_json(f) back = facts_from_json(data) assert [e.name for e in back.entities] == [e.name for e in f.entities] assert back.entities[1].first_seen_hint == datetime(2025, 12, 31, tzinfo=UTC) and back.entities[1].family and back.entities[1].family.name == "Zeta" assert back.entities[1].attributes["released"] == datetime(2026, 1, 2, tzinfo=UTC) and back.entities[1].organization.identifiers == {"domain": "zorg.example"} assert back.claims[0].observed_at == datetime(2026, 1, 3, tzinfo=UTC) and back.claims[0].entity.key() == m.key() assert back.relations[0].attributes == {"role": "developer"} and back.events[0].effective_at == datetime(2026, 1, 2, tzinfo=UTC) assert back.prices[0].input_per_mtok == 1.5 and back.results[0].evaluated_at == datetime(2026, 1, 5, tzinfo=UTC) and back.results[0].config == {"variant": "Diamond"} assert back.targets[0].key == "more" and back.document_entity.key() == m.key() and back.document_title == "Zeta 9" assert facts_to_json(back) == data # ---------------------------------------------------------------------------------------------- taxonomy at write time async def test_license_normalised_with_raw_and_mapping(conn: AsyncConnection) -> None: t = _tag() f = Facts() m = f.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}, attributes={"license": "Apache 2.0", "openness": "restricted", "modalities": ["Text", "pdf"]}) w = _writer(conn) await w.write(f) claims = {r["property"]: r for r in await fetch_all(conn, "select * from claims where entity_id = :e and status = 'current'", e=m.id)} assert claims["license"]["value"] == "Apache-2.0" and claims["license"]["value_raw"] == "Apache 2.0" and claims["license"]["run_id"] == "run_test" assert claims["license_key"]["value"] == "Apache-2.0" assert claims["openness"]["value"] == "restricted-weights" and claims["openness"]["value_raw"] == "restricted" assert claims["modalities"]["value"] == ["document", "text"] ent = await fetch_one(conn, "select attributes from entities where id = :e", e=m.id) assert ent["attributes"]["license"] == "Apache-2.0" and ent["attributes"]["license_raw"] == "Apache 2.0" and ent["attributes"]["license_key"] == "Apache-2.0" assert ent["attributes"]["modalities"] == ["document", "text"] and ent["attributes"]["modalities_raw"] == "Text, pdf" mapping = await fetch_one(conn, "select canonical from taxonomy_mappings where domain = 'license' and raw = 'Apache 2.0'") assert mapping and mapping["canonical"] == "Apache-2.0" # a second source spelling the same licence differently confirms, never a LICENSE_CHANGED event f2 = Facts() m2 = f2.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}, attributes={"license": "apache-2.0"}) await _writer(conn, tier=2, url="https://other.example/x").write(f2) assert m2.id == m.id n = await fetch_one(conn, "select count(*) as n from claims where entity_id = :e and property = 'license' and status = 'current'", e=m.id) assert n["n"] == 1 ev = await fetch_all(conn, "select event_type from change_events where entity_id = :e", e=m.id) assert {e["event_type"] for e in ev} == {"NEW_MODEL"} async def test_unknown_taxonomy_value_kept(conn: AsyncConnection) -> None: t = _tag() f = Facts() m = f.entity("model", f"Zeta {t}", attributes={"license": f"custom-{t}"}) await _writer(conn).write(f) c = await fetch_one(conn, "select value, value_raw from claims where entity_id = :e and property = 'license'", e=m.id) assert c["value"] == f"custom-{t}" and c["value_raw"] is None mp = await fetch_one(conn, "select canonical from taxonomy_mappings where domain = 'license' and raw = :r", r=f"custom-{t}") assert mp and mp["canonical"] is None # ---------------------------------------------------------------------------------------------- same-source loophole & supersede async def test_llm_never_supersedes_deterministic_same_source(conn: AsyncConnection) -> None: t = _tag() url = f"https://docs.example/{t}" f = Facts() m = f.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}, attributes={"context_length": 100_000}) await _writer(conn, tier=1, url=url).write(f) # LLM extraction from the same URL, one tier lower f2 = Facts() m2 = f2.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}, attributes={"context_length": 200_000}) llm = _writer(conn, tier=2, extractor="llm", url=url) await llm.write(f2) rows = await fetch_all(conn, "select value, status, extractor from claims where entity_id = :e and property = 'context_length' order by value_num", e=m.id) assert [(r["value"], r["status"]) for r in rows] == [(100_000, "current"), (200_000, "conflicting")] assert llm.stats.conflicts == 1 assert await fetch_one(conn, "select 1 from review_queue where kind = 'conflict' and :e = any(entity_ids)", e=m.id) # the deterministic extractor correcting itself from the same URL supersedes → CONTEXT_CHANGED with deterministic importance f3 = Facts() f3.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}, attributes={"context_length": 1_000_000}) await _writer(conn, tier=1, url=url).write(f3) cur = await fetch_one(conn, "select value from claims where entity_id = :e and property = 'context_length' and status = 'current'", e=m.id) assert cur["value"] == 1_000_000 and m2.id == m.id ev = await fetch_one(conn, "select importance, is_backfill, run_id, recorded_at from change_events where entity_id = :e and event_type = 'CONTEXT_CHANGED'", e=m.id) assert ev and ev["importance"] == 3 and ev["is_backfill"] is False and ev["run_id"] == "run_test" and ev["recorded_at"] is not None async def test_derived_writer_conflicts_without_review_items(conn: AsyncConnection) -> None: t = _tag() f = Facts() m = f.entity("model", f"Zeta {t}", attributes={"openness": "open-weights"}) await _writer(conn, tier=1).write(f) f2 = Facts() f2.claim(EntityRef("model", f"Zeta {t}", id=m.id), "openness", "restricted-weights") w = _writer(conn, tier=2, extractor="derived", url=None) await w.write(f2) assert w.stats.conflicts == 1 assert (await fetch_one(conn, "select value from claims where entity_id = :e and property = 'openness' and status = 'current'", e=m.id))["value"] == "open-weights" assert not await fetch_one(conn, "select 1 from review_queue where kind = 'conflict' and :e = any(entity_ids)", e=m.id) # ---------------------------------------------------------------------------------------------- results: comparability, one current row per key, bounds async def test_results_one_current_per_config_key(conn: AsyncConnection) -> None: t = _tag() f = Facts() m = f.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}) b = f.entity("benchmark", f"LiveBench {t}", identifiers={"registry_benchmark": f"livebench-{t}"}) f.result(model=m, benchmark=b, score=61.0, metric="global_average", unit="%", config={"release": "2026-05-01", "livebench_model_id": "zeta"}) await _writer(conn, tier=2, source_key="livebench.ai").write(f) first = await fetch_one(conn, "select * from benchmark_results where model_id = :m", m=m.id) assert first["config_key"] and first["trust_level"] == "official-benchmark" and first["run_group"] == "2026-05-01" and first["is_current"] is True assert first["run_id"] == "run_test" and first["extractor"] == "deterministic" f2 = Facts() m2 = f2.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}) b2 = f2.entity("benchmark", f"LiveBench {t}", identifiers={"registry_benchmark": f"livebench-{t}"}) f2.result(model=m2, benchmark=b2, score=64.0, metric="global_average", unit="%", config={"release": "2026-06-25", "livebench_model_id": "zeta"}) await _writer(conn, tier=2, source_key="livebench.ai", observed_at=datetime.now(UTC) + timedelta(seconds=1)).write(f2) rows = await fetch_all(conn, "select score, is_current, valid_to, config_key, run_group from benchmark_results where model_id = :m order by observed_at", m=m.id) assert [(r["score"], r["is_current"], r["valid_to"] is None) for r in rows] == [(61.0, False, False), (64.0, True, True)] assert rows[0]["config_key"] == rows[1]["config_key"] # same run group, different condition (reasoning effort) → both stay current, same config_key f3 = Facts() m3 = f3.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}) b3 = f3.entity("benchmark", f"LiveBench {t}", identifiers={"registry_benchmark": f"livebench-{t}"}) f3.result(model=m3, benchmark=b3, score=66.0, metric="global_average", unit="%", config={"release": "2026-06-25", "livebench_model_id": "zeta", "reasoning_effort": "high"}) await _writer(conn, tier=2, source_key="livebench.ai", observed_at=datetime.now(UTC) + timedelta(seconds=2)).write(f3) cur = await fetch_all(conn, "select score from benchmark_results where model_id = :m and is_current order by score", m=m.id) assert [r["score"] for r in cur] == [64.0, 66.0] async def test_result_out_of_bounds_flagged(conn: AsyncConnection) -> None: t = _tag() f = Facts() m = f.entity("model", f"Zeta {t}") b = f.entity("benchmark", f"Bench {t}") f.result(model=m, benchmark=b, score=140.0, metric="accuracy", unit="%") await _writer(conn, tier=2).write(f) r = await fetch_one(conn, "select confidence from benchmark_results where model_id = :m", m=m.id) assert r["confidence"] == "low" a = await fetch_one(conn, "select check_name, severity, status from anomalies where entity_id = :m", m=m.id) assert a and a["check_name"] == "score_above_max" and a["severity"] == "critical" and a["status"] == "open" async def test_effort_variant_result_config_augmented(conn: AsyncConnection) -> None: """An evaluator row named '-high' lands on the canonical model with reasoning_effort in the config.""" t = _tag() base = Facts() m = base.entity("model", f"Zeta {t}", identifiers={"artificial_analysis": f"zeta-{t}"}) await _writer(conn, tier=2, source_key="artificialanalysis.ai").write(base) f = Facts() v = f.entity("model", f"zeta-{t}-high", identifiers={"artificial_analysis": f"zeta-{t}-high"}) b = f.entity("benchmark", f"GPQA {t}", identifiers={"registry_benchmark": f"gpqa-{t}"}) f.result(model=v, benchmark=b, score=80.0, metric="accuracy", unit="%", config={"evaluator": "Artificial Analysis", "index_version": "4.3", "aa_slug": f"zeta-{t}-high"}) w = _writer(conn, tier=2, source_key="artificialanalysis.ai") await w.write(f) assert v.id == m.id and w.stats.folded_variants == 1 r = await fetch_one(conn, "select config, trust_level from benchmark_results where model_id = :m", m=m.id) assert r["config"]["reasoning_effort"] == "high" and r["config"]["aa_variant_slug"] == f"zeta-{t}-high" and r["trust_level"] == "independent-evaluator" ids = await fetch_all(conn, "select value from entity_identifiers where entity_id = :m and scheme = 'artificial_analysis' order by value", m=m.id) assert [i["value"] for i in ids] == [f"zeta-{t}", f"zeta-{t}-high"] assert not await fetch_one(conn, "select 1 from entities where canonical_name = :n", n=f"zeta-{t}-high") # ---------------------------------------------------------------------------------------------- events: backfill, group key, NEW_* importance async def test_new_entity_events_backfill_and_grouping(conn: AsyncConnection) -> None: t = _tag() f = Facts() m = f.entity("model", f"Zeta {t}", attributes={"release_date": "2024-01-15"}) await _writer(conn, tier=1).write(f) ev = await fetch_one(conn, "select is_backfill, group_key, effective_at, importance from change_events where entity_id = :e and event_type = 'NEW_MODEL'", e=m.id) assert ev["is_backfill"] is True and ev["group_key"] == f"release:{m.id}:2024-01" and ev["importance"] == 2 # first run of a connector → everything is backfill even without dates f2 = Facts() m2 = f2.entity("model", f"Zeta {t} b") await _writer(conn, tier=1, is_first_run=True).write(f2) ev2 = await fetch_one(conn, "select is_backfill from change_events where entity_id = :e", e=m2.id) assert ev2["is_backfill"] is True # artifacts and families never make importance-3 news f3 = Facts() art = f3.entity("artifact", f"unsloth/Zeta-{t}-GGUF", identifiers={"hf_repo": f"unsloth/Zeta-{t}-GGUF"}, canonical=EntityRef("model", f"Zeta {t}", id=m.id), artifact_kind="quantization") fam = f3.entity("model_family", f"Zeta family {t}") await _writer(conn, tier=2).write(f3) imps = {r["event_type"]: r["importance"] for r in await fetch_all(conn, "select event_type, importance from change_events where entity_id in (:a, :f)", a=art.id, f=fam.id)} assert imps == {"NEW_ARTIFACT": 0, "NEW_MODEL_FAMILY": 1} row = await fetch_one(conn, "select canonical_id, artifact_kind from entities where id = :a", a=art.id) assert row["canonical_id"] == m.id and row["artifact_kind"] == "quantization" assert await fetch_one(conn, "select 1 from relations where subject_id = :a and predicate = 'artifact_of' and object_id = :m and valid_to is null", a=art.id, m=m.id) async def test_family_hint_and_first_seen_hint(conn: AsyncConnection) -> None: t = _tag() f = Facts() m = f.entity("model", f"Llama {t} 9B", family=EntityRef("model_family", f"Llama {t}"), first_seen_hint=datetime(2024, 3, 1, tzinfo=UTC), identity_confidence="medium") await _writer(conn, tier=1).write(f) row = await fetch_one(conn, "select family_id, first_seen_at, identity_confidence from entities where id = :m", m=m.id) assert row["family_id"] == m.family.id and row["first_seen_at"] == datetime(2024, 3, 1, tzinfo=UTC) and row["identity_confidence"] == "medium" assert await fetch_one(conn, "select 1 from relations where subject_id = :m and predicate = 'member_of_family' and object_id = :f", m=m.id, f=m.family.id) fam = await fetch_one(conn, "select entity_type, slug from entities where id = :f", f=m.family.id) assert fam["entity_type"] == "model_family" async def test_price_change_importance_and_run_id(conn: AsyncConnection) -> None: t = _tag() f = Facts() m = f.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}) p = f.entity("provider", f"Zorg Cloud {t}") f.price(model=m, provider=p, provider_model_id="zeta", input_per_mtok=10.0, output_per_mtok=30.0) await _writer(conn, tier=1).write(f) f2 = Facts() m2 = f2.entity("model", f"Zeta {t}", identifiers={"hf_repo": f"zorg/zeta-{t}"}) p2 = f2.entity("provider", f"Zorg Cloud {t}") f2.price(model=m2, provider=p2, provider_model_id="zeta", input_per_mtok=4.0, output_per_mtok=30.0) await _writer(conn, tier=1, run_id="run_2", observed_at=datetime.now(UTC) + timedelta(seconds=1)).write(f2) ev = await fetch_one(conn, "select importance, run_id from change_events where entity_id = :m and event_type = 'PRICE_CHANGED'", m=m.id) assert ev["importance"] == 3 and ev["run_id"] == "run_2" # -60 % prices = await fetch_all(conn, "select input_per_mtok, run_id, valid_to from prices where model_id = :m order by valid_from", m=m.id) assert [(r["input_per_mtok"], r["run_id"], r["valid_to"] is None) for r in prices] == [(10.0, "run_test", False), (4.0, "run_2", True)] async def test_target_and_unused_imports_keep_dataclasses_stable() -> None: assert Target(url="https://x").doc_type == "page" and Claim(EntityRef("model", "x"), "p", 1).unit is None assert Event("RELEASE", "model", "s").importance == 2 and PriceObs(EntityRef("model", "m"), EntityRef("provider", "p")).currency == "USD" assert ResultObs(EntityRef("model", "m"), EntityRef("benchmark", "b"), 1.0).trust_level is None