LLM facts: one tier below source, plausible model names only, status vocabulary normalised

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 13 days ago (Sep 11, 2026) parent 58605ba

1 changed file +34 −6

modified src/aiatlas/services/handlers.py +34 −6

		`@@ -53,7 +53,8 @@ async def llm_extract(payload: dict[str, Any], job: dict[str, Any]) -> dict[str,`
53	53	facts = facts_from_llm(task, res.data, entity_id=snap["doc_entity_id"], entity_type=snap["entity_type"], entity_name=snap["canonical_name"], url=snap["url"])
54	54	async with transaction() as conn:
55	55	tier = await fetch_one(conn, "select tier from sources where id = :id", id=snap["source_id"]) if snap["source_id"] else None
56		− writer = FactWriter(conn, source_id=snap["source_id"], snapshot_id=snapshot_id, source_url=snap["url"], tier=(tier["tier"] if tier else 2),
	56	+ # LLM output never outranks a deterministic statement from the same source: one tier lower (disagreement → conflicting + review)
	57	+ writer = FactWriter(conn, source_id=snap["source_id"], snapshot_id=snapshot_id, source_url=snap["url"], tier=min(4, (tier["tier"] if tier else 2) + 1),
57	58	connector_name=snap["connector_name"], extractor="llm", extractor_version=res.model, observed_at=snap["observed_at"].astimezone(UTC))
58	59	ws = await writer.write(facts)
59	60	await execute(conn, "update snapshots set processing_status = 'llm_done' where id = :id", id=snapshot_id)
		`@@ -78,11 +79,34 @@ def _guess_task(doc_type: str \| None, entity_type: str \| None) -> str:`
78	79	return "classify_then_extract"
79	80
80	81
	82	+GENERIC_MODEL_NAMES = {"claude", "gpt", "gemini", "llama", "mistral", "qwen", "deepseek", "grok", "gemma", "phi", "command", "codex", "sora", "veo", "imagen",
	83	+ "model", "models", "the model", "new model", "ai model", "llm", "openai", "anthropic", "google", "meta", "microsoft", "nvidia"}
	84	+STATUS_ALIASES = {"available": "active", "ga": "active", "generally available": "active", "live": "active", "released": "active", "beta": "preview",
	85	+ "experimental": "preview", "coming soon": "announced", "sunset": "retired", "discontinued": "retired", "legacy": "deprecated"}
	86	+
	87	+
	88	+def plausible_model_name(name: str \| None) -> bool:
	89	+ """Reject family/vendor names and vague phrases the LLM sometimes returns as a model name."""
	90	+ if not name:
	91	+ return False
	92	+ n = name.strip()
	93	+ low = n.lower()
	94	+ if low in GENERIC_MODEL_NAMES or len(n) < 3 or len(n) > 80:
	95	+ return False
	96	+ return any(ch.isdigit() for ch in n) or len(n.split()) >= 2 or "-" in n
	97	+
	98	+
	99	+def _norm_status(v: Any) -> Any:
	100	+ return STATUS_ALIASES.get(v.strip().lower(), v.strip().lower()) if isinstance(v, str) else v
	101	+
	102	+
81	103	def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str \| None, entity_type: str \| None, entity_name: str \| None, url: str) -> Facts:
82		− """Map a validated LLM output onto facts. LLM claims default to 'medium' confidence and never outrank tier-1 deterministic ones
83		− (the writer stores disagreements as conflicting)."""
	104	+ """Map a validated LLM output onto facts. LLM claims default to 'medium' confidence and are written one tier below their source,
	105	+ so they never outrank deterministic statements (the writer stores disagreements as conflicting)."""
84	106	facts = Facts()
85	107	conf = "medium"
	108	+ if "status" in data:
	109	+ data = {**data, "status": _norm_status(data.get("status"))}
86	110
87	111	def model_ref(name: str, developer: str \| None = None) -> EntityRef:
88	112	org = facts.entity("company", developer) if developer else None
		`@@ -90,7 +114,7 @@ def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str \| None, en`
90	114
91	115	if task == "model_passport":
92	116	name = data.get("name") or entity_name
93		− if not name:
	117	+ if not name or (not (entity_id and entity_type == "model") and not plausible_model_name(name)):
94	118	return facts
95	119	ref = EntityRef(entity_type="model", name=name, id=entity_id) if entity_id and entity_type == "model" else model_ref(name, data.get("developer"))
96	120	if ref not in facts.entities:
		`@@ -144,7 +168,8 @@ def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str \| None, en`
144	168	if data.get("description"):
145	169	facts.claim(ref, "description", data["description"], confidence=conf)
146	170	for m in data.get("models") or []:
147		− facts.relate(ref, "develops", facts.entity("model", m, organization=ref), confidence="low")
	171	+ if plausible_model_name(m):
	172	+ facts.relate(ref, "develops", facts.entity("model", m, organization=ref), confidence="low")
148	173	for inv in data.get("investors") or []:
149	174	facts.relate(ref, "funded_by", facts.entity("company", inv), confidence="low")
150	175	if data.get("parent_company"):
		`@@ -160,7 +185,8 @@ def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str \| None, en`
160	185	for prop in ("authors", "affiliations", "date", "field", "summary", "methods", "key_claims", "results", "limitations", "code_url"):
161	186	facts.claim(ref, prop, data.get(prop), confidence=conf)
162	187	for m in data.get("models") or []:
163		− facts.relate(facts.entity("model", m), "described_by", ref, confidence="low")
	188	+ if plausible_model_name(m):
	189	+ facts.relate(facts.entity("model", m), "described_by", ref, confidence="low")
164	190	for d in data.get("datasets") or []:
165	191	facts.relate(ref, "uses_dataset", facts.entity("dataset", d), confidence="low")
166	192	for b in data.get("benchmarks") or []:
		`@@ -192,6 +218,8 @@ def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str \| None, en`
192	218	elif task == "release_announcement":
193	219	org = facts.entity("company", data["organization"]) if data.get("organization") else None
194	220	for m in data.get("models") or []:
	221	+ if not plausible_model_name(m):
	222	+ continue
195	223	ref = facts.entity("model", m, organization=org)
196	224	if org:
197	225	facts.relate(org, "develops", ref, confidence="low")
198	226