LLM facts: one tier below source, plausible model names only, status vocabulary normalised
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 changed file +34 −6
modified
src/aiatlas/services/handlers.py
+34 −6
@@ -53,7 +53,8 @@ async def llm_extract(payload: dict[str, Any], job: dict[str, Any]) -> dict[str, | |||||
| 53 | facts = facts_from_llm(task, res.data, entity_id=snap["doc_entity_id"], entity_type=snap["entity_type"], entity_name=snap["canonical_name"], url=snap["url"]) | 53 | facts = facts_from_llm(task, res.data, entity_id=snap["doc_entity_id"], entity_type=snap["entity_type"], entity_name=snap["canonical_name"], url=snap["url"]) | ||
| 54 | async with transaction() as conn: | 54 | async with transaction() as conn: | ||
| 55 | tier = await fetch_one(conn, "select tier from sources where id = :id", id=snap["source_id"]) if snap["source_id"] else None | 55 | tier = await fetch_one(conn, "select tier from sources where id = :id", id=snap["source_id"]) if snap["source_id"] else None | ||
| 56 | − writer = FactWriter(conn, source_id=snap["source_id"], snapshot_id=snapshot_id, source_url=snap["url"], tier=(tier["tier"] if tier else 2), | 56 | + # LLM output never outranks a deterministic statement from the same source: one tier lower (disagreement → conflicting + review) | ||
| 57 | + writer = FactWriter(conn, source_id=snap["source_id"], snapshot_id=snapshot_id, source_url=snap["url"], tier=min(4, (tier["tier"] if tier else 2) + 1), | ||||
| 57 | connector_name=snap["connector_name"], extractor="llm", extractor_version=res.model, observed_at=snap["observed_at"].astimezone(UTC)) | 58 | connector_name=snap["connector_name"], extractor="llm", extractor_version=res.model, observed_at=snap["observed_at"].astimezone(UTC)) | ||
| 58 | ws = await writer.write(facts) | 59 | ws = await writer.write(facts) | ||
| 59 | await execute(conn, "update snapshots set processing_status = 'llm_done' where id = :id", id=snapshot_id) | 60 | await execute(conn, "update snapshots set processing_status = 'llm_done' where id = :id", id=snapshot_id) | ||
@@ -78,11 +79,34 @@ def _guess_task(doc_type: str | None, entity_type: str | None) -> str: | |||||
| 78 | return "classify_then_extract" | 79 | return "classify_then_extract" | ||
| 79 | 80 | ||||
| 80 | 81 | ||||
| 82 | +GENERIC_MODEL_NAMES = {"claude", "gpt", "gemini", "llama", "mistral", "qwen", "deepseek", "grok", "gemma", "phi", "command", "codex", "sora", "veo", "imagen", | ||||
| 83 | + "model", "models", "the model", "new model", "ai model", "llm", "openai", "anthropic", "google", "meta", "microsoft", "nvidia"} | ||||
| 84 | +STATUS_ALIASES = {"available": "active", "ga": "active", "generally available": "active", "live": "active", "released": "active", "beta": "preview", | ||||
| 85 | + "experimental": "preview", "coming soon": "announced", "sunset": "retired", "discontinued": "retired", "legacy": "deprecated"} | ||||
| 86 | + | ||||
| 87 | + | ||||
| 88 | +def plausible_model_name(name: str | None) -> bool: | ||||
| 89 | + """Reject family/vendor names and vague phrases the LLM sometimes returns as a model name.""" | ||||
| 90 | + if not name: | ||||
| 91 | + return False | ||||
| 92 | + n = name.strip() | ||||
| 93 | + low = n.lower() | ||||
| 94 | + if low in GENERIC_MODEL_NAMES or len(n) < 3 or len(n) > 80: | ||||
| 95 | + return False | ||||
| 96 | + return any(ch.isdigit() for ch in n) or len(n.split()) >= 2 or "-" in n | ||||
| 97 | + | ||||
| 98 | + | ||||
| 99 | +def _norm_status(v: Any) -> Any: | ||||
| 100 | + return STATUS_ALIASES.get(v.strip().lower(), v.strip().lower()) if isinstance(v, str) else v | ||||
| 101 | + | ||||
| 102 | + | ||||
| 81 | def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str | None, entity_type: str | None, entity_name: str | None, url: str) -> Facts: | 103 | def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str | None, entity_type: str | None, entity_name: str | None, url: str) -> Facts: | ||
| 82 | − """Map a validated LLM output onto facts. LLM claims default to 'medium' confidence and never outrank tier-1 deterministic ones | 104 | + """Map a validated LLM output onto facts. LLM claims default to 'medium' confidence and are written one tier below their source, | ||
| 83 | − (the writer stores disagreements as conflicting).""" | 105 | + so they never outrank deterministic statements (the writer stores disagreements as conflicting).""" | ||
| 84 | facts = Facts() | 106 | facts = Facts() | ||
| 85 | conf = "medium" | 107 | conf = "medium" | ||
| 108 | + if "status" in data: | ||||
| 109 | + data = {**data, "status": _norm_status(data.get("status"))} | ||||
| 86 | 110 | ||||
| 87 | def model_ref(name: str, developer: str | None = None) -> EntityRef: | 111 | def model_ref(name: str, developer: str | None = None) -> EntityRef: | ||
| 88 | org = facts.entity("company", developer) if developer else None | 112 | org = facts.entity("company", developer) if developer else None | ||
@@ -90,7 +114,7 @@ def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str | None, en | |||||
| 90 | 114 | ||||
| 91 | if task == "model_passport": | 115 | if task == "model_passport": | ||
| 92 | name = data.get("name") or entity_name | 116 | name = data.get("name") or entity_name | ||
| 93 | − if not name: | 117 | + if not name or (not (entity_id and entity_type == "model") and not plausible_model_name(name)): | ||
| 94 | return facts | 118 | return facts | ||
| 95 | ref = EntityRef(entity_type="model", name=name, id=entity_id) if entity_id and entity_type == "model" else model_ref(name, data.get("developer")) | 119 | ref = EntityRef(entity_type="model", name=name, id=entity_id) if entity_id and entity_type == "model" else model_ref(name, data.get("developer")) | ||
| 96 | if ref not in facts.entities: | 120 | if ref not in facts.entities: | ||
@@ -144,7 +168,8 @@ def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str | None, en | |||||
| 144 | if data.get("description"): | 168 | if data.get("description"): | ||
| 145 | facts.claim(ref, "description", data["description"], confidence=conf) | 169 | facts.claim(ref, "description", data["description"], confidence=conf) | ||
| 146 | for m in data.get("models") or []: | 170 | for m in data.get("models") or []: | ||
| 147 | − facts.relate(ref, "develops", facts.entity("model", m, organization=ref), confidence="low") | 171 | + if plausible_model_name(m): | ||
| 172 | + facts.relate(ref, "develops", facts.entity("model", m, organization=ref), confidence="low") | ||||
| 148 | for inv in data.get("investors") or []: | 173 | for inv in data.get("investors") or []: | ||
| 149 | facts.relate(ref, "funded_by", facts.entity("company", inv), confidence="low") | 174 | facts.relate(ref, "funded_by", facts.entity("company", inv), confidence="low") | ||
| 150 | if data.get("parent_company"): | 175 | if data.get("parent_company"): | ||
@@ -160,7 +185,8 @@ def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str | None, en | |||||
| 160 | for prop in ("authors", "affiliations", "date", "field", "summary", "methods", "key_claims", "results", "limitations", "code_url"): | 185 | for prop in ("authors", "affiliations", "date", "field", "summary", "methods", "key_claims", "results", "limitations", "code_url"): | ||
| 161 | facts.claim(ref, prop, data.get(prop), confidence=conf) | 186 | facts.claim(ref, prop, data.get(prop), confidence=conf) | ||
| 162 | for m in data.get("models") or []: | 187 | for m in data.get("models") or []: | ||
| 163 | − facts.relate(facts.entity("model", m), "described_by", ref, confidence="low") | 188 | + if plausible_model_name(m): | ||
| 189 | + facts.relate(facts.entity("model", m), "described_by", ref, confidence="low") | ||||
| 164 | for d in data.get("datasets") or []: | 190 | for d in data.get("datasets") or []: | ||
| 165 | facts.relate(ref, "uses_dataset", facts.entity("dataset", d), confidence="low") | 191 | facts.relate(ref, "uses_dataset", facts.entity("dataset", d), confidence="low") | ||
| 166 | for b in data.get("benchmarks") or []: | 192 | for b in data.get("benchmarks") or []: | ||
@@ -192,6 +218,8 @@ def facts_from_llm(task: str, data: dict[str, Any], *, entity_id: str | None, en | |||||
| 192 | elif task == "release_announcement": | 218 | elif task == "release_announcement": | ||
| 193 | org = facts.entity("company", data["organization"]) if data.get("organization") else None | 219 | org = facts.entity("company", data["organization"]) if data.get("organization") else None | ||
| 194 | for m in data.get("models") or []: | 220 | for m in data.get("models") or []: | ||
| 221 | + if not plausible_model_name(m): | ||||
| 222 | + continue | ||||
| 195 | ref = facts.entity("model", m, organization=org) | 223 | ref = facts.entity("model", m, organization=org) | ||
| 196 | if org: | 224 | if org: | ||
| 197 | facts.relate(org, "develops", ref, confidence="low") | 225 | facts.relate(org, "develops", ref, confidence="low") | ||
| 198 | 226 | ||||