HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""What a connector produces: entity references, temporal claims, relations, events, prices, benchmark results.2Connectors never touch the database — they return `Facts`; the writer resolves, versions and records them."""3from __future__ import annotations45from dataclasses import dataclass, field6from datetime import datetime7from typing import Any89CONFIDENCE = ("verified", "high", "medium", "low", "conflicted")101112@dataclass13class EntityRef:14 entity_type: str15 name: str16 identifiers: dict[str, str] = field(default_factory=dict) # scheme -> value (hf_repo, github_repo, arxiv, doi, domain, pypi, provider_model_id…)17 aliases: list[str] = field(default_factory=list)18 slug_hint: str | None = None19 organization: EntityRef | None = None # developer / publisher / owner20 description: str | None = None21 status: str | None = None22 attributes: dict[str, Any] = field(default_factory=dict) # convenience: each item becomes a Claim23 first_seen_hint: datetime | None = None # e.g. release date, to backdate `first_seen_at` for historical backfill24 # canonical hierarchy hints (model_family → model → artifact): the writer materialises `entities.family_id` / `canonical_id`25 family: EntityRef | None = None # model → its model_family ("Qwen3.6", "Llama 4", "Claude")26 canonical: EntityRef | None = None # artifact → the model it packages/quantises; alias entity → canonical entity27 artifact_kind: str | None = None # artifacts only: checkpoint|quantization|conversion|packaging28 identity_confidence: str | None = None # high|medium|low — how sure the connector is that this is one real thing29 id: str | None = field(default=None, compare=False)3031 def key(self) -> str:32 if self.identifiers:33 scheme, value = sorted(self.identifiers.items())[0]34 return f"{self.entity_type}:{scheme}={value}"35 return f"{self.entity_type}:name={self.name.strip().lower()}"363738@dataclass39class Claim:40 entity: EntityRef41 property: str42 value: Any43 unit: str | None = None44 confidence: str | None = None # default from source tier45 observed_at: datetime | None = None46 effective_at: datetime | None = None47 source_url: str | None = None484950@dataclass51class Relation:52 subject: EntityRef53 predicate: str54 object: EntityRef55 attributes: dict[str, Any] = field(default_factory=dict)56 confidence: str | None = None57 source_url: str | None = None585960@dataclass61class Event:62 event_type: str63 category: str64 summary: str65 entity: EntityRef | None = None66 old_value: Any = None67 new_value: Any = None68 importance: int = 269 effective_at: datetime | None = None70 dedupe_key: str | None = None71 source_url: str | None = None72 meta: dict[str, Any] = field(default_factory=dict)737475@dataclass76class PriceObs:77 model: EntityRef78 provider: EntityRef79 provider_model_id: str | None = None80 input_per_mtok: float | None = None81 output_per_mtok: float | None = None82 cached_input_per_mtok: float | None = None83 cache_write_per_mtok: float | None = None84 batch_input_per_mtok: float | None = None85 batch_output_per_mtok: float | None = None86 per_image: float | None = None87 per_request: float | None = None88 currency: str = "USD"89 context_length: int | None = None90 max_output_tokens: int | None = None91 features: dict[str, Any] = field(default_factory=dict)92 source_url: str | None = None93 meta: dict[str, Any] = field(default_factory=dict)9495 def price_tuple(self) -> tuple[Any, ...]:96 return (self.input_per_mtok, self.output_per_mtok, self.cached_input_per_mtok, self.cache_write_per_mtok,97 self.batch_input_per_mtok, self.batch_output_per_mtok, self.per_image, self.per_request, self.currency)9899100@dataclass101class ResultObs:102 model: EntityRef103 benchmark: EntityRef104 score: float105 metric: str | None = None106 unit: str | None = "%"107 higher_is_better: bool = True108 config: dict[str, Any] = field(default_factory=dict)109 evaluated_at: datetime | None = None110 source_url: str | None = None111 confidence: str | None = None112 trust_level: str | None = None # ontology.benchmarks.TRUST_LEVELS; derived from the source when None113 variant: str | None = None # benchmark variant label (GPQA Diamond, Verified…); derived from config when None114 run_group: str | None = None # evaluation run/release the row belongs to; derived from config when None115116117@dataclass118class Target:119 """Something to fetch. Connectors return targets from `discover()` and may add more from `extract()`."""120 url: str121 doc_type: str = "page"122 entity: EntityRef | None = None123 meta: dict[str, Any] = field(default_factory=dict)124 accept: str | None = None125 min_bytes: int = 64126 escalate: bool = False127 needs_llm: bool = False128 priority: int = 2129 key: str | None = None # short handle for --file overrides / fixtures130 rate_per_min: int | None = None131132133@dataclass134class Facts:135 entities: list[EntityRef] = field(default_factory=list)136 claims: list[Claim] = field(default_factory=list)137 relations: list[Relation] = field(default_factory=list)138 events: list[Event] = field(default_factory=list)139 prices: list[PriceObs] = field(default_factory=list)140 results: list[ResultObs] = field(default_factory=list)141 targets: list[Target] = field(default_factory=list) # follow-up fetches discovered while extracting142 document_title: str | None = None143 document_entity: EntityRef | None = None # main entity described by the document144 llm_hint: str | None = None # ask the LLM factory for a specific extraction task145146 # ---------------------------------------------------------------------------------------------- builders147 def entity(self, entity_type: str, name: str, **kw: Any) -> EntityRef:148 ref = EntityRef(entity_type=entity_type, name=name.strip(), **kw)149 self.entities.append(ref)150 return ref151152 def claim(self, entity: EntityRef, property: str, value: Any, **kw: Any) -> Claim | None:153 if value is None or value == "" or value == [] or value == {}:154 return None155 c = Claim(entity=entity, property=property, value=value, **kw)156 self.claims.append(c)157 return c158159 def relate(self, subject: EntityRef, predicate: str, obj: EntityRef, **kw: Any) -> Relation:160 r = Relation(subject=subject, predicate=predicate, object=obj, **kw)161 self.relations.append(r)162 return r163164 def event(self, event_type: str, category: str, summary: str, **kw: Any) -> Event:165 e = Event(event_type=event_type, category=category, summary=summary[:500], **kw)166 self.events.append(e)167 return e168169 def price(self, **kw: Any) -> PriceObs:170 p = PriceObs(**kw)171 self.prices.append(p)172 return p173174 def result(self, **kw: Any) -> ResultObs:175 r = ResultObs(**kw)176 self.results.append(r)177 return r178179 def follow(self, url: str, **kw: Any) -> Target:180 t = Target(url=url, **kw)181 self.targets.append(t)182 return t183184 def extend(self, other: Facts) -> None:185 self.entities += other.entities186 self.claims += other.claims187 self.relations += other.relations188 self.events += other.events189 self.prices += other.prices190 self.results += other.results191 self.targets += other.targets192193 def is_empty(self) -> bool:194 return not (self.entities or self.claims or self.relations or self.events or self.prices or self.results)195196197# Property → change-event mapping (material properties emit events; noisy metrics never do).198MATERIAL_PROPERTIES: dict[str, tuple[str, int]] = {199 "context_length": ("CONTEXT_CHANGED", 2),200 "max_output_tokens": ("MAX_OUTPUT_CHANGED", 1),201 "status": ("STATUS_CHANGED", 2),202 "license": ("LICENSE_CHANGED", 2),203 "parameter_count": ("PARAMETERS_CHANGED", 1),204 "active_parameter_count": ("PARAMETERS_CHANGED", 1),205 "weights_availability": ("OPENNESS_CHANGED", 3),206 "openness": ("OPENNESS_CHANGED", 3),207 "license_key": ("LICENSE_CHANGED", 2),208 "release_date": ("RELEASE_DATE_CHANGED", 1),209 "knowledge_cutoff": ("KNOWLEDGE_CUTOFF_CHANGED", 1),210 "latest_version": ("VERSION_RELEASED", 2),211 "deprecation_date": ("DEPRECATION_ANNOUNCED", 3),212 "retirement_date": ("RETIREMENT_ANNOUNCED", 3),213 "modalities": ("CAPABILITIES_CHANGED", 2),214 "capabilities": ("CAPABILITIES_CHANGED", 1),215 "rate_limit": ("RATE_LIMIT_CHANGED", 1),216 "memory_gb": ("SPEC_CHANGED", 1),217 "price_usd": ("PRICE_CHANGED", 2),218}219220NOISY_PREFIXES = ("metric.", "stats.", "counts.")221222EVENT_CATEGORY_BY_TYPE: dict[str, str] = {223 "model": "model", "model_family": "model", "artifact": "model", "company": "company", "organization": "company", "lab": "company", "university": "company",224 "paper": "paper", "dataset": "dataset", "benchmark": "benchmark", "provider": "provider", "framework": "framework",225 "library": "framework", "repository": "repository", "tool": "tool", "agent": "tool", "hardware": "hardware",226 "runtime": "framework", "quantization": "model", "regulation": "regulation", "incident": "incident", "release": "release",227 "mcp_server": "tool", "researcher": "company", "license": "model", "product": "tool", "robot": "hardware",228}229230231# ---------------------------------------------------------------------------------------------- JSON round-trip (quarantine)232# Quarantined runs hold their Facts in `quarantined_runs.facts` (jsonb) until an operator releases or discards them. Datetimes are233# tagged so they come back as real datetime objects (asyncpg needs them, not ISO strings). Entity references are inlined; the234# resolver caches by `EntityRef.key()`, so equal refs resolve to the same entity after a round-trip.235_DT_TAG = "__datetime__"236237238def _encode(value: Any) -> Any:239 if isinstance(value, datetime):240 return {_DT_TAG: value.isoformat()}241 if isinstance(value, EntityRef):242 return {"entity_type": value.entity_type, "name": value.name, "identifiers": dict(value.identifiers), "aliases": list(value.aliases),243 "slug_hint": value.slug_hint, "organization": _encode(value.organization), "description": value.description, "status": value.status,244 "attributes": _encode(value.attributes), "first_seen_hint": _encode(value.first_seen_hint), "family": _encode(value.family),245 "canonical": _encode(value.canonical), "artifact_kind": value.artifact_kind, "identity_confidence": value.identity_confidence, "id": value.id}246 if isinstance(value, dict):247 return {str(k): _encode(v) for k, v in value.items()}248 if isinstance(value, (list, tuple, set)):249 return [_encode(v) for v in value]250 if isinstance(value, (str, int, float, bool)) or value is None:251 return value252 return str(value)253254255def _decode(value: Any) -> Any:256 if isinstance(value, dict):257 if set(value) == {_DT_TAG}:258 return datetime.fromisoformat(value[_DT_TAG])259 return {k: _decode(v) for k, v in value.items()}260 if isinstance(value, list):261 return [_decode(v) for v in value]262 return value263264265def _ref_from(d: dict[str, Any] | None) -> EntityRef | None:266 if not d:267 return None268 return EntityRef(entity_type=d["entity_type"], name=d["name"], identifiers=dict(d.get("identifiers") or {}), aliases=list(d.get("aliases") or []),269 slug_hint=d.get("slug_hint"), organization=_ref_from(d.get("organization")), description=d.get("description"), status=d.get("status"),270 attributes=_decode(d.get("attributes") or {}), first_seen_hint=_decode(d.get("first_seen_hint")), family=_ref_from(d.get("family")),271 canonical=_ref_from(d.get("canonical")), artifact_kind=d.get("artifact_kind"), identity_confidence=d.get("identity_confidence"), id=d.get("id"))272273274def facts_to_json(facts: Facts) -> dict[str, Any]:275 """Serialise a Facts object to a JSON-compatible dict (see `facts_from_json`)."""276 def obj(o: Any) -> dict[str, Any]:277 return {k: _encode(v) for k, v in o.__dict__.items()}278279 return {280 "version": 1,281 "entities": [_encode(e) for e in facts.entities],282 "claims": [obj(c) for c in facts.claims],283 "relations": [obj(r) for r in facts.relations],284 "events": [obj(e) for e in facts.events],285 "prices": [obj(p) for p in facts.prices],286 "results": [obj(r) for r in facts.results],287 "targets": [obj(t) for t in facts.targets],288 "document_title": facts.document_title,289 "document_entity": _encode(facts.document_entity),290 "llm_hint": facts.llm_hint,291 }292293294def facts_from_json(data: dict[str, Any]) -> Facts:295 """Inverse of `facts_to_json`."""296 def ref_fields(d: dict[str, Any], *keys: str) -> dict[str, Any]:297 out = {k: _decode(v) for k, v in d.items() if k not in keys}298 for k in keys:299 out[k] = _ref_from(d.get(k))300 return out301302 facts = Facts()303 facts.entities = [r for r in (_ref_from(e) for e in data.get("entities") or []) if r]304 facts.claims = [Claim(**ref_fields(c, "entity")) for c in data.get("claims") or []]305 facts.relations = [Relation(**ref_fields(r, "subject", "object")) for r in data.get("relations") or []]306 facts.events = [Event(**ref_fields(e, "entity")) for e in data.get("events") or []]307 facts.prices = [PriceObs(**ref_fields(p, "model", "provider")) for p in data.get("prices") or []]308 facts.results = [ResultObs(**ref_fields(r, "model", "benchmark")) for r in data.get("results") or []]309 facts.targets = [Target(**ref_fields(t, "entity")) for t in data.get("targets") or []]310 facts.document_title = data.get("document_title")311 facts.document_entity = _ref_from(data.get("document_entity"))312 facts.llm_hint = data.get("llm_hint")313 return facts314315316__all__ = [317 "CONFIDENCE",318 "EVENT_CATEGORY_BY_TYPE",319 "MATERIAL_PROPERTIES",320 "NOISY_PREFIXES",321 "Claim",322 "EntityRef",323 "Event",324 "Facts",325 "PriceObs",326 "Relation",327 "ResultObs",328 "Target",329 "facts_from_json",330 "facts_to_json",331]332