1"""What a connector produces: entity references, temporal claims, relations, events, prices, benchmark results.2Connectors never touch the database — they return `Facts`; the writer resolves, versions and records them."""3from __future__ import annotations45from dataclasses import dataclass, field6from datetime import datetime7from typing import Any89CONFIDENCE = ("verified", "high", "medium", "low", "conflicted")101112@dataclass13class EntityRef:14 entity_type: str15 name: str16 identifiers: dict[str, str] = field(default_factory=dict) # scheme -> value (hf_repo, github_repo, arxiv, doi, domain, pypi, provider_model_id…)17 aliases: list[str] = field(default_factory=list)18 slug_hint: str | None = None19 organization: EntityRef | None = None # developer / publisher / owner20 description: str | None = None21 status: str | None = None22 attributes: dict[str, Any] = field(default_factory=dict) # convenience: each item becomes a Claim23 first_seen_hint: datetime | None = None # e.g. release date, to backdate `first_seen_at` for historical backfill24 id: str | None = field(default=None, compare=False)2526 def key(self) -> str:27 if self.identifiers:28 scheme, value = sorted(self.identifiers.items())[0]29 return f"{self.entity_type}:{scheme}={value}"30 return f"{self.entity_type}:name={self.name.strip().lower()}"313233@dataclass34class Claim:35 entity: EntityRef36 property: str37 value: Any38 unit: str | None = None39 confidence: str | None = None # default from source tier40 observed_at: datetime | None = None41 effective_at: datetime | None = None42 source_url: str | None = None434445@dataclass46class Relation:47 subject: EntityRef48 predicate: str49 object: EntityRef50 attributes: dict[str, Any] = field(default_factory=dict)51 confidence: str | None = None52 source_url: str | None = None535455@dataclass56class Event:57 event_type: str58 category: str59 summary: str60 entity: EntityRef | None = None61 old_value: Any = None62 new_value: Any = None63 importance: int = 264 effective_at: datetime | None = None65 dedupe_key: str | None = None66 source_url: str | None = None67 meta: dict[str, Any] = field(default_factory=dict)686970@dataclass71class PriceObs:72 model: EntityRef73 provider: EntityRef74 provider_model_id: str | None = None75 input_per_mtok: float | None = None76 output_per_mtok: float | None = None77 cached_input_per_mtok: float | None = None78 cache_write_per_mtok: float | None = None79 batch_input_per_mtok: float | None = None80 batch_output_per_mtok: float | None = None81 per_image: float | None = None82 per_request: float | None = None83 currency: str = "USD"84 context_length: int | None = None85 max_output_tokens: int | None = None86 features: dict[str, Any] = field(default_factory=dict)87 source_url: str | None = None88 meta: dict[str, Any] = field(default_factory=dict)8990 def price_tuple(self) -> tuple[Any, ...]:91 return (self.input_per_mtok, self.output_per_mtok, self.cached_input_per_mtok, self.cache_write_per_mtok,92 self.batch_input_per_mtok, self.batch_output_per_mtok, self.per_image, self.per_request, self.currency)939495@dataclass96class ResultObs:97 model: EntityRef98 benchmark: EntityRef99 score: float100 metric: str | None = None101 unit: str | None = "%"102 higher_is_better: bool = True103 config: dict[str, Any] = field(default_factory=dict)104 evaluated_at: datetime | None = None105 source_url: str | None = None106 confidence: str | None = None107108109@dataclass110class Target:111 """Something to fetch. Connectors return targets from `discover()` and may add more from `extract()`."""112 url: str113 doc_type: str = "page"114 entity: EntityRef | None = None115 meta: dict[str, Any] = field(default_factory=dict)116 accept: str | None = None117 min_bytes: int = 64118 escalate: bool = False119 needs_llm: bool = False120 priority: int = 2121 key: str | None = None # short handle for --file overrides / fixtures122 rate_per_min: int | None = None123124125@dataclass126class Facts:127 entities: list[EntityRef] = field(default_factory=list)128 claims: list[Claim] = field(default_factory=list)129 relations: list[Relation] = field(default_factory=list)130 events: list[Event] = field(default_factory=list)131 prices: list[PriceObs] = field(default_factory=list)132 results: list[ResultObs] = field(default_factory=list)133 targets: list[Target] = field(default_factory=list) # follow-up fetches discovered while extracting134 document_title: str | None = None135 document_entity: EntityRef | None = None # main entity described by the document136 llm_hint: str | None = None # ask the LLM factory for a specific extraction task137138 # ---------------------------------------------------------------------------------------------- builders139 def entity(self, entity_type: str, name: str, **kw: Any) -> EntityRef:140 ref = EntityRef(entity_type=entity_type, name=name.strip(), **kw)141 self.entities.append(ref)142 return ref143144 def claim(self, entity: EntityRef, property: str, value: Any, **kw: Any) -> Claim | None:145 if value is None or value == "" or value == [] or value == {}:146 return None147 c = Claim(entity=entity, property=property, value=value, **kw)148 self.claims.append(c)149 return c150151 def relate(self, subject: EntityRef, predicate: str, obj: EntityRef, **kw: Any) -> Relation:152 r = Relation(subject=subject, predicate=predicate, object=obj, **kw)153 self.relations.append(r)154 return r155156 def event(self, event_type: str, category: str, summary: str, **kw: Any) -> Event:157 e = Event(event_type=event_type, category=category, summary=summary[:500], **kw)158 self.events.append(e)159 return e160161 def price(self, **kw: Any) -> PriceObs:162 p = PriceObs(**kw)163 self.prices.append(p)164 return p165166 def result(self, **kw: Any) -> ResultObs:167 r = ResultObs(**kw)168 self.results.append(r)169 return r170171 def follow(self, url: str, **kw: Any) -> Target:172 t = Target(url=url, **kw)173 self.targets.append(t)174 return t175176 def extend(self, other: Facts) -> None:177 self.entities += other.entities178 self.claims += other.claims179 self.relations += other.relations180 self.events += other.events181 self.prices += other.prices182 self.results += other.results183 self.targets += other.targets184185 def is_empty(self) -> bool:186 return not (self.entities or self.claims or self.relations or self.events or self.prices or self.results)187188189# Property → change-event mapping (material properties emit events; noisy metrics never do).190MATERIAL_PROPERTIES: dict[str, tuple[str, int]] = {191 "context_length": ("CONTEXT_CHANGED", 2),192 "max_output_tokens": ("MAX_OUTPUT_CHANGED", 1),193 "status": ("STATUS_CHANGED", 2),194 "license": ("LICENSE_CHANGED", 2),195 "parameter_count": ("PARAMETERS_CHANGED", 1),196 "active_parameter_count": ("PARAMETERS_CHANGED", 1),197 "weights_availability": ("OPENNESS_CHANGED", 3),198 "openness": ("OPENNESS_CHANGED", 3),199 "release_date": ("RELEASE_DATE_CHANGED", 1),200 "knowledge_cutoff": ("KNOWLEDGE_CUTOFF_CHANGED", 1),201 "latest_version": ("VERSION_RELEASED", 2),202 "deprecation_date": ("DEPRECATION_ANNOUNCED", 3),203 "retirement_date": ("RETIREMENT_ANNOUNCED", 3),204 "modalities": ("CAPABILITIES_CHANGED", 2),205 "capabilities": ("CAPABILITIES_CHANGED", 1),206 "rate_limit": ("RATE_LIMIT_CHANGED", 1),207 "memory_gb": ("SPEC_CHANGED", 1),208 "price_usd": ("PRICE_CHANGED", 2),209}210211NOISY_PREFIXES = ("metric.", "stats.", "counts.")212213EVENT_CATEGORY_BY_TYPE: dict[str, str] = {214 "model": "model", "company": "company", "organization": "company", "lab": "company", "university": "company",215 "paper": "paper", "dataset": "dataset", "benchmark": "benchmark", "provider": "provider", "framework": "framework",216 "library": "framework", "repository": "repository", "tool": "tool", "agent": "tool", "hardware": "hardware",217 "runtime": "framework", "quantization": "model", "regulation": "regulation", "incident": "incident", "release": "release",218 "mcp_server": "tool", "researcher": "company", "license": "model", "product": "tool", "robot": "hardware",219}220221__all__ = ["EntityRef", "Claim", "Relation", "Event", "PriceObs", "ResultObs", "Target", "Facts", "MATERIAL_PROPERTIES",222 "NOISY_PREFIXES", "EVENT_CATEGORY_BY_TYPE", "CONFIDENCE"]223