"""Connector SDK (spec §8, §105–106): metadata, base class, registry. @register class GreenhouseConnector(Connector): meta = ConnectorMeta(connector_id="greenhouse-v1", name="Greenhouse job board", version="1", category=Surface.JOBS_BOARD, …) def extract(self, sensor, result) -> Extraction: ... A *connector* is a strategy; a *sensor* row binds a connector to a URL for one company. `connector_id` (which embeds the version) is stored on every observation/snapshot as `connector_version`, so history remains reproducible when connectors evolve — bump the version (new id) when extraction behaviour changes materially. Only `fetch.Fetcher` touches the network; connectors receive it and must not open sockets themselves. """ from __future__ import annotations import importlib import logging import pkgutil import re from collections.abc import Mapping from dataclasses import dataclass, field from typing import Any, ClassVar from companyatlas.fetch import Fetcher, FetchResult from companyatlas.sdk.models import DiscoveredUrl, Extraction from companyatlas.taxonomy import SURFACE_BASE_INTERVAL_S, FetchMode, Surface log = logging.getLogger(__name__) @dataclass(frozen=True, slots=True) class ConnectorMeta: connector_id: str # e.g. "generic-html-v1" — stable id incl. version name: str version: str category: str # primary Surface value fetch_mode: str = FetchMode.HTTP supports_discovery: bool = False supports_incremental: bool = True # honours etag / last-modified / delta semantics default_interval_s: int = 86400 surfaces: tuple[str, ...] = () # surfaces this connector can serve (empty = category only) url_pattern: str | None = None # regex on the sensor URL; when it matches the connector is preferred pattern_required: bool = False # vendor / API connectors: never auto-selected unless the URL pattern matches priority: int = 10 # higher wins among candidates for the same surface accept: str | None = None # Accept header override max_bytes: int | None = None description: str = "" @dataclass(slots=True) class ConnectorContext: """What a connector may know about the company it is observing (read-only).""" company: Mapping[str, Any] extra: dict[str, Any] = field(default_factory=dict) @property def canonical_domain(self) -> str: return str(self.company.get("canonical_domain") or "") class Connector: meta: ClassVar[ConnectorMeta] # ---------------------------------------------------------------- fetch async def fetch(self, ctx: ConnectorContext, sensor: Mapping[str, Any], fetcher: Fetcher) -> FetchResult: """Default: conditional GET using the sensor's stored validators. Raises NotModified / FetchError.""" return await fetcher.get(str(sensor["url"]), etag=sensor.get("etag"), last_modified=sensor.get("last_modified"), accept=self.meta.accept, max_bytes=self.meta.max_bytes) # ---------------------------------------------------------------- extract def extract(self, sensor: Mapping[str, Any], result: FetchResult) -> Extraction: # pragma: no cover - abstract raise NotImplementedError # ---------------------------------------------------------------- discover (optional) async def discover(self, ctx: ConnectorContext, sensor: Mapping[str, Any], extraction: Extraction) -> list[DiscoveredUrl]: return list(extraction.discovered) # ---------------------------------------------------------------- selection @classmethod def score(cls, surface: str, url: str) -> int: """0 = cannot serve; higher = better fit.""" m = cls.meta s = 0 if m.url_pattern and re.search(m.url_pattern, url, re.IGNORECASE): s += 100 elif m.pattern_required: return 0 if surface == m.category: s += 20 elif surface in m.surfaces: s += 10 elif "*" in m.surfaces: s += 5 return s + m.priority if s > 0 else 0 @property def connector_id(self) -> str: return self.meta.connector_id def __repr__(self) -> str: return f"" # ------------------------------------------------------------------------------------------------------------ registry _REGISTRY: dict[str, type[Connector]] = {} _INSTANCES: dict[str, Connector] = {} _LOADED = False def register(cls: type[Connector]) -> type[Connector]: meta = getattr(cls, "meta", None) if not isinstance(meta, ConnectorMeta): raise TypeError(f"{cls.__name__} must define `meta = ConnectorMeta(...)`") if meta.connector_id in _REGISTRY and _REGISTRY[meta.connector_id] is not cls: log.warning("connector id re-registered", extra={"connector_id": meta.connector_id}) _REGISTRY[meta.connector_id] = cls _INSTANCES.pop(meta.connector_id, None) return cls def load_all() -> None: """Import every `companyatlas.connectors.*` module once so decorators run.""" global _LOADED if _LOADED: return _LOADED = True try: import companyatlas.connectors as pkg except ImportError: # pragma: no cover return for mod in pkgutil.iter_modules(pkg.__path__): if mod.name.startswith("_"): continue try: importlib.import_module(f"companyatlas.connectors.{mod.name}") except Exception: log.exception("connector module failed to import", extra={"connector_module": mod.name}) def get(connector_id: str) -> Connector: load_all() inst = _INSTANCES.get(connector_id) if inst is None: cls = _REGISTRY.get(connector_id) if cls is None: # tolerate an older version id (e.g. "greenhouse-v0") by falling back to the newest of the same family family = connector_id.rsplit("-v", 1)[0] candidates = sorted((k for k in _REGISTRY if k.rsplit("-v", 1)[0] == family), reverse=True) if not candidates: raise KeyError(f"unknown connector {connector_id!r}") cls = _REGISTRY[candidates[0]] inst = _INSTANCES[connector_id] = cls() return inst def all_connectors() -> list[Connector]: load_all() return [get(k) for k in sorted(_REGISTRY)] def for_surface(surface: str, url: str) -> Connector: """Best connector for a (surface, url) pair — ATS/JSON/feed/sitemap by URL pattern, else the generic HTML connector.""" load_all() best: tuple[int, str] | None = None for cid, cls in _REGISTRY.items(): s = cls.score(surface, url) if s > 0 and (best is None or s > best[0] or (s == best[0] and cid < best[1])): best = (s, cid) if best is None: return get("generic-html-v1") return get(best[1]) def default_interval(surface: str, connector: Connector | None = None) -> int: if connector is not None and connector.meta.default_interval_s: return min(int(SURFACE_BASE_INTERVAL_S.get(surface, 86400)), connector.meta.default_interval_s) return int(SURFACE_BASE_INTERVAL_S.get(surface, 86400)) async def sync_connectors_table(conn: Any) -> int: """Upsert one `connectors` row per registered connector (idempotent). Returns the number of connectors.""" from companyatlas.db import execute n = 0 for c in all_connectors(): m = c.meta await execute(conn, """ insert into connectors (id, name, version, category, fetch_mode, supports_discovery, supports_incremental, default_interval_s) values (:id, :name, :version, :category, :fetch_mode, :sd, :si, :interval) on conflict (id) do update set name = excluded.name, version = excluded.version, category = excluded.category, fetch_mode = excluded.fetch_mode, supports_discovery = excluded.supports_discovery, supports_incremental = excluded.supports_incremental, default_interval_s = excluded.default_interval_s, updated_at = now() """, id=m.connector_id, name=m.name, version=m.version, category=str(m.category), fetch_mode=str(m.fetch_mode), sd=m.supports_discovery, si=m.supports_incremental, interval=int(m.default_interval_s)) n += 1 return n __all__ = ["Connector", "ConnectorContext", "ConnectorMeta", "Surface", "all_connectors", "default_interval", "for_surface", "get", "load_all", "register", "sync_connectors_table"]