1"""Connector interface: discover() → capabilities() → fetch_range() → normalize() → validate() → checkpoint().23Connectors never log full URLs, never persist credentials, and return *raw bytes + provenance* separately from4normalized Arrow tables so that raw records stay immutable.5"""6from __future__ import annotations78from abc import ABC, abstractmethod9from dataclasses import dataclass, field10from datetime import date, datetime11from pathlib import Path12from typing import Any1314import pyarrow as pa151617@dataclass(frozen=True)18class Capabilities:19 source_id: str20 asset_classes: tuple[str, ...]21 intervals: tuple[str, ...]22 adjustments: tuple[str, ...]23 supports_symbol_filter: bool # can the vendor serve one symbol? (FirstRate: no)24 supports_date_range: bool # can the vendor serve an arbitrary range? (FirstRate: no)25 incremental_windows: tuple[str, ...] # e.g. day, week, month26 license_class: str27 notes: dict[str, Any] = field(default_factory=dict)282930@dataclass(frozen=True)31class FetchRequest:32 dataset_id: str33 params: dict[str, Any] # vendor-specific, *without* credentials34 start: date | None = None35 end: date | None = None363738@dataclass39class FetchResult:40 request: FetchRequest41 path: Path # raw archive stored immutably under lake/raw42 sha256: str43 bytes: int44 fetched_at: datetime45 vendor_last_update: date | None46 file_count: int | None = None47 source_url_sanitized: str = ""484950@dataclass51class NormalizedBatch:52 table: pa.Table53 symbol: str54 session_dates: tuple[date, date]55 row_count: int56 raw_rows: int # rows read from the vendor file (before dedupe)575859@dataclass60class ValidationReport:61 rows: int62 flagged_rows: int63 anomalies: list[dict[str, Any]]64 ok: bool656667@dataclass68class Checkpoint:69 scope: str70 key: str71 value: dict[str, Any]727374class Connector(ABC):75 source_id: str7677 @abstractmethod78 def discover(self) -> dict[str, Any]:79 """Light-weight live probe (e.g. last_update, listing counts). Never bulk-downloads."""8081 @abstractmethod82 def capabilities(self) -> Capabilities: ...8384 @abstractmethod85 def fetch_range(self, request: FetchRequest, dest_dir: Path) -> FetchResult: ...8687 @abstractmethod88 def normalize(self, result: FetchResult, **kwargs: Any): # -> Iterator[NormalizedBatch]89 ...9091 @abstractmethod92 def validate(self, batch: NormalizedBatch) -> ValidationReport: ...9394 @abstractmethod95 def checkpoint(self, result: FetchResult, batches_published: int) -> Checkpoint: ...96