Python 76.2%
JavaScript 11.3%
CSS 6.3%
HTML 5.9%
1"""Modèles Pydantic v2 — le contrat de l'API. Aucun dict non typé ne sort de l'API."""23from __future__ import annotations45from datetime import UTC, datetime6from enum import StrEnum7from typing import Any, Literal89from pydantic import BaseModel, Field, field_validator1011Format = Literal["markdown", "html", "raw_html", "json", "links", "screenshot", "chunks", "metadata"]12FetchMode = Literal["auto", "http", "browser", "stealth"]13ResolvedMode = Literal["http", "browser", "stealth"]141516class ErrorCode(StrEnum):17 TIMEOUT_DNS = "TIMEOUT_DNS"18 TIMEOUT_CONNECT = "TIMEOUT_CONNECT"19 TIMEOUT_TTFB = "TIMEOUT_TTFB"20 TIMEOUT_RENDER = "TIMEOUT_RENDER"21 HTTP_4XX = "HTTP_4XX"22 HTTP_5XX = "HTTP_5XX"23 RATE_LIMITED = "RATE_LIMITED"24 BLOCKED = "BLOCKED"25 ROBOTS_DISALLOWED = "ROBOTS_DISALLOWED"26 TOO_LARGE = "TOO_LARGE"27 UNSUPPORTED_CONTENT = "UNSUPPORTED_CONTENT"28 PARSE_FAILED = "PARSE_FAILED"29 SSL_ERROR = "SSL_ERROR"30 CIRCUIT_OPEN = "CIRCUIT_OPEN"31 BUDGET_EXCEEDED = "BUDGET_EXCEEDED"32 LLM_INVALID_OUTPUT = "LLM_INVALID_OUTPUT"33 SSRF_REFUSED = "SSRF_REFUSED"34 INVALID_URL = "INVALID_URL"35 NETWORK = "NETWORK"36 INTERNAL = "INTERNAL"37 CANCELLED = "CANCELLED"383940RETRYABLE: frozenset[ErrorCode] = frozenset(41 {42 ErrorCode.TIMEOUT_DNS,43 ErrorCode.TIMEOUT_CONNECT,44 ErrorCode.TIMEOUT_TTFB,45 ErrorCode.TIMEOUT_RENDER,46 ErrorCode.HTTP_5XX,47 ErrorCode.RATE_LIMITED,48 ErrorCode.CIRCUIT_OPEN,49 ErrorCode.NETWORK,50 ErrorCode.LLM_INVALID_OUTPUT,51 }52)535455class ErrorInfo(BaseModel):56 code: ErrorCode57 message: str58 retryable: bool59 attempts: int = 160 details: dict[str, Any] | None = None6162 @classmethod63 def make(cls, code: ErrorCode, message: str, attempts: int = 1, **details: Any) -> ErrorInfo:64 return cls(65 code=code,66 message=message[:1000],67 retryable=code in RETRYABLE,68 attempts=attempts,69 details=details or None,70 )717273class Cookie(BaseModel):74 name: str75 value: str76 domain: str | None = None77 path: str = "/"787980class BrowserAction(BaseModel):81 type: Literal["click", "scroll", "type", "wait", "press", "screenshot", "evaluate"]82 selector: str | None = None83 text: str | None = None84 key: str | None = None85 ms: int | None = None86 direction: Literal["up", "down"] = "down"87 amount: int = 100088 script: str | None = None899091class Location(BaseModel):92 country: str = "CA"93 languages: list[str] = ["fr-CA", "fr", "en"]949596class ChunkOptions(BaseModel):97 strategy: Literal["by_heading", "by_tokens"] = "by_heading"98 size_tokens: int = 51299 overlap_tokens: int = 64100 min_tokens: int = 64101 max_tokens: int = 1024102103104class CssField(BaseModel):105 selector: str106 attr: str = "text"107 type: Literal["str", "int", "float", "date", "url", "list", "bool"] = "str"108 multiple: bool = False109110111class ExtractOptions(BaseModel):112 mode: Literal["css", "llm"] = "css"113 schema_: dict[str, Any] | None = Field(default=None, alias="schema")114 css: dict[str, CssField] | None = None115 prompt: str | None = None116117 model_config = {"populate_by_name": True}118119120class ScrapeOptions(BaseModel):121 formats: list[Format] = ["markdown"]122 only_main_content: bool = True123 include_tags: list[str] = []124 exclude_tags: list[str] = []125 wait_for: str | int | None = None126 timeout_ms: int = 30_000127 mode: FetchMode = "auto"128 headers: dict[str, str] = {}129 cookies: list[Cookie] = []130 proxy: str | None = None131 actions: list[BrowserAction] = []132 location: Location | None = None133 remove_base64_images: bool = True134 chunk: ChunkOptions | None = None135 extract: ExtractOptions | None = None136 cache: Literal["use", "bypass", "refresh"] = "use"137 max_age_s: int = 86_400138 citations: bool = False139 verify_ssl: bool = True140 respect_robots: bool = True141142 @field_validator("timeout_ms")143 @classmethod144 def _clamp_timeout(cls, v: int) -> int:145 return max(1_000, min(v, 180_000))146147148class Link(BaseModel):149 href: str150 text: str = ""151 rel: str | None = None152 kind: Literal["internal", "external", "asset", "mailto", "tel", "other"] = "other"153 nofollow: bool = False154155156class PageMetadata(BaseModel):157 title: str | None = None158 description: str | None = None159 language: str | None = None160 author: str | None = None161 published_at: str | None = None162 modified_at: str | None = None163 canonical_url: str | None = None164 site_name: str | None = None165 og_image: str | None = None166 og_type: str | None = None167 keywords: list[str] = []168 favicon: str | None = None169 content_type: str | None = None170 charset: str | None = None171 word_count: int = 0172 jsonld: list[dict[str, Any]] = []173 is_pdf: bool = False174 pdf_pages: int | None = None175 pdf_is_scanned: bool | None = None176 extra: dict[str, Any] = {}177178179class Chunk(BaseModel):180 index: int181 text: str182 heading_path: str = ""183 token_count: int184 char_range: tuple[int, int]185 url: str186187188class Timings(BaseModel):189 dns_ms: float | None = None190 connect_ms: float | None = None191 ttfb_ms: float | None = None192 fetch_ms: float | None = None193 render_ms: float | None = None194 process_ms: float | None = None195 total_ms: float | None = None196197198class PageResult(BaseModel):199 url: str200 final_url: str201 status: Literal["ok", "failed", "skipped"]202 http_status: int | None = None203 fetch_mode_used: ResolvedMode | None = None204 markdown: str | None = None205 html: str | None = None206 raw_html: str | None = None207 json_data: dict[str, Any] | None = None208 links: list[Link] = []209 metadata: PageMetadata = Field(default_factory=PageMetadata)210 chunks: list[Chunk] | None = None211 screenshot_url: str | None = None212 error: ErrorInfo | None = None213 timings: Timings = Field(default_factory=Timings)214 fetched_at: datetime = Field(default_factory=lambda: datetime.now(UTC))215 depth: int = 0216 from_cache: bool = False217218 @classmethod219 def failed(cls, url: str, err: ErrorInfo, mode: ResolvedMode | None = None, **kw: Any) -> PageResult:220 return cls(url=url, final_url=url, status="failed", error=err, fetch_mode_used=mode, **kw)221222 @classmethod223 def skipped(cls, url: str, reason: str, **kw: Any) -> PageResult:224 return cls(225 url=url,226 final_url=url,227 status="skipped",228 error=ErrorInfo(code=ErrorCode.INTERNAL, message=reason, retryable=False),229 **kw,230 )231232233class CrawlOptions(BaseModel):234 max_depth: int = 3235 max_pages: int = 100236 include_paths: list[str] = []237 exclude_paths: list[str] = []238 allow_subdomains: bool = False239 allow_external_links: bool = False240 ignore_sitemap: bool = False241 respect_robots: bool = True242 delay_ms: int = 0243 concurrency: int = 5244 strategy: Literal["bfs", "dfs", "best_first"] = "bfs"245 search: str | None = None246 max_duration_s: int = 1800247248 @field_validator("max_pages")249 @classmethod250 def _clamp_pages(cls, v: int) -> int:251 return max(1, min(v, 20_000))252253254class MapOptions(BaseModel):255 search: str | None = None256 include_subdomains: bool = False257 limit: int = 5000258 ignore_sitemap: bool = False259 include_titles: bool = False260 crawl_depth: int = 2261 timeout_s: int = 30262263264class MappedUrl(BaseModel):265 url: str266 sources: list[str] = []267 lastmod: str | None = None268 depth: int | None = None269 title: str | None = None270 score: float | None = None271272273JobStatus = Literal["queued", "running", "paused", "completed", "failed", "cancelled"]274JobKind = Literal["crawl", "batch", "extract", "agent", "map"]275276277class JobSummary(BaseModel):278 id: str279 kind: JobKind280 status: JobStatus281 created_at: datetime282 started_at: datetime | None = None283 finished_at: datetime | None = None284 total: int = 0285 completed: int = 0286 failed: int = 0287 skipped: int = 0288 root_url: str | None = None289 error: ErrorInfo | None = None290 credits_used: int = 0291 meta: dict[str, Any] = {}292293294class ApiError(BaseModel):295 code: str296 message: str297 retryable: bool = False298 details: dict[str, Any] | None = None299300301class ApiErrorEnvelope(BaseModel):302 error: ApiError303