spb/trouve-ka Public

1# Trouve-KA — fetcher HTTP sécurisé2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""Fetch HTTP avec garde SSRF revalidée à chaque redirection, limites de taille,6gestion des content-types et cache HTTP conditionnel (ETag / Last-Modified).78Jamais de Chromium ici (CLAUDE.md §5.10) : fetch HTTP pur.9"""1011import time12from urllib.parse import urlsplit1314import httpx1516from trouveka.config import Settings17from trouveka.shared import canonicalize_url, is_safe_url18from trouveka.types import ErrorCode, FetchResult1920ACCEPTED_CONTENT_TYPES = ("text/html", "application/xhtml+xml", "text/plain")212223class Fetcher:24    def __init__(self, client: httpx.AsyncClient, settings: Settings):25        self._client = client26        self._s = settings2728    async def fetch(29        self, url: str, *, etag: str | None = None, last_modified: str | None = None30    ) -> FetchResult:31        """Fetch une URL en suivant manuellement les redirections (revalidation SSRF à chaque saut)."""32        start = time.monotonic()33        chain: list[str] = []34        current = url3536        for _hop in range(self._s.max_redirects + 1):37            if not is_safe_url(current):38                return self._fail(url, current, ErrorCode.SSRF_BLOCKED, start, chain)3940            # Accept-Encoding volontairement absent : httpx annonce lui-même les41            # encodages qu'il sait décoder (br seulement si brotli est installé).42            # L'annoncer à la main fait indexer du binaire compressé (bug itum.qc.ca).43            headers = {44                "User-Agent": self._s.crawler_user_agent,45                "Accept": "text/html,application/xhtml+xml;q=0.9,text/plain;q=0.5,*/*;q=0.1",46                "Accept-Language": "fr-CA,fr;q=0.9,en-CA;q=0.8,en;q=0.7",47            }48            if etag and current == url:49                headers["If-None-Match"] = etag50            if last_modified and current == url:51                headers["If-Modified-Since"] = last_modified5253            try:54                async with self._client.stream(55                    "GET", current, headers=headers, timeout=self._s.fetch_timeout, follow_redirects=False56                ) as resp:57                    if resp.status_code in (301, 302, 303, 307, 308):58                        location = resp.headers.get("location")59                        if not location:60                            return self._fail(url, current, ErrorCode.HTTP_4XX, start, chain, resp.status_code)61                        next_url = canonicalize_url(location, base=current)62                        if not next_url:63                            return self._fail(url, current, ErrorCode.SSRF_BLOCKED, start, chain, resp.status_code)64                        chain.append(next_url)65                        current = next_url66                        continue6768                    if resp.status_code == 304:69                        return FetchResult(70                            url=url, final_url=current, status_code=304,71                            duration_ms=self._ms(start), redirect_chain=chain,72                        )73                    if 400 <= resp.status_code < 500:74                        return self._fail(url, current, ErrorCode.HTTP_4XX, start, chain, resp.status_code)75                    if resp.status_code >= 500:76                        return self._fail(url, current, ErrorCode.HTTP_5XX, start, chain, resp.status_code)7778                    # X-Robots-Tag : noindex au niveau HTTP79                    x_robots = (resp.headers.get("x-robots-tag") or "").lower()8081                    raw_ct = resp.headers.get("content-type") or ""82                    content_type = raw_ct.split(";")[0].strip().lower()83                    charset = None84                    if "charset=" in raw_ct.lower():85                        charset = raw_ct.lower().split("charset=")[-1].split(";")[0].strip(' "\'') or None86                    if content_type and not any(content_type.startswith(t) for t in ACCEPTED_CONTENT_TYPES):87                        return self._fail(88                            url, current, ErrorCode.UNSUPPORTED_CONTENT, start, chain, resp.status_code89                        )9091                    declared = resp.headers.get("content-length")92                    if declared and int(declared) > self._s.max_response_bytes:93                        return self._fail(url, current, ErrorCode.TOO_LARGE, start, chain, resp.status_code)9495                    body = b""96                    async for part in resp.aiter_bytes():97                        body += part98                        if len(body) > self._s.max_response_bytes:99                            return self._fail(url, current, ErrorCode.TOO_LARGE, start, chain, resp.status_code)100101                    result = FetchResult(102                        url=url,103                        final_url=current,104                        status_code=resp.status_code,105                        content_type=content_type or None,106                        charset=charset,107                        body=body,108                        etag=resp.headers.get("etag"),109                        last_modified=resp.headers.get("last-modified"),110                        duration_ms=self._ms(start),111                        redirect_chain=chain,112                    )113                    if "noindex" in x_robots:114                        result.error_code = ErrorCode.ROBOTS_DENIED115                    return result116117            except httpx.TimeoutException:118                return self._fail(url, current, ErrorCode.TIMEOUT, start, chain)119            except httpx.ConnectError as exc:120                code = ErrorCode.TLS if "SSL" in str(exc) or "certificate" in str(exc).lower() else ErrorCode.DNS121                return self._fail(url, current, code, start, chain)122            except httpx.HTTPError:123                return self._fail(url, current, ErrorCode.CONNECTION, start, chain)124125        return self._fail(url, current, ErrorCode.TOO_MANY_REDIRECTS, start, chain)126127    @staticmethod128    def _ms(start: float) -> int:129        return int((time.monotonic() - start) * 1000)130131    def _fail(132        self,133        url: str,134        final_url: str,135        code: ErrorCode,136        start: float,137        chain: list[str],138        status: int | None = None,139    ) -> FetchResult:140        return FetchResult(141            url=url, final_url=final_url, status_code=status, error_code=code,142            duration_ms=self._ms(start), redirect_chain=chain,143        )144145146def scheme_host(url: str) -> str:147    parts = urlsplit(url)148    return f"{parts.scheme}://{parts.netloc}"149