# Trouve-KA — tests du fetcher (web de fixtures via MockTransport) # Author: Simon-Pierre Boucher # Contact: contact@spboucher.ai import functools import httpx import pytest import trouveka.crawler.fetcher as fetcher_mod from trouveka.config import Settings from trouveka.crawler.fetcher import Fetcher from trouveka.shared import is_safe_url from trouveka.types import ErrorCode @pytest.fixture(autouse=True) def no_dns(monkeypatch): # Les hôtes de fixtures ne résolvent pas en vrai DNS : on garde toute la # logique SSRF (schémas, hôtes bloqués, IP littérales) sans résolution. monkeypatch.setattr(fetcher_mod, "is_safe_url", functools.partial(is_safe_url, resolve=False)) def make_fetcher(handler, **overrides) -> Fetcher: settings = Settings(max_response_bytes=1000, max_redirects=3, fetch_timeout=5, **overrides) client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) return Fetcher(client, settings) async def test_simple_fetch(): def handler(request): return httpx.Response(200, html="ok", headers={"content-type": "text/html; charset=utf-8"}) result = await make_fetcher(handler).fetch("https://site.qc.ca/page") assert result.status_code == 200 assert result.error_code == ErrorCode.OK assert b"ok" in result.body async def test_redirect_followed_with_ssrf_recheck(): def handler(request): if request.url.path == "/a": return httpx.Response(301, headers={"location": "/b"}) return httpx.Response(200, html="fini", headers={"content-type": "text/html"}) result = await make_fetcher(handler).fetch("https://site.qc.ca/a") assert result.status_code == 200 assert result.final_url == "https://site.qc.ca/b" assert result.redirect_chain == ["https://site.qc.ca/b"] async def test_redirect_to_private_ip_blocked(): def handler(request): return httpx.Response(302, headers={"location": "http://169.254.169.254/latest/meta-data/"}) result = await make_fetcher(handler).fetch("https://site.qc.ca/evil") assert result.error_code == ErrorCode.SSRF_BLOCKED async def test_too_many_redirects(): def handler(request): n = int(request.url.path.strip("/") or 0) return httpx.Response(301, headers={"location": f"/{n + 1}"}) result = await make_fetcher(handler).fetch("https://site.qc.ca/0") assert result.error_code == ErrorCode.TOO_MANY_REDIRECTS async def test_body_size_limit(): def handler(request): return httpx.Response(200, content=b"x" * 5000, headers={"content-type": "text/html"}) result = await make_fetcher(handler).fetch("https://site.qc.ca/gros") assert result.error_code == ErrorCode.TOO_LARGE async def test_unsupported_content_type(): def handler(request): return httpx.Response(200, content=b"%PDF-1.4", headers={"content-type": "application/pdf"}) result = await make_fetcher(handler).fetch("https://site.qc.ca/doc.pdf") assert result.error_code == ErrorCode.UNSUPPORTED_CONTENT async def test_conditional_304(): def handler(request): assert request.headers.get("if-none-match") == 'W/"abc"' return httpx.Response(304) result = await make_fetcher(handler).fetch("https://site.qc.ca/page", etag='W/"abc"') assert result.status_code == 304 async def test_4xx_and_5xx_classified(): async def run(status): def handler(request): return httpx.Response(status) return await make_fetcher(handler).fetch("https://site.qc.ca/x") assert (await run(404)).error_code == ErrorCode.HTTP_4XX assert (await run(503)).error_code == ErrorCode.HTTP_5XX async def test_ssrf_blocked_upfront(): def handler(request): # pragma: no cover — ne doit jamais être appelé raise AssertionError("fetch ne doit pas partir") result = await make_fetcher(handler).fetch("http://127.0.0.1:8080/admin") assert result.error_code == ErrorCode.SSRF_BLOCKED