# Trouve-KA — tests du fetcher (web de fixtures via MockTransport)
# Author: Simon-Pierre Boucher
# Contact: contact@spboucher.ai
import functools
import httpx
import pytest
import trouveka.crawler.fetcher as fetcher_mod
from trouveka.config import Settings
from trouveka.crawler.fetcher import Fetcher
from trouveka.shared import is_safe_url
from trouveka.types import ErrorCode
@pytest.fixture(autouse=True)
def no_dns(monkeypatch):
# Les hôtes de fixtures ne résolvent pas en vrai DNS : on garde toute la
# logique SSRF (schémas, hôtes bloqués, IP littérales) sans résolution.
monkeypatch.setattr(fetcher_mod, "is_safe_url", functools.partial(is_safe_url, resolve=False))
def make_fetcher(handler, **overrides) -> Fetcher:
settings = Settings(max_response_bytes=1000, max_redirects=3, fetch_timeout=5, **overrides)
client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
return Fetcher(client, settings)
async def test_simple_fetch():
def handler(request):
return httpx.Response(200, html="
ok",
headers={"content-type": "text/html; charset=utf-8"})
result = await make_fetcher(handler).fetch("https://site.qc.ca/page")
assert result.status_code == 200
assert result.error_code == ErrorCode.OK
assert b"ok" in result.body
async def test_redirect_followed_with_ssrf_recheck():
def handler(request):
if request.url.path == "/a":
return httpx.Response(301, headers={"location": "/b"})
return httpx.Response(200, html="fini", headers={"content-type": "text/html"})
result = await make_fetcher(handler).fetch("https://site.qc.ca/a")
assert result.status_code == 200
assert result.final_url == "https://site.qc.ca/b"
assert result.redirect_chain == ["https://site.qc.ca/b"]
async def test_redirect_to_private_ip_blocked():
def handler(request):
return httpx.Response(302, headers={"location": "http://169.254.169.254/latest/meta-data/"})
result = await make_fetcher(handler).fetch("https://site.qc.ca/evil")
assert result.error_code == ErrorCode.SSRF_BLOCKED
async def test_too_many_redirects():
def handler(request):
n = int(request.url.path.strip("/") or 0)
return httpx.Response(301, headers={"location": f"/{n + 1}"})
result = await make_fetcher(handler).fetch("https://site.qc.ca/0")
assert result.error_code == ErrorCode.TOO_MANY_REDIRECTS
async def test_body_size_limit():
def handler(request):
return httpx.Response(200, content=b"x" * 5000, headers={"content-type": "text/html"})
result = await make_fetcher(handler).fetch("https://site.qc.ca/gros")
assert result.error_code == ErrorCode.TOO_LARGE
async def test_unsupported_content_type():
def handler(request):
return httpx.Response(200, content=b"%PDF-1.4", headers={"content-type": "application/pdf"})
result = await make_fetcher(handler).fetch("https://site.qc.ca/doc.pdf")
assert result.error_code == ErrorCode.UNSUPPORTED_CONTENT
async def test_conditional_304():
def handler(request):
assert request.headers.get("if-none-match") == 'W/"abc"'
return httpx.Response(304)
result = await make_fetcher(handler).fetch("https://site.qc.ca/page", etag='W/"abc"')
assert result.status_code == 304
async def test_4xx_and_5xx_classified():
async def run(status):
def handler(request):
return httpx.Response(status)
return await make_fetcher(handler).fetch("https://site.qc.ca/x")
assert (await run(404)).error_code == ErrorCode.HTTP_4XX
assert (await run(503)).error_code == ErrorCode.HTTP_5XX
async def test_ssrf_blocked_upfront():
def handler(request): # pragma: no cover — ne doit jamais être appelé
raise AssertionError("fetch ne doit pas partir")
result = await make_fetcher(handler).fetch("http://127.0.0.1:8080/admin")
assert result.error_code == ErrorCode.SSRF_BLOCKED