spb/trouve-ka Public

1# Trouve-KA — gestion robots.txt2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""Cache robots.txt conforme aux standards (Protego, le parseur de Scrapy).67Les règles sont persistées en base (domains.robots_body) et rafraîchies8après 24 h. Un robots.txt introuvable (404) = tout permis; une erreur9serveur = prudence, on refuse temporairement.10"""1112from datetime import UTC, datetime, timedelta1314import httpx15from protego import Protego1617from trouveka.logging import get_logger1819log = get_logger("crawler.robots")2021ROBOTS_TTL = timedelta(hours=24)22_MAX_ROBOTS_BYTES = 512_000232425class RobotsCache:26    def __init__(self, db, client: httpx.AsyncClient, user_agent: str):27        self._db = db28        self._client = client29        self._ua = user_agent30        self._parsers: dict[int, tuple[Protego | None, datetime]] = {}3132    async def _fetch_robots(self, scheme_host: str) -> tuple[str | None, str]:33        """Retourne (body, status) : ok | not_found | error."""34        try:35            resp = await self._client.get(36                f"{scheme_host}/robots.txt",37                headers={"User-Agent": self._ua},38                timeout=15,39                follow_redirects=True,40            )41        except httpx.HTTPError:42            return None, "error"43        if resp.status_code == 200:44            return resp.text[:_MAX_ROBOTS_BYTES], "ok"45        if 400 <= resp.status_code < 500:46            return None, "not_found"  # 4xx = pas de robots → tout permis (standard)47        return None, "error"4849    async def get_parser(self, domain_id: int, scheme_host: str) -> tuple[Protego | None, str, int | None]:50        """Parseur robots pour un domaine. Retourne (parser|None, status, crawl_delay_ms)."""51        cached = self._parsers.get(domain_id)52        now = datetime.now(UTC)53        if cached and now - cached[1] < ROBOTS_TTL:54            parser = cached[0]55            return parser, "ok" if parser else "not_found", None5657        row = await self._db.get_domain_by_id(domain_id)58        if row and row["robots_fetched_at"] and now - row["robots_fetched_at"] < ROBOTS_TTL:59            body, status = row["robots_body"], row["robots_status"]60        else:61            body, status = await self._fetch_robots(scheme_host)62            delay_ms = None63            if body:64                delay = Protego.parse(body).crawl_delay(self._ua)65                delay_ms = int(delay * 1000) if delay else None66            await self._db.save_robots(domain_id, body, status, delay_ms)6768        parser = Protego.parse(body) if body else None69        self._parsers[domain_id] = (parser, now)70        if len(self._parsers) > 5000:  # borne mémoire du cache local71            self._parsers.pop(next(iter(self._parsers)))72        return parser, status, row["crawl_delay_ms"] if row else None7374    async def allowed(self, domain_id: int, url: str, scheme_host: str) -> tuple[bool, float | None]:75        """(autorisé, crawl_delay_secondes). Erreur robots = refus temporaire (prudence)."""76        parser, status, delay_ms = await self.get_parser(domain_id, scheme_host)77        if status == "error":78            return False, None79        if parser is None:80            return True, None81        delay = parser.crawl_delay(self._ua)82        if delay is None and delay_ms:83            delay = delay_ms / 100084        return parser.can_fetch(url, self._ua), delay85