# Trouve-KA — gestion robots.txt
# Author: Simon-Pierre Boucher
# Contact: contact@spboucher.ai

"""Cache robots.txt conforme aux standards (Protego, le parseur de Scrapy).

Les règles sont persistées en base (domains.robots_body) et rafraîchies
après 24 h. Un robots.txt introuvable (404) = tout permis; une erreur
serveur = prudence, on refuse temporairement.
"""

from datetime import UTC, datetime, timedelta

import httpx
from protego import Protego

from trouveka.logging import get_logger

log = get_logger("crawler.robots")

ROBOTS_TTL = timedelta(hours=24)
_MAX_ROBOTS_BYTES = 512_000


class RobotsCache:
    def __init__(self, db, client: httpx.AsyncClient, user_agent: str):
        self._db = db
        self._client = client
        self._ua = user_agent
        self._parsers: dict[int, tuple[Protego | None, datetime]] = {}

    async def _fetch_robots(self, scheme_host: str) -> tuple[str | None, str]:
        """Retourne (body, status) : ok | not_found | error."""
        try:
            resp = await self._client.get(
                f"{scheme_host}/robots.txt",
                headers={"User-Agent": self._ua},
                timeout=15,
                follow_redirects=True,
            )
        except httpx.HTTPError:
            return None, "error"
        if resp.status_code == 200:
            return resp.text[:_MAX_ROBOTS_BYTES], "ok"
        if 400 <= resp.status_code < 500:
            return None, "not_found"  # 4xx = pas de robots → tout permis (standard)
        return None, "error"

    async def get_parser(self, domain_id: int, scheme_host: str) -> tuple[Protego | None, str, int | None]:
        """Parseur robots pour un domaine. Retourne (parser|None, status, crawl_delay_ms)."""
        cached = self._parsers.get(domain_id)
        now = datetime.now(UTC)
        if cached and now - cached[1] < ROBOTS_TTL:
            parser = cached[0]
            return parser, "ok" if parser else "not_found", None

        row = await self._db.get_domain_by_id(domain_id)
        if row and row["robots_fetched_at"] and now - row["robots_fetched_at"] < ROBOTS_TTL:
            body, status = row["robots_body"], row["robots_status"]
        else:
            body, status = await self._fetch_robots(scheme_host)
            delay_ms = None
            if body:
                delay = Protego.parse(body).crawl_delay(self._ua)
                delay_ms = int(delay * 1000) if delay else None
            await self._db.save_robots(domain_id, body, status, delay_ms)

        parser = Protego.parse(body) if body else None
        self._parsers[domain_id] = (parser, now)
        if len(self._parsers) > 5000:  # borne mémoire du cache local
            self._parsers.pop(next(iter(self._parsers)))
        return parser, status, row["crawl_delay_ms"] if row else None

    async def allowed(self, domain_id: int, url: str, scheme_host: str) -> tuple[bool, float | None]:
        """(autorisé, crawl_delay_secondes). Erreur robots = refus temporaire (prudence)."""
        parser, status, delay_ms = await self.get_parser(domain_id, scheme_host)
        if status == "error":
            return False, None
        if parser is None:
            return True, None
        delay = parser.crawl_delay(self._ua)
        if delay is None and delay_ms:
            delay = delay_ms / 1000
        return parser.can_fetch(url, self._ua), delay