spb/trouve-ka Public

1# Trouve-KA — détection de pièges de crawl2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""Heuristiques anti-pièges (CLAUDE.md §5.8) : calendriers infinis, session IDs,6explosions de facettes, pagination infinie, chemins répétitifs."""78import re9from urllib.parse import parse_qsl, urlsplit1011_SESSION_PARAMS = {"sid", "sessionid", "session_id", "phpsessid", "jsessionid", "aspsessionid", "cfid", "cftoken"}12_CALENDAR_RE = re.compile(r"/(?:19|20)\d{2}[-/](?:0?\d|1[0-2])(?:[-/](?:0?\d|[12]\d|3[01]))?/?$")13_LONG_NUMBER_RE = re.compile(r"\d{10,}")141516def looks_like_trap(17    url: str,18    *,19    max_query_params: int = 8,20    max_path_segments: int = 12,21    max_url_length: int = 1024,22) -> bool:23    """True si l'URL ressemble à un piège de crawl et ne doit pas entrer au frontier."""24    if len(url) > max_url_length:25        return True26    try:27        parts = urlsplit(url)28    except ValueError:29        return True3031    params = parse_qsl(parts.query, keep_blank_values=True)32    if len(params) > max_query_params:33        return True34    keys = {k.lower() for k, _ in params}35    if keys & _SESSION_PARAMS:36        return True37    # Même paramètre répété (facettes qui explosent : ?filter=a&filter=b&filter=c…)38    raw_keys = [k.lower() for k, _ in params]39    if any(raw_keys.count(k) > 3 for k in set(raw_keys)):40        return True4142    segments = [s for s in parts.path.split("/") if s]43    if len(segments) > max_path_segments:44        return True45    # Segment répété (boucles : /a/b/a/b/a/b)46    if any(segments.count(s) > 3 for s in set(segments)):47        return True48    # Calendriers profonds (au-delà de l'année-mois raisonnable) : /events/2031/05/1749    if _CALENDAR_RE.search(parts.path):50        year_match = re.search(r"/((?:19|20)\d{2})[-/]", parts.path)51        if year_match and not (1995 <= int(year_match.group(1)) <= 2027):52            return True53    # Pagination excessive54    for key, value in params:55        if key.lower() in ("page", "p", "offset", "start") and value.isdigit() and int(value) > 500:56            return True57    if _LONG_NUMBER_RE.search(parts.query):58        return True59    return False60