spb/trouve-ka Public

1# Trouve-KA — API FastAPI2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""API de Trouve-KA.67Publique : /api/search, /api/status, /api/submit, /api/health8Protégée (X-Admin-Token) : /api/admin/* — contrôle du crawler, jamais public sans auth.910Dégradation gracieuse : si OpenSearch tombe, /api/status répond quand même;11si Postgres tombe, la recherche répond quand même (analytics sautées).12"""1314import html15import re16import time17from contextlib import asynccontextmanager18from typing import Annotated1920from fastapi import Depends, FastAPI, Header, HTTPException, Query21from fastapi.middleware.cors import CORSMiddleware22from pydantic import BaseModel2324from trouveka.config import get_settings25from trouveka.database import Database26from trouveka.logging import get_logger27from trouveka.queue import Coordination28from trouveka.ranking import build_search_body29from trouveka.search_core import SearchCore30from trouveka.shared import canonicalize_url, display_url, extract_domain, is_http_url3132log = get_logger("api")33settings = get_settings()3435db = Database(settings.database_url, pool_min=settings.pg_pool_min, pool_max=settings.pg_pool_max)36coord = Coordination(settings.redis_url)37search = SearchCore(settings.search_url, settings.search_index)383940@asynccontextmanager41async def lifespan(_app: FastAPI):42    await db.connect()43    await search.ensure_index()44    yield45    await search.close()46    await coord.close()47    await db.close()484950app = FastAPI(title="Trouve-KA API", version="0.1.0", lifespan=lifespan)5152# Derrière ngrok (www.trouve-ka.com), le web app proxifie /api : CORS permissif inutile53# en prod, mais pratique en dev local (web sur :3000, API sur :8080).54app.add_middleware(55    CORSMiddleware,56    allow_origins=["http://localhost:3000", settings.public_url],57    allow_methods=["GET", "POST"],58    allow_headers=["*", "X-Admin-Token"],59)606162def require_admin(x_admin_token: Annotated[str | None, Header()] = None) -> None:63    if not x_admin_token or x_admin_token != settings.admin_token:64        raise HTTPException(status_code=401, detail="Jeton admin invalide")656667_TAG_RE = re.compile(r"<(?!/?em>)[^>]*>")686970def _safe_snippet(fragments: list[str]) -> str:71    """Ne laisse passer que <em>/</em> (highlight); tout le reste est échappé par OpenSearch."""72    return _TAG_RE.sub("", " … ".join(fragments))[:400]737475BADGE_LABELS = {"government": "Gouvernement", "news": "Actualités", "education": "Éducation"}767778# ---------------------------------------------------------------------- publique7980@app.get("/api/health")81async def health():82    return {"ok": True, "search_ok": await search.ping()}838485@app.get("/api/search")86async def api_search(87    q: str = Query(..., min_length=1, max_length=200),88    page: int = Query(1, ge=1, le=100),89    limit: int = Query(10, ge=1, le=50),90    language: str | None = Query(None, pattern="^(fr|en)$"),91    location: str | None = None,92    category: str | None = Query(None, max_length=40),93    quebec_only: bool = False,94    freshness: str | None = Query(None, pattern="^(day|week|month|year)$"),95):96    started = time.monotonic()97    query_text = f"{q} {location}" if location else q98    body, analysis = build_search_body(99        query_text, page=page, limit=limit, language=language,100        category=category, quebec_only=quebec_only, freshness=freshness,101    )102    try:103        res = await search.search(body)104    except Exception:105        log.exception("recherche échouée", extra={"ctx": {"q": q}})106        raise HTTPException(status_code=503, detail="Le moteur de recherche est temporairement indisponible")107108    took_ms = int((time.monotonic() - started) * 1000)109    results = []110    for hit in res["hits"]["hits"]:111        src = hit["_source"]112        highlight = hit.get("highlight", {})113        fragments = highlight.get("body") or highlight.get("description") or []114        snippet = _safe_snippet(fragments) if fragments else html.escape(src.get("description") or "")[:400]115        badges = [BADGE_LABELS[c] for c in src.get("categories", []) if c in BADGE_LABELS]116        if src.get("page_quebec_score", 0) >= 0.45 or src.get("domain_quebec_score", 0) >= 0.6:117            badges.insert(0, "Québec")118        results.append({119            "title": src.get("title") or src["url"],120            "url": src["url"],121            "display_url": display_url(src["url"]),122            "snippet": snippet,123            "domain": src["domain"],124            "language": src.get("language"),125            "quebec_score": src.get("page_quebec_score", 0),126            "badges": badges,127            "published_at": src.get("published_at"),128        })129130    total = res["hits"]["total"]["value"]131    # Analytics agrégées, respectueuses de la vie privée — jamais bloquantes132    try:133        await db.record_search_query(q, analysis["language"], total, took_ms)134    except Exception:135        log.exception("analytics de recherche sautées")136137    return {"query": q, "total": total, "took_ms": took_ms, "page": page, "limit": limit, "results": results}138139140@app.get("/api/status")141async def api_status():142    snapshot: dict = {}143    try:144        snapshot = await db.status_snapshot()145    except Exception:146        log.exception("statut PG indisponible")147    search_ok = await search.ping()148    try:149        paused = await coord.is_paused()150    except Exception:151        paused = False152    return {153        "pages_indexed": snapshot.get("pages_indexed", 0),154        "domains_count": snapshot.get("domains_count", 0),155        "indexed_last_hour": snapshot.get("indexed_last_hour", 0),156        "fetched_last_hour": snapshot.get("fetched_last_hour", 0),157        "errors_last_hour": snapshot.get("errors_last_hour", 0),158        "frontier_pending": snapshot.get("frontier_pending", 0),159        "frontier_in_progress": snapshot.get("frontier_in_progress", 0),160        "crawler_state": "paused" if paused else "running",161        "search_ok": search_ok,162    }163164165@app.get("/api/live")166async def api_live():167    """Dernière page visitée par TrouveKABot — alimente le flux temps réel du footer.168169    Public mais volontairement minimal : domaine + URL + horodatage, rien d'interne.170    """171    try:172        events = await db.recent_events(1)173    except Exception:174        return {"event": None}175    if not events:176        return {"event": None}177    e = events[0]178    return {179        "event": {180            "at": e["at"],181            "url": e["url"],182            "domain": extract_domain(e["url"]),183            "outcome": e["outcome"],184        }185    }186187188class SubmitBody(BaseModel):189    url: str190191192@app.post("/api/submit")193async def api_submit(payload: SubmitBody):194    """Soumettre un site québécois. Soumission ≠ inclusion : le crawler valide."""195    if not is_http_url(payload.url):196        raise HTTPException(status_code=422, detail="URL invalide (http/https seulement)")197    url = canonicalize_url(payload.url)198    domain = extract_domain(url) if url else None199    if not url or not domain:200        raise HTTPException(status_code=422, detail="URL invalide")201    await db.add_submission(url)202    await db.enqueue_url(url, domain, priority=0.7, depth=0)203    return {204        "accepted": True,205        "message": "Merci! Le site sera visité par TrouveKABot. La soumission ne garantit pas l'inclusion.",206    }207208209# ---------------------------------------------------------------------- admin210211class SeedsBody(BaseModel):212    urls: list[str]213214215class RecrawlBody(BaseModel):216    url: str | None = None217    domain: str | None = None218219220class DomainBody(BaseModel):221    domain: str222223224@app.get("/api/admin/overview", dependencies=[Depends(require_admin)])225async def admin_overview():226    overview = await db.admin_overview()227    overview["paused"] = await coord.is_paused()228    try:229        overview["enrich_backlog"] = await coord.enrich_backlog()230    except Exception:231        overview["enrich_backlog"] = None232    return overview233234235@app.get("/api/admin/recent", dependencies=[Depends(require_admin)])236async def admin_recent(limit: int = Query(50, ge=1, le=200)):237    return {"events": await db.recent_events(limit)}238239240@app.post("/api/admin/pause", dependencies=[Depends(require_admin)])241async def admin_pause():242    await coord.pause_crawler()243    return {"paused": True}244245246@app.post("/api/admin/resume", dependencies=[Depends(require_admin)])247async def admin_resume():248    await coord.resume_crawler()249    return {"paused": False}250251252@app.post("/api/admin/seeds", dependencies=[Depends(require_admin)])253async def admin_seeds(payload: SeedsBody):254    added = 0255    for raw in payload.urls[:500]:256        url = canonicalize_url(raw.strip())257        domain = extract_domain(url) if url else None258        if url and domain:259            if await db.enqueue_url(url, domain, priority=1.0, depth=0, is_seed=True):260                added += 1261    return {"added": added}262263264@app.post("/api/admin/recrawl", dependencies=[Depends(require_admin)])265async def admin_recrawl(payload: RecrawlBody):266    if payload.url:267        url = canonicalize_url(payload.url)268        ok = await db.requeue_url(url) if url else False269        return {"requeued": 1 if ok else 0}270    if payload.domain:271        return {"requeued": await db.requeue_domain(payload.domain.lower())}272    raise HTTPException(status_code=422, detail="url ou domain requis")273274275@app.post("/api/admin/domains/block", dependencies=[Depends(require_admin)])276async def admin_block_domain(payload: DomainBody):277    await db.block_domain(payload.domain.lower())278    return {"blocked": payload.domain.lower()}279280281@app.get("/api/admin/frontier", dependencies=[Depends(require_admin)])282async def admin_frontier(283    domain: str | None = None,284    status: str | None = Query(None, pattern="^(pending|in_progress|done|failed|blocked)$"),285    limit: int = Query(100, ge=1, le=500),286):287    return {"items": await db.frontier_inspect(domain, status, limit)}288289290def main() -> None:291    import uvicorn292293    uvicorn.run("trouveka.api.main:app", host=settings.api_host, port=settings.api_port, workers=1)294295296if __name__ == "__main__":297    main()298