spb/trouve-ka Public

1# Trouve-KA — API FastAPI2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""API de Trouve-KA.67Publique : /api/search, /api/status, /api/submit, /api/health8Protégée (X-Admin-Token) : /api/admin/* — contrôle du crawler, jamais public sans auth.910Dégradation gracieuse : si OpenSearch tombe, /api/status répond quand même;11si Postgres tombe, la recherche répond quand même (analytics sautées).12"""1314import html15import re16import time17from contextlib import asynccontextmanager18from typing import Annotated1920from fastapi import Depends, FastAPI, Header, HTTPException, Query21from fastapi.middleware.cors import CORSMiddleware22from pydantic import BaseModel2324from trouveka.config import get_settings25from trouveka.database import Database26from trouveka.logging import get_logger27from trouveka.queue import Coordination28from trouveka.ranking import build_search_body29from trouveka.search_core import SearchCore30from trouveka.shared import canonicalize_url, display_url, extract_domain, is_http_url3132log = get_logger("api")33settings = get_settings()3435db = Database(settings.database_url, pool_min=settings.pg_pool_min, pool_max=settings.pg_pool_max)36coord = Coordination(settings.redis_url)37search = SearchCore(settings.search_url, settings.search_index)383940@asynccontextmanager41async def lifespan(_app: FastAPI):42    await db.connect()43    await search.ensure_index()44    yield45    await search.close()46    await coord.close()47    await db.close()484950app = FastAPI(title="Trouve-KA API", version="0.1.0", lifespan=lifespan)5152# Derrière ngrok (www.trouve-ka.com), le web app proxifie /api : CORS permissif inutile53# en prod, mais pratique en dev local (web sur :3000, API sur :8080).54app.add_middleware(55    CORSMiddleware,56    allow_origins=["http://localhost:3000", settings.public_url],57    allow_methods=["GET", "POST"],58    allow_headers=["*", "X-Admin-Token"],59)606162def require_admin(x_admin_token: Annotated[str | None, Header()] = None) -> None:63    if not x_admin_token or x_admin_token != settings.admin_token:64        raise HTTPException(status_code=401, detail="Jeton admin invalide")656667_TAG_RE = re.compile(r"<(?!/?em>)[^>]*>")686970def _safe_snippet(fragments: list[str]) -> str:71    """Ne laisse passer que <em>/</em> (highlight); tout le reste est échappé par OpenSearch."""72    snippet = _TAG_RE.sub("", " … ".join(fragments))[:400]73    # La troncature peut couper une balise en deux (« …Plombier</em ») : on nettoie74    snippet = re.sub(r"<[^>]*$", "", snippet)75    if snippet.count("<em>") > snippet.count("</em>"):76        snippet += "</em>"77    return snippet787980BADGE_LABELS = {"government": "Gouvernement", "news": "Actualités", "education": "Éducation"}818283# ---------------------------------------------------------------------- publique8485@app.get("/api/health")86async def health():87    return {"ok": True, "search_ok": await search.ping()}888990@app.get("/api/search")91async def api_search(92    q: str = Query(..., min_length=1, max_length=200),93    page: int = Query(1, ge=1, le=100),94    limit: int = Query(10, ge=1, le=50),95    language: str | None = Query(None, pattern="^(fr|en)$"),96    location: str | None = None,97    category: str | None = Query(None, max_length=40),98    quebec_only: bool = False,99    freshness: str | None = Query(None, pattern="^(day|week|month|year)$"),100    images: bool = False,101):102    started = time.monotonic()103    query_text = f"{q} {location}" if location else q104    body, analysis = build_search_body(105        query_text, page=page, limit=limit, language=language,106        category=category, quebec_only=quebec_only, freshness=freshness,107        images_only=images,108    )109    try:110        res = await search.search(body)111    except Exception:112        log.exception("recherche échouée", extra={"ctx": {"q": q}})113        raise HTTPException(status_code=503, detail="Le moteur de recherche est temporairement indisponible")114115    took_ms = int((time.monotonic() - started) * 1000)116    results = []117    for hit in res["hits"]["hits"]:118        src = hit["_source"]119        highlight = hit.get("highlight", {})120        fragments = highlight.get("body") or highlight.get("description") or []121        snippet = _safe_snippet(fragments) if fragments else html.escape(src.get("description") or "")[:400]122        badges = [BADGE_LABELS[c] for c in src.get("categories", []) if c in BADGE_LABELS]123        if src.get("page_quebec_score", 0) >= 0.45 or src.get("domain_quebec_score", 0) >= 0.6:124            badges.insert(0, "Québec")125        results.append({126            "title": src.get("title") or src["url"],127            "url": src["url"],128            "display_url": display_url(src["url"]),129            "snippet": snippet,130            "domain": src["domain"],131            "language": src.get("language"),132            "quebec_score": src.get("page_quebec_score", 0),133            "badges": badges,134            "published_at": src.get("published_at"),135            "image": src.get("image_url"),136        })137138    total = res["hits"]["total"]["value"]139    # Analytics agrégées, respectueuses de la vie privée — jamais bloquantes140    try:141        await db.record_search_query(q, analysis["language"], total, took_ms)142    except Exception:143        log.exception("analytics de recherche sautées")144145    return {"query": q, "total": total, "took_ms": took_ms, "page": page, "limit": limit, "results": results}146147148@app.get("/api/status")149async def api_status():150    snapshot: dict = {}151    try:152        snapshot = await db.status_snapshot()153    except Exception:154        log.exception("statut PG indisponible")155    search_ok = await search.ping()156    try:157        paused = await coord.is_paused()158    except Exception:159        paused = False160    return {161        "pages_indexed": snapshot.get("pages_indexed", 0),162        "domains_count": snapshot.get("domains_count", 0),163        "indexed_last_hour": snapshot.get("indexed_last_hour", 0),164        "fetched_last_hour": snapshot.get("fetched_last_hour", 0),165        "errors_last_hour": snapshot.get("errors_last_hour", 0),166        "frontier_pending": snapshot.get("frontier_pending", 0),167        "frontier_in_progress": snapshot.get("frontier_in_progress", 0),168        "crawler_state": "paused" if paused else "running",169        "search_ok": search_ok,170    }171172173@app.get("/api/live")174async def api_live():175    """Dernière page visitée par TrouveKABot — alimente le flux temps réel du footer.176177    Public mais volontairement minimal : domaine + URL + horodatage, rien d'interne.178    """179    try:180        events = await db.recent_events(1)181    except Exception:182        return {"event": None}183    if not events:184        return {"event": None}185    e = events[0]186    return {187        "event": {188            "at": e["at"],189            "url": e["url"],190            "domain": extract_domain(e["url"]),191            "outcome": e["outcome"],192        }193    }194195196class SubmitBody(BaseModel):197    url: str198199200@app.post("/api/submit")201async def api_submit(payload: SubmitBody):202    """Soumettre un site québécois. Soumission ≠ inclusion : le crawler valide."""203    if not is_http_url(payload.url):204        raise HTTPException(status_code=422, detail="URL invalide (http/https seulement)")205    url = canonicalize_url(payload.url)206    domain = extract_domain(url) if url else None207    if not url or not domain:208        raise HTTPException(status_code=422, detail="URL invalide")209    await db.add_submission(url)210    await db.enqueue_url(url, domain, priority=0.7, depth=0)211    return {212        "accepted": True,213        "message": "Merci! Le site sera visité par TrouveKABot. La soumission ne garantit pas l'inclusion.",214    }215216217# ---------------------------------------------------------------------- admin218219class SeedsBody(BaseModel):220    urls: list[str]221222223class RecrawlBody(BaseModel):224    url: str | None = None225    domain: str | None = None226227228class DomainBody(BaseModel):229    domain: str230231232@app.get("/api/admin/overview", dependencies=[Depends(require_admin)])233async def admin_overview():234    overview = await db.admin_overview()235    overview["paused"] = await coord.is_paused()236    try:237        overview["enrich_backlog"] = await coord.enrich_backlog()238    except Exception:239        overview["enrich_backlog"] = None240    return overview241242243@app.get("/api/admin/recent", dependencies=[Depends(require_admin)])244async def admin_recent(limit: int = Query(50, ge=1, le=200)):245    return {"events": await db.recent_events(limit)}246247248@app.post("/api/admin/pause", dependencies=[Depends(require_admin)])249async def admin_pause():250    await coord.pause_crawler()251    return {"paused": True}252253254@app.post("/api/admin/resume", dependencies=[Depends(require_admin)])255async def admin_resume():256    await coord.resume_crawler()257    return {"paused": False}258259260@app.post("/api/admin/seeds", dependencies=[Depends(require_admin)])261async def admin_seeds(payload: SeedsBody):262    added = 0263    for raw in payload.urls[:500]:264        url = canonicalize_url(raw.strip())265        domain = extract_domain(url) if url else None266        if url and domain:267            if await db.enqueue_url(url, domain, priority=1.0, depth=0, is_seed=True):268                added += 1269    return {"added": added}270271272@app.post("/api/admin/recrawl", dependencies=[Depends(require_admin)])273async def admin_recrawl(payload: RecrawlBody):274    if payload.url:275        url = canonicalize_url(payload.url)276        ok = await db.requeue_url(url) if url else False277        return {"requeued": 1 if ok else 0}278    if payload.domain:279        return {"requeued": await db.requeue_domain(payload.domain.lower())}280    raise HTTPException(status_code=422, detail="url ou domain requis")281282283@app.post("/api/admin/domains/block", dependencies=[Depends(require_admin)])284async def admin_block_domain(payload: DomainBody):285    await db.block_domain(payload.domain.lower())286    return {"blocked": payload.domain.lower()}287288289@app.get("/api/admin/frontier", dependencies=[Depends(require_admin)])290async def admin_frontier(291    domain: str | None = None,292    status: str | None = Query(None, pattern="^(pending|in_progress|done|failed|blocked)$"),293    limit: int = Query(100, ge=1, le=500),294):295    return {"items": await db.frontier_inspect(domain, status, limit)}296297298def main() -> None:299    import uvicorn300301    uvicorn.run("trouveka.api.main:app", host=settings.api_host, port=settings.api_port, workers=1)302303304if __name__ == "__main__":305    main()306