spb/api-ka

1# ============================================2# Projet   : API-KA3# Fichier  : tests/test_collectors.py4# Node     : m3u96b5# Author   : Simon-Pierre Boucher6# Contact  : contact@spboucher.ai7# Date     : 2026-08-168# ============================================9"""Tests des collecteurs : pipeline complet, déduplication, échecs et statut retried."""1011from __future__ import annotations1213import datetime1415import pytest16from sqlalchemy import func, select1718from src.collectors import COLLECTORS, build_collectors, get_collector19from src.collectors.louka_collector import LoukaCollector20from src.config import SERVICES21from src.database.db import session_scope22from src.database.models import DATA_MODELS, CollectionRun2324SAMPLE_PAYLOAD = [25    {"id": 1, "nom": "enregistrement-a"},26    {"id": 2, "nom": "enregistrement-b"},27]282930def _count_rows(service: str, date_key: datetime.date) -> int:31    model = DATA_MODELS[service]32    with session_scope() as session:33        return (34            session.execute(35                select(func.count())36                .select_from(model)37                .where(model.source == service, model.date_key == date_key)38            ).scalar()39            or 040        )414243def _last_run(service: str, date_key: datetime.date) -> CollectionRun | None:44    with session_scope() as session:45        return (46            session.execute(47                select(CollectionRun)48                .where(49                    CollectionRun.service == service, CollectionRun.date_key == date_key50                )51                .order_by(CollectionRun.id.desc())52                .limit(1)53            )54            .scalars()55            .first()56        )575859def test_registry_covers_all_services() -> None:60    """Chaque service KA a un collecteur enregistré."""61    assert set(COLLECTORS) == set(SERVICES)62    assert len(build_collectors()) == 963    with pytest.raises(ValueError):64        get_collector("inconnu")656667def test_run_success(monkeypatch: pytest.MonkeyPatch) -> None:68    """Le pipeline complet insère les données et journalise un run success."""69    date_key = datetime.date(2026, 8, 1)70    collector = LoukaCollector(attempts=1, delays=(0,))71    monkeypatch.setattr(collector, "fetch", lambda: SAMPLE_PAYLOAD)7273    result = collector.run(date_key=date_key)7475    assert result["status"] == "success"76    assert result["records_count"] == 277    assert _count_rows("louka", date_key) == 278    run = _last_run("louka", date_key)79    assert run is not None80    assert run.status == "success"81    assert run.records_count == 282    assert run.node838485def test_run_deduplicates_by_checksum(monkeypatch: pytest.MonkeyPatch) -> None:86    """Une seconde collecte du même payload n'insère aucun doublon."""87    date_key = datetime.date(2026, 8, 2)88    collector = LoukaCollector(attempts=1, delays=(0,))89    monkeypatch.setattr(collector, "fetch", lambda: SAMPLE_PAYLOAD)9091    first = collector.run(date_key=date_key)92    second = collector.run(date_key=date_key)9394    assert first["records_count"] == 295    assert second["status"] == "success"96    assert second["records_count"] == 097    assert _count_rows("louka", date_key) == 29899100def test_run_failure_after_retries(monkeypatch: pytest.MonkeyPatch) -> None:101    """Après épuisement des relances : run failed + message d'erreur journalisé."""102    date_key = datetime.date(2026, 8, 3)103    collector = LoukaCollector(attempts=2, delays=(0, 0))104105    def broken() -> None:106        raise RuntimeError("source indisponible")107108    monkeypatch.setattr(collector, "fetch", broken)109110    result = collector.run(date_key=date_key)111112    assert result["status"] == "failed"113    assert "source indisponible" in (result["error"] or "")114    run = _last_run("louka", date_key)115    assert run is not None116    assert run.status == "failed"117    assert run.records_count == 0118    assert "source indisponible" in (run.error_message or "")119120121def test_run_retried_status(monkeypatch: pytest.MonkeyPatch) -> None:122    """Un succès après relance est journalisé avec le statut retried."""123    date_key = datetime.date(2026, 8, 4)124    collector = LoukaCollector(attempts=2, delays=(0, 0))125    calls = {"count": 0}126127    def flaky() -> list[dict]:128        calls["count"] += 1129        if calls["count"] == 1:130            raise RuntimeError("transitoire")131        return SAMPLE_PAYLOAD132133    monkeypatch.setattr(collector, "fetch", flaky)134135    result = collector.run(date_key=date_key)136137    assert result["status"] == "retried"138    assert result["records_count"] == 2139    run = _last_run("louka", date_key)140    assert run is not None141    assert run.status == "retried"142143144def test_validate_rejects_empty_payload() -> None:145    """Un payload vide ou None est rejeté par la validation."""146    collector = LoukaCollector(attempts=1, delays=(0,))147    with pytest.raises(ValueError):148        collector.validate(None)149    with pytest.raises(ValueError):150        collector.validate([])151152153def test_checksum_is_stable_and_order_insensitive() -> None:154    """Le checksum SHA-256 est canonique (indépendant de l'ordre des clés)."""155    a = LoukaCollector.checksum({"x": 1, "y": 2})156    b = LoukaCollector.checksum({"y": 2, "x": 1})157    c = LoukaCollector.checksum({"x": 1, "y": 3})158    assert a == b159    assert a != c160    assert len(a) == 64161162163# ------------------------------------------------------------------- job-ka164165166def test_jobka_pipeline_inserts_into_jobka_data(167    monkeypatch: pytest.MonkeyPatch,168) -> None:169    """Le collecteur job-ka insère dans jobka_data et journalise un run success."""170    from src.collectors.jobka_collector import JobkaCollector171172    date_key = datetime.date(2026, 8, 5)173    collector = JobkaCollector(attempts=1, delays=(0,))174    monkeypatch.setattr(175        collector,176        "fetch",177        lambda: [178            {"uid": "a1", "title": "Analyste", "employer": "Ville de Québec"},179            {"uid": "b2", "title": "Technicien", "employer": "CHU de Québec"},180        ],181    )182183    result = collector.run(date_key=date_key)184185    assert result["service"] == "jobka"186    assert result["status"] == "success"187    assert result["records_count"] == 2188    assert _count_rows("jobka", date_key) == 2189    run = _last_run("jobka", date_key)190    assert run is not None191    assert run.status == "success"192193194def test_jobka_fetch_paginates_offset_until_total(195    monkeypatch: pytest.MonkeyPatch,196) -> None:197    """fetch() pagine /api/jobs (limit/offset, clé "jobs") jusqu'au total annoncé."""198    from src.collectors import base_collector199    from src.collectors.jobka_collector import JobkaCollector200201    total = 7202    jobs = [{"uid": f"job-{i}", "title": f"Poste {i}"} for i in range(total)]203    calls: list[dict] = []204205    class FakeResponse:206        def __init__(self, offset: int, limit: int) -> None:207            self._batch = jobs[offset : offset + limit]208209        def raise_for_status(self) -> None:210            return None211212        def json(self) -> dict:213            return {"total": total, "count": len(self._batch), "jobs": self._batch}214215    class FakeClient:216        def __init__(self, *args, **kwargs) -> None:217            pass218219        def __enter__(self) -> "FakeClient":220            return self221222        def __exit__(self, *exc) -> None:223            return None224225        def get(self, url: str, params: dict) -> FakeResponse:226            calls.append({"url": url, **params})227            return FakeResponse(params["offset"], params["limit"])228229    monkeypatch.setattr(base_collector.httpx, "Client", FakeClient)230    collector = JobkaCollector(attempts=1, delays=(0,))231    monkeypatch.setattr(type(collector), "page_size", 3)232233    items = collector.fetch()234235    assert [i["uid"] for i in items] == [f"job-{i}" for i in range(total)]236    # 3 pages : offsets 0, 3, 6 — l'arrêt vient du total annoncé par l'API.237    assert [c["offset"] for c in calls] == [0, 3, 6]238    assert all(c["limit"] == 3 for c in calls)239240241def test_jobka_registered_with_expected_settings() -> None:242    """job-ka est enregistré dans le registre avec la bonne configuration."""243    from src.collectors.jobka_collector import JobkaCollector244245    assert COLLECTORS["jobka"] is JobkaCollector246    collector = get_collector("jobka")247    assert collector.service == "jobka"248    assert collector.items_key == "jobs"249    assert collector.pagination == "offset"250    assert DATA_MODELS["jobka"].__tablename__ == "jobka_data"251