spb/countryatlas

1"""Infrastructure behaviour: run header, cache invalidation on snapshot swap, empty DB (503), rate limit, admin guard, OpenAPI, formatting."""2from __future__ import annotations34import os5import shutil6import time7from pathlib import Path89import duckdb10from fastapi.testclient import TestClient1112from countryatlas.api.cache import ResponseCache13from countryatlas.api.db import invalidate_registry_caches14from countryatlas.api.formatting import compact_number, format_change, format_short, format_value15from countryatlas.api.main import create_app16from countryatlas.api.provenance import source_url17from countryatlas.config import settings18from tests.api.conftest import RUN_ID192021def test_run_header_and_cache(client):22    r1 = client.get("/api/v1/countries/FRA")23    assert r1.headers["x-countryatlas-run"] == RUN_ID and r1.headers["x-cache"] == "MISS"24    r2 = client.get("/api/v1/countries/FRA")25    assert r2.headers["x-cache"] == "HIT" and r2.json() == r1.json()26    assert "x-cache" not in {k.lower() for k in client.get("/api/v1/countries/FRA/download.csv").headers}  # downloads not cached27    assert client.get("/api/v1/countries/FRA?x=1").headers["x-cache"] == "MISS"  # different query → different key282930def test_cache_key_includes_run_id():31    c = ResponseCache(maxsize=2)32    c.set(ResponseCache.key("run1", "/x", "a=1"), 1)33    assert c.get(ResponseCache.key("run1", "/x", "a=1")) == 134    assert c.get(ResponseCache.key("run2", "/x", "a=1")) is None35    assert ResponseCache.key("r", "/x", {"b": 2, "a": 1}) == ("r", "/x", "a=1&b=2")36    c.set(("k2",), 2)37    c.set(("k3",), 3)38    assert c.get(ResponseCache.key("run1", "/x", "a=1")) is None  # LRU evicted39    assert c.stats()["size"] == 2404142def test_snapshot_swap_reopens(tmp_path: Path, fixture_db: Path):43    db_path = tmp_path / "atlas.duckdb"44    shutil.copy(fixture_db, db_path)45    app = create_app(db_path, rate_limit_per_minute=0, cache=ResponseCache())46    with TestClient(app) as c:47        assert c.get("/api/v1/health").json()["run_id"] == RUN_ID48        assert c.get("/api/v1/countries/CAN").headers["x-cache"] == "MISS"49        assert c.get("/api/v1/countries/CAN").headers["x-cache"] == "HIT"50        # build a new snapshot with a different run_id and swap it atomically (new inode)51        new = tmp_path / "build.duckdb"52        shutil.copy(fixture_db, new)53        con = duckdb.connect(str(new))54        con.execute("UPDATE meta SET value = 'fixture-NEXT' WHERE key = 'build_run_id'")55        con.execute("UPDATE observations SET value = 12345 WHERE country_id = 'CAN' AND indicator_id = 'population' AND year = 2024")56        con.execute("UPDATE latest SET value = 12345 WHERE country_id = 'CAN' AND indicator_id = 'population'")57        con.close()58        os.replace(new, db_path)59        h = c.get("/api/v1/health").json()60        assert h["run_id"] == "fixture-NEXT"61        r = c.get("/api/v1/countries/CAN")62        assert r.headers["x-countryatlas-run"] == "fixture-NEXT" and r.headers["x-cache"] == "MISS"63        pop = next(m for m in r.json()["headline"] if m["indicator"] == "population")64        assert pop["value"] == 1234565        # file removed → 503 problem+json, health = empty; file back → recovers66        os.remove(db_path)67        assert c.get("/api/v1/health").json()["status"] == "empty"68        r = c.get("/api/v1/countries/CAN")69        assert r.status_code == 503 and r.json()["title"] == "Data not built yet" and r.headers["content-type"].startswith("application/problem+json")70        shutil.copy(fixture_db, db_path)71        assert c.get("/api/v1/countries/CAN").status_code == 200727374def test_empty_database(tmp_path: Path):75    app = create_app(tmp_path / "missing.duckdb", rate_limit_per_minute=0, cache=ResponseCache())76    with TestClient(app, raise_server_exceptions=False) as c:77        h = c.get("/api/v1/health")78        assert h.status_code == 200 and h.json()["status"] == "empty" and h.json()["observations"] == 079        for p in ("/api/v1/countries", "/api/v1/home", "/api/v1/search?q=x", "/api/v1/indicators/gdp", "/api/v1/rankings/gdp"):80            r = c.get(p)81            assert r.status_code == 503, p82            assert r.json()["title"] == "Data not built yet" and r.headers.get("retry-after")83        assert c.get("/api/v1/methodology").status_code == 200  # registry-only84        assert c.get("/api/v1/openapi.json").status_code == 200858687def test_rate_limit(fixture_db: Path):88    app = create_app(fixture_db, rate_limit_per_minute=3, cache=ResponseCache())89    with TestClient(app, raise_server_exceptions=False) as c:90        codes = [c.get("/api/v1/health").status_code for _ in range(5)]91        assert codes == [200] * 5  # health exempt92        codes = [c.get("/api/v1/countries/CAN", headers={"X-Forwarded-For": "203.0.113.9"}).status_code for _ in range(5)]93        assert codes[:3] == [200, 200, 200] and codes[3] == 42994        r = c.get("/api/v1/countries/CAN", headers={"X-Forwarded-For": "203.0.113.9"})95        assert r.headers["retry-after"] and r.json()["title"] == "Too many requests"96        assert c.get("/api/v1/countries/CAN", headers={"X-Forwarded-For": "203.0.113.10"}).status_code == 200  # other IP979899def test_admin_guard(client):100    assert client.get("/api/v1/admin/overview").status_code == 403101    assert client.get("/api/v1/admin/overview", headers={"X-Admin-Token": "wrong"}).status_code == 403102    ok = client.get("/api/v1/admin/overview", headers={"X-Admin-Token": "test-admin-token"})103    assert ok.status_code == 200104    body = ok.json()105    assert body["status"] == "ok" and body["counts"]["observations"] > 3000 and body["db"]["exists"]106    conns = {c["connector"]: c for c in body["connectors"]}107    assert conns["who"]["last_status"] == "failed" and conns["worldbank"]["n_ok"] == 2108    assert body["sources"] and body["cache"]["maxsize"] and body["scheduler"]["alive"] is None109    saved = settings.admin_token110    try:111        settings.admin_token = None112        r = client.get("/api/v1/admin/overview", headers={"X-Admin-Token": "test-admin-token"})113        assert r.status_code == 503 and r.json()["title"] == "Admin disabled"114    finally:115        settings.admin_token = saved116117118def test_admin_endpoints(client, tmp_path: Path):119    h = {"X-Admin-Token": "test-admin-token"}120    runs = client.get("/api/v1/admin/runs?limit=3", headers=h).json()121    assert runs["n"] == 3 and runs["items"][0]["run_id"] == RUN_ID122    assert client.get("/api/v1/admin/runs?connector=who", headers=h).json()["items"][0]["status"] == "failed"123    issues = client.get("/api/v1/admin/issues?severity=warning", headers=h).json()124    assert issues["n"] == 2 and issues["summary"][0]["code"] == "extreme_jump"125    cov = client.get("/api/v1/admin/coverage", headers=h).json()126    assert cov["n_indicators"] == 14 and cov["indicators"][0]["n_countries"] == 8 and cov["countries"][0]["coverage_pct"] == 100.0127    raw = client.get(f"/api/v1/admin/raw?run_id={RUN_ID}", headers=h).json()128    assert raw["n_runs"] == 5 and any(f.get("missing") for f in raw["files"])129    # refresh: no pid file → 409 ; pid file with own pid → SIGUSR1 delivered (handler installed)130    saved = settings.data_dir131    settings.data_dir = tmp_path132    try:133        r = client.post("/api/v1/admin/refresh", headers=h)134        assert r.status_code == 409135        import signal136137        got = []138        old = signal.signal(signal.SIGUSR1, lambda *a: got.append(1))139        (tmp_path / "scheduler.pid").write_text(str(os.getpid()))140        r = client.post("/api/v1/admin/refresh", headers=h)141        time.sleep(0.05)142        signal.signal(signal.SIGUSR1, old)143        assert r.status_code == 200 and r.json()["signal"] == "SIGUSR1" and got144        assert client.post("/api/v1/admin/cache/clear", headers=h).json()["ok"]145    finally:146        settings.data_dir = saved147148149def test_openapi_and_docs(client):150    spec = client.get("/api/v1/openapi.json").json()151    paths = set(spec["paths"])152    for p in ("/api/v1/health", "/api/v1/countries", "/api/v1/countries/{id}", "/api/v1/countries/{id}/topics/{topic}",153              "/api/v1/countries/{id}/series/{indicator}", "/api/v1/countries/{id}/changes", "/api/v1/countries/{id}/events",154              "/api/v1/countries/{id}/similar", "/api/v1/countries/{id}/insights", "/api/v1/countries/{id}/dna",155              "/api/v1/countries/{id}/download.{fmt}", "/api/v1/indicators", "/api/v1/indicators/{slug}", "/api/v1/indicators/{slug}/map",156              "/api/v1/indicators/{slug}/trend", "/api/v1/indicators/{slug}/download.{fmt}", "/api/v1/series", "/api/v1/rankings",157              "/api/v1/rankings/{indicator}", "/api/v1/rankings/{indicator}/history", "/api/v1/compare", "/api/v1/compare/snapshot",158              "/api/v1/compare/download.{fmt}", "/api/v1/regions", "/api/v1/regions/{slug}", "/api/v1/search", "/api/v1/home",159              "/api/v1/changes", "/api/v1/sources", "/api/v1/sources/{id}", "/api/v1/methodology", "/api/v1/admin/overview",160              "/api/v1/admin/runs", "/api/v1/admin/issues", "/api/v1/admin/coverage", "/api/v1/admin/raw", "/api/v1/admin/refresh"):161        assert p in paths, p162    assert "Provenance" in spec["components"]["schemas"]163    assert client.get("/api/v1/docs").status_code == 200 and client.get("/api/v1/redoc").status_code == 200164165166def test_unknown_route_and_validation(client):167    r = client.get("/api/v1/nothing-here")168    assert r.status_code == 404 and r.headers["content-type"].startswith("application/problem+json") and "/api/v1/docs" in r.json()["detail"]169    r = client.get("/api/v1/rankings/gdp?limit=0")170    assert r.status_code == 422 and r.json()["errors"][0]["loc"] == ["query", "limit"]171172173def test_formatting():174    assert format_value(53372.1, {"format": "currency", "unit_short": "US$"}) == "US$53.4k"175    assert format_value(1.23e12, {"format": "currency"}) == "US$1.2T"  # default prefix176    assert format_value(45.3e9, {"format": "currency", "unit_short": "intl $"}) == "intl $45.3B"177    assert format_short(53372.1, {"format": "currency", "unit_short": "US$"}) == "53.4k"178    assert format_short(3.44, {"format": "percent"}) == "3.4"179    assert format_value(3.44, {"format": "percent", "precision": 1}) == "3.4 %"180    assert format_value(82.13, {"format": "years"}) == "82.1 yrs"181    assert format_value(5.234, {"format": "tonnes", "precision": 2}) == "5.23 t"182    assert format_value(3.2, {"format": "per_1000"}) == "3.2 per 1,000"183    assert format_value(1_234_567, {"format": "number"}) == "1.2M"184    assert format_value(812.4, {"format": "number", "precision": 0}) == "812"185    assert format_value(None, {"format": "number"}) == "—"186    assert format_value(float("nan"), {"format": "number"}) == "—"187    assert compact_number(-2_500_000) == "-2.5M"188    assert format_change(1.2, 30.0, {"format": "percent"}) == "+1.2 pts"189    assert format_change(-100.0, -3.4, {"format": "currency"}) == "−3.4 %"190191192def test_registry_caches_cleared_on_snapshot_open(tmp_path: Path, fixture_db: Path):193    from countryatlas import registry194195    registry.topics()196    assert registry.topics.cache_info().currsize == 1197    invalidate_registry_caches()198    assert registry.topics.cache_info().currsize == 0 and registry.indicators.cache_info().currsize == 0199    # opening a new snapshot (new run_id) clears the loaders again200    registry.topics()201    db_path = tmp_path / "atlas.duckdb"202    shutil.copy(fixture_db, db_path)203    app = create_app(db_path, rate_limit_per_minute=0, cache=ResponseCache())204    with TestClient(app) as c:205        assert c.get("/api/v1/health").json()["status"] == "ok"206    assert registry.topics.cache_info().currsize == 0207208209def test_source_urls():210    assert source_url("worldbank", "WDI", "NY.GDP.PCAP.CD", "CA") == "https://data.worldbank.org/indicator/NY.GDP.PCAP.CD?locations=CA"211    assert source_url("owid", "grapher", "median-age") == "https://ourworldindata.org/grapher/median-age"212    assert source_url("owid", "energy", "renewables_share_elec") == "https://github.com/owid/energy-data"213    assert source_url("eurostat", "prc_hpi_a", "") == "https://ec.europa.eu/eurostat/databrowser/view/prc_hpi_a/default/table"214    assert source_url("who", "GHO", "WHOSIS_000001") == "https://www.who.int/data/gho/data/indicators/indicator-details/GHO/WHOSIS_000001"215    assert source_url("fred", "", "UNRATE") == "https://fred.stlouisfed.org/series/UNRATE"216    assert source_url("imf", "WEO", "NGDPD") == "https://data.imf.org/"217    assert source_url("bis", "", "") == "https://data.bis.org/" and source_url("ilo", "", "") == "https://ilostat.ilo.org/"218    assert source_url("oecd", "X", "Y") == "https://data-explorer.oecd.org/"219    assert source_url("unknown", "", "") is None220