spb/countryatlas

1"""FRED connector — offline tests on recorded /series + /series/observations payloads, plus a live smoke test."""2from __future__ import annotations34import json5from datetime import date67import pytest89from countryatlas.connectors import fred as fred_mod10from countryatlas.connectors._util import ConnectorError11from countryatlas.connectors.fred import FREDConnector, parse_fred_datetime, redact12from countryatlas.models import IndicatorSourceSpec13from tests.connectors._helpers import make_response, raw_from_file1415KEY = "0123456789abcdef0123456789abcdef"161718def _series_meta(fixtures_dir, code):19    return json.load(open(fixtures_dir / "fred" / f"series_{code}.json"))["seriess"][0]202122@pytest.fixture23def conn(monkeypatch):24    monkeypatch.setattr(fred_mod.settings, "fred_api_key", KEY)25    c = FREDConnector()26    c.MIN_SPACING_S = 0.0  # no pacing in unit tests27    return c282930def test_redaction_and_datetime_parsing():31    assert redact(f"https://x/fred/series?series_id=A&api_key={KEY}&file_type=json") == \32        "https://x/fred/series?series_id=A&api_key=<redacted>&file_type=json"33    dt = parse_fred_datetime("2026-09-01 15:16:43-05")34    assert dt is not None and dt.utcoffset().total_seconds() == -5 * 3600 and dt.year == 202635    assert parse_fred_datetime("2026-09-01").tzinfo is not None363738def test_missing_key_raises(monkeypatch):39    monkeypatch.setattr(fred_mod.settings, "fred_api_key", None)40    c = FREDConnector()41    with pytest.raises(ConnectorError, match="FRED_API_KEY"):42        c.get("https://api.stlouisfed.org/fred/series", params={"series_id": "FEDFUNDS"})434445def test_fetch_adds_key_aggregates_weekly_and_redacts_url(conn, fixtures_dir, monkeypatch):46    series = (fixtures_dir / "fred" / "series_MORTGAGE30US.json").read_bytes()47    obs = (fixtures_dir / "fred" / "obs_MORTGAGE30US_m.json").read_bytes()48    calls: list[dict] = []4950    # patch the underlying Connector.get (the FRED override adds the key + pacing on top of it)51    import countryatlas.connectors.base as base_mod5253    def _base_get(self, url, params=None, headers=None):54        calls.append({"url": url, "params": dict(params or {})})55        body = series if url.endswith("/series") else obs56        return make_response(url, body, "application/json", params)5758    monkeypatch.setattr(base_mod.Connector, "get", _base_get)59    spec = IndicatorSourceSpec(indicator_id="mortgage-rate", connector="fred", dataset="FRED", code="MORTGAGE30US",60                               countries=["USA"], frequency="M", priority=1)61    p = conn.fetch(spec)62    assert calls[0]["params"]["api_key"] == KEY and calls[0]["params"]["file_type"] == "json"63    assert calls[1]["params"]["frequency"] == "m" and calls[1]["params"]["aggregation_method"] == "avg"64    assert calls[1]["params"]["observation_start"] == "1950-01-01"65    assert KEY not in p.url and "api_key=<redacted>" in p.url66    assert KEY not in json.dumps(p.meta)67    assert p.source_updated_at is not None and p.source_updated_at.year == 202668    assert p.meta["series_meta"]["frequency_short"] == "W"69    rows = conn.normalize(p, spec)70    assert [(r.period.isoformat(), r.value) for r in rows] == [("2026-05-01", 6.44), ("2026-06-01", 6.49),71                                                               ("2026-07-01", 6.54), ("2026-08-01", 6.67)]72    assert all(r.frequency == "M" and r.country_id == "USA" for r in rows)  # the "." (incomplete Sept) is skipped73    assert rows[0].metadata["aggregated_from"] == "W"747576def test_normalize_monthly_series_and_missing_dot(conn, fixtures_dir):77    spec = IndicatorSourceSpec(indicator_id="policy-rate", connector="fred", dataset="FRED", code="FEDFUNDS",78                               countries=["USA"], frequency="M", priority=2)79    raw = raw_from_file("fred", "FRED", "FEDFUNDS", fixtures_dir / "fred" / "obs_FEDFUNDS.json", "application/json",80                        series_meta=_series_meta(fixtures_dir, "FEDFUNDS"), request={"series_id": "FEDFUNDS"})81    rows = conn.normalize(raw, spec)82    assert len(rows) == 6 and rows[0].period == date(2024, 1, 1) and rows[0].frequency == "M"83    assert rows[0].unit == "% per annum" and rows[0].value == pytest.approx(5.33)84    # inject a missing value85    doc = json.loads(raw.body)86    doc["observations"].append({"date": "2024-07-01", "value": "."})87    raw.body = json.dumps(doc).encode()88    assert len(conn.normalize(raw, spec)) == 6899091def test_yoy_pct_transform_computes_inflation(conn, fixtures_dir):92    spec = IndicatorSourceSpec(indicator_id="inflation", connector="fred", dataset="FRED", code="CPIAUCSL",93                               countries=["USA"], frequency="M", priority=3, transform="yoy_pct")94    raw = raw_from_file("fred", "FRED", "CPIAUCSL", fixtures_dir / "fred" / "obs_CPIAUCSL.json", "application/json",95                        series_meta=_series_meta(fixtures_dir, "CPIAUCSL"), request={"series_id": "CPIAUCSL"})96    obs = {o["date"]: float(o["value"]) for o in json.loads(raw.body)["observations"]}97    rows = conn.normalize(raw, spec)98    # 2024 has no predecessor in the fixture → only 2025-01..03 remain99    assert [r.period.isoformat() for r in rows] == ["2025-01-01", "2025-02-01", "2025-03-01"]100    expected = (obs["2025-01-01"] / obs["2024-01-01"] - 1) * 100101    assert rows[0].value == pytest.approx(expected)102    assert rows[0].unit == "annual %" and rows[0].metadata["transform"] == "yoy_pct"103104105def test_bis_series_derives_country_and_rebases(conn, fixtures_dir):106    spec = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="fred", dataset="FRED", code="QCAR628BIS",107                               frequency="Q", priority=3, transform="rebase:2015")108    assert FREDConnector.country_for(spec) == "CAN"109    assert FREDConnector.country_for(spec.model_copy(update={"code": "QXMR628BIS"})) is None  # euro area110    raw = raw_from_file("fred", "FRED", "QCAR628BIS", fixtures_dir / "fred" / "obs_QCAR628BIS.json", "application/json",111                        series_meta=_series_meta(fixtures_dir, "QCAR628BIS"), request={"series_id": "QCAR628BIS"})112    rows = conn.normalize(raw, spec)113    assert all(r.country_id == "CAN" and r.frequency == "Q" for r in rows)114    y2015 = [r.value for r in rows if r.year == 2015]115    assert len(y2015) == 4 and sum(y2015) / 4 == pytest.approx(100.0)116    assert rows[0].period == date(2015, 1, 1) and rows[1].period == date(2015, 4, 1)117    assert rows[0].unit == "index (2015 = 100)"118119120def test_quarterly_and_annual_periods(conn, fixtures_dir):121    meta = _series_meta(fixtures_dir, "FEDFUNDS")122    body = json.dumps({"observations": [{"date": "2025-04-01", "value": "65.1"}, {"date": "2025-07-01", "value": "65.3"}]}).encode()123    raw = raw_from_file("fred", "FRED", "RHORUSQ156N", fixtures_dir / "fred" / "obs_FEDFUNDS.json", "application/json",124                        series_meta={**meta, "frequency_short": "Q"}, request={})125    raw.body = body126    spec = IndicatorSourceSpec(indicator_id="homeownership-rate", connector="fred", dataset="FRED", code="RHORUSQ156N",127                               countries=["USA"], frequency="Q")128    rows = conn.normalize(raw, spec)129    assert [(r.period, r.frequency) for r in rows] == [(date(2025, 4, 1), "Q"), (date(2025, 7, 1), "Q")]130    raw.meta["series_meta"]["frequency_short"] = "A"131    raw.body = json.dumps({"observations": [{"date": "2024-01-01", "value": "83730"}]}).encode()132    spec = spec.model_copy(update={"indicator_id": "median-household-income", "code": "MEHOINUSA672N", "frequency": "A"})133    rows = conn.normalize(raw, spec)134    assert rows[0].period == date(2024, 1, 1) and rows[0].frequency == "A" and rows[0].year == 2024135136137@pytest.mark.live138def test_live_fedfunds(monkeypatch):139    import os140141    if not os.environ.get("FRED_API_KEY"):142        pytest.skip("FRED_API_KEY not set")143    monkeypatch.setattr(fred_mod.settings, "fred_api_key", os.environ["FRED_API_KEY"])144    conn = FREDConnector()145    spec = IndicatorSourceSpec(indicator_id="policy-rate", connector="fred", dataset="FRED", code="FEDFUNDS",146                               countries=["USA"], frequency="M", params={"observation_start": "2024-01-01"})147    rows = conn.normalize(conn.fetch(spec), spec)148    assert len(rows) >= 20 and rows[0].country_id == "USA"149