spb/countryatlas

1"""OECD SDMX connector — recorded csvfilewithlabels fixtures (house prices RHP quarterly; KEI production index monthly)."""2from __future__ import annotations34from datetime import date56import pytest78from countryatlas.connectors._util import ConnectorError9from countryatlas.connectors.oecd import OECDConnector, _clean_key, _with_version10from countryatlas.models import IndicatorSourceSpec, RawPayload1112HP = "OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0"13KEI = "OECD.SDD.STES,DSD_KEI@DF_KEI,4.0"141516def _raw(fixtures_dir, name: str, dataset: str, code: str, retrieved_at) -> RawPayload:17    body = (fixtures_dir / "oecd" / name).read_bytes()18    return RawPayload(connector="oecd", dataset=dataset, code=code, url="fixture://oecd", retrieved_at=retrieved_at,19                      status_code=200, content_type="text/csv", body=body)202122@pytest.fixture23def oecd() -> OECDConnector:24    return OECDConnector()252627def test_key_and_version_helpers():28    assert _clean_key("*.Q.RHP.IX") == ".Q.RHP.IX"29    assert _clean_key(".Q.RHP.IX") == ".Q.RHP.IX"30    assert _with_version("OECD.ELS.SPD,DSD_SOCX_AGG@DF_SOCX_AGG") == "OECD.ELS.SPD,DSD_SOCX_AGG@DF_SOCX_AGG,"31    assert _with_version(HP) == HP323334def test_quarterly_index(oecd, fixtures_dir, retrieved_at):35    spec = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="oecd", dataset=HP, code="*.Q.RHP.IX",36                               priority=1, frequency="Q")37    rows = oecd.normalize(_raw(fixtures_dir, "house_prices_RHP_sample.csv", HP, "*.Q.RHP.IX", retrieved_at), spec)38    assert {r.country_id for r in rows} == {"CAN", "USA"}39    usa = sorted((r for r in rows if r.country_id == "USA"), key=lambda r: r.period)40    assert usa[0].period == date(2023, 1, 1) and usa[0].frequency == "Q" and usa[0].year == 202341    assert any(r.period == date(2025, 4, 1) for r in usa), "2025-Q2 → 2025-04-01"42    q2 = next(r for r in usa if r.period == date(2025, 4, 1))43    assert q2.value == pytest.approx(154.108309038445)44    assert q2.unit == "index (2015 = 100)"45    assert q2.metadata["source_unit"] == "IX" and q2.metadata["base_period"] == "2015"46    assert not q2.is_estimate and not q2.is_forecast47    assert oecd.validate(rows).errors == 0484950def test_yoy_transform_derives_growth(oecd, fixtures_dir, retrieved_at):51    spec = IndicatorSourceSpec(indicator_id="house-price-growth", connector="oecd", dataset=HP, code="*.Q.RHP.IX",52                               priority=1, frequency="Q", transform="yoy")53    raw = _raw(fixtures_dir, "house_prices_RHP_sample.csv", HP, "*.Q.RHP.IX", retrieved_at)54    idx = {(r.country_id, r.period): r.value for r in oecd.normalize(55        raw, IndicatorSourceSpec(indicator_id="real-house-price-index", connector="oecd", dataset=HP, code="*.Q.RHP.IX"))}56    rows = oecd.normalize(raw, spec)57    assert rows, "growth rows exist once a full year of history is present"58    # first four quarters (2023) have no previous-year value → dropped59    assert min(r.period for r in rows) == date(2024, 1, 1)60    r = next(r for r in rows if r.country_id == "CAN" and r.period == date(2025, 1, 1))61    expected = (idx[("CAN", date(2025, 1, 1))] / idx[("CAN", date(2024, 1, 1))] - 1) * 10062    assert r.value == pytest.approx(expected)63    assert r.unit == "annual %" and r.metadata["derived"].startswith("year-on-year")646566def test_monthly_kei_periods(oecd, fixtures_dir, retrieved_at):67    spec = IndicatorSourceSpec(indicator_id="industrial-production-index", connector="oecd", dataset=KEI,68                               code="*.M.PRVM.IX.BTE.Y._Z", priority=1, frequency="M")69    rows = oecd.normalize(_raw(fixtures_dir, "kei_PRVM_sample.csv", KEI, spec.code, retrieved_at), spec)70    can = sorted((r for r in rows if r.country_id == "CAN"), key=lambda r: r.period)71    assert can[0].frequency == "M" and can[0].period.day == 1 and can[0].period.year == 202672    assert any(r.period == date(2026, 6, 1) for r in can)73    assert next(r for r in can if r.period == date(2026, 6, 1)).value == pytest.approx(109.273561067984)747576def test_post_hoc_filter_and_estimate_flag(oecd, fixtures_dir, retrieved_at):77    body = (fixtures_dir / "oecd" / "kei_PRVM_sample.csv").read_text()78    # mark one row as estimate and another as provisional79    lines = body.splitlines()80    lines[1] = lines[1].replace(",A,Normal value,", ",E,Estimated value,", 1)81    lines[2] = lines[2].replace(",A,Normal value,", ",P,Provisional value,", 1)82    raw = RawPayload(connector="oecd", dataset=KEI, code="*.M.PRVM.IX.BTE.Y._Z", url="fixture://oecd",83                     retrieved_at=retrieved_at, status_code=200, content_type="text/csv", body="\n".join(lines).encode())84    spec = IndicatorSourceSpec(indicator_id="industrial-production-index", connector="oecd", dataset=KEI,85                               code="*.M.PRVM.IX.BTE.Y._Z", priority=1, frequency="M", params={"filter": {"ACTIVITY": "BTE"}})86    rows = oecd.normalize(raw, spec)87    assert sum(r.is_estimate for r in rows) == 288    assert {r.metadata.get("obs_status") for r in rows if r.is_estimate} == {"E", "P"}89    bad = IndicatorSourceSpec(indicator_id="industrial-production-index", connector="oecd", dataset=KEI,90                              code="*.M.PRVM.IX.BTE.Y._Z", params={"filter": {"ACTIVITY": "ZZZ"}})91    assert oecd.normalize(raw, bad) == []929394def test_underspecified_key_raises(oecd, fixtures_dir, retrieved_at):95    body = (fixtures_dir / "oecd" / "house_prices_RHP_sample.csv").read_text()96    dup = body + "\n" + body.splitlines()[1].replace(",RHP,Real house price indices,", ",HPI,Nominal house price indices,")97    raw = RawPayload(connector="oecd", dataset=HP, code="*.Q..IX", url="fixture://oecd", retrieved_at=retrieved_at,98                     status_code=200, content_type="text/csv", body=dup.encode())99    spec = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="oecd", dataset=HP, code="*.Q..IX")100    with pytest.raises(ConnectorError, match="under-specified"):101        oecd.normalize(raw, spec)102103104def test_registry_specs_for_oecd():105    from countryatlas import registry106107    specs = registry.source_specs("oecd")108    by_ind = {s.indicator_id: s for s in specs}109    assert {"real-house-price-index", "tax-revenue-pct-gdp", "average-annual-wages", "industrial-production-index",110            "tertiary-attainment-25-64"} <= set(by_ind)111    assert by_ind["industrial-production-index"].frequency == "M"112    assert by_ind["house-price-growth"].transform == "yoy"113    for s in specs:114        assert s.dataset.count(",") in (1, 2), s.dataset115        assert s.code.startswith("*."), s.code116117118@pytest.mark.live119def test_live_house_prices_two_countries():120    oecd = OECDConnector()121    spec = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="oecd", dataset=HP, code="CAN+USA.Q.RHP.IX",122                               priority=1, frequency="Q", params={"startPeriod": "2024-Q1"})123    raw = oecd.fetch(spec)124    rows = oecd.normalize(raw, spec)125    assert {r.country_id for r in rows} == {"CAN", "USA"}126    assert all(r.frequency == "Q" for r in rows)127    oecd.close()128