Connectors IMF/OECD/Eurostat/WHO/FRED/BIS/ILO + per-connector mapping files; registry fixes
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
169 changed files +19,065 −19
added
apps/web/.env.example
+4 −0
@@ -0,0 +1,4 @@ | ||
| 1 | +# Server-side API base (FastAPI, loopback). Client code calls same-origin /api/v1/* which Next rewrites here. | |
| 2 | +API_URL=http://127.0.0.1:8291 | |
| 3 | +# Public site URL used for canonical links, OpenGraph and the sitemap. | |
| 4 | +NEXT_PUBLIC_SITE_URL=https://www.countryatlas.co | |
added
apps/web/next.config.ts
+50 −0
@@ -0,0 +1,50 @@ | ||
| 1 | +import type { NextConfig } from 'next'; | |
| 2 | +import { existsSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | + | |
| 5 | +// Monorepo: a single `.env` may live at the repository root; Next only reads the app directory. | |
| 6 | +for (const candidate of [path.resolve(process.cwd(), '../../.env'), path.resolve(process.cwd(), '.env')]) { | |
| 7 | + if (existsSync(candidate)) { | |
| 8 | + try { | |
| 9 | + process.loadEnvFile(candidate); | |
| 10 | + } catch { | |
| 11 | + /* ignore malformed env */ | |
| 12 | + } | |
| 13 | + } | |
| 14 | +} | |
| 15 | + | |
| 16 | +const API_URL = process.env.API_URL ?? 'http://127.0.0.1:8291'; | |
| 17 | + | |
| 18 | +const nextConfig: NextConfig = { | |
| 19 | + reactStrictMode: true, | |
| 20 | + poweredByHeader: false, | |
| 21 | + allowedDevOrigins: ['127.0.0.1', 'localhost'], | |
| 22 | + outputFileTracingRoot: path.resolve(__dirname, '../..'), | |
| 23 | + experimental: { | |
| 24 | + optimizePackageImports: ['lucide-react'], | |
| 25 | + }, | |
| 26 | + // Browser-side fetches go to the same origin; the FastAPI service is loopback-only (ARCHITECTURE §1, §9). | |
| 27 | + async rewrites() { | |
| 28 | + return [{ source: '/api/v1/:path*', destination: `${API_URL}/api/v1/:path*` }]; | |
| 29 | + }, | |
| 30 | + async headers() { | |
| 31 | + const PUBLIC_CACHE = { key: 'Cache-Control', value: 'public, s-maxage=600, stale-while-revalidate=3600' }; | |
| 32 | + const NO_STORE = { key: 'Cache-Control', value: 'private, no-store' }; | |
| 33 | + return [ | |
| 34 | + { | |
| 35 | + source: '/(.*)', | |
| 36 | + headers: [ | |
| 37 | + { key: 'X-Content-Type-Options', value: 'nosniff' }, | |
| 38 | + { key: 'Referrer-Policy', value: 'strict-origin-when-cross-origin' }, | |
| 39 | + { key: 'X-Frame-Options', value: 'SAMEORIGIN' }, | |
| 40 | + { key: 'Permissions-Policy', value: 'camera=(), microphone=(), geolocation=()' }, | |
| 41 | + ], | |
| 42 | + }, | |
| 43 | + { source: '/countries/:path*', headers: [PUBLIC_CACHE] }, | |
| 44 | + { source: '/api/:path*', headers: [NO_STORE] }, | |
| 45 | + { source: '/admin/:path*', headers: [NO_STORE] }, | |
| 46 | + ]; | |
| 47 | + }, | |
| 48 | +}; | |
| 49 | + | |
| 50 | +export default nextConfig; | |
added
apps/web/package.json
+40 −0
@@ -0,0 +1,40 @@ | ||
| 1 | +{ | |
| 2 | + "name": "@countryatlas/web", | |
| 3 | + "version": "0.1.0", | |
| 4 | + "private": true, | |
| 5 | + "scripts": { | |
| 6 | + "dev": "next dev -p 8290", | |
| 7 | + "build": "next build", | |
| 8 | + "start": "next start -p 8290 -H 0.0.0.0", | |
| 9 | + "typecheck": "tsc -p tsconfig.json --noEmit", | |
| 10 | + "lint": "next lint", | |
| 11 | + "qa": "node qa/screens.mjs", | |
| 12 | + "mock-api": "node qa/mock-api.mjs" | |
| 13 | + }, | |
| 14 | + "dependencies": { | |
| 15 | + "d3-geo": "^3.1.1", | |
| 16 | + "d3-scale": "^4.0.2", | |
| 17 | + "d3-shape": "^3.2.0", | |
| 18 | + "lucide-react": "^1.0.0", | |
| 19 | + "next": "16.3.4", | |
| 20 | + "react": "19.2.8", | |
| 21 | + "react-dom": "19.2.8", | |
| 22 | + "server-only": "^0.0.1", | |
| 23 | + "topojson-client": "^3.1.0", | |
| 24 | + "world-atlas": "^2.0.2" | |
| 25 | + }, | |
| 26 | + "devDependencies": { | |
| 27 | + "@tailwindcss/postcss": "^4", | |
| 28 | + "@types/d3-geo": "^3.1.1", | |
| 29 | + "@types/d3-scale": "^4.0.9", | |
| 30 | + "@types/d3-shape": "^3.1.7", | |
| 31 | + "@types/geojson": "^7946.0.16", | |
| 32 | + "@types/node": "^24.0.0", | |
| 33 | + "@types/react": "^19", | |
| 34 | + "@types/react-dom": "^19", | |
| 35 | + "@types/topojson-client": "^3.1.5", | |
| 36 | + "@types/topojson-specification": "^1.0.5", | |
| 37 | + "tailwindcss": "^4", | |
| 38 | + "typescript": "^5.9.3" | |
| 39 | + } | |
| 40 | +} | |
added
apps/web/postcss.config.mjs
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +const config = { | |
| 2 | + plugins: { | |
| 3 | + '@tailwindcss/postcss': {}, | |
| 4 | + }, | |
| 5 | +}; | |
| 6 | + | |
| 7 | +export default config; | |
added
apps/web/src/app/globals.css
+348 −0
@@ -0,0 +1,348 @@ | ||
| 1 | +@import 'tailwindcss'; | |
| 2 | + | |
| 3 | +/* | |
| 4 | + CountryAtlas design tokens. | |
| 5 | + Editorial, calm, data-dense. Raw values live on :root / .dark; Tailwind utilities read them through | |
| 6 | + `@theme inline`, so `bg-paper`, `text-ink-2`, `border-rule`, `text-up`, `fill-series-1` … follow the theme. | |
| 7 | + Series/sequential colours are the validated dataviz palette (skills/dataviz references/palette.md), | |
| 8 | + re-validated against these surfaces on 2026-09-11 (see apps/web/README.md → Design tokens). | |
| 9 | +*/ | |
| 10 | +@custom-variant dark (&:where(.dark, .dark *)); | |
| 11 | + | |
| 12 | +:root { | |
| 13 | + color-scheme: light; | |
| 14 | + --paper: #fbfaf7; /* page plane */ | |
| 15 | + --surface: #ffffff; /* chart surface, sheets, dialogs */ | |
| 16 | + --surface-2: #f3f1ec; /* hover wash, chips */ | |
| 17 | + --ink: #1a1917; | |
| 18 | + --ink-2: #5c5a55; | |
| 19 | + --ink-3: #8a877f; | |
| 20 | + --rule: #e6e3dc; | |
| 21 | + --rule-strong: #c9c6bd; | |
| 22 | + --accent: #1c5cab; | |
| 23 | + --accent-ink: #ffffff; | |
| 24 | + --accent-soft: #e8f0fb; | |
| 25 | + --up: #006300; | |
| 26 | + --down: #b02a2a; | |
| 27 | + --warn: #9a6a00; | |
| 28 | + --nodata: #d9d6ce; | |
| 29 | + --forecast: #8a877f; | |
| 30 | + | |
| 31 | + --series-1: #2a78d6; | |
| 32 | + --series-2: #eb6834; | |
| 33 | + --series-3: #1baf7a; | |
| 34 | + --series-4: #eda100; | |
| 35 | + --series-5: #e87ba4; | |
| 36 | + --series-6: #008300; | |
| 37 | + --series-7: #4a3aa7; | |
| 38 | + --series-8: #e34948; | |
| 39 | + | |
| 40 | + --seq-1: #cde2fb; | |
| 41 | + --seq-2: #9ec5f4; | |
| 42 | + --seq-3: #6da7ec; | |
| 43 | + --seq-4: #3987e5; | |
| 44 | + --seq-5: #256abf; | |
| 45 | + --seq-6: #184f95; | |
| 46 | + --seq-7: #0d366b; | |
| 47 | + | |
| 48 | + --shadow-sheet: 0 -8px 32px rgba(26, 25, 23, 0.12); | |
| 49 | + --shadow-pop: 0 4px 24px rgba(26, 25, 23, 0.14); | |
| 50 | +} | |
| 51 | + | |
| 52 | +.dark { | |
| 53 | + color-scheme: dark; | |
| 54 | + --paper: #151513; | |
| 55 | + --surface: #1c1c1a; | |
| 56 | + --surface-2: #24241f; | |
| 57 | + --ink: #f2f0ea; | |
| 58 | + --ink-2: #c9c6bd; | |
| 59 | + --ink-3: #8a877f; | |
| 60 | + --rule: #2c2b28; | |
| 61 | + --rule-strong: #3a3935; | |
| 62 | + --accent: #5598e7; | |
| 63 | + --accent-ink: #0d1b2e; | |
| 64 | + --accent-soft: #17304f; | |
| 65 | + --up: #0ca30c; | |
| 66 | + --down: #e66767; | |
| 67 | + --warn: #e0a100; | |
| 68 | + --nodata: #33322f; | |
| 69 | + --forecast: #8a877f; | |
| 70 | + | |
| 71 | + --series-1: #3987e5; | |
| 72 | + --series-2: #d95926; | |
| 73 | + --series-3: #199e70; | |
| 74 | + --series-4: #c98500; | |
| 75 | + --series-5: #d55181; | |
| 76 | + --series-6: #008300; | |
| 77 | + --series-7: #9085e9; | |
| 78 | + --series-8: #e66767; | |
| 79 | + | |
| 80 | + --seq-1: #0d366b; | |
| 81 | + --seq-2: #184f95; | |
| 82 | + --seq-3: #256abf; | |
| 83 | + --seq-4: #3987e5; | |
| 84 | + --seq-5: #6da7ec; | |
| 85 | + --seq-6: #9ec5f4; | |
| 86 | + --seq-7: #cde2fb; | |
| 87 | + | |
| 88 | + --shadow-sheet: 0 -8px 32px rgba(0, 0, 0, 0.5); | |
| 89 | + --shadow-pop: 0 4px 24px rgba(0, 0, 0, 0.55); | |
| 90 | +} | |
| 91 | + | |
| 92 | +@theme inline { | |
| 93 | + --color-paper: var(--paper); | |
| 94 | + --color-surface: var(--surface); | |
| 95 | + --color-surface-2: var(--surface-2); | |
| 96 | + --color-ink: var(--ink); | |
| 97 | + --color-ink-2: var(--ink-2); | |
| 98 | + --color-ink-3: var(--ink-3); | |
| 99 | + --color-rule: var(--rule); | |
| 100 | + --color-rule-strong: var(--rule-strong); | |
| 101 | + --color-accent: var(--accent); | |
| 102 | + --color-accent-ink: var(--accent-ink); | |
| 103 | + --color-accent-soft: var(--accent-soft); | |
| 104 | + --color-up: var(--up); | |
| 105 | + --color-down: var(--down); | |
| 106 | + --color-warn: var(--warn); | |
| 107 | + --color-nodata: var(--nodata); | |
| 108 | + --color-forecast: var(--forecast); | |
| 109 | + --color-series-1: var(--series-1); | |
| 110 | + --color-series-2: var(--series-2); | |
| 111 | + --color-series-3: var(--series-3); | |
| 112 | + --color-series-4: var(--series-4); | |
| 113 | + --color-series-5: var(--series-5); | |
| 114 | + --color-series-6: var(--series-6); | |
| 115 | + --color-series-7: var(--series-7); | |
| 116 | + --color-series-8: var(--series-8); | |
| 117 | + --color-seq-1: var(--seq-1); | |
| 118 | + --color-seq-2: var(--seq-2); | |
| 119 | + --color-seq-3: var(--seq-3); | |
| 120 | + --color-seq-4: var(--seq-4); | |
| 121 | + --color-seq-5: var(--seq-5); | |
| 122 | + --color-seq-6: var(--seq-6); | |
| 123 | + --color-seq-7: var(--seq-7); | |
| 124 | + | |
| 125 | + --font-ui: var(--font-ui), ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif; | |
| 126 | + --font-display: var(--font-display), 'Iowan Old Style', 'Palatino Linotype', Georgia, serif; | |
| 127 | + --font-mono: ui-monospace, 'SF Mono', Menlo, Consolas, monospace; | |
| 128 | + | |
| 129 | + /* Typographic scale (rem). Body 15px on mobile, 16px from md. */ | |
| 130 | + --text-2xs: 0.6875rem; | |
| 131 | + --text-2xs--line-height: 1rem; | |
| 132 | + --text-xs: 0.75rem; | |
| 133 | + --text-xs--line-height: 1.1rem; | |
| 134 | + --text-sm: 0.875rem; | |
| 135 | + --text-sm--line-height: 1.3rem; | |
| 136 | + --text-base: 1rem; | |
| 137 | + --text-base--line-height: 1.55rem; | |
| 138 | + --text-lg: 1.125rem; | |
| 139 | + --text-lg--line-height: 1.6rem; | |
| 140 | + --text-xl: 1.3125rem; | |
| 141 | + --text-xl--line-height: 1.75rem; | |
| 142 | + --text-2xl: 1.625rem; | |
| 143 | + --text-2xl--line-height: 2rem; | |
| 144 | + --text-3xl: 2rem; | |
| 145 | + --text-3xl--line-height: 2.3rem; | |
| 146 | + --text-4xl: 2.5rem; | |
| 147 | + --text-4xl--line-height: 2.75rem; | |
| 148 | + --text-5xl: 3.25rem; | |
| 149 | + --text-5xl--line-height: 3.4rem; | |
| 150 | + | |
| 151 | + --shadow-sheet: var(--shadow-sheet); | |
| 152 | + --shadow-pop: var(--shadow-pop); | |
| 153 | + | |
| 154 | + --radius-xs: 2px; | |
| 155 | + --radius-sm: 4px; | |
| 156 | + --radius-md: 6px; | |
| 157 | +} | |
| 158 | + | |
| 159 | +@layer base { | |
| 160 | + html { | |
| 161 | + font-family: var(--font-ui); | |
| 162 | + background: var(--paper); | |
| 163 | + color: var(--ink); | |
| 164 | + -webkit-text-size-adjust: 100%; | |
| 165 | + text-rendering: optimizeLegibility; | |
| 166 | + scroll-padding-top: 6.5rem; | |
| 167 | + } | |
| 168 | + html, | |
| 169 | + body { | |
| 170 | + /* Never scroll horizontally; individual scrollers opt in with overflow-x-auto. */ | |
| 171 | + overflow-x: clip; | |
| 172 | + max-width: 100%; | |
| 173 | + } | |
| 174 | + body { | |
| 175 | + font-size: 15px; | |
| 176 | + line-height: 1.55; | |
| 177 | + min-height: 100dvh; | |
| 178 | + background: var(--paper); | |
| 179 | + color: var(--ink); | |
| 180 | + } | |
| 181 | + @media (min-width: 48rem) { | |
| 182 | + body { | |
| 183 | + font-size: 16px; | |
| 184 | + } | |
| 185 | + } | |
| 186 | + h1, | |
| 187 | + h2, | |
| 188 | + h3 { | |
| 189 | + text-wrap: balance; | |
| 190 | + } | |
| 191 | + p { | |
| 192 | + text-wrap: pretty; | |
| 193 | + } | |
| 194 | + a { | |
| 195 | + text-underline-offset: 0.15em; | |
| 196 | + } | |
| 197 | + :focus-visible { | |
| 198 | + outline: 2px solid var(--accent); | |
| 199 | + outline-offset: 2px; | |
| 200 | + border-radius: 2px; | |
| 201 | + } | |
| 202 | + ::selection { | |
| 203 | + background: var(--accent-soft); | |
| 204 | + color: var(--ink); | |
| 205 | + } | |
| 206 | + svg { | |
| 207 | + display: block; | |
| 208 | + } | |
| 209 | + button { | |
| 210 | + cursor: pointer; | |
| 211 | + } | |
| 212 | + [hidden] { | |
| 213 | + display: none !important; | |
| 214 | + } | |
| 215 | + @media (prefers-reduced-motion: reduce) { | |
| 216 | + *, | |
| 217 | + *::before, | |
| 218 | + *::after { | |
| 219 | + animation-duration: 0.01ms !important; | |
| 220 | + animation-iteration-count: 1 !important; | |
| 221 | + transition-duration: 0.01ms !important; | |
| 222 | + scroll-behavior: auto !important; | |
| 223 | + } | |
| 224 | + } | |
| 225 | +} | |
| 226 | + | |
| 227 | +@utility tnum { | |
| 228 | + font-variant-numeric: tabular-nums lining-nums; | |
| 229 | + font-feature-settings: 'tnum' 1, 'lnum' 1; | |
| 230 | +} | |
| 231 | +@utility pnum { | |
| 232 | + font-variant-numeric: proportional-nums lining-nums; | |
| 233 | +} | |
| 234 | +@utility display { | |
| 235 | + font-family: var(--font-display); | |
| 236 | + font-weight: 500; | |
| 237 | + letter-spacing: -0.01em; | |
| 238 | +} | |
| 239 | +@utility eyebrow { | |
| 240 | + font-size: var(--text-2xs); | |
| 241 | + line-height: 1rem; | |
| 242 | + letter-spacing: 0.08em; | |
| 243 | + text-transform: uppercase; | |
| 244 | + font-weight: 600; | |
| 245 | + color: var(--ink-3); | |
| 246 | +} | |
| 247 | +@utility hairline { | |
| 248 | + border-top: 1px solid var(--rule); | |
| 249 | +} | |
| 250 | +@utility tap { | |
| 251 | + min-height: 44px; | |
| 252 | + min-width: 44px; | |
| 253 | +} | |
| 254 | +@utility scrollbar-none { | |
| 255 | + scrollbar-width: none; | |
| 256 | + &::-webkit-scrollbar { | |
| 257 | + display: none; | |
| 258 | + } | |
| 259 | +} | |
| 260 | +@utility safe-bottom { | |
| 261 | + padding-bottom: env(safe-area-inset-bottom, 0px); | |
| 262 | +} | |
| 263 | +@utility no-data-hatch { | |
| 264 | + background-color: var(--nodata); | |
| 265 | + background-image: repeating-linear-gradient(135deg, transparent 0 3px, rgba(26, 25, 23, 0.14) 3px 4px); | |
| 266 | +} | |
| 267 | +@utility link-quiet { | |
| 268 | + color: inherit; | |
| 269 | + text-decoration: none; | |
| 270 | + &:hover { | |
| 271 | + color: var(--accent); | |
| 272 | + } | |
| 273 | +} | |
| 274 | +@utility container-x { | |
| 275 | + padding-left: 1rem; | |
| 276 | + padding-right: 1rem; | |
| 277 | + @media (min-width: 40rem) { | |
| 278 | + padding-left: 1.5rem; | |
| 279 | + padding-right: 1.5rem; | |
| 280 | + } | |
| 281 | +} | |
| 282 | + | |
| 283 | +/* Chart chrome shared by the SVG kit (class names instead of inline styles so theme swaps repaint). */ | |
| 284 | +.ca-chart text { | |
| 285 | + font-family: var(--font-ui); | |
| 286 | + font-variant-numeric: tabular-nums; | |
| 287 | + fill: var(--ink-3); | |
| 288 | + font-size: 11px; | |
| 289 | +} | |
| 290 | +.ca-chart .axis line, | |
| 291 | +.ca-chart .grid line { | |
| 292 | + stroke: var(--rule); | |
| 293 | + stroke-width: 1; | |
| 294 | + shape-rendering: crispEdges; | |
| 295 | +} | |
| 296 | +.ca-chart .baseline { | |
| 297 | + stroke: var(--rule-strong); | |
| 298 | +} | |
| 299 | +.ca-chart .series-line { | |
| 300 | + fill: none; | |
| 301 | + stroke-width: 2; | |
| 302 | + stroke-linejoin: round; | |
| 303 | + stroke-linecap: round; | |
| 304 | +} | |
| 305 | +.ca-chart .forecast { | |
| 306 | + stroke-dasharray: 4 4; | |
| 307 | +} | |
| 308 | +.ca-chart .ring { | |
| 309 | + stroke: var(--surface); | |
| 310 | + stroke-width: 2; | |
| 311 | +} | |
| 312 | +.ca-chart .label { | |
| 313 | + fill: var(--ink-2); | |
| 314 | + font-size: 11px; | |
| 315 | +} | |
| 316 | +.ca-chart .label-strong { | |
| 317 | + fill: var(--ink); | |
| 318 | + font-weight: 600; | |
| 319 | +} | |
| 320 | +.ca-hatch { | |
| 321 | + fill: url(#ca-nodata-hatch); | |
| 322 | +} | |
| 323 | + | |
| 324 | +/* Motion tokens */ | |
| 325 | +@keyframes ca-sheet-up { | |
| 326 | + from { | |
| 327 | + transform: translateY(24px); | |
| 328 | + opacity: 0; | |
| 329 | + } | |
| 330 | + to { | |
| 331 | + transform: none; | |
| 332 | + opacity: 1; | |
| 333 | + } | |
| 334 | +} | |
| 335 | +@keyframes ca-fade { | |
| 336 | + from { | |
| 337 | + opacity: 0; | |
| 338 | + } | |
| 339 | + to { | |
| 340 | + opacity: 1; | |
| 341 | + } | |
| 342 | +} | |
| 343 | +.animate-sheet { | |
| 344 | + animation: ca-sheet-up 220ms cubic-bezier(0.2, 0.8, 0.2, 1); | |
| 345 | +} | |
| 346 | +.animate-fade { | |
| 347 | + animation: ca-fade 160ms ease-out; | |
| 348 | +} | |
added
apps/web/src/app/icon.svg
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 32 32" fill="none" stroke="#1c5cab" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><rect width="32" height="32" rx="7" fill="#fbfaf7"/><circle cx="16" cy="16" r="12"/><path d="M5 19.3h22"/><path d="M10.1 26.6 16 7.6l5.9 19"/><path d="M16 7.6c-3 2.9-4.3 7.1-4.3 11.7" opacity="0.55"/><path d="M16 7.6c3 2.9 4.3 7.1 4.3 11.7" opacity="0.55"/></svg> | |
added
apps/web/src/app/layout.tsx
+62 −0
@@ -0,0 +1,62 @@ | ||
| 1 | +import type { Metadata, Viewport } from 'next'; | |
| 2 | +import './globals.css'; | |
| 3 | +import { t } from '@/i18n'; | |
| 4 | +import { fontDisplay, fontUi } from '@/lib/fonts'; | |
| 5 | +import { SITE_NAME, SITE_URL, TAGLINE } from '@/lib/site'; | |
| 6 | +import { ProvenanceProvider } from '@/components/data/provenance-context'; | |
| 7 | +import { ProvenanceSheet } from '@/components/data/provenance-sheet'; | |
| 8 | +import { MobileTabBar } from '@/components/layout/mobile-tab-bar'; | |
| 9 | +import { SearchContainer } from '@/components/layout/search-container'; | |
| 10 | +import { SearchProvider } from '@/components/layout/search-context'; | |
| 11 | +import { SiteFooter } from '@/components/layout/site-footer'; | |
| 12 | +import { SiteHeader } from '@/components/layout/site-header'; | |
| 13 | +import { THEME_BOOT } from '@/components/layout/theme-toggle'; | |
| 14 | + | |
| 15 | +export const metadata: Metadata = { | |
| 16 | + metadataBase: new URL(SITE_URL), | |
| 17 | + title: { default: `${SITE_NAME} — ${TAGLINE}`, template: `%s — ${SITE_NAME}` }, | |
| 18 | + description: t('site.description'), | |
| 19 | + applicationName: SITE_NAME, | |
| 20 | + robots: { index: true, follow: true }, | |
| 21 | + alternates: { canonical: '/' }, | |
| 22 | + openGraph: { type: 'website', siteName: SITE_NAME, url: SITE_URL, title: `${SITE_NAME} — ${TAGLINE}`, description: t('site.description') }, | |
| 23 | + twitter: { card: 'summary_large_image', title: `${SITE_NAME} — ${TAGLINE}`, description: t('site.description') }, | |
| 24 | + icons: { icon: [{ url: '/icon.svg', type: 'image/svg+xml' }, { url: '/icon.png', sizes: '512x512', type: 'image/png' }], apple: [{ url: '/apple-icon.png', sizes: '180x180' }] }, | |
| 25 | +}; | |
| 26 | + | |
| 27 | +export const viewport: Viewport = { | |
| 28 | + width: 'device-width', | |
| 29 | + initialScale: 1, | |
| 30 | + viewportFit: 'cover', | |
| 31 | + themeColor: [ | |
| 32 | + { media: '(prefers-color-scheme: light)', color: '#fbfaf7' }, | |
| 33 | + { media: '(prefers-color-scheme: dark)', color: '#151513' }, | |
| 34 | + ], | |
| 35 | +}; | |
| 36 | + | |
| 37 | +export default function RootLayout({ children }: { children: React.ReactNode }) { | |
| 38 | + return ( | |
| 39 | + <html lang="en" className={`${fontUi.variable} ${fontDisplay.variable} h-full antialiased`} suppressHydrationWarning> | |
| 40 | + <head> | |
| 41 | + <script dangerouslySetInnerHTML={{ __html: THEME_BOOT }} /> | |
| 42 | + </head> | |
| 43 | + <body className="flex min-h-full flex-col pb-[calc(56px+env(safe-area-inset-bottom,0px))] md:pb-0"> | |
| 44 | + <a href="#main" className="sr-only focus:not-sr-only focus:fixed focus:left-3 focus:top-3 focus:z-[200] focus:rounded-sm focus:bg-accent focus:px-3 focus:py-2 focus:text-sm focus:text-accent-ink"> | |
| 45 | + {t('site.skip')} | |
| 46 | + </a> | |
| 47 | + <ProvenanceProvider> | |
| 48 | + <SearchProvider> | |
| 49 | + <SiteHeader /> | |
| 50 | + <main id="main" className="container-x mx-auto w-full max-w-[1400px] flex-1"> | |
| 51 | + {children} | |
| 52 | + </main> | |
| 53 | + <SiteFooter /> | |
| 54 | + <MobileTabBar /> | |
| 55 | + <SearchContainer /> | |
| 56 | + <ProvenanceSheet /> | |
| 57 | + </SearchProvider> | |
| 58 | + </ProvenanceProvider> | |
| 59 | + </body> | |
| 60 | + </html> | |
| 61 | + ); | |
| 62 | +} | |
added
apps/web/src/app/manifest.ts
+19 −0
@@ -0,0 +1,19 @@ | ||
| 1 | +import type { MetadataRoute } from 'next'; | |
| 2 | +import { SITE_NAME, TAGLINE } from '@/lib/site'; | |
| 3 | + | |
| 4 | +export default function manifest(): MetadataRoute.Manifest { | |
| 5 | + return { | |
| 6 | + name: SITE_NAME, | |
| 7 | + short_name: SITE_NAME, | |
| 8 | + description: TAGLINE, | |
| 9 | + start_url: '/', | |
| 10 | + display: 'standalone', | |
| 11 | + background_color: '#fbfaf7', | |
| 12 | + theme_color: '#1c5cab', | |
| 13 | + icons: [ | |
| 14 | + { src: '/icon.svg', type: 'image/svg+xml', sizes: 'any' }, | |
| 15 | + { src: '/icon.png', type: 'image/png', sizes: '512x512' }, | |
| 16 | + { src: '/apple-icon.png', type: 'image/png', sizes: '180x180' }, | |
| 17 | + ], | |
| 18 | + }; | |
| 19 | +} | |
added
apps/web/src/components/brand/Logo.tsx
+53 −0
@@ -0,0 +1,53 @@ | ||
| 1 | +import { cn } from '@/lib/cn'; | |
| 2 | +import { t } from '@/i18n'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * CountryAtlas brand. | |
| 6 | + * Mark: a globe ring whose equator doubles as the crossbar of an "A" — two meridian-like legs rise to a | |
| 7 | + * single apex, so the glyph reads as globe + atlas + the letter A at once. Pure geometry, strokes only, | |
| 8 | + * `currentColor` → works on light/dark and in monochrome. Wordmark: "Country" regular + "Atlas" semibold. | |
| 9 | + * | |
| 10 | + * Usage: <Logo variant="full" /> (header), <Logo variant="mark" size={24} /> (favicons, OG), <Logo variant="wordmark" />. | |
| 11 | + */ | |
| 12 | +export function LogoMark({ size = 28, className, title, strokeWidth = 2 }: { size?: number; className?: string; title?: string; strokeWidth?: number }) { | |
| 13 | + return ( | |
| 14 | + <svg width={size} height={size} viewBox="0 0 32 32" fill="none" stroke="currentColor" strokeWidth={strokeWidth} strokeLinecap="round" strokeLinejoin="round" className={cn('shrink-0', className)} role={title ? 'img' : undefined} aria-hidden={title ? undefined : true} aria-label={title}> | |
| 15 | + {title ? <title>{title}</title> : null} | |
| 16 | + {/* globe ring */} | |
| 17 | + <circle cx="16" cy="16" r="13" /> | |
| 18 | + {/* equator = crossbar of the A, clipped to the ring */} | |
| 19 | + <path d="M4.1 19.5h23.8" /> | |
| 20 | + {/* the A: two legs to a single apex */} | |
| 21 | + <path d="M9.6 27.2 16 6.8l6.4 20.4" /> | |
| 22 | + {/* inner meridian hint */} | |
| 23 | + <path d="M16 6.8c-3.2 3.1-4.6 7.7-4.6 12.7" opacity="0.55" /> | |
| 24 | + <path d="M16 6.8c3.2 3.1 4.6 7.7 4.6 12.7" opacity="0.55" /> | |
| 25 | + </svg> | |
| 26 | + ); | |
| 27 | +} | |
| 28 | + | |
| 29 | +export function Wordmark({ className }: { className?: string }) { | |
| 30 | + return ( | |
| 31 | + <span className={cn('select-none whitespace-nowrap font-ui text-[1.0625rem] tracking-[-0.02em] text-ink', className)}> | |
| 32 | + <span className="font-normal">Country</span> | |
| 33 | + <span className="font-semibold">Atlas</span> | |
| 34 | + </span> | |
| 35 | + ); | |
| 36 | +} | |
| 37 | + | |
| 38 | +export function Logo({ variant = 'full', size = 26, className }: { variant?: 'full' | 'mark' | 'wordmark'; size?: number; className?: string }) { | |
| 39 | + if (variant === 'mark') return <LogoMark size={size} className={className} title={t('brand.logoAlt')} />; | |
| 40 | + if (variant === 'wordmark') return <Wordmark className={className} />; | |
| 41 | + return ( | |
| 42 | + <span className={cn('inline-flex items-center gap-2', className)}> | |
| 43 | + <LogoMark size={size} className="text-accent" /> | |
| 44 | + <Wordmark /> | |
| 45 | + </span> | |
| 46 | + ); | |
| 47 | +} | |
| 48 | + | |
| 49 | +/** Raw SVG string of the mark (for favicons / OG rasterisation); `color` is a CSS colour. */ | |
| 50 | +export function logoMarkSvg({ size = 512, color = '#1c5cab', background, radius = 0 }: { size?: number; color?: string; background?: string; radius?: number } = {}): string { | |
| 51 | + const bg = background ? `<rect width="32" height="32" rx="${radius}" fill="${background}"/>` : ''; | |
| 52 | + return `<svg xmlns="http://www.w3.org/2000/svg" width="${size}" height="${size}" viewBox="0 0 32 32" fill="none" stroke="${color}" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">${bg}<circle cx="16" cy="16" r="13"/><path d="M4.1 19.5h23.8"/><path d="M9.6 27.2 16 6.8l6.4 20.4"/><path d="M16 6.8c-3.2 3.1-4.6 7.7-4.6 12.7" opacity="0.55"/><path d="M16 6.8c3.2 3.1 4.6 7.7 4.6 12.7" opacity="0.55"/></svg>`; | |
| 53 | +} | |
added
apps/web/src/components/charts/chart-frame.tsx
+136 −0
@@ -0,0 +1,136 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { Table2 } from 'lucide-react'; | |
| 3 | +import { useId, useState, type ReactNode } from 'react'; | |
| 4 | +import { t } from '@/i18n'; | |
| 5 | +import { cn } from '@/lib/cn'; | |
| 6 | +import type { Provenance } from '@/lib/types'; | |
| 7 | +import type { ProvenancePayload } from '@/components/data/provenance-context'; | |
| 8 | +import { SourceLine } from './source-line'; | |
| 9 | + | |
| 10 | +export interface TableColumn { | |
| 11 | + key: string; | |
| 12 | + label: string; | |
| 13 | + numeric?: boolean; | |
| 14 | +} | |
| 15 | +export interface TableData { | |
| 16 | + columns: TableColumn[]; | |
| 17 | + rows: Array<Record<string, string>>; | |
| 18 | +} | |
| 19 | + | |
| 20 | +/** | |
| 21 | + * Wraps any chart: optional heading, the chart, a source attribution line (clickable → provenance), | |
| 22 | + * and the accessible data-table toggle. `summary` is the auto-generated text summary announced to AT. | |
| 23 | + */ | |
| 24 | +export function ChartFrame({ | |
| 25 | + title, | |
| 26 | + subtitle, | |
| 27 | + summary, | |
| 28 | + provenance, | |
| 29 | + payload, | |
| 30 | + table, | |
| 31 | + actions, | |
| 32 | + legend, | |
| 33 | + note, | |
| 34 | + children, | |
| 35 | + className, | |
| 36 | + minHeight, | |
| 37 | +}: { | |
| 38 | + title?: ReactNode; | |
| 39 | + subtitle?: ReactNode; | |
| 40 | + summary: string; | |
| 41 | + provenance?: Provenance | null; | |
| 42 | + payload?: ProvenancePayload | null; | |
| 43 | + table?: TableData; | |
| 44 | + actions?: ReactNode; | |
| 45 | + legend?: ReactNode; | |
| 46 | + note?: ReactNode; | |
| 47 | + children: ReactNode; | |
| 48 | + className?: string; | |
| 49 | + /** Reserve height to avoid layout shift while the chart mounts. */ | |
| 50 | + minHeight?: number; | |
| 51 | +}) { | |
| 52 | + const [showTable, setShowTable] = useState(false); | |
| 53 | + const id = useId(); | |
| 54 | + return ( | |
| 55 | + <figure className={cn('min-w-0', className)} aria-describedby={`${id}-sum`}> | |
| 56 | + {title || actions ? ( | |
| 57 | + <figcaption className="mb-2 flex items-start justify-between gap-3"> | |
| 58 | + <div className="min-w-0"> | |
| 59 | + {title ? <div className="text-sm font-semibold text-ink">{title}</div> : null} | |
| 60 | + {subtitle ? <div className="text-xs text-ink-2">{subtitle}</div> : null} | |
| 61 | + </div> | |
| 62 | + {actions ? <div className="flex shrink-0 items-center gap-1">{actions}</div> : null} | |
| 63 | + </figcaption> | |
| 64 | + ) : null} | |
| 65 | + {legend} | |
| 66 | + <div style={minHeight ? { minHeight } : undefined}>{children}</div> | |
| 67 | + <p id={`${id}-sum`} className="sr-only"> | |
| 68 | + {summary} | |
| 69 | + </p> | |
| 70 | + <div className="mt-1.5 flex flex-wrap items-center justify-between gap-x-3 gap-y-1"> | |
| 71 | + <div className="min-w-0 flex-1"> | |
| 72 | + <SourceLine provenance={provenance} payload={payload} /> | |
| 73 | + {note ? <p className="text-2xs text-ink-3">{note}</p> : null} | |
| 74 | + </div> | |
| 75 | + {table ? ( | |
| 76 | + <button | |
| 77 | + type="button" | |
| 78 | + onClick={() => setShowTable((s) => !s)} | |
| 79 | + aria-expanded={showTable} | |
| 80 | + aria-controls={`${id}-table`} | |
| 81 | + className="inline-flex min-h-[32px] items-center gap-1 rounded-sm px-1.5 text-2xs text-ink-3 hover:bg-surface-2 hover:text-ink" | |
| 82 | + > | |
| 83 | + <Table2 size={12} aria-hidden /> | |
| 84 | + {showTable ? t('common.hideTable') : t('common.viewTable')} | |
| 85 | + </button> | |
| 86 | + ) : null} | |
| 87 | + </div> | |
| 88 | + {table && showTable ? ( | |
| 89 | + <div id={`${id}-table`} className="mt-2 max-h-72 overflow-auto border-y border-rule"> | |
| 90 | + <table className="w-full text-xs tnum"> | |
| 91 | + <caption className="sr-only">{t('chart.table.caption')}</caption> | |
| 92 | + <thead className="sticky top-0 bg-paper"> | |
| 93 | + <tr> | |
| 94 | + {table.columns.map((c) => ( | |
| 95 | + <th key={c.key} scope="col" className={cn('px-2 py-1.5 text-left font-medium text-ink-2', c.numeric && 'text-right')}> | |
| 96 | + {c.label} | |
| 97 | + </th> | |
| 98 | + ))} | |
| 99 | + </tr> | |
| 100 | + </thead> | |
| 101 | + <tbody className="divide-y divide-rule"> | |
| 102 | + {table.rows.map((r, i) => ( | |
| 103 | + <tr key={i}> | |
| 104 | + {table.columns.map((c) => ( | |
| 105 | + <td key={c.key} className={cn('px-2 py-1', c.numeric && 'text-right')}> | |
| 106 | + {r[c.key] ?? t('common.na')} | |
| 107 | + </td> | |
| 108 | + ))} | |
| 109 | + </tr> | |
| 110 | + ))} | |
| 111 | + </tbody> | |
| 112 | + </table> | |
| 113 | + </div> | |
| 114 | + ) : null} | |
| 115 | + </figure> | |
| 116 | + ); | |
| 117 | +} | |
| 118 | + | |
| 119 | +/** Legend for ≥ 2 series (a single series needs none — the title names it). */ | |
| 120 | +export function Legend({ items }: { items: Array<{ label: string; color: string; dashed?: boolean; shape?: 'line' | 'rect' }> }) { | |
| 121 | + if (items.length < 2) return null; | |
| 122 | + return ( | |
| 123 | + <ul className="mb-2 flex flex-wrap gap-x-4 gap-y-1 text-xs text-ink-2" aria-label={t('common.legend')}> | |
| 124 | + {items.map((it) => ( | |
| 125 | + <li key={it.label} className="inline-flex items-center gap-1.5"> | |
| 126 | + {it.shape === 'rect' ? ( | |
| 127 | + <span aria-hidden className="inline-block h-2.5 w-2.5 rounded-xs" style={{ background: it.color }} /> | |
| 128 | + ) : ( | |
| 129 | + <span aria-hidden className="inline-block h-0.5 w-4 rounded-full" style={{ background: it.color, ...(it.dashed ? { backgroundImage: `repeating-linear-gradient(90deg, ${it.color} 0 3px, transparent 3px 6px)`, backgroundColor: 'transparent' } : {}) }} /> | |
| 130 | + )} | |
| 131 | + <span>{it.label}</span> | |
| 132 | + </li> | |
| 133 | + ))} | |
| 134 | + </ul> | |
| 135 | + ); | |
| 136 | +} | |
added
apps/web/src/components/charts/choropleth-view.tsx
+137 −0
@@ -0,0 +1,137 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { useRouter } from 'next/navigation'; | |
| 3 | +import { useCallback, useId, useState } from 'react'; | |
| 4 | +import { t } from '@/i18n'; | |
| 5 | +import { cn } from '@/lib/cn'; | |
| 6 | +import { formatValue } from '@/lib/format'; | |
| 7 | +import { MAP_HEIGHT, MAP_WIDTH } from '@/lib/map-geo'; | |
| 8 | +import { routes } from '@/lib/site'; | |
| 9 | +import type { FormatSpec as Spec } from '@/lib/types'; | |
| 10 | +import { seqVar } from './palette'; | |
| 11 | + | |
| 12 | +export interface ChoroplethFeature { | |
| 13 | + iso3: string | null; | |
| 14 | + name: string; | |
| 15 | + slug: string | null; | |
| 16 | + flag: string | null; | |
| 17 | + d: string; | |
| 18 | + value: number | null; | |
| 19 | + cls: number | null; | |
| 20 | +} | |
| 21 | + | |
| 22 | +/** Map a class index (0..k-1) onto the 7-step sequential ramp. */ | |
| 23 | +function stepFor(cls: number, k: number): number { | |
| 24 | + if (k <= 1) return 4; | |
| 25 | + const start = k >= 6 ? 1 : 2; | |
| 26 | + const end = 7; | |
| 27 | + return Math.round(start + (cls / (k - 1)) * (end - start)); | |
| 28 | +} | |
| 29 | + | |
| 30 | +/** | |
| 31 | + * Interactive SVG world map: hover (mouse) shows the floating label; on touch the first tap selects and | |
| 32 | + * shows the label with an "Open" link, a click with a mouse navigates. Explicit hatched fill for no data. | |
| 33 | + */ | |
| 34 | +export function ChoroplethView({ features, sphere, legend, k, spec, summary, title, height, compact, className }: { features: ChoroplethFeature[]; sphere: string; legend: Array<{ cls: number; label: string }>; k: number; spec: Spec; summary: string; title: string; height?: number; compact?: boolean; className?: string }) { | |
| 35 | + const router = useRouter(); | |
| 36 | + const id = useId(); | |
| 37 | + const [active, setActive] = useState<{ f: ChoroplethFeature; x: number; y: number; sticky: boolean } | null>(null); | |
| 38 | + | |
| 39 | + const place = useCallback((e: React.PointerEvent<SVGPathElement>, f: ChoroplethFeature, sticky: boolean) => { | |
| 40 | + const box = e.currentTarget.ownerSVGElement?.parentElement?.getBoundingClientRect(); | |
| 41 | + if (!box) return; | |
| 42 | + setActive({ f, x: e.clientX - box.left, y: e.clientY - box.top, sticky }); | |
| 43 | + }, []); | |
| 44 | + | |
| 45 | + const open = (f: ChoroplethFeature) => { | |
| 46 | + if (f.slug) router.push(routes.country(f.slug)); | |
| 47 | + }; | |
| 48 | + | |
| 49 | + return ( | |
| 50 | + <figure className={cn('min-w-0', className)}> | |
| 51 | + <div className="relative w-full" style={{ aspectRatio: `${MAP_WIDTH} / ${MAP_HEIGHT}`, maxHeight: height }} onPointerLeave={() => setActive((a) => (a?.sticky ? a : null))}> | |
| 52 | + <svg viewBox={`0 0 ${MAP_WIDTH} ${MAP_HEIGHT}`} className="h-full w-full" role="img" aria-label={summary}> | |
| 53 | + <title>{title}</title> | |
| 54 | + <desc>{summary}</desc> | |
| 55 | + <defs> | |
| 56 | + <pattern id={`${id}-hatch`} width="6" height="6" patternUnits="userSpaceOnUse" patternTransform="rotate(45)"> | |
| 57 | + <rect width="6" height="6" fill="var(--nodata)" /> | |
| 58 | + <line x1="0" y1="0" x2="0" y2="6" stroke="var(--rule-strong)" strokeWidth="1.5" /> | |
| 59 | + </pattern> | |
| 60 | + </defs> | |
| 61 | + <path d={sphere} fill="var(--surface)" stroke="var(--rule)" strokeWidth={1} /> | |
| 62 | + <g stroke="var(--surface)" strokeWidth={0.6} strokeLinejoin="round"> | |
| 63 | + {features.map((f, i) => { | |
| 64 | + const hasData = f.cls != null; | |
| 65 | + const isActive = active?.f === f; | |
| 66 | + return ( | |
| 67 | + <path | |
| 68 | + key={f.iso3 ?? `${f.name}-${i}`} | |
| 69 | + d={f.d} | |
| 70 | + fill={hasData ? seqVar(stepFor(f.cls!, k)) : `url(#${id}-hatch)`} | |
| 71 | + className={cn(f.slug && 'cursor-pointer', 'transition-[fill-opacity] duration-100')} | |
| 72 | + fillOpacity={isActive ? 0.75 : 1} | |
| 73 | + tabIndex={f.slug ? 0 : -1} | |
| 74 | + role={f.slug ? 'link' : undefined} | |
| 75 | + aria-label={`${f.name}: ${formatValue(f.value, spec)}`} | |
| 76 | + onPointerMove={(e) => { | |
| 77 | + if (e.pointerType === 'mouse') place(e, f, false); | |
| 78 | + }} | |
| 79 | + onPointerDown={(e) => { | |
| 80 | + if (e.pointerType !== 'mouse') { | |
| 81 | + e.preventDefault(); | |
| 82 | + place(e, f, true); | |
| 83 | + } | |
| 84 | + }} | |
| 85 | + onClick={(e) => { | |
| 86 | + // Mouse: navigate. Touch: the label carries the link (first tap selects). | |
| 87 | + if ((e.nativeEvent as PointerEvent).pointerType === 'mouse' || (e as unknown as { detail: number }).detail === 0) open(f); | |
| 88 | + }} | |
| 89 | + onKeyDown={(e) => { | |
| 90 | + if (e.key === 'Enter' && f.slug) open(f); | |
| 91 | + }} | |
| 92 | + onFocus={(e) => { | |
| 93 | + const b = e.currentTarget.getBBox(); | |
| 94 | + setActive({ f, x: (b.x + b.width / 2) / MAP_WIDTH * (e.currentTarget.ownerSVGElement?.clientWidth ?? MAP_WIDTH), y: (b.y / MAP_HEIGHT) * (e.currentTarget.ownerSVGElement?.clientHeight ?? MAP_HEIGHT), sticky: false }); | |
| 95 | + }} | |
| 96 | + > | |
| 97 | + <title>{`${f.name}: ${formatValue(f.value, spec)}`}</title> | |
| 98 | + </path> | |
| 99 | + ); | |
| 100 | + })} | |
| 101 | + </g> | |
| 102 | + </svg> | |
| 103 | + {active ? ( | |
| 104 | + <div className="pointer-events-none absolute z-10 rounded-sm border border-rule bg-surface px-2.5 py-1.5 text-xs shadow-pop" style={{ left: Math.min(active.x + 10, Math.max(0, (typeof window !== 'undefined' ? 0 : 0) + active.x + 10)), top: active.y - 44, maxWidth: 220 }}> | |
| 105 | + <div className="flex items-center gap-1.5 font-medium text-ink"> | |
| 106 | + {active.f.flag ? <span aria-hidden>{active.f.flag}</span> : null} | |
| 107 | + <span className="truncate">{active.f.name}</span> | |
| 108 | + </div> | |
| 109 | + <div className="tnum text-ink-2">{formatValue(active.f.value, spec)}</div> | |
| 110 | + {active.sticky && active.f.slug ? ( | |
| 111 | + <button type="button" className="pointer-events-auto mt-1 text-accent underline" onClick={() => open(active.f)}> | |
| 112 | + {t('metric.open', { name: active.f.name })} → | |
| 113 | + </button> | |
| 114 | + ) : null} | |
| 115 | + </div> | |
| 116 | + ) : null} | |
| 117 | + </div> | |
| 118 | + <figcaption className="mt-2 flex flex-wrap items-center gap-x-3 gap-y-1.5 text-2xs text-ink-2"> | |
| 119 | + {!compact ? <span className="mr-1 font-medium text-ink">{title}</span> : null} | |
| 120 | + <ul className="flex flex-wrap items-center gap-x-2.5 gap-y-1"> | |
| 121 | + {legend.map((l) => ( | |
| 122 | + <li key={l.cls} className="inline-flex items-center gap-1 tnum"> | |
| 123 | + <span aria-hidden className="inline-block h-2.5 w-3.5 rounded-xs" style={{ background: seqVar(stepFor(l.cls, k)) }} /> | |
| 124 | + {l.label} | |
| 125 | + </li> | |
| 126 | + ))} | |
| 127 | + <li className="inline-flex items-center gap-1"> | |
| 128 | + <span aria-hidden className="no-data-hatch inline-block h-2.5 w-3.5 rounded-xs" /> | |
| 129 | + {t('chart.legend.noData')} | |
| 130 | + </li> | |
| 131 | + </ul> | |
| 132 | + <span className="ml-auto hidden text-ink-3 md:inline">{t('chart.map.hoverHint')}</span> | |
| 133 | + <span className="ml-auto text-ink-3 md:hidden">{t('chart.map.tapHint')}</span> | |
| 134 | + </figcaption> | |
| 135 | + </figure> | |
| 136 | + ); | |
| 137 | +} | |
added
apps/web/src/components/charts/choropleth.tsx
+70 −0
@@ -0,0 +1,70 @@ | ||
| 1 | +import { t } from '@/i18n'; | |
| 2 | +import { formatValue } from '@/lib/format'; | |
| 3 | +import { classIndex, isoLookupFromCountries, worldPaths } from '@/lib/map-geo'; | |
| 4 | +import type { CountrySummary, MapResponse } from '@/lib/types'; | |
| 5 | +import { ChoroplethView, type ChoroplethFeature } from './choropleth-view'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Server component: joins the world geometry with `/indicators/{slug}/map` values and the country list | |
| 9 | + * (for slugs and the ISO-numeric lookup), computes 5–7 quantile classes from the API legend, and hands | |
| 10 | + * plain data to the interactive client view. Sequential single-hue ramp (light → dark = low → high). | |
| 11 | + */ | |
| 12 | +export function Choropleth({ | |
| 13 | + map, | |
| 14 | + countries, | |
| 15 | + height, | |
| 16 | + compact = false, | |
| 17 | + className, | |
| 18 | + title, | |
| 19 | +}: { | |
| 20 | + map: MapResponse; | |
| 21 | + countries: CountrySummary[]; | |
| 22 | + height?: number; | |
| 23 | + compact?: boolean; | |
| 24 | + className?: string; | |
| 25 | + title?: string; | |
| 26 | +}) { | |
| 27 | + const { paths, sphere } = worldPaths(); | |
| 28 | + const byId = new Map(countries.map((c) => [c.id, c])); | |
| 29 | + // Registry numeric → ISO3 lookup (when the API exposes iso_numeric on the summary); static table otherwise. | |
| 30 | + const numeric = isoLookupFromCountries(countries as Array<{ id: string; iso_numeric?: string | null }>); | |
| 31 | + const breaks = (map.legend?.breaks ?? []).slice(0, 6); | |
| 32 | + const k = breaks.length + 1; // classes | |
| 33 | + const features: ChoroplethFeature[] = paths.map((p) => { | |
| 34 | + // Prefer registry numeric lookup when the static table has no code. | |
| 35 | + const iso3 = p.iso3 ?? (p.numeric ? numeric.get(p.numeric) ?? null : null); | |
| 36 | + const c = iso3 ? byId.get(iso3) : undefined; | |
| 37 | + const v = iso3 ? map.values[iso3] : undefined; | |
| 38 | + return { | |
| 39 | + iso3, | |
| 40 | + name: c?.name ?? p.name, | |
| 41 | + slug: c?.slug ?? null, | |
| 42 | + flag: c?.flag ?? null, | |
| 43 | + d: p.d, | |
| 44 | + value: typeof v === 'number' ? v : null, | |
| 45 | + cls: typeof v === 'number' ? classIndex(v, breaks) : null, | |
| 46 | + }; | |
| 47 | + }); | |
| 48 | + const spec = map.indicator; | |
| 49 | + const legendItems = Array.from({ length: k }, (_, i) => { | |
| 50 | + const lo = i === 0 ? map.legend.min : breaks[i - 1]!; | |
| 51 | + const hi = i === k - 1 ? map.legend.max : breaks[i]!; | |
| 52 | + return { cls: i, label: `${formatValue(lo, spec)} – ${formatValue(hi, spec)}` }; | |
| 53 | + }); | |
| 54 | + const year = map.year_used ?? map.year ?? ''; | |
| 55 | + const summary = t('chart.summary.map', { name: spec.name, year, n: map.n, min: formatValue(map.legend.min, spec), max: formatValue(map.legend.max, spec) }); | |
| 56 | + return ( | |
| 57 | + <ChoroplethView | |
| 58 | + features={features} | |
| 59 | + sphere={sphere} | |
| 60 | + legend={legendItems} | |
| 61 | + k={k} | |
| 62 | + spec={{ format: spec.format, unit: spec.unit, unit_short: spec.unit_short, precision: spec.precision, name: spec.name }} | |
| 63 | + summary={summary} | |
| 64 | + title={title ?? t('chart.map.legend', { name: spec.name, year })} | |
| 65 | + height={height} | |
| 66 | + compact={compact} | |
| 67 | + className={className} | |
| 68 | + /> | |
| 69 | + ); | |
| 70 | +} | |
added
apps/web/src/components/charts/dna-radial.tsx
+71 −0
@@ -0,0 +1,71 @@ | ||
| 1 | +import { t } from '@/i18n'; | |
| 2 | +import { cn } from '@/lib/cn'; | |
| 3 | +import { isNum } from '@/lib/format'; | |
| 4 | +import type { DNAResponse, DnaDimension } from '@/lib/types'; | |
| 5 | +import { CHART } from './palette'; | |
| 6 | + | |
| 7 | +export const DNA_DIMS: DnaDimension[] = ['income', 'demographics', 'urbanization', 'trade', 'energy', 'emissions', 'innovation', 'education', 'public_spending']; | |
| 8 | + | |
| 9 | +/** | |
| 10 | + * Radial "fingerprint": 9 dimensions, 0–100 percentile rank, drawn as a filled polygon on a 100-radius | |
| 11 | + * ring with 25/50/75 guide rings and labelled spokes. Server-rendered SVG in a square viewBox; a missing | |
| 12 | + * dimension collapses to the centre and is marked in the legend. | |
| 13 | + */ | |
| 14 | +export function DnaRadial({ dna, name, size = 320, className, compact = false }: { dna: Pick<DNAResponse, 'dims' | 'year_ref'> | null; name: string; size?: number; className?: string; compact?: boolean }) { | |
| 15 | + const dims = DNA_DIMS.map((d) => ({ key: d, label: t(`country.dna.${d}` as const), value: dna?.dims?.[d] ?? null })); | |
| 16 | + const present = dims.filter((d) => isNum(d.value)); | |
| 17 | + if (!dna || present.length === 0) return <p className="text-sm text-ink-3">{t('country.dna.none')}</p>; | |
| 18 | + | |
| 19 | + const R = 100; | |
| 20 | + const pad = compact ? 16 : 44; | |
| 21 | + const cx = R + pad; | |
| 22 | + const cy = R + pad; | |
| 23 | + const W = 2 * (R + pad); | |
| 24 | + const angle = (i: number) => -Math.PI / 2 + (i / dims.length) * 2 * Math.PI; | |
| 25 | + const pt = (i: number, r: number): [number, number] => [cx + r * Math.cos(angle(i)), cy + r * Math.sin(angle(i))]; | |
| 26 | + const poly = dims.map((d, i) => pt(i, isNum(d.value) ? (d.value / 100) * R : 0)); | |
| 27 | + const summary = t('chart.summary.dna', { name, dims: present.map((d) => `${d.label} ${Math.round(d.value as number)}`).join(', ') }); | |
| 28 | + | |
| 29 | + return ( | |
| 30 | + <figure className={cn('min-w-0', className)}> | |
| 31 | + <svg viewBox={`0 0 ${W} ${W}`} width="100%" style={{ maxWidth: size, aspectRatio: '1 / 1', margin: '0 auto' }} role="img" aria-label={summary} className="ca-chart"> | |
| 32 | + <title>{t('country.dna.title')}</title> | |
| 33 | + <desc>{summary}</desc> | |
| 34 | + {[25, 50, 75, 100].map((r) => ( | |
| 35 | + <circle key={r} cx={cx} cy={cy} r={(r / 100) * R} fill="none" stroke={CHART.rule} strokeWidth={r === 100 ? 1 : 0.75} /> | |
| 36 | + ))} | |
| 37 | + {dims.map((_, i) => { | |
| 38 | + const [x, y] = pt(i, R); | |
| 39 | + return <line key={i} x1={cx} y1={cy} x2={x} y2={y} stroke={CHART.rule} strokeWidth={0.75} />; | |
| 40 | + })} | |
| 41 | + <polygon points={poly.map((p) => p.join(',')).join(' ')} fill={CHART.accent} fillOpacity={0.16} stroke={CHART.accent} strokeWidth={2} strokeLinejoin="round" /> | |
| 42 | + {poly.map(([x, y], i) => ( | |
| 43 | + <circle key={i} cx={x} cy={y} r={isNum(dims[i]!.value) ? 4 : 0} fill={CHART.accent} stroke={CHART.surface} strokeWidth={2} /> | |
| 44 | + ))} | |
| 45 | + {!compact | |
| 46 | + ? dims.map((d, i) => { | |
| 47 | + const [x, y] = pt(i, R + 22); | |
| 48 | + const a = angle(i); | |
| 49 | + const anchor = Math.abs(Math.cos(a)) < 0.2 ? 'middle' : Math.cos(a) > 0 ? 'start' : 'end'; | |
| 50 | + return ( | |
| 51 | + <text key={d.key} x={x} y={y} textAnchor={anchor} dy="0.32em" className={isNum(d.value) ? 'label' : ''} style={{ fontSize: 10, fill: isNum(d.value) ? CHART.ink2 : CHART.ink3 }}> | |
| 52 | + {d.label} | |
| 53 | + {isNum(d.value) ? ` ${Math.round(d.value)}` : ' —'} | |
| 54 | + </text> | |
| 55 | + ); | |
| 56 | + }) | |
| 57 | + : null} | |
| 58 | + </svg> | |
| 59 | + {compact ? ( | |
| 60 | + <figcaption className="mt-2 grid grid-cols-3 gap-x-3 gap-y-1 text-2xs text-ink-2"> | |
| 61 | + {dims.map((d) => ( | |
| 62 | + <span key={d.key} className="flex justify-between gap-1"> | |
| 63 | + <span className="truncate">{d.label}</span> | |
| 64 | + <span className="tnum text-ink">{isNum(d.value) ? Math.round(d.value) : '—'}</span> | |
| 65 | + </span> | |
| 66 | + ))} | |
| 67 | + </figcaption> | |
| 68 | + ) : null} | |
| 69 | + </figure> | |
| 70 | + ); | |
| 71 | +} | |
added
apps/web/src/components/charts/line-chart.tsx
+302 −0
@@ -0,0 +1,302 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { area as d3Area, line as d3Line, stack as d3Stack } from 'd3-shape'; | |
| 3 | +import { useCallback, useMemo, useState, type PointerEvent } from 'react'; | |
| 4 | +import { t } from '@/i18n'; | |
| 5 | +import { formatPeriod, formatTick, formatValue } from '@/lib/format'; | |
| 6 | +import type { FormatSpec as Spec, Provenance } from '@/lib/types'; | |
| 7 | +import type { ProvenancePayload } from '@/components/data/provenance-context'; | |
| 8 | +import { ChartFrame, Legend, type TableData } from './chart-frame'; | |
| 9 | +import { CHART, MARK, seriesVar } from './palette'; | |
| 10 | +import { DEFAULT_MARGIN, extent, periodToX, splitForecast, xYearScale, yScale, yearTicks, type Margin, type SeriesPoint } from './scales'; | |
| 11 | +import { summarizeMulti, summarizeSeries } from './summary'; | |
| 12 | +import { ChartTooltip, TooltipRow } from './tooltip'; | |
| 13 | +import { useMeasure } from './use-measure'; | |
| 14 | + | |
| 15 | +export interface LineSeries { | |
| 16 | + id: string; | |
| 17 | + name: string; | |
| 18 | + points: SeriesPoint[]; | |
| 19 | + /** Fixed colour slot (0-based). Defaults to the array index — colour follows the entity, never its rank. */ | |
| 20 | + colorIndex?: number; | |
| 21 | + color?: string; | |
| 22 | +} | |
| 23 | + | |
| 24 | +export interface LineChartProps { | |
| 25 | + series: LineSeries[]; | |
| 26 | + spec: Spec; | |
| 27 | + /** Subject for the accessible summary ("Canada's GDP per capita"). */ | |
| 28 | + subject?: string; | |
| 29 | + variant?: 'line' | 'area' | 'stacked'; | |
| 30 | + log?: boolean; | |
| 31 | + height?: number; | |
| 32 | + margin?: Partial<Margin>; | |
| 33 | + title?: React.ReactNode; | |
| 34 | + subtitle?: React.ReactNode; | |
| 35 | + provenance?: Provenance | null; | |
| 36 | + payload?: ProvenancePayload | null; | |
| 37 | + actions?: React.ReactNode; | |
| 38 | + /** Direct end-labels for ≤ 4 series. */ | |
| 39 | + endLabels?: boolean; | |
| 40 | + /** Draw a highlighted marker on the latest actual point. */ | |
| 41 | + endDot?: boolean; | |
| 42 | + className?: string; | |
| 43 | + defaultWidth?: number; | |
| 44 | +} | |
| 45 | + | |
| 46 | +interface XY { | |
| 47 | + x: number; | |
| 48 | + y: number; | |
| 49 | + p: SeriesPoint; | |
| 50 | +} | |
| 51 | + | |
| 52 | +/** | |
| 53 | + * Multi-series line / area / stacked-area chart. Dashed segments for `is_forecast`, optional log scale, | |
| 54 | + * crosshair + one tooltip listing every series at the nearest X (pointer events → touch friendly). | |
| 55 | + * Index=100 transformations are data-side (pass already-indexed points). | |
| 56 | + */ | |
| 57 | +export function LineChart({ | |
| 58 | + series, | |
| 59 | + spec, | |
| 60 | + subject, | |
| 61 | + variant = 'line', | |
| 62 | + log = false, | |
| 63 | + height = 240, | |
| 64 | + margin: marginIn, | |
| 65 | + title, | |
| 66 | + subtitle, | |
| 67 | + provenance, | |
| 68 | + payload, | |
| 69 | + actions, | |
| 70 | + endLabels = true, | |
| 71 | + endDot = true, | |
| 72 | + className, | |
| 73 | + defaultWidth = 640, | |
| 74 | +}: LineChartProps) { | |
| 75 | + const { ref, width } = useMeasure<HTMLDivElement>(defaultWidth); | |
| 76 | + const margin: Margin = { ...DEFAULT_MARGIN, ...marginIn }; | |
| 77 | + if (endLabels && series.length > 1 && series.length <= 4) margin.right = Math.max(margin.right, 72); | |
| 78 | + const [hover, setHover] = useState<number | null>(null); // x-value (fractional year) | |
| 79 | + | |
| 80 | + const model = useMemo(() => { | |
| 81 | + const clean = series.map((s) => ({ ...s, points: s.points.filter((p) => p.value != null && Number.isFinite(p.value)) })); | |
| 82 | + const allX = clean.flatMap((s) => s.points.map(periodToX)); | |
| 83 | + const xDom = extent(allX) ?? [2000, 2024]; | |
| 84 | + const innerW = Math.max(10, width - margin.left - margin.right); | |
| 85 | + const innerH = Math.max(10, height - margin.top - margin.bottom); | |
| 86 | + const x = xYearScale(xDom, [0, innerW]); | |
| 87 | + | |
| 88 | + let stacks: Array<Array<[number, number, number]>> | null = null; // per series: [x, y0, y1] | |
| 89 | + let yDom: [number, number]; | |
| 90 | + if (variant === 'stacked') { | |
| 91 | + // Build a wide table keyed by x. | |
| 92 | + const keys = clean.map((s) => s.id); | |
| 93 | + const xs = Array.from(new Set(allX)).sort((a, b) => a - b); | |
| 94 | + const table = xs.map((xv) => { | |
| 95 | + const row: Record<string, number> = { __x: xv }; | |
| 96 | + for (const s of clean) { | |
| 97 | + const p = s.points.find((q) => periodToX(q) === xv); | |
| 98 | + row[s.id] = p?.value ?? 0; | |
| 99 | + } | |
| 100 | + return row; | |
| 101 | + }); | |
| 102 | + const st = d3Stack<Record<string, number>>().keys(keys)(table); | |
| 103 | + stacks = st.map((layer) => layer.map((d) => [d.data.__x!, d[0], d[1]] as [number, number, number])); | |
| 104 | + const top = Math.max(0, ...stacks.flat().map((d) => d[2])); | |
| 105 | + yDom = [0, top]; | |
| 106 | + } else { | |
| 107 | + yDom = extent(clean.flatMap((s) => s.points.map((p) => p.value))) ?? [0, 1]; | |
| 108 | + } | |
| 109 | + const y = yScale(yDom, [innerH, 0], { log, includeZero: variant !== 'line' || !log }); | |
| 110 | + const xTicks = yearTicks(xDom, Math.max(2, Math.floor(innerW / 90))); | |
| 111 | + const yTicks = y.ticks(4); | |
| 112 | + const pathFor = d3Line<XY>() | |
| 113 | + .x((d) => d.x) | |
| 114 | + .y((d) => d.y); | |
| 115 | + const areaFor = d3Area<XY>() | |
| 116 | + .x((d) => d.x) | |
| 117 | + .y0(() => y(Math.max(0, y.domain()[0]!))) | |
| 118 | + .y1((d) => d.y); | |
| 119 | + | |
| 120 | + const layers = clean.map((s, i) => { | |
| 121 | + const color = s.color ?? seriesVar(s.colorIndex ?? i); | |
| 122 | + const runs = splitForecast(s.points).map((r) => ({ | |
| 123 | + forecast: r.forecast, | |
| 124 | + xy: r.points.map((p) => ({ x: x(periodToX(p)), y: y(p.value!), p })), | |
| 125 | + })); | |
| 126 | + const all = s.points.map((p) => ({ x: x(periodToX(p)), y: y(p.value!), p })); | |
| 127 | + const actual = s.points.filter((p) => !p.is_forecast); | |
| 128 | + const last = actual[actual.length - 1] ?? null; | |
| 129 | + return { s, color, runs, all, last: last ? { x: x(periodToX(last)), y: y(last.value!), p: last } : null }; | |
| 130 | + }); | |
| 131 | + | |
| 132 | + const stackPaths = | |
| 133 | + stacks && | |
| 134 | + stacks.map((layer, i) => { | |
| 135 | + const a = d3Area<[number, number, number]>() | |
| 136 | + .x((d) => x(d[0])) | |
| 137 | + .y0((d) => y(d[1])) | |
| 138 | + .y1((d) => y(d[2])); | |
| 139 | + return { d: a(layer) ?? '', color: clean[i]!.color ?? seriesVar(clean[i]!.colorIndex ?? i) }; | |
| 140 | + }); | |
| 141 | + | |
| 142 | + return { clean, x, y, innerW, innerH, xTicks, yTicks, pathFor, areaFor, layers, stackPaths, xDom }; | |
| 143 | + }, [series, width, height, margin.left, margin.right, margin.top, margin.bottom, variant, log]); | |
| 144 | + | |
| 145 | + const onMove = useCallback( | |
| 146 | + (e: PointerEvent<SVGRectElement>) => { | |
| 147 | + const rect = e.currentTarget.getBoundingClientRect(); | |
| 148 | + const px = e.clientX - rect.left; | |
| 149 | + const xv = model.x.invert(px); | |
| 150 | + // snap to nearest existing x | |
| 151 | + let best: number | null = null; | |
| 152 | + let bestD = Infinity; | |
| 153 | + for (const s of model.clean) | |
| 154 | + for (const p of s.points) { | |
| 155 | + const d = Math.abs(periodToX(p) - xv); | |
| 156 | + if (d < bestD) { | |
| 157 | + bestD = d; | |
| 158 | + best = periodToX(p); | |
| 159 | + } | |
| 160 | + } | |
| 161 | + setHover(best); | |
| 162 | + }, | |
| 163 | + [model], | |
| 164 | + ); | |
| 165 | + | |
| 166 | + const hoverRows = useMemo(() => { | |
| 167 | + if (hover == null) return null; | |
| 168 | + return model.layers | |
| 169 | + .map((l) => { | |
| 170 | + const p = l.s.points.find((q) => periodToX(q) === hover); | |
| 171 | + return p ? { name: l.s.name, color: l.color, p } : null; | |
| 172 | + }) | |
| 173 | + .filter((r): r is { name: string; color: string; p: SeriesPoint } => !!r); | |
| 174 | + }, [hover, model]); | |
| 175 | + | |
| 176 | + const summary = | |
| 177 | + series.length === 1 ? summarizeSeries(subject ?? series[0]!.name, series[0]!.points, spec) : summarizeMulti(series.map((s) => s.name), series.map((s) => s.points)); | |
| 178 | + | |
| 179 | + const table: TableData = useMemo(() => { | |
| 180 | + const xs = Array.from(new Set(model.clean.flatMap((s) => s.points.map((p) => p.period)))).sort(); | |
| 181 | + return { | |
| 182 | + columns: [{ key: 'period', label: t('common.period') }, ...model.clean.map((s) => ({ key: s.id, label: s.name, numeric: true }))], | |
| 183 | + rows: xs.map((period) => { | |
| 184 | + const row: Record<string, string> = { period: formatPeriod(period, spec.frequency ?? 'A') }; | |
| 185 | + for (const s of model.clean) { | |
| 186 | + const p = s.points.find((q) => q.period === period); | |
| 187 | + row[s.id] = p ? `${formatValue(p.value, spec)}${p.is_forecast ? ' *' : ''}` : t('common.na'); | |
| 188 | + } | |
| 189 | + return row; | |
| 190 | + }), | |
| 191 | + }; | |
| 192 | + }, [model, spec]); | |
| 193 | + | |
| 194 | + const hasForecast = series.some((s) => s.points.some((p) => p.is_forecast)); | |
| 195 | + const hoverX = hover != null ? model.x(hover) : null; | |
| 196 | + const legendItems = model.layers.map((l) => ({ label: l.s.name, color: l.color, shape: variant === 'line' ? ('line' as const) : ('rect' as const) })); | |
| 197 | + const empty = model.clean.every((s) => s.points.length === 0); | |
| 198 | + | |
| 199 | + return ( | |
| 200 | + <ChartFrame | |
| 201 | + title={title} | |
| 202 | + subtitle={subtitle} | |
| 203 | + summary={summary} | |
| 204 | + provenance={provenance} | |
| 205 | + payload={payload} | |
| 206 | + table={empty ? undefined : table} | |
| 207 | + actions={actions} | |
| 208 | + legend={<Legend items={legendItems} />} | |
| 209 | + note={hasForecast ? t('chart.forecastNote') : undefined} | |
| 210 | + className={className} | |
| 211 | + minHeight={height} | |
| 212 | + > | |
| 213 | + <div ref={ref} className="relative w-full" style={{ height }}> | |
| 214 | + {empty ? ( | |
| 215 | + <div className="grid h-full place-items-center text-sm text-ink-3">{t('chart.noData')}</div> | |
| 216 | + ) : ( | |
| 217 | + <svg className="ca-chart" width={width} height={height} viewBox={`0 0 ${width} ${height}`} role="img" aria-label={summary}> | |
| 218 | + <title>{typeof title === 'string' ? title : (spec.name ?? subject ?? '')}</title> | |
| 219 | + <desc>{summary}</desc> | |
| 220 | + <g transform={`translate(${margin.left},${margin.top})`}> | |
| 221 | + <g className="grid"> | |
| 222 | + {model.yTicks.map((tk) => ( | |
| 223 | + <line key={tk} x1={0} x2={model.innerW} y1={model.y(tk)} y2={model.y(tk)} /> | |
| 224 | + ))} | |
| 225 | + </g> | |
| 226 | + <g className="axis"> | |
| 227 | + <line className="baseline" x1={0} x2={model.innerW} y1={model.innerH} y2={model.innerH} /> | |
| 228 | + {model.xTicks.map((yr) => ( | |
| 229 | + <text key={yr} x={model.x(yr)} y={model.innerH + 16} textAnchor="middle"> | |
| 230 | + {yr} | |
| 231 | + </text> | |
| 232 | + ))} | |
| 233 | + {model.yTicks.map((tk) => ( | |
| 234 | + <text key={tk} x={-6} y={model.y(tk)} dy="0.32em" textAnchor="end"> | |
| 235 | + {formatTick(tk, spec)} | |
| 236 | + </text> | |
| 237 | + ))} | |
| 238 | + </g> | |
| 239 | + | |
| 240 | + {variant === 'stacked' && model.stackPaths | |
| 241 | + ? model.stackPaths.map((sp, i) => <path key={i} d={sp.d} fill={sp.color} fillOpacity={0.85} stroke={CHART.surface} strokeWidth={MARK.gap} />) | |
| 242 | + : null} | |
| 243 | + | |
| 244 | + {variant !== 'stacked' | |
| 245 | + ? model.layers.map((l) => ( | |
| 246 | + <g key={l.s.id}> | |
| 247 | + {variant === 'area' ? <path d={model.areaFor(l.all) ?? ''} fill={l.color} fillOpacity={MARK.areaOpacity} /> : null} | |
| 248 | + {l.runs.map((r, i) => ( | |
| 249 | + <path key={i} className={`series-line${r.forecast ? ' forecast' : ''}`} d={model.pathFor(r.xy) ?? ''} stroke={l.color} /> | |
| 250 | + ))} | |
| 251 | + {endDot && l.last ? ( | |
| 252 | + <g> | |
| 253 | + <circle className="ring" cx={l.last.x} cy={l.last.y} r={MARK.dotR + MARK.ringW / 2} fill={l.color} /> | |
| 254 | + </g> | |
| 255 | + ) : null} | |
| 256 | + {endLabels && series.length > 1 && series.length <= 4 && l.last ? ( | |
| 257 | + <text className="label" x={l.last.x + 8} y={l.last.y} dy="0.32em"> | |
| 258 | + {l.s.name} | |
| 259 | + </text> | |
| 260 | + ) : null} | |
| 261 | + </g> | |
| 262 | + )) | |
| 263 | + : null} | |
| 264 | + | |
| 265 | + {series.length === 1 && model.layers[0]?.last ? ( | |
| 266 | + <text className="label label-strong" x={Math.min(model.layers[0].last.x + 8, model.innerW - 4)} y={model.layers[0].last.y} dy="0.32em" textAnchor={model.layers[0].last.x + 60 > model.innerW ? 'end' : 'start'} dx={model.layers[0].last.x + 60 > model.innerW ? -10 : 0}> | |
| 267 | + {formatValue(model.layers[0].last.p.value, spec)} | |
| 268 | + </text> | |
| 269 | + ) : null} | |
| 270 | + | |
| 271 | + {hoverX != null ? <line x1={hoverX} x2={hoverX} y1={0} y2={model.innerH} stroke={CHART.ruleStrong} strokeWidth={1} /> : null} | |
| 272 | + {hoverRows?.map((r) => { | |
| 273 | + const l = model.layers.find((ly) => ly.s.name === r.name)!; | |
| 274 | + return <circle key={r.name} className="ring" cx={model.x(periodToX(r.p))} cy={model.y(r.p.value!)} r={MARK.dotR} fill={l.color} />; | |
| 275 | + })} | |
| 276 | + | |
| 277 | + <rect x={0} y={0} width={model.innerW} height={model.innerH} fill="transparent" style={{ touchAction: 'pan-y' }} onPointerMove={onMove} onPointerDown={onMove} onPointerLeave={() => setHover(null)} /> | |
| 278 | + </g> | |
| 279 | + </svg> | |
| 280 | + )} | |
| 281 | + {hover != null && hoverRows && hoverRows.length > 0 && hoverX != null ? ( | |
| 282 | + <ChartTooltip x={margin.left + hoverX} y={margin.top} width={width}> | |
| 283 | + <div className="mb-0.5 text-2xs text-ink-3"> | |
| 284 | + {formatPeriod(hoverRows[0]!.p.period, spec.frequency ?? 'A')} | |
| 285 | + {hoverRows.some((r) => r.p.is_forecast) ? ` · ${t('chart.tooltipForecast')}` : ''} | |
| 286 | + </div> | |
| 287 | + {hoverRows.map((r) => ( | |
| 288 | + <TooltipRow key={r.name} color={series.length > 1 ? r.color : undefined} label={series.length > 1 ? r.name : (spec.name ?? '')} value={formatValue(r.p.value, spec)} /> | |
| 289 | + ))} | |
| 290 | + </ChartTooltip> | |
| 291 | + ) : null} | |
| 292 | + </div> | |
| 293 | + </ChartFrame> | |
| 294 | + ); | |
| 295 | +} | |
| 296 | + | |
| 297 | +export function AreaChart(props: Omit<LineChartProps, 'variant'>) { | |
| 298 | + return <LineChart {...props} variant="area" />; | |
| 299 | +} | |
| 300 | +export function StackedArea(props: Omit<LineChartProps, 'variant' | 'log'>) { | |
| 301 | + return <LineChart {...props} variant="stacked" />; | |
| 302 | +} | |
added
apps/web/src/components/charts/palette.ts
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +/** | |
| 2 | + * Chart colour roles — all CSS custom properties so light/dark swap without re-render. | |
| 3 | + * Categorical slots are assigned in FIXED order (dataviz rule: colour follows the entity, never its rank; | |
| 4 | + * never cycle past 8 — fold to "Other" or facet). | |
| 5 | + */ | |
| 6 | +export const SERIES_MAX = 8; | |
| 7 | + | |
| 8 | +export function seriesVar(index: number): string { | |
| 9 | + const i = Math.min(Math.max(index, 0), SERIES_MAX - 1) + 1; | |
| 10 | + return `var(--series-${i})`; | |
| 11 | +} | |
| 12 | + | |
| 13 | +/** Sequential (choropleth) class fill, k ∈ 1..7 light→dark. */ | |
| 14 | +export function seqVar(step: number): string { | |
| 15 | + const i = Math.min(Math.max(step, 1), 7); | |
| 16 | + return `var(--seq-${i})`; | |
| 17 | +} | |
| 18 | + | |
| 19 | +export const CHART = { | |
| 20 | + ink: 'var(--ink)', | |
| 21 | + ink2: 'var(--ink-2)', | |
| 22 | + ink3: 'var(--ink-3)', | |
| 23 | + rule: 'var(--rule)', | |
| 24 | + ruleStrong: 'var(--rule-strong)', | |
| 25 | + surface: 'var(--surface)', | |
| 26 | + accent: 'var(--accent)', | |
| 27 | + accentSoft: 'var(--accent-soft)', | |
| 28 | + up: 'var(--up)', | |
| 29 | + down: 'var(--down)', | |
| 30 | + nodata: 'var(--nodata)', | |
| 31 | + forecast: 'var(--forecast)', | |
| 32 | +} as const; | |
| 33 | + | |
| 34 | +/** Mark specs (dataviz marks-and-anatomy): thin marks, 2px lines, ≥8px end dots, 24px max bar thickness. */ | |
| 35 | +export const MARK = { | |
| 36 | + line: 2, | |
| 37 | + dotR: 4, | |
| 38 | + ringW: 2, | |
| 39 | + barMax: 24, | |
| 40 | + barRadius: 4, | |
| 41 | + gap: 2, | |
| 42 | + areaOpacity: 0.12, | |
| 43 | +} as const; | |
added
apps/web/src/components/charts/population-pyramid.tsx
+58 −0
@@ -0,0 +1,58 @@ | ||
| 1 | +import { t } from '@/i18n'; | |
| 2 | +import { fixed, isNum } from '@/lib/format'; | |
| 3 | +import { seriesVar } from './palette'; | |
| 4 | + | |
| 5 | +/** | |
| 6 | + * Age structure as a single stacked bar (0–14 / 15–64 / 65+ shares, % of population). Ordinal colouring | |
| 7 | + * (one hue, three lightness steps via the sequential ramp) because the categories are ordered. | |
| 8 | + * A 2 px surface gap separates segments; labels inside only when the segment is wide enough. | |
| 9 | + */ | |
| 10 | +export function PopulationPyramid({ | |
| 11 | + shares, | |
| 12 | + name, | |
| 13 | + year, | |
| 14 | + className, | |
| 15 | +}: { | |
| 16 | + shares: { young: number | null; working: number | null; old: number | null }; | |
| 17 | + name: string; | |
| 18 | + year?: number | null; | |
| 19 | + className?: string; | |
| 20 | +}) { | |
| 21 | + const parts = [ | |
| 22 | + { key: 'young', label: '0–14', value: shares.young, color: 'var(--seq-2)' }, | |
| 23 | + { key: 'working', label: '15–64', value: shares.working, color: 'var(--seq-4)' }, | |
| 24 | + { key: 'old', label: '65+', value: shares.old, color: 'var(--seq-6)' }, | |
| 25 | + ]; | |
| 26 | + const valid = parts.filter((p) => isNum(p.value)); | |
| 27 | + if (valid.length === 0) return <p className="text-sm text-ink-3">{t('chart.noData')}</p>; | |
| 28 | + const total = valid.reduce((a, p) => a + (p.value as number), 0) || 100; | |
| 29 | + const summary = t('chart.summary.pyramid', { name, parts: valid.map((p) => `${p.label}: ${fixed(p.value as number, 1)} %`).join(', ') }); | |
| 30 | + return ( | |
| 31 | + <figure className={className}> | |
| 32 | + <div className="flex h-6 w-full overflow-hidden rounded-xs" role="img" aria-label={summary} style={{ gap: 2 }}> | |
| 33 | + {valid.map((p) => { | |
| 34 | + const pct = ((p.value as number) / total) * 100; | |
| 35 | + return ( | |
| 36 | + <div key={p.key} className="relative h-full" style={{ width: `${pct}%`, background: p.color }} title={`${p.label}: ${fixed(p.value as number, 1)} %`}> | |
| 37 | + {pct > 14 ? ( | |
| 38 | + <span className="tnum absolute inset-0 grid place-items-center text-2xs font-medium" style={{ color: p.key === 'young' ? 'var(--ink)' : '#fff' }}> | |
| 39 | + {fixed(p.value as number, 0)} % | |
| 40 | + </span> | |
| 41 | + ) : null} | |
| 42 | + </div> | |
| 43 | + ); | |
| 44 | + })} | |
| 45 | + </div> | |
| 46 | + <figcaption className="mt-1.5 flex flex-wrap gap-x-4 gap-y-1 text-xs text-ink-2"> | |
| 47 | + {valid.map((p) => ( | |
| 48 | + <span key={p.key} className="inline-flex items-center gap-1.5"> | |
| 49 | + <span aria-hidden className="inline-block h-2.5 w-2.5 rounded-xs" style={{ background: p.color }} /> | |
| 50 | + {p.label} <span className="tnum text-ink">{fixed(p.value as number, 1)} %</span> | |
| 51 | + </span> | |
| 52 | + ))} | |
| 53 | + {year ? <span className="text-ink-3">{year}</span> : null} | |
| 54 | + </figcaption> | |
| 55 | + {seriesVar(0) ? null : null} | |
| 56 | + </figure> | |
| 57 | + ); | |
| 58 | +} | |
added
apps/web/src/components/charts/ranked-bars.tsx
+108 −0
@@ -0,0 +1,108 @@ | ||
| 1 | +import Link from 'next/link'; | |
| 2 | +import { t } from '@/i18n'; | |
| 3 | +import { cn } from '@/lib/cn'; | |
| 4 | +import { formatValue, isNum, ordinal } from '@/lib/format'; | |
| 5 | +import { routes } from '@/lib/site'; | |
| 6 | +import type { FormatSpec as Spec, Provenance } from '@/lib/types'; | |
| 7 | +import { MARK } from './palette'; | |
| 8 | + | |
| 9 | +export interface RankedBarRow { | |
| 10 | + id: string; | |
| 11 | + label: string; | |
| 12 | + flag?: string | null; | |
| 13 | + href?: string | null; | |
| 14 | + value: number | null; | |
| 15 | + rank?: number | null; | |
| 16 | + /** Secondary text shown right of the value (e.g. change). */ | |
| 17 | + hint?: string | null; | |
| 18 | +} | |
| 19 | + | |
| 20 | +/** | |
| 21 | + * Horizontal ranked bars rendered as HTML (server component, fully responsive, no SVG text scaling). | |
| 22 | + * One series → one colour (slot 1); `highlightId` marks the country of interest with the accent and bold label. | |
| 23 | + * Bars ≤ 24 px thick, 4 px rounded data-end, square at the baseline; value labelled at the tip (text tokens). | |
| 24 | + */ | |
| 25 | +export function RankedBars({ | |
| 26 | + rows, | |
| 27 | + spec, | |
| 28 | + highlightId, | |
| 29 | + showRank = true, | |
| 30 | + className, | |
| 31 | + provenance, | |
| 32 | + ariaLabel, | |
| 33 | + linkRows = true, | |
| 34 | +}: { | |
| 35 | + rows: RankedBarRow[]; | |
| 36 | + spec: Spec; | |
| 37 | + highlightId?: string | null; | |
| 38 | + showRank?: boolean; | |
| 39 | + className?: string; | |
| 40 | + provenance?: Provenance | null; | |
| 41 | + ariaLabel?: string; | |
| 42 | + linkRows?: boolean; | |
| 43 | +}) { | |
| 44 | + const values = rows.map((r) => r.value).filter(isNum); | |
| 45 | + const max = values.length ? Math.max(...values.map(Math.abs)) : 0; | |
| 46 | + const top = rows[0]; | |
| 47 | + const summary = top ? t('chart.summary.bars', { top: top.label, value: formatValue(top.value, spec), n: rows.length }) : t('chart.noData'); | |
| 48 | + if (rows.length === 0) return <p className="text-sm text-ink-3">{t('chart.noData')}</p>; | |
| 49 | + return ( | |
| 50 | + <div className={cn('min-w-0', className)} role="img" aria-label={ariaLabel ?? summary}> | |
| 51 | + <ol className="divide-y divide-rule"> | |
| 52 | + {rows.map((r, i) => { | |
| 53 | + const pct = isNum(r.value) && max > 0 ? Math.max(0, (Math.abs(r.value) / max) * 100) : 0; | |
| 54 | + const hl = highlightId && r.id === highlightId; | |
| 55 | + const label = ( | |
| 56 | + <span className={cn('flex min-w-0 items-center gap-1.5 truncate text-sm', hl ? 'font-semibold text-ink' : 'text-ink')}> | |
| 57 | + {r.flag ? ( | |
| 58 | + <span aria-hidden className="text-base leading-none"> | |
| 59 | + {r.flag} | |
| 60 | + </span> | |
| 61 | + ) : null} | |
| 62 | + <span className="truncate">{r.label}</span> | |
| 63 | + </span> | |
| 64 | + ); | |
| 65 | + return ( | |
| 66 | + <li key={r.id} className="grid grid-cols-[minmax(0,11rem)_1fr] items-center gap-x-3 py-1.5 sm:grid-cols-[minmax(0,13rem)_1fr]"> | |
| 67 | + <div className="flex min-w-0 items-center gap-2"> | |
| 68 | + {showRank ? <span className="tnum w-6 shrink-0 text-right text-xs text-ink-3">{r.rank ?? i + 1}</span> : null} | |
| 69 | + {linkRows && r.href ? ( | |
| 70 | + <Link href={r.href} className="link-quiet min-w-0 truncate"> | |
| 71 | + {label} | |
| 72 | + </Link> | |
| 73 | + ) : ( | |
| 74 | + label | |
| 75 | + )} | |
| 76 | + </div> | |
| 77 | + <div className="flex min-w-0 items-center gap-2"> | |
| 78 | + <div className="h-4 min-w-0 flex-1" style={{ maxHeight: MARK.barMax }}> | |
| 79 | + <div | |
| 80 | + className={cn('h-full rounded-r-sm', hl ? 'bg-accent' : 'bg-series-1')} | |
| 81 | + style={{ width: `${pct}%`, minWidth: isNum(r.value) ? 2 : 0, borderRadius: `0 ${MARK.barRadius}px ${MARK.barRadius}px 0`, opacity: hl ? 1 : 0.85 }} | |
| 82 | + /> | |
| 83 | + </div> | |
| 84 | + <span className={cn('tnum shrink-0 text-right text-sm', hl ? 'font-semibold text-ink' : 'text-ink')} style={{ minWidth: '4.5rem' }}> | |
| 85 | + {formatValue(r.value, spec)} | |
| 86 | + </span> | |
| 87 | + {r.hint ? <span className="tnum hidden shrink-0 text-xs text-ink-3 sm:inline">{r.hint}</span> : null} | |
| 88 | + </div> | |
| 89 | + </li> | |
| 90 | + ); | |
| 91 | + })} | |
| 92 | + </ol> | |
| 93 | + {provenance ? ( | |
| 94 | + <p className="mt-1.5 text-2xs text-ink-2"> | |
| 95 | + <span className="text-ink-3">{t('common.source')}: </span> | |
| 96 | + {[provenance.source_name, provenance.dataset].filter(Boolean).join(' — ')} · {provenance.series_code} | |
| 97 | + </p> | |
| 98 | + ) : null} | |
| 99 | + </div> | |
| 100 | + ); | |
| 101 | +} | |
| 102 | + | |
| 103 | +/** Convenience: a country row for RankedBars from a ranking row. */ | |
| 104 | +export function rankedRowFromCountry(c: { id: string; slug: string | null; name: string | null; flag: string | null }, value: number | null, rank?: number | null, hint?: string | null): RankedBarRow { | |
| 105 | + return { id: c.id, label: c.name ?? c.id, flag: c.flag, href: c.slug ? routes.country(c.slug) : null, value, rank, hint }; | |
| 106 | +} | |
| 107 | + | |
| 108 | +export { ordinal }; | |
added
apps/web/src/components/charts/scales.ts
+133 −0
@@ -0,0 +1,133 @@ | ||
| 1 | +import { scaleLinear, scaleLog, type ScaleContinuousNumeric } from 'd3-scale'; | |
| 2 | +import type { SeriesValue, SparkPoint } from '@/lib/types'; | |
| 3 | + | |
| 4 | +/** Chart-side point. Build with `pointsFromSpark` / `pointsFromSeries`. */ | |
| 5 | +export interface SeriesPoint { | |
| 6 | + period: string; | |
| 7 | + year: number; | |
| 8 | + value: number | null; | |
| 9 | + is_forecast?: boolean; | |
| 10 | + is_estimate?: boolean; | |
| 11 | +} | |
| 12 | + | |
| 13 | +/** API sparkline `[year, value]` → points (annual, non-forecast). */ | |
| 14 | +export function pointsFromSpark(spark: SparkPoint[] | null | undefined): SeriesPoint[] { | |
| 15 | + if (!spark) return []; | |
| 16 | + return spark.filter((p) => Array.isArray(p) && typeof p[0] === 'number').map(([year, value]) => ({ period: `${year}-01-01`, year, value })); | |
| 17 | +} | |
| 18 | + | |
| 19 | +/** API series values → points (keeps forecast flags, drops rows without a period). */ | |
| 20 | +export function pointsFromSeries(values: SeriesValue[] | null | undefined): SeriesPoint[] { | |
| 21 | + if (!values) return []; | |
| 22 | + const out: SeriesPoint[] = []; | |
| 23 | + for (const v of values) { | |
| 24 | + const year = v.year ?? (v.period ? Number(v.period.slice(0, 4)) : NaN); | |
| 25 | + if (!Number.isFinite(year)) continue; | |
| 26 | + out.push({ period: v.period ?? `${year}-01-01`, year, value: v.value, is_forecast: v.is_forecast, is_estimate: v.is_estimate }); | |
| 27 | + } | |
| 28 | + return out; | |
| 29 | +} | |
| 30 | + | |
| 31 | +export interface Margin { | |
| 32 | + top: number; | |
| 33 | + right: number; | |
| 34 | + bottom: number; | |
| 35 | + left: number; | |
| 36 | +} | |
| 37 | + | |
| 38 | +export const DEFAULT_MARGIN: Margin = { top: 12, right: 12, bottom: 24, left: 44 }; | |
| 39 | + | |
| 40 | +export function extent(values: Array<number | null | undefined>): [number, number] | null { | |
| 41 | + let lo = Infinity; | |
| 42 | + let hi = -Infinity; | |
| 43 | + for (const v of values) { | |
| 44 | + if (typeof v !== 'number' || !Number.isFinite(v)) continue; | |
| 45 | + if (v < lo) lo = v; | |
| 46 | + if (v > hi) hi = v; | |
| 47 | + } | |
| 48 | + if (lo === Infinity) return null; | |
| 49 | + return [lo, hi]; | |
| 50 | +} | |
| 51 | + | |
| 52 | +/** Y scale: linear (zero-anchored when the data allows) or log for strictly positive values. */ | |
| 53 | +export function yScale( | |
| 54 | + domain: [number, number], | |
| 55 | + range: [number, number], | |
| 56 | + opts: { log?: boolean; includeZero?: boolean } = {}, | |
| 57 | +): ScaleContinuousNumeric<number, number> { | |
| 58 | + let [lo, hi] = domain; | |
| 59 | + if (opts.log && lo > 0) { | |
| 60 | + return scaleLog().domain([lo, hi]).range(range).nice(); | |
| 61 | + } | |
| 62 | + if (opts.includeZero ?? true) { | |
| 63 | + if (lo > 0) lo = 0; | |
| 64 | + if (hi < 0) hi = 0; | |
| 65 | + } | |
| 66 | + if (lo === hi) { | |
| 67 | + const pad = Math.abs(lo) * 0.1 || 1; | |
| 68 | + lo -= pad; | |
| 69 | + hi += pad; | |
| 70 | + } | |
| 71 | + return scaleLinear().domain([lo, hi]).range(range).nice(5); | |
| 72 | +} | |
| 73 | + | |
| 74 | +export function xYearScale(domain: [number, number], range: [number, number]) { | |
| 75 | + const [lo, hi] = domain[0] === domain[1] ? [domain[0] - 1, domain[1] + 1] : domain; | |
| 76 | + return scaleLinear().domain([lo, hi]).range(range); | |
| 77 | +} | |
| 78 | + | |
| 79 | +/** ~n year ticks at round intervals (1, 2, 5, 10, 20, 25, 50). */ | |
| 80 | +export function yearTicks(domain: [number, number], n = 5): number[] { | |
| 81 | + const [lo, hi] = domain; | |
| 82 | + const span = Math.max(1, hi - lo); | |
| 83 | + const steps = [1, 2, 5, 10, 20, 25, 50, 100]; | |
| 84 | + const step = steps.find((s) => span / s <= n) ?? 100; | |
| 85 | + const start = Math.ceil(lo / step) * step; | |
| 86 | + const out: number[] = []; | |
| 87 | + for (let y = start; y <= hi; y += step) out.push(y); | |
| 88 | + if (out.length === 0) out.push(lo, hi); | |
| 89 | + return out; | |
| 90 | +} | |
| 91 | + | |
| 92 | +/** Fractional year for a period (annual → year, quarterly → year + q/4, monthly → year + m/12). */ | |
| 93 | +export function periodToX(p: Pick<SeriesPoint, 'period' | 'year'>): number { | |
| 94 | + const m = /^(\d{4})-(\d{2})/.exec(p.period ?? ''); | |
| 95 | + if (!m) return p.year; | |
| 96 | + const month = Number(m[2]); | |
| 97 | + return Number(m[1]) + (month - 1) / 12; | |
| 98 | +} | |
| 99 | + | |
| 100 | +export function clamp(v: number, lo: number, hi: number): number { | |
| 101 | + return Math.min(hi, Math.max(lo, v)); | |
| 102 | +} | |
| 103 | + | |
| 104 | +/** Split a series into consecutive runs of the same `is_forecast` flag (so projections draw dashed). */ | |
| 105 | +export function splitForecast(points: SeriesPoint[]): Array<{ forecast: boolean; points: SeriesPoint[] }> { | |
| 106 | + const runs: Array<{ forecast: boolean; points: SeriesPoint[] }> = []; | |
| 107 | + let prev: SeriesPoint | null = null; | |
| 108 | + for (const p of points) { | |
| 109 | + if (p.value == null) { | |
| 110 | + prev = null; | |
| 111 | + continue; | |
| 112 | + } | |
| 113 | + const f = !!p.is_forecast; | |
| 114 | + const last = runs[runs.length - 1]; | |
| 115 | + if (!last || last.forecast !== f || prev === null) { | |
| 116 | + const run = { forecast: f, points: [] as SeriesPoint[] }; | |
| 117 | + // connect segments: a forecast run starts from the last actual point | |
| 118 | + if (prev && last && last.forecast !== f) run.points.push(prev); | |
| 119 | + runs.push(run); | |
| 120 | + } | |
| 121 | + runs[runs.length - 1]!.points.push(p); | |
| 122 | + prev = p; | |
| 123 | + } | |
| 124 | + return runs; | |
| 125 | +} | |
| 126 | + | |
| 127 | +export function firstLast(points: SeriesPoint[]): { first: SeriesPoint; last: SeriesPoint } | null { | |
| 128 | + const valid = points.filter((p) => typeof p.value === 'number' && Number.isFinite(p.value) && !p.is_forecast); | |
| 129 | + const first = valid[0]; | |
| 130 | + const last = valid[valid.length - 1]; | |
| 131 | + if (!first || !last) return null; | |
| 132 | + return { first, last }; | |
| 133 | +} | |
added
apps/web/src/components/charts/scatter.tsx
+118 −0
@@ -0,0 +1,118 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { scaleSqrt } from 'd3-scale'; | |
| 3 | +import { useMemo, useState } from 'react'; | |
| 4 | +import { t } from '@/i18n'; | |
| 5 | +import { formatTick, formatValue } from '@/lib/format'; | |
| 6 | +import type { FormatSpec as Spec } from '@/lib/types'; | |
| 7 | +import { ChartFrame } from './chart-frame'; | |
| 8 | +import { CHART, MARK } from './palette'; | |
| 9 | +import { DEFAULT_MARGIN, extent, yScale } from './scales'; | |
| 10 | +import { ChartTooltip, TooltipRow } from './tooltip'; | |
| 11 | +import { useMeasure } from './use-measure'; | |
| 12 | + | |
| 13 | +export interface ScatterPoint { | |
| 14 | + id: string; | |
| 15 | + label: string; | |
| 16 | + x: number | null; | |
| 17 | + y: number | null; | |
| 18 | + size?: number | null; | |
| 19 | + href?: string; | |
| 20 | + highlight?: boolean; | |
| 21 | +} | |
| 22 | + | |
| 23 | +/** | |
| 24 | + * Scatter / bubble. One colour for all points (nominal, no series), highlighted points in the accent; | |
| 25 | + * labels only for highlighted points and the extremes (never every point). Nearest-point hover with a | |
| 26 | + * 24 px hit radius. | |
| 27 | + */ | |
| 28 | +export function Scatter({ points, xSpec, ySpec, logX, logY, height = 300, title, className, labelCount = 4 }: { points: ScatterPoint[]; xSpec: Spec; ySpec: Spec; logX?: boolean; logY?: boolean; height?: number; title?: React.ReactNode; className?: string; labelCount?: number }) { | |
| 29 | + const { ref, width } = useMeasure<HTMLDivElement>(560); | |
| 30 | + const [hover, setHover] = useState<string | null>(null); | |
| 31 | + const m = { ...DEFAULT_MARGIN, left: 52, bottom: 32 }; | |
| 32 | + const model = useMemo(() => { | |
| 33 | + const clean = points.filter((p) => p.x != null && p.y != null) as Array<ScatterPoint & { x: number; y: number }>; | |
| 34 | + const innerW = Math.max(10, width - m.left - m.right); | |
| 35 | + const innerH = Math.max(10, height - m.top - m.bottom); | |
| 36 | + const xd = extent(clean.map((p) => p.x)) ?? [0, 1]; | |
| 37 | + const yd = extent(clean.map((p) => p.y)) ?? [0, 1]; | |
| 38 | + const x = yScale(xd, [0, innerW], { log: logX, includeZero: !logX }); | |
| 39 | + const y = yScale(yd, [innerH, 0], { log: logY, includeZero: !logY }); | |
| 40 | + const sizes = clean.map((p) => p.size ?? null).filter((v): v is number => v != null); | |
| 41 | + const r = sizes.length ? scaleSqrt().domain([0, Math.max(...sizes)]).range([3, 18]) : null; | |
| 42 | + const labelled = new Set<string>(); | |
| 43 | + for (const p of clean) if (p.highlight) labelled.add(p.id); | |
| 44 | + const byY = [...clean].sort((a, b) => b.y - a.y); | |
| 45 | + const byX = [...clean].sort((a, b) => b.x - a.x); | |
| 46 | + for (const p of [byY[0], byY[byY.length - 1], byX[0], byX[byX.length - 1]]) if (p && labelled.size < labelCount + 2) labelled.add(p.id); | |
| 47 | + return { clean, x, y, r, innerW, innerH, labelled }; | |
| 48 | + }, [points, width, height, logX, logY, labelCount, m.left, m.right, m.top, m.bottom]); | |
| 49 | + | |
| 50 | + const summary = t('chart.summary.scatter', { x: xSpec.name ?? 'x', y: ySpec.name ?? 'y', n: model.clean.length }); | |
| 51 | + const hp = hover ? model.clean.find((p) => p.id === hover) : null; | |
| 52 | + const onMove = (e: React.PointerEvent<SVGRectElement>) => { | |
| 53 | + const rect = e.currentTarget.getBoundingClientRect(); | |
| 54 | + const px = e.clientX - rect.left; | |
| 55 | + const py = e.clientY - rect.top; | |
| 56 | + let best: string | null = null; | |
| 57 | + let bd = 24 * 24; | |
| 58 | + for (const p of model.clean) { | |
| 59 | + const dx = model.x(p.x) - px; | |
| 60 | + const dy = model.y(p.y) - py; | |
| 61 | + const d = dx * dx + dy * dy; | |
| 62 | + if (d < bd) { | |
| 63 | + bd = d; | |
| 64 | + best = p.id; | |
| 65 | + } | |
| 66 | + } | |
| 67 | + setHover(best); | |
| 68 | + }; | |
| 69 | + return ( | |
| 70 | + <ChartFrame title={title} summary={summary} className={className} minHeight={height} table={{ columns: [{ key: 'label', label: '' }, { key: 'x', label: xSpec.name ?? 'x', numeric: true }, { key: 'y', label: ySpec.name ?? 'y', numeric: true }], rows: model.clean.map((p) => ({ label: p.label, x: formatValue(p.x, xSpec), y: formatValue(p.y, ySpec) })) }}> | |
| 71 | + <div ref={ref} className="relative" style={{ height }}> | |
| 72 | + <svg className="ca-chart" width={width} height={height} viewBox={`0 0 ${width} ${height}`} role="img" aria-label={summary}> | |
| 73 | + <g transform={`translate(${m.left},${m.top})`}> | |
| 74 | + <g className="grid"> | |
| 75 | + {model.y.ticks(4).map((tk) => ( | |
| 76 | + <line key={tk} x1={0} x2={model.innerW} y1={model.y(tk)} y2={model.y(tk)} /> | |
| 77 | + ))} | |
| 78 | + </g> | |
| 79 | + <g className="axis"> | |
| 80 | + <line className="baseline" x1={0} x2={model.innerW} y1={model.innerH} y2={model.innerH} /> | |
| 81 | + {model.x.ticks(5).map((tk) => ( | |
| 82 | + <text key={tk} x={model.x(tk)} y={model.innerH + 16} textAnchor="middle"> | |
| 83 | + {formatTick(tk, xSpec)} | |
| 84 | + </text> | |
| 85 | + ))} | |
| 86 | + {model.y.ticks(4).map((tk) => ( | |
| 87 | + <text key={tk} x={-6} y={model.y(tk)} dy="0.32em" textAnchor="end"> | |
| 88 | + {formatTick(tk, ySpec)} | |
| 89 | + </text> | |
| 90 | + ))} | |
| 91 | + <text x={model.innerW} y={model.innerH + 28} textAnchor="end" className="label"> | |
| 92 | + {xSpec.name} | |
| 93 | + </text> | |
| 94 | + </g> | |
| 95 | + {model.clean.map((p) => ( | |
| 96 | + <circle key={p.id} className="ring" cx={model.x(p.x)} cy={model.y(p.y)} r={model.r ? model.r(p.size ?? 0) : MARK.dotR} fill={p.highlight ? CHART.accent : 'var(--series-1)'} fillOpacity={p.highlight ? 1 : 0.7} /> | |
| 97 | + ))} | |
| 98 | + {model.clean | |
| 99 | + .filter((p) => model.labelled.has(p.id)) | |
| 100 | + .map((p) => ( | |
| 101 | + <text key={`l-${p.id}`} className={`label${p.highlight ? ' label-strong' : ''}`} x={model.x(p.x) + 7} y={model.y(p.y)} dy="0.32em"> | |
| 102 | + {p.label} | |
| 103 | + </text> | |
| 104 | + ))} | |
| 105 | + <rect x={0} y={0} width={model.innerW} height={model.innerH} fill="transparent" onPointerMove={onMove} onPointerDown={onMove} onPointerLeave={() => setHover(null)} style={{ touchAction: 'pan-y' }} /> | |
| 106 | + </g> | |
| 107 | + </svg> | |
| 108 | + {hp ? ( | |
| 109 | + <ChartTooltip x={m.left + model.x(hp.x)} y={m.top + model.y(hp.y)} width={width}> | |
| 110 | + <div className="mb-0.5 font-medium text-ink">{hp.label}</div> | |
| 111 | + <TooltipRow label={xSpec.name ?? 'x'} value={formatValue(hp.x, xSpec)} /> | |
| 112 | + <TooltipRow label={ySpec.name ?? 'y'} value={formatValue(hp.y, ySpec)} /> | |
| 113 | + </ChartTooltip> | |
| 114 | + ) : null} | |
| 115 | + </div> | |
| 116 | + </ChartFrame> | |
| 117 | + ); | |
| 118 | +} | |
added
apps/web/src/components/charts/slope-chart.tsx
+70 −0
@@ -0,0 +1,70 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { useMemo } from 'react'; | |
| 3 | +import { t } from '@/i18n'; | |
| 4 | +import { formatValue } from '@/lib/format'; | |
| 5 | +import type { FormatSpec as Spec } from '@/lib/types'; | |
| 6 | +import { ChartFrame } from './chart-frame'; | |
| 7 | +import { CHART, MARK, seriesVar } from './palette'; | |
| 8 | +import { yScale } from './scales'; | |
| 9 | +import { useMeasure } from './use-measure'; | |
| 10 | + | |
| 11 | +export interface SlopeRow { | |
| 12 | + id: string; | |
| 13 | + label: string; | |
| 14 | + a: number | null; | |
| 15 | + b: number | null; | |
| 16 | + colorIndex?: number; | |
| 17 | + highlight?: boolean; | |
| 18 | +} | |
| 19 | + | |
| 20 | +/** Two-period slope chart; labels on both ends, highlighted row in the accent, others in muted ink. */ | |
| 21 | +export function SlopeChart({ rows, spec, yearA, yearB, height = 260, title, className }: { rows: SlopeRow[]; spec: Spec; yearA: number; yearB: number; height?: number; title?: React.ReactNode; className?: string }) { | |
| 22 | + const { ref, width } = useMeasure<HTMLDivElement>(480); | |
| 23 | + const model = useMemo(() => { | |
| 24 | + const clean = rows.filter((r) => r.a != null && r.b != null) as Array<SlopeRow & { a: number; b: number }>; | |
| 25 | + const vals = clean.flatMap((r) => [r.a, r.b]); | |
| 26 | + const lo = Math.min(...vals); | |
| 27 | + const hi = Math.max(...vals); | |
| 28 | + const m = { top: 20, bottom: 24, left: 110, right: 110 }; | |
| 29 | + const innerH = height - m.top - m.bottom; | |
| 30 | + const y = yScale([lo, hi], [innerH, 0], { includeZero: false }); | |
| 31 | + const x0 = m.left; | |
| 32 | + const x1 = width - m.right; | |
| 33 | + return { clean, y, m, x0, x1, innerH }; | |
| 34 | + }, [rows, width, height]); | |
| 35 | + const summary = t('chart.summary.slope', { y0: yearA, y1: yearB, n: model.clean.length }); | |
| 36 | + return ( | |
| 37 | + <ChartFrame title={title} summary={summary} className={className} minHeight={height} table={{ columns: [{ key: 'label', label: '' }, { key: 'a', label: String(yearA), numeric: true }, { key: 'b', label: String(yearB), numeric: true }], rows: model.clean.map((r) => ({ label: r.label, a: formatValue(r.a, spec), b: formatValue(r.b, spec) })) }}> | |
| 38 | + <div ref={ref} style={{ height }}> | |
| 39 | + <svg className="ca-chart" width={width} height={height} viewBox={`0 0 ${width} ${height}`} role="img" aria-label={summary}> | |
| 40 | + <g transform={`translate(0,${model.m.top})`}> | |
| 41 | + <line x1={model.x0} x2={model.x0} y1={0} y2={model.innerH} stroke={CHART.rule} /> | |
| 42 | + <line x1={model.x1} x2={model.x1} y1={0} y2={model.innerH} stroke={CHART.rule} /> | |
| 43 | + <text x={model.x0} y={model.innerH + 16} textAnchor="middle"> | |
| 44 | + {yearA} | |
| 45 | + </text> | |
| 46 | + <text x={model.x1} y={model.innerH + 16} textAnchor="middle"> | |
| 47 | + {yearB} | |
| 48 | + </text> | |
| 49 | + {model.clean.map((r, i) => { | |
| 50 | + const color = r.highlight ? CHART.accent : rows.length > 8 ? CHART.ruleStrong : seriesVar(r.colorIndex ?? i); | |
| 51 | + return ( | |
| 52 | + <g key={r.id}> | |
| 53 | + <line x1={model.x0} x2={model.x1} y1={model.y(r.a)} y2={model.y(r.b)} stroke={color} strokeWidth={MARK.line} strokeLinecap="round" opacity={r.highlight || rows.length <= 8 ? 1 : 0.6} /> | |
| 54 | + <circle className="ring" cx={model.x0} cy={model.y(r.a)} r={MARK.dotR} fill={color} /> | |
| 55 | + <circle className="ring" cx={model.x1} cy={model.y(r.b)} r={MARK.dotR} fill={color} /> | |
| 56 | + <text className={`label${r.highlight ? ' label-strong' : ''}`} x={model.x0 - 8} y={model.y(r.a)} dy="0.32em" textAnchor="end"> | |
| 57 | + {r.label} {formatValue(r.a, spec)} | |
| 58 | + </text> | |
| 59 | + <text className={`label${r.highlight ? ' label-strong' : ''}`} x={model.x1 + 8} y={model.y(r.b)} dy="0.32em"> | |
| 60 | + {formatValue(r.b, spec)} | |
| 61 | + </text> | |
| 62 | + </g> | |
| 63 | + ); | |
| 64 | + })} | |
| 65 | + </g> | |
| 66 | + </svg> | |
| 67 | + </div> | |
| 68 | + </ChartFrame> | |
| 69 | + ); | |
| 70 | +} | |
added
apps/web/src/components/charts/small-multiples.tsx
+21 −0
@@ -0,0 +1,21 @@ | ||
| 1 | +import type { ReactNode } from 'react'; | |
| 2 | +import { cn } from '@/lib/cn'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Grid wrapper for small multiples: 1 col ≤ 360 px, 2 cols on phones, 3–4 on desktop. Each child should be a | |
| 6 | + * chart with the same axes so facets compare (dataviz rule: facet instead of a 5th+ series on all-pairs forms). | |
| 7 | + */ | |
| 8 | +export function SmallMultiples({ children, columns = 3, className }: { children: ReactNode; columns?: 2 | 3 | 4; className?: string }) { | |
| 9 | + return ( | |
| 10 | + <div | |
| 11 | + className={cn( | |
| 12 | + 'grid gap-x-6 gap-y-6 min-[361px]:grid-cols-2', | |
| 13 | + columns >= 3 && 'lg:grid-cols-3', | |
| 14 | + columns === 4 && 'xl:grid-cols-4', | |
| 15 | + className, | |
| 16 | + )} | |
| 17 | + > | |
| 18 | + {children} | |
| 19 | + </div> | |
| 20 | + ); | |
| 21 | +} | |
added
apps/web/src/components/charts/source-line.tsx
+37 −0
@@ -0,0 +1,37 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { Info } from 'lucide-react'; | |
| 3 | +import { t } from '@/i18n'; | |
| 4 | +import { formatDate } from '@/lib/format'; | |
| 5 | +import type { Provenance } from '@/lib/types'; | |
| 6 | +import { useProvenance, type ProvenancePayload } from '@/components/data/provenance-context'; | |
| 7 | + | |
| 8 | +/** | |
| 9 | + * "Source: World Bank — WDI · NY.GDP.PCAP.CD · retrieved 11 Sep 2026". Click → provenance sheet | |
| 10 | + * (when `payload` is given) — otherwise a plain text line. | |
| 11 | + */ | |
| 12 | +export function SourceLine({ provenance, payload, className }: { provenance: Provenance | null | undefined; payload?: ProvenancePayload | null; className?: string }) { | |
| 13 | + const { open } = useProvenance(); | |
| 14 | + if (!provenance) return null; | |
| 15 | + const parts = [provenance.source_name, provenance.dataset].filter(Boolean).join(' — '); | |
| 16 | + const text = [parts, provenance.series_code, provenance.retrieved_at ? t('common.retrieved', { date: formatDate(provenance.retrieved_at) }) : null] | |
| 17 | + .filter(Boolean) | |
| 18 | + .join(' · '); | |
| 19 | + const inner = ( | |
| 20 | + <> | |
| 21 | + <span className="text-ink-3">{t('common.source')}: </span> | |
| 22 | + <span>{text}</span> | |
| 23 | + </> | |
| 24 | + ); | |
| 25 | + if (!payload) return <p className={`text-2xs text-ink-2 ${className ?? ''}`}>{inner}</p>; | |
| 26 | + return ( | |
| 27 | + <button | |
| 28 | + type="button" | |
| 29 | + onClick={() => open(payload)} | |
| 30 | + className={`group inline-flex min-h-[32px] max-w-full items-center gap-1 text-left text-2xs text-ink-2 hover:text-accent ${className ?? ''}`} | |
| 31 | + aria-label={t('common.openProvenance')} | |
| 32 | + > | |
| 33 | + <span className="truncate">{inner}</span> | |
| 34 | + <Info size={12} aria-hidden className="shrink-0 text-ink-3 group-hover:text-accent" /> | |
| 35 | + </button> | |
| 36 | + ); | |
| 37 | +} | |
added
apps/web/src/components/charts/sparkline.tsx
+52 −0
@@ -0,0 +1,52 @@ | ||
| 1 | +import { line as d3Line } from 'd3-shape'; | |
| 2 | +import { scaleLinear } from 'd3-scale'; | |
| 3 | +import { CHART, MARK } from './palette'; | |
| 4 | +import { extent, periodToX, splitForecast, type SeriesPoint } from './scales'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Tiny server-renderable line (no axes) with an end dot. Fixed viewBox so it scales with its box; | |
| 8 | + * strokes use `vector-effect: non-scaling-stroke` so lines stay 1.5 px at any size. | |
| 9 | + */ | |
| 10 | +export function Sparkline({ | |
| 11 | + points, | |
| 12 | + width = 96, | |
| 13 | + height = 28, | |
| 14 | + color = CHART.accent, | |
| 15 | + className, | |
| 16 | + ariaLabel, | |
| 17 | + direction, | |
| 18 | +}: { | |
| 19 | + points: SeriesPoint[]; | |
| 20 | + width?: number; | |
| 21 | + height?: number; | |
| 22 | + color?: string; | |
| 23 | + className?: string; | |
| 24 | + ariaLabel?: string; | |
| 25 | + /** Colour the end dot by direction (up/down) — the line stays neutral. */ | |
| 26 | + direction?: 'up' | 'down' | 'flat' | null; | |
| 27 | +}) { | |
| 28 | + const clean = points.filter((p) => p.value != null && Number.isFinite(p.value)); | |
| 29 | + if (clean.length < 2) return <span className={className} style={{ display: 'inline-block', width, height }} aria-hidden />; | |
| 30 | + const xs = clean.map(periodToX); | |
| 31 | + const xDom = extent(xs)!; | |
| 32 | + const yDom = extent(clean.map((p) => p.value))!; | |
| 33 | + const pad = 3; | |
| 34 | + const x = scaleLinear().domain(xDom).range([pad, width - pad]); | |
| 35 | + const y = scaleLinear() | |
| 36 | + .domain(yDom[0] === yDom[1] ? [yDom[0] - 1, yDom[1] + 1] : yDom) | |
| 37 | + .range([height - pad, pad]); | |
| 38 | + const gen = d3Line<SeriesPoint>() | |
| 39 | + .x((p) => x(periodToX(p))) | |
| 40 | + .y((p) => y(p.value!)); | |
| 41 | + const runs = splitForecast(clean); | |
| 42 | + const last = clean.filter((p) => !p.is_forecast).at(-1) ?? clean.at(-1)!; | |
| 43 | + const dotColor = direction === 'up' ? CHART.up : direction === 'down' ? CHART.down : color; | |
| 44 | + return ( | |
| 45 | + <svg className={className} width={width} height={height} viewBox={`0 0 ${width} ${height}`} role={ariaLabel ? 'img' : undefined} aria-label={ariaLabel} aria-hidden={ariaLabel ? undefined : true} preserveAspectRatio="none"> | |
| 46 | + {runs.map((r, i) => ( | |
| 47 | + <path key={i} d={gen(r.points) ?? ''} fill="none" stroke={r.forecast ? CHART.forecast : color} strokeWidth={1.5} strokeLinecap="round" strokeLinejoin="round" strokeDasharray={r.forecast ? '3 3' : undefined} vectorEffect="non-scaling-stroke" opacity={r.forecast ? 0.8 : 0.9} /> | |
| 48 | + ))} | |
| 49 | + <circle cx={x(periodToX(last))} cy={y(last.value!)} r={MARK.dotR - 1} fill={dotColor} stroke={CHART.surface} strokeWidth={1.5} vectorEffect="non-scaling-stroke" /> | |
| 50 | + </svg> | |
| 51 | + ); | |
| 52 | +} | |
added
apps/web/src/components/charts/summary.ts
+22 −0
@@ -0,0 +1,22 @@ | ||
| 1 | +import { t } from '@/i18n'; | |
| 2 | +import { formatValue } from '@/lib/format'; | |
| 3 | +import type { FormatSpec as Spec } from '@/lib/types'; | |
| 4 | +import { firstLast, type SeriesPoint } from './scales'; | |
| 5 | + | |
| 6 | +/** "Canada's GDP per capita rose from US$21.4k in 1990 to US$53.4k in 2024." */ | |
| 7 | +export function summarizeSeries(subject: string, points: SeriesPoint[], spec: Spec): string { | |
| 8 | + const fl = firstLast(points); | |
| 9 | + if (!fl) return t('chart.noData'); | |
| 10 | + const from = formatValue(fl.first.value, spec); | |
| 11 | + const to = formatValue(fl.last.value, spec); | |
| 12 | + const key = | |
| 13 | + fl.last.value! > fl.first.value! ? 'chart.summary.rose' : fl.last.value! < fl.first.value! ? 'chart.summary.fell' : 'chart.summary.flat'; | |
| 14 | + return t(key, { subject, from, y0: fl.first.year, to, y1: fl.last.year }); | |
| 15 | +} | |
| 16 | + | |
| 17 | +export function summarizeMulti(names: string[], allPoints: SeriesPoint[][]): string { | |
| 18 | + const years = allPoints.flat().map((p) => p.year); | |
| 19 | + const y0 = Math.min(...years); | |
| 20 | + const y1 = Math.max(...years); | |
| 21 | + return t('chart.summary.multi', { n: names.length, y0, y1, names: names.join(', ') }); | |
| 22 | +} | |
added
apps/web/src/components/charts/tooltip.tsx
+45 −0
@@ -0,0 +1,45 @@ | ||
| 1 | +'use client'; | |
| 2 | +import type { CSSProperties, ReactNode } from 'react'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Floating HTML tooltip anchored inside a `relative` chart container. Values lead (strong), labels follow. | |
| 6 | + * Pass `x`/`y` in container pixels; it flips to stay inside `width`. | |
| 7 | + */ | |
| 8 | +export function ChartTooltip({ | |
| 9 | + x, | |
| 10 | + y, | |
| 11 | + width, | |
| 12 | + children, | |
| 13 | +}: { | |
| 14 | + x: number; | |
| 15 | + y: number; | |
| 16 | + width: number; | |
| 17 | + children: ReactNode; | |
| 18 | +}) { | |
| 19 | + const flip = x > width * 0.6; | |
| 20 | + const style: CSSProperties = { | |
| 21 | + left: flip ? undefined : x + 10, | |
| 22 | + right: flip ? width - x + 10 : undefined, | |
| 23 | + top: Math.max(0, y - 8), | |
| 24 | + }; | |
| 25 | + return ( | |
| 26 | + <div | |
| 27 | + role="status" | |
| 28 | + aria-live="polite" | |
| 29 | + className="pointer-events-none absolute z-10 max-w-[min(260px,80vw)] rounded-sm border border-rule bg-surface px-2.5 py-1.5 text-xs shadow-pop tnum" | |
| 30 | + style={style} | |
| 31 | + > | |
| 32 | + {children} | |
| 33 | + </div> | |
| 34 | + ); | |
| 35 | +} | |
| 36 | + | |
| 37 | +export function TooltipRow({ color, label, value, muted }: { color?: string; label: string; value: string; muted?: boolean }) { | |
| 38 | + return ( | |
| 39 | + <div className="flex items-baseline gap-2 py-0.5"> | |
| 40 | + {color ? <span aria-hidden className="inline-block h-0.5 w-3 shrink-0 self-center rounded-full" style={{ background: color }} /> : null} | |
| 41 | + <span className={muted ? 'text-ink-3' : 'text-ink-2'}>{label}</span> | |
| 42 | + <span className="ml-auto font-semibold text-ink">{value}</span> | |
| 43 | + </div> | |
| 44 | + ); | |
| 45 | +} | |
added
apps/web/src/components/charts/use-measure.ts
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { useEffect, useRef, useState } from 'react'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Measure an element's width with ResizeObserver. Returns `defaultWidth` during SSR and the first client | |
| 6 | + * render so server and client markup match; the chart re-renders once the real width is known. | |
| 7 | + */ | |
| 8 | +export function useMeasure<T extends HTMLElement = HTMLDivElement>(defaultWidth = 640) { | |
| 9 | + const ref = useRef<T | null>(null); | |
| 10 | + const [width, setWidth] = useState(defaultWidth); | |
| 11 | + useEffect(() => { | |
| 12 | + const el = ref.current; | |
| 13 | + if (!el) return; | |
| 14 | + const apply = () => { | |
| 15 | + const w = Math.round(el.getBoundingClientRect().width); | |
| 16 | + if (w > 0) setWidth(w); | |
| 17 | + }; | |
| 18 | + apply(); | |
| 19 | + if (typeof ResizeObserver === 'undefined') return; | |
| 20 | + const ro = new ResizeObserver(() => apply()); | |
| 21 | + ro.observe(el); | |
| 22 | + return () => ro.disconnect(); | |
| 23 | + }, []); | |
| 24 | + return { ref, width } as const; | |
| 25 | +} | |
added
apps/web/src/components/layout/mobile-tab-bar.tsx
+46 −0
@@ -0,0 +1,46 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { BarChart3, Globe2, Home, Scale, Search } from 'lucide-react'; | |
| 3 | +import Link from 'next/link'; | |
| 4 | +import { usePathname } from 'next/navigation'; | |
| 5 | +import { t } from '@/i18n'; | |
| 6 | +import { cn } from '@/lib/cn'; | |
| 7 | +import { routes } from '@/lib/site'; | |
| 8 | +import { useOpenSearch } from './search-context'; | |
| 9 | + | |
| 10 | +/** | |
| 11 | + * Fixed bottom tab bar (< md): Home · Countries · Compare · Rankings · Search. Safe-area padding; the | |
| 12 | + * body reserves space (`pb-[calc(56px+env(safe-area-inset-bottom))]`) so it never covers the footer. | |
| 13 | + */ | |
| 14 | +export function MobileTabBar() { | |
| 15 | + const pathname = usePathname(); | |
| 16 | + const openSearch = useOpenSearch(); | |
| 17 | + const tabs = [ | |
| 18 | + { href: routes.home(), label: t('nav.home'), Icon: Home, exact: true }, | |
| 19 | + { href: routes.countries(), label: t('nav.countries'), Icon: Globe2 }, | |
| 20 | + { href: routes.compare(), label: t('nav.compare'), Icon: Scale }, | |
| 21 | + { href: routes.rankings(), label: t('nav.rankings'), Icon: BarChart3 }, | |
| 22 | + ]; | |
| 23 | + return ( | |
| 24 | + <nav aria-label={t('nav.primary')} className="fixed inset-x-0 bottom-0 z-30 border-t border-rule bg-paper/95 backdrop-blur supports-[backdrop-filter]:bg-paper/90 md:hidden safe-bottom"> | |
| 25 | + <ul className="grid h-14 grid-cols-5"> | |
| 26 | + {tabs.map(({ href, label, Icon, exact }) => { | |
| 27 | + const active = exact ? pathname === href : pathname === href || pathname.startsWith(`${href}/`); | |
| 28 | + return ( | |
| 29 | + <li key={href} className="min-w-0"> | |
| 30 | + <Link href={href} aria-current={active ? 'page' : undefined} className={cn('flex h-full flex-col items-center justify-center gap-0.5 text-2xs', active ? 'text-accent' : 'text-ink-2')}> | |
| 31 | + <Icon size={20} aria-hidden strokeWidth={active ? 2.25 : 1.75} /> | |
| 32 | + <span className="truncate">{label}</span> | |
| 33 | + </Link> | |
| 34 | + </li> | |
| 35 | + ); | |
| 36 | + })} | |
| 37 | + <li className="min-w-0"> | |
| 38 | + <button type="button" onClick={openSearch} className="flex h-full w-full flex-col items-center justify-center gap-0.5 text-2xs text-ink-2"> | |
| 39 | + <Search size={20} aria-hidden strokeWidth={1.75} /> | |
| 40 | + <span className="truncate">{t('nav.search')}</span> | |
| 41 | + </button> | |
| 42 | + </li> | |
| 43 | + </ul> | |
| 44 | + </nav> | |
| 45 | + ); | |
| 46 | +} | |
added
apps/web/src/components/layout/nav-links.tsx
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +'use client'; | |
| 2 | +import Link from 'next/link'; | |
| 3 | +import { usePathname } from 'next/navigation'; | |
| 4 | +import { t } from '@/i18n'; | |
| 5 | +import { cn } from '@/lib/cn'; | |
| 6 | + | |
| 7 | +export function NavLinks({ items }: { items: Array<{ href: string; label: string }> }) { | |
| 8 | + const pathname = usePathname(); | |
| 9 | + return ( | |
| 10 | + <nav aria-label={t('nav.primary')} className="hidden md:block"> | |
| 11 | + <ul className="flex items-center gap-0.5"> | |
| 12 | + {items.map((it) => { | |
| 13 | + const active = pathname === it.href || pathname.startsWith(`${it.href}/`); | |
| 14 | + return ( | |
| 15 | + <li key={it.href}> | |
| 16 | + <Link href={it.href} aria-current={active ? 'page' : undefined} className={cn('inline-flex h-9 items-center rounded-sm px-2.5 text-sm', active ? 'font-medium text-ink' : 'text-ink-2 hover:bg-surface-2 hover:text-ink')}> | |
| 17 | + {it.label} | |
| 18 | + </Link> | |
| 19 | + </li> | |
| 20 | + ); | |
| 21 | + })} | |
| 22 | + </ul> | |
| 23 | + </nav> | |
| 24 | + ); | |
| 25 | +} | |
added
apps/web/src/components/layout/search-container.tsx
+23 −0
@@ -0,0 +1,23 @@ | ||
| 1 | +'use client'; | |
| 2 | +import dynamic from 'next/dynamic'; | |
| 3 | +import { useSearch } from './search-context'; | |
| 4 | + | |
| 5 | +const SearchDialog = dynamic(() => import('./search-dialog').then((m) => m.SearchDialog), { ssr: false }); | |
| 6 | + | |
| 7 | +/** Loads the search dialog bundle only once the user opens it (or has opened it before in this session). */ | |
| 8 | +export function SearchContainer() { | |
| 9 | + const { open } = useSearch(); | |
| 10 | + const [everOpened, setEver] = useStateOnce(open); | |
| 11 | + if (!everOpened) return null; | |
| 12 | + void setEver; | |
| 13 | + return <SearchDialog />; | |
| 14 | +} | |
| 15 | + | |
| 16 | +import { useEffect, useState } from 'react'; | |
| 17 | +function useStateOnce(flag: boolean) { | |
| 18 | + const [v, setV] = useState(flag); | |
| 19 | + useEffect(() => { | |
| 20 | + if (flag) setV(true); | |
| 21 | + }, [flag]); | |
| 22 | + return [v, setV] as const; | |
| 23 | +} | |
added
apps/web/src/components/layout/search-context.tsx
+39 −0
@@ -0,0 +1,39 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { createContext, useCallback, useContext, useEffect, useMemo, useState, type ReactNode } from 'react'; | |
| 3 | + | |
| 4 | +interface Ctx { | |
| 5 | + open: boolean; | |
| 6 | + setOpen: (v: boolean) => void; | |
| 7 | +} | |
| 8 | +const SearchCtx = createContext<Ctx>({ open: false, setOpen: () => {} }); | |
| 9 | + | |
| 10 | +/** Global search state + ⌘K / Ctrl+K / "/" shortcuts. */ | |
| 11 | +export function SearchProvider({ children }: { children: ReactNode }) { | |
| 12 | + const [open, setOpen] = useState(false); | |
| 13 | + useEffect(() => { | |
| 14 | + const onKey = (e: KeyboardEvent) => { | |
| 15 | + const target = e.target as HTMLElement | null; | |
| 16 | + const typing = target && (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA' || target.isContentEditable); | |
| 17 | + if ((e.metaKey || e.ctrlKey) && e.key.toLowerCase() === 'k') { | |
| 18 | + e.preventDefault(); | |
| 19 | + setOpen((o) => !o); | |
| 20 | + } else if (e.key === '/' && !typing && !e.metaKey && !e.ctrlKey && !e.altKey) { | |
| 21 | + e.preventDefault(); | |
| 22 | + setOpen(true); | |
| 23 | + } | |
| 24 | + }; | |
| 25 | + window.addEventListener('keydown', onKey); | |
| 26 | + return () => window.removeEventListener('keydown', onKey); | |
| 27 | + }, []); | |
| 28 | + const value = useMemo(() => ({ open, setOpen }), [open]); | |
| 29 | + return <SearchCtx.Provider value={value}>{children}</SearchCtx.Provider>; | |
| 30 | +} | |
| 31 | + | |
| 32 | +export function useSearch() { | |
| 33 | + return useContext(SearchCtx); | |
| 34 | +} | |
| 35 | + | |
| 36 | +export function useOpenSearch() { | |
| 37 | + const { setOpen } = useSearch(); | |
| 38 | + return useCallback(() => setOpen(true), [setOpen]); | |
| 39 | +} | |
added
apps/web/src/components/layout/search-dialog.tsx
+224 −0
@@ -0,0 +1,224 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { Clock, CornerDownLeft, Search, X } from 'lucide-react'; | |
| 3 | +import { useRouter } from 'next/navigation'; | |
| 4 | +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; | |
| 5 | +import { t } from '@/i18n'; | |
| 6 | +import { clientApi } from '@/lib/client-api'; | |
| 7 | +import { cn } from '@/lib/cn'; | |
| 8 | +import { routes } from '@/lib/site'; | |
| 9 | +import type { SearchHit, SearchHitType } from '@/lib/types'; | |
| 10 | +import { BottomSheet } from '@/components/data/bottom-sheet'; | |
| 11 | +import { useSearch } from './search-context'; | |
| 12 | + | |
| 13 | +const RECENT_KEY = 'ca-recent-searches'; | |
| 14 | +const MAX_RECENT = 8; | |
| 15 | + | |
| 16 | +function hrefFor(h: SearchHit): string { | |
| 17 | + if (h.url) return h.url; | |
| 18 | + const slug = h.slug ?? h.id; | |
| 19 | + switch (h.type) { | |
| 20 | + case 'country': | |
| 21 | + return routes.country(slug); | |
| 22 | + case 'indicator': | |
| 23 | + return routes.indicator(slug); | |
| 24 | + case 'topic': | |
| 25 | + return routes.indicators(slug); | |
| 26 | + case 'region': | |
| 27 | + return routes.region(slug); | |
| 28 | + case 'source': | |
| 29 | + return routes.source(h.id); | |
| 30 | + default: | |
| 31 | + return '/'; | |
| 32 | + } | |
| 33 | +} | |
| 34 | + | |
| 35 | +function loadRecent(): SearchHit[] { | |
| 36 | + try { | |
| 37 | + const raw = localStorage.getItem(RECENT_KEY); | |
| 38 | + return raw ? (JSON.parse(raw) as SearchHit[]) : []; | |
| 39 | + } catch { | |
| 40 | + return []; | |
| 41 | + } | |
| 42 | +} | |
| 43 | +function saveRecent(h: SearchHit) { | |
| 44 | + try { | |
| 45 | + const cur = loadRecent().filter((x) => !(x.type === h.type && x.id === h.id)); | |
| 46 | + localStorage.setItem(RECENT_KEY, JSON.stringify([h, ...cur].slice(0, MAX_RECENT))); | |
| 47 | + } catch { | |
| 48 | + /* ignore */ | |
| 49 | + } | |
| 50 | +} | |
| 51 | + | |
| 52 | +const TYPE_ORDER: string[] = ['country', 'country_topic', 'country_indicator', 'indicator', 'topic', 'region', 'source']; | |
| 53 | + | |
| 54 | +/** | |
| 55 | + * Global search (⌘K / "/" / tab bar). Debounced `/api/v1/search?q=`, grouped hits with type chips, keyboard | |
| 56 | + * navigation, recent searches in localStorage. Full-screen sheet on mobile, centred panel on desktop. | |
| 57 | + */ | |
| 58 | +export function SearchDialog() { | |
| 59 | + const { open, setOpen } = useSearch(); | |
| 60 | + const router = useRouter(); | |
| 61 | + const [q, setQ] = useState(''); | |
| 62 | + const [hits, setHits] = useState<SearchHit[]>([]); | |
| 63 | + const [state, setState] = useState<'idle' | 'loading' | 'error'>('idle'); | |
| 64 | + const [recent, setRecent] = useState<SearchHit[]>([]); | |
| 65 | + const [active, setActive] = useState(0); | |
| 66 | + const inputRef = useRef<HTMLInputElement>(null); | |
| 67 | + const abortRef = useRef<AbortController | null>(null); | |
| 68 | + | |
| 69 | + useEffect(() => { | |
| 70 | + if (open) { | |
| 71 | + setRecent(loadRecent()); | |
| 72 | + setTimeout(() => inputRef.current?.focus(), 30); | |
| 73 | + } else { | |
| 74 | + setQ(''); | |
| 75 | + setHits([]); | |
| 76 | + setState('idle'); | |
| 77 | + setActive(0); | |
| 78 | + } | |
| 79 | + }, [open]); | |
| 80 | + | |
| 81 | + useEffect(() => { | |
| 82 | + const term = q.trim(); | |
| 83 | + abortRef.current?.abort(); | |
| 84 | + if (term.length < 1) { | |
| 85 | + setHits([]); | |
| 86 | + setState('idle'); | |
| 87 | + return; | |
| 88 | + } | |
| 89 | + const ctrl = new AbortController(); | |
| 90 | + abortRef.current = ctrl; | |
| 91 | + setState('loading'); | |
| 92 | + const timer = setTimeout(async () => { | |
| 93 | + try { | |
| 94 | + const res = await clientApi.search(term, 14, ctrl.signal); | |
| 95 | + if (ctrl.signal.aborted) return; | |
| 96 | + setHits(res.hits); | |
| 97 | + setActive(0); | |
| 98 | + setState('idle'); | |
| 99 | + } catch (e) { | |
| 100 | + if ((e as Error).name === 'AbortError') return; | |
| 101 | + setState('error'); | |
| 102 | + } | |
| 103 | + }, 160); | |
| 104 | + return () => clearTimeout(timer); | |
| 105 | + }, [q]); | |
| 106 | + | |
| 107 | + const list = useMemo(() => { | |
| 108 | + const src = q.trim() ? hits : recent; | |
| 109 | + return [...src].sort((a, b) => (q.trim() ? b.score - a.score : 0) || TYPE_ORDER.indexOf(a.type) - TYPE_ORDER.indexOf(b.type)); | |
| 110 | + }, [hits, recent, q]); | |
| 111 | + | |
| 112 | + const go = useCallback( | |
| 113 | + (h: SearchHit) => { | |
| 114 | + saveRecent(h); | |
| 115 | + setOpen(false); | |
| 116 | + router.push(hrefFor(h)); | |
| 117 | + }, | |
| 118 | + [router, setOpen], | |
| 119 | + ); | |
| 120 | + | |
| 121 | + const onKey = (e: React.KeyboardEvent) => { | |
| 122 | + if (e.key === 'ArrowDown') { | |
| 123 | + e.preventDefault(); | |
| 124 | + setActive((a) => Math.min(list.length - 1, a + 1)); | |
| 125 | + } else if (e.key === 'ArrowUp') { | |
| 126 | + e.preventDefault(); | |
| 127 | + setActive((a) => Math.max(0, a - 1)); | |
| 128 | + } else if (e.key === 'Enter') { | |
| 129 | + const h = list[active]; | |
| 130 | + if (h) go(h); | |
| 131 | + } | |
| 132 | + }; | |
| 133 | + | |
| 134 | + const clearRecent = () => { | |
| 135 | + try { | |
| 136 | + localStorage.removeItem(RECENT_KEY); | |
| 137 | + } catch { | |
| 138 | + /* ignore */ | |
| 139 | + } | |
| 140 | + setRecent([]); | |
| 141 | + }; | |
| 142 | + | |
| 143 | + return ( | |
| 144 | + <BottomSheet | |
| 145 | + open={open} | |
| 146 | + onClose={() => setOpen(false)} | |
| 147 | + side="full" | |
| 148 | + labelledBy="search-title" | |
| 149 | + title={ | |
| 150 | + <div className="relative flex items-center"> | |
| 151 | + <Search size={16} aria-hidden className="absolute left-2 text-ink-3" /> | |
| 152 | + <input | |
| 153 | + ref={inputRef} | |
| 154 | + id="search-title" | |
| 155 | + type="search" | |
| 156 | + value={q} | |
| 157 | + onChange={(e) => setQ(e.target.value)} | |
| 158 | + onKeyDown={onKey} | |
| 159 | + placeholder={t('search.placeholder')} | |
| 160 | + aria-label={t('nav.search')} | |
| 161 | + aria-controls="search-results" | |
| 162 | + aria-activedescendant={list[active] ? `hit-${list[active].type}-${list[active].id}` : undefined} | |
| 163 | + autoComplete="off" | |
| 164 | + enterKeyHint="go" | |
| 165 | + className="h-10 w-full rounded-sm border border-rule bg-paper pl-8 pr-8 text-base outline-none placeholder:text-ink-3 focus:border-accent" | |
| 166 | + /> | |
| 167 | + {q ? ( | |
| 168 | + <button type="button" onClick={() => setQ('')} className="absolute right-1 grid h-8 w-8 place-items-center text-ink-3 hover:text-ink" aria-label={t('search.clear')}> | |
| 169 | + <X size={14} aria-hidden /> | |
| 170 | + </button> | |
| 171 | + ) : null} | |
| 172 | + </div> | |
| 173 | + } | |
| 174 | + > | |
| 175 | + <div className="text-sm"> | |
| 176 | + {!q.trim() && recent.length > 0 ? ( | |
| 177 | + <div className="mb-1 flex items-center justify-between text-2xs text-ink-3"> | |
| 178 | + <span className="inline-flex items-center gap-1"> | |
| 179 | + <Clock size={11} aria-hidden /> {t('search.recent')} | |
| 180 | + </span> | |
| 181 | + <button type="button" onClick={clearRecent} className="min-h-[32px] px-1 hover:text-ink"> | |
| 182 | + {t('search.clear')} | |
| 183 | + </button> | |
| 184 | + </div> | |
| 185 | + ) : null} | |
| 186 | + {!q.trim() && recent.length === 0 ? <p className="py-6 text-center text-ink-3">{t('search.start')}</p> : null} | |
| 187 | + {state === 'error' ? <p className="py-6 text-center text-down">{t('search.error')}</p> : null} | |
| 188 | + {q.trim() && state !== 'error' && state !== 'loading' && list.length === 0 ? <p className="py-6 text-center text-ink-3">{t('search.empty', { q: q.trim() })}</p> : null} | |
| 189 | + <ul id="search-results" role="listbox" aria-label={t('nav.search')} className="divide-y divide-rule"> | |
| 190 | + {list.map((h, i) => ( | |
| 191 | + <li key={`${h.type}-${h.id}`} id={`hit-${h.type}-${h.id}`} role="option" aria-selected={i === active}> | |
| 192 | + <button | |
| 193 | + type="button" | |
| 194 | + onClick={() => go(h)} | |
| 195 | + onMouseEnter={() => setActive(i)} | |
| 196 | + className={cn('flex min-h-[48px] w-full items-center gap-3 rounded-sm px-2 py-2 text-left', i === active ? 'bg-surface-2' : 'hover:bg-surface-2/60')} | |
| 197 | + > | |
| 198 | + <span aria-hidden className="w-6 text-center text-lg leading-none"> | |
| 199 | + {h.country?.flag ?? ''} | |
| 200 | + </span> | |
| 201 | + <span className="min-w-0 flex-1"> | |
| 202 | + <span className="block truncate text-ink">{h.name}</span> | |
| 203 | + {h.hint ? <span className="block truncate text-xs text-ink-3">{h.hint}</span> : null} | |
| 204 | + </span> | |
| 205 | + <TypeChip type={h.type} /> | |
| 206 | + {i === active ? <CornerDownLeft size={14} aria-hidden className="hidden text-ink-3 md:block" /> : null} | |
| 207 | + </button> | |
| 208 | + </li> | |
| 209 | + ))} | |
| 210 | + </ul> | |
| 211 | + <p className="mt-3 hidden text-2xs text-ink-3 md:block">{t('search.hint')}</p> | |
| 212 | + <div aria-live="polite" className="sr-only"> | |
| 213 | + {state === 'loading' ? t('search.loading') : q.trim() ? t('search.results', { n: list.length }) : ''} | |
| 214 | + </div> | |
| 215 | + </div> | |
| 216 | + </BottomSheet> | |
| 217 | + ); | |
| 218 | +} | |
| 219 | + | |
| 220 | +const CHIP_TYPE: Record<string, SearchHitType> = { country: 'country', indicator: 'indicator', topic: 'topic', region: 'region', source: 'source', country_topic: 'topic', country_indicator: 'indicator' }; | |
| 221 | +export function TypeChip({ type }: { type: SearchHitType | string }) { | |
| 222 | + const key = CHIP_TYPE[type] ?? 'country'; | |
| 223 | + return <span className="shrink-0 rounded-xs border border-rule px-1.5 py-0.5 text-2xs uppercase tracking-wide text-ink-3">{t(`search.type.${key}` as 'search.type.country')}</span>; | |
| 224 | +} | |
added
apps/web/src/components/layout/search-trigger.tsx
+33 −0
@@ -0,0 +1,33 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { Search } from 'lucide-react'; | |
| 3 | +import { t } from '@/i18n'; | |
| 4 | +import { cn } from '@/lib/cn'; | |
| 5 | +import { useOpenSearch } from './search-context'; | |
| 6 | + | |
| 7 | +/** Header search field look-alike (button) — desktop shows the ⌘K hint, mobile an icon button. */ | |
| 8 | +export function SearchTrigger({ variant = 'field', className }: { variant?: 'field' | 'icon' | 'hero'; className?: string }) { | |
| 9 | + const open = useOpenSearch(); | |
| 10 | + if (variant === 'icon') { | |
| 11 | + return ( | |
| 12 | + <button type="button" onClick={open} className={cn('tap grid place-items-center rounded-sm text-ink-2 hover:bg-surface-2 hover:text-ink', className)} aria-label={t('search.open')}> | |
| 13 | + <Search size={18} aria-hidden /> | |
| 14 | + </button> | |
| 15 | + ); | |
| 16 | + } | |
| 17 | + if (variant === 'hero') { | |
| 18 | + return ( | |
| 19 | + <button type="button" onClick={open} className={cn('flex h-12 w-full items-center gap-3 rounded-md border border-rule-strong bg-surface px-4 text-left text-base text-ink-3 shadow-[0_1px_0_rgba(0,0,0,0.03)] hover:border-accent hover:text-ink-2', className)}> | |
| 20 | + <Search size={18} aria-hidden className="text-ink-3" /> | |
| 21 | + <span className="flex-1 truncate">{t('search.placeholder')}</span> | |
| 22 | + <kbd className="hidden rounded-xs border border-rule px-1.5 py-0.5 font-ui text-2xs text-ink-3 md:inline">{t('search.shortcut')}</kbd> | |
| 23 | + </button> | |
| 24 | + ); | |
| 25 | + } | |
| 26 | + return ( | |
| 27 | + <button type="button" onClick={open} className={cn('flex h-9 w-56 items-center gap-2 rounded-sm border border-rule bg-surface px-2.5 text-left text-sm text-ink-3 hover:border-rule-strong hover:text-ink-2 lg:w-72', className)} aria-label={t('search.open')}> | |
| 28 | + <Search size={15} aria-hidden /> | |
| 29 | + <span className="flex-1 truncate">{t('search.placeholder.short')}</span> | |
| 30 | + <kbd className="rounded-xs border border-rule px-1 py-px font-ui text-2xs">{t('search.shortcut')}</kbd> | |
| 31 | + </button> | |
| 32 | + ); | |
| 33 | +} | |
added
apps/web/src/components/layout/site-footer.tsx
+86 −0
@@ -0,0 +1,86 @@ | ||
| 1 | +import Link from 'next/link'; | |
| 2 | +import { t } from '@/i18n'; | |
| 3 | +import { formatDate } from '@/lib/format'; | |
| 4 | +import { routes } from '@/lib/site'; | |
| 5 | +import { Logo } from '@/components/brand/Logo'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Site footer: explore/reference link columns, licence note, credits ("Made by … · contact", "Hosted on MacLustr"). | |
| 9 | + * `builtAt`/`runId` (optional) come from the API `meta` of the page that renders it. Reuse `<FooterCredits />` | |
| 10 | + * on /sources and /api pages for the contact line. | |
| 11 | + */ | |
| 12 | +export function SiteFooter({ builtAt, runId }: { builtAt?: string | null; runId?: string | null }) { | |
| 13 | + const explore = [ | |
| 14 | + { href: routes.countries(), label: t('nav.countries') }, | |
| 15 | + { href: routes.compare(), label: t('nav.compare') }, | |
| 16 | + { href: routes.rankings(), label: t('nav.rankings') }, | |
| 17 | + { href: routes.indicators(), label: t('nav.indicators') }, | |
| 18 | + { href: routes.regions(), label: t('nav.regions') }, | |
| 19 | + { href: routes.changes(), label: t('site.footer.changes') }, | |
| 20 | + ]; | |
| 21 | + const reference = [ | |
| 22 | + { href: routes.sources(), label: t('site.footer.sources') }, | |
| 23 | + { href: routes.methodology(), label: t('site.footer.methodology') }, | |
| 24 | + { href: routes.api(), label: t('site.footer.api') }, | |
| 25 | + { href: routes.data(), label: t('site.footer.data') }, | |
| 26 | + ]; | |
| 27 | + return ( | |
| 28 | + <footer className="mt-16 border-t border-rule bg-surface-2/40 text-sm"> | |
| 29 | + <div className="container-x mx-auto max-w-[1400px] py-10"> | |
| 30 | + <div className="grid gap-8 md:grid-cols-[1.4fr_1fr_1fr]"> | |
| 31 | + <div className="max-w-md"> | |
| 32 | + <Logo variant="full" /> | |
| 33 | + <p className="mt-3 text-ink-2">{t('site.tagline')}</p> | |
| 34 | + <p className="mt-3 text-xs leading-relaxed text-ink-3">{t('site.licence')}</p> | |
| 35 | + {builtAt ? ( | |
| 36 | + <p className="tnum mt-3 text-xs text-ink-3"> | |
| 37 | + {t('site.footer.refreshed', { date: formatDate(builtAt) })} | |
| 38 | + {runId ? ` · ${t('site.footer.build', { run: runId })}` : ''} | |
| 39 | + </p> | |
| 40 | + ) : null} | |
| 41 | + </div> | |
| 42 | + <FooterColumn title={t('site.footer.explore')} items={explore} /> | |
| 43 | + <FooterColumn title={t('site.footer.reference')} items={reference} /> | |
| 44 | + </div> | |
| 45 | + <div className="mt-8 flex flex-col gap-2 border-t border-rule pt-5 text-xs text-ink-2 md:flex-row md:items-center md:justify-between"> | |
| 46 | + <FooterCredits /> | |
| 47 | + <p className="text-ink-3">{t('site.footer.made')}</p> | |
| 48 | + </div> | |
| 49 | + </div> | |
| 50 | + </footer> | |
| 51 | + ); | |
| 52 | +} | |
| 53 | + | |
| 54 | +function FooterColumn({ title, items }: { title: string; items: Array<{ href: string; label: string }> }) { | |
| 55 | + return ( | |
| 56 | + <div> | |
| 57 | + <div className="eyebrow mb-2">{title}</div> | |
| 58 | + <ul className="space-y-1"> | |
| 59 | + {items.map((it) => ( | |
| 60 | + <li key={it.href}> | |
| 61 | + <Link href={it.href} className="inline-flex min-h-[32px] items-center text-ink-2 hover:text-accent"> | |
| 62 | + {it.label} | |
| 63 | + </Link> | |
| 64 | + </li> | |
| 65 | + ))} | |
| 66 | + </ul> | |
| 67 | + </div> | |
| 68 | + ); | |
| 69 | +} | |
| 70 | + | |
| 71 | +/** "Made by Simon-Pierre Boucher · contact@spboucher.ai" + "Hosted on MacLustr". */ | |
| 72 | +export function FooterCredits({ className }: { className?: string }) { | |
| 73 | + return ( | |
| 74 | + <p className={className}> | |
| 75 | + {t('site.footer.madeBy')} <span className="text-ink">{t('site.footer.author')}</span> ·{' '} | |
| 76 | + <a href={`mailto:${t('site.footer.contact')}`} className="text-accent hover:underline"> | |
| 77 | + {t('site.footer.contact')} | |
| 78 | + </a> | |
| 79 | + <span className="mx-2 text-ink-3">·</span> | |
| 80 | + {t('site.footer.hostedOn')}{' '} | |
| 81 | + <a href="https://www.maclustr.io" target="_blank" rel="noopener noreferrer" className="text-accent hover:underline"> | |
| 82 | + {t('site.footer.host')} | |
| 83 | + </a> | |
| 84 | + </p> | |
| 85 | + ); | |
| 86 | +} | |
added
apps/web/src/components/layout/site-header.tsx
+39 −0
@@ -0,0 +1,39 @@ | ||
| 1 | +import Link from 'next/link'; | |
| 2 | +import { t } from '@/i18n'; | |
| 3 | +import { routes } from '@/lib/site'; | |
| 4 | +import { Logo } from '@/components/brand/Logo'; | |
| 5 | +import { NavLinks } from './nav-links'; | |
| 6 | +import { SearchTrigger } from './search-trigger'; | |
| 7 | +import { ThemeToggle } from './theme-toggle'; | |
| 8 | + | |
| 9 | +export const PRIMARY_NAV = [ | |
| 10 | + { key: 'nav.countries', href: routes.countries() }, | |
| 11 | + { key: 'nav.compare', href: routes.compare() }, | |
| 12 | + { key: 'nav.rankings', href: routes.rankings() }, | |
| 13 | + { key: 'nav.indicators', href: routes.indicators() }, | |
| 14 | + { key: 'nav.regions', href: routes.regions() }, | |
| 15 | + { key: 'nav.data', href: routes.data() }, | |
| 16 | +] as const; | |
| 17 | + | |
| 18 | +/** Sticky top bar: wordmark, primary nav (md+), search trigger, theme toggle. 52 px tall on mobile, 56 on md. */ | |
| 19 | +export function SiteHeader() { | |
| 20 | + return ( | |
| 21 | + <header className="sticky top-0 z-30 border-b border-rule bg-paper/95 backdrop-blur supports-[backdrop-filter]:bg-paper/85"> | |
| 22 | + <div className="container-x mx-auto flex h-[52px] max-w-[1400px] items-center gap-3 md:h-14"> | |
| 23 | + <Link href={routes.home()} className="flex h-11 items-center rounded-sm pr-1" aria-label={t('brand.home')}> | |
| 24 | + <Logo variant="full" /> | |
| 25 | + </Link> | |
| 26 | + <NavLinks items={PRIMARY_NAV.map((n) => ({ href: n.href, label: t(n.key) }))} /> | |
| 27 | + <div className="ml-auto flex items-center gap-1"> | |
| 28 | + <div className="hidden md:block"> | |
| 29 | + <SearchTrigger variant="field" /> | |
| 30 | + </div> | |
| 31 | + <div className="md:hidden"> | |
| 32 | + <SearchTrigger variant="icon" /> | |
| 33 | + </div> | |
| 34 | + <ThemeToggle /> | |
| 35 | + </div> | |
| 36 | + </div> | |
| 37 | + </header> | |
| 38 | + ); | |
| 39 | +} | |
added
apps/web/src/components/layout/theme-toggle.tsx
+41 −0
@@ -0,0 +1,41 @@ | ||
| 1 | +'use client'; | |
| 2 | +import { Moon, Sun } from 'lucide-react'; | |
| 3 | +import { useEffect, useState } from 'react'; | |
| 4 | +import { t } from '@/i18n'; | |
| 5 | + | |
| 6 | +export const THEME_KEY = 'ca-theme'; | |
| 7 | + | |
| 8 | +/** Applied before first paint by the inline script in layout.tsx; here we only toggle and persist. */ | |
| 9 | +export function ThemeToggle({ className }: { className?: string }) { | |
| 10 | + const [dark, setDark] = useState<boolean | null>(null); | |
| 11 | + useEffect(() => { | |
| 12 | + setDark(document.documentElement.classList.contains('dark')); | |
| 13 | + const mq = window.matchMedia('(prefers-color-scheme: dark)'); | |
| 14 | + const onChange = () => { | |
| 15 | + if (!localStorage.getItem(THEME_KEY)) { | |
| 16 | + document.documentElement.classList.toggle('dark', mq.matches); | |
| 17 | + setDark(mq.matches); | |
| 18 | + } | |
| 19 | + }; | |
| 20 | + mq.addEventListener('change', onChange); | |
| 21 | + return () => mq.removeEventListener('change', onChange); | |
| 22 | + }, []); | |
| 23 | + const toggle = () => { | |
| 24 | + const next = !document.documentElement.classList.contains('dark'); | |
| 25 | + document.documentElement.classList.toggle('dark', next); | |
| 26 | + try { | |
| 27 | + localStorage.setItem(THEME_KEY, next ? 'dark' : 'light'); | |
| 28 | + } catch { | |
| 29 | + /* private mode */ | |
| 30 | + } | |
| 31 | + setDark(next); | |
| 32 | + }; | |
| 33 | + return ( | |
| 34 | + <button type="button" onClick={toggle} className={`tap grid place-items-center rounded-sm text-ink-2 hover:bg-surface-2 hover:text-ink ${className ?? ''}`} aria-label={t('theme.toggle')} aria-pressed={dark ?? undefined} title={dark ? t('theme.light') : t('theme.dark')}> | |
| 35 | + {dark ? <Sun size={18} aria-hidden /> : <Moon size={18} aria-hidden />} | |
| 36 | + </button> | |
| 37 | + ); | |
| 38 | +} | |
| 39 | + | |
| 40 | +/** Inline bootstrap: apply stored/system theme before paint (no flash). */ | |
| 41 | +export const THEME_BOOT = `try{var k='${THEME_KEY}',s=localStorage.getItem(k),d=s?s==='dark':matchMedia('(prefers-color-scheme: dark)').matches;if(d)document.documentElement.classList.add('dark')}catch(e){}`; | |
added
apps/web/src/i18n/en.ts
+368 −0
@@ -0,0 +1,368 @@ | ||
| 1 | +/** | |
| 2 | + * English UI dictionary. Every visible string in a component goes through `t()` (src/i18n/index.ts). | |
| 3 | + * Placeholders use `{name}`. Keys are flat, dot-namespaced; add new sections at the end of the block | |
| 4 | + * they belong to so diffs stay readable. | |
| 5 | + */ | |
| 6 | +export const en = { | |
| 7 | + // --- site | |
| 8 | + 'site.name': 'CountryAtlas', | |
| 9 | + 'site.tagline': 'Understand the world, one country at a time.', | |
| 10 | + 'site.description': | |
| 11 | + 'Country-level data and statistics for 218 countries and territories: economy, population, health, energy, climate and more, every number traceable to its source.', | |
| 12 | + 'site.skip': 'Skip to content', | |
| 13 | + 'site.licence': | |
| 14 | + 'Data: World Bank, IMF, OECD, Eurostat, WHO, Our World in Data, BIS, ILO — CC BY 4.0 where applicable. Each value links to its source.', | |
| 15 | + 'site.footer.made': 'CountryAtlas is an independent, non-commercial data project.', | |
| 16 | + 'site.footer.sources': 'Sources', | |
| 17 | + 'site.footer.methodology': 'Methodology', | |
| 18 | + 'site.footer.api': 'API', | |
| 19 | + 'site.footer.data': 'Data downloads', | |
| 20 | + 'site.footer.about': 'About', | |
| 21 | + 'site.footer.explore': 'Explore', | |
| 22 | + 'site.footer.reference': 'Reference', | |
| 23 | + 'site.footer.refreshed': 'Data refreshed {date}', | |
| 24 | + 'site.footer.build': 'Snapshot {run}', | |
| 25 | + 'site.footer.madeBy': 'Made by', | |
| 26 | + 'site.footer.author': 'Simon-Pierre Boucher', | |
| 27 | + 'site.footer.contact': 'contact@spboucher.ai', | |
| 28 | + 'site.footer.hostedOn': 'Hosted on', | |
| 29 | + 'site.footer.host': 'MacLustr', | |
| 30 | + 'site.footer.credits': 'Credits', | |
| 31 | + 'site.footer.changes': 'Recent changes', | |
| 32 | + 'site.footer.explorePage': 'Explore', | |
| 33 | + 'brand.logoAlt': 'CountryAtlas logo', | |
| 34 | + 'brand.home': 'CountryAtlas — home', | |
| 35 | + | |
| 36 | + // --- navigation | |
| 37 | + 'nav.home': 'Home', | |
| 38 | + 'nav.countries': 'Countries', | |
| 39 | + 'nav.compare': 'Compare', | |
| 40 | + 'nav.rankings': 'Rankings', | |
| 41 | + 'nav.indicators': 'Indicators', | |
| 42 | + 'nav.regions': 'Regions', | |
| 43 | + 'nav.data': 'Data', | |
| 44 | + 'nav.search': 'Search', | |
| 45 | + 'nav.menu': 'Menu', | |
| 46 | + 'nav.primary': 'Primary', | |
| 47 | + 'nav.close': 'Close', | |
| 48 | + | |
| 49 | + // --- theme | |
| 50 | + 'theme.toggle': 'Toggle colour theme', | |
| 51 | + 'theme.light': 'Light', | |
| 52 | + 'theme.dark': 'Dark', | |
| 53 | + 'theme.system': 'System', | |
| 54 | + | |
| 55 | + // --- search | |
| 56 | + 'search.placeholder': 'Search countries, indicators, topics…', | |
| 57 | + 'search.placeholder.short': 'Search…', | |
| 58 | + 'search.hint': 'Type to search. ↑↓ to navigate, ↵ to open, esc to close.', | |
| 59 | + 'search.recent': 'Recent', | |
| 60 | + 'search.clear': 'Clear', | |
| 61 | + 'search.empty': 'No results for “{q}”.', | |
| 62 | + 'search.error': 'Search is unavailable right now.', | |
| 63 | + 'search.loading': 'Searching…', | |
| 64 | + 'search.shortcut': '⌘K', | |
| 65 | + 'search.open': 'Open search', | |
| 66 | + 'search.type.country': 'Country', | |
| 67 | + 'search.type.indicator': 'Indicator', | |
| 68 | + 'search.type.topic': 'Topic', | |
| 69 | + 'search.type.region': 'Region', | |
| 70 | + 'search.type.source': 'Source', | |
| 71 | + 'search.results': '{n} results', | |
| 72 | + 'search.start': 'Start typing to search 218 countries and 260 indicators.', | |
| 73 | + | |
| 74 | + // --- common | |
| 75 | + 'common.noData': 'No data', | |
| 76 | + 'common.noDataLong': 'No data available', | |
| 77 | + 'common.loading': 'Loading…', | |
| 78 | + 'common.seeAll': 'See all', | |
| 79 | + 'common.seeFullRanking': 'See full ranking', | |
| 80 | + 'common.more': 'More', | |
| 81 | + 'common.less': 'Less', | |
| 82 | + 'common.compare': 'Compare', | |
| 83 | + 'common.ranking': 'Ranking', | |
| 84 | + 'common.download': 'Download', | |
| 85 | + 'common.downloadCsv': 'Download CSV', | |
| 86 | + 'common.share': 'Share', | |
| 87 | + 'common.copied': 'Link copied', | |
| 88 | + 'common.copyLink': 'Copy link', | |
| 89 | + 'common.close': 'Close', | |
| 90 | + 'common.of': 'of', | |
| 91 | + 'common.and': 'and', | |
| 92 | + 'common.vs': 'vs {period}', | |
| 93 | + 'common.source': 'Source', | |
| 94 | + 'common.retrieved': 'retrieved {date}', | |
| 95 | + 'common.forecast': 'forecast', | |
| 96 | + 'common.estimate': 'estimate', | |
| 97 | + 'common.viewTable': 'View data as table', | |
| 98 | + 'common.hideTable': 'Hide data table', | |
| 99 | + 'common.showMore': 'Show {n} more', | |
| 100 | + 'common.back': 'Back', | |
| 101 | + 'common.year': 'Year', | |
| 102 | + 'common.value': 'Value', | |
| 103 | + 'common.country': 'Country', | |
| 104 | + 'common.indicator': 'Indicator', | |
| 105 | + 'common.period': 'Period', | |
| 106 | + 'common.change': 'Change', | |
| 107 | + 'common.rank': 'Rank', | |
| 108 | + 'common.world': 'World', | |
| 109 | + 'common.region': 'Region', | |
| 110 | + 'common.income': 'Income group', | |
| 111 | + 'common.all': 'All', | |
| 112 | + 'common.filters': 'Filters', | |
| 113 | + 'common.reset': 'Reset', | |
| 114 | + 'common.apply': 'Apply', | |
| 115 | + 'common.sortBy': 'Sort by', | |
| 116 | + 'common.name': 'Name', | |
| 117 | + 'common.population': 'Population', | |
| 118 | + 'common.gdpPerCapita': 'GDP per capita', | |
| 119 | + 'common.gdp': 'GDP', | |
| 120 | + 'common.coverage': 'Coverage', | |
| 121 | + 'common.na': '—', | |
| 122 | + 'common.up': 'up', | |
| 123 | + 'common.down': 'down', | |
| 124 | + 'common.flat': 'unchanged', | |
| 125 | + 'common.log': 'Log scale', | |
| 126 | + 'common.linear': 'Linear', | |
| 127 | + 'common.legend': 'Legend', | |
| 128 | + 'common.why': 'Why?', | |
| 129 | + 'common.openProvenance': 'Show source and provenance', | |
| 130 | + 'common.new': 'New', | |
| 131 | + 'common.updated': 'Updated', | |
| 132 | + 'common.notBuilt': 'Data is being prepared', | |
| 133 | + 'common.notBuiltHint': 'The first data snapshot is still being built. This page will fill in automatically once it is ready.', | |
| 134 | + 'common.errorTitle': 'Something went wrong', | |
| 135 | + 'common.errorHint': 'The data service did not respond. Try again in a moment.', | |
| 136 | + 'common.retry': 'Try again', | |
| 137 | + 'common.notFound': 'Page not found', | |
| 138 | + 'common.notFoundHint': 'We could not find that page. Try the search, or start from the list of countries.', | |
| 139 | + 'common.goHome': 'Go to the home page', | |
| 140 | + 'common.browseCountries': 'Browse all countries', | |
| 141 | + 'common.freshness.fresh': 'Fresh', | |
| 142 | + 'common.freshness.recent': 'Recent', | |
| 143 | + 'common.freshness.stale': 'Stale', | |
| 144 | + 'common.freshness.updated': 'Updated {when}', | |
| 145 | + 'common.status.verified': 'Verified', | |
| 146 | + 'common.status.imported': 'Imported', | |
| 147 | + 'common.status.warning': 'Flagged', | |
| 148 | + 'common.status.stale': 'Stale', | |
| 149 | + 'common.status.quarantined': 'Quarantined', | |
| 150 | + 'common.byline': 'CountryAtlas', | |
| 151 | + 'common.ago.now': 'just now', | |
| 152 | + 'common.ago.minutes': '{n} min ago', | |
| 153 | + 'common.ago.hours': '{n} h ago', | |
| 154 | + 'common.ago.days': '{n} d ago', | |
| 155 | + 'common.ago.months': '{n} mo ago', | |
| 156 | + 'common.ago.years': '{n} y ago', | |
| 157 | + | |
| 158 | + // --- provenance | |
| 159 | + 'prov.title': 'Provenance', | |
| 160 | + 'prov.source': 'Source', | |
| 161 | + 'prov.dataset': 'Dataset', | |
| 162 | + 'prov.series': 'Series code', | |
| 163 | + 'prov.observation': 'Observation date', | |
| 164 | + 'prov.retrieved': 'Retrieved', | |
| 165 | + 'prov.sourceUpdated': 'Source updated', | |
| 166 | + 'prov.unit': 'Unit', | |
| 167 | + 'prov.transform': 'Transformation', | |
| 168 | + 'prov.methodology': 'Methodology', | |
| 169 | + 'prov.licence': 'Licence', | |
| 170 | + 'prov.status': 'Status', | |
| 171 | + 'prov.openSource': 'Open at source', | |
| 172 | + 'prov.none': 'No transformation', | |
| 173 | + 'prov.indicatorPage': 'Indicator page', | |
| 174 | + 'prov.value': 'Value', | |
| 175 | + | |
| 176 | + // --- metric | |
| 177 | + 'metric.rankWorld': '{rank} of {n}', | |
| 178 | + 'metric.rankRegion': '{rank} in {region}', | |
| 179 | + 'metric.rankYearNote': 'Ranked among {year} values', | |
| 180 | + 'metric.change': '{change} vs {period}', | |
| 181 | + 'metric.history': 'History', | |
| 182 | + 'metric.open': 'Open {name}', | |
| 183 | + | |
| 184 | + // --- chart | |
| 185 | + 'chart.summary.rose': '{subject} rose from {from} in {y0} to {to} in {y1}.', | |
| 186 | + 'chart.summary.fell': '{subject} fell from {from} in {y0} to {to} in {y1}.', | |
| 187 | + 'chart.summary.flat': '{subject} was {from} in {y0} and {to} in {y1}.', | |
| 188 | + 'chart.summary.multi': '{n} series from {y0} to {y1}: {names}.', | |
| 189 | + 'chart.summary.bars': 'Ranked bars: {top} leads with {value}; {n} entries.', | |
| 190 | + 'chart.summary.map': 'World map of {name} ({year}) for {n} countries, from {min} to {max}.', | |
| 191 | + 'chart.summary.dna': 'Country DNA fingerprint of {name}: {dims}.', | |
| 192 | + 'chart.summary.pyramid': 'Population by age group for {name}: {parts}.', | |
| 193 | + 'chart.summary.scatter': 'Scatter of {y} against {x} for {n} countries.', | |
| 194 | + 'chart.summary.slope': 'Change from {y0} to {y1} for {n} entries.', | |
| 195 | + 'chart.noData': 'No data to chart.', | |
| 196 | + 'chart.forecastNote': 'Dashed segments are projections.', | |
| 197 | + 'chart.tooltipForecast': 'Projection', | |
| 198 | + 'chart.legend.noData': 'No data', | |
| 199 | + 'chart.map.tapHint': 'Tap a country for its value', | |
| 200 | + 'chart.map.hoverHint': 'Hover a country for its value; click to open it', | |
| 201 | + 'chart.map.legend': '{name}, {year}', | |
| 202 | + 'chart.table.caption': 'Data table for the chart above', | |
| 203 | + 'chart.download': 'Download this series', | |
| 204 | + 'chart.axisYear': 'Year', | |
| 205 | + | |
| 206 | + // --- home | |
| 207 | + 'home.hero.title': 'Understand the world, one country at a time.', | |
| 208 | + 'home.hero.sub': | |
| 209 | + 'Compare 218 countries and territories across 260 indicators — with every number traceable to its source.', | |
| 210 | + 'home.snapshot.title': 'Global snapshot', | |
| 211 | + 'home.snapshot.population': 'World population', | |
| 212 | + 'home.snapshot.gdp': 'World GDP', | |
| 213 | + 'home.snapshot.lifeExpectancy': 'Median life expectancy', | |
| 214 | + 'home.snapshot.countries': 'Countries & territories', | |
| 215 | + 'home.snapshot.indicators': 'Indicators', | |
| 216 | + 'home.snapshot.observations': 'Observations', | |
| 217 | + 'home.snapshot.refreshed': 'Refreshed {date}', | |
| 218 | + 'home.explore.title': 'Explore countries', | |
| 219 | + 'home.explore.sub': 'Pick a region, or start from the map.', | |
| 220 | + 'home.explore.mapTitle': 'GDP per capita, PPP', | |
| 221 | + 'home.explore.allCountries': 'All countries', | |
| 222 | + 'home.compare.title': 'Compare countries', | |
| 223 | + 'home.compare.sub': 'Side by side across every indicator.', | |
| 224 | + 'home.compare.custom': 'Build your own comparison', | |
| 225 | + 'home.topics.title': 'Major topics', | |
| 226 | + 'home.topics.sub': '19 topics, 260 indicators.', | |
| 227 | + 'home.featured.title': 'Featured indicators', | |
| 228 | + 'home.featured.sub': 'Global trends and this year’s leaders.', | |
| 229 | + 'home.changes.title': 'Biggest recent changes', | |
| 230 | + 'home.changes.sub': 'Detected automatically from the latest data — no editorial picks.', | |
| 231 | + 'home.changes.all': 'All recent changes', | |
| 232 | + 'home.rankings.economies': 'Largest economies', | |
| 233 | + 'home.rankings.growth': 'Fastest population growth', | |
| 234 | + 'home.rankings.life': 'Highest life expectancy', | |
| 235 | + 'home.rankings.energy': 'Energy transition leaders', | |
| 236 | + 'home.updated.title': 'Recently updated data', | |
| 237 | + 'home.updated.sub': 'Datasets refreshed in the latest snapshot.', | |
| 238 | + 'home.notBuilt': 'The first data snapshot is being built. Countries, rankings and charts appear here as soon as it is ready.', | |
| 239 | + | |
| 240 | + // --- countries directory | |
| 241 | + 'countries.title': 'Countries', | |
| 242 | + 'countries.sub': '{n} countries and territories. Filter by region or income group, sort, or jump by letter.', | |
| 243 | + 'countries.search': 'Filter by name…', | |
| 244 | + 'countries.count': '{n} of {total}', | |
| 245 | + 'countries.noMatch': 'No country matches these filters.', | |
| 246 | + 'countries.sort.name': 'Name', | |
| 247 | + 'countries.sort.population': 'Population', | |
| 248 | + 'countries.sort.gdp': 'GDP per capita', | |
| 249 | + 'countries.sort.coverage': 'Coverage', | |
| 250 | + 'countries.letterIndex': 'Jump to letter', | |
| 251 | + 'countries.territory': 'Territory', | |
| 252 | + 'countries.filters': 'Filters', | |
| 253 | + 'countries.filtersActive': '{n} active', | |
| 254 | + 'countries.populationTrend': 'Population trend', | |
| 255 | + | |
| 256 | + // --- country page | |
| 257 | + 'country.title': '{name} Data & Statistics', | |
| 258 | + 'country.description': | |
| 259 | + '{name}: population, GDP, growth, inflation, unemployment, life expectancy, energy, climate and 250+ more indicators with sources.', | |
| 260 | + 'country.capital': 'Capital', | |
| 261 | + 'country.region': 'Region', | |
| 262 | + 'country.incomeGroup': 'Income group', | |
| 263 | + 'country.population': 'Population', | |
| 264 | + 'country.area': 'Area', | |
| 265 | + 'country.currency': 'Currency', | |
| 266 | + 'country.refreshed': 'Data refreshed {date}', | |
| 267 | + 'country.coverage': '{pct} coverage · {n} indicators', | |
| 268 | + 'country.compare': 'Compare', | |
| 269 | + 'country.compareWith': 'Compare {name} with another country', | |
| 270 | + 'country.download': 'Download', | |
| 271 | + 'country.downloadHint': 'Full dataset (CSV)', | |
| 272 | + 'country.share': 'Share', | |
| 273 | + 'country.headline.title': 'At a glance', | |
| 274 | + 'country.headline.sub': 'Twelve headline indicators, latest available year.', | |
| 275 | + 'country.changes.title': 'What changed in {name}', | |
| 276 | + 'country.changes.sub': 'Statistically notable movements in the latest data.', | |
| 277 | + 'country.changes.none': 'No notable changes detected in the latest snapshot.', | |
| 278 | + 'country.similar.title': 'Countries similar to {name}', | |
| 279 | + 'country.similar.sub': 'Nearest peers by standardised distance on the selected dimension.', | |
| 280 | + 'country.similar.none': 'Not enough data to compute peers.', | |
| 281 | + 'country.similar.mode.overall': 'Overall', | |
| 282 | + 'country.similar.mode.economic': 'Economic', | |
| 283 | + 'country.similar.mode.demographic': 'Demographic', | |
| 284 | + 'country.similar.mode.energy': 'Energy', | |
| 285 | + 'country.similar.mode.social': 'Social', | |
| 286 | + 'country.similar.score': 'Similarity {score}', | |
| 287 | + 'country.similar.why': 'Why these are similar', | |
| 288 | + 'country.similar.contrib': '{indicator}: {a} vs {b}', | |
| 289 | + 'country.dna.title': 'Country DNA', | |
| 290 | + 'country.dna.sub': 'Percentile rank among all countries on nine dimensions. Descriptive, not a score.', | |
| 291 | + 'country.dna.none': 'DNA not available for this country yet.', | |
| 292 | + 'country.facts.title': 'Key facts', | |
| 293 | + 'country.facts.sub': 'Computed from the data — no model-generated numbers.', | |
| 294 | + 'country.facts.none': 'No verified insights yet.', | |
| 295 | + 'country.timeline.title': 'Timeline', | |
| 296 | + 'country.timeline.sub': 'Records, reversals and turning points across the whole history.', | |
| 297 | + 'country.timeline.none': 'No events detected.', | |
| 298 | + 'country.topics.title': 'Explore {name} by topic', | |
| 299 | + 'country.topics.sub': '{n} topics · {m} indicators with data.', | |
| 300 | + 'country.topics.count': '{n} indicators', | |
| 301 | + 'country.topics.withData': '{n} with data', | |
| 302 | + 'country.landlocked': 'Landlocked', | |
| 303 | + 'country.unMember': 'UN member', | |
| 304 | + 'country.borders': 'Borders', | |
| 305 | + 'country.languages': 'Languages', | |
| 306 | + 'country.officialName': 'Official name', | |
| 307 | + 'country.similar.peerOf': 'Peers of {name}', | |
| 308 | + 'country.notFound': 'Unknown country', | |
| 309 | + 'country.notFoundHint': 'No country or territory matches “{slug}”. Check the spelling or browse the directory.', | |
| 310 | + 'country.severity': 'Severity {n}', | |
| 311 | + 'country.dna.income': 'Income', | |
| 312 | + 'country.dna.demographics': 'Demographics', | |
| 313 | + 'country.dna.urbanization': 'Urbanisation', | |
| 314 | + 'country.dna.trade': 'Trade', | |
| 315 | + 'country.dna.energy': 'Energy', | |
| 316 | + 'country.dna.emissions': 'Emissions', | |
| 317 | + 'country.dna.innovation': 'Innovation', | |
| 318 | + 'country.dna.education': 'Education', | |
| 319 | + 'country.dna.public_spending': 'Public spending', | |
| 320 | + 'country.overview': 'Overview', | |
| 321 | + | |
| 322 | + // --- topic page | |
| 323 | + 'topic.title': '{country} — {topic}', | |
| 324 | + 'topic.description': '{topic} indicators for {country}: {list} and more, with full history, ranks and sources.', | |
| 325 | + 'topic.noData': 'No data available ({n})', | |
| 326 | + 'topic.noDataHint': 'These indicators exist in the registry but no source reports a value for {country}.', | |
| 327 | + 'topic.indicators': '{n} indicators', | |
| 328 | + 'topic.withData': '{n} with data', | |
| 329 | + 'topic.latest': 'Latest', | |
| 330 | + 'topic.rankIn': 'Rank', | |
| 331 | + 'topic.history': 'Full history', | |
| 332 | + 'topic.compareLink': 'Compare', | |
| 333 | + 'topic.rankingLink': 'Ranking', | |
| 334 | + 'topic.indicatorLink': 'Indicator page', | |
| 335 | + 'topic.otherTopics': 'Other topics', | |
| 336 | + 'topic.notFound': 'Unknown topic', | |
| 337 | + 'topic.overview': 'Overview', | |
| 338 | + | |
| 339 | + // --- empty state | |
| 340 | + 'empty.title': 'No data for {indicator} in {country}.', | |
| 341 | + 'empty.sourcesChecked': 'Sources checked: {sources}.', | |
| 342 | + 'empty.generic': 'Nothing to show here yet.', | |
| 343 | + | |
| 344 | + // --- data table | |
| 345 | + 'table.definitionList': 'Table shown as a list on small screens', | |
| 346 | + 'table.sortAsc': 'Sorted ascending', | |
| 347 | + 'table.sortDesc': 'Sorted descending', | |
| 348 | + | |
| 349 | + // --- changes / severity | |
| 350 | + 'change.kind.yoy_drop': 'Sharp drop', | |
| 351 | + 'change.kind.yoy_jump': 'Sharp rise', | |
| 352 | + 'change.kind.record_high': 'Record high', | |
| 353 | + 'change.kind.record_low': 'Record low', | |
| 354 | + 'change.kind.n_year_high': '{n}-year high', | |
| 355 | + 'change.kind.n_year_low': '{n}-year low', | |
| 356 | + 'change.kind.sign_flip': 'Reversal', | |
| 357 | + 'change.kind.accelerating': 'Accelerating', | |
| 358 | + 'change.kind.decelerating': 'Decelerating', | |
| 359 | + 'change.severity.high': 'Major', | |
| 360 | + 'change.severity.medium': 'Notable', | |
| 361 | + 'change.severity.low': 'Minor', | |
| 362 | + | |
| 363 | + // --- opengraph | |
| 364 | + 'og.alt': '{name} — key statistics on CountryAtlas', | |
| 365 | + 'og.site': 'countryatlas.co', | |
| 366 | +} as const; | |
| 367 | + | |
| 368 | +export type DictKey = keyof typeof en; | |
added
apps/web/src/i18n/index.ts
+28 −0
@@ -0,0 +1,28 @@ | ||
| 1 | +import { en, type DictKey } from './en'; | |
| 2 | + | |
| 3 | +/** Fixed locale for every Intl call, server and client, so hydration never diverges. */ | |
| 4 | +export const LOCALE = 'en-US' as const; | |
| 5 | + | |
| 6 | +type Params = Record<string, string | number | null | undefined>; | |
| 7 | + | |
| 8 | +/** | |
| 9 | + * Translate a dictionary key, interpolating `{name}` placeholders. | |
| 10 | + * Usage: `t('metric.rankWorld', { rank: '12th', n: 190 })`. | |
| 11 | + * Missing keys return the key itself so a typo is visible rather than silent. | |
| 12 | + */ | |
| 13 | +export function t(key: DictKey, params?: Params): string { | |
| 14 | + const raw: string = en[key] ?? key; | |
| 15 | + if (!params) return raw; | |
| 16 | + return raw.replace(/\{(\w+)\}/g, (_, k: string) => { | |
| 17 | + const v = params[k]; | |
| 18 | + return v == null ? '' : String(v); | |
| 19 | + }); | |
| 20 | +} | |
| 21 | + | |
| 22 | +/** Optional-key variant for dynamic keys (e.g. `change.kind.${kind}`); returns `fallback` when the key is unknown. */ | |
| 23 | +export function tOpt(key: string, fallback: string, params?: Params): string { | |
| 24 | + if (key in en) return t(key as DictKey, params); | |
| 25 | + return fallback; | |
| 26 | +} | |
| 27 | + | |
| 28 | +export type { DictKey }; | |
added
apps/web/src/lib/api.ts
+155 −0
@@ -0,0 +1,155 @@ | ||
| 1 | +import 'server-only'; | |
| 2 | +import type { | |
| 3 | + ChangesResponse, | |
| 4 | + CountriesResponse, | |
| 5 | + CountryResponse, | |
| 6 | + CountryTopicResponse, | |
| 7 | + DNAResponse, | |
| 8 | + HealthResponse, | |
| 9 | + HomeResponse, | |
| 10 | + InsightsResponse, | |
| 11 | + MapResponse, | |
| 12 | + Problem, | |
| 13 | + RankingResponse, | |
| 14 | + SearchResponse, | |
| 15 | + SeriesResponse, | |
| 16 | + SimilarResponse, | |
| 17 | + SimilarityMode, | |
| 18 | +} from './types'; | |
| 19 | + | |
| 20 | +/** | |
| 21 | + * Typed fetch wrapper for the CountryAtlas API (server components only — client code calls the same-origin | |
| 22 | + * `/api/v1/*` rewrite through `src/lib/client-api.ts`). | |
| 23 | + * | |
| 24 | + * - default cache: ISR `next: { revalidate: 900 }` (15 min); search is `no-store`. | |
| 25 | + * - non-2xx → `ApiError` (status + RFC 7807 problem body when available); network failure → status 0. | |
| 26 | + * - 503 "Data not built yet" (no snapshot) → `ApiError.notBuilt`; pages render <NotBuiltState/> — never crash. | |
| 27 | + * - `safe(promise)` turns any error into `null` for the optional panels fetched in parallel with Promise.all. | |
| 28 | + */ | |
| 29 | + | |
| 30 | +export const API_URL = process.env.API_URL ?? 'http://127.0.0.1:8291'; | |
| 31 | +const BASE = `${API_URL.replace(/\/$/, '')}/api/v1`; | |
| 32 | + | |
| 33 | +export class ApiError extends Error { | |
| 34 | + readonly status: number; | |
| 35 | + readonly problem: Problem | null; | |
| 36 | + readonly path: string; | |
| 37 | + constructor(status: number, path: string, problem: Problem | null, message?: string) { | |
| 38 | + super(message ?? problem?.detail ?? problem?.title ?? `API ${status} on ${path}`); | |
| 39 | + this.name = 'ApiError'; | |
| 40 | + this.status = status; | |
| 41 | + this.problem = problem; | |
| 42 | + this.path = path; | |
| 43 | + } | |
| 44 | + /** The snapshot does not exist yet (fresh deployment, pipeline still running) — or the API is down. */ | |
| 45 | + get notBuilt(): boolean { | |
| 46 | + return this.status === 503 || this.status === 0; | |
| 47 | + } | |
| 48 | + get notFound(): boolean { | |
| 49 | + return this.status === 404; | |
| 50 | + } | |
| 51 | +} | |
| 52 | + | |
| 53 | +export interface FetchOptions { | |
| 54 | + /** Seconds; `false` → `cache: 'no-store'`. Default 900. */ | |
| 55 | + revalidate?: number | false; | |
| 56 | + tags?: string[]; | |
| 57 | +} | |
| 58 | + | |
| 59 | +type Query = Record<string, string | number | boolean | null | undefined>; | |
| 60 | + | |
| 61 | +function qs(query?: Query): string { | |
| 62 | + if (!query) return ''; | |
| 63 | + const p = new URLSearchParams(); | |
| 64 | + for (const [k, v] of Object.entries(query)) { | |
| 65 | + if (v === undefined || v === null || v === '') continue; | |
| 66 | + p.set(k, String(v)); | |
| 67 | + } | |
| 68 | + const s = p.toString(); | |
| 69 | + return s ? `?${s}` : ''; | |
| 70 | +} | |
| 71 | + | |
| 72 | +export async function request<T>(path: string, query?: Query, opts: FetchOptions = {}): Promise<T> { | |
| 73 | + const url = `${BASE}${path}${qs(query)}`; | |
| 74 | + const init: RequestInit & { next?: { revalidate?: number | false; tags?: string[] } } = { headers: { accept: 'application/json' } }; | |
| 75 | + if (opts.revalidate === false) init.cache = 'no-store'; | |
| 76 | + else init.next = { revalidate: opts.revalidate ?? 900, tags: opts.tags }; | |
| 77 | + | |
| 78 | + let res: Response; | |
| 79 | + try { | |
| 80 | + res = await fetch(url, init); | |
| 81 | + } catch (e) { | |
| 82 | + throw new ApiError(0, path, null, `API unreachable at ${BASE} (${(e as Error).message})`); | |
| 83 | + } | |
| 84 | + if (!res.ok) { | |
| 85 | + let problem: Problem | null = null; | |
| 86 | + try { | |
| 87 | + const body = (await res.json()) as unknown; | |
| 88 | + if (body && typeof body === 'object' && 'status' in body && 'title' in body) problem = body as Problem; | |
| 89 | + else if (body && typeof body === 'object' && 'detail' in body) problem = { title: String((body as { detail: unknown }).detail), status: res.status }; | |
| 90 | + } catch { | |
| 91 | + /* non-JSON error body */ | |
| 92 | + } | |
| 93 | + throw new ApiError(res.status, path, problem); | |
| 94 | + } | |
| 95 | + return (await res.json()) as T; | |
| 96 | +} | |
| 97 | + | |
| 98 | +/** Resolve to `null` on any API error (optional panels). */ | |
| 99 | +export async function safe<T>(p: Promise<T>): Promise<T | null> { | |
| 100 | + try { | |
| 101 | + return await p; | |
| 102 | + } catch { | |
| 103 | + return null; | |
| 104 | + } | |
| 105 | +} | |
| 106 | + | |
| 107 | +/** True when the error means "render the calm not-built state" rather than an error boundary. */ | |
| 108 | +export function isNotBuilt(e: unknown): boolean { | |
| 109 | + return e instanceof ApiError && e.notBuilt; | |
| 110 | +} | |
| 111 | +export function isNotFound(e: unknown): boolean { | |
| 112 | + return e instanceof ApiError && e.notFound; | |
| 113 | +} | |
| 114 | + | |
| 115 | +// ---------------------------------------------------------------------------------------------- endpoints (§8) | |
| 116 | + | |
| 117 | +export const api = { | |
| 118 | + health: () => request<HealthResponse>('/health', undefined, { revalidate: 60 }), | |
| 119 | + | |
| 120 | + home: () => request<HomeResponse>('/home'), | |
| 121 | + | |
| 122 | + countries: (q: { region?: string; income?: string; q?: string; sort?: 'name' | 'population' | 'gdp' | 'gdp_per_capita' | 'coverage'; kind?: string; limit?: number; offset?: number } = {}) => | |
| 123 | + request<CountriesResponse>('/countries', { limit: 1000, ...q }), | |
| 124 | + | |
| 125 | + country: (id: string) => request<CountryResponse>(`/countries/${encodeURIComponent(id)}`), | |
| 126 | + | |
| 127 | + countryTopic: (id: string, topic: string) => request<CountryTopicResponse>(`/countries/${encodeURIComponent(id)}/topics/${encodeURIComponent(topic)}`), | |
| 128 | + | |
| 129 | + countrySeries: (id: string, indicator: string, q: { from?: number; to?: number; freq?: Frequency; include_forecast?: boolean; include_alt?: boolean } = {}) => | |
| 130 | + request<SeriesResponse>(`/countries/${encodeURIComponent(id)}/series/${encodeURIComponent(indicator)}`, q), | |
| 131 | + | |
| 132 | + countryChanges: (id: string, limit = 12, kind?: string) => request<ChangesResponse>(`/countries/${encodeURIComponent(id)}/changes`, { limit, kind }), | |
| 133 | + | |
| 134 | + countryEvents: (id: string, limit = 40, q: { kind?: string; indicator?: string } = {}) => | |
| 135 | + request<ChangesResponse>(`/countries/${encodeURIComponent(id)}/events`, { limit, ...q }), | |
| 136 | + | |
| 137 | + countrySimilar: (id: string, mode: SimilarityMode | string = 'overall', limit = 12) => | |
| 138 | + request<SimilarResponse>(`/countries/${encodeURIComponent(id)}/similar`, { mode, limit }), | |
| 139 | + | |
| 140 | + countryInsights: (id: string) => request<InsightsResponse>(`/countries/${encodeURIComponent(id)}/insights`), | |
| 141 | + | |
| 142 | + countryDna: (id: string) => request<DNAResponse>(`/countries/${encodeURIComponent(id)}/dna`), | |
| 143 | + | |
| 144 | + indicatorMap: (slug: string, q: { year?: number; nearest?: boolean } = {}) => request<MapResponse>(`/indicators/${encodeURIComponent(slug)}/map`, q), | |
| 145 | + | |
| 146 | + ranking: (slug: string, q: { year?: number; group?: string; sort?: 'asc' | 'desc'; limit?: number; offset?: number; sparkline?: boolean } = {}) => | |
| 147 | + request<RankingResponse>(`/rankings/${encodeURIComponent(slug)}`, q), | |
| 148 | + | |
| 149 | + search: (q: string, limit = 12, type?: string) => request<SearchResponse>('/search', { q, limit, type }, { revalidate: false }), | |
| 150 | + | |
| 151 | + changes: (q: { limit?: number; kind?: string } = {}) => request<ChangesResponse>('/changes', q), | |
| 152 | +}; | |
| 153 | + | |
| 154 | +type Frequency = 'A' | 'Q' | 'M'; | |
| 155 | +export type Api = typeof api; | |
added
apps/web/src/lib/client-api.ts
+29 −0
@@ -0,0 +1,29 @@ | ||
| 1 | +'use client'; | |
| 2 | +import type { SearchResponse, SeriesResponse } from './types'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Browser-side fetch helpers: same-origin `/api/v1/*` (rewritten by next.config.ts to the FastAPI service). | |
| 6 | + * Kept deliberately small — server components should do the heavy lifting. | |
| 7 | + */ | |
| 8 | +export class ClientApiError extends Error { | |
| 9 | + constructor( | |
| 10 | + readonly status: number, | |
| 11 | + message: string, | |
| 12 | + ) { | |
| 13 | + super(message); | |
| 14 | + this.name = 'ClientApiError'; | |
| 15 | + } | |
| 16 | +} | |
| 17 | + | |
| 18 | +async function get<T>(path: string, signal?: AbortSignal): Promise<T> { | |
| 19 | + const res = await fetch(`/api/v1${path}`, { headers: { accept: 'application/json' }, signal }); | |
| 20 | + if (!res.ok) throw new ClientApiError(res.status, `API ${res.status}`); | |
| 21 | + return (await res.json()) as T; | |
| 22 | +} | |
| 23 | + | |
| 24 | +export const clientApi = { | |
| 25 | + search: (q: string, limit = 12, signal?: AbortSignal) => | |
| 26 | + get<SearchResponse>(`/search?q=${encodeURIComponent(q)}&limit=${limit}`, signal), | |
| 27 | + countrySeries: (id: string, indicator: string, signal?: AbortSignal) => | |
| 28 | + get<SeriesResponse>(`/countries/${encodeURIComponent(id)}/series/${encodeURIComponent(indicator)}`, signal), | |
| 29 | +}; | |
added
apps/web/src/lib/cn.ts
+4 −0
@@ -0,0 +1,4 @@ | ||
| 1 | +/** Tiny class-name joiner (no dependency). */ | |
| 2 | +export function cn(...parts: Array<string | false | null | undefined>): string { | |
| 3 | + return parts.filter(Boolean).join(' '); | |
| 4 | +} | |
added
apps/web/src/lib/fonts.system.ts
+3 −0
@@ -0,0 +1,3 @@ | ||
| 1 | +/** Offline fallback for lib/fonts.ts: exposes the same variables bound to the system stack. */ | |
| 2 | +export const fontUi = { variable: 'font-ui-system', className: '', style: { fontFamily: 'ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif' } }; | |
| 3 | +export const fontDisplay = { variable: 'font-display-system', className: '', style: { fontFamily: '"Iowan Old Style", "Palatino Linotype", Georgia, serif' } }; | |
added
apps/web/src/lib/fonts.ts
+8 −0
@@ -0,0 +1,8 @@ | ||
| 1 | +/** | |
| 2 | + * Fonts via next/font/google (self-hosted at build). If the build machine has no network, swap the import in | |
| 3 | + * layout.tsx for `./fonts.system` (same exported names, system stack) — the build must never fail on fonts. | |
| 4 | + */ | |
| 5 | +import { Inter, Newsreader } from 'next/font/google'; | |
| 6 | + | |
| 7 | +export const fontUi = Inter({ variable: '--font-ui', subsets: ['latin'], display: 'swap', axes: ['opsz'] }); | |
| 8 | +export const fontDisplay = Newsreader({ variable: '--font-display', subsets: ['latin'], display: 'swap', weight: ['400', '500', '600'], style: ['normal', 'italic'], axes: ['opsz'] }); | |
added
apps/web/src/lib/format.ts
+258 −0
@@ -0,0 +1,258 @@ | ||
| 1 | +import { LOCALE, t } from '@/i18n'; | |
| 2 | +import type { FormatSpec, IndicatorFormat } from './types'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Number/date formatting. All Intl calls use the FIXED locale `en-US` (server and client alike) so | |
| 6 | + * server-rendered markup never differs from the client render (hydration). Never rely on the browser default. | |
| 7 | + */ | |
| 8 | + | |
| 9 | + | |
| 10 | +const NF_CACHE = new Map<string, Intl.NumberFormat>(); | |
| 11 | +function nf(opts: Intl.NumberFormatOptions): Intl.NumberFormat { | |
| 12 | + const key = JSON.stringify(opts); | |
| 13 | + let f = NF_CACHE.get(key); | |
| 14 | + if (!f) { | |
| 15 | + f = new Intl.NumberFormat(LOCALE, opts); | |
| 16 | + NF_CACHE.set(key, f); | |
| 17 | + } | |
| 18 | + return f; | |
| 19 | +} | |
| 20 | + | |
| 21 | +const NA = t('common.na'); | |
| 22 | + | |
| 23 | +export function isNum(v: unknown): v is number { | |
| 24 | + return typeof v === 'number' && Number.isFinite(v); | |
| 25 | +} | |
| 26 | + | |
| 27 | +/** 1.2T / 45.3B / 12.4M / 53.4k — fixed suffix set (never "trillion"), 3 significant digits max. */ | |
| 28 | +export function compact(v: number, maxSig = 3): string { | |
| 29 | + const abs = Math.abs(v); | |
| 30 | + const units: Array<[number, string]> = [ | |
| 31 | + [1e12, 'T'], | |
| 32 | + [1e9, 'B'], | |
| 33 | + [1e6, 'M'], | |
| 34 | + [1e3, 'k'], | |
| 35 | + ]; | |
| 36 | + for (const [div, suffix] of units) { | |
| 37 | + if (abs >= div) { | |
| 38 | + const n = v / div; | |
| 39 | + const digits = Math.abs(n) >= 100 ? 0 : Math.abs(n) >= 10 ? 1 : Math.min(2, maxSig - 1); | |
| 40 | + return nf({ maximumFractionDigits: digits, minimumFractionDigits: 0 }).format(n) + suffix; | |
| 41 | + } | |
| 42 | + } | |
| 43 | + return nf({ maximumFractionDigits: abs >= 100 ? 0 : abs >= 10 ? 1 : 2 }).format(v); | |
| 44 | +} | |
| 45 | + | |
| 46 | +/** Grouped integer-ish number: 1,234,567. */ | |
| 47 | +export function grouped(v: number, digits = 0): string { | |
| 48 | + return nf({ maximumFractionDigits: digits, minimumFractionDigits: 0 }).format(v); | |
| 49 | +} | |
| 50 | + | |
| 51 | +export function fixed(v: number, digits = 1): string { | |
| 52 | + return nf({ maximumFractionDigits: digits, minimumFractionDigits: digits }).format(v); | |
| 53 | +} | |
| 54 | + | |
| 55 | +/** Currency prefix from the unit string (US$, intl $, €…). Defaults to US$. */ | |
| 56 | +function currencyPrefix(spec: FormatSpec): string { | |
| 57 | + const u = (spec.unit_short ?? spec.unit ?? '').toLowerCase(); | |
| 58 | + if (u.includes('intl') || u.includes('international') || u.includes('ppp')) return 'intl $'; | |
| 59 | + if (u.includes('€') || u.includes('eur')) return '€'; | |
| 60 | + if (u.includes('pps')) return 'PPS '; | |
| 61 | + return 'US$'; | |
| 62 | +} | |
| 63 | + | |
| 64 | +/** | |
| 65 | + * Format a value according to the indicator's `format` (ARCHITECTURE §4). | |
| 66 | + * `opts.compactBelow` (default 1e6) → numbers below it are grouped, above are compacted. | |
| 67 | + */ | |
| 68 | +export function formatValue( | |
| 69 | + value: number | null | undefined, | |
| 70 | + spec: FormatSpec, | |
| 71 | + opts: { compactBelow?: number; withUnit?: boolean } = {}, | |
| 72 | +): string { | |
| 73 | + if (!isNum(value)) return NA; | |
| 74 | + const withUnit = opts.withUnit ?? true; | |
| 75 | + const precision = spec.precision ?? 1; | |
| 76 | + const format = (spec.format ?? 'number') as IndicatorFormat; | |
| 77 | + const compactBelow = opts.compactBelow ?? 1e6; | |
| 78 | + const abs = Math.abs(value); | |
| 79 | + switch (format) { | |
| 80 | + case 'currency': { | |
| 81 | + const prefix = currencyPrefix(spec); | |
| 82 | + const body = abs >= 1e5 ? compact(value) : grouped(value, abs < 10 ? 2 : 0); | |
| 83 | + const suffix = spec.unit_short && /\/h$/.test(spec.unit_short) ? '/h' : ''; | |
| 84 | + return `${prefix}${body}${suffix}`; | |
| 85 | + } | |
| 86 | + case 'percent': | |
| 87 | + return withUnit ? `${fixed(value, precision)} %` : fixed(value, precision); | |
| 88 | + case 'years': | |
| 89 | + return withUnit ? `${fixed(value, precision)} yrs` : fixed(value, precision); | |
| 90 | + case 'per_1000': | |
| 91 | + return withUnit ? `${fixed(value, precision)} ‰` : fixed(value, precision); | |
| 92 | + case 'per_100k': | |
| 93 | + return withUnit ? `${fixed(value, precision)} /100k` : fixed(value, precision); | |
| 94 | + case 'per_million': | |
| 95 | + return withUnit ? `${fixed(value, precision)} /M` : fixed(value, precision); | |
| 96 | + case 'index': | |
| 97 | + return fixed(value, precision); | |
| 98 | + case 'ratio': | |
| 99 | + return fixed(value, Math.max(precision, 2)); | |
| 100 | + case 'celsius': | |
| 101 | + return `${fixed(value, Math.max(precision, 2))} °C`; | |
| 102 | + case 'tonnes': { | |
| 103 | + const u = spec.unit_short ?? 't'; | |
| 104 | + return withUnit ? `${abs >= compactBelow ? compact(value) : fixed(value, precision)} ${u}` : fixed(value, precision); | |
| 105 | + } | |
| 106 | + case 'kwh': { | |
| 107 | + const u = spec.unit_short ?? 'kWh'; | |
| 108 | + return withUnit ? `${abs >= compactBelow ? compact(value) : grouped(value, precision)} ${u}` : grouped(value, precision); | |
| 109 | + } | |
| 110 | + case 'km': | |
| 111 | + return withUnit ? `${abs >= compactBelow ? compact(value) : grouped(value)} km` : grouped(value); | |
| 112 | + case 'ha': | |
| 113 | + return withUnit ? `${abs >= compactBelow ? compact(value) : grouped(value)} ha` : grouped(value); | |
| 114 | + case 'number': | |
| 115 | + default: { | |
| 116 | + if (abs >= compactBelow) return compact(value); | |
| 117 | + if (Number.isInteger(value) || abs >= 1000) return grouped(value); | |
| 118 | + return fixed(value, precision); | |
| 119 | + } | |
| 120 | + } | |
| 121 | +} | |
| 122 | + | |
| 123 | +/** Units whose change is expressed in points rather than percent. */ | |
| 124 | +export function isPointsUnit(spec: FormatSpec): boolean { | |
| 125 | + return spec.format === 'percent' || spec.format === 'index' || spec.format === 'ratio' || spec.format === 'years'; | |
| 126 | +} | |
| 127 | + | |
| 128 | +/** | |
| 129 | + * Signed change: "+2.3 pts" for percent-type units, "+4.1 %" for relative change, "+1.2M" absolute. | |
| 130 | + * Prefer `changePct` for level indicators and `changeAbs` for point-type units. | |
| 131 | + */ | |
| 132 | +export function formatChange( | |
| 133 | + changeAbs: number | null | undefined, | |
| 134 | + changePct: number | null | undefined, | |
| 135 | + spec: FormatSpec, | |
| 136 | +): { text: string; direction: 'up' | 'down' | 'flat' } | null { | |
| 137 | + if (isPointsUnit(spec)) { | |
| 138 | + if (!isNum(changeAbs)) return null; | |
| 139 | + const d = changeAbs > 0 ? 'up' : changeAbs < 0 ? 'down' : 'flat'; | |
| 140 | + const unit = spec.format === 'years' ? ' yrs' : spec.format === 'index' || spec.format === 'ratio' ? '' : ' pts'; | |
| 141 | + return { text: `${sign(changeAbs)}${fixed(Math.abs(changeAbs), spec.precision ?? 1)}${unit}`, direction: d }; | |
| 142 | + } | |
| 143 | + if (isNum(changePct)) { | |
| 144 | + const d = changePct > 0 ? 'up' : changePct < 0 ? 'down' : 'flat'; | |
| 145 | + return { text: `${sign(changePct)}${fixed(Math.abs(changePct), 1)} %`, direction: d }; | |
| 146 | + } | |
| 147 | + if (isNum(changeAbs)) { | |
| 148 | + const d = changeAbs > 0 ? 'up' : changeAbs < 0 ? 'down' : 'flat'; | |
| 149 | + return { text: `${sign(changeAbs)}${formatValue(Math.abs(changeAbs), spec)}`, direction: d }; | |
| 150 | + } | |
| 151 | + return null; | |
| 152 | +} | |
| 153 | + | |
| 154 | +function sign(v: number): string { | |
| 155 | + return v > 0 ? '+' : v < 0 ? '−' : ''; | |
| 156 | +} | |
| 157 | + | |
| 158 | +const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; | |
| 159 | + | |
| 160 | +/** "2024" for annual, "Q2 2026" quarterly, "Jun 2026" monthly (period = first day, ISO date). */ | |
| 161 | +export function formatPeriod(period: string | null | undefined, frequency: 'A' | 'Q' | 'M' | string | null | undefined = 'A'): string { | |
| 162 | + if (!period) return NA; | |
| 163 | + const m = /^(\d{4})-(\d{2})/.exec(period); | |
| 164 | + if (!m) return period; | |
| 165 | + const year = m[1]!; | |
| 166 | + const month = Number(m[2]); | |
| 167 | + if (frequency === 'Q') return `Q${Math.floor((month - 1) / 3) + 1} ${year}`; | |
| 168 | + if (frequency === 'M') return `${MONTHS[month - 1] ?? ''} ${year}`; | |
| 169 | + return year; | |
| 170 | +} | |
| 171 | + | |
| 172 | +/** "12th of 190". */ | |
| 173 | +export function formatRank(rank: number | null | undefined, n: number | null | undefined): string { | |
| 174 | + if (!isNum(rank) || !isNum(n)) return NA; | |
| 175 | + return t('metric.rankWorld', { rank: ordinal(rank), n: grouped(n) }); | |
| 176 | +} | |
| 177 | + | |
| 178 | +export function ordinal(n: number): string { | |
| 179 | + const s = ['th', 'st', 'nd', 'rd']; | |
| 180 | + const v = n % 100; | |
| 181 | + return `${grouped(n)}${s[(v - 20) % 10] ?? s[v] ?? s[0]}`; | |
| 182 | +} | |
| 183 | + | |
| 184 | +/** "11 Sep 2026" — deterministic (UTC) date formatting. */ | |
| 185 | +export function formatDate(iso: string | null | undefined, opts: { month?: 'short' | 'long' } = {}): string { | |
| 186 | + if (!iso) return NA; | |
| 187 | + const d = new Date(iso); | |
| 188 | + if (Number.isNaN(d.getTime())) return iso; | |
| 189 | + const day = d.getUTCDate(); | |
| 190 | + const mon = MONTHS[d.getUTCMonth()] ?? ''; | |
| 191 | + const month = opts.month === 'long' ? LONG_MONTHS[d.getUTCMonth()] : mon; | |
| 192 | + return `${day} ${month} ${d.getUTCFullYear()}`; | |
| 193 | +} | |
| 194 | +const LONG_MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; | |
| 195 | + | |
| 196 | +/** Human "3 d ago" style, computed against `now` (pass a fixed `now` on the server to keep SSR stable). */ | |
| 197 | +export function relativeFreshness(iso: string | null | undefined, now: Date | number = Date.now()): string { | |
| 198 | + if (!iso) return NA; | |
| 199 | + const then = new Date(iso).getTime(); | |
| 200 | + if (Number.isNaN(then)) return NA; | |
| 201 | + const nowMs = typeof now === 'number' ? now : now.getTime(); | |
| 202 | + const s = Math.max(0, (nowMs - then) / 1000); | |
| 203 | + if (s < 90) return t('common.ago.now'); | |
| 204 | + const min = s / 60; | |
| 205 | + if (min < 60) return t('common.ago.minutes', { n: Math.round(min) }); | |
| 206 | + const h = min / 60; | |
| 207 | + if (h < 36) return t('common.ago.hours', { n: Math.round(h) }); | |
| 208 | + const d = h / 24; | |
| 209 | + if (d < 45) return t('common.ago.days', { n: Math.round(d) }); | |
| 210 | + const mo = d / 30.44; | |
| 211 | + if (mo < 18) return t('common.ago.months', { n: Math.round(mo) }); | |
| 212 | + return t('common.ago.years', { n: Math.round(d / 365.25) }); | |
| 213 | +} | |
| 214 | + | |
| 215 | +/** Freshness class for a retrieval date: fresh < 45 d, recent < 400 d, else stale. */ | |
| 216 | +export function freshnessLevel(iso: string | null | undefined, now: number = Date.now()): 'fresh' | 'recent' | 'stale' | null { | |
| 217 | + if (!iso) return null; | |
| 218 | + const then = new Date(iso).getTime(); | |
| 219 | + if (Number.isNaN(then)) return null; | |
| 220 | + const days = (now - then) / 86_400_000; | |
| 221 | + if (days < 45) return 'fresh'; | |
| 222 | + if (days < 400) return 'recent'; | |
| 223 | + return 'stale'; | |
| 224 | +} | |
| 225 | + | |
| 226 | +/** Percent 0–1 or 0–100 → "83 %". */ | |
| 227 | +export function formatPct(v: number | null | undefined, digits = 0): string { | |
| 228 | + if (!isNum(v)) return NA; | |
| 229 | + const p = v <= 1 ? v * 100 : v; | |
| 230 | + return `${fixed(p, digits)} %`; | |
| 231 | +} | |
| 232 | + | |
| 233 | +/** Axis tick label: compact for large magnitudes, otherwise ≤ 2 decimals, plus the short unit when useful. */ | |
| 234 | +export function formatTick(v: number, spec: FormatSpec): string { | |
| 235 | + const abs = Math.abs(v); | |
| 236 | + if (spec.format === 'percent') return `${fixed(v, abs < 1 && abs > 0 ? 1 : 0)}%`; | |
| 237 | + if (spec.format === 'currency') return `${currencyPrefix(spec)}${abs >= 1e3 ? compact(v, 2) : grouped(v)}`; | |
| 238 | + if (abs >= 1e4) return compact(v, 2); | |
| 239 | + if (Number.isInteger(v)) return grouped(v); | |
| 240 | + return fixed(v, abs < 1 ? 2 : 1); | |
| 241 | +} | |
| 242 | + | |
| 243 | +export const IndicatorFormats: readonly IndicatorFormat[] = [ | |
| 244 | + 'number', | |
| 245 | + 'percent', | |
| 246 | + 'currency', | |
| 247 | + 'index', | |
| 248 | + 'years', | |
| 249 | + 'per_1000', | |
| 250 | + 'per_100k', | |
| 251 | + 'per_million', | |
| 252 | + 'ratio', | |
| 253 | + 'celsius', | |
| 254 | + 'tonnes', | |
| 255 | + 'kwh', | |
| 256 | + 'ha', | |
| 257 | + 'km', | |
| 258 | +]; | |
added
apps/web/src/lib/iso-numeric.ts
+48 −0
@@ -0,0 +1,48 @@ | ||
| 1 | +/** | |
| 2 | + * ISO 3166-1 numeric → alpha-3, for the `world-atlas` TopoJSON (Natural Earth 1:110m), whose | |
| 3 | + * geometries carry only the numeric code as `id` and an English short `name` in properties. | |
| 4 | + * Pure data; covers every id present in `countries-110m.json` plus the territories that appear in | |
| 5 | + * the registry (they have no polygon at 1:110m but keep a stable code). | |
| 6 | + */ | |
| 7 | +export const ISO_NUMERIC_TO_ALPHA3: Readonly<Record<string, string>> = { | |
| 8 | + '004': 'AFG', '008': 'ALB', '010': 'ATA', '012': 'DZA', '016': 'ASM', '020': 'AND', '024': 'AGO', '028': 'ATG', '031': 'AZE', '032': 'ARG', | |
| 9 | + '036': 'AUS', '040': 'AUT', '044': 'BHS', '048': 'BHR', '050': 'BGD', '051': 'ARM', '052': 'BRB', '056': 'BEL', '060': 'BMU', '064': 'BTN', | |
| 10 | + '068': 'BOL', '070': 'BIH', '072': 'BWA', '076': 'BRA', '084': 'BLZ', '090': 'SLB', '092': 'VGB', '096': 'BRN', '100': 'BGR', '104': 'MMR', | |
| 11 | + '108': 'BDI', '112': 'BLR', '116': 'KHM', '120': 'CMR', '124': 'CAN', '132': 'CPV', '136': 'CYM', '140': 'CAF', '144': 'LKA', '148': 'TCD', | |
| 12 | + '152': 'CHL', '156': 'CHN', '158': 'TWN', '170': 'COL', '174': 'COM', '175': 'MYT', '178': 'COG', '180': 'COD', '188': 'CRI', '191': 'HRV', | |
| 13 | + '192': 'CUB', '196': 'CYP', '203': 'CZE', '204': 'BEN', '208': 'DNK', '212': 'DMA', '214': 'DOM', '218': 'ECU', '222': 'SLV', '226': 'GNQ', | |
| 14 | + '231': 'ETH', '232': 'ERI', '233': 'EST', '234': 'FRO', '238': 'FLK', '242': 'FJI', '246': 'FIN', '250': 'FRA', '254': 'GUF', '258': 'PYF', | |
| 15 | + '260': 'ATF', '262': 'DJI', '266': 'GAB', '268': 'GEO', '270': 'GMB', '275': 'PSE', '276': 'DEU', '288': 'GHA', '292': 'GIB', '300': 'GRC', | |
| 16 | + '304': 'GRL', '308': 'GRD', '312': 'GLP', '316': 'GUM', '320': 'GTM', '324': 'GIN', '328': 'GUY', '332': 'HTI', '340': 'HND', '344': 'HKG', | |
| 17 | + '348': 'HUN', '352': 'ISL', '356': 'IND', '360': 'IDN', '364': 'IRN', '368': 'IRQ', '372': 'IRL', '376': 'ISR', '380': 'ITA', '384': 'CIV', | |
| 18 | + '388': 'JAM', '392': 'JPN', '398': 'KAZ', '400': 'JOR', '404': 'KEN', '408': 'PRK', '410': 'KOR', '414': 'KWT', '417': 'KGZ', '418': 'LAO', | |
| 19 | + '422': 'LBN', '426': 'LSO', '428': 'LVA', '430': 'LBR', '434': 'LBY', '438': 'LIE', '440': 'LTU', '442': 'LUX', '446': 'MAC', '450': 'MDG', | |
| 20 | + '454': 'MWI', '458': 'MYS', '462': 'MDV', '466': 'MLI', '470': 'MLT', '474': 'MTQ', '478': 'MRT', '480': 'MUS', '484': 'MEX', '492': 'MCO', | |
| 21 | + '496': 'MNG', '498': 'MDA', '499': 'MNE', '504': 'MAR', '508': 'MOZ', '512': 'OMN', '516': 'NAM', '524': 'NPL', '528': 'NLD', '531': 'CUW', | |
| 22 | + '533': 'ABW', '540': 'NCL', '548': 'VUT', '554': 'NZL', '558': 'NIC', '562': 'NER', '566': 'NGA', '578': 'NOR', '580': 'MNP', '586': 'PAK', | |
| 23 | + '591': 'PAN', '598': 'PNG', '600': 'PRY', '604': 'PER', '608': 'PHL', '616': 'POL', '620': 'PRT', '624': 'GNB', '626': 'TLS', '630': 'PRI', | |
| 24 | + '634': 'QAT', '638': 'REU', '642': 'ROU', '643': 'RUS', '646': 'RWA', '659': 'KNA', '662': 'LCA', '670': 'VCT', '674': 'SMR', '678': 'STP', | |
| 25 | + '682': 'SAU', '686': 'SEN', '688': 'SRB', '690': 'SYC', '694': 'SLE', '702': 'SGP', '703': 'SVK', '704': 'VNM', '705': 'SVN', '706': 'SOM', | |
| 26 | + '710': 'ZAF', '716': 'ZWE', '724': 'ESP', '728': 'SSD', '729': 'SDN', '732': 'ESH', '740': 'SUR', '748': 'SWZ', '752': 'SWE', '756': 'CHE', | |
| 27 | + '760': 'SYR', '762': 'TJK', '764': 'THA', '768': 'TGO', '776': 'TON', '780': 'TTO', '784': 'ARE', '788': 'TUN', '792': 'TUR', '795': 'TKM', | |
| 28 | + '800': 'UGA', '804': 'UKR', '807': 'MKD', '818': 'EGY', '826': 'GBR', '834': 'TZA', '840': 'USA', '850': 'VIR', '854': 'BFA', '858': 'URY', | |
| 29 | + '860': 'UZB', '862': 'VEN', '882': 'WSM', '887': 'YEM', '894': 'ZMB', | |
| 30 | +}; | |
| 31 | + | |
| 32 | +/** | |
| 33 | + * Natural Earth draws three polygons with no ISO numeric id (`id` undefined in the TopoJSON): | |
| 34 | + * Kosovo (user-assigned XKX, used by the World Bank), Northern Cyprus and Somaliland (no code — | |
| 35 | + * they render as "no data"). Keyed by the Natural Earth `name` property. | |
| 36 | + */ | |
| 37 | +export const ATLAS_NAME_TO_ALPHA3: Readonly<Record<string, string | null>> = { Kosovo: 'XKX', 'N. Cyprus': null, Somaliland: null }; | |
| 38 | + | |
| 39 | +/** Alpha-3 for a world-atlas geometry (`id` numeric string, `name` from properties); null when it has no code. */ | |
| 40 | +export function atlasGeometryIso3(id: string | number | null | undefined, name?: string | null): string | null { | |
| 41 | + if (id != null && id !== '') { | |
| 42 | + const key = String(id).padStart(3, '0'); | |
| 43 | + const iso = ISO_NUMERIC_TO_ALPHA3[key]; | |
| 44 | + if (iso) return iso; | |
| 45 | + } | |
| 46 | + if (name && name in ATLAS_NAME_TO_ALPHA3) return ATLAS_NAME_TO_ALPHA3[name] ?? null; | |
| 47 | + return null; | |
| 48 | +} | |
added
apps/web/src/lib/map-geo.ts
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +import { geoEqualEarth, geoPath, type GeoPermissibleObjects, type GeoProjection } from 'd3-geo'; | |
| 2 | +import { feature } from 'topojson-client'; | |
| 3 | +import type { Topology, GeometryCollection } from 'topojson-specification'; | |
| 4 | +import type { Feature, FeatureCollection, Geometry } from 'geojson'; | |
| 5 | +import world from 'world-atlas/countries-110m.json'; | |
| 6 | +import { atlasGeometryIso3 } from './iso-numeric'; | |
| 7 | + | |
| 8 | +/** | |
| 9 | + * World geometry for the Choropleth: world-atlas 110m TopoJSON → Equal Earth projected SVG paths keyed by | |
| 10 | + * ISO3. Computed once per server process (module scope); ≈ 90 KB of path data shipped to the client view. | |
| 11 | + * The numeric → alpha-3 lookup prefers the API's `iso_numeric` (registry) and falls back to the static table. | |
| 12 | + */ | |
| 13 | +export const MAP_WIDTH = 960; | |
| 14 | +export const MAP_HEIGHT = 470; | |
| 15 | + | |
| 16 | +export interface CountryPath { | |
| 17 | + iso3: string | null; | |
| 18 | + /** ISO numeric id from the TopoJSON (3 digits) when present. */ | |
| 19 | + numeric: string | null; | |
| 20 | + name: string; | |
| 21 | + d: string; | |
| 22 | +} | |
| 23 | + | |
| 24 | +type CountriesTopology = Topology<{ countries: GeometryCollection<{ name: string }> }>; | |
| 25 | + | |
| 26 | +function fitProjection(width: number, height: number, pad = 4): GeoProjection { | |
| 27 | + return geoEqualEarth().fitExtent( | |
| 28 | + [ | |
| 29 | + [pad, pad], | |
| 30 | + [width - pad, height - pad], | |
| 31 | + ], | |
| 32 | + { type: 'Sphere' } as GeoPermissibleObjects, | |
| 33 | + ); | |
| 34 | +} | |
| 35 | + | |
| 36 | +let cache: { paths: CountryPath[]; sphere: string } | null = null; | |
| 37 | + | |
| 38 | +export function worldPaths(): { paths: CountryPath[]; sphere: string } { | |
| 39 | + if (cache) return cache; | |
| 40 | + const topology = world as unknown as CountriesTopology; | |
| 41 | + const fc = feature(topology, topology.objects.countries) as FeatureCollection<Geometry, { name: string }>; | |
| 42 | + const projection = fitProjection(MAP_WIDTH, MAP_HEIGHT); | |
| 43 | + const path = geoPath(projection).digits(1); | |
| 44 | + const paths: CountryPath[] = []; | |
| 45 | + for (const f of fc.features as Array<Feature<Geometry, { name: string }>>) { | |
| 46 | + if (f.id === '010') continue; // Antarctica: no data, dominates the lower band | |
| 47 | + const d = path(f); | |
| 48 | + if (!d) continue; | |
| 49 | + const numeric = f.id != null && f.id !== '' ? String(f.id).padStart(3, '0') : null; | |
| 50 | + paths.push({ iso3: atlasGeometryIso3(f.id as string | number | undefined, f.properties?.name), numeric, name: f.properties?.name ?? '', d }); | |
| 51 | + } | |
| 52 | + const sphere = path({ type: 'Sphere' }) ?? ''; | |
| 53 | + cache = { paths, sphere }; | |
| 54 | + return cache; | |
| 55 | +} | |
| 56 | + | |
| 57 | +/** Re-key a path's ISO3 using the registry's iso_numeric when available (API `/countries` → iso_numeric). */ | |
| 58 | +export function isoLookupFromCountries(countries: Array<{ id: string; iso_numeric?: string | null }>): Map<string, string> { | |
| 59 | + const m = new Map<string, string>(); | |
| 60 | + for (const c of countries) if (c.iso_numeric) m.set(String(c.iso_numeric).padStart(3, '0'), c.id); | |
| 61 | + return m; | |
| 62 | +} | |
| 63 | + | |
| 64 | +/** Class index 0..k for a value against sorted quantile breaks (k = breaks.length). */ | |
| 65 | +export function classIndex(value: number, breaks: number[]): number { | |
| 66 | + let i = 0; | |
| 67 | + while (i < breaks.length && value >= breaks[i]!) i++; | |
| 68 | + return i; | |
| 69 | +} | |
added
apps/web/src/lib/regions.ts
+39 −0
@@ -0,0 +1,39 @@ | ||
| 1 | +import type { IncomeGroup } from './types'; | |
| 2 | + | |
| 3 | +/** Static mirror of `registry/groups.yaml` regions (World Bank, 7) and income groups (4) used for filters/chips. */ | |
| 4 | +export interface GroupDef { | |
| 5 | + id: string; | |
| 6 | + slug: string; | |
| 7 | + name: string; | |
| 8 | + short: string; | |
| 9 | +} | |
| 10 | + | |
| 11 | +export const WB_REGIONS: readonly GroupDef[] = [ | |
| 12 | + { id: 'NAC', slug: 'north-america', name: 'North America', short: 'N. America' }, | |
| 13 | + { id: 'LCN', slug: 'latin-america-caribbean', name: 'Latin America & Caribbean', short: 'Latin America' }, | |
| 14 | + { id: 'ECS', slug: 'europe-central-asia', name: 'Europe & Central Asia', short: 'Europe & C. Asia' }, | |
| 15 | + { id: 'MEA', slug: 'middle-east-north-africa', name: 'Middle East, North Africa, Afghanistan & Pakistan', short: 'MENA+' }, | |
| 16 | + { id: 'SAS', slug: 'south-asia', name: 'South Asia', short: 'South Asia' }, | |
| 17 | + { id: 'EAS', slug: 'east-asia-pacific', name: 'East Asia & Pacific', short: 'East Asia & Pacific' }, | |
| 18 | + { id: 'SSF', slug: 'sub-saharan-africa', name: 'Sub-Saharan Africa', short: 'Sub-Saharan Africa' }, | |
| 19 | +]; | |
| 20 | + | |
| 21 | +export const INCOME_GROUPS: readonly (GroupDef & { id: IncomeGroup })[] = [ | |
| 22 | + { id: 'HIC', slug: 'high-income', name: 'High income', short: 'High' }, | |
| 23 | + { id: 'UMC', slug: 'upper-middle-income', name: 'Upper middle income', short: 'Upper middle' }, | |
| 24 | + { id: 'LMC', slug: 'lower-middle-income', name: 'Lower middle income', short: 'Lower middle' }, | |
| 25 | + { id: 'LIC', slug: 'low-income', name: 'Low income', short: 'Low' }, | |
| 26 | +]; | |
| 27 | + | |
| 28 | +export function regionName(id: string | null | undefined): string | null { | |
| 29 | + if (!id) return null; | |
| 30 | + return WB_REGIONS.find((r) => r.id === id.toUpperCase())?.name ?? null; | |
| 31 | +} | |
| 32 | +export function regionShort(id: string | null | undefined): string | null { | |
| 33 | + if (!id) return null; | |
| 34 | + return WB_REGIONS.find((r) => r.id === id.toUpperCase())?.short ?? null; | |
| 35 | +} | |
| 36 | +export function regionSlug(id: string | null | undefined): string | null { | |
| 37 | + if (!id) return null; | |
| 38 | + return WB_REGIONS.find((r) => r.id === id.toUpperCase())?.slug ?? null; | |
| 39 | +} | |
added
apps/web/src/lib/site.ts
+32 −0
@@ -0,0 +1,32 @@ | ||
| 1 | +/** Site-wide constants (URL, name) used by metadata, sitemap and OG images. */ | |
| 2 | +export const SITE_URL = (process.env.NEXT_PUBLIC_SITE_URL ?? 'https://www.countryatlas.co').replace(/\/$/, ''); | |
| 3 | +export const SITE_NAME = 'CountryAtlas'; | |
| 4 | +export const SITE_DOMAIN = 'countryatlas.co'; | |
| 5 | +export const TAGLINE = 'Understand the world, one country at a time.'; | |
| 6 | +export const OG_SIZE = { width: 1200, height: 630 } as const; | |
| 7 | + | |
| 8 | +/** Route helpers — keep every internal href in one place so the next agent can add pages consistently. */ | |
| 9 | +export const routes = { | |
| 10 | + home: () => '/', | |
| 11 | + countries: () => '/countries', | |
| 12 | + country: (slug: string) => `/countries/${slug}`, | |
| 13 | + countryTopic: (slug: string, topic: string) => `/countries/${slug}/${topic}`, | |
| 14 | + countryIndicator: (slug: string, topic: string, indicator: string) => `/countries/${slug}/${topic}#${indicator}`, | |
| 15 | + countryDownload: (id: string, fmt: 'csv' | 'json' = 'csv') => `/api/v1/countries/${id}/download.${fmt}`, | |
| 16 | + compare: (...slugs: string[]) => (slugs.length ? `/compare/${slugs.join('/')}` : '/compare'), | |
| 17 | + rankings: () => '/rankings', | |
| 18 | + ranking: (indicator: string) => `/rankings/${indicator}`, | |
| 19 | + indicators: (topic?: string) => (topic ? `/indicators?topic=${encodeURIComponent(topic)}` : '/indicators'), | |
| 20 | + indicator: (slug: string) => `/indicators/${slug}`, | |
| 21 | + indicatorDownload: (slug: string, fmt: 'csv' | 'json' = 'csv') => `/api/v1/indicators/${slug}/download.${fmt}`, | |
| 22 | + regions: () => '/regions', | |
| 23 | + region: (slug: string) => `/regions/${slug}`, | |
| 24 | + sources: () => '/sources', | |
| 25 | + source: (id: string) => `/sources/${id}`, | |
| 26 | + methodology: () => '/methodology', | |
| 27 | + data: () => '/data', | |
| 28 | + api: () => '/api', | |
| 29 | + changes: () => '/changes', | |
| 30 | + explore: () => '/explore', | |
| 31 | + search: (q: string) => `/search?q=${encodeURIComponent(q)}`, | |
| 32 | +} as const; | |
added
apps/web/src/lib/topics.ts
+76 −0
@@ -0,0 +1,76 @@ | ||
| 1 | +import type { TopicId } from './types'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Static mirror of `registry/topics.yaml` (id, name, short, order, blurb). The API is the source of truth for | |
| 5 | + * indicator membership and counts; this list exists so navigation, the home grid and metadata never need a | |
| 6 | + * fetch. Keep in sync when the registry changes (19 topics, fixed set — ARCHITECTURE §4). | |
| 7 | + */ | |
| 8 | +export interface TopicDef { | |
| 9 | + id: TopicId; | |
| 10 | + name: string; | |
| 11 | + short: string; | |
| 12 | + order: number; | |
| 13 | + blurb: string; | |
| 14 | +} | |
| 15 | + | |
| 16 | +export const TOPICS: readonly TopicDef[] = [ | |
| 17 | + { id: 'economy', name: 'Economy', short: 'Economy', order: 1, blurb: 'Output, growth, prices, productivity and external balance.' }, | |
| 18 | + { id: 'government', name: 'Government & public finance', short: 'Government', order: 2, blurb: 'Public debt, deficits, revenue, spending and taxation.' }, | |
| 19 | + { id: 'population', name: 'Population & demographics', short: 'Population', order: 3, blurb: 'Size, growth, ages, births, deaths, urbanisation and migration.' }, | |
| 20 | + { id: 'labor', name: 'Employment & wages', short: 'Labor', order: 4, blurb: 'Unemployment, participation, employment, youth, wages and hours.' }, | |
| 21 | + { id: 'income', name: 'Income & inequality', short: 'Income', order: 5, blurb: 'Living standards, poverty and distribution.' }, | |
| 22 | + { id: 'housing', name: 'Housing', short: 'Housing', order: 6, blurb: 'Prices, rents, affordability, mortgages and construction.' }, | |
| 23 | + { id: 'health', name: 'Health', short: 'Health', order: 7, blurb: 'Longevity, mortality, health system capacity and risk factors.' }, | |
| 24 | + { id: 'education', name: 'Education', short: 'Education', order: 8, blurb: 'Literacy, enrolment, attainment and spending.' }, | |
| 25 | + { id: 'trade', name: 'Trade', short: 'Trade', order: 9, blurb: 'Exports, imports, balances, openness and composition.' }, | |
| 26 | + { id: 'energy', name: 'Energy', short: 'Energy', order: 10, blurb: 'Generation, consumption, mix, dependence and intensity.' }, | |
| 27 | + { id: 'climate', name: 'Climate', short: 'Climate', order: 11, blurb: 'Greenhouse gases, intensity and temperature contribution.' }, | |
| 28 | + { id: 'environment', name: 'Environment', short: 'Environment', order: 12, blurb: 'Forests, air quality, land, water and protected areas.' }, | |
| 29 | + { id: 'infrastructure', name: 'Infrastructure & transportation', short: 'Infrastructure', order: 13, blurb: 'Transport networks, passengers, freight and access.' }, | |
| 30 | + { id: 'digital', name: 'Digital economy & Internet', short: 'Digital', order: 14, blurb: 'Connectivity, adoption and digital infrastructure.' }, | |
| 31 | + { id: 'innovation', name: 'Innovation, research & technology', short: 'Innovation', order: 15, blurb: 'R&D, patents, publications, researchers and high-tech.' }, | |
| 32 | + { id: 'agriculture', name: 'Agriculture & natural resources', short: 'Agriculture', order: 16, blurb: 'Land, production, employment and resource rents.' }, | |
| 33 | + { id: 'tourism', name: 'Tourism', short: 'Tourism', order: 17, blurb: 'Arrivals, departures and tourism receipts.' }, | |
| 34 | + { id: 'security', name: 'Security & governance', short: 'Security', order: 18, blurb: 'Homicide, military spending, refugees, displacement and institutions.' }, | |
| 35 | + { id: 'quality-of-life', name: 'Quality of life', short: 'Quality of life', order: 19, blurb: 'Human development, happiness, safety, access and environment.' }, | |
| 36 | +]; | |
| 37 | + | |
| 38 | +const BY_ID = new Map(TOPICS.map((tp) => [tp.id, tp])); | |
| 39 | +export function topicById(id: string): TopicDef | undefined { | |
| 40 | + return BY_ID.get(id as TopicId); | |
| 41 | +} | |
| 42 | +export function isTopicId(id: string): id is TopicId { | |
| 43 | + return BY_ID.has(id as TopicId); | |
| 44 | +} | |
| 45 | + | |
| 46 | +/** Headline indicators on the country overview, in order (registry/topics.yaml `headline`). */ | |
| 47 | +export const HEADLINE_INDICATORS = [ | |
| 48 | + 'population', | |
| 49 | + 'gdp', | |
| 50 | + 'gdp-per-capita', | |
| 51 | + 'gdp-growth', | |
| 52 | + 'inflation', | |
| 53 | + 'unemployment-rate', | |
| 54 | + 'life-expectancy', | |
| 55 | + 'median-age', | |
| 56 | + 'government-debt-pct-gdp', | |
| 57 | + 'co2-per-capita', | |
| 58 | + 'renewable-electricity-share', | |
| 59 | + 'internet-users', | |
| 60 | +] as const; | |
| 61 | + | |
| 62 | +/** Indicator slug → topic id fallback (used to build `/countries/[slug]/[topic]#indicator` links for headline metrics). */ | |
| 63 | +export const HEADLINE_TOPIC: Record<(typeof HEADLINE_INDICATORS)[number], TopicId> = { | |
| 64 | + population: 'population', | |
| 65 | + gdp: 'economy', | |
| 66 | + 'gdp-per-capita': 'economy', | |
| 67 | + 'gdp-growth': 'economy', | |
| 68 | + inflation: 'economy', | |
| 69 | + 'unemployment-rate': 'labor', | |
| 70 | + 'life-expectancy': 'health', | |
| 71 | + 'median-age': 'population', | |
| 72 | + 'government-debt-pct-gdp': 'government', | |
| 73 | + 'co2-per-capita': 'climate', | |
| 74 | + 'renewable-electricity-share': 'energy', | |
| 75 | + 'internet-users': 'digital', | |
| 76 | +}; | |
added
apps/web/src/lib/types.ts
+575 −0
@@ -0,0 +1,575 @@ | ||
| 1 | +/** | |
| 2 | + * TypeScript mirror of the FastAPI response models in `src/countryatlas/api/schemas.py` (ARCHITECTURE §8). | |
| 3 | + * Aligned on 2026-09-11 against that file — when it changes, change this one. The next agent extends it for | |
| 4 | + * compare / rankings / indicators / regions / sources pages (the corresponding pydantic models already exist: | |
| 5 | + * CompareResponse, RankingResponse, IndicatorResponse, RegionResponse, SourcesResponse … add them here). | |
| 6 | + * | |
| 7 | + * Conventions: every top-level response carries `meta`. Optional fields are `| null` (the API sends explicit | |
| 8 | + * nulls, never omits) except where pydantic gives a default (`is_forecast: false`, lists `[]`). Dates are ISO | |
| 9 | + * strings (`period` = first day of the period, `YYYY-MM-DD`). Models are `extra="allow"` server-side, so unknown | |
| 10 | + * keys may appear — never rely on them without adding them here. | |
| 11 | + */ | |
| 12 | + | |
| 13 | +export type Frequency = 'A' | 'Q' | 'M'; | |
| 14 | +export type ObservationStatus = 'verified' | 'imported' | 'warning' | 'stale' | 'quarantined'; | |
| 15 | +export type IndicatorFormat = | |
| 16 | + | 'number' | |
| 17 | + | 'percent' | |
| 18 | + | 'currency' | |
| 19 | + | 'index' | |
| 20 | + | 'years' | |
| 21 | + | 'per_1000' | |
| 22 | + | 'per_100k' | |
| 23 | + | 'per_million' | |
| 24 | + | 'ratio' | |
| 25 | + | 'celsius' | |
| 26 | + | 'tonnes' | |
| 27 | + | 'kwh' | |
| 28 | + | 'ha' | |
| 29 | + | 'km'; | |
| 30 | +export type IncomeGroup = 'HIC' | 'UMC' | 'LMC' | 'LIC'; | |
| 31 | +export type CountryStatus = 'country' | 'territory' | 'historical'; | |
| 32 | +export type SimilarityMode = 'overall' | 'economic' | 'demographic' | 'energy' | 'social'; | |
| 33 | +export type ChangeKind = | |
| 34 | + | 'yoy_drop' | |
| 35 | + | 'yoy_jump' | |
| 36 | + | 'record_high' | |
| 37 | + | 'record_low' | |
| 38 | + | 'n_year_high' | |
| 39 | + | 'n_year_low' | |
| 40 | + | 'sign_flip' | |
| 41 | + | 'accelerating' | |
| 42 | + | 'decelerating'; | |
| 43 | +export type SearchHitType = 'country' | 'indicator' | 'topic' | 'region' | 'source' | 'country_topic' | 'country_indicator'; | |
| 44 | +export type TopicId = | |
| 45 | + | 'economy' | |
| 46 | + | 'government' | |
| 47 | + | 'population' | |
| 48 | + | 'labor' | |
| 49 | + | 'income' | |
| 50 | + | 'housing' | |
| 51 | + | 'health' | |
| 52 | + | 'education' | |
| 53 | + | 'trade' | |
| 54 | + | 'energy' | |
| 55 | + | 'climate' | |
| 56 | + | 'environment' | |
| 57 | + | 'infrastructure' | |
| 58 | + | 'digital' | |
| 59 | + | 'innovation' | |
| 60 | + | 'agriculture' | |
| 61 | + | 'tourism' | |
| 62 | + | 'security' | |
| 63 | + | 'quality-of-life'; | |
| 64 | +export type DnaDimension = 'income' | 'demographics' | 'urbanization' | 'trade' | 'energy' | 'emissions' | 'innovation' | 'education' | 'public_spending'; | |
| 65 | + | |
| 66 | +// ---------------------------------------------------------------------------------------------- envelope & errors | |
| 67 | + | |
| 68 | +export interface Meta { | |
| 69 | + built_at: string | null; | |
| 70 | + run_id: string | null; | |
| 71 | + generated_at: string; | |
| 72 | +} | |
| 73 | + | |
| 74 | +/** RFC 7807 problem+json (errors.py `problem_body`). */ | |
| 75 | +export interface Problem { | |
| 76 | + type?: string; | |
| 77 | + title: string; | |
| 78 | + status: number; | |
| 79 | + detail?: string; | |
| 80 | + instance?: string; | |
| 81 | + resource?: string; | |
| 82 | + id?: string; | |
| 83 | +} | |
| 84 | + | |
| 85 | +// ---------------------------------------------------------------------------------------------- provenance & values | |
| 86 | + | |
| 87 | +export interface Provenance { | |
| 88 | + source: string | null; // worldbank | imf | oecd | eurostat | who | fred | owid | bis | ilo | |
| 89 | + source_name: string | null; | |
| 90 | + dataset: string | null; | |
| 91 | + series_code: string | null; | |
| 92 | + retrieved_at: string | null; | |
| 93 | + source_updated_at: string | null; | |
| 94 | + url: string | null; | |
| 95 | + transform: string | null; | |
| 96 | + licence: string | null; | |
| 97 | + /** Series endpoint `sources[]` only. */ | |
| 98 | + n_values?: number; | |
| 99 | +} | |
| 100 | + | |
| 101 | +export interface ChangeValue { | |
| 102 | + abs: number | null; | |
| 103 | + pct: number | null; | |
| 104 | + formatted: string | null; | |
| 105 | + /** `change_10y` only. */ | |
| 106 | + value_10y_ago?: number | null; | |
| 107 | +} | |
| 108 | + | |
| 109 | +export interface Prev { | |
| 110 | + period: string | null; | |
| 111 | + value: number | null; | |
| 112 | +} | |
| 113 | + | |
| 114 | +/** `[year, value]` — API sparklines (last ≤ 30 annual, non-forecast points). */ | |
| 115 | +export type SparkPoint = [number, number | null]; | |
| 116 | + | |
| 117 | +/** One indicator's latest value for a country (schemas.MetricValue) — headline & topic pages. */ | |
| 118 | +export interface MetricValue { | |
| 119 | + indicator: string; // slug | |
| 120 | + indicator_name: string | null; | |
| 121 | + has_data: boolean; | |
| 122 | + value: number | null; | |
| 123 | + /** Server-formatted display string (same rules as lib/format.ts) — prefer it when present. */ | |
| 124 | + formatted: string | null; | |
| 125 | + period: string | null; | |
| 126 | + year: number | null; | |
| 127 | + frequency: Frequency | string | null; | |
| 128 | + unit: string | null; | |
| 129 | + unit_short: string | null; | |
| 130 | + format: IndicatorFormat | string | null; | |
| 131 | + is_estimate: boolean; | |
| 132 | + is_forecast: boolean; | |
| 133 | + status: ObservationStatus | string | null; | |
| 134 | + prev: Prev | null; | |
| 135 | + change: ChangeValue | null; | |
| 136 | + change_10y: ChangeValue | null; | |
| 137 | + rank_world: number | null; | |
| 138 | + n_world: number | null; | |
| 139 | + rank_region: number | null; | |
| 140 | + n_region: number | null; | |
| 141 | + rank_income: number | null; | |
| 142 | + n_income: number | null; | |
| 143 | + rank_year: number | null; | |
| 144 | + rank_is_stale: boolean; | |
| 145 | + higher_is_better: boolean | null; | |
| 146 | + sparkline: SparkPoint[]; | |
| 147 | + provenance: Provenance | null; | |
| 148 | +} | |
| 149 | + | |
| 150 | +// ---------------------------------------------------------------------------------------------- cards | |
| 151 | + | |
| 152 | +export interface CountryCard { | |
| 153 | + id: string; // ISO3 | |
| 154 | + iso2: string | null; | |
| 155 | + slug: string | null; | |
| 156 | + name: string | null; | |
| 157 | + flag: string | null; | |
| 158 | + region: string | null; // WB region id (ECS, NAC …) | |
| 159 | + region_name: string | null; | |
| 160 | + income: IncomeGroup | string | null; | |
| 161 | + income_name: string | null; | |
| 162 | + kind: 'country' | 'aggregate' | string | null; | |
| 163 | +} | |
| 164 | + | |
| 165 | +export interface IndicatorCard { | |
| 166 | + id: string; | |
| 167 | + slug: string; | |
| 168 | + name: string | null; | |
| 169 | + short_name: string | null; | |
| 170 | + topic: TopicId | string | null; | |
| 171 | + subtopic: string | null; | |
| 172 | + unit: string | null; | |
| 173 | + unit_short: string | null; | |
| 174 | + format: IndicatorFormat | string | null; | |
| 175 | + precision: number | null; | |
| 176 | + frequency: Frequency | string | null; | |
| 177 | + aggregation: string | null; | |
| 178 | + higher_is_better: boolean | null; | |
| 179 | + ranking_eligible: boolean | null; | |
| 180 | + featured: boolean | null; | |
| 181 | +} | |
| 182 | + | |
| 183 | +export interface IndicatorSummary extends IndicatorCard { | |
| 184 | + description: string | null; | |
| 185 | + n_countries: number | null; | |
| 186 | + n_observations: number | null; | |
| 187 | + first_year: number | null; | |
| 188 | + last_year: number | null; | |
| 189 | + latest_source_updated_at: string | null; | |
| 190 | + primary_source_id: string | null; | |
| 191 | + coverage_pct: number | null; | |
| 192 | + tags?: string[]; | |
| 193 | +} | |
| 194 | + | |
| 195 | +export interface GroupCard { | |
| 196 | + id: string; | |
| 197 | + slug: string | null; | |
| 198 | + name: string | null; | |
| 199 | + kind: 'world' | 'region' | 'continent' | 'income' | 'org' | 'custom' | string | null; | |
| 200 | + wb_code: string | null; | |
| 201 | + n_members: number | null; | |
| 202 | +} | |
| 203 | + | |
| 204 | +// ---------------------------------------------------------------------------------------------- countries | |
| 205 | + | |
| 206 | +/** `/countries` item. */ | |
| 207 | +export interface CountrySummary extends CountryCard { | |
| 208 | + capital: string | null; | |
| 209 | + continent: string | null; | |
| 210 | + subregion: string | null; | |
| 211 | + population_latest: number | null; | |
| 212 | + population_year: number | null; | |
| 213 | + gdp_latest: number | null; | |
| 214 | + gdp_year: number | null; | |
| 215 | + gdp_per_capita_latest: number | null; | |
| 216 | + gdp_per_capita_year: number | null; | |
| 217 | + coverage_pct: number | null; | |
| 218 | + n_indicators: number | null; | |
| 219 | +} | |
| 220 | + | |
| 221 | +export interface CountriesResponse { | |
| 222 | + meta: Meta; | |
| 223 | + n: number; | |
| 224 | + filters: { region: string | null; income: string | null; q: string | null; sort: string }; | |
| 225 | + items: CountrySummary[]; | |
| 226 | +} | |
| 227 | + | |
| 228 | +export interface Country extends CountryCard { | |
| 229 | + official_name: string | null; | |
| 230 | + iso3: string | null; | |
| 231 | + iso_numeric: string | null; | |
| 232 | + capital: string | null; | |
| 233 | + continent: string | null; | |
| 234 | + subregion: string | null; | |
| 235 | + currency_code: string | null; | |
| 236 | + currency_name: string | null; | |
| 237 | + area_km2: number | null; | |
| 238 | + latitude: number | null; | |
| 239 | + longitude: number | null; | |
| 240 | + un_member: boolean | null; | |
| 241 | + independent: boolean | null; | |
| 242 | + landlocked: boolean | null; | |
| 243 | + borders: string[] | null; | |
| 244 | + languages: string[] | null; | |
| 245 | + demonym: string | null; | |
| 246 | + status: CountryStatus | string | null; | |
| 247 | +} | |
| 248 | + | |
| 249 | +export interface Coverage { | |
| 250 | + n_indicators: number | null; | |
| 251 | + n_observations: number | null; | |
| 252 | + latest_year: number | null; | |
| 253 | + coverage_pct: number | null; | |
| 254 | + updated_at: string | null; | |
| 255 | +} | |
| 256 | + | |
| 257 | +export interface Freshness { | |
| 258 | + source_updated_at: string | null; | |
| 259 | + retrieved_at: string | null; | |
| 260 | + built_at: string | null; | |
| 261 | +} | |
| 262 | + | |
| 263 | +export interface TopicSummary { | |
| 264 | + id: TopicId | string; | |
| 265 | + name: string; | |
| 266 | + short: string | null; | |
| 267 | + order: number | null; | |
| 268 | + blurb: string | null; | |
| 269 | + n_indicators: number; | |
| 270 | + n_with_data: number; | |
| 271 | +} | |
| 272 | + | |
| 273 | +export interface CountryResponse { | |
| 274 | + meta: Meta; | |
| 275 | + country: Country; | |
| 276 | + groups: GroupCard[]; | |
| 277 | + coverage: Coverage | null; | |
| 278 | + freshness: Freshness; | |
| 279 | + headline: MetricValue[]; | |
| 280 | + topics: TopicSummary[]; | |
| 281 | + neighbours: CountryCard[]; | |
| 282 | +} | |
| 283 | + | |
| 284 | +export interface SubtopicBlock { | |
| 285 | + subtopic: string; | |
| 286 | + indicators: MetricValue[]; | |
| 287 | +} | |
| 288 | + | |
| 289 | +export interface CountryTopicResponse { | |
| 290 | + meta: Meta; | |
| 291 | + country: CountryCard; | |
| 292 | + topic: { id: TopicId | string; name: string; short: string | null; order: number | null; blurb: string | null }; | |
| 293 | + n_with_data: number; | |
| 294 | + n_indicators: number; | |
| 295 | + subtopics: SubtopicBlock[]; | |
| 296 | +} | |
| 297 | + | |
| 298 | +// ---------------------------------------------------------------------------------------------- series | |
| 299 | + | |
| 300 | +export interface SeriesValue { | |
| 301 | + period: string | null; | |
| 302 | + year: number | null; | |
| 303 | + frequency: Frequency | string | null; | |
| 304 | + value: number | null; | |
| 305 | + is_forecast: boolean; | |
| 306 | + is_estimate: boolean; | |
| 307 | + status: ObservationStatus | string | null; | |
| 308 | + source_id: string | null; | |
| 309 | + provenance: Provenance | null; | |
| 310 | +} | |
| 311 | + | |
| 312 | +export interface SeriesStats { | |
| 313 | + min: { year: number; value: number } | null; | |
| 314 | + max: { year: number; value: number } | null; | |
| 315 | + first: { year: number; value: number } | null; | |
| 316 | + last: { year: number; value: number } | null; | |
| 317 | + cagr: number | null; // % per year | |
| 318 | + n: number; | |
| 319 | +} | |
| 320 | + | |
| 321 | +export interface Series { | |
| 322 | + indicator: IndicatorCard; | |
| 323 | + country: CountryCard; | |
| 324 | + unit: string | null; | |
| 325 | + frequency: Frequency | string | null; | |
| 326 | + values: SeriesValue[]; | |
| 327 | + alternatives: SeriesValue[] | null; | |
| 328 | + provenance: Provenance | null; | |
| 329 | + sources: Provenance[]; | |
| 330 | + stats: SeriesStats; | |
| 331 | +} | |
| 332 | + | |
| 333 | +export interface SeriesResponse extends Series { | |
| 334 | + meta: Meta; | |
| 335 | +} | |
| 336 | + | |
| 337 | +export interface MultiSeriesResponse { | |
| 338 | + meta: Meta; | |
| 339 | + n: number; | |
| 340 | + series: Series[]; | |
| 341 | +} | |
| 342 | + | |
| 343 | +// ---------------------------------------------------------------------------------------------- changes / events | |
| 344 | + | |
| 345 | +export interface ChangeItem { | |
| 346 | + id: string | null; | |
| 347 | + country: CountryCard | null; | |
| 348 | + indicator: IndicatorCard | { id: string; slug: string }; | |
| 349 | + kind: ChangeKind | string | null; | |
| 350 | + period: string | null; | |
| 351 | + year: number | null; | |
| 352 | + value: number | null; | |
| 353 | + ref_value: number | null; | |
| 354 | + delta: number | null; | |
| 355 | + delta_pct: number | null; | |
| 356 | + window_years: number | null; | |
| 357 | + severity: number | null; // 0–1 | |
| 358 | + headline: string | null; | |
| 359 | + detail: unknown; | |
| 360 | + detected_at: string | null; | |
| 361 | + formatted: string | null; | |
| 362 | + provenance: Provenance | null; | |
| 363 | +} | |
| 364 | + | |
| 365 | +export interface ChangesResponse { | |
| 366 | + meta: Meta; | |
| 367 | + n: number; | |
| 368 | + items: ChangeItem[]; | |
| 369 | +} | |
| 370 | + | |
| 371 | +// ---------------------------------------------------------------------------------------------- similar / insights / dna | |
| 372 | + | |
| 373 | +/** `contributions` JSON: {indicator: {z_a, z_b, weight, contribution}} (ARCHITECTURE §2.1). */ | |
| 374 | +export type Contributions = Record<string, { z_a?: number | null; z_b?: number | null; weight?: number | null; contribution?: number | null; value_a?: number | null; value_b?: number | null }>; | |
| 375 | + | |
| 376 | +export interface SimilarPeer { | |
| 377 | + country: CountryCard; | |
| 378 | + score: number | null; // 0–100 | |
| 379 | + rank: number | null; | |
| 380 | + contributions: Contributions | string | null; | |
| 381 | +} | |
| 382 | + | |
| 383 | +export interface SimilarResponse { | |
| 384 | + meta: Meta; | |
| 385 | + country: CountryCard; | |
| 386 | + mode: SimilarityMode | string; | |
| 387 | + modes: string[]; | |
| 388 | + peers: SimilarPeer[]; | |
| 389 | +} | |
| 390 | + | |
| 391 | +export interface Insight { | |
| 392 | + id: string | null; | |
| 393 | + template_id: string | null; | |
| 394 | + text: string; | |
| 395 | + values: unknown; | |
| 396 | + indicators: string[]; | |
| 397 | + computed_at: string | null; | |
| 398 | + provenance: Provenance[]; | |
| 399 | +} | |
| 400 | + | |
| 401 | +export interface InsightsResponse { | |
| 402 | + meta: Meta; | |
| 403 | + country: CountryCard; | |
| 404 | + items: Insight[]; | |
| 405 | +} | |
| 406 | + | |
| 407 | +export interface DnaDimensionRow { | |
| 408 | + id: DnaDimension | string; | |
| 409 | + label: string; | |
| 410 | + indicator: string | null; | |
| 411 | + value: number | null; | |
| 412 | +} | |
| 413 | + | |
| 414 | +export interface DNAResponse { | |
| 415 | + meta: Meta; | |
| 416 | + country: CountryCard; | |
| 417 | + dims: Partial<Record<DnaDimension, number | null>>; | |
| 418 | + year_ref: number | null; | |
| 419 | + dimensions: DnaDimensionRow[]; | |
| 420 | +} | |
| 421 | + | |
| 422 | +// ---------------------------------------------------------------------------------------------- maps & rankings | |
| 423 | + | |
| 424 | +export interface MapLegend { | |
| 425 | + min: number | null; | |
| 426 | + max: number | null; | |
| 427 | + /** k − 1 interior quantile breaks (3–7 classes). */ | |
| 428 | + breaks: number[]; | |
| 429 | + n_classes: number; | |
| 430 | +} | |
| 431 | + | |
| 432 | +export interface MapResponse { | |
| 433 | + meta: Meta; | |
| 434 | + indicator: IndicatorCard; | |
| 435 | + year: number | null; | |
| 436 | + year_used: number | null; | |
| 437 | + nearest: boolean; | |
| 438 | + values: Record<string, number | null>; // ISO3 → value | |
| 439 | + years: Record<string, number> | null; | |
| 440 | + formatted: Record<string, string> | null; | |
| 441 | + legend: MapLegend; | |
| 442 | + n: number; | |
| 443 | + provenance: Provenance | null; | |
| 444 | + sources: Provenance[]; | |
| 445 | +} | |
| 446 | + | |
| 447 | +export interface RankingRow { | |
| 448 | + rank: number; | |
| 449 | + rank_world: number | null; | |
| 450 | + n_world: number | null; | |
| 451 | + pct_rank: number | null; | |
| 452 | + country: CountryCard; | |
| 453 | + value: number | null; | |
| 454 | + formatted: string | null; | |
| 455 | + year: number | null; | |
| 456 | + change_1y: ChangeValue | null; | |
| 457 | + change_10y: ChangeValue | null; | |
| 458 | + sparkline: SparkPoint[]; | |
| 459 | + provenance: Provenance | null; | |
| 460 | +} | |
| 461 | + | |
| 462 | +export interface RankingResponse { | |
| 463 | + meta: Meta; | |
| 464 | + indicator: IndicatorCard; | |
| 465 | + group: GroupCard; | |
| 466 | + year: number | null; | |
| 467 | + year_used: number | null; | |
| 468 | + years_available: number[]; | |
| 469 | + sort: 'asc' | 'desc' | string; | |
| 470 | + n: number; | |
| 471 | + limit: number; | |
| 472 | + offset: number; | |
| 473 | + rows: RankingRow[]; | |
| 474 | +} | |
| 475 | + | |
| 476 | +// ---------------------------------------------------------------------------------------------- home / search / health | |
| 477 | + | |
| 478 | +export interface GlobalSnapshot { | |
| 479 | + world_population: number | null; | |
| 480 | + world_population_formatted: string | null; | |
| 481 | + world_population_year: number | null; | |
| 482 | + world_gdp: number | null; | |
| 483 | + world_gdp_formatted: string | null; | |
| 484 | + world_gdp_year: number | null; | |
| 485 | + median_life_expectancy: number | null; | |
| 486 | + median_life_expectancy_year: number | null; | |
| 487 | + n_countries: number; | |
| 488 | + n_territories: number; | |
| 489 | + n_indicators: number; | |
| 490 | + n_indicators_with_data: number; | |
| 491 | + n_observations: number; | |
| 492 | + n_sources: number; | |
| 493 | + built_at: string | null; | |
| 494 | + run_id: string | null; | |
| 495 | + note: string | null; | |
| 496 | +} | |
| 497 | + | |
| 498 | +export interface HomeListRow { | |
| 499 | + rank: number; | |
| 500 | + country: CountryCard; | |
| 501 | + value: number | null; | |
| 502 | + formatted: string | null; | |
| 503 | + year: number | null; | |
| 504 | + change_pct: number | null; | |
| 505 | + change_abs: number | null; | |
| 506 | + rank_world: number | null; | |
| 507 | + n_world: number | null; | |
| 508 | + provenance: Provenance | null; | |
| 509 | +} | |
| 510 | + | |
| 511 | +export type HomeListKey = 'largest_economies' | 'fastest_population_growth' | 'highest_life_expectancy' | 'energy_transition_leaders' | 'highest_gdp_per_capita_ppp' | 'lowest_unemployment'; | |
| 512 | + | |
| 513 | +export interface HomeList { | |
| 514 | + title: string; | |
| 515 | + description: string | null; | |
| 516 | + indicator: IndicatorCard; | |
| 517 | + sort: 'asc' | 'desc'; | |
| 518 | + rows: HomeListRow[]; | |
| 519 | +} | |
| 520 | + | |
| 521 | +export interface HomeResponse { | |
| 522 | + meta: Meta; | |
| 523 | + snapshot: GlobalSnapshot; | |
| 524 | + lists: Partial<Record<HomeListKey, HomeList>> & Record<string, HomeList>; | |
| 525 | + recent_changes: ChangeItem[]; | |
| 526 | + recently_updated: IndicatorSummary[]; | |
| 527 | + featured_indicators: IndicatorSummary[]; | |
| 528 | + trending: IndicatorSummary[]; | |
| 529 | +} | |
| 530 | + | |
| 531 | +export interface SearchHit { | |
| 532 | + type: SearchHitType | string; | |
| 533 | + id: string; | |
| 534 | + slug: string | null; | |
| 535 | + name: string; | |
| 536 | + hint: string | null; | |
| 537 | + score: number; | |
| 538 | + /** Site-relative URL chosen by the API (`/countries/canada`, `/countries/canada/economy` …). */ | |
| 539 | + url: string | null; | |
| 540 | + country: CountryCard | null; | |
| 541 | + topic: string | null; | |
| 542 | + indicator: string | null; | |
| 543 | +} | |
| 544 | + | |
| 545 | +export interface SearchResponse { | |
| 546 | + meta: Meta; | |
| 547 | + q: string; | |
| 548 | + n: number; | |
| 549 | + hits: SearchHit[]; | |
| 550 | +} | |
| 551 | + | |
| 552 | +export interface HealthResponse { | |
| 553 | + status: 'ok' | 'empty' | 'degraded' | string; | |
| 554 | + run_id: string | null; | |
| 555 | + built_at: string | null; | |
| 556 | + observations: number | null; | |
| 557 | + countries: number | null; | |
| 558 | + indicators: number | null; | |
| 559 | + db_path: string | null; | |
| 560 | + version: string | null; | |
| 561 | + cache: Record<string, number> | null; | |
| 562 | +} | |
| 563 | + | |
| 564 | +// ---------------------------------------------------------------------------------------------- helpers shared by components | |
| 565 | + | |
| 566 | +/** Minimal display spec derived from a MetricValue or an IndicatorCard (what lib/format.ts needs). */ | |
| 567 | +export interface FormatSpec { | |
| 568 | + format: IndicatorFormat | string | null | undefined; | |
| 569 | + unit?: string | null; | |
| 570 | + unit_short?: string | null; | |
| 571 | + precision?: number | null; | |
| 572 | + frequency?: Frequency | string | null; | |
| 573 | + name?: string | null; | |
| 574 | + higher_is_better?: boolean | null; | |
| 575 | +} | |
added
apps/web/tsconfig.json
+22 −0
@@ -0,0 +1,22 @@ | ||
| 1 | +{ | |
| 2 | + "compilerOptions": { | |
| 3 | + "target": "ES2022", | |
| 4 | + "lib": ["dom", "dom.iterable", "esnext"], | |
| 5 | + "allowJs": true, | |
| 6 | + "skipLibCheck": true, | |
| 7 | + "strict": true, | |
| 8 | + "noUncheckedIndexedAccess": true, | |
| 9 | + "noEmit": true, | |
| 10 | + "esModuleInterop": true, | |
| 11 | + "module": "esnext", | |
| 12 | + "moduleResolution": "bundler", | |
| 13 | + "resolveJsonModule": true, | |
| 14 | + "isolatedModules": true, | |
| 15 | + "jsx": "react-jsx", | |
| 16 | + "incremental": true, | |
| 17 | + "plugins": [{ "name": "next" }], | |
| 18 | + "paths": { "@/*": ["./src/*"] } | |
| 19 | + }, | |
| 20 | + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", ".next/dev/types/**/*.ts"], | |
| 21 | + "exclude": ["node_modules", "qa"] | |
| 22 | +} | |
added
docs/sources-research.md
+316 −0
@@ -0,0 +1,316 @@ | ||
| 1 | +# CountryAtlas — Public data sources, verified 2026-09-11 | |
| 2 | + | |
| 3 | +Every URL below was called with `curl` on 2026-09-11 from Québec (all times UTC). "VERIFIED" = HTTP 200 with data shown. | |
| 4 | +"UNVERIFIED" = could not confirm; what was tried is stated. Samples are trimmed. | |
| 5 | + | |
| 6 | +Summary table | |
| 7 | + | |
| 8 | +| Source | Base URL | Auth | Rate limit (observed / documented) | Country code | Time format | Licence | | |
| 9 | +|---|---|---|---|---|---|---| | |
| 10 | +| World Bank WDI v2 | `https://api.worldbank.org/v2/` | none | none published; Cloudflare-cached (`cache-control: max-age≈30000`) | ISO3 in `countryiso3code`, WB 2-letter in `country.id` | `"2025"` (year string) | CC BY 4.0 (attribute "World Bank, World Development Indicators") | | |
| 11 | +| IMF WEO (SDMX) | `https://api.imf.org/external/sdmx/{2.1,3.0}/` | none | none observed; 3.6 MB CSV for all countries in <10 s | ISO3 (`CAN`) + IMF group codes (`G001`…) | year `2024` | © IMF, free reuse with citation (see `LICENSE` field) | | |
| 12 | +| OECD SDMX | `https://sdmx.oecd.org/public/rest/` | none | doc: 60 req/h per IP for anonymous… see §3 (not enforced on our tests) | ISO3 (`REF_AREA`) | `2024`, `2026-06`, `2026-Q1` | CC BY 4.0 | | |
| 13 | +| Eurostat JSON-stat | `https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/` | none | doc: no hard limit; large queries may be refused | Eurostat geo (`EL`, `EU27_2020`) | `2025`, `2025-M06`, `2025-Q1` | CC BY 4.0 | | |
| 14 | +| WHO GHO OData | `https://ghoapi.azureedge.net/api/` | none | none published; CDN `max-age=3600` | ISO3 (`SpatialDim`) | `TimeDim` int year | CC BY-NC-SA 3.0 IGO | | |
| 15 | +| FRED | `https://api.stlouisfed.org/fred/` | `api_key` | 120 req/min; bursts trigger 429 then Akamai **403** for several minutes | US mostly; ISO2 embedded in ids (`QCAR628BIS`) | `2026-08-01` (ISO date) | FRED ToU; third-party series keep their own terms | | |
| 16 | +| OWID | `raw.githubusercontent.com/owid/...`, `ourworldindata.org/grapher/` | none | GitHub raw: reasonable; grapher CSV: none published | ISO3 in `iso_code`/`code`; `OWID_*` for aggregates | year int | CC BY 4.0 | | |
| 17 | +| UN WPP portal API | `https://population.un.org/dataportalapi/api/v1/` | **Bearer token for `/data/*`** | n/a | UN M49 numeric location ids | year | CC BY 3.0 IGO | | |
| 18 | +| UN Comtrade | `https://comtradeapi.un.org/` | **subscription key** (free `/public/v1/preview` limited) | preview: 500 records/call | M49 numeric (`124`) | `period=2023` | UN ToU | | |
| 19 | +| BIS SDMX v2 | `https://stats.bis.org/api/v2/` | none | none observed | **ISO2** (`CA`, `US`, `XM`=euro area) | `2026-08`, `2026-Q1` | BIS terms (free with attribution) | | |
| 20 | +| ILOSTAT SDMX | `https://sdmx.ilo.org/rest/` | none | `cache-control: no-store`; none published | ISO3 | year | CC BY 4.0 | | |
| 21 | + | |
| 22 | +--- | |
| 23 | + | |
| 24 | +## 1. World Bank Indicators API v2 — VERIFIED | |
| 25 | + | |
| 26 | +Base `https://api.worldbank.org/v2/`, no auth, `format=json`. Response is a 2-element array: `[meta, rows]`. | |
| 27 | +Meta: `{"page":1,"pages":1,"per_page":20000,"total":17490,"sourceid":"2","lastupdated":"2026-07-13"}` — **`lastupdated` is the WDI vintage** (2026-07-13 on all WDI queries). | |
| 28 | + | |
| 29 | +### Country list (regions, income groups, aggregates) | |
| 30 | +`GET /country?format=json&per_page=400` → `total: 295` (217 economies + 78 aggregates). Aggregates have `region.id == "NA"` (and `incomeLevel.id == "NA"`). | |
| 31 | +```json | |
| 32 | +{"id":"CAN","iso2Code":"CA","name":"Canada","region":{"id":"NAC","iso2code":"XU","value":"North America"}, | |
| 33 | + "adminregion":{"id":"","iso2code":"","value":""},"incomeLevel":{"id":"HIC","iso2code":"XD","value":"High income"}, | |
| 34 | + "lendingType":{"id":"LNX","iso2code":"XX","value":"Not classified"},"capitalCity":"Ottawa","longitude":"-75.6919","latitude":"45.4215"} | |
| 35 | +{"id":"WLD","iso2Code":"1W","name":"World","region":{"id":"NA","iso2code":"NA","value":"Aggregates"}, ...} | |
| 36 | +``` | |
| 37 | +Filter aggregates with `region.id == "NA"` → 78 (WLD, EUU, HIC, OED, LIC, LMC, UMC, AFE, …). Real economies: 217. | |
| 38 | + | |
| 39 | +### Indicator data, all countries | |
| 40 | +`GET /country/all/indicator/NY.GDP.PCAP.CD?format=json&per_page=20000&date=1960:2026` → 17 490 rows, 1 page. | |
| 41 | +```json | |
| 42 | +{"indicator":{"id":"NY.GDP.PCAP.CD","value":"GDP per capita (current US$)"},"country":{"id":"1W","value":"World"}, | |
| 43 | + "countryiso3code":"WLD","date":"2025","value":14405.8484595528,"unit":"","obs_status":"","decimal":1} | |
| 44 | +``` | |
| 45 | +- Rows are sorted country → year **descending**. Missing values: `"value": null` (2 745 of 17 490 here). `obs_status` is essentially always `""`. | |
| 46 | +- **Data rows include aggregates** (WLD, EUU, HIC…). Join on `countryiso3code` with the country list to drop them. | |
| 47 | +- **Quirk**: 5 income-group aggregates (`XD` High income, `XM` Low, `XN` Lower-middle, `XT` Upper-middle, `XY` Not classified) come back with **`countryiso3code: ""`** — use `country.id` (2-letter) as a fallback key. | |
| 48 | +- `per_page`: default 50 (→ `pages: 350`). Tested 20 000, 25 000 and 32 000 → OK. **50 000 → HTTP "Request Error" XML page**. Use ≤ 32 000; one indicator for all countries/years fits in one page (~17.5k rows). | |
| 49 | +- `date=1960:2026` (range) or `date=2024`; `mrv=N` = N most-recent values per country; `mrnev=1` = most recent **non-empty** value (returns fewer rows, e.g. Gini CAN → 2022); `gapfill=Y` with `mrv` forward-fills (Gini CAN 2023–2025 all 31.5 — beware). | |
| 50 | +- Several countries: `/country/CAN;USA/indicator/...`. Several indicators: `/country/CAN/indicator/SP.POP.TOTL;NY.GDP.MKTP.CD?source=2` (needs `source=`). | |
| 51 | +- Retired code → HTTP 200 with `[{"message":[{"id":"175","key":"Invalid format","value":"The indicator was not found. It may have been deleted or archived."}]}]` — **check for `.[0].message`**, not the status code. | |
| 52 | + | |
| 53 | +### Indicator metadata | |
| 54 | +`GET /indicator/NY.GDP.PCAP.CD?format=json` → | |
| 55 | +```json | |
| 56 | +{"id":"NY.GDP.PCAP.CD","name":"GDP per capita (current US$)","unit":"","source":{"id":"2","value":"World Development Indicators"}, | |
| 57 | + "sourceNote":"Gross domestic product is the total income earned ...","sourceOrganization":"Country official statistics, ...; OECD; Staff estimates, World Bank (WB)", | |
| 58 | + "topics":[{"id":"3","value":"Economy & Growth"}]} | |
| 59 | +``` | |
| 60 | +Full catalogue: `/indicator?format=json&per_page=30000` (29 544 indicators, 6 MB) — filter locally (`source.id=="2"` → 1 498 WDI indicators). The `q=` search parameter is **ignored** (returns the full list). `/source?format=json&per_page=100` → 71 sources; `/source/2?format=json` → `{"id":"2","lastupdated":"2026-07-13","name":"World Development Indicators","code":"WDI"}`. | |
| 61 | + | |
| 62 | +### Indicator code audit (`mrnev=1&per_page=400` on `/country/all/...`) | |
| 63 | +**Exist and return data (94/96)** — latest year with data (CAN value in brackets): | |
| 64 | + | |
| 65 | +| Code | Latest | Code | Latest | Code | Latest | | |
| 66 | +|---|---|---|---|---|---| | |
| 67 | +| NY.GDP.MKTP.CD | 2025 | NY.GDP.PCAP.CD | 2025 (55 698) | NY.GDP.PCAP.PP.CD | 2025 | | |
| 68 | +| NY.GDP.MKTP.KD.ZG | 2025 | FP.CPI.TOTL.ZG | 2025 | SL.UEM.TOTL.ZS | 2025 | | |
| 69 | +| SP.POP.TOTL | 2025 | SP.POP.GROW | 2025 | SP.DYN.LE00.IN | 2024 | | |
| 70 | +| SP.DYN.TFRT.IN | 2024 | SP.URB.TOTL.IN.ZS | 2025 | GC.DOD.TOTL.GD.ZS | 2024 (120 countries only) | | |
| 71 | +| **EN.GHG.CO2.PC.CE.AR5** | 2024 (14.0 t) | EG.ELC.RNEW.ZS | **2021** | IT.NET.USER.ZS | 2025 | | |
| 72 | +| GB.XPD.RSDV.GD.ZS | 2024 | NE.EXP.GNFS.ZS | 2025 | NE.TRD.GNFS.ZS | 2025 | | |
| 73 | +| SH.XPD.CHEX.GD.ZS | 2024 | SE.XPD.TOTL.GD.ZS | 2025 | SP.DYN.IMRT.IN | 2024 | | |
| 74 | +| SI.POV.GINI | 2025 (CAN 2022) | SL.TLF.CACT.ZS | 2025 | MS.MIL.XPND.GD.ZS | 2024 | | |
| 75 | +| EG.USE.PCAP.KG.OE | 2024 | EG.FEC.RNEW.ZS | 2022 | AG.LND.FRST.ZS | 2023 | | |
| 76 | +| EN.ATM.PM25.MC.M3 | 2023 | IT.CEL.SETS.P2 | 2025 | IT.NET.BBND.P2 | 2025 | | |
| 77 | +| ST.INT.ARVL | **2020** | BN.CAB.XOKA.GD.ZS | 2025 | GC.REV.XGRT.GD.ZS | 2024 | | |
| 78 | +| GC.XPN.TOTL.GD.ZS | 2024 | SH.MED.PHYS.ZS | 2023 | SH.MED.BEDS.ZS | 2023 | | |
| 79 | +| SM.POP.NETM | 2025 | SP.POP.65UP.TO.ZS | 2025 | SP.POP.0014.TO.ZS | 2025 | | |
| 80 | +| NV.AGR.TOTL.ZS | 2025 | NV.IND.TOTL.ZS | 2025 | NV.SRV.TOTL.ZS | 2025 | | |
| 81 | +| FR.INR.LEND | 2025 (CAN stops 2017) | PA.NUS.FCRF | 2025 | EG.ELC.ACCS.ZS | 2024 | | |
| 82 | +| SH.STA.SUIC.P5 | 2021 | SH.PRV.SMOK | 2024 | SE.ADT.LITR.ZS | 2024 (no CAN) | | |
| 83 | +| SE.TER.ENRR | 2025 | SE.SEC.ENRR | 2025 | SE.PRM.ENRR | 2025 | | |
| 84 | +| SL.UEM.1524.ZS | 2025 | AG.PRD.FOOD.XD | 2022 | AG.LND.AGRI.ZS | 2023 | | |
| 85 | +| NY.GDP.TOTL.RT.ZS | 2021 | ER.LND.PTLD.ZS | 2025 | EN.POP.DNST | 2023 | | |
| 86 | +| SP.POP.DPND | 2025 | VC.IHR.PSRC.P5 | 2023 | SH.XPD.CHEX.PC.CD | 2024 | | |
| 87 | +| NY.GDP.PCAP.KD.ZG | 2025 | NE.GDI.TOTL.ZS | 2025 | NE.CON.PRVT.ZS | 2025 | | |
| 88 | +| FM.LBL.BMNY.GD.ZS | 2025 (CAN stops 2008) | BX.KLT.DINV.WD.GD.ZS | 2025 | DT.DOD.DECT.GN.ZS | 2024 (133 low/middle income only) | | |
| 89 | +| SL.GDP.PCAP.EM.KD | 2025 | EG.IMP.CONS.ZS | 2023 | EG.EGY.PRIM.PP.KD | 2022 | | |
| 90 | +| SH.DYN.MORT | 2024 | SH.DYN.NCOM.ZS | 2021 | SP.ADO.TFRT | 2024 | | |
| 91 | +| SH.STA.OWAD.ZS | 2022 (**overweight**, not obesity; source 16 HNP) | SP.DYN.CBRT.IN | 2024 | SP.DYN.CDRT.IN | 2024 | | |
| 92 | +| SM.POP.TOTL.ZS | 2024 | IP.PAT.RESD | 2021 | IP.JRN.ARTC.SC | 2023 | | |
| 93 | +| TX.VAL.TECH.MF.ZS | 2024 | LP.LPI.OVRL.XQ | 2022 | IS.AIR.PSGR | 2023 | | |
| 94 | +| IS.RRS.TOTL.KM | 2021 | EG.ELC.FOSL.ZS | 2023 | EG.ELC.HYRO.ZS | 2024 | | |
| 95 | +| EG.ELC.NUCL.ZS | 2024 | EG.ELC.RNWX.ZS | 2021 | **EN.GHG.CO2.MT.CE.AR5** | 2024 | | |
| 96 | +| **EN.GHG.CH4.MT.CE.AR5** | 2024 | GC.TAX.TOTL.GD.ZS | 2024 | SH.H2O.BASW.ZS | 2024 | | |
| 97 | +| SE.PRM.CMPT.ZS | 2025 | | | | | | |
| 98 | + | |
| 99 | +**Missing (2/96)**: | |
| 100 | +- `EN.ATM.CO2E.PC` — retired. Replacement **`EN.GHG.CO2.PC.CE.AR5`** (t CO2e/capita, excl. LULUCF, source EDGAR). Sibling codes verified: `EN.GHG.CO2.MT.CE.AR5`, `EN.GHG.CH4.MT.CE.AR5`, `EN.GHG.N2O.MT.CE.AR5`, `EN.GHG.ALL.MT.CE.AR5` (total GHG Mt), `EN.GHG.ALL.PC.CE.AR5` (total GHG per capita, CAN 2024 = 18.6). | |
| 101 | +- `SM.POP.REFG` — listed in the catalogue under source 57 "WDI Database Archives" but **returns "not found" both with and without `source=57`**. Replacement: UNHCR Refugee Data Finder API (`https://api.unhcr.org/population/v1/population/?year=2024&coa=CAN`, no key) or OWID grapher `refugee-population-by-country-or-territory-of-asylum`. Not tested here. | |
| 102 | +- Obesity: WB has no adult obesity (BMI≥30) indicator (catalogue search found only overweight `SH.STA.OWAD.ZS` and HEFPI mean-BMI). Use **WHO `NCD_BMI_30A`** (§5). | |
| 103 | + | |
| 104 | +--- | |
| 105 | + | |
| 106 | +## 2. IMF World Economic Outlook — VERIFIED (no key needed) | |
| 107 | + | |
| 108 | +- Legacy `dataservices.imf.org/REST/SDMX_JSON.svc` → connection fails (retired). | |
| 109 | +- New portal `https://data.imf.org` exposes **SDMX 2.1 and 3.0 REST at `https://api.imf.org/external/sdmx/`**, **no API key**, no subscription. (An IMF "Data Portal Subscription" only raises quotas; anonymous works.) | |
| 110 | +- Dataflow **`IMF.RES,WEO`** (agency `IMF.RES`, id `WEO`, version 9.0.0), DSD `DSD_WEO`. | |
| 111 | +- **Dimension order: `COUNTRY.INDICATOR.FREQUENCY`** (+ `TIME_PERIOD`); FREQUENCY is always `A`. | |
| 112 | +- Current vintage: `PUBLICATION_DATE 2026-04-14` = **April 2026 WEO** (visible in CSV metadata). Years 1980–2031 (52 periods). | |
| 113 | + | |
| 114 | +### Sample query (2.1, CSV — recommended: flat, includes projection metadata) | |
| 115 | +``` | |
| 116 | +GET https://api.imf.org/external/sdmx/2.1/data/IMF.RES,WEO/CAN+USA.NGDP_RPCH+PCPIPCH+LUR+GGXWDG_NGDP+GGXCNL_NGDP+BCA_NGDPD+PPPPC.A?startPeriod=2024&endPeriod=2031 | |
| 117 | +Accept: text/csv | |
| 118 | +``` | |
| 119 | +``` | |
| 120 | +DATAFLOW,COUNTRY,INDICATOR,FREQUENCY,TIME_PERIOD,OBS_VALUE,SCALE,...,LATEST_ACTUAL_ANNUAL_DATA,... | |
| 121 | +IMF.RES:WEO(9.0.0),CAN,LUR,A,2024,6.366667,0,...,2025,... | |
| 122 | +IMF.RES:WEO(9.0.0),CAN,LUR,A,2025,6.858333,0,...,2025,... | |
| 123 | +IMF.RES:WEO(9.0.0),CAN,LUR,A,2026,6.521412,0,...,2025,... | |
| 124 | +``` | |
| 125 | +- **Projection flag: column `LATEST_ACTUAL_ANNUAL_DATA`** (per country×indicator; here `2025`) = "estimates start after". Any `TIME_PERIOD > LATEST_ACTUAL_ANNUAL_DATA` is a **forecast**. Present in CSV output only (2.1 XML has it as series attribute too; SDMX-JSON 3.0 exposes it in `attributes` if requested with `attributes=all`, not confirmed). | |
| 126 | +- CSV also carries `LICENSE` ("© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm") and `SUGGESTED_CITATION`: *"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [date]."* | |
| 127 | +- All countries: leave the COUNTRY segment empty: `/data/IMF.RES,WEO/.NGDP_RPCH.A?startPeriod=2024` → 1 640 rows, 213 areas (incl. aggregates `G001` World etc.), 3.6 MB CSV (the CSV repeats ~40 metadata columns per row — parse `COUNTRY,INDICATOR,TIME_PERIOD,OBS_VALUE,LATEST_ACTUAL_ANNUAL_DATA`). | |
| 128 | +- Default 2.1 output (no Accept) is **SDMX-ML StructureSpecific XML**: `<Series COUNTRY="CAN" INDICATOR="NGDP_RPCH" FREQUENCY="A" ...><Obs TIME_PERIOD="2026" OBS_VALUE="1.499992"/>`. Asking `Accept: application/vnd.sdmx.data+json` on 2.1 still returns XML (JSON not served on 2.1). | |
| 129 | +- Quirk: the `+` separator in the 2.1 key with the JSON Accept header once returned HTTP 500; CSV/XML fine. `dataflow/IMF.RES/WEO/+` on 2.1 → 204 (use 3.0 for structures). | |
| 130 | + | |
| 131 | +### SDMX 3.0 (JSON) | |
| 132 | +``` | |
| 133 | +GET https://api.imf.org/external/sdmx/3.0/data/dataflow/IMF.RES/WEO/+/CAN.NGDP_RPCH.A | |
| 134 | +Accept: application/vnd.sdmx.data+json;version=2.0.0 | |
| 135 | +``` | |
| 136 | +```json | |
| 137 | +{"data":{"dataSets":[{"series":{"0:0:0":{"attributes":[0,0,0,"9/25/2025"],"observations":{"0":["2.162741"],"1":["3.503078"],...}}}}], | |
| 138 | + "structures":[{"dimensions":{"series":[{"id":"COUNTRY","keyPosition":0},{"id":"INDICATOR","keyPosition":1},{"id":"FREQUENCY","keyPosition":2}], | |
| 139 | + "observation":[{"id":"TIME_PERIOD","values":[{"value":"1980"},...]}]}}]}} | |
| 140 | +``` | |
| 141 | +- Wildcard all countries: `.../WEO/+/*.NGDP_RPCH.A` → 210 areas, 194 KB. | |
| 142 | +- Time filter must be **URL-encoded**: `?c%5BTIME_PERIOD%5D=ge:2024` (raw `c[TIME_PERIOD]=` → HTTP 400 Tomcat page). Observation values are **strings** (`"2.162741"`). | |
| 143 | +- Structures: `/3.0/structure/dataflow/IMF.RES/WEO/+`, `/3.0/structure/datastructure/IMF.RES/DSD_WEO/+`, codelists `IMF.RES/CL_WEO_INDICATOR` (145 codes), `IMF.RES/CL_WEO_COUNTRY` (344 codes) with `Accept: application/vnd.sdmx.structure+json;version=2.0.0`. | |
| 144 | +- Verified indicator codes: `NGDP_RPCH` (real GDP growth %), `PCPIPCH` (CPI inflation avg %), `LUR` (unemployment rate), `GGXWDG_NGDP` (gross debt % GDP), `GGXCNL_NGDP` (net lending % GDP), `BCA_NGDPD` (current account % GDP), `PPPPC` (GDP per capita PPP intl $), plus `NGDPD`, `NGDPDPC`, `PPPGDP`, `LP` (population), `GGXWDN_NGDP` (net debt). | |
| 145 | +- Quirk: `COUNTRY_UPDATE_DATE` attribute reads `9/25/2025` even in the April 2026 vintage — do not use it as vintage; use `PUBLICATION_DATE`/`UPDATE_DATE` from CSV or the dataflow version. | |
| 146 | + | |
| 147 | +### Bulk download fallback — UNVERIFIED | |
| 148 | +`https://www.imf.org/-/media/Files/Publications/WEO/WEO-Database/2026/april/WEOApr2026all.ashx` → 302 to Azure blob → **`BlobNotFound`**; `.xls` variants and the page `imf.org/en/Publications/WEO/weo-database/2026/april/download-entire-database` → **403 Akamai "Access Denied"** for curl (browser UA too). The Oct-2025 and Apr-2025 `.ashx` URLs also 302→BlobNotFound. Conclusion: the tab-separated "WEO…all.xls" files are no longer served from that path; use the SDMX API above (or data.imf.org bulk "Download entire dataset" button, browser only). | |
| 149 | + | |
| 150 | +--- | |
| 151 | + | |
| 152 | +## 3. OECD SDMX API — VERIFIED (5+ dataflows end-to-end) | |
| 153 | + | |
| 154 | +Base `https://sdmx.oecd.org/public/rest/`. No key. Licence **CC BY 4.0** (cite "OECD (2026), <dataset>, https://data-explorer.oecd.org"). Documented limits (OECD API docs): 60 data requests/hour/IP anonymous, 1 000 000 obs/query; not hit during ~40 requests here. `cache-control: public, max-age=7200` on data. | |
| 155 | + | |
| 156 | +URL pattern: `/data/{AGENCY},{DSD@DF},{version|blank}/{key}?startPeriod=…&format=csvfilewithlabels` (CSV code+label columns) or `format=csvfile` or `format=jsondata` (SDMX-JSON 1.0). **Key must contain every dimension** (dots), empty = wildcard; missing dims → HTTP **403** `Not enough key values in query, expecting N got M`. Nothing found → HTTP **404** `NoRecordsFound`. Country dim `REF_AREA` = ISO3. Time: `2024`, `2026-06`, `2026-Q1`. Current dataflow ids: `GET /dataflow/all` (XML, 8.9 MB, 1 546 flows; JSON Accept → 406). Structure: `/dataflow/{AGENCY}/{DSD@DF}/latest?references=all` or `/datastructure/{AGENCY}/{DSD}/latest` with `Accept: application/vnd.sdmx.structure+json` (dimension list under `dataStructureComponents.dimensionList.dimensions`). | |
| 157 | + | |
| 158 | +| Topic | Dataflow (verified id) | Dims (order) | Working query (CAN+USA) | Sample | | |
| 159 | +|---|---|---|---|---| | |
| 160 | +| House prices | `OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES` | REF_AREA.FREQ.MEASURE.UNIT_MEASURE | `/CAN+USA.A.RHP+HPI+RPI+HPI_RPI+HPI_YDH.?startPeriod=2021` | `CAN,A,RHP,IX,2021,144.49` (base 2015=100) | | |
| 161 | +| Short/long rates | `OECD.SDD.STES,DSD_STES@DF_FINMARK` (4.0) | REF_AREA.FREQ.MEASURE.UNIT_MEASURE.ACTIVITY.ADJUSTMENT.TRANSFORMATION.TIME_HORIZ.METHODOLOGY (9) | `/CAN+USA.M.IRSTCI+IR3TIB+IRLT.PA......?startPeriod=2026-06` | `CAN IRLT 2026-08 3.675` | | |
| 162 | +| Rates (KEI) | `OECD.SDD.STES,DSD_KEI@DF_KEI` (4.0) | REF_AREA.FREQ.MEASURE.UNIT_MEASURE.ACTIVITY.ADJUSTMENT.TRANSFORMATION (7) | `/CAN+USA.M.IR3TIB+IRLT.....?startPeriod=2026-06` | `CAN IR3TIB 2026-08 2.28` | | |
| 163 | +| Avg annual wages | `OECD.ELS.SAE,DSD_EARNINGS@AV_AN_WAGE` | REF_AREA.MEASURE.UNIT_MEASURE.PAY_PERIOD.PRICE_BASE.AGGREGATION_OPERATION.SEX (7) | `/CAN+USA.......?startPeriod=2021` | `CAN WG USD_PPP Q(constant) 2024 67767.6` | | |
| 164 | +| Hours worked | `OECD.ELS.SAE,DSD_HW@DF_AVG_ANN_HRS_WKD` | 13 dims (REF_AREA.MEASURE.UNIT_MEASURE.SEX.AGE.LABOUR_FORCE_STATUS.WORK_PERIOD.HOURS_TYPE.WORKER_STATUS.WORK_TIME_ARNGMNT.AGGREGATION_OPERATION.HOUR_BANDS.JOB_COVERAGE) | `/CAN+USA............?startPeriod=2021` then filter `WORKER_STATUS=_T` | `CAN 2022 1693 h`, `USA 2023 1804.95` | | |
| 165 | +| Tax revenue | `OECD.CTP.TPS,DSD_REV_COMP_OECD@DF_RSOECD` (2.0) | REF_AREA.MEASURE.SECTOR.STANDARD_REVENUE.CTRY_SPECIFIC_REVENUE.UNIT_MEASURE.FREQ (7) | `/CAN+USA.TAX_REV.S13._T._T.PT_B1GQ.A?startPeriod=2022` | `CAN _T 2023 34.79 % GDP` (`T_AA` cash-basis is 0 for accrual reporters — use `_T`) | | |
| 166 | +| Social expenditure | `OECD.ELS.SPD,DSD_SOCX_AGG@DF_SOCX_AGG` | REF_AREA.FREQ.MEASURE.UNIT_MEASURE.EXPEND_SOURCE.SPENDING_TYPE.PROGRAMME_TYPE.PRICE_BASE (8) | `/CAN+USA.A.SOCX.PT_B1GQ.ES10._T._T.?startPeriod=2021` | `CAN ES10 (public) 2021 21.858 % GDP` | | |
| 167 | +| Hospital beds | `OECD.ELS.HD,DSD_HEALTH_REAC_HOSP@DF_HOSP_REAC` (1.1) | REF_AREA.MEASURE.UNIT_MEASURE.STATISTICAL_OPERATION.OWNERSHIP_TYPE.HEALTH_FUNCTION.CARE_TYPE.MEDICAL_TECH.HEALTH_CARE_PROVIDER (9) | `/CAN+USA.HB.10P3HB.._T._T...?startPeriod=2022` | `CAN 2024 2.08 beds/1000` (HEALTH_FUNCTION=_T; `HC0` rows are 0) | | |
| 168 | +| Physicians / nurses | `OECD.ELS.HD,DSD_HEALTH_EMP_REAC@DF_PHYS` (also `@DF_NURSE`) — **not** `DSD_HEALTH_REAC_EMP` | REF_AREA.MEASURE.UNIT_MEASURE.AGE.SEX.HEALTH_PROF.HEALTH_PROF_ACTIVITY_STATUS.… (9) | `/CAN+USA.........?startPeriod=2022` filter `UNIT_MEASURE=10P3HB` | `CAN HSE PHYS LP(practising) 2022 2.93/1000` | | |
| 169 | +| R&D (MSTI) | `OECD.STI.STP,DSD_MSTI@DF_MSTI` (1.3) | REF_AREA.FREQ.MEASURE.UNIT_MEASURE.PRICE_BASE.TRANSFORMATION (6) | `/CAN.A.G.PT_B1GQ._Z._Z?startPeriod=2021` | `G PT_B1GQ 2023 1.935` (GERD % GDP); researchers `T_RS FTE 2023 235170` | | |
| 170 | +| Education attainment | `OECD.EDU.IMEP,DSD_EAG_LSO_EA@DF_LSO_NEAC_DISTR_EA` (the `DSD_EAG_UOE_NEAC@DF_EAG_NEAC` id **does not exist** anymore) | 17 dims; REF_AREA first | `/CAN+USA................?startPeriod=2023` filter `AGE=Y25T64, SEX=_T, ATTAINMENT_LEV=ISCED11A_5T8, STATISTICAL_OPERATION=V` | structure verified; returned 3 945 rows; `SE` rows (std error) have empty OBS_VALUE — value rows use `STATISTICAL_OPERATION=V` (not individually displayed; partially verified) | | |
| 171 | +| Productivity | `OECD.SDD.TPS,DSD_PDB@DF_PDB_LV` | REF_AREA.FREQ.MEASURE.ACTIVITY.UNIT_MEASURE.PRICE_BASE.TRANSFORMATION.ASSET_CODE.CONVERSION_TYPE (9) | `/CAN+USA.A.......?startPeriod=2022` | **UNVERIFIED — HTTP 500 "Internal server error"** on two attempts (structure endpoint OK). Retry later or use `DSD_PDB@DF_PDB_GR`. | | |
| 172 | +| CLI / confidence | `OECD.SDD.STES,DSD_STES@DF_CLI` (4.1) | as FINMARK | not tested | id present in flow list | | |
| 173 | + | |
| 174 | +CSV (`csvfilewithlabels`) columns: `STRUCTURE,STRUCTURE_ID,STRUCTURE_NAME,ACTION,REF_AREA,Reference area,…,TIME_PERIOD,Time period,OBS_VALUE,Observation value,OBS_STATUS,…` — every dimension is followed by its label column; parse with a CSV reader keyed by header (labels can contain commas). `OBS_STATUS`: `A` normal, `E` estimate, `P` provisional, `B` break. Empty `OBS_VALUE` = missing. | |
| 175 | +`format=jsondata` sample (house prices): `{"data":{"dataSets":[{"series":{"0:0:0:0":{"observations":{"0":[139.53,0],"1":[144.16,0],"2":[143.35,0]}}}}],"structure":{"dimensions":{"observation":[{"id":"TIME_PERIOD","values":[{"id":"2025"},{"id":"2024"},{"id":"2023"}]}]}}}}` — **time values come in descending order**; map observation index → `values[i].id`, never assume ascending. | |
| 176 | + | |
| 177 | +### Connector implementation notes (2026-09-11, `registry/sources/oecd.yaml`) | |
| 178 | +Full-dimension keys that returned data for ALL countries (`REF_AREA` empty) — the shorter keys in the table above 404 (`NoRecordsFound`) when used with an empty REF_AREA: | |
| 179 | +- Wages `DSD_EARNINGS@AV_AN_WAGE`: `.WG.USD_PPP.A.Q.MEAN._Z` (PRICE_BASE `Q` = constant prices, base 2025; `V` = current). | |
| 180 | +- Hours `DSD_HW@DF_AVG_ANN_HRS_WKD`: `.HW.H_Y_PS._Z._Z.EMP.A.ACTUAL._T._Z.MEAN._Z._T` (12 dims after REF_AREA). | |
| 181 | +- Hospital beds `DF_HOSP_REAC`: `.HB.10P3HB._Z._Z._T._T._Z._Z` — the total is `OWNERSHIP_TYPE=_Z, HEALTH_FUNCTION=_T, CARE_TYPE=_T`; the `.._T._T...` form above no longer matches. | |
| 182 | +- Physicians `DF_PHYS`: `.HSE.10P3HB._Z._Z.PHYS._Z.LP._Z` (practising; USA has `LP` too). **`DF_NURSE` does not exist** (404 "Could not find Dataflow"). | |
| 183 | +- Education `DF_LSO_NEAC_DISTR_EA` (17 dims): `._T.Y25T64.ISCED11A_5T8._T.POP._Z._T._Z.ED_NED.POP._Z.PT_POP_SEX_AGE.OBS._Z.NEAC.A` — `STATISTICAL_OPERATION` is `OBS`/`SE` (not `V`); no ISCED 3–8 aggregate → "at least upper secondary" = 100 − `ISCED11A_0T2`. Includes `OECD`, `G20`, `EU25` aggregates. | |
| 184 | +- SOCX: last segment PRICE_BASE is `_Z` (`.A.SOCX.PT_B1GQ.ES10._T._T._Z`). KEI industrial production: `.M.PRVM.IX.BTE.Y._Z` (2015=100, monthly, back to 1919 for some countries). | |
| 185 | +- `DF_PDB_LV` (GDP per hour worked) still HTTP 500 on 2026-09-11 (3 attempts) — not mapped. | |
| 186 | +- Rate limit is real: 17 requests in ~40 s → one **HTTP 429** (retried later). Keep ≤ 12/min. | |
| 187 | + | |
| 188 | +--- | |
| 189 | + | |
| 190 | +## 4. Eurostat JSON-stat 2.0 — VERIFIED (13/13 — ilc_li02 works with `statinfo=MED_EI`, see note) | |
| 191 | + | |
| 192 | +Base `https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/{dataset}?format=JSON&lang=EN&{dim}={code}&…`. No key. Filters: repeat `geo=DE&geo=FR`; `time=2024` or `sinceTimePeriod=2022` (`untilTimePeriod`, `lastTimePeriod=N`). Unknown dimension name → HTTP 400 `{"error":[{"status":400,"id":150,"label":"INVALID_QUERY_DIMENSION: … Dimension \"INDIC_IL\" is not defined"}]}`. Unknown code → 200 with `size: [...,0,...]` and empty `value`. Discover a dataset's dimensions/codes by requesting a single `geo`+`time` (`.dimension.<dim>.category.index`). Licence CC BY 4.0 ("Source: Eurostat"). | |
| 193 | + | |
| 194 | +Structure (HICP example) — value is a **sparse object keyed by the flattened row-major index over `size`**: | |
| 195 | +```json | |
| 196 | +{"version":"2.0","class":"dataset","label":"HICP - annual data …","source":"ESTAT","updated":"2026-02-06T23:00:00+0100", | |
| 197 | + "id":["freq","unit","coicop","geo","time"],"size":[1,1,1,4,4], | |
| 198 | + "dimension":{"geo":{"category":{"index":{"EU27_2020":0,"DE":1,"EL":2,"FR":3},"label":{"EL":"Greece",…}}}, | |
| 199 | + "time":{"category":{"index":{"2022":0,"2023":1,"2024":2,"2025":3}}}}, | |
| 200 | + "value":{"0":9.2,"1":6.4,"2":2.6,"3":2.5,"4":8.7,"5":6.0,…}, | |
| 201 | + "status":{"8":"d","9":"d"}} | |
| 202 | +``` | |
| 203 | +Index = `geoIdx*size[time] + timeIdx` (generally Σ idx_k·∏size_{k+1..}). Missing cells are simply absent from `value`. `status` flags: `p` provisional, `e` estimated, `b` break, `d` definition differs, `u` low reliability, `:` n/a. `updated` = dataset last update (freshness). | |
| 204 | + | |
| 205 | +| Dataset | Verified query (after `?format=JSON&lang=EN&`) | `updated` | Sample | | |
| 206 | +|---|---|---|---| | |
| 207 | +| `prc_hicp_aind` | `unit=RCH_A_AVG&coicop=CP00&geo=DE&geo=FR&geo=EL&geo=EU27_2020&sinceTimePeriod=2022` | 2026-02-06 | EU27 2022 9.2 % | | |
| 208 | +| `prc_hpi_a` | `purchase=TOTAL&unit=RCH_A_AVG&geo=DE&geo=FR&sinceTimePeriod=2022` | 2026-07-02 | DE 2022 6.0, 2023 -8.4 | | |
| 209 | +| `une_rt_a` | `age=Y15-74&unit=PC_ACT&sex=T&geo=…` | 2026-09-10 | DE 2022 3.1 | | |
| 210 | +| `gov_10dd_edpt1` | `na_item=GD&unit=PC_GDP§or=S13&geo=DE&geo=FR&geo=IT` | 2026-04-22 | DE 2022 64.4 % | | |
| 211 | +| `lfsi_emp_a` | `indic_em=EMP_LFS&unit=PC_POP&age=Y20-64&sex=T` | 2026-09-10 | DE 2022 80.8 | | |
| 212 | +| `ilc_di03` | `statinfo=MED_EI&age=TOTAL&sex=T&unit=EUR` (dimension is `statinfo`, **not** `indic_il`) | 2026-06-08 | DE 2022 24 925 € | | |
| 213 | +| `ilc_li02` | dims `freq,statinfo,unit,rskpovth,sex,age,geo,time`; `rskpovth=A_60&unit=PC&sex=T&age=TOTAL` | 2026-06-10 | **UNVERIFIED values: every combination tested (DE/FR, 2019–2025, both `statinfo`, also SDMX-CSV endpoint) returns an empty `value`** — dataset appears to be mid-revision. Use `ilc_peps01n` (AROPE rate; dims `unit,age,sex`; 204 values for DE 2024 — verified) or retry later. | | |
| 214 | +| `rd_e_gerdtot` | `sectperf=TOTAL&unit=PC_GDP` | 2026-03-18 | DE 2022 3.07 (2024 `p`) | | |
| 215 | +| `nrg_ind_ren` | `nrg_bal=REN&unit=PC` | 2026-07-14 | DE 2021 19.3 | | |
| 216 | +| `env_air_gge` | `airpol=GHG&src_crf=TOTX4_MEMO&unit=MIO_T` (**not** `TOTX4_MEMONIA`; units are `MIO_T`/`THS_T`, no `T_HAB` here) | 2026-06-02 | DE 2022 749.3, 2024 649.8 Mt CO2e | | |
| 217 | +| `demo_mlexpec` | `sex=T&age=Y_LT1` | 2026-06-04 | DE 2022 80.7 | | |
| 218 | +| `edat_lfse_03` | `sex=T&age=Y25-64&isced11=ED5-8&unit=PC` | 2026-09-10 | DE 2022 32.1 | | |
| 219 | +| `ilc_lvho07a` | `rskpovth=TOTAL&age=TOTAL&sex=T&unit=PC` (**no `incgrp`** dimension) | 2026-06-08 | DE 2022 11.9 | | |
| 220 | + | |
| 221 | +**Connector implementation notes (2026-09-11, `registry/sources/eurostat.yaml`)** — `ilc_li02` is NOT empty: the `statinfo` codes are `MED_EI`/`MEAN_EI` (there is no `RT`), so `statinfo=MED_EI&rskpovth=B_60&unit=PC&sex=T&age=TOTAL` returns the at-risk-of-poverty rate (DE 2024 15.5 %). Other verified filters: `ilc_lvho02` (tenure) `rskpovth=TOTAL&hhcomp=TOTAL&tenure=OWN&unit=PC` (DE 2024 47.2 %); `une_ltu_a` `indic_em=LTU&age=Y15-74&sex=T&unit=PC_UNE`; `lfsa_eppga` `sex=T&age=Y15-64&unit=PC`; `lfsi_emp_a` has `age=Y15-64`; `ilc_di03` `unit=PPS` (DE 2024 25 192); `prc_hpi_a` `unit=I15_A_AVG` for the 2015=100 index. All 18 mapped datasets returned data for all geos with no `geo`/`time` filter (≤ 2 000 cells each). | |
| 222 | + | |
| 223 | +Geo codes (une_rt_a 2024): `EU27_2020, EA21 (from 2026), EA20, BE, BG, CZ, DK, DE, EE, IE, EL, ES, FR, HR, IT, CY, LV, LT, LU, HU, MT, NL, AT, PL, PT, RO, SI, SK, FI, SE, IS, NO, CH, BA, ME, MK, RS, TR`. Map to ISO2: `EL→GR`, `UK→GB` (UK present in some datasets but **no values after 2020**: `geo=UK` on une_rt_a → 0 values), everything else identical. Aggregates: `EU27_2020`, `EA20`, `EA21`, `EU28` — drop or map to your own aggregate ids. Alternate SDMX 2.1 endpoint for CSV: `…/api/dissemination/sdmx/2.1/data/{ds}/{key}?format=SDMX-CSV` (header `DATAFLOW,LAST UPDATE,freq,…,TIME_PERIOD,OBS_VALUE,OBS_FLAG`). | |
| 224 | + | |
| 225 | +--- | |
| 226 | + | |
| 227 | +## 5. WHO GHO OData — VERIFIED | |
| 228 | + | |
| 229 | +Base `https://ghoapi.azureedge.net/api/`, OData v4 (`$filter`, `$select`, `$top`, `$orderby`; URL-encode spaces as `%20`). No key. `cache-control: public, max-age=3600`. Licence CC BY-NC-SA 3.0 IGO (cite "WHO Global Health Observatory"). | |
| 230 | + | |
| 231 | +- `GET /Indicator` → 3 099 rows `{"IndicatorCode":"WHOSIS_000001","IndicatorName":"Life expectancy at birth (years)","Language":"EN"}`; search: `/Indicator?$filter=contains(IndicatorName,'obesity')`. | |
| 232 | +- Dimensions: `/DIMENSION/SEX/DimensionValues` → `SEX_BTSX` Both, `SEX_FMLE`, `SEX_MLE`, `SEX_NOA`; `/DIMENSION/COUNTRY/DimensionValues` → 234 with `ParentCode` = WHO region (`AMR`…). | |
| 233 | +- Data row (`/WHOSIS_000001?$filter=SpatialDim eq 'CAN' and Dim1 eq 'SEX_BTSX' and TimeDim ge 2019`): | |
| 234 | +```json | |
| 235 | +{"IndicatorCode":"WHOSIS_000001","SpatialDimType":"COUNTRY","SpatialDim":"CAN","ParentLocationCode":"AMR","ParentLocation":"Americas", | |
| 236 | + "TimeDimType":"YEAR","TimeDim":2021,"Dim1Type":"SEX","Dim1":"SEX_BTSX","Dim2Type":null,"Dim2":null, | |
| 237 | + "Value":"81.6 [81.5-81.7]","NumericValue":81.58276248,"Low":81.52940857,"High":81.68435338,"Date":"2024-08-02T09:43:39.193+02:00"} | |
| 238 | +``` | |
| 239 | +- `SpatialDimType` ∈ `COUNTRY | REGION (AFR, AMR, EMR, EUR, SEAR, WPR) | GLOBAL | WORLDBANKINCOMEGROUP` — filter `SpatialDimType eq 'COUNTRY'`. `SpatialDim` = ISO3. `TimeDim` = int year. Use `NumericValue` (the `Value` string embeds CIs). `Date` = row last modified (freshness). | |
| 240 | + | |
| 241 | +| Indicator | Code | Dim1 / Dim2 to filter | CAN latest | | |
| 242 | +|---|---|---|---| | |
| 243 | +| Life expectancy | `WHOSIS_000001` | `Dim1 eq 'SEX_BTSX'` | 2021 81.6 (4 312 rows all sexes=BTSX) | | |
| 244 | +| Healthy life expectancy | `WHOSIS_000002` | `SEX_BTSX` | 2021 69.8 | | |
| 245 | +| Adult obesity (age-std) | `NCD_BMI_30A` | `SEX_BTSX`, Dim2 `AGEGROUP_YEARS18-PLUS` | 2024 26.0 % | | |
| 246 | +| Tobacco use | `M_Est_tob_curr` | `SEX_BTSX` | **contains projections to 2030** (2030: 8.3 %) — cap at ≤ current year | | |
| 247 | +| Medical doctors /10 000 | `HWF_0001` | none | 2024 28.54 | | |
| 248 | +| Measles MCV1 | **`WHS8_110`** (`WHS4_100` is **DTP3**, not measles) | none | 2025 92 % | | |
| 249 | +| Suicide rate | `SDGSUICIDE` | `SEX_BTSX`, Dim2 `AGEGROUP_YEARSALL` | 2021 | | |
| 250 | +| Infant mortality | `MDG_0000000001` | `SEX_BTSX`, Dim2 `AGEGROUP_MONTHS0-11` | 2024 4.7 | | |
| 251 | +| Health exp. % GDP | `GHED_CHEGDP_SHA2011` | none | 2023 11.19 | | |
| 252 | +| Safely managed water | `WSH_WATER_SAFELY_MANAGED` | Dim1 `RESIDENCEAREATYPE_TOTL` | 2024 96.9 | | |
| 253 | +| Alcohol per capita | `SA_0000001688` | `SEX_BTSX` | 2024 9.24 L | | |
| 254 | + | |
| 255 | +--- | |
| 256 | + | |
| 257 | +## 6. FRED — VERIFIED | |
| 258 | + | |
| 259 | +Base `https://api.stlouisfed.org/fred/`, `api_key=<FRED_API_KEY>&file_type=json`. Limit **120 requests/minute** (documented); observed: a burst of 130 parallel calls → 60×200, 23×**429**, 47×**403 Akamai "Access Denied"** and the 403 ban lasted several minutes for all series endpoints. Throttle to ≤ 2 req/s sequential. Licence: FRED terms; third-party series (BIS, OECD, S&P) keep their own terms. | |
| 260 | + | |
| 261 | +`GET /series/observations?series_id=FEDFUNDS&api_key=…&file_type=json&observation_start=2026-01-01` → | |
| 262 | +```json | |
| 263 | +{"realtime_start":"2026-09-03","observation_start":"2026-01-01","units":"lin","output_type":1,"count":8, | |
| 264 | + "observations":[{"realtime_start":"2026-09-03","realtime_end":"2026-09-03","date":"2026-08-01","value":"3.63"}]} | |
| 265 | +``` | |
| 266 | +- Values are **strings**; missing = `"."` (e.g. DGS10 on 2026-09-07 holiday). Params: `frequency=a|q|m|w|d` with `aggregation_method=avg|sum|eop` (tested: FEDFUNDS annual avg 2024 5.14, 2025 4.21, 2026 `"."` incomplete year), `units=pc1|pch|lin…`, `observation_start/end`, `limit`, `sort_order`. | |
| 267 | +- `GET /series?series_id=X` → `seriess[0]` with `title, frequency_short, units_short, observation_start, observation_end, last_updated, seasonal_adjustment_short, notes`. **Quirk: `notes` may contain raw control characters → strict JSON parsers (jq) fail; use a lenient parser** (Python `json.JSONDecoder(strict=False)`). | |
| 268 | +- Search: `/series/search?search_text=…&limit=1000`; `/release/series?release_id=…`. | |
| 269 | + | |
| 270 | +US series — all 30 exist (obs end / last update): CPIAUCSL (2026-07), UNRATE (2026-08), GDPC1 & GDP (2026-Q2), FEDFUNDS (2026-08), MORTGAGE30US (2026-09-10, W), CSUSHPINSA (2026-06, now titled "S&P Cotality Case-Shiller"), HOUST, PERMIT (2026-07), RHORUSQ156N (2026-Q2), MEHOINUSA672N (A, 2024), CIVPART, AHETPI, CES0500000003 (2026-08), GFDEGDQ188S (2026-Q1), DGS10 (D), DEXCAUS, DEXUSEU, DEXJPUS (D, 2026-09-04), T10Y2Y, PAYEMS, PCEPILFE, INDPRO, RSAFS, UMCSENT (2026-07), TOTALSA, PSAVERT, M2SL (2026-07), WALCL (W), BOPGSTB (2026-07). | |
| 271 | + | |
| 272 | +International series (all exist): | |
| 273 | +- BIS house prices: `Q<ISO2>R628BIS` real / `Q<ISO2>N628BIS` nominal, Index 2010=100, quarterly — verified CA, US, FR, DE, JP, GB, AU, ES, IT, KR, NZ, SE, CH (obs end 2026-Q1, JP 2025-Q4; updated 2026-06/08). 61 `Q..R628BIS` series found via search. (`QCAN628BIS` = nominal Canada.) | |
| 274 | +- OECD MEI-derived (still present, but **CPI ones discontinued at 2025-03**): `IRSTCI01CAM156N` (CA immediate rate, to 2026-06), `IR3TIB01CAM156N`, `IRLTLT01CAM156N`, `IRLTLT01DEM156N` (10-y yields, 2026-06), `LRHUTTTTCAM156S` (CA unemployment, 2026-07), `LRHUTTTTDEM156S`, `LRHUTTTTGBM156S`, `NAEXKP01CAQ657S` (CA real GDP q/q, 2026-Q2), `CLVMNACSCAB1GQDE` (Eurostat DE real GDP, 2026-Q2). **Stale**: `CPALTT01CAM659N`, `CPALTT01DEM659N`, `CPHPTT01GBM659N` end 2025-03; `CPALTT01JPM659N` ends 2021-06. Prefer OECD/Eurostat/WB for non-US CPI. | |
| 275 | + | |
| 276 | +--- | |
| 277 | + | |
| 278 | +## 7. Our World in Data — VERIFIED (CC BY 4.0; cite "Our World in Data" + underlying source from metadata) | |
| 279 | + | |
| 280 | +| File | Size | Rows × cols | Years | Notes | | |
| 281 | +|---|---|---|---|---| | |
| 282 | +| `https://raw.githubusercontent.com/owid/co2-data/master/owid-co2-data.csv` | 14.4 MB | 50 411 × 79 | 1750–2024 | codebook `owid-co2-codebook.csv` (200) | | |
| 283 | +| `https://raw.githubusercontent.com/owid/energy-data/master/owid-energy-data.csv` | 9.2 MB | 23 377 × 130 | 1900–2025 | codebook `owid-energy-codebook.csv` (200) | | |
| 284 | + | |
| 285 | +- Key columns `country,year,iso_code,population,gdp`. **Aggregates/regions have an empty `iso_code`** (co2: 36 of 254 entities, e.g. `World`, `Europe`, `European Union (27)`, `High-income countries`, `Asia (GCP)`, `International aviation`, `Kuwaiti Oil Fires`; energy: 94 of 314 incl. `Africa (EI)`, `EU (Ember)`, historical `Czechoslovakia`, `East Germany`). **No `OWID_` codes in these two CSVs** (unlike the grapher API) — also `Kosovo` has empty `iso_code`. Filter `iso_code != ""` for countries (then map `OWID_KOS`→`XKX` yourself if you take Kosovo from grapher). | |
| 286 | +- All requested co2 columns exist: `co2, co2_per_capita, co2_per_gdp, methane, nitrous_oxide, total_ghg, share_global_co2, temperature_change_from_co2, consumption_co2_per_capita, coal_co2, oil_co2, gas_co2, cumulative_co2, land_use_change_co2` (+ `co2_including_luc`, `cement_co2`, `flaring_co2`, `trade_co2_share`…). CAN 2024: co2 533.34 Mt, co2_per_capita 13.42, total_ghg 802.7, methane 115.3, land_use_change_co2 111.9, `consumption_co2_per_capita` empty for 2024 (lags a year). | |
| 287 | +- All requested energy columns exist: `electricity_generation, renewables_share_elec, solar_share_elec, wind_share_elec, hydro_share_elec, nuclear_share_elec, fossil_share_elec, low_carbon_share_elec, energy_per_capita, energy_per_gdp, primary_energy_consumption, per_capita_electricity, net_elec_imports_share_demand, carbon_intensity_elec`. CAN 2025: generation 652.4 TWh, renewables 63.9 %, hydro 52.8 %, nuclear 13.1 %, fossil 23.0 %, per_capita_electricity 16 259 kWh, carbon_intensity_elec 190.7 gCO2/kWh; `energy_per_capita` empty for 2025 (Energy Institute data lags Ember). | |
| 288 | +- Missing values = empty string. Units per codebook (co2 in Mt; shares in %; energy_per_capita kWh/person). | |
| 289 | + | |
| 290 | +Grapher API: `https://ourworldindata.org/grapher/life-expectancy.csv?v=1&csvType=full&useColumnShortNames=true` → 605 KB, header `entity,code,year,life_expectancy_0`; 21 564 rows; aggregates carry **`OWID_` codes** (`OWID_WRL, OWID_EUR, OWID_HIC, OWID_KOS, OWID_USS…`, 765 rows). `csvType=filtered&country=CAN~USA&time=2020..latest` works (`Canada,CAN,2022,81.2493`). Sibling `….metadata.json?…` → `{"chart":{"title","citation":"Riley (2005); Zijdeman et al. (2015); HMD (2025); UN WPP (2024)"},"columns":{"life_expectancy_0":{"titleShort","unit":"years","lastUpdated":"2025-10-22","citationShort":…}}}` — use `columns.*.lastUpdated` for freshness and `citationShort` for attribution. | |
| 291 | + | |
| 292 | +--- | |
| 293 | + | |
| 294 | +## 8. UN and others | |
| 295 | + | |
| 296 | +**UN WPP Data Portal API** (`https://population.un.org/dataportalapi/api/v1/`): metadata endpoints `/indicators`, `/locations` answer **200 without a token**, but every data endpoint (`/data/indicators/49/locations/124/start/2023/end/2024`) → **HTTP 401, `www-authenticate: Bearer`** — token required (confirmed). Token-free fallback: bulk CSV `https://population.un.org/wpp/assets/Excel%20Files/1_Indicator%20(Standard)/CSV_FILES/WPP2024_Demographic_Indicators_Medium.csv.gz` (200, 16.6 MB gz, includes ISO3 + projections to 2100). | |
| 297 | + | |
| 298 | +**UN Comtrade**: `https://comtradeapi.un.org/data/v1/get/C/A/HS?reporterCode=124&period=2023&partnerCode=0&cmdCode=TOTAL&flowCode=X` → **401 "missing subscription key"** (confirmed). Free preview **works without key**: `https://comtradeapi.un.org/public/v1/preview/C/A/HS?reporterCode=124&period=2023&partnerCode=0&cmdCode=TOTAL&flowCode=X` → `{"count":12,"data":[{"period":"2023","flowCode":"X","cmdCode":"TOTAL","primaryValue":224462800586.53,…}]}` (≤ 500 records, `reporterISO` null in preview, M49 numeric codes; rate-limited). | |
| 299 | + | |
| 300 | +**BIS Data Portal SDMX v2** (`https://stats.bis.org/api/v2/`, no key). CSV via `?format=csv`; JSON needs header `Accept: application/vnd.sdmx.data+json` (`format=json` → 406). Country dim **ISO2** (`CA`, `US`, `XM` euro area, `4T`/`5R`/`XW` aggregates). Structures: `/structure/dataflow/BIS/WS_CBPOL/1.0?references=all` (`Accept: application/vnd.sdmx.structure+json`). | |
| 301 | +- Policy rates `WS_CBPOL`, key `FREQ.REF_AREA`: `GET /data/dataflow/BIS/WS_CBPOL/1.0/M.CA+US?startPeriod=2026-06&format=csv` → | |
| 302 | + `FREQ,REF_AREA,UNIT_MEASURE,…,TIME_PERIOD,OBS_VALUE,OBS_STATUS` → `M,CA,368,…,2026-08,2.25,A` / `M,US,368,…,2026-08,3.625,A` (unit 368 = per cent per year; US = midpoint of target range). All countries: `/M.?startPeriod=2026-08` → 34 areas (AU BR CA CH CL CN CO CZ DK GB HK HU ID IS JP KW MA MK MX MY NO NZ PE PH PL RO RS RU SE TH TR US XM ZA). Daily also available (`D.`). | |
| 303 | +- Property prices `WS_SPP`, key `FREQ.REF_AREA.VALUE.UNIT_MEASURE` (VALUE `N` nominal / `R` real; UNIT `628` index 2010=100, `771` y/y %): `GET /data/dataflow/BIS/WS_SPP/1.0/Q.CA+US.R.628?startPeriod=2025-Q1&format=csv` → `Q,US,R,628,…,2025-Q1,159.7398,A`. All countries latest: `/Q..N+R.628?startPeriod=2026-Q1` → 59 areas, 118 rows. Quoted `TITLE/COMPILATION` fields contain commas — use a real CSV parser. | |
| 304 | + | |
| 305 | +**ILOSTAT SDMX** (`https://sdmx.ilo.org/rest/`, no key, `cache-control: no-store`). The id `DF_UNE_DEAP_SEX_AGE_RT_A` **does not exist** (404 "Could not find Dataflow"); current flows (from `/dataflow/ILO`, 1 212 flows): **`DF_UNE_DEAP_SEX_AGE_RT`** (survey-based, annual+quarterly) and **`DF_UNE_2EAP_SEX_AGE_RT`** ("ILO modelled estimates, Nov. 2025", incl. projections). Dims `REF_AREA.FREQ.MEASURE.SEX.AGE`. | |
| 306 | +`GET /data/ILO,DF_UNE_DEAP_SEX_AGE_RT/.A..SEX_T.AGE_YTHADULT_YGE15?startPeriod=2024&format=csv` → 537 rows, 200 areas: | |
| 307 | +`DATAFLOW,REF_AREA,FREQ,MEASURE,SEX,AGE,TIME_PERIOD,OBS_VALUE,OBS_STATUS,…,SOURCE` → `ILO:DF_UNE_DEAP_SEX_AGE_RT(1.0),CAN,A,UNE_DEAP_RT,SEX_T,AGE_YTHADULT_YGE15,2023,5.389,,RT,PT,0,LFS - Labour Force Survey,…`. | |
| 308 | +Modelled: `/data/ILO,DF_UNE_2EAP_SEX_AGE_RT/.A..SEX_T.AGE_YTHADULT_YGE15?startPeriod=2025` → 271 areas, years 2025–2027 (**2026–2027 are projections**; CAN 2025 6.907 = WB SL.UEM.TOTL.ZS 2025). Age codes: `AGE_YTHADULT_YGE15` (15+), `AGE_YTHADULT_Y15-64`, `AGE_YTHADULT_Y15-24` youth. | |
| 309 | + | |
| 310 | +--- | |
| 311 | + | |
| 312 | +## Cross-source notes for connectors | |
| 313 | +- Prefer ISO3 as the canonical key. Convert: Eurostat `EL→GR`, `UK→GB` (ISO2→ISO3 table); BIS ISO2→ISO3; OWID `OWID_KOS→XKX`, drop other `OWID_*`; WB `XKX` Kosovo exists; IMF uses `KOS`; WHO uses ISO3 but no Kosovo. | |
| 314 | +- Forecast flags: IMF `TIME_PERIOD > LATEST_ACTUAL_ANNUAL_DATA`; ILO modelled `year > 2025`; WHO tobacco has 2030 values; Eurostat status `p`/`e`; FRED incomplete-year aggregates return `"."`. | |
| 315 | +- Freshness fields: WB `lastupdated` (meta) and `/source/2.lastupdated`; IMF CSV `UPDATE_DATE`; OECD none per row (HTTP `last-modified` absent) → store fetch date; Eurostat `updated`; WHO `Date`; FRED `last_updated`; OWID metadata `columns.*.lastUpdated`, GitHub `etag`. | |
| 316 | +- Parsing hazards: WB empty `countryiso3code` for 5 income aggregates; FRED control chars in `notes`; OECD/BIS labels with commas; OECD jsondata descending time; Eurostat sparse index arithmetic. | |
added
package.json
+15 −0
@@ -0,0 +1,15 @@ | ||
| 1 | +{ | |
| 2 | + "name": "countryatlas", | |
| 3 | + "version": "0.1.0", | |
| 4 | + "private": true, | |
| 5 | + "description": "CountryAtlas — Understand the world, one country at a time. Web app workspace (the data platform and API are Python, see pyproject.toml).", | |
| 6 | + "packageManager": "pnpm@11.1.2", | |
| 7 | + "engines": { "node": ">=22" }, | |
| 8 | + "scripts": { | |
| 9 | + "dev:web": "pnpm --filter @countryatlas/web run dev", | |
| 10 | + "build": "pnpm --filter @countryatlas/web run build", | |
| 11 | + "start:web": "pnpm --filter @countryatlas/web run start", | |
| 12 | + "typecheck": "pnpm -r run typecheck", | |
| 13 | + "qa": "pnpm --filter @countryatlas/web run qa" | |
| 14 | + } | |
| 15 | +} | |
added
pnpm-lock.yaml
+1220 −0
@@ -0,0 +1,1220 @@ | ||
| 1 | +lockfileVersion: '9.0' | |
| 2 | + | |
| 3 | +settings: | |
| 4 | + autoInstallPeers: true | |
| 5 | + excludeLinksFromLockfile: false | |
| 6 | + | |
| 7 | +importers: | |
| 8 | + | |
| 9 | + .: {} | |
| 10 | + | |
| 11 | + apps/web: | |
| 12 | + dependencies: | |
| 13 | + d3-geo: | |
| 14 | + specifier: ^3.1.1 | |
| 15 | + version: 3.1.1 | |
| 16 | + d3-scale: | |
| 17 | + specifier: ^4.0.2 | |
| 18 | + version: 4.0.2 | |
| 19 | + d3-shape: | |
| 20 | + specifier: ^3.2.0 | |
| 21 | + version: 3.2.0 | |
| 22 | + lucide-react: | |
| 23 | + specifier: ^1.0.0 | |
| 24 | + version: 1.43.0(react@19.2.8) | |
| 25 | + next: | |
| 26 | + specifier: 16.3.4 | |
| 27 | + version: 16.3.4(@types/node@24.13.4)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) | |
| 28 | + react: | |
| 29 | + specifier: 19.2.8 | |
| 30 | + version: 19.2.8 | |
| 31 | + react-dom: | |
| 32 | + specifier: 19.2.8 | |
| 33 | + version: 19.2.8(react@19.2.8) | |
| 34 | + server-only: | |
| 35 | + specifier: ^0.0.1 | |
| 36 | + version: 0.0.1 | |
| 37 | + topojson-client: | |
| 38 | + specifier: ^3.1.0 | |
| 39 | + version: 3.1.0 | |
| 40 | + world-atlas: | |
| 41 | + specifier: ^2.0.2 | |
| 42 | + version: 2.0.2 | |
| 43 | + devDependencies: | |
| 44 | + '@tailwindcss/postcss': | |
| 45 | + specifier: ^4 | |
| 46 | + version: 4.3.3 | |
| 47 | + '@types/d3-geo': | |
| 48 | + specifier: ^3.1.1 | |
| 49 | + version: 3.1.1 | |
| 50 | + '@types/d3-scale': | |
| 51 | + specifier: ^4.0.9 | |
| 52 | + version: 4.0.9 | |
| 53 | + '@types/d3-shape': | |
| 54 | + specifier: ^3.1.7 | |
| 55 | + version: 3.2.0 | |
| 56 | + '@types/geojson': | |
| 57 | + specifier: ^7946.0.16 | |
| 58 | + version: 7946.0.16 | |
| 59 | + '@types/node': | |
| 60 | + specifier: ^24.0.0 | |
| 61 | + version: 24.13.4 | |
| 62 | + '@types/react': | |
| 63 | + specifier: ^19 | |
| 64 | + version: 19.3.0 | |
| 65 | + '@types/react-dom': | |
| 66 | + specifier: ^19 | |
| 67 | + version: 19.3.0(@types/react@19.3.0) | |
| 68 | + '@types/topojson-client': | |
| 69 | + specifier: ^3.1.5 | |
| 70 | + version: 3.1.5 | |
| 71 | + '@types/topojson-specification': | |
| 72 | + specifier: ^1.0.5 | |
| 73 | + version: 1.0.5 | |
| 74 | + tailwindcss: | |
| 75 | + specifier: ^4 | |
| 76 | + version: 4.3.3 | |
| 77 | + typescript: | |
| 78 | + specifier: ^5.9.3 | |
| 79 | + version: 5.9.3 | |
| 80 | + | |
| 81 | +packages: | |
| 82 | + | |
| 83 | + '@alloc/quick-lru@5.3.0': | |
| 84 | + resolution: {integrity: sha512-U4+70Pc5ZS9osnCBCE5Jha/ciHM+Yp+CNMNC/7HvYbNRk1Ldd+f7qO65W5qfhu/TCv+/ozljlXXe9Nj8419DMA==} | |
| 85 | + engines: {node: '>=10'} | |
| 86 | + | |
| 87 | + '@emnapi/runtime@1.11.3': | |
| 88 | + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} | |
| 89 | + | |
| 90 | + '@img/colour@1.1.0': | |
| 91 | + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} | |
| 92 | + engines: {node: '>=18'} | |
| 93 | + | |
| 94 | + '@img/sharp-darwin-arm64@0.35.4': | |
| 95 | + resolution: {integrity: sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==} | |
| 96 | + engines: {node: '>=20.9.0'} | |
| 97 | + cpu: [arm64] | |
| 98 | + os: [darwin] | |
| 99 | + | |
| 100 | + '@img/sharp-darwin-x64@0.35.4': | |
| 101 | + resolution: {integrity: sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==} | |
| 102 | + engines: {node: '>=20.9.0'} | |
| 103 | + cpu: [x64] | |
| 104 | + os: [darwin] | |
| 105 | + | |
| 106 | + '@img/sharp-freebsd-wasm32@0.35.4': | |
| 107 | + resolution: {integrity: sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==} | |
| 108 | + engines: {node: '>=20.9.0'} | |
| 109 | + os: [freebsd] | |
| 110 | + | |
| 111 | + '@img/sharp-libvips-darwin-arm64@1.3.3': | |
| 112 | + resolution: {integrity: sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==} | |
| 113 | + cpu: [arm64] | |
| 114 | + os: [darwin] | |
| 115 | + | |
| 116 | + '@img/sharp-libvips-darwin-x64@1.3.3': | |
| 117 | + resolution: {integrity: sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==} | |
| 118 | + cpu: [x64] | |
| 119 | + os: [darwin] | |
| 120 | + | |
| 121 | + '@img/sharp-libvips-linux-arm64@1.3.3': | |
| 122 | + resolution: {integrity: sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==} | |
| 123 | + cpu: [arm64] | |
| 124 | + os: [linux] | |
| 125 | + libc: [glibc] | |
| 126 | + | |
| 127 | + '@img/sharp-libvips-linux-arm@1.3.3': | |
| 128 | + resolution: {integrity: sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==} | |
| 129 | + cpu: [arm] | |
| 130 | + os: [linux] | |
| 131 | + libc: [glibc] | |
| 132 | + | |
| 133 | + '@img/sharp-libvips-linux-ppc64@1.3.3': | |
| 134 | + resolution: {integrity: sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==} | |
| 135 | + cpu: [ppc64] | |
| 136 | + os: [linux] | |
| 137 | + libc: [glibc] | |
| 138 | + | |
| 139 | + '@img/sharp-libvips-linux-riscv64@1.3.3': | |
| 140 | + resolution: {integrity: sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==} | |
| 141 | + cpu: [riscv64] | |
| 142 | + os: [linux] | |
| 143 | + libc: [glibc] | |
| 144 | + | |
| 145 | + '@img/sharp-libvips-linux-s390x@1.3.3': | |
| 146 | + resolution: {integrity: sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==} | |
| 147 | + cpu: [s390x] | |
| 148 | + os: [linux] | |
| 149 | + libc: [glibc] | |
| 150 | + | |
| 151 | + '@img/sharp-libvips-linux-x64@1.3.3': | |
| 152 | + resolution: {integrity: sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==} | |
| 153 | + cpu: [x64] | |
| 154 | + os: [linux] | |
| 155 | + libc: [glibc] | |
| 156 | + | |
| 157 | + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': | |
| 158 | + resolution: {integrity: sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==} | |
| 159 | + cpu: [arm64] | |
| 160 | + os: [linux] | |
| 161 | + libc: [musl] | |
| 162 | + | |
| 163 | + '@img/sharp-libvips-linuxmusl-x64@1.3.3': | |
| 164 | + resolution: {integrity: sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==} | |
| 165 | + cpu: [x64] | |
| 166 | + os: [linux] | |
| 167 | + libc: [musl] | |
| 168 | + | |
| 169 | + '@img/sharp-linux-arm64@0.35.4': | |
| 170 | + resolution: {integrity: sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==} | |
| 171 | + engines: {node: '>=20.9.0'} | |
| 172 | + cpu: [arm64] | |
| 173 | + os: [linux] | |
| 174 | + libc: [glibc] | |
| 175 | + | |
| 176 | + '@img/sharp-linux-arm@0.35.4': | |
| 177 | + resolution: {integrity: sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==} | |
| 178 | + engines: {node: '>=20.9.0'} | |
| 179 | + cpu: [arm] | |
| 180 | + os: [linux] | |
| 181 | + libc: [glibc] | |
| 182 | + | |
| 183 | + '@img/sharp-linux-ppc64@0.35.4': | |
| 184 | + resolution: {integrity: sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==} | |
| 185 | + engines: {node: '>=20.9.0'} | |
| 186 | + cpu: [ppc64] | |
| 187 | + os: [linux] | |
| 188 | + libc: [glibc] | |
| 189 | + | |
| 190 | + '@img/sharp-linux-riscv64@0.35.4': | |
| 191 | + resolution: {integrity: sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==} | |
| 192 | + engines: {node: '>=20.9.0'} | |
| 193 | + cpu: [riscv64] | |
| 194 | + os: [linux] | |
| 195 | + libc: [glibc] | |
| 196 | + | |
| 197 | + '@img/sharp-linux-s390x@0.35.4': | |
| 198 | + resolution: {integrity: sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==} | |
| 199 | + engines: {node: '>=20.9.0'} | |
| 200 | + cpu: [s390x] | |
| 201 | + os: [linux] | |
| 202 | + libc: [glibc] | |
| 203 | + | |
| 204 | + '@img/sharp-linux-x64@0.35.4': | |
| 205 | + resolution: {integrity: sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==} | |
| 206 | + engines: {node: '>=20.9.0'} | |
| 207 | + cpu: [x64] | |
| 208 | + os: [linux] | |
| 209 | + libc: [glibc] | |
| 210 | + | |
| 211 | + '@img/sharp-linuxmusl-arm64@0.35.4': | |
| 212 | + resolution: {integrity: sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==} | |
| 213 | + engines: {node: '>=20.9.0'} | |
| 214 | + cpu: [arm64] | |
| 215 | + os: [linux] | |
| 216 | + libc: [musl] | |
| 217 | + | |
| 218 | + '@img/sharp-linuxmusl-x64@0.35.4': | |
| 219 | + resolution: {integrity: sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==} | |
| 220 | + engines: {node: '>=20.9.0'} | |
| 221 | + cpu: [x64] | |
| 222 | + os: [linux] | |
| 223 | + libc: [musl] | |
| 224 | + | |
| 225 | + '@img/sharp-wasm32@0.35.4': | |
| 226 | + resolution: {integrity: sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==} | |
| 227 | + engines: {node: '>=20.9.0'} | |
| 228 | + | |
| 229 | + '@img/sharp-webcontainers-wasm32@0.35.4': | |
| 230 | + resolution: {integrity: sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==} | |
| 231 | + engines: {node: '>=20.9.0'} | |
| 232 | + cpu: [wasm32] | |
| 233 | + | |
| 234 | + '@img/sharp-win32-arm64@0.35.4': | |
| 235 | + resolution: {integrity: sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==} | |
| 236 | + engines: {node: '>=20.9.0'} | |
| 237 | + cpu: [arm64] | |
| 238 | + os: [win32] | |
| 239 | + | |
| 240 | + '@img/sharp-win32-ia32@0.35.4': | |
| 241 | + resolution: {integrity: sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==} | |
| 242 | + engines: {node: ^20.9.0} | |
| 243 | + cpu: [ia32] | |
| 244 | + os: [win32] | |
| 245 | + | |
| 246 | + '@img/sharp-win32-x64@0.35.4': | |
| 247 | + resolution: {integrity: sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==} | |
| 248 | + engines: {node: '>=20.9.0'} | |
| 249 | + cpu: [x64] | |
| 250 | + os: [win32] | |
| 251 | + | |
| 252 | + '@jridgewell/gen-mapping@0.3.13': | |
| 253 | + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} | |
| 254 | + | |
| 255 | + '@jridgewell/remapping@2.3.5': | |
| 256 | + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} | |
| 257 | + | |
| 258 | + '@jridgewell/resolve-uri@3.1.2': | |
| 259 | + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} | |
| 260 | + engines: {node: '>=6.0.0'} | |
| 261 | + | |
| 262 | + '@jridgewell/sourcemap-codec@1.6.0': | |
| 263 | + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} | |
| 264 | + | |
| 265 | + '@jridgewell/trace-mapping@0.3.31': | |
| 266 | + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} | |
| 267 | + | |
| 268 | + '@next/env@16.3.4': | |
| 269 | + resolution: {integrity: sha512-cjWZnUUa6jZq2kFaNe/ZyJdZonOZ/QoN0Zka2nz/FLOrfx14pQuM9c5RaSVkWMqgdt4ksgPAMWPyHSs/CyV48Q==} | |
| 270 | + | |
| 271 | + '@next/swc-darwin-arm64@16.3.4': | |
| 272 | + resolution: {integrity: sha512-iBr3I5LZNk5/bgl5//iTgD2tcym14MX0Xo7fD//u9dYAEgGzza1y9oywluPtf74YnOswVdH1908aK9xVz7zQTw==} | |
| 273 | + engines: {node: '>= 10'} | |
| 274 | + cpu: [arm64] | |
| 275 | + os: [darwin] | |
| 276 | + | |
| 277 | + '@next/swc-darwin-x64@16.3.4': | |
| 278 | + resolution: {integrity: sha512-2dpiSyl2Jw/NrBPaU2MAKGSa+2MR82pJIn4Sm5Rjr+gxAeuh0z158Su3Z2O8zn7UNNq+ej4bToed6RcRN/Lydg==} | |
| 279 | + engines: {node: '>= 10'} | |
| 280 | + cpu: [x64] | |
| 281 | + os: [darwin] | |
| 282 | + | |
| 283 | + '@next/swc-linux-arm64-gnu@16.3.4': | |
| 284 | + resolution: {integrity: sha512-+t+U8HZT+fApePCS5h89CSH3datz29MkzyfCn+6fpsZBG/oiEOhINcb9rtkv6sdpToLGFn2e6146NzaKCXkqrA==} | |
| 285 | + engines: {node: '>= 10'} | |
| 286 | + cpu: [arm64] | |
| 287 | + os: [linux] | |
| 288 | + libc: [glibc] | |
| 289 | + | |
| 290 | + '@next/swc-linux-arm64-musl@16.3.4': | |
| 291 | + resolution: {integrity: sha512-mx03GNs1ocQA5JQ4FxDMmIsNkdrZh8cuezKCrId28e5/gIPU/l7Kcy2+vmCCzdjnnmXJy+iOAu+7K0QppO6Urg==} | |
| 292 | + engines: {node: '>= 10'} | |
| 293 | + cpu: [arm64] | |
| 294 | + os: [linux] | |
| 295 | + libc: [musl] | |
| 296 | + | |
| 297 | + '@next/swc-linux-x64-gnu@16.3.4': | |
| 298 | + resolution: {integrity: sha512-YIhGY6fSMfha52bnVxnzc9zaVBzJg+cqQTOD8tXIBSx4fuv0pVMxQTE0PaS59YhnMOiYiG09IMwxJAf/CFm/Dw==} | |
| 299 | + engines: {node: '>= 10'} | |
| 300 | + cpu: [x64] | |
| 301 | + os: [linux] | |
| 302 | + libc: [glibc] | |
| 303 | + | |
| 304 | + '@next/swc-linux-x64-musl@16.3.4': | |
| 305 | + resolution: {integrity: sha512-+eaaX6axpDb0yF1GCpiERe6njplvdC+nks/fKfcHu3XPGRrald8P3/X7yv7QLdjA51knnxwl9pxdIJsg+w1L+Q==} | |
| 306 | + engines: {node: '>= 10'} | |
| 307 | + cpu: [x64] | |
| 308 | + os: [linux] | |
| 309 | + libc: [musl] | |
| 310 | + | |
| 311 | + '@next/swc-win32-arm64-msvc@16.3.4': | |
| 312 | + resolution: {integrity: sha512-0jcXW7Xs/uzICrmgV3MhDYDeRy++1CqnpDIerlPIqYO4bhzB4WNbX/aRnQclustsAyTkFKB0z6rbcjmNg5tR8A==} | |
| 313 | + engines: {node: '>= 10'} | |
| 314 | + cpu: [arm64] | |
| 315 | + os: [win32] | |
| 316 | + | |
| 317 | + '@next/swc-win32-x64-msvc@16.3.4': | |
| 318 | + resolution: {integrity: sha512-vvBzwu1pYQCp92maZCFCIw/XgOTMR5tur9GjakwIo2cmwRTMKajRZZDS9+e4KsUZWKu1E007WUeAFXRRjZeuzw==} | |
| 319 | + engines: {node: '>= 10'} | |
| 320 | + cpu: [x64] | |
| 321 | + os: [win32] | |
| 322 | + | |
| 323 | + '@swc/helpers@0.5.23': | |
| 324 | + resolution: {integrity: sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==} | |
| 325 | + | |
| 326 | + '@tailwindcss/node@4.3.3': | |
| 327 | + resolution: {integrity: sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==} | |
| 328 | + | |
| 329 | + '@tailwindcss/oxide-android-arm64@4.3.3': | |
| 330 | + resolution: {integrity: sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==} | |
| 331 | + engines: {node: '>= 20'} | |
| 332 | + cpu: [arm64] | |
| 333 | + os: [android] | |
| 334 | + | |
| 335 | + '@tailwindcss/oxide-darwin-arm64@4.3.3': | |
| 336 | + resolution: {integrity: sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==} | |
| 337 | + engines: {node: '>= 20'} | |
| 338 | + cpu: [arm64] | |
| 339 | + os: [darwin] | |
| 340 | + | |
| 341 | + '@tailwindcss/oxide-darwin-x64@4.3.3': | |
| 342 | + resolution: {integrity: sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==} | |
| 343 | + engines: {node: '>= 20'} | |
| 344 | + cpu: [x64] | |
| 345 | + os: [darwin] | |
| 346 | + | |
| 347 | + '@tailwindcss/oxide-freebsd-x64@4.3.3': | |
| 348 | + resolution: {integrity: sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==} | |
| 349 | + engines: {node: '>= 20'} | |
| 350 | + cpu: [x64] | |
| 351 | + os: [freebsd] | |
| 352 | + | |
| 353 | + '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3': | |
| 354 | + resolution: {integrity: sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==} | |
| 355 | + engines: {node: '>= 20'} | |
| 356 | + cpu: [arm] | |
| 357 | + os: [linux] | |
| 358 | + | |
| 359 | + '@tailwindcss/oxide-linux-arm64-gnu@4.3.3': | |
| 360 | + resolution: {integrity: sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==} | |
| 361 | + engines: {node: '>= 20'} | |
| 362 | + cpu: [arm64] | |
| 363 | + os: [linux] | |
| 364 | + libc: [glibc] | |
| 365 | + | |
| 366 | + '@tailwindcss/oxide-linux-arm64-musl@4.3.3': | |
| 367 | + resolution: {integrity: sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==} | |
| 368 | + engines: {node: '>= 20'} | |
| 369 | + cpu: [arm64] | |
| 370 | + os: [linux] | |
| 371 | + libc: [musl] | |
| 372 | + | |
| 373 | + '@tailwindcss/oxide-linux-x64-gnu@4.3.3': | |
| 374 | + resolution: {integrity: sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==} | |
| 375 | + engines: {node: '>= 20'} | |
| 376 | + cpu: [x64] | |
| 377 | + os: [linux] | |
| 378 | + libc: [glibc] | |
| 379 | + | |
| 380 | + '@tailwindcss/oxide-linux-x64-musl@4.3.3': | |
| 381 | + resolution: {integrity: sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==} | |
| 382 | + engines: {node: '>= 20'} | |
| 383 | + cpu: [x64] | |
| 384 | + os: [linux] | |
| 385 | + libc: [musl] | |
| 386 | + | |
| 387 | + '@tailwindcss/oxide-wasm32-wasi@4.3.3': | |
| 388 | + resolution: {integrity: sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==} | |
| 389 | + engines: {node: '>=14.0.0'} | |
| 390 | + cpu: [wasm32] | |
| 391 | + bundledDependencies: | |
| 392 | + - '@napi-rs/wasm-runtime' | |
| 393 | + - '@emnapi/core' | |
| 394 | + - '@emnapi/runtime' | |
| 395 | + - '@tybys/wasm-util' | |
| 396 | + - '@emnapi/wasi-threads' | |
| 397 | + - tslib | |
| 398 | + | |
| 399 | + '@tailwindcss/oxide-win32-arm64-msvc@4.3.3': | |
| 400 | + resolution: {integrity: sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==} | |
| 401 | + engines: {node: '>= 20'} | |
| 402 | + cpu: [arm64] | |
| 403 | + os: [win32] | |
| 404 | + | |
| 405 | + '@tailwindcss/oxide-win32-x64-msvc@4.3.3': | |
| 406 | + resolution: {integrity: sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==} | |
| 407 | + engines: {node: '>= 20'} | |
| 408 | + cpu: [x64] | |
| 409 | + os: [win32] | |
| 410 | + | |
| 411 | + '@tailwindcss/oxide@4.3.3': | |
| 412 | + resolution: {integrity: sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==} | |
| 413 | + engines: {node: '>= 20'} | |
| 414 | + | |
| 415 | + '@tailwindcss/postcss@4.3.3': | |
| 416 | + resolution: {integrity: sha512-JTSZZGQi1AyKirbLN3azmjVzef92tcX7h+iSqPdaeStyFpGpDlKvvpxeOE8njhbUanbRwr3z8DyzhICWnMtQeg==} | |
| 417 | + | |
| 418 | + '@types/d3-geo@3.1.1': | |
| 419 | + resolution: {integrity: sha512-65Emv9fQiQQqphLlRkuQ5ypPsOmWPhtBGCMv61JDPEPMvsx+gzhGf74yw1a78xFKPj6zw4AgQICJoQv0vK9M2w==} | |
| 420 | + | |
| 421 | + '@types/d3-path@3.1.1': | |
| 422 | + resolution: {integrity: sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==} | |
| 423 | + | |
| 424 | + '@types/d3-scale@4.0.9': | |
| 425 | + resolution: {integrity: sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==} | |
| 426 | + | |
| 427 | + '@types/d3-shape@3.2.0': | |
| 428 | + resolution: {integrity: sha512-kVd74ta9eof3eJOvbNd1vGKS/XERRyQbT26Og63hIsvDO84cjD5gEOhsXf26w3FSoNlPVz84DOFcKv/oou+fMw==} | |
| 429 | + | |
| 430 | + '@types/d3-time@3.0.4': | |
| 431 | + resolution: {integrity: sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==} | |
| 432 | + | |
| 433 | + '@types/geojson@7946.0.16': | |
| 434 | + resolution: {integrity: sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==} | |
| 435 | + | |
| 436 | + '@types/node@24.13.4': | |
| 437 | + resolution: {integrity: sha512-YJ7EqCstVTzIr0fMr7qul/977en+pQHrfmuKIo6Zr9i75Be21dr3MovcfvGtyvi2HAUrRerWps5sMO9I7WaxDw==} | |
| 438 | + | |
| 439 | + '@types/react-dom@19.3.0': | |
| 440 | + resolution: {integrity: sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==} | |
| 441 | + peerDependencies: | |
| 442 | + '@types/react': ^19.3.0 | |
| 443 | + | |
| 444 | + '@types/react@19.3.0': | |
| 445 | + resolution: {integrity: sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==} | |
| 446 | + | |
| 447 | + '@types/topojson-client@3.1.5': | |
| 448 | + resolution: {integrity: sha512-C79rySTyPxnQNNguTZNI1Ct4D7IXgvyAs3p9HPecnl6mNrJ5+UhvGNYcZfpROYV2lMHI48kJPxwR+F9C6c7nmw==} | |
| 449 | + | |
| 450 | + '@types/topojson-specification@1.0.5': | |
| 451 | + resolution: {integrity: sha512-C7KvcQh+C2nr6Y2Ub4YfgvWvWCgP2nOQMtfhlnwsRL4pYmmwzBS7HclGiS87eQfDOU/DLQpX6GEscviaz4yLIQ==} | |
| 452 | + | |
| 453 | + baseline-browser-mapping@2.11.21: | |
| 454 | + resolution: {integrity: sha512-uh8vpY/1/YyFkunIDFH/12p7/7VdPKA1hejMVEbdkEaWnUz0Hesvx5EbiU6XxjyHZIOju+ZMbQJkRh+es3/spQ==} | |
| 455 | + engines: {node: '>=6.0.0'} | |
| 456 | + hasBin: true | |
| 457 | + | |
| 458 | + caniuse-lite@1.0.30001810: | |
| 459 | + resolution: {integrity: sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==} | |
| 460 | + | |
| 461 | + client-only@0.0.1: | |
| 462 | + resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} | |
| 463 | + | |
| 464 | + commander@2.20.3: | |
| 465 | + resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} | |
| 466 | + | |
| 467 | + csstype@3.2.3: | |
| 468 | + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} | |
| 469 | + | |
| 470 | + d3-array@3.2.4: | |
| 471 | + resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} | |
| 472 | + engines: {node: '>=12'} | |
| 473 | + | |
| 474 | + d3-color@3.1.0: | |
| 475 | + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} | |
| 476 | + engines: {node: '>=12'} | |
| 477 | + | |
| 478 | + d3-format@3.1.2: | |
| 479 | + resolution: {integrity: sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==} | |
| 480 | + engines: {node: '>=12'} | |
| 481 | + | |
| 482 | + d3-geo@3.1.1: | |
| 483 | + resolution: {integrity: sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==} | |
| 484 | + engines: {node: '>=12'} | |
| 485 | + | |
| 486 | + d3-interpolate@3.0.1: | |
| 487 | + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} | |
| 488 | + engines: {node: '>=12'} | |
| 489 | + | |
| 490 | + d3-path@3.1.0: | |
| 491 | + resolution: {integrity: sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==} | |
| 492 | + engines: {node: '>=12'} | |
| 493 | + | |
| 494 | + d3-scale@4.0.2: | |
| 495 | + resolution: {integrity: sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==} | |
| 496 | + engines: {node: '>=12'} | |
| 497 | + | |
| 498 | + d3-shape@3.2.0: | |
| 499 | + resolution: {integrity: sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==} | |
| 500 | + engines: {node: '>=12'} | |
| 501 | + | |
| 502 | + d3-time-format@4.1.0: | |
| 503 | + resolution: {integrity: sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==} | |
| 504 | + engines: {node: '>=12'} | |
| 505 | + | |
| 506 | + d3-time@3.1.0: | |
| 507 | + resolution: {integrity: sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==} | |
| 508 | + engines: {node: '>=12'} | |
| 509 | + | |
| 510 | + detect-libc@2.1.2: | |
| 511 | + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} | |
| 512 | + engines: {node: '>=8'} | |
| 513 | + | |
| 514 | + enhanced-resolve@5.24.5: | |
| 515 | + resolution: {integrity: sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==} | |
| 516 | + engines: {node: '>=10.13.0'} | |
| 517 | + | |
| 518 | + graceful-fs@4.2.11: | |
| 519 | + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} | |
| 520 | + | |
| 521 | + internmap@2.0.3: | |
| 522 | + resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} | |
| 523 | + engines: {node: '>=12'} | |
| 524 | + | |
| 525 | + jiti@2.7.0: | |
| 526 | + resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} | |
| 527 | + hasBin: true | |
| 528 | + | |
| 529 | + lightningcss-android-arm64@1.32.0: | |
| 530 | + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} | |
| 531 | + engines: {node: '>= 12.0.0'} | |
| 532 | + cpu: [arm64] | |
| 533 | + os: [android] | |
| 534 | + | |
| 535 | + lightningcss-darwin-arm64@1.32.0: | |
| 536 | + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} | |
| 537 | + engines: {node: '>= 12.0.0'} | |
| 538 | + cpu: [arm64] | |
| 539 | + os: [darwin] | |
| 540 | + | |
| 541 | + lightningcss-darwin-x64@1.32.0: | |
| 542 | + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} | |
| 543 | + engines: {node: '>= 12.0.0'} | |
| 544 | + cpu: [x64] | |
| 545 | + os: [darwin] | |
| 546 | + | |
| 547 | + lightningcss-freebsd-x64@1.32.0: | |
| 548 | + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} | |
| 549 | + engines: {node: '>= 12.0.0'} | |
| 550 | + cpu: [x64] | |
| 551 | + os: [freebsd] | |
| 552 | + | |
| 553 | + lightningcss-linux-arm-gnueabihf@1.32.0: | |
| 554 | + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} | |
| 555 | + engines: {node: '>= 12.0.0'} | |
| 556 | + cpu: [arm] | |
| 557 | + os: [linux] | |
| 558 | + | |
| 559 | + lightningcss-linux-arm64-gnu@1.32.0: | |
| 560 | + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} | |
| 561 | + engines: {node: '>= 12.0.0'} | |
| 562 | + cpu: [arm64] | |
| 563 | + os: [linux] | |
| 564 | + libc: [glibc] | |
| 565 | + | |
| 566 | + lightningcss-linux-arm64-musl@1.32.0: | |
| 567 | + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} | |
| 568 | + engines: {node: '>= 12.0.0'} | |
| 569 | + cpu: [arm64] | |
| 570 | + os: [linux] | |
| 571 | + libc: [musl] | |
| 572 | + | |
| 573 | + lightningcss-linux-x64-gnu@1.32.0: | |
| 574 | + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} | |
| 575 | + engines: {node: '>= 12.0.0'} | |
| 576 | + cpu: [x64] | |
| 577 | + os: [linux] | |
| 578 | + libc: [glibc] | |
| 579 | + | |
| 580 | + lightningcss-linux-x64-musl@1.32.0: | |
| 581 | + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} | |
| 582 | + engines: {node: '>= 12.0.0'} | |
| 583 | + cpu: [x64] | |
| 584 | + os: [linux] | |
| 585 | + libc: [musl] | |
| 586 | + | |
| 587 | + lightningcss-win32-arm64-msvc@1.32.0: | |
| 588 | + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} | |
| 589 | + engines: {node: '>= 12.0.0'} | |
| 590 | + cpu: [arm64] | |
| 591 | + os: [win32] | |
| 592 | + | |
| 593 | + lightningcss-win32-x64-msvc@1.32.0: | |
| 594 | + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} | |
| 595 | + engines: {node: '>= 12.0.0'} | |
| 596 | + cpu: [x64] | |
| 597 | + os: [win32] | |
| 598 | + | |
| 599 | + lightningcss@1.32.0: | |
| 600 | + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} | |
| 601 | + engines: {node: '>= 12.0.0'} | |
| 602 | + | |
| 603 | + lucide-react@1.43.0: | |
| 604 | + resolution: {integrity: sha512-ubtnda1fVK5ky0PNEpOmB0wiwhpZUyJiol4K14KCm+QKvuCkfUu54Et/rIjyupJpwiJ5Hg+BLQE86/GEsS+IgQ==} | |
| 605 | + peerDependencies: | |
| 606 | + react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 | |
| 607 | + | |
| 608 | + magic-string@0.30.21: | |
| 609 | + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} | |
| 610 | + | |
| 611 | + nanoid@3.3.18: | |
| 612 | + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} | |
| 613 | + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} | |
| 614 | + hasBin: true | |
| 615 | + | |
| 616 | + next@16.3.4: | |
| 617 | + resolution: {integrity: sha512-/Ztf6CeRH+ejEXUrYtqI4gkS66eFIHuSwqi60RgcpWKodxFZx2/dqVCMKBwILfAHXQ+F1b1vAudgj3mnxqtoIA==} | |
| 618 | + engines: {node: '>=20.9.0'} | |
| 619 | + hasBin: true | |
| 620 | + peerDependencies: | |
| 621 | + '@opentelemetry/api': ^1.1.0 | |
| 622 | + '@playwright/test': ^1.51.1 | |
| 623 | + babel-plugin-react-compiler: '*' | |
| 624 | + react: ^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0 | |
| 625 | + react-dom: ^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0 | |
| 626 | + sass: ^1.3.0 | |
| 627 | + peerDependenciesMeta: | |
| 628 | + '@opentelemetry/api': | |
| 629 | + optional: true | |
| 630 | + '@playwright/test': | |
| 631 | + optional: true | |
| 632 | + babel-plugin-react-compiler: | |
| 633 | + optional: true | |
| 634 | + sass: | |
| 635 | + optional: true | |
| 636 | + | |
| 637 | + picocolors@1.1.1: | |
| 638 | + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} | |
| 639 | + | |
| 640 | + postcss@8.5.23: | |
| 641 | + resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==} | |
| 642 | + engines: {node: ^10 || ^12 || >=14} | |
| 643 | + | |
| 644 | + postcss@8.5.28: | |
| 645 | + resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==} | |
| 646 | + engines: {node: ^10 || ^12 || >=14} | |
| 647 | + | |
| 648 | + react-dom@19.2.8: | |
| 649 | + resolution: {integrity: sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==} | |
| 650 | + peerDependencies: | |
| 651 | + react: ^19.2.8 | |
| 652 | + | |
| 653 | + react@19.2.8: | |
| 654 | + resolution: {integrity: sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==} | |
| 655 | + engines: {node: '>=0.10.0'} | |
| 656 | + | |
| 657 | + scheduler@0.27.0: | |
| 658 | + resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} | |
| 659 | + | |
| 660 | + semver@7.8.5: | |
| 661 | + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} | |
| 662 | + engines: {node: '>=10'} | |
| 663 | + hasBin: true | |
| 664 | + | |
| 665 | + server-only@0.0.1: | |
| 666 | + resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==} | |
| 667 | + | |
| 668 | + sharp@0.35.4: | |
| 669 | + resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} | |
| 670 | + engines: {node: '>=20.9.0'} | |
| 671 | + peerDependencies: | |
| 672 | + '@types/node': '*' | |
| 673 | + peerDependenciesMeta: | |
| 674 | + '@types/node': | |
| 675 | + optional: true | |
| 676 | + | |
| 677 | + source-map-js@1.2.1: | |
| 678 | + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} | |
| 679 | + engines: {node: '>=0.10.0'} | |
| 680 | + | |
| 681 | + styled-jsx@5.1.6: | |
| 682 | + resolution: {integrity: sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==} | |
| 683 | + engines: {node: '>= 12.0.0'} | |
| 684 | + peerDependencies: | |
| 685 | + '@babel/core': '*' | |
| 686 | + babel-plugin-macros: '*' | |
| 687 | + react: '>= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0' | |
| 688 | + peerDependenciesMeta: | |
| 689 | + '@babel/core': | |
| 690 | + optional: true | |
| 691 | + babel-plugin-macros: | |
| 692 | + optional: true | |
| 693 | + | |
| 694 | + tailwindcss@4.3.3: | |
| 695 | + resolution: {integrity: sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==} | |
| 696 | + | |
| 697 | + tapable@2.3.3: | |
| 698 | + resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} | |
| 699 | + engines: {node: '>=6'} | |
| 700 | + | |
| 701 | + topojson-client@3.1.0: | |
| 702 | + resolution: {integrity: sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw==} | |
| 703 | + hasBin: true | |
| 704 | + | |
| 705 | + tslib@2.8.1: | |
| 706 | + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} | |
| 707 | + | |
| 708 | + typescript@5.9.3: | |
| 709 | + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} | |
| 710 | + engines: {node: '>=14.17'} | |
| 711 | + hasBin: true | |
| 712 | + | |
| 713 | + undici-types@7.18.2: | |
| 714 | + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} | |
| 715 | + | |
| 716 | + world-atlas@2.0.2: | |
| 717 | + resolution: {integrity: sha512-IXfV0qwlKXpckz1FhwXVwKRjiIhOnWttOskm5CtxMsjgE/MXAYRHWJqgXOpM8IkcPBoXnyTU5lFHcYa5ChG0LQ==} | |
| 718 | + | |
| 719 | +snapshots: | |
| 720 | + | |
| 721 | + '@alloc/quick-lru@5.3.0': {} | |
| 722 | + | |
| 723 | + '@emnapi/runtime@1.11.3': | |
| 724 | + dependencies: | |
| 725 | + tslib: 2.8.1 | |
| 726 | + optional: true | |
| 727 | + | |
| 728 | + '@img/colour@1.1.0': | |
| 729 | + optional: true | |
| 730 | + | |
| 731 | + '@img/sharp-darwin-arm64@0.35.4': | |
| 732 | + optionalDependencies: | |
| 733 | + '@img/sharp-libvips-darwin-arm64': 1.3.3 | |
| 734 | + optional: true | |
| 735 | + | |
| 736 | + '@img/sharp-darwin-x64@0.35.4': | |
| 737 | + optionalDependencies: | |
| 738 | + '@img/sharp-libvips-darwin-x64': 1.3.3 | |
| 739 | + optional: true | |
| 740 | + | |
| 741 | + '@img/sharp-freebsd-wasm32@0.35.4': | |
| 742 | + dependencies: | |
| 743 | + '@img/sharp-wasm32': 0.35.4 | |
| 744 | + optional: true | |
| 745 | + | |
| 746 | + '@img/sharp-libvips-darwin-arm64@1.3.3': | |
| 747 | + optional: true | |
| 748 | + | |
| 749 | + '@img/sharp-libvips-darwin-x64@1.3.3': | |
| 750 | + optional: true | |
| 751 | + | |
| 752 | + '@img/sharp-libvips-linux-arm64@1.3.3': | |
| 753 | + optional: true | |
| 754 | + | |
| 755 | + '@img/sharp-libvips-linux-arm@1.3.3': | |
| 756 | + optional: true | |
| 757 | + | |
| 758 | + '@img/sharp-libvips-linux-ppc64@1.3.3': | |
| 759 | + optional: true | |
| 760 | + | |
| 761 | + '@img/sharp-libvips-linux-riscv64@1.3.3': | |
| 762 | + optional: true | |
| 763 | + | |
| 764 | + '@img/sharp-libvips-linux-s390x@1.3.3': | |
| 765 | + optional: true | |
| 766 | + | |
| 767 | + '@img/sharp-libvips-linux-x64@1.3.3': | |
| 768 | + optional: true | |
| 769 | + | |
| 770 | + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': | |
| 771 | + optional: true | |
| 772 | + | |
| 773 | + '@img/sharp-libvips-linuxmusl-x64@1.3.3': | |
| 774 | + optional: true | |
| 775 | + | |
| 776 | + '@img/sharp-linux-arm64@0.35.4': | |
| 777 | + optionalDependencies: | |
| 778 | + '@img/sharp-libvips-linux-arm64': 1.3.3 | |
| 779 | + optional: true | |
| 780 | + | |
| 781 | + '@img/sharp-linux-arm@0.35.4': | |
| 782 | + optionalDependencies: | |
| 783 | + '@img/sharp-libvips-linux-arm': 1.3.3 | |
| 784 | + optional: true | |
| 785 | + | |
| 786 | + '@img/sharp-linux-ppc64@0.35.4': | |
| 787 | + optionalDependencies: | |
| 788 | + '@img/sharp-libvips-linux-ppc64': 1.3.3 | |
| 789 | + optional: true | |
| 790 | + | |
| 791 | + '@img/sharp-linux-riscv64@0.35.4': | |
| 792 | + optionalDependencies: | |
| 793 | + '@img/sharp-libvips-linux-riscv64': 1.3.3 | |
| 794 | + optional: true | |
| 795 | + | |
| 796 | + '@img/sharp-linux-s390x@0.35.4': | |
| 797 | + optionalDependencies: | |
| 798 | + '@img/sharp-libvips-linux-s390x': 1.3.3 | |
| 799 | + optional: true | |
| 800 | + | |
| 801 | + '@img/sharp-linux-x64@0.35.4': | |
| 802 | + optionalDependencies: | |
| 803 | + '@img/sharp-libvips-linux-x64': 1.3.3 | |
| 804 | + optional: true | |
| 805 | + | |
| 806 | + '@img/sharp-linuxmusl-arm64@0.35.4': | |
| 807 | + optionalDependencies: | |
| 808 | + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 | |
| 809 | + optional: true | |
| 810 | + | |
| 811 | + '@img/sharp-linuxmusl-x64@0.35.4': | |
| 812 | + optionalDependencies: | |
| 813 | + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 | |
| 814 | + optional: true | |
| 815 | + | |
| 816 | + '@img/sharp-wasm32@0.35.4': | |
| 817 | + dependencies: | |
| 818 | + '@emnapi/runtime': 1.11.3 | |
| 819 | + optional: true | |
| 820 | + | |
| 821 | + '@img/sharp-webcontainers-wasm32@0.35.4': | |
| 822 | + dependencies: | |
| 823 | + '@img/sharp-wasm32': 0.35.4 | |
| 824 | + optional: true | |
| 825 | + | |
| 826 | + '@img/sharp-win32-arm64@0.35.4': | |
| 827 | + optional: true | |
| 828 | + | |
| 829 | + '@img/sharp-win32-ia32@0.35.4': | |
| 830 | + optional: true | |
| 831 | + | |
| 832 | + '@img/sharp-win32-x64@0.35.4': | |
| 833 | + optional: true | |
| 834 | + | |
| 835 | + '@jridgewell/gen-mapping@0.3.13': | |
| 836 | + dependencies: | |
| 837 | + '@jridgewell/sourcemap-codec': 1.6.0 | |
| 838 | + '@jridgewell/trace-mapping': 0.3.31 | |
| 839 | + | |
| 840 | + '@jridgewell/remapping@2.3.5': | |
| 841 | + dependencies: | |
| 842 | + '@jridgewell/gen-mapping': 0.3.13 | |
| 843 | + '@jridgewell/trace-mapping': 0.3.31 | |
| 844 | + | |
| 845 | + '@jridgewell/resolve-uri@3.1.2': {} | |
| 846 | + | |
| 847 | + '@jridgewell/sourcemap-codec@1.6.0': {} | |
| 848 | + | |
| 849 | + '@jridgewell/trace-mapping@0.3.31': | |
| 850 | + dependencies: | |
| 851 | + '@jridgewell/resolve-uri': 3.1.2 | |
| 852 | + '@jridgewell/sourcemap-codec': 1.6.0 | |
| 853 | + | |
| 854 | + '@next/env@16.3.4': {} | |
| 855 | + | |
| 856 | + '@next/swc-darwin-arm64@16.3.4': | |
| 857 | + optional: true | |
| 858 | + | |
| 859 | + '@next/swc-darwin-x64@16.3.4': | |
| 860 | + optional: true | |
| 861 | + | |
| 862 | + '@next/swc-linux-arm64-gnu@16.3.4': | |
| 863 | + optional: true | |
| 864 | + | |
| 865 | + '@next/swc-linux-arm64-musl@16.3.4': | |
| 866 | + optional: true | |
| 867 | + | |
| 868 | + '@next/swc-linux-x64-gnu@16.3.4': | |
| 869 | + optional: true | |
| 870 | + | |
| 871 | + '@next/swc-linux-x64-musl@16.3.4': | |
| 872 | + optional: true | |
| 873 | + | |
| 874 | + '@next/swc-win32-arm64-msvc@16.3.4': | |
| 875 | + optional: true | |
| 876 | + | |
| 877 | + '@next/swc-win32-x64-msvc@16.3.4': | |
| 878 | + optional: true | |
| 879 | + | |
| 880 | + '@swc/helpers@0.5.23': | |
| 881 | + dependencies: | |
| 882 | + tslib: 2.8.1 | |
| 883 | + | |
| 884 | + '@tailwindcss/node@4.3.3': | |
| 885 | + dependencies: | |
| 886 | + '@jridgewell/remapping': 2.3.5 | |
| 887 | + enhanced-resolve: 5.24.5 | |
| 888 | + jiti: 2.7.0 | |
| 889 | + lightningcss: 1.32.0 | |
| 890 | + magic-string: 0.30.21 | |
| 891 | + source-map-js: 1.2.1 | |
| 892 | + tailwindcss: 4.3.3 | |
| 893 | + | |
| 894 | + '@tailwindcss/oxide-android-arm64@4.3.3': | |
| 895 | + optional: true | |
| 896 | + | |
| 897 | + '@tailwindcss/oxide-darwin-arm64@4.3.3': | |
| 898 | + optional: true | |
| 899 | + | |
| 900 | + '@tailwindcss/oxide-darwin-x64@4.3.3': | |
| 901 | + optional: true | |
| 902 | + | |
| 903 | + '@tailwindcss/oxide-freebsd-x64@4.3.3': | |
| 904 | + optional: true | |
| 905 | + | |
| 906 | + '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3': | |
| 907 | + optional: true | |
| 908 | + | |
| 909 | + '@tailwindcss/oxide-linux-arm64-gnu@4.3.3': | |
| 910 | + optional: true | |
| 911 | + | |
| 912 | + '@tailwindcss/oxide-linux-arm64-musl@4.3.3': | |
| 913 | + optional: true | |
| 914 | + | |
| 915 | + '@tailwindcss/oxide-linux-x64-gnu@4.3.3': | |
| 916 | + optional: true | |
| 917 | + | |
| 918 | + '@tailwindcss/oxide-linux-x64-musl@4.3.3': | |
| 919 | + optional: true | |
| 920 | + | |
| 921 | + '@tailwindcss/oxide-wasm32-wasi@4.3.3': | |
| 922 | + optional: true | |
| 923 | + | |
| 924 | + '@tailwindcss/oxide-win32-arm64-msvc@4.3.3': | |
| 925 | + optional: true | |
| 926 | + | |
| 927 | + '@tailwindcss/oxide-win32-x64-msvc@4.3.3': | |
| 928 | + optional: true | |
| 929 | + | |
| 930 | + '@tailwindcss/oxide@4.3.3': | |
| 931 | + optionalDependencies: | |
| 932 | + '@tailwindcss/oxide-android-arm64': 4.3.3 | |
| 933 | + '@tailwindcss/oxide-darwin-arm64': 4.3.3 | |
| 934 | + '@tailwindcss/oxide-darwin-x64': 4.3.3 | |
| 935 | + '@tailwindcss/oxide-freebsd-x64': 4.3.3 | |
| 936 | + '@tailwindcss/oxide-linux-arm-gnueabihf': 4.3.3 | |
| 937 | + '@tailwindcss/oxide-linux-arm64-gnu': 4.3.3 | |
| 938 | + '@tailwindcss/oxide-linux-arm64-musl': 4.3.3 | |
| 939 | + '@tailwindcss/oxide-linux-x64-gnu': 4.3.3 | |
| 940 | + '@tailwindcss/oxide-linux-x64-musl': 4.3.3 | |
| 941 | + '@tailwindcss/oxide-wasm32-wasi': 4.3.3 | |
| 942 | + '@tailwindcss/oxide-win32-arm64-msvc': 4.3.3 | |
| 943 | + '@tailwindcss/oxide-win32-x64-msvc': 4.3.3 | |
| 944 | + | |
| 945 | + '@tailwindcss/postcss@4.3.3': | |
| 946 | + dependencies: | |
| 947 | + '@alloc/quick-lru': 5.3.0 | |
| 948 | + '@tailwindcss/node': 4.3.3 | |
| 949 | + '@tailwindcss/oxide': 4.3.3 | |
| 950 | + postcss: 8.5.28 | |
| 951 | + tailwindcss: 4.3.3 | |
| 952 | + | |
| 953 | + '@types/d3-geo@3.1.1': | |
| 954 | + dependencies: | |
| 955 | + '@types/geojson': 7946.0.16 | |
| 956 | + | |
| 957 | + '@types/d3-path@3.1.1': {} | |
| 958 | + | |
| 959 | + '@types/d3-scale@4.0.9': | |
| 960 | + dependencies: | |
| 961 | + '@types/d3-time': 3.0.4 | |
| 962 | + | |
| 963 | + '@types/d3-shape@3.2.0': | |
| 964 | + dependencies: | |
| 965 | + '@types/d3-path': 3.1.1 | |
| 966 | + | |
| 967 | + '@types/d3-time@3.0.4': {} | |
| 968 | + | |
| 969 | + '@types/geojson@7946.0.16': {} | |
| 970 | + | |
| 971 | + '@types/node@24.13.4': | |
| 972 | + dependencies: | |
| 973 | + undici-types: 7.18.2 | |
| 974 | + | |
| 975 | + '@types/react-dom@19.3.0(@types/react@19.3.0)': | |
| 976 | + dependencies: | |
| 977 | + '@types/react': 19.3.0 | |
| 978 | + | |
| 979 | + '@types/react@19.3.0': | |
| 980 | + dependencies: | |
| 981 | + csstype: 3.2.3 | |
| 982 | + | |
| 983 | + '@types/topojson-client@3.1.5': | |
| 984 | + dependencies: | |
| 985 | + '@types/geojson': 7946.0.16 | |
| 986 | + '@types/topojson-specification': 1.0.5 | |
| 987 | + | |
| 988 | + '@types/topojson-specification@1.0.5': | |
| 989 | + dependencies: | |
| 990 | + '@types/geojson': 7946.0.16 | |
| 991 | + | |
| 992 | + baseline-browser-mapping@2.11.21: {} | |
| 993 | + | |
| 994 | + caniuse-lite@1.0.30001810: {} | |
| 995 | + | |
| 996 | + client-only@0.0.1: {} | |
| 997 | + | |
| 998 | + commander@2.20.3: {} | |
| 999 | + | |
| 1000 | + csstype@3.2.3: {} | |
| 1001 | + | |
| 1002 | + d3-array@3.2.4: | |
| 1003 | + dependencies: | |
| 1004 | + internmap: 2.0.3 | |
| 1005 | + | |
| 1006 | + d3-color@3.1.0: {} | |
| 1007 | + | |
| 1008 | + d3-format@3.1.2: {} | |
| 1009 | + | |
| 1010 | + d3-geo@3.1.1: | |
| 1011 | + dependencies: | |
| 1012 | + d3-array: 3.2.4 | |
| 1013 | + | |
| 1014 | + d3-interpolate@3.0.1: | |
| 1015 | + dependencies: | |
| 1016 | + d3-color: 3.1.0 | |
| 1017 | + | |
| 1018 | + d3-path@3.1.0: {} | |
| 1019 | + | |
| 1020 | + d3-scale@4.0.2: | |
| 1021 | + dependencies: | |
| 1022 | + d3-array: 3.2.4 | |
| 1023 | + d3-format: 3.1.2 | |
| 1024 | + d3-interpolate: 3.0.1 | |
| 1025 | + d3-time: 3.1.0 | |
| 1026 | + d3-time-format: 4.1.0 | |
| 1027 | + | |
| 1028 | + d3-shape@3.2.0: | |
| 1029 | + dependencies: | |
| 1030 | + d3-path: 3.1.0 | |
| 1031 | + | |
| 1032 | + d3-time-format@4.1.0: | |
| 1033 | + dependencies: | |
| 1034 | + d3-time: 3.1.0 | |
| 1035 | + | |
| 1036 | + d3-time@3.1.0: | |
| 1037 | + dependencies: | |
| 1038 | + d3-array: 3.2.4 | |
| 1039 | + | |
| 1040 | + detect-libc@2.1.2: {} | |
| 1041 | + | |
| 1042 | + enhanced-resolve@5.24.5: | |
| 1043 | + dependencies: | |
| 1044 | + graceful-fs: 4.2.11 | |
| 1045 | + tapable: 2.3.3 | |
| 1046 | + | |
| 1047 | + graceful-fs@4.2.11: {} | |
| 1048 | + | |
| 1049 | + internmap@2.0.3: {} | |
| 1050 | + | |
| 1051 | + jiti@2.7.0: {} | |
| 1052 | + | |
| 1053 | + lightningcss-android-arm64@1.32.0: | |
| 1054 | + optional: true | |
| 1055 | + | |
| 1056 | + lightningcss-darwin-arm64@1.32.0: | |
| 1057 | + optional: true | |
| 1058 | + | |
| 1059 | + lightningcss-darwin-x64@1.32.0: | |
| 1060 | + optional: true | |
| 1061 | + | |
| 1062 | + lightningcss-freebsd-x64@1.32.0: | |
| 1063 | + optional: true | |
| 1064 | + | |
| 1065 | + lightningcss-linux-arm-gnueabihf@1.32.0: | |
| 1066 | + optional: true | |
| 1067 | + | |
| 1068 | + lightningcss-linux-arm64-gnu@1.32.0: | |
| 1069 | + optional: true | |
| 1070 | + | |
| 1071 | + lightningcss-linux-arm64-musl@1.32.0: | |
| 1072 | + optional: true | |
| 1073 | + | |
| 1074 | + lightningcss-linux-x64-gnu@1.32.0: | |
| 1075 | + optional: true | |
| 1076 | + | |
| 1077 | + lightningcss-linux-x64-musl@1.32.0: | |
| 1078 | + optional: true | |
| 1079 | + | |
| 1080 | + lightningcss-win32-arm64-msvc@1.32.0: | |
| 1081 | + optional: true | |
| 1082 | + | |
| 1083 | + lightningcss-win32-x64-msvc@1.32.0: | |
| 1084 | + optional: true | |
| 1085 | + | |
| 1086 | + lightningcss@1.32.0: | |
| 1087 | + dependencies: | |
| 1088 | + detect-libc: 2.1.2 | |
| 1089 | + optionalDependencies: | |
| 1090 | + lightningcss-android-arm64: 1.32.0 | |
| 1091 | + lightningcss-darwin-arm64: 1.32.0 | |
| 1092 | + lightningcss-darwin-x64: 1.32.0 | |
| 1093 | + lightningcss-freebsd-x64: 1.32.0 | |
| 1094 | + lightningcss-linux-arm-gnueabihf: 1.32.0 | |
| 1095 | + lightningcss-linux-arm64-gnu: 1.32.0 | |
| 1096 | + lightningcss-linux-arm64-musl: 1.32.0 | |
| 1097 | + lightningcss-linux-x64-gnu: 1.32.0 | |
| 1098 | + lightningcss-linux-x64-musl: 1.32.0 | |
| 1099 | + lightningcss-win32-arm64-msvc: 1.32.0 | |
| 1100 | + lightningcss-win32-x64-msvc: 1.32.0 | |
| 1101 | + | |
| 1102 | + lucide-react@1.43.0(react@19.2.8): | |
| 1103 | + dependencies: | |
| 1104 | + react: 19.2.8 | |
| 1105 | + | |
| 1106 | + magic-string@0.30.21: | |
| 1107 | + dependencies: | |
| 1108 | + '@jridgewell/sourcemap-codec': 1.6.0 | |
| 1109 | + | |
| 1110 | + nanoid@3.3.18: {} | |
| 1111 | + | |
| 1112 | + next@16.3.4(@types/node@24.13.4)(react-dom@19.2.8(react@19.2.8))(react@19.2.8): | |
| 1113 | + dependencies: | |
| 1114 | + '@next/env': 16.3.4 | |
| 1115 | + '@swc/helpers': 0.5.23 | |
| 1116 | + baseline-browser-mapping: 2.11.21 | |
| 1117 | + caniuse-lite: 1.0.30001810 | |
| 1118 | + postcss: 8.5.23 | |
| 1119 | + react: 19.2.8 | |
| 1120 | + react-dom: 19.2.8(react@19.2.8) | |
| 1121 | + styled-jsx: 5.1.6(react@19.2.8) | |
| 1122 | + optionalDependencies: | |
| 1123 | + '@next/swc-darwin-arm64': 16.3.4 | |
| 1124 | + '@next/swc-darwin-x64': 16.3.4 | |
| 1125 | + '@next/swc-linux-arm64-gnu': 16.3.4 | |
| 1126 | + '@next/swc-linux-arm64-musl': 16.3.4 | |
| 1127 | + '@next/swc-linux-x64-gnu': 16.3.4 | |
| 1128 | + '@next/swc-linux-x64-musl': 16.3.4 | |
| 1129 | + '@next/swc-win32-arm64-msvc': 16.3.4 | |
| 1130 | + '@next/swc-win32-x64-msvc': 16.3.4 | |
| 1131 | + sharp: 0.35.4(@types/node@24.13.4) | |
| 1132 | + transitivePeerDependencies: | |
| 1133 | + - '@babel/core' | |
| 1134 | + - '@types/node' | |
| 1135 | + - babel-plugin-macros | |
| 1136 | + | |
| 1137 | + picocolors@1.1.1: {} | |
| 1138 | + | |
| 1139 | + postcss@8.5.23: | |
| 1140 | + dependencies: | |
| 1141 | + nanoid: 3.3.18 | |
| 1142 | + picocolors: 1.1.1 | |
| 1143 | + source-map-js: 1.2.1 | |
| 1144 | + | |
| 1145 | + postcss@8.5.28: | |
| 1146 | + dependencies: | |
| 1147 | + nanoid: 3.3.18 | |
| 1148 | + picocolors: 1.1.1 | |
| 1149 | + source-map-js: 1.2.1 | |
| 1150 | + | |
| 1151 | + react-dom@19.2.8(react@19.2.8): | |
| 1152 | + dependencies: | |
| 1153 | + react: 19.2.8 | |
| 1154 | + scheduler: 0.27.0 | |
| 1155 | + | |
| 1156 | + react@19.2.8: {} | |
| 1157 | + | |
| 1158 | + scheduler@0.27.0: {} | |
| 1159 | + | |
| 1160 | + semver@7.8.5: | |
| 1161 | + optional: true | |
| 1162 | + | |
| 1163 | + server-only@0.0.1: {} | |
| 1164 | + | |
| 1165 | + sharp@0.35.4(@types/node@24.13.4): | |
| 1166 | + dependencies: | |
| 1167 | + '@img/colour': 1.1.0 | |
| 1168 | + detect-libc: 2.1.2 | |
| 1169 | + semver: 7.8.5 | |
| 1170 | + optionalDependencies: | |
| 1171 | + '@img/sharp-darwin-arm64': 0.35.4 | |
| 1172 | + '@img/sharp-darwin-x64': 0.35.4 | |
| 1173 | + '@img/sharp-freebsd-wasm32': 0.35.4 | |
| 1174 | + '@img/sharp-libvips-darwin-arm64': 1.3.3 | |
| 1175 | + '@img/sharp-libvips-darwin-x64': 1.3.3 | |
| 1176 | + '@img/sharp-libvips-linux-arm': 1.3.3 | |
| 1177 | + '@img/sharp-libvips-linux-arm64': 1.3.3 | |
| 1178 | + '@img/sharp-libvips-linux-ppc64': 1.3.3 | |
| 1179 | + '@img/sharp-libvips-linux-riscv64': 1.3.3 | |
| 1180 | + '@img/sharp-libvips-linux-s390x': 1.3.3 | |
| 1181 | + '@img/sharp-libvips-linux-x64': 1.3.3 | |
| 1182 | + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 | |
| 1183 | + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 | |
| 1184 | + '@img/sharp-linux-arm': 0.35.4 | |
| 1185 | + '@img/sharp-linux-arm64': 0.35.4 | |
| 1186 | + '@img/sharp-linux-ppc64': 0.35.4 | |
| 1187 | + '@img/sharp-linux-riscv64': 0.35.4 | |
| 1188 | + '@img/sharp-linux-s390x': 0.35.4 | |
| 1189 | + '@img/sharp-linux-x64': 0.35.4 | |
| 1190 | + '@img/sharp-linuxmusl-arm64': 0.35.4 | |
| 1191 | + '@img/sharp-linuxmusl-x64': 0.35.4 | |
| 1192 | + '@img/sharp-webcontainers-wasm32': 0.35.4 | |
| 1193 | + '@img/sharp-win32-arm64': 0.35.4 | |
| 1194 | + '@img/sharp-win32-ia32': 0.35.4 | |
| 1195 | + '@img/sharp-win32-x64': 0.35.4 | |
| 1196 | + '@types/node': 24.13.4 | |
| 1197 | + optional: true | |
| 1198 | + | |
| 1199 | + source-map-js@1.2.1: {} | |
| 1200 | + | |
| 1201 | + styled-jsx@5.1.6(react@19.2.8): | |
| 1202 | + dependencies: | |
| 1203 | + client-only: 0.0.1 | |
| 1204 | + react: 19.2.8 | |
| 1205 | + | |
| 1206 | + tailwindcss@4.3.3: {} | |
| 1207 | + | |
| 1208 | + tapable@2.3.3: {} | |
| 1209 | + | |
| 1210 | + topojson-client@3.1.0: | |
| 1211 | + dependencies: | |
| 1212 | + commander: 2.20.3 | |
| 1213 | + | |
| 1214 | + tslib@2.8.1: {} | |
| 1215 | + | |
| 1216 | + typescript@5.9.3: {} | |
| 1217 | + | |
| 1218 | + undici-types@7.18.2: {} | |
| 1219 | + | |
| 1220 | + world-atlas@2.0.2: {} | |
added
pnpm-workspace.yaml
+2 −0
@@ -0,0 +1,2 @@ | ||
| 1 | +packages: | |
| 2 | + - apps/* | |
modified
pyproject.toml
+2 −0
@@ -25,6 +25,7 @@ dev = ["pytest>=8", "pytest-asyncio>=0.24", "ruff>=0.6", "respx>=0.21"] | ||
| 25 | 25 | |
| 26 | 26 | [project.scripts] |
| 27 | 27 | ca = "countryatlas.cli:app" |
| 28 | +ca-api = "countryatlas.api.main:run" | |
| 28 | 29 | |
| 29 | 30 | [build-system] |
| 30 | 31 | requires = ["hatchling"] |
@@ -39,3 +40,4 @@ target-version = "py312" | ||
| 39 | 40 | |
| 40 | 41 | [tool.pytest.ini_options] |
| 41 | 42 | testpaths = ["tests"] |
| 43 | +markers = ["live: hits real external endpoints (skipped unless --run-live or -m live)"] | |
modified
registry/indicators.yaml
+15 −10
@@ -29,7 +29,7 @@ indicators: | ||
| 29 | 29 | description: Gross domestic product at purchaser's prices, converted to US dollars at official exchange rates. |
| 30 | 30 | sources: |
| 31 | 31 | - {connector: worldbank, dataset: WDI, code: NY.GDP.MKTP.CD, priority: 1} |
| 32 | − - {connector: imf, dataset: WEO, code: NGDPD, priority: 2, transform: "x*1e9"} | |
| 32 | + - {connector: imf, dataset: WEO, code: NGDPD, priority: 2, notes: "SDMX CSV OBS_VALUE is already in US$ (SCALE=9 is display-only) — no transform"} | |
| 33 | 33 | - slug: gdp-ppp |
| 34 | 34 | name: GDP, PPP (current international $) |
| 35 | 35 | short_name: GDP (PPP) |
@@ -44,7 +44,7 @@ indicators: | ||
| 44 | 44 | description: GDP converted to international dollars using purchasing power parity rates. |
| 45 | 45 | sources: |
| 46 | 46 | - {connector: worldbank, dataset: WDI, code: NY.GDP.MKTP.PP.CD, priority: 1} |
| 47 | − - {connector: imf, dataset: WEO, code: PPPGDP, priority: 2, transform: "x*1e9"} | |
| 47 | + - {connector: imf, dataset: WEO, code: PPPGDP, priority: 2, notes: "SDMX CSV OBS_VALUE is already in international $ (SCALE=9 is display-only) — no transform"} | |
| 48 | 48 | - slug: gdp-per-capita |
| 49 | 49 | name: GDP per capita (current US$) |
| 50 | 50 | short_name: GDP per capita |
@@ -160,7 +160,7 @@ indicators: | ||
| 160 | 160 | precision: 2 |
| 161 | 161 | frequency: M |
| 162 | 162 | ranking_eligible: false |
| 163 | − bounds: [-2, 200] | |
| 163 | + bounds: [-2, 5000] | |
| 164 | 164 | description: Official policy interest rate set by the central bank (end of period). |
| 165 | 165 | sources: [] # bis WS_CBPOL (all), fred FEDFUNDS (USA) — added by connector agents |
| 166 | 166 | - slug: lending-rate |
@@ -475,7 +475,7 @@ indicators: | ||
| 475 | 475 | format: percent |
| 476 | 476 | bounds: [0, 100] |
| 477 | 477 | sources: |
| 478 | − - {connector: worldbank, dataset: WDI, code: GC.TAX.TOTL.GD.ZS, priority: 1} | |
| 478 | + - {connector: worldbank, dataset: WDI, code: GC.TAX.TOTL.GD.ZS, priority: 2, notes: "central government, cash basis — OECD Revenue Statistics (general government, accrual) is priority 1 where available"} | |
| 479 | 479 | - slug: social-expenditure-pct-gdp |
| 480 | 480 | name: Public social expenditure (% of GDP) |
| 481 | 481 | short_name: Social spending |
@@ -836,8 +836,7 @@ indicators: | ||
| 836 | 836 | precision: 0 |
| 837 | 837 | aggregation: sum |
| 838 | 838 | bounds: [0, null] |
| 839 | − sources: | |
| 840 | − - {connector: worldbank, dataset: WDI, code: SM.POP.REFG, priority: 1} | |
| 839 | + sources: [] # WB SM.POP.REFG archived (2026) → UNHCR via owid grapher, added by connector agents | |
| 841 | 840 | - slug: refugees-by-origin |
| 842 | 841 | name: Refugee population by country of origin |
| 843 | 842 | topic: security |
@@ -1382,7 +1381,7 @@ indicators: | ||
| 1382 | 1381 | format: per_1000 |
| 1383 | 1382 | higher_is_better: false |
| 1384 | 1383 | featured: true |
| 1385 | − bounds: [0, 300] | |
| 1384 | + bounds: [0, 600] | |
| 1386 | 1385 | sources: |
| 1387 | 1386 | - {connector: worldbank, dataset: WDI, code: SP.DYN.IMRT.IN, priority: 1} |
| 1388 | 1387 | - slug: under-5-mortality-rate |
@@ -1393,7 +1392,7 @@ indicators: | ||
| 1393 | 1392 | unit_short: /1,000 |
| 1394 | 1393 | format: per_1000 |
| 1395 | 1394 | higher_is_better: false |
| 1396 | − bounds: [0, 500] | |
| 1395 | + bounds: [0, 800] | |
| 1397 | 1396 | sources: |
| 1398 | 1397 | - {connector: worldbank, dataset: WDI, code: SH.DYN.MORT, priority: 1} |
| 1399 | 1398 | - slug: maternal-mortality-ratio |
@@ -1404,7 +1403,7 @@ indicators: | ||
| 1404 | 1403 | unit_short: /100k |
| 1405 | 1404 | format: per_100k |
| 1406 | 1405 | higher_is_better: false |
| 1407 | − bounds: [0, 5000] | |
| 1406 | + bounds: [0, 10000] | |
| 1408 | 1407 | sources: |
| 1409 | 1408 | - {connector: worldbank, dataset: WDI, code: SH.STA.MMRT, priority: 1} |
| 1410 | 1409 | - slug: health-expenditure-per-capita |
@@ -1441,7 +1440,7 @@ indicators: | ||
| 1441 | 1440 | bounds: [0, 30] |
| 1442 | 1441 | sources: |
| 1443 | 1442 | - {connector: worldbank, dataset: WDI, code: SH.MED.PHYS.ZS, priority: 1} |
| 1444 | − - {connector: who, dataset: GHO, code: HWF_0001, priority: 2, transform: "x/10"} | |
| 1443 | + - {connector: who, dataset: GHO, code: HWF_0001, priority: 3, transform: "x/10"} | |
| 1445 | 1444 | - slug: nurses-per-1000 |
| 1446 | 1445 | name: Nurses and midwives |
| 1447 | 1446 | topic: health |
@@ -2312,6 +2311,7 @@ indicators: | ||
| 2312 | 2311 | bounds: [0, null] |
| 2313 | 2312 | sources: |
| 2314 | 2313 | - {connector: owid, dataset: co2, code: co2, priority: 1} |
| 2314 | + - {connector: worldbank, dataset: WDI, code: EN.GHG.CO2.MT.CE.AR5, priority: 2} | |
| 2315 | 2315 | - slug: co2-per-capita |
| 2316 | 2316 | name: CO₂ emissions per capita |
| 2317 | 2317 | short_name: CO₂ per capita |
@@ -2327,6 +2327,7 @@ indicators: | ||
| 2327 | 2327 | change_floor: 0.5 |
| 2328 | 2328 | sources: |
| 2329 | 2329 | - {connector: owid, dataset: co2, code: co2_per_capita, priority: 1} |
| 2330 | + - {connector: worldbank, dataset: WDI, code: EN.GHG.CO2.PC.CE.AR5, priority: 2} | |
| 2330 | 2331 | - slug: co2-per-gdp |
| 2331 | 2332 | name: CO₂ emissions per unit of GDP |
| 2332 | 2333 | short_name: CO₂ intensity |
@@ -2387,6 +2388,7 @@ indicators: | ||
| 2387 | 2388 | bounds: [0, null] |
| 2388 | 2389 | sources: |
| 2389 | 2390 | - {connector: owid, dataset: co2, code: methane, priority: 1} |
| 2391 | + - {connector: worldbank, dataset: WDI, code: EN.GHG.CH4.MT.CE.AR5, priority: 2} | |
| 2390 | 2392 | - slug: nitrous-oxide-emissions |
| 2391 | 2393 | name: Nitrous oxide emissions |
| 2392 | 2394 | topic: climate |
@@ -2399,6 +2401,7 @@ indicators: | ||
| 2399 | 2401 | bounds: [0, null] |
| 2400 | 2402 | sources: |
| 2401 | 2403 | - {connector: owid, dataset: co2, code: nitrous_oxide, priority: 1} |
| 2404 | + - {connector: worldbank, dataset: WDI, code: EN.GHG.N2O.MT.CE.AR5, priority: 2} | |
| 2402 | 2405 | - slug: total-ghg-emissions |
| 2403 | 2406 | name: Total greenhouse gas emissions |
| 2404 | 2407 | topic: climate |
@@ -2411,6 +2414,7 @@ indicators: | ||
| 2411 | 2414 | bounds: [0, null] |
| 2412 | 2415 | sources: |
| 2413 | 2416 | - {connector: owid, dataset: co2, code: total_ghg, priority: 1} |
| 2417 | + - {connector: worldbank, dataset: WDI, code: EN.GHG.ALL.MT.CE.AR5, priority: 2} | |
| 2414 | 2418 | - slug: ghg-per-capita |
| 2415 | 2419 | name: Greenhouse gas emissions per capita |
| 2416 | 2420 | topic: climate |
@@ -2423,6 +2427,7 @@ indicators: | ||
| 2423 | 2427 | bounds: [0, 200] |
| 2424 | 2428 | sources: |
| 2425 | 2429 | - {connector: owid, dataset: co2, code: ghg_per_capita, priority: 1} |
| 2430 | + - {connector: worldbank, dataset: WDI, code: EN.GHG.ALL.PC.CE.AR5, priority: 2} | |
| 2426 | 2431 | - slug: coal-co2 |
| 2427 | 2432 | name: CO₂ emissions from coal |
| 2428 | 2433 | topic: climate |
added
registry/insights.yaml
+103 −0
@@ -0,0 +1,103 @@ | ||
| 1 | +# Insight templates (ARCHITECTURE §7). Every number is computed from the snapshot; the text is a plain template. | |
| 2 | +# | |
| 3 | +# kinds: | |
| 4 | +# change_since value at `since` (nearest year within ±3) vs latest → {v0} {v1} {y0} {y1} {delta} {pct} {abs_delta} {abs_pct} | |
| 5 | +# {verb} = grew|fell (up|down for `verb_style: updown`, rose|declined for `verb_style: rise`) | |
| 6 | +# rank_in_group rank of the country's latest value among the group's members for the same year → {rank} {n} {group} {v1} {y1} | |
| 7 | +# group ∈ world | region | income | <group id such as oecd, eu, g20>; needs ≥ `min_n` members with data | |
| 8 | +# vs_median latest value vs the group's median for the same year → {median} {ratio} {diff} {above_below} {v1} {y1} | |
| 9 | +# avg_growth mean of the indicator over [from, latest] (for growth-rate indicators) → {avg} {y0} {y1} {n_years} | |
| 10 | +# Placeholders {country} and {indicator} are always available. Values are formatted with the indicator's format/unit. | |
| 11 | +templates: | |
| 12 | + - id: population-since-2000 | |
| 13 | + kind: change_since | |
| 14 | + indicator: population | |
| 15 | + since: 2000 | |
| 16 | + verb_style: grow | |
| 17 | + text: "{country}'s population {verb} {abs_pct} between {y0} and {y1}, from {v0} to {v1}." | |
| 18 | + - id: gdp-per-capita-since-2000 | |
| 19 | + kind: change_since | |
| 20 | + indicator: gdp-per-capita | |
| 21 | + since: 2000 | |
| 22 | + verb_style: rise | |
| 23 | + text: "GDP per capita {verb} from {v0} in {y0} to {v1} in {y1} ({signed_pct} in current US dollars)." | |
| 24 | + - id: gdp-growth-decade-average | |
| 25 | + kind: avg_growth | |
| 26 | + indicator: gdp-growth | |
| 27 | + from: 2014 | |
| 28 | + text: "GDP grew {avg} a year on average over {y0}–{y1}." | |
| 29 | + - id: gdp-per-capita-rank-oecd | |
| 30 | + kind: rank_in_group | |
| 31 | + indicator: gdp-per-capita-ppp | |
| 32 | + group: oecd | |
| 33 | + min_n: 20 | |
| 34 | + text: "{country} ranks {rank} of {n} OECD members for GDP per capita (PPP), at {v1} in {y1}." | |
| 35 | + - id: gdp-per-capita-rank-region | |
| 36 | + kind: rank_in_group | |
| 37 | + indicator: gdp-per-capita-ppp | |
| 38 | + group: region | |
| 39 | + min_n: 5 | |
| 40 | + text: "Within {group}, {country} ranks {rank} of {n} for GDP per capita (PPP) ({v1}, {y1})." | |
| 41 | + - id: life-expectancy-rank-region | |
| 42 | + kind: rank_in_group | |
| 43 | + indicator: life-expectancy | |
| 44 | + group: region | |
| 45 | + min_n: 5 | |
| 46 | + text: "Life expectancy of {v1} ranks {rank} of {n} in {group} ({y1})." | |
| 47 | + - id: life-expectancy-since-2000 | |
| 48 | + kind: change_since | |
| 49 | + indicator: life-expectancy | |
| 50 | + since: 2000 | |
| 51 | + verb_style: gain | |
| 52 | + text: "Life expectancy {verb} {abs_delta} since {y0}, from {v0} to {v1} in {y1}." | |
| 53 | + - id: renewables-since-2010 | |
| 54 | + kind: change_since | |
| 55 | + indicator: renewable-electricity-share | |
| 56 | + since: 2010 | |
| 57 | + verb_style: rise | |
| 58 | + text: "Renewables' share of electricity generation {verb} from {v0} in {y0} to {v1} in {y1} ({signed_delta} points)." | |
| 59 | + - id: urbanisation-since-2000 | |
| 60 | + kind: change_since | |
| 61 | + indicator: urban-population-share | |
| 62 | + since: 2000 | |
| 63 | + verb_style: rise | |
| 64 | + text: "{v1} of the population lived in urban areas in {y1}, compared with {v0} in {y0}." | |
| 65 | + - id: debt-since-2010 | |
| 66 | + kind: change_since | |
| 67 | + indicator: government-debt-pct-gdp | |
| 68 | + since: 2010 | |
| 69 | + verb_style: rise | |
| 70 | + text: "Central government debt {verb} from {v0} of GDP in {y0} to {v1} in {y1}." | |
| 71 | + - id: co2-per-capita-vs-world | |
| 72 | + kind: vs_median | |
| 73 | + indicator: co2-per-capita | |
| 74 | + group: world | |
| 75 | + text: "CO₂ emissions of {v1} per person are {ratio} the world median of {median} ({y1})." | |
| 76 | + - id: median-age-rank-world | |
| 77 | + kind: rank_in_group | |
| 78 | + indicator: median-age | |
| 79 | + group: world | |
| 80 | + min_n: 50 | |
| 81 | + text: "With a median age of {v1}, {country} ranks {rank} of {n} countries worldwide ({y1})." | |
| 82 | + - id: internet-users-since-2010 | |
| 83 | + kind: change_since | |
| 84 | + indicator: internet-users | |
| 85 | + since: 2010 | |
| 86 | + verb_style: rise | |
| 87 | + text: "{v1} of people used the internet in {y1}, up from {v0} in {y0}." | |
| 88 | + - id: unemployment-vs-region | |
| 89 | + kind: vs_median | |
| 90 | + indicator: unemployment-rate | |
| 91 | + group: region | |
| 92 | + text: "Unemployment of {v1} in {y1} is {above_below} the {group} median of {median}." | |
| 93 | + - id: trade-vs-world | |
| 94 | + kind: vs_median | |
| 95 | + indicator: trade-pct-gdp | |
| 96 | + group: world | |
| 97 | + text: "Trade is worth {v1} of GDP ({y1}), {above_below} the world median of {median}." | |
| 98 | + - id: fertility-since-2000 | |
| 99 | + kind: change_since | |
| 100 | + indicator: fertility-rate | |
| 101 | + since: 2000 | |
| 102 | + verb_style: rise | |
| 103 | + text: "The fertility rate {verb} from {v0} children per woman in {y0} to {v1} in {y1}." | |
added
registry/similarity.yaml
+88 −0
@@ -0,0 +1,88 @@ | ||
| 1 | +# Similarity features per mode (ARCHITECTURE §7) and Country DNA dimensions. | |
| 2 | +# | |
| 3 | +# Each feature: indicator (registry slug), transform: none|log|log1p, weight (default 1). | |
| 4 | +# Optional `per: <slug>` divides the indicator by another one (e.g. net migration per 1,000 people, scale multiplies). | |
| 5 | +# Countries need ≥ 70 % of a mode's features (by weight) to be scored; z-scores are computed across countries from the | |
| 6 | +# `latest` table; distance = weighted Euclidean over shared features, rescaled to the full weight; score = 100·exp(−d/d0) | |
| 7 | +# where d0 = median pairwise distance of the mode. Top 12 peers are stored with per-feature contributions. | |
| 8 | +modes: | |
| 9 | + overall: | |
| 10 | + features: | |
| 11 | + - {indicator: gdp-per-capita-ppp, transform: log, weight: 1.5} | |
| 12 | + - {indicator: population, transform: log} | |
| 13 | + - {indicator: median-age} | |
| 14 | + - {indicator: urban-population-share} | |
| 15 | + - {indicator: trade-pct-gdp} | |
| 16 | + - {indicator: fertility-rate} | |
| 17 | + - {indicator: government-expenditure-pct-gdp} | |
| 18 | + - {indicator: life-expectancy} | |
| 19 | + - {indicator: internet-users} | |
| 20 | + - {indicator: co2-per-capita, transform: log1p} | |
| 21 | + - {indicator: renewable-electricity-share} | |
| 22 | + - {indicator: services-value-added-pct-gdp} | |
| 23 | + economic: | |
| 24 | + features: | |
| 25 | + - {indicator: gdp-per-capita-ppp, transform: log, weight: 1.5} | |
| 26 | + - {indicator: gdp-growth} | |
| 27 | + - {indicator: inflation} | |
| 28 | + - {indicator: trade-pct-gdp} | |
| 29 | + - {indicator: services-value-added-pct-gdp} | |
| 30 | + - {indicator: industry-value-added-pct-gdp} | |
| 31 | + - {indicator: agriculture-value-added-pct-gdp} | |
| 32 | + - {indicator: gross-capital-formation-pct-gdp} | |
| 33 | + - {indicator: unemployment-rate} | |
| 34 | + - {indicator: government-expenditure-pct-gdp} | |
| 35 | + demographic: | |
| 36 | + features: | |
| 37 | + - {indicator: median-age, weight: 1.5} | |
| 38 | + - {indicator: fertility-rate} | |
| 39 | + - {indicator: population-growth} | |
| 40 | + - {indicator: urban-population-share} | |
| 41 | + - {indicator: population-65-plus-share} | |
| 42 | + - {indicator: life-expectancy} | |
| 43 | + - {indicator: net-migration, per: population, scale: 1000} | |
| 44 | + - {indicator: population-density, transform: log} | |
| 45 | + energy: | |
| 46 | + features: | |
| 47 | + - {indicator: energy-use-per-capita, transform: log, weight: 1.5} | |
| 48 | + - {indicator: renewable-electricity-share} | |
| 49 | + - {indicator: fossil-electricity-share} | |
| 50 | + - {indicator: nuclear-electricity-share} | |
| 51 | + - {indicator: energy-imports-share} | |
| 52 | + - {indicator: co2-per-capita, transform: log1p} | |
| 53 | + - {indicator: carbon-intensity-electricity} | |
| 54 | + social: | |
| 55 | + features: | |
| 56 | + - {indicator: life-expectancy, weight: 1.5} | |
| 57 | + - {indicator: tertiary-enrollment} | |
| 58 | + - {indicator: gini-index} | |
| 59 | + - {indicator: internet-users} | |
| 60 | + - {indicator: health-expenditure-pct-gdp} | |
| 61 | + - {indicator: education-expenditure-pct-gdp} | |
| 62 | + - {indicator: homicide-rate, transform: log1p} | |
| 63 | + - {indicator: infant-mortality-rate, transform: log1p} | |
| 64 | + | |
| 65 | +# Country DNA: 9 dimensions in [0, 100] = percentile rank among countries (mean of the listed indicators' percentiles; | |
| 66 | +# `invert: true` uses 100 − percentile so that the dimension reads in the same direction as its label). Descriptive only. | |
| 67 | +dna: | |
| 68 | + income: | |
| 69 | + - {indicator: gdp-per-capita-ppp} | |
| 70 | + demographics: # "older population" → high | |
| 71 | + - {indicator: median-age} | |
| 72 | + - {indicator: fertility-rate, invert: true} | |
| 73 | + urbanization: | |
| 74 | + - {indicator: urban-population-share} | |
| 75 | + trade: | |
| 76 | + - {indicator: trade-pct-gdp} | |
| 77 | + energy: | |
| 78 | + - {indicator: energy-use-per-capita} | |
| 79 | + emissions: | |
| 80 | + - {indicator: co2-per-capita} | |
| 81 | + innovation: | |
| 82 | + - {indicator: rd-expenditure-pct-gdp} | |
| 83 | + - {indicator: patent-applications-residents, per: population, scale: 1000000} | |
| 84 | + education: | |
| 85 | + - {indicator: tertiary-enrollment} | |
| 86 | + - {indicator: expected-years-of-schooling} | |
| 87 | + public_spending: | |
| 88 | + - {indicator: government-expenditure-pct-gdp} | |
added
registry/sources/bis.yaml
+13 −0
@@ -0,0 +1,13 @@ | ||
| 1 | +# BIS Data Portal (SDMX v2, CSV) mappings — connector `bis`. `dataset` = dataflow, `code` = SDMX key (REF_AREA empty = | |
| 2 | +# all areas; ISO2, `XM` euro area and `4T`/`5R`/`XW` aggregates are dropped). Verified 2026-09-11 (docs §8). | |
| 3 | +# BIS indices are 2010=100; the registry indices are 2015=100 → `rebase:2015` (per-country mean of 2015 = 100). | |
| 4 | +sources: | |
| 5 | +- {indicator: policy-rate, dataset: WS_CBPOL, code: "M.", frequency: M, priority: 1, | |
| 6 | + notes: "Central bank policy rates, monthly, end of period, % per annum (unit 368); ~34 economies. US = midpoint of the | |
| 7 | + Fed target range; euro area (XM) dropped (not a country)."} | |
| 8 | +- {indicator: real-house-price-index, dataset: WS_SPP, code: "Q..R.628", frequency: Q, priority: 1, transform: "rebase:2015", | |
| 9 | + notes: "Selected residential property prices, real (CPI-deflated) index, 2010=100 re-based to 2015=100, ~57 countries."} | |
| 10 | +- {indicator: nominal-house-price-index, dataset: WS_SPP, code: "Q..N.628", frequency: Q, priority: 1, transform: "rebase:2015", | |
| 11 | + notes: "Selected residential property prices, nominal index, 2010=100 re-based to 2015=100."} | |
| 12 | +- {indicator: house-price-growth, dataset: WS_SPP, code: "Q..R.771", frequency: Q, priority: 1, | |
| 13 | + notes: "Real residential property prices, year-on-year % change (unit 771) as published by BIS."} | |
added
registry/sources/eurostat.yaml
+119 −0
@@ -0,0 +1,119 @@ | ||
| 1 | +# Eurostat JSON-stat mappings for the `eurostat` connector (verified with curl on 2026-09-11 — docs/sources-research.md §4). | |
| 2 | +# dataset = Eurostat dataset code; params = dimension filters (every dimension except geo/time must be pinned to ONE code). | |
| 3 | +# geo EL→GR, UK→GB; EU27_2020/EA20/EA21 aggregates are dropped. Status p/e/s → is_estimate, f → is_forecast. | |
| 4 | +sources: | |
| 5 | +# ---------------------------------------------------------------- economy | |
| 6 | +- indicator: inflation | |
| 7 | + dataset: prc_hicp_aind | |
| 8 | + code: RCH_A_AVG.CP00 | |
| 9 | + params: {unit: RCH_A_AVG, coicop: CP00} | |
| 10 | + priority: 2 | |
| 11 | + notes: HICP all-items, annual average rate of change (%). WB CPI stays priority 1; IMF WEO is also 2 for non-EU areas. | |
| 12 | +- indicator: nominal-house-price-index | |
| 13 | + dataset: prc_hpi_a | |
| 14 | + code: TOTAL.I15_A_AVG | |
| 15 | + params: {purchase: TOTAL, unit: I15_A_AVG} | |
| 16 | + priority: 2 | |
| 17 | + frequency: A | |
| 18 | + notes: House price index (total purchases), annual average, 2015=100 — annual series behind the quarterly OECD index. | |
| 19 | +# ---------------------------------------------------------------- government | |
| 20 | +- indicator: general-government-gross-debt-pct-gdp | |
| 21 | + dataset: gov_10dd_edpt1 | |
| 22 | + code: GD.S13.PC_GDP | |
| 23 | + params: {na_item: GD, sector: S13, unit: PC_GDP} | |
| 24 | + priority: 2 | |
| 25 | + notes: Government consolidated gross debt (EDP, Maastricht definition), % of GDP. | |
| 26 | +# ---------------------------------------------------------------- labour | |
| 27 | +- indicator: unemployment-rate | |
| 28 | + dataset: une_rt_a | |
| 29 | + code: Y15-74.PC_ACT.T | |
| 30 | + params: {age: Y15-74, unit: PC_ACT, sex: T} | |
| 31 | + priority: 3 | |
| 32 | + notes: LFS unemployment rate, 15–74, % of labour force. | |
| 33 | +- indicator: employment-rate | |
| 34 | + dataset: lfsi_emp_a | |
| 35 | + code: EMP_LFS.Y15-64.PC_POP.T | |
| 36 | + params: {indic_em: EMP_LFS, age: Y15-64, unit: PC_POP, sex: T} | |
| 37 | + priority: 1 | |
| 38 | + notes: LFS employment rate, 15–64, % of population. | |
| 39 | +- indicator: long-term-unemployment-share | |
| 40 | + dataset: une_ltu_a | |
| 41 | + code: LTU.Y15-74.PC_UNE.T | |
| 42 | + params: {indic_em: LTU, age: Y15-74, unit: PC_UNE, sex: T} | |
| 43 | + priority: 1 | |
| 44 | + notes: Long-term unemployed (12 months+) as % of total unemployment, 15–74. | |
| 45 | +- indicator: part-time-employment-share | |
| 46 | + dataset: lfsa_eppga | |
| 47 | + code: Y15-64.PC.T | |
| 48 | + params: {age: Y15-64, unit: PC, sex: T} | |
| 49 | + priority: 1 | |
| 50 | + notes: Part-time employment as % of total employment, 15–64. | |
| 51 | +# ---------------------------------------------------------------- income & poverty | |
| 52 | +- indicator: median-household-income | |
| 53 | + dataset: ilc_di03 | |
| 54 | + code: MED_EI.TOTAL.T.PPS | |
| 55 | + params: {statinfo: MED_EI, age: TOTAL, sex: T, unit: PPS} | |
| 56 | + priority: 1 | |
| 57 | + notes: Median equivalised net income, purchasing power standard (PPS), total population. | |
| 58 | +- indicator: at-risk-of-poverty-rate | |
| 59 | + dataset: ilc_li02 | |
| 60 | + code: MED_EI.B_60.PC.T.TOTAL | |
| 61 | + params: {statinfo: MED_EI, rskpovth: B_60, unit: PC, sex: T, age: TOTAL} | |
| 62 | + priority: 1 | |
| 63 | + notes: "At-risk-of-poverty rate (income below 60 % of national median equivalised income), % of population. statinfo=MED_EI is the only valid code (there is no RT)." | |
| 64 | +# ---------------------------------------------------------------- housing | |
| 65 | +- indicator: housing-cost-overburden-rate | |
| 66 | + dataset: ilc_lvho07a | |
| 67 | + code: TOTAL.TOTAL.T.PC | |
| 68 | + params: {rskpovth: TOTAL, age: TOTAL, sex: T, unit: PC} | |
| 69 | + priority: 1 | |
| 70 | + notes: Share of population living in households where housing costs exceed 40 % of disposable income. | |
| 71 | +- indicator: homeownership-rate | |
| 72 | + dataset: ilc_lvho02 | |
| 73 | + code: TOTAL.TOTAL.OWN.PC | |
| 74 | + params: {rskpovth: TOTAL, hhcomp: TOTAL, tenure: OWN, unit: PC} | |
| 75 | + priority: 1 | |
| 76 | + notes: Distribution of population by tenure status — owners (with or without mortgage), % of population (EU-SILC). | |
| 77 | +# ---------------------------------------------------------------- innovation, energy, climate, health, education | |
| 78 | +- indicator: rd-expenditure-pct-gdp | |
| 79 | + dataset: rd_e_gerdtot | |
| 80 | + code: TOTAL.PC_GDP | |
| 81 | + params: {sectperf: TOTAL, unit: PC_GDP} | |
| 82 | + priority: 3 | |
| 83 | + notes: GERD all sectors, % of GDP (behind WB 1 and OECD MSTI 2). | |
| 84 | +- indicator: renewable-energy-consumption-share | |
| 85 | + dataset: nrg_ind_ren | |
| 86 | + code: REN.PC | |
| 87 | + params: {nrg_bal: REN, unit: PC} | |
| 88 | + priority: 2 | |
| 89 | + notes: Share of renewable energy in gross final energy consumption (SHARES), %. | |
| 90 | +- indicator: total-ghg-emissions | |
| 91 | + dataset: env_air_gge | |
| 92 | + code: GHG.TOTX4_MEMO.MIO_T | |
| 93 | + params: {airpol: GHG, src_crf: TOTX4_MEMO, unit: MIO_T} | |
| 94 | + priority: 3 | |
| 95 | + notes: Total GHG excluding LULUCF and memo items, million tonnes CO2-equivalent (UNFCCC inventories). Behind OWID 1 and WB 2. | |
| 96 | +- indicator: life-expectancy | |
| 97 | + dataset: demo_mlexpec | |
| 98 | + code: T.Y_LT1 | |
| 99 | + params: {sex: T, age: Y_LT1} | |
| 100 | + priority: 3 | |
| 101 | + notes: Life expectancy at birth, total. | |
| 102 | +- indicator: tertiary-attainment-25-34 | |
| 103 | + dataset: edat_lfse_03 | |
| 104 | + code: T.Y25-34.ED5-8.PC | |
| 105 | + params: {sex: T, age: Y25-34, isced11: ED5-8, unit: PC} | |
| 106 | + priority: 2 | |
| 107 | + notes: Share of 25–34 year-olds with tertiary education (ISCED 5–8). | |
| 108 | +- indicator: tertiary-attainment-25-64 | |
| 109 | + dataset: edat_lfse_03 | |
| 110 | + code: T.Y25-64.ED5-8.PC | |
| 111 | + params: {sex: T, age: Y25-64, isced11: ED5-8, unit: PC} | |
| 112 | + priority: 2 | |
| 113 | + notes: Share of 25–64 year-olds with tertiary education (ISCED 5–8). | |
| 114 | +- indicator: upper-secondary-attainment | |
| 115 | + dataset: edat_lfse_03 | |
| 116 | + code: T.Y25-64.ED3-8.PC | |
| 117 | + params: {sex: T, age: Y25-64, isced11: ED3-8, unit: PC} | |
| 118 | + priority: 3 | |
| 119 | + notes: Share of 25–64 year-olds with at least upper secondary education (ISCED 3–8). Behind OECD 1 and WB 2. | |
added
registry/sources/fred.yaml
+44 −0
@@ -0,0 +1,44 @@ | ||
| 1 | +# FRED mappings — connector `fred`. All series verified on 2026-09-11 via /fred/series (docs/sources-research.md §6). | |
| 2 | +# One spec = one series = one country (default USA). `frequency` follows the series (weekly → monthly mean via | |
| 3 | +# frequency=m&aggregation_method=avg, done automatically by the connector). Special transforms handled in code: | |
| 4 | +# `yoy_pct` (year-on-year % change) and `rebase:2015` (re-index so 2015 mean = 100). | |
| 5 | +sources: | |
| 6 | +# --- money & rates ------------------------------------------------------------------------------------------------ | |
| 7 | +- {indicator: policy-rate, dataset: FRED, code: FEDFUNDS, countries: [USA], frequency: M, priority: 2, | |
| 8 | + notes: "Federal funds effective rate, monthly average (BIS WS_CBPOL = priority 1, end-of-period target midpoint)."} | |
| 9 | +- {indicator: mortgage-rate, dataset: FRED, code: MORTGAGE30US, countries: [USA], frequency: M, priority: 1, | |
| 10 | + notes: "Freddie Mac 30-year fixed mortgage average; weekly series aggregated to monthly mean by FRED."} | |
| 11 | +# --- housing ------------------------------------------------------------------------------------------------------ | |
| 12 | +- {indicator: housing-starts, dataset: FRED, code: HOUST, countries: [USA], frequency: M, priority: 1, | |
| 13 | + notes: "New privately-owned housing units started, thousands of units, seasonally adjusted annual rate."} | |
| 14 | +- {indicator: building-permits, dataset: FRED, code: PERMIT, countries: [USA], frequency: M, priority: 1, | |
| 15 | + notes: "New privately-owned housing units authorized by building permits, thousands, SAAR."} | |
| 16 | +- {indicator: homeownership-rate, dataset: FRED, code: RHORUSQ156N, countries: [USA], frequency: Q, priority: 1, | |
| 17 | + notes: "Homeownership rate in the United States (Census HVS), quarterly, not seasonally adjusted."} | |
| 18 | +# --- income: MEHOINUSA672N dropped (unit mismatch: 2024 C-CPI-U dollars vs registry euro PPS) ----------------- | |
| 19 | +# --- economy / labour --------------------------------------------------------------------------------------------- | |
| 20 | +- {indicator: industrial-production-index, dataset: FRED, code: INDPRO, countries: [USA], frequency: M, priority: 2, | |
| 21 | + transform: "rebase:2015", notes: "Industrial Production: Total Index (2017=100, SA) re-based to 2015=100."} | |
| 22 | +- {indicator: unemployment-rate, dataset: FRED, code: UNRATE, countries: [USA], frequency: M, priority: 3, | |
| 23 | + notes: "Civilian unemployment rate, monthly, SA (monthly granularity behind the WB/IMF annual series)."} | |
| 24 | +- {indicator: inflation, dataset: FRED, code: CPIAUCSL, countries: [USA], frequency: M, priority: 3, transform: "yoy_pct", | |
| 25 | + notes: "CPI-U all items (1982-84=100, SA) → year-on-year % change computed in normalize."} | |
| 26 | +- {indicator: labor-force-participation-rate, dataset: FRED, code: CIVPART, countries: [USA], frequency: M, priority: 3, | |
| 27 | + notes: "Civilian labor force participation rate, 16+, monthly, SA."} | |
| 28 | +- {indicator: government-debt-pct-gdp, dataset: FRED, code: GFDEGDQ188S, countries: [USA], frequency: Q, priority: 2, | |
| 29 | + notes: "Federal debt: total public debt as % of GDP, quarterly, SA."} | |
| 30 | +# --- BIS real residential property prices republished by FRED (2010=100 → re-based to 2015=100) — priority 3 behind | |
| 31 | +# BIS (1) and OECD (2). The country is derived from the code (Q{ISO2}R628BIS). All 13 ids verified via /fred/series/search. | |
| 32 | +- {indicator: real-house-price-index, dataset: FRED, code: QUSR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 33 | +- {indicator: real-house-price-index, dataset: FRED, code: QCAR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 34 | +- {indicator: real-house-price-index, dataset: FRED, code: QGBR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 35 | +- {indicator: real-house-price-index, dataset: FRED, code: QFRR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 36 | +- {indicator: real-house-price-index, dataset: FRED, code: QDER628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 37 | +- {indicator: real-house-price-index, dataset: FRED, code: QJPR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 38 | +- {indicator: real-house-price-index, dataset: FRED, code: QITR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 39 | +- {indicator: real-house-price-index, dataset: FRED, code: QESR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 40 | +- {indicator: real-house-price-index, dataset: FRED, code: QAUR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 41 | +- {indicator: real-house-price-index, dataset: FRED, code: QKRR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 42 | +- {indicator: real-house-price-index, dataset: FRED, code: QNLR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 43 | +- {indicator: real-house-price-index, dataset: FRED, code: QSER628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
| 44 | +- {indicator: real-house-price-index, dataset: FRED, code: QCHR628BIS, frequency: Q, priority: 3, transform: "rebase:2015"} | |
added
registry/sources/ilo.yaml
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +# ILOSTAT SDMX mappings — connector `ilo`. `dataset` = dataflow id, `code` = key REF_AREA.FREQ.MEASURE.SEX.AGE with | |
| 2 | +# REF_AREA empty (all areas). DF_UNE_2EAP_SEX_AGE_RT = "ILO modelled estimates, Nov. 2025" incl. projections | |
| 3 | +# (years > release year → is_forecast; OBS_STATUS != R → is_estimate). Verified 2026-09-11 (docs §8). | |
| 4 | +# WB already republishes these modelled estimates (priority 1) → ILO direct is a low-priority fallback (4). | |
| 5 | +sources: | |
| 6 | +- {indicator: unemployment-rate, dataset: DF_UNE_2EAP_SEX_AGE_RT, code: ".A..SEX_T.AGE_YTHADULT_YGE15", frequency: A, priority: 4, | |
| 7 | + notes: "Unemployment rate, total, 15+, ILO modelled estimates (Nov. 2025) with projections."} | |
| 8 | +- {indicator: youth-unemployment-rate, dataset: DF_UNE_2EAP_SEX_AGE_RT, code: ".A..SEX_T.AGE_YTHADULT_Y15-24", frequency: A, priority: 4, | |
| 9 | + notes: "Youth (15–24) unemployment rate, total, ILO modelled estimates (Nov. 2025) with projections."} | |
added
registry/sources/imf.yaml
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +# IMF World Economic Outlook (April 2026) — extra mappings for the `imf` connector. | |
| 2 | +# The core WEO series (NGDPD, PPPGDP, NGDPDPC, PPPPC, NGDP_RPCH, PCPIPCH, BCA_NGDPD, GGXWDG_NGDP, GGXCNL_NGDP, GGR_NGDP, | |
| 3 | +# GGX_NGDP, LUR) are declared inline in registry/indicators.yaml — do not repeat them here. | |
| 4 | +# OBS_VALUE in the SDMX CSV is already in base units (US$, persons): no scale transforms. | |
| 5 | +sources: | |
| 6 | +- indicator: gross-savings-pct-gdp | |
| 7 | + dataset: WEO | |
| 8 | + code: NGSD_NGDP | |
| 9 | + priority: 2 | |
| 10 | + notes: Gross national savings, % of GDP (WEO, incl. projections). | |
| 11 | +- indicator: gross-capital-formation-pct-gdp | |
| 12 | + dataset: WEO | |
| 13 | + code: NID_NGDP | |
| 14 | + priority: 2 | |
| 15 | + notes: Total investment (gross capital formation), % of GDP (WEO, incl. projections). | |
| 16 | +- indicator: population | |
| 17 | + dataset: WEO | |
| 18 | + code: LP | |
| 19 | + priority: 3 | |
| 20 | + notes: WEO population (persons; SCALE=6 in the CSV only describes the display unit). Includes projections. | |
| 21 | +- indicator: current-account-balance | |
| 22 | + dataset: WEO | |
| 23 | + code: BCA | |
| 24 | + priority: 2 | |
| 25 | + notes: Current account balance, US$ (base units in the CSV; SCALE=9 is display-only). Includes projections. | |
added
registry/sources/oecd.yaml
+105 −0
@@ -0,0 +1,105 @@ | ||
| 1 | +# OECD SDMX mappings for the `oecd` connector (verified with curl on 2026-09-11 — see docs/sources-research.md §3). | |
| 2 | +# dataset = "{AGENCY},{DSD@DF}[,{version}]"; code = full SDMX key, "*" = all REF_AREA. Dimension order is the | |
| 3 | +# dataflow's own order (listed in the notes). Aggregates (OECD, EU27_2020, EA20, G20 …) are dropped at normalize time. | |
| 4 | +sources: | |
| 5 | +# ---------------------------------------------------------------- housing (Analytical house prices, quarterly, 2015=100) | |
| 6 | +- indicator: real-house-price-index | |
| 7 | + dataset: OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0 | |
| 8 | + code: "*.Q.RHP.IX" | |
| 9 | + priority: 1 | |
| 10 | + frequency: Q | |
| 11 | + notes: "REF_AREA.FREQ.MEASURE.UNIT_MEASURE — real house price index, seasonally adjusted, 2015=100." | |
| 12 | +- indicator: nominal-house-price-index | |
| 13 | + dataset: OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0 | |
| 14 | + code: "*.Q.HPI.IX" | |
| 15 | + priority: 1 | |
| 16 | + frequency: Q | |
| 17 | + notes: Nominal house price index, seasonally adjusted, 2015=100. | |
| 18 | +- indicator: rent-price-index | |
| 19 | + dataset: OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0 | |
| 20 | + code: "*.Q.RPI.IX" | |
| 21 | + priority: 1 | |
| 22 | + frequency: Q | |
| 23 | + notes: Rent price index, 2015=100. | |
| 24 | +- indicator: price-to-income-ratio | |
| 25 | + dataset: OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0 | |
| 26 | + code: "*.Q.HPI_YDH.IX" | |
| 27 | + priority: 1 | |
| 28 | + frequency: Q | |
| 29 | + notes: Nominal house prices / nominal disposable income per head, 2015=100. | |
| 30 | +- indicator: price-to-rent-ratio | |
| 31 | + dataset: OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0 | |
| 32 | + code: "*.Q.HPI_RPI.IX" | |
| 33 | + priority: 1 | |
| 34 | + frequency: Q | |
| 35 | + notes: Nominal house prices / rent prices, 2015=100. | |
| 36 | +- indicator: house-price-growth | |
| 37 | + dataset: OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0 | |
| 38 | + code: "*.Q.RHP.IX" | |
| 39 | + priority: 1 | |
| 40 | + frequency: Q | |
| 41 | + transform: yoy | |
| 42 | + notes: "Derived in the connector: year-on-year % change of the real house price index (RHP), quarter vs same quarter a year earlier. No direct growth series in the dataflow." | |
| 43 | +# ---------------------------------------------------------------- labour | |
| 44 | +- indicator: average-annual-wages | |
| 45 | + dataset: OECD.ELS.SAE,DSD_EARNINGS@AV_AN_WAGE | |
| 46 | + code: "*.WG.USD_PPP.A.Q.MEAN._Z" | |
| 47 | + priority: 1 | |
| 48 | + notes: "REF_AREA.MEASURE.UNIT_MEASURE.PAY_PERIOD.PRICE_BASE.AGGREGATION_OPERATION.SEX — average annual wages, USD PPP, constant prices (base 2025)." | |
| 49 | +- indicator: hours-worked | |
| 50 | + dataset: OECD.ELS.SAE,DSD_HW@DF_AVG_ANN_HRS_WKD | |
| 51 | + code: "*.HW.H_Y_PS._Z._Z.EMP.A.ACTUAL._T._Z.MEAN._Z._T" | |
| 52 | + priority: 1 | |
| 53 | + notes: "REF_AREA.MEASURE.UNIT_MEASURE.SEX.AGE.LABOUR_FORCE_STATUS.WORK_PERIOD.HOURS_TYPE.WORKER_STATUS.WORK_TIME_ARNGMNT.AGGREGATION_OPERATION.HOUR_BANDS.JOB_COVERAGE — average actual annual hours worked per person employed, total employment." | |
| 54 | +# ---------------------------------------------------------------- government | |
| 55 | +- indicator: tax-revenue-pct-gdp | |
| 56 | + dataset: OECD.CTP.TPS,DSD_REV_COMP_OECD@DF_RSOECD,2.0 | |
| 57 | + code: "*.TAX_REV.S13._T._T.PT_B1GQ.A" | |
| 58 | + priority: 1 | |
| 59 | + notes: "REF_AREA.MEASURE.SECTOR.STANDARD_REVENUE.CTRY_SPECIFIC_REVENUE.UNIT_MEASURE.FREQ — Revenue Statistics, total tax revenue (_T, accrual) of general government, % of GDP. OECD members + partners; WB GC.TAX.TOTL.GD.ZS (central government, cash) is priority 2." | |
| 60 | +- indicator: social-expenditure-pct-gdp | |
| 61 | + dataset: OECD.ELS.SPD,DSD_SOCX_AGG@DF_SOCX_AGG | |
| 62 | + code: "*.A.SOCX.PT_B1GQ.ES10._T._T._Z" | |
| 63 | + priority: 1 | |
| 64 | + notes: "REF_AREA.FREQ.MEASURE.UNIT_MEASURE.EXPEND_SOURCE.SPENDING_TYPE.PROGRAMME_TYPE.PRICE_BASE — SOCX public social expenditure (ES10), all programmes, % of GDP." | |
| 65 | +# ---------------------------------------------------------------- health | |
| 66 | +- indicator: hospital-beds-per-1000 | |
| 67 | + dataset: OECD.ELS.HD,DSD_HEALTH_REAC_HOSP@DF_HOSP_REAC | |
| 68 | + code: "*.HB.10P3HB._Z._Z._T._T._Z._Z" | |
| 69 | + priority: 2 | |
| 70 | + notes: "REF_AREA.MEASURE.UNIT_MEASURE.STATISTICAL_OPERATION.OWNERSHIP_TYPE.HEALTH_FUNCTION.CARE_TYPE.MEDICAL_TECH.HEALTH_CARE_PROVIDER — total hospital beds per 1 000 population (all functions, all care types)." | |
| 71 | +- indicator: physicians-per-1000 | |
| 72 | + dataset: OECD.ELS.HD,DSD_HEALTH_EMP_REAC@DF_PHYS | |
| 73 | + code: "*.HSE.10P3HB._Z._Z.PHYS._Z.LP._Z" | |
| 74 | + priority: 2 | |
| 75 | + notes: "REF_AREA.MEASURE.UNIT_MEASURE.AGE.SEX.HEALTH_PROF.WORKER_STATUS.HEALTH_PROF_ACTIVITY_STATUS.PRICE_BASE — practising physicians (LP) per 1 000 population, head counts. DF_NURSE does not exist (404) so nurses stay with WB." | |
| 76 | +# ---------------------------------------------------------------- education (Education at a Glance, adult attainment) | |
| 77 | +- indicator: tertiary-attainment-25-64 | |
| 78 | + dataset: OECD.EDU.IMEP,DSD_EAG_LSO_EA@DF_LSO_NEAC_DISTR_EA | |
| 79 | + code: "*._T.Y25T64.ISCED11A_5T8._T.POP._Z._T._Z.ED_NED.POP._Z.PT_POP_SEX_AGE.OBS._Z.NEAC.A" | |
| 80 | + priority: 1 | |
| 81 | + notes: "REF_AREA.SEX.AGE.ATTAINMENT_LEV.EDUCATION_FIELD.MEASURE.INCOME.BIRTH_PLACE.MIGRATION_AGE.EDU_STATUS.LABOUR_FORCE_STATUS.DURATION_UNEMP.UNIT_MEASURE.STATISTICAL_OPERATION.WORK_TIME_ARNGMNT.QUESTIONNAIRE.FREQ — share of 25–64 year-olds with tertiary education (ISCED 5–8); STATISTICAL_OPERATION=OBS (SE rows are standard errors)." | |
| 82 | +- indicator: tertiary-attainment-25-34 | |
| 83 | + dataset: OECD.EDU.IMEP,DSD_EAG_LSO_EA@DF_LSO_NEAC_DISTR_EA | |
| 84 | + code: "*._T.Y25T34.ISCED11A_5T8._T.POP._Z._T._Z.ED_NED.POP._Z.PT_POP_SEX_AGE.OBS._Z.NEAC.A" | |
| 85 | + priority: 1 | |
| 86 | + notes: Share of 25–34 year-olds with tertiary education (ISCED 5–8). | |
| 87 | +- indicator: upper-secondary-attainment | |
| 88 | + dataset: OECD.EDU.IMEP,DSD_EAG_LSO_EA@DF_LSO_NEAC_DISTR_EA | |
| 89 | + code: "*._T.Y25T64.ISCED11A_0T2._T.POP._Z._T._Z.ED_NED.POP._Z.PT_POP_SEX_AGE.OBS._Z.NEAC.A" | |
| 90 | + priority: 1 | |
| 91 | + transform: "100 - x" | |
| 92 | + notes: "At least upper secondary = 100 − share of 25–64 year-olds with below upper secondary attainment (ISCED 0–2); the dataflow has no ISCED 3–8 aggregate." | |
| 93 | +# ---------------------------------------------------------------- innovation | |
| 94 | +- indicator: rd-expenditure-pct-gdp | |
| 95 | + dataset: OECD.STI.STP,DSD_MSTI@DF_MSTI | |
| 96 | + code: "*.A.G.PT_B1GQ._Z._Z" | |
| 97 | + priority: 2 | |
| 98 | + notes: "REF_AREA.FREQ.MEASURE.UNIT_MEASURE.PRICE_BASE.TRANSFORMATION — MSTI gross domestic expenditure on R&D (GERD), % of GDP." | |
| 99 | +# ---------------------------------------------------------------- economy (Key short-term economic indicators) | |
| 100 | +- indicator: industrial-production-index | |
| 101 | + dataset: OECD.SDD.STES,DSD_KEI@DF_KEI,4.0 | |
| 102 | + code: "*.M.PRVM.IX.BTE.Y._Z" | |
| 103 | + priority: 1 | |
| 104 | + frequency: M | |
| 105 | + notes: "REF_AREA.FREQ.MEASURE.UNIT_MEASURE.ACTIVITY.ADJUSTMENT.TRANSFORMATION — production volume index, industry except construction (BTE), calendar and seasonally adjusted, 2015=100." | |
added
registry/sources/owid.yaml
+8 −0
@@ -0,0 +1,8 @@ | ||
| 1 | +# Our World in Data grapher mappings — connector `owid` (dataset `grapher`, code = grapher slug). | |
| 2 | +# UNHCR refugee statistics (WB SM.POP.REFG is retired — docs/sources-research.md §1). Both slugs verified 2026-09-11: | |
| 3 | +# CSV header `entity,code,year,refugees[,owid_region]`, unit "people", column lastUpdated 2025-07-03, data to 2024. | |
| 4 | +sources: | |
| 5 | +- {indicator: refugee-population, dataset: grapher, code: refugee-population-by-country-or-territory-of-asylum, priority: 1, | |
| 6 | + notes: "UNHCR Refugee Data Finder via OWID: refugees under UNHCR's mandate by country of asylum (end of year)."} | |
| 7 | +- {indicator: refugees-by-origin, dataset: grapher, code: refugee-population-by-country-or-territory-of-origin, priority: 2, | |
| 8 | + notes: "UNHCR via OWID: refugees by country of origin (end of year). WB SM.POP.REFG.OR stays priority 1 if it resolves."} | |
added
registry/sources/who.yaml
+34 −0
@@ -0,0 +1,34 @@ | ||
| 1 | +# WHO Global Health Observatory (GHO) mappings — connector `who` (id = this file name). | |
| 2 | +# Every code below was verified on 2026-09-11 against https://ghoapi.azureedge.net/api/Indicator and the data endpoint | |
| 3 | +# (see docs/sources-research.md §5). `params` become OData `$filter` clauses (Dim1/Dim2/Dim3 = SEX / AGEGROUP / …); | |
| 4 | +# each combination was checked to yield ONE row per (country, year). Inline WHO mappings for life-expectancy, | |
| 5 | +# healthy-life-expectancy, physicians-per-1000 and obesity-prevalence live in indicators.yaml. | |
| 6 | +# WB carries the same series for most of these (priority 1) → WHO is the fallback at priority 2. | |
| 7 | +sources: | |
| 8 | +- {indicator: smoking-prevalence, dataset: GHO, code: M_Est_tob_curr, params: {Dim1: SEX_BTSX}, priority: 2, | |
| 9 | + notes: "Estimate of current tobacco use prevalence (%), age-standardised, 15+. Includes projections to 2030 (is_forecast)."} | |
| 10 | +- {indicator: suicide-rate, dataset: GHO, code: SDGSUICIDE, params: {Dim1: SEX_BTSX, Dim2: AGEGROUP_YEARSALL}, priority: 2, | |
| 11 | + notes: "Crude suicide rate per 100 000 population, both sexes, all ages."} | |
| 12 | +- {indicator: infant-mortality-rate, dataset: GHO, code: MDG_0000000001, params: {Dim1: SEX_BTSX, Dim2: AGEGROUP_MONTHS0-11}, priority: 2, | |
| 13 | + notes: "UN IGME infant mortality rate per 1 000 live births, both sexes."} | |
| 14 | +- {indicator: under-5-mortality-rate, dataset: GHO, code: MDG_0000000007, | |
| 15 | + params: {Dim1: SEX_BTSX, Dim2: AGEGROUP_YEARSUNDER5, Dim3: WEALTHQUINTILE_TOTL}, priority: 2, | |
| 16 | + notes: "UN IGME under-five mortality rate per 1 000 live births, both sexes, all wealth quintiles."} | |
| 17 | +- {indicator: maternal-mortality-ratio, dataset: GHO, code: MDG_0000000026, priority: 2, | |
| 18 | + notes: "MMEIG maternal mortality ratio per 100 000 live births."} | |
| 19 | +- {indicator: health-expenditure-pct-gdp, dataset: GHO, code: GHED_CHEGDP_SHA2011, priority: 2, | |
| 20 | + notes: "Current health expenditure (CHE) as % of GDP, Global Health Expenditure Database (SHA 2011)."} | |
| 21 | +- {indicator: safely-managed-water, dataset: GHO, code: WSH_WATER_SAFELY_MANAGED, params: {Dim1: RESIDENCEAREATYPE_TOTL}, priority: 2, | |
| 22 | + notes: "JMP: population using safely managed drinking-water services (%), total (urban + rural)."} | |
| 23 | +- {indicator: alcohol-consumption, dataset: GHO, code: SA_0000001688, params: {Dim1: SEX_BTSX}, priority: 2, | |
| 24 | + notes: "Total alcohol per capita (15+) consumption in litres of pure alcohol, three-year average (SDG 3.5.2)."} | |
| 25 | +- {indicator: measles-immunization, dataset: GHO, code: WHS8_110, priority: 2, | |
| 26 | + notes: "MCV1 coverage among 1-year-olds (%), WUENIC. (WHS4_100 is DTP3, not measles.)"} | |
| 27 | +- {indicator: dtp3-immunization, dataset: GHO, code: WHS4_100, priority: 2, | |
| 28 | + notes: "DTP3 coverage among 1-year-olds (%), WUENIC."} | |
| 29 | +- {indicator: nurses-per-1000, dataset: GHO, code: HWF_0006, priority: 2, transform: "x/10", | |
| 30 | + notes: "Nursing and midwifery personnel per 10 000 population → per 1 000 (x/10)."} | |
| 31 | +- {indicator: hospital-beds-per-1000, dataset: GHO, code: WHS6_102, priority: 2, transform: "x/10", | |
| 32 | + notes: "Hospital beds per 10 000 population → per 1 000 (x/10)."} | |
| 33 | +- {indicator: road-traffic-deaths, dataset: GHO, code: RS_198, priority: 2, | |
| 34 | + notes: "Estimated road traffic death rate per 100 000 population (Global status report; one year per edition)."} | |
modified
registry/topics.yaml
+8 −9
@@ -59,7 +59,7 @@ topics: | ||
| 59 | 59 | order: 5 |
| 60 | 60 | blurb: Living standards, poverty and distribution. |
| 61 | 61 | indicators: [gni-per-capita, gni-per-capita-ppp, gini-index, income-share-top-10, income-share-bottom-20, poverty-headcount-215, |
| 62 | − poverty-headcount-national, median-household-income, at-risk-of-poverty-rate, palma-ratio, household-consumption-per-capita] | |
| 62 | + poverty-headcount-national, median-household-income, at-risk-of-poverty-rate, household-consumption-per-capita] | |
| 63 | 63 | - id: housing |
| 64 | 64 | name: Housing |
| 65 | 65 | short: Housing |
@@ -119,7 +119,7 @@ topics: | ||
| 119 | 119 | order: 12 |
| 120 | 120 | blurb: Forests, air quality, land, water and protected areas. |
| 121 | 121 | indicators: [forest-area-share, forest-area, pm25-exposure, protected-areas-share, agricultural-land-share, arable-land-share, |
| 122 | − freshwater-withdrawal-share, renewable-freshwater-per-capita, threatened-species, deforestation-rate, environmental-performance, | |
| 122 | + freshwater-withdrawal-share, renewable-freshwater-per-capita, | |
| 123 | 123 | safely-managed-water, safely-managed-sanitation] |
| 124 | 124 | - id: infrastructure |
| 125 | 125 | name: Infrastructure & transportation |
@@ -127,22 +127,21 @@ topics: | ||
| 127 | 127 | order: 13 |
| 128 | 128 | blurb: Transport networks, passengers, freight and access. |
| 129 | 129 | indicators: [air-passengers, air-freight, rail-lines, rail-passengers, rail-freight, container-port-traffic, logistics-performance-index, |
| 130 | − access-to-electricity, road-traffic-deaths, vehicles-per-1000, paved-roads-share, liner-shipping-connectivity] | |
| 130 | + access-to-electricity, road-traffic-deaths, liner-shipping-connectivity] | |
| 131 | 131 | - id: digital |
| 132 | 132 | name: Digital economy & Internet |
| 133 | 133 | short: Digital |
| 134 | 134 | order: 14 |
| 135 | 135 | blurb: Connectivity, adoption and digital infrastructure. |
| 136 | − indicators: [internet-users, fixed-broadband-subscriptions, mobile-subscriptions, mobile-broadband-subscriptions, secure-internet-servers, | |
| 137 | − ict-goods-exports-share, ict-service-exports-share, households-with-internet, fixed-broadband-speed, mobile-broadband-speed] | |
| 136 | + indicators: [internet-users, fixed-broadband-subscriptions, mobile-subscriptions, secure-internet-servers, | |
| 137 | + ict-goods-exports-share, ict-service-exports-share] | |
| 138 | 138 | - id: innovation |
| 139 | 139 | name: Innovation, research & technology |
| 140 | 140 | short: Innovation |
| 141 | 141 | order: 15 |
| 142 | 142 | blurb: R&D, patents, publications, researchers and high-tech. |
| 143 | 143 | indicators: [rd-expenditure-pct-gdp, researchers-per-million, patent-applications-residents, patent-applications-nonresidents, |
| 144 | − scientific-articles, high-tech-exports-share, high-tech-exports, ict-service-exports-share, trademark-applications, | |
| 145 | − rd-expenditure, business-rd-share] | |
| 144 | + scientific-articles, high-tech-exports-share, high-tech-exports, ict-service-exports-share, trademark-applications] | |
| 146 | 145 | - id: agriculture |
| 147 | 146 | name: Agriculture & natural resources |
| 148 | 147 | short: Agriculture |
@@ -157,7 +156,7 @@ topics: | ||
| 157 | 156 | short: Tourism |
| 158 | 157 | order: 17 |
| 159 | 158 | blurb: Arrivals, departures and tourism receipts. |
| 160 | − indicators: [tourist-arrivals, tourist-departures, tourism-receipts, tourism-receipts-pct-exports, tourism-expenditures, tourism-receipts-per-arrival] | |
| 159 | + indicators: [tourist-arrivals, tourist-departures, tourism-receipts, tourism-receipts-pct-exports, tourism-expenditures] | |
| 161 | 160 | - id: security |
| 162 | 161 | name: Security & governance |
| 163 | 162 | short: Security |
@@ -173,4 +172,4 @@ topics: | ||
| 173 | 172 | blurb: Human development, happiness, safety, access and environment. |
| 174 | 173 | indicators: [human-development-index, life-satisfaction, life-expectancy, healthy-life-expectancy, gini-index, pm25-exposure, |
| 175 | 174 | homicide-rate, safely-managed-water, access-to-electricity, internet-users, expected-years-of-schooling, unemployment-rate, |
| 176 | − housing-cost-overburden-rate, gender-inequality-index, women-in-parliament-share] | |
| 175 | + housing-cost-overburden-rate, women-in-parliament-share] | |
added
scripts/run_connector.py
+81 −0
@@ -0,0 +1,81 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Run one or more connectors end to end (fetch → store_raw → normalize → validate → staging parquet) and print a | |
| 3 | +per-spec summary: rows, countries, period range, forecast rows, status/message. | |
| 4 | + | |
| 5 | + CA_DATA_DIR=~/countryatlas-data .venv/bin/python scripts/run_connector.py who fred bis ilo [--indicator SLUG] | |
| 6 | + [--mode fetch|normalize] [--concurrency N] | |
| 7 | + | |
| 8 | +Uses the pipeline's `run_fetch` (same code path as `ca fetch`), so the staging files land in | |
| 9 | +`staging/<connector>/<dataset>__<code>__<indicator>.parquet` with the .run.json / .meta.json / .issues.json sidecars. | |
| 10 | +FRED calls are serialised by the connector itself (≥ 1.1 s spacing), whatever the concurrency. | |
| 11 | +""" | |
| 12 | +from __future__ import annotations | |
| 13 | + | |
| 14 | +import argparse | |
| 15 | +import logging | |
| 16 | +import sys | |
| 17 | +from pathlib import Path | |
| 18 | + | |
| 19 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 20 | + | |
| 21 | +import polars as pl | |
| 22 | + | |
| 23 | +from countryatlas.config import settings | |
| 24 | +from countryatlas.pipeline.fetch import run_fetch | |
| 25 | +from countryatlas.pipeline.staging import spec_paths | |
| 26 | +from countryatlas.registry import source_specs | |
| 27 | + | |
| 28 | + | |
| 29 | +def summarize(connector: str, indicator: str | None) -> list[str]: | |
| 30 | + lines = [] | |
| 31 | + for spec in source_specs(connector=connector, indicator=indicator): | |
| 32 | + p = spec_paths(spec)["parquet"] | |
| 33 | + label = f"{spec.dataset}:{spec.code} → {spec.indicator_id}" | |
| 34 | + if not p.exists(): | |
| 35 | + lines.append(f" ✗ {label}: no staging file") | |
| 36 | + continue | |
| 37 | + df = pl.read_parquet(p) | |
| 38 | + if df.is_empty(): | |
| 39 | + lines.append(f" ✗ {label}: empty") | |
| 40 | + continue | |
| 41 | + n_c = df["country_id"].n_unique() | |
| 42 | + pmin, pmax = df["period"].min(), df["period"].max() | |
| 43 | + n_fc = int(df["is_forecast"].sum()) | |
| 44 | + n_q = int((df["status"] == "quarantined").sum()) | |
| 45 | + n_w = int((df["status"] == "warning").sum()) | |
| 46 | + freq = ",".join(sorted(df["frequency"].unique().to_list())) | |
| 47 | + lines.append( | |
| 48 | + f" ✓ {label}: {df.height} rows, {n_c} countries, {pmin}…{pmax} [{freq}]" | |
| 49 | + + (f", {n_fc} forecast" if n_fc else "") | |
| 50 | + + (f", {n_q} quarantined" if n_q else "") | |
| 51 | + + (f", {n_w} warnings" if n_w else "") | |
| 52 | + ) | |
| 53 | + return lines | |
| 54 | + | |
| 55 | + | |
| 56 | +def main() -> int: | |
| 57 | + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| 58 | + ap.add_argument("connectors", nargs="+", help="connector ids (who fred bis ilo …)") | |
| 59 | + ap.add_argument("--indicator", default=None) | |
| 60 | + ap.add_argument("--mode", choices=["fetch", "normalize"], default="fetch") | |
| 61 | + ap.add_argument("--concurrency", type=int, default=None) | |
| 62 | + ap.add_argument("-v", "--verbose", action="store_true") | |
| 63 | + args = ap.parse_args() | |
| 64 | + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, | |
| 65 | + format="%(asctime)s %(levelname)s %(name)s: %(message)s") | |
| 66 | + logging.getLogger("httpx").setLevel(logging.WARNING) | |
| 67 | + print(f"data dir: {settings.data_dir}") | |
| 68 | + summary = run_fetch(connectors=args.connectors, indicator=args.indicator, mode=args.mode, concurrency=args.concurrency) | |
| 69 | + print(f"\nrun {summary.run_id}: {len(summary.ok)} ok / {len(summary.runs)} specs in {summary.duration_s:.0f}s") | |
| 70 | + for cid in args.connectors: | |
| 71 | + print(f"\n[{cid}]") | |
| 72 | + for line in summarize(cid, args.indicator): | |
| 73 | + print(line) | |
| 74 | + for r in summary.runs: | |
| 75 | + if r.connector == cid and r.status not in ("ok",): | |
| 76 | + print(f" ! {r.status.upper()} {r.dataset}: {r.message}") | |
| 77 | + return 0 if not summary.failed else 1 | |
| 78 | + | |
| 79 | + | |
| 80 | +if __name__ == "__main__": | |
| 81 | + raise SystemExit(main()) | |
added
src/countryatlas/api/cache.py
+61 −0
@@ -0,0 +1,61 @@ | ||
| 1 | +"""Small in-process LRU + TTL cache for rendered responses. | |
| 2 | + | |
| 3 | +Keys include the snapshot `run_id`, so everything is naturally invalidated when a new snapshot is swapped in. | |
| 4 | +""" | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import threading | |
| 8 | +import time | |
| 9 | +from collections import OrderedDict | |
| 10 | +from collections.abc import Hashable | |
| 11 | +from typing import Any | |
| 12 | + | |
| 13 | + | |
| 14 | +class ResponseCache: | |
| 15 | + def __init__(self, maxsize: int = 2048, ttl: float = 3600.0) -> None: | |
| 16 | + self.maxsize = maxsize | |
| 17 | + self.ttl = ttl | |
| 18 | + self._data: OrderedDict[Hashable, tuple[float, Any]] = OrderedDict() | |
| 19 | + self._lock = threading.Lock() | |
| 20 | + self.hits = 0 | |
| 21 | + self.misses = 0 | |
| 22 | + | |
| 23 | + @staticmethod | |
| 24 | + def key(run_id: str, path: str, query: str | dict[str, Any] | None = None) -> tuple: | |
| 25 | + if isinstance(query, dict): | |
| 26 | + query = "&".join(f"{k}={v}" for k, v in sorted(query.items())) | |
| 27 | + return (run_id, path, query or "") | |
| 28 | + | |
| 29 | + def get(self, key: Hashable) -> Any | None: | |
| 30 | + now = time.monotonic() | |
| 31 | + with self._lock: | |
| 32 | + item = self._data.get(key) | |
| 33 | + if item is None: | |
| 34 | + self.misses += 1 | |
| 35 | + return None | |
| 36 | + expires, value = item | |
| 37 | + if expires < now: | |
| 38 | + del self._data[key] | |
| 39 | + self.misses += 1 | |
| 40 | + return None | |
| 41 | + self._data.move_to_end(key) | |
| 42 | + self.hits += 1 | |
| 43 | + return value | |
| 44 | + | |
| 45 | + def set(self, key: Hashable, value: Any, ttl: float | None = None) -> None: | |
| 46 | + with self._lock: | |
| 47 | + self._data[key] = (time.monotonic() + (ttl or self.ttl), value) | |
| 48 | + self._data.move_to_end(key) | |
| 49 | + while len(self._data) > self.maxsize: | |
| 50 | + self._data.popitem(last=False) | |
| 51 | + | |
| 52 | + def clear(self) -> None: | |
| 53 | + with self._lock: | |
| 54 | + self._data.clear() | |
| 55 | + | |
| 56 | + def stats(self) -> dict[str, int]: | |
| 57 | + with self._lock: | |
| 58 | + return {"size": len(self._data), "hits": self.hits, "misses": self.misses, "maxsize": self.maxsize} | |
| 59 | + | |
| 60 | + | |
| 61 | +response_cache = ResponseCache() | |
added
src/countryatlas/api/common.py
+380 −0
@@ -0,0 +1,380 @@ | ||
| 1 | +"""Shared helpers for routers: entity resolution, metric objects, sparklines, meta block.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import math | |
| 5 | +from collections.abc import Iterable | |
| 6 | +from datetime import UTC, datetime | |
| 7 | +from typing import Any | |
| 8 | + | |
| 9 | +from countryatlas.api.db import Snapshot | |
| 10 | +from countryatlas.api.errors import bad_request, not_found | |
| 11 | +from countryatlas.api.formatting import format_change, format_value | |
| 12 | +from countryatlas.api.provenance import build_provenance | |
| 13 | +from countryatlas.registry import indicators_by_id as registry_indicators | |
| 14 | +from countryatlas.registry import topics as registry_topics | |
| 15 | + | |
| 16 | +# --------------------------------------------------------------------------------------------- meta | |
| 17 | + | |
| 18 | + | |
| 19 | +def meta_block(snap: Snapshot) -> dict[str, Any]: | |
| 20 | + return { | |
| 21 | + "built_at": snap.built_at, | |
| 22 | + "run_id": snap.run_id, | |
| 23 | + "generated_at": datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z"), | |
| 24 | + } | |
| 25 | + | |
| 26 | + | |
| 27 | +def clean_float(v: Any) -> float | None: | |
| 28 | + if v is None: | |
| 29 | + return None | |
| 30 | + try: | |
| 31 | + f = float(v) | |
| 32 | + except (TypeError, ValueError): | |
| 33 | + return None | |
| 34 | + if math.isnan(f) or math.isinf(f): | |
| 35 | + return None | |
| 36 | + return f | |
| 37 | + | |
| 38 | + | |
| 39 | +def parse_csv(value: str | None, upper: bool = False, limit: int = 50) -> list[str]: | |
| 40 | + if not value: | |
| 41 | + return [] | |
| 42 | + items = [x.strip() for x in value.split(",") if x.strip()] | |
| 43 | + if upper: | |
| 44 | + items = [x.upper() for x in items] | |
| 45 | + if len(items) > limit: | |
| 46 | + raise bad_request(f"Too many items ({len(items)} > {limit}).") | |
| 47 | + return list(dict.fromkeys(items)) | |
| 48 | + | |
| 49 | + | |
| 50 | +# --------------------------------------------------------------------------------------------- resolution | |
| 51 | + | |
| 52 | + | |
| 53 | +def resolve_country(snap: Snapshot, ident: str) -> dict[str, Any]: | |
| 54 | + """ISO3 or slug, case-insensitive. 404 otherwise.""" | |
| 55 | + key = (ident or "").strip() | |
| 56 | + c = snap.countries().get(key.upper()) or snap.countries_by_slug().get(key.lower()) | |
| 57 | + if c is None and len(key) == 2: | |
| 58 | + for row in snap.countries().values(): | |
| 59 | + if (row.get("iso2") or "").upper() == key.upper(): | |
| 60 | + c = row | |
| 61 | + break | |
| 62 | + if c is None: | |
| 63 | + raise not_found("country", ident, "Use an ISO3 code (e.g. CAN) or a slug (e.g. canada); see /api/v1/countries.") | |
| 64 | + return c | |
| 65 | + | |
| 66 | + | |
| 67 | +def resolve_indicator(snap: Snapshot, slug: str) -> dict[str, Any]: | |
| 68 | + key = (slug or "").strip().lower() | |
| 69 | + ind = snap.indicators_by_slug().get(key) or snap.indicators().get(key) | |
| 70 | + if ind is None: | |
| 71 | + raise not_found("indicator", slug, "See /api/v1/indicators for the list of slugs (e.g. gdp-per-capita).") | |
| 72 | + return merged_indicator(ind) | |
| 73 | + | |
| 74 | + | |
| 75 | +def merged_indicator(ind: dict[str, Any]) -> dict[str, Any]: | |
| 76 | + """DB indicator row with registry metadata filling gaps (format, precision, higher_is_better, ...).""" | |
| 77 | + reg = registry_indicators().get(ind["id"]) | |
| 78 | + if reg is None: | |
| 79 | + return ind | |
| 80 | + out = dict(ind) | |
| 81 | + for k in ("name", "short_name", "description", "topic", "subtopic", "unit", "unit_short", "frequency", "precision", | |
| 82 | + "aggregation", "higher_is_better", "ranking_eligible", "featured", "format", "scale", "methodology", "tags", | |
| 83 | + "per_capita_of"): | |
| 84 | + if out.get(k) is None or out.get(k) == "": | |
| 85 | + out[k] = getattr(reg, k, None) | |
| 86 | + if out.get("bounds_min") is None and reg.bounds: | |
| 87 | + out["bounds_min"] = reg.bounds[0] | |
| 88 | + if out.get("bounds_max") is None and reg.bounds and len(reg.bounds) > 1: | |
| 89 | + out["bounds_max"] = reg.bounds[1] | |
| 90 | + out["source_priority"] = [s.connector for s in reg.sources] | |
| 91 | + return out | |
| 92 | + | |
| 93 | + | |
| 94 | +def indicator_meta(snap: Snapshot, slug: str) -> dict[str, Any] | None: | |
| 95 | + """Indicator metadata from the DB row when present, else from the registry (so 'no data' indicators still render).""" | |
| 96 | + row = snap.indicators().get(slug) | |
| 97 | + if row is not None: | |
| 98 | + return merged_indicator(row) | |
| 99 | + reg = registry_indicators().get(slug) | |
| 100 | + if reg is None: | |
| 101 | + return None | |
| 102 | + return { | |
| 103 | + "id": reg.slug, "slug": reg.slug, "name": reg.name, "short_name": reg.short_name, "description": reg.description, "topic": reg.topic, | |
| 104 | + "subtopic": reg.subtopic, "unit": reg.unit, "unit_short": reg.unit_short, "frequency": reg.frequency, "precision": reg.precision, | |
| 105 | + "aggregation": reg.aggregation, "higher_is_better": reg.higher_is_better, "ranking_eligible": reg.ranking_eligible, | |
| 106 | + "featured": reg.featured, "format": reg.format, "scale": reg.scale, "bounds_min": reg.bounds[0] if reg.bounds else None, | |
| 107 | + "bounds_max": reg.bounds[1] if reg.bounds and len(reg.bounds) > 1 else None, "methodology": reg.methodology, "tags": list(reg.tags), | |
| 108 | + "per_capita_of": reg.per_capita_of, "source_priority": [s.connector for s in reg.sources], "n_countries": 0, "n_observations": 0, | |
| 109 | + "first_year": None, "last_year": None, "latest_source_updated_at": None, "primary_source_id": None, "in_snapshot": False, | |
| 110 | + } | |
| 111 | + | |
| 112 | + | |
| 113 | +def resolve_group(snap: Snapshot, ident: str) -> dict[str, Any]: | |
| 114 | + key = (ident or "").strip().lower() | |
| 115 | + g = snap.groups().get(key) or snap.groups_by_slug().get(key) | |
| 116 | + if g is None: | |
| 117 | + raise not_found("region", ident, "Use a group id or slug (world, oecd, g7, europe-central-asia…); see /api/v1/regions.") | |
| 118 | + return g | |
| 119 | + | |
| 120 | + | |
| 121 | +def resolve_topic(topic: str) -> dict[str, Any]: | |
| 122 | + for t in registry_topics()["topics"]: | |
| 123 | + if t["id"] == (topic or "").strip().lower(): | |
| 124 | + return t | |
| 125 | + raise not_found("topic", topic, "Topics: " + ", ".join(t["id"] for t in registry_topics()["topics"]) + ".") | |
| 126 | + | |
| 127 | + | |
| 128 | +def country_card(c: dict[str, Any]) -> dict[str, Any]: | |
| 129 | + return { | |
| 130 | + "id": c["id"], | |
| 131 | + "iso2": c.get("iso2"), | |
| 132 | + "slug": c.get("slug"), | |
| 133 | + "name": c.get("short_name"), | |
| 134 | + "flag": c.get("flag_emoji"), | |
| 135 | + "region": c.get("region_wb"), | |
| 136 | + "region_name": c.get("region_wb_name"), | |
| 137 | + "income": c.get("income_group"), | |
| 138 | + "income_name": c.get("income_group_name"), | |
| 139 | + "kind": c.get("kind"), | |
| 140 | + } | |
| 141 | + | |
| 142 | + | |
| 143 | +def indicator_card(ind: dict[str, Any]) -> dict[str, Any]: | |
| 144 | + return { | |
| 145 | + "id": ind["id"], | |
| 146 | + "slug": ind.get("slug") or ind["id"], | |
| 147 | + "name": ind.get("name"), | |
| 148 | + "short_name": ind.get("short_name") or ind.get("name"), | |
| 149 | + "topic": ind.get("topic"), | |
| 150 | + "subtopic": ind.get("subtopic"), | |
| 151 | + "unit": ind.get("unit"), | |
| 152 | + "unit_short": ind.get("unit_short"), | |
| 153 | + "format": ind.get("format"), | |
| 154 | + "precision": ind.get("precision"), | |
| 155 | + "frequency": ind.get("frequency"), | |
| 156 | + "aggregation": ind.get("aggregation"), | |
| 157 | + "higher_is_better": ind.get("higher_is_better"), | |
| 158 | + "ranking_eligible": ind.get("ranking_eligible"), | |
| 159 | + "featured": ind.get("featured"), | |
| 160 | + } | |
| 161 | + | |
| 162 | + | |
| 163 | +def group_card(g: dict[str, Any]) -> dict[str, Any]: | |
| 164 | + return {"id": g["id"], "slug": g.get("slug"), "name": g.get("name"), "kind": g.get("kind"), "wb_code": g.get("wb_code"), | |
| 165 | + "n_members": g.get("n_members")} | |
| 166 | + | |
| 167 | + | |
| 168 | +# --------------------------------------------------------------------------------------------- metrics | |
| 169 | + | |
| 170 | +LATEST_WITH_OBS_SQL = """ | |
| 171 | +SELECT l.country_id, l.indicator_id, l.period, l.year, l.frequency, l.value, l.prev_period, l.prev_value, l.change_abs, | |
| 172 | + l.change_pct, l.rank_world, l.n_world, l.rank_region, l.n_region, l.rank_income, l.n_income, l.rank_year, | |
| 173 | + l.source_id, l.is_forecast, l.is_estimate, l.status, l.value_10y_ago, l.change_10y_abs, l.change_10y_pct, | |
| 174 | + o.unit, o.source_dataset, o.source_series_code, o.retrieved_at, o.source_updated_at | |
| 175 | +FROM latest l | |
| 176 | +LEFT JOIN observations o | |
| 177 | + ON o.country_id = l.country_id AND o.indicator_id = l.indicator_id AND o.period = l.period AND o.frequency = l.frequency | |
| 178 | +""" | |
| 179 | + | |
| 180 | + | |
| 181 | +def latest_rows(snap: Snapshot, country_id: str | None = None, indicator_ids: Iterable[str] | None = None, | |
| 182 | + country_ids: Iterable[str] | None = None) -> list[dict[str, Any]]: | |
| 183 | + where, params = [], [] | |
| 184 | + if country_id: | |
| 185 | + where.append("l.country_id = ?") | |
| 186 | + params.append(country_id) | |
| 187 | + if country_ids is not None: | |
| 188 | + ids = list(country_ids) | |
| 189 | + if not ids: | |
| 190 | + return [] | |
| 191 | + where.append(f"l.country_id IN ({','.join('?' * len(ids))})") | |
| 192 | + params.extend(ids) | |
| 193 | + if indicator_ids is not None: | |
| 194 | + ids = list(indicator_ids) | |
| 195 | + if not ids: | |
| 196 | + return [] | |
| 197 | + where.append(f"l.indicator_id IN ({','.join('?' * len(ids))})") | |
| 198 | + params.extend(ids) | |
| 199 | + sql = LATEST_WITH_OBS_SQL + (" WHERE " + " AND ".join(where) if where else "") | |
| 200 | + return snap.query(sql, params) | |
| 201 | + | |
| 202 | + | |
| 203 | +def sparklines(snap: Snapshot, country_id: str, indicator_ids: Iterable[str], n: int = 30) -> dict[str, list[list[float]]]: | |
| 204 | + """indicator_id → [[year, value], …] last `n` non-forecast points at the indicator's `latest` frequency.""" | |
| 205 | + ids = list(indicator_ids) | |
| 206 | + if not ids: | |
| 207 | + return {} | |
| 208 | + ph = ",".join("?" * len(ids)) | |
| 209 | + rows = snap.query_rows( | |
| 210 | + f""" | |
| 211 | + WITH lf AS (SELECT indicator_id, frequency FROM latest WHERE country_id = ? AND indicator_id IN ({ph})) | |
| 212 | + SELECT indicator_id, year, period, value FROM ( | |
| 213 | + SELECT o.indicator_id, o.year, o.period, o.value, | |
| 214 | + row_number() OVER (PARTITION BY o.indicator_id ORDER BY o.period DESC) AS rn | |
| 215 | + FROM observations o JOIN lf ON lf.indicator_id = o.indicator_id AND lf.frequency = o.frequency | |
| 216 | + WHERE o.country_id = ? AND o.indicator_id IN ({ph}) AND NOT o.is_forecast AND o.value IS NOT NULL | |
| 217 | + ) WHERE rn <= ? ORDER BY indicator_id, period | |
| 218 | + """, | |
| 219 | + [country_id, *ids, country_id, *ids, n], | |
| 220 | + ) | |
| 221 | + out: dict[str, list[list[float]]] = {} | |
| 222 | + for iid, year, _period, value in rows: | |
| 223 | + out.setdefault(iid, []).append([int(year), clean_float(value)]) | |
| 224 | + return out | |
| 225 | + | |
| 226 | + | |
| 227 | +def sparklines_for_countries(snap: Snapshot, indicator_id: str, country_ids: Iterable[str], n: int = 30, | |
| 228 | + max_year: int | None = None) -> dict[str, list[list[float]]]: | |
| 229 | + ids = list(country_ids) | |
| 230 | + if not ids: | |
| 231 | + return {} | |
| 232 | + ph = ",".join("?" * len(ids)) | |
| 233 | + params: list[Any] = [indicator_id, *ids] | |
| 234 | + extra = "" | |
| 235 | + if max_year is not None: | |
| 236 | + extra = " AND year <= ?" | |
| 237 | + params.append(max_year) | |
| 238 | + params.append(n) | |
| 239 | + rows = snap.query_rows( | |
| 240 | + f""" | |
| 241 | + SELECT country_id, year, value FROM ( | |
| 242 | + SELECT country_id, year, value, row_number() OVER (PARTITION BY country_id ORDER BY period DESC) AS rn | |
| 243 | + FROM observations WHERE indicator_id = ? AND country_id IN ({ph}) AND frequency = 'A' AND NOT is_forecast | |
| 244 | + AND value IS NOT NULL{extra} | |
| 245 | + ) WHERE rn <= ? ORDER BY country_id, year | |
| 246 | + """, | |
| 247 | + params, | |
| 248 | + ) | |
| 249 | + out: dict[str, list[list[float]]] = {} | |
| 250 | + for cid, year, value in rows: | |
| 251 | + out.setdefault(cid, []).append([int(year), clean_float(value)]) | |
| 252 | + return out | |
| 253 | + | |
| 254 | + | |
| 255 | +def metric_from_latest(snap: Snapshot, row: dict[str, Any], ind: dict[str, Any], country: dict[str, Any] | None = None, | |
| 256 | + sparkline: list[list[float]] | None = None) -> dict[str, Any]: | |
| 257 | + """MetricValue object with change, ranks, formatted strings and provenance.""" | |
| 258 | + value = clean_float(row.get("value")) | |
| 259 | + max_year = snap.max_years().get(ind["id"]) | |
| 260 | + rank_year = row.get("rank_year") or row.get("year") | |
| 261 | + out: dict[str, Any] = { | |
| 262 | + "indicator": ind["id"], | |
| 263 | + "indicator_name": ind.get("short_name") or ind.get("name"), | |
| 264 | + "has_data": value is not None, | |
| 265 | + "value": value, | |
| 266 | + "formatted": format_value(value, ind), | |
| 267 | + "period": row.get("period"), | |
| 268 | + "year": row.get("year"), | |
| 269 | + "frequency": row.get("frequency"), | |
| 270 | + "unit": row.get("unit") or ind.get("unit"), | |
| 271 | + "unit_short": ind.get("unit_short"), | |
| 272 | + "format": ind.get("format"), | |
| 273 | + "is_estimate": bool(row.get("is_estimate")) if row.get("is_estimate") is not None else False, | |
| 274 | + "is_forecast": bool(row.get("is_forecast")) if row.get("is_forecast") is not None else False, | |
| 275 | + "status": row.get("status"), | |
| 276 | + "prev": {"period": row.get("prev_period"), "value": clean_float(row.get("prev_value"))} if row.get("prev_value") is not None else None, | |
| 277 | + "change": { | |
| 278 | + "abs": clean_float(row.get("change_abs")), | |
| 279 | + "pct": clean_float(row.get("change_pct")), | |
| 280 | + "formatted": format_change(clean_float(row.get("change_abs")), clean_float(row.get("change_pct")), ind), | |
| 281 | + } if row.get("change_abs") is not None or row.get("change_pct") is not None else None, | |
| 282 | + "change_10y": { | |
| 283 | + "abs": clean_float(row.get("change_10y_abs")), | |
| 284 | + "pct": clean_float(row.get("change_10y_pct")), | |
| 285 | + "value_10y_ago": clean_float(row.get("value_10y_ago")), | |
| 286 | + "formatted": format_change(clean_float(row.get("change_10y_abs")), clean_float(row.get("change_10y_pct")), ind), | |
| 287 | + } if row.get("change_10y_abs") is not None or row.get("change_10y_pct") is not None else None, | |
| 288 | + "rank_world": row.get("rank_world"), | |
| 289 | + "n_world": row.get("n_world"), | |
| 290 | + "rank_region": row.get("rank_region"), | |
| 291 | + "n_region": row.get("n_region"), | |
| 292 | + "rank_income": row.get("rank_income"), | |
| 293 | + "n_income": row.get("n_income"), | |
| 294 | + "rank_year": rank_year, | |
| 295 | + "rank_is_stale": bool(max_year is not None and rank_year is not None and int(rank_year) < int(max_year) - 2), | |
| 296 | + "higher_is_better": ind.get("higher_is_better"), | |
| 297 | + "sparkline": sparkline or [], | |
| 298 | + "provenance": build_provenance( | |
| 299 | + snap, ind["id"], row.get("source_id"), row.get("source_dataset"), row.get("source_series_code"), | |
| 300 | + row.get("retrieved_at"), row.get("source_updated_at"), (country or {}).get("iso2"), | |
| 301 | + ), | |
| 302 | + } | |
| 303 | + return out | |
| 304 | + | |
| 305 | + | |
| 306 | +def empty_metric(ind: dict[str, Any]) -> dict[str, Any]: | |
| 307 | + return {"indicator": ind["id"], "indicator_name": ind.get("short_name") or ind.get("name"), "has_data": False, | |
| 308 | + "value": None, "formatted": "—", "unit": ind.get("unit"), "unit_short": ind.get("unit_short"), | |
| 309 | + "format": ind.get("format"), "higher_is_better": ind.get("higher_is_better"), "sparkline": [], "provenance": None} | |
| 310 | + | |
| 311 | + | |
| 312 | +def observation_value(snap: Snapshot, row: dict[str, Any], ind: dict[str, Any], iso2: str | None = None) -> dict[str, Any]: | |
| 313 | + """Value object for one observation row (series / downloads).""" | |
| 314 | + return { | |
| 315 | + "period": row.get("period"), | |
| 316 | + "year": row.get("year"), | |
| 317 | + "frequency": row.get("frequency"), | |
| 318 | + "value": clean_float(row.get("value")), | |
| 319 | + "is_forecast": bool(row.get("is_forecast")), | |
| 320 | + "is_estimate": bool(row.get("is_estimate")), | |
| 321 | + "status": row.get("status"), | |
| 322 | + "source_id": row.get("source_id"), | |
| 323 | + "provenance": build_provenance(snap, ind["id"], row.get("source_id"), row.get("source_dataset"), | |
| 324 | + row.get("source_series_code"), row.get("retrieved_at"), row.get("source_updated_at"), iso2), | |
| 325 | + } | |
| 326 | + | |
| 327 | + | |
| 328 | +def series_stats(values: list[dict[str, Any]]) -> dict[str, Any]: | |
| 329 | + pts = [(v["year"], v["value"]) for v in values if v.get("value") is not None and not v.get("is_forecast")] | |
| 330 | + if not pts: | |
| 331 | + return {"min": None, "max": None, "first": None, "last": None, "cagr": None, "n": 0} | |
| 332 | + vals = [p[1] for p in pts] | |
| 333 | + first_year, first = pts[0] | |
| 334 | + last_year, last = pts[-1] | |
| 335 | + cagr = None | |
| 336 | + if first and last and first > 0 and last > 0 and last_year > first_year: | |
| 337 | + cagr = (last / first) ** (1.0 / (last_year - first_year)) - 1.0 | |
| 338 | + return { | |
| 339 | + "min": {"year": pts[vals.index(min(vals))][0], "value": min(vals)}, | |
| 340 | + "max": {"year": pts[vals.index(max(vals))][0], "value": max(vals)}, | |
| 341 | + "first": {"year": first_year, "value": first}, | |
| 342 | + "last": {"year": last_year, "value": last}, | |
| 343 | + "cagr": clean_float(cagr * 100) if cagr is not None else None, | |
| 344 | + "n": len(pts), | |
| 345 | + } | |
| 346 | + | |
| 347 | + | |
| 348 | +def quantile_breaks(values: list[float], k: int = 6) -> list[float]: | |
| 349 | + """k-1 interior quantile breaks (5–7 classes) over the sorted values.""" | |
| 350 | + vals = sorted(v for v in values if v is not None) | |
| 351 | + if len(vals) < 2: | |
| 352 | + return [] | |
| 353 | + n = len(vals) | |
| 354 | + k = max(5, min(7, k)) if n >= 40 else max(3, min(5, n)) | |
| 355 | + breaks: list[float] = [] | |
| 356 | + for i in range(1, k): | |
| 357 | + q = i / k | |
| 358 | + pos = q * (n - 1) | |
| 359 | + lo = math.floor(pos) | |
| 360 | + hi = min(lo + 1, n - 1) | |
| 361 | + val = vals[lo] + (vals[hi] - vals[lo]) * (pos - lo) | |
| 362 | + breaks.append(val) | |
| 363 | + # dedupe while preserving order (many identical values → fewer classes) | |
| 364 | + out: list[float] = [] | |
| 365 | + for b in breaks: | |
| 366 | + if not out or b > out[-1]: | |
| 367 | + out.append(b) | |
| 368 | + return out | |
| 369 | + | |
| 370 | + | |
| 371 | +def is_per_capita_or_share(ind: dict[str, Any]) -> bool: | |
| 372 | + fmt = (ind.get("format") or "").lower() | |
| 373 | + slug = ind.get("slug") or ind["id"] | |
| 374 | + unit = (ind.get("unit") or "").lower() | |
| 375 | + return ( | |
| 376 | + fmt in ("percent", "per_1000", "per_100k", "per_million", "years", "ratio", "index", "celsius") | |
| 377 | + or "per-capita" in slug or "per-person" in slug or "per-hour" in slug or "share" in slug or "rate" in slug | |
| 378 | + or "per capita" in unit or "%" in unit or "per person" in unit | |
| 379 | + or ind.get("aggregation") == "weighted_mean" | |
| 380 | + ) | |
added
src/countryatlas/api/db.py
+250 −0
@@ -0,0 +1,250 @@ | ||
| 1 | +"""Read-only DuckDB access with snapshot-swap detection (docs/ARCHITECTURE.md §2). | |
| 2 | + | |
| 3 | +* The API never writes. It opens `atlas.duckdb` with `read_only=True`. | |
| 4 | +* Before every request `Database.current()` compares `os.stat()` (st_ino, st_mtime_ns) with the open snapshot and | |
| 5 | + reopens when the pipeline has swapped a new file in (atomic `os.replace`). | |
| 6 | +* DuckDB connections are not thread-safe: every query runs under the snapshot's lock (the API runs with one worker; | |
| 7 | + queries are millisecond-scale so a lock is the simplest correct option). | |
| 8 | +* If the file does not exist the API still starts: `current()` returns None and data endpoints raise `DataNotBuilt` | |
| 9 | + (503 problem+json), `/health` reports `status: "empty"`. | |
| 10 | +""" | |
| 11 | +from __future__ import annotations | |
| 12 | + | |
| 13 | +import logging | |
| 14 | +import os | |
| 15 | +import threading | |
| 16 | +from pathlib import Path | |
| 17 | +from typing import Any | |
| 18 | + | |
| 19 | +import duckdb | |
| 20 | + | |
| 21 | +from countryatlas.config import settings | |
| 22 | + | |
| 23 | +log = logging.getLogger("countryatlas.api.db") | |
| 24 | + | |
| 25 | +SOURCE_URL_FALLBACKS = { | |
| 26 | + "worldbank": "https://data.worldbank.org/", | |
| 27 | + "owid": "https://ourworldindata.org/", | |
| 28 | + "imf": "https://data.imf.org/", | |
| 29 | + "oecd": "https://data-explorer.oecd.org/", | |
| 30 | + "eurostat": "https://ec.europa.eu/eurostat/databrowser/", | |
| 31 | + "who": "https://www.who.int/data/gho", | |
| 32 | + "fred": "https://fred.stlouisfed.org/", | |
| 33 | + "bis": "https://data.bis.org/", | |
| 34 | + "ilo": "https://ilostat.ilo.org/", | |
| 35 | +} | |
| 36 | + | |
| 37 | + | |
| 38 | +class DataNotBuilt(Exception): | |
| 39 | + """Raised by data endpoints when no snapshot exists yet.""" | |
| 40 | + | |
| 41 | + | |
| 42 | +class Snapshot: | |
| 43 | + """One open read-only connection to one physical DuckDB file, plus lazily loaded static lookups.""" | |
| 44 | + | |
| 45 | + def __init__(self, path: Path, stat: os.stat_result) -> None: | |
| 46 | + self.path = path | |
| 47 | + self.ino = stat.st_ino | |
| 48 | + self.mtime_ns = stat.st_mtime_ns | |
| 49 | + self.conn = duckdb.connect(str(path), read_only=True) | |
| 50 | + self._lock = threading.RLock() | |
| 51 | + self._static: dict[str, Any] = {} | |
| 52 | + self.meta: dict[str, str] = {} | |
| 53 | + try: | |
| 54 | + self.meta = {k: v for k, v in self.query_rows("SELECT key, value FROM meta")} | |
| 55 | + except duckdb.Error as e: # pragma: no cover - defensive | |
| 56 | + log.warning("meta table unreadable: %s", e) | |
| 57 | + self.run_id: str = self.meta.get("build_run_id") or self.meta.get("run_id") or f"mtime-{self.mtime_ns}" | |
| 58 | + self.built_at: str | None = self.meta.get("built_at") | |
| 59 | + | |
| 60 | + # ------------------------------------------------------------------ querying | |
| 61 | + def query_rows(self, sql: str, params: Any = None) -> list[tuple]: | |
| 62 | + with self._lock: | |
| 63 | + cur = self.conn.cursor() | |
| 64 | + try: | |
| 65 | + cur.execute(sql, params or []) | |
| 66 | + return cur.fetchall() | |
| 67 | + finally: | |
| 68 | + cur.close() | |
| 69 | + | |
| 70 | + def query(self, sql: str, params: Any = None) -> list[dict[str, Any]]: | |
| 71 | + with self._lock: | |
| 72 | + cur = self.conn.cursor() | |
| 73 | + try: | |
| 74 | + cur.execute(sql, params or []) | |
| 75 | + cols = [d[0] for d in cur.description or []] | |
| 76 | + return [dict(zip(cols, r)) for r in cur.fetchall()] | |
| 77 | + finally: | |
| 78 | + cur.close() | |
| 79 | + | |
| 80 | + def one(self, sql: str, params: Any = None) -> dict[str, Any] | None: | |
| 81 | + rows = self.query(sql, params) | |
| 82 | + return rows[0] if rows else None | |
| 83 | + | |
| 84 | + def scalar(self, sql: str, params: Any = None) -> Any: | |
| 85 | + rows = self.query_rows(sql, params) | |
| 86 | + return rows[0][0] if rows and rows[0] else None | |
| 87 | + | |
| 88 | + def table_count(self, table: str) -> int: | |
| 89 | + try: | |
| 90 | + return int(self.scalar(f"SELECT count(*) FROM {table}") or 0) | |
| 91 | + except duckdb.Error: | |
| 92 | + return 0 | |
| 93 | + | |
| 94 | + def close(self) -> None: | |
| 95 | + with self._lock: | |
| 96 | + try: | |
| 97 | + self.conn.close() | |
| 98 | + except Exception as e: # noqa: BLE001 | |
| 99 | + log.debug("closing snapshot %s: %s", self.run_id, e) | |
| 100 | + | |
| 101 | + # ------------------------------------------------------------------ static lookups (per snapshot) | |
| 102 | + def _cached(self, key: str, loader): # type: ignore[no-untyped-def] | |
| 103 | + if key not in self._static: | |
| 104 | + with self._lock: | |
| 105 | + if key not in self._static: | |
| 106 | + self._static[key] = loader() | |
| 107 | + return self._static[key] | |
| 108 | + | |
| 109 | + def countries(self) -> dict[str, dict[str, Any]]: | |
| 110 | + """All countries (kind='country' and territories) keyed by id (ISO3).""" | |
| 111 | + return self._cached("countries", lambda: {r["id"]: r for r in self.query("SELECT * FROM countries ORDER BY short_name")}) | |
| 112 | + | |
| 113 | + def countries_by_slug(self) -> dict[str, dict[str, Any]]: | |
| 114 | + return self._cached("countries_by_slug", lambda: {c["slug"]: c for c in self.countries().values() if c.get("slug")}) | |
| 115 | + | |
| 116 | + def indicators(self) -> dict[str, dict[str, Any]]: | |
| 117 | + return self._cached("indicators", lambda: {r["id"]: r for r in self.query("SELECT * FROM indicators ORDER BY name")}) | |
| 118 | + | |
| 119 | + def indicators_by_slug(self) -> dict[str, dict[str, Any]]: | |
| 120 | + return self._cached("indicators_by_slug", lambda: {i["slug"]: i for i in self.indicators().values() if i.get("slug")}) | |
| 121 | + | |
| 122 | + def groups(self) -> dict[str, dict[str, Any]]: | |
| 123 | + def load() -> dict[str, dict[str, Any]]: | |
| 124 | + rows = self.query("SELECT * FROM groups") | |
| 125 | + if not rows: # fall back to the registry when the pipeline did not materialise groups | |
| 126 | + from countryatlas.registry import groups as reg_groups | |
| 127 | + | |
| 128 | + rows = [ | |
| 129 | + {"id": g.id, "slug": g.slug, "name": g.name, "kind": g.kind, "description": g.description, | |
| 130 | + "wb_code": g.wb_code, "n_members": len(g.members)} | |
| 131 | + for g in reg_groups() | |
| 132 | + ] | |
| 133 | + return {r["id"]: r for r in rows} | |
| 134 | + | |
| 135 | + return self._cached("groups", load) | |
| 136 | + | |
| 137 | + def groups_by_slug(self) -> dict[str, dict[str, Any]]: | |
| 138 | + return self._cached("groups_by_slug", lambda: {g["slug"]: g for g in self.groups().values() if g.get("slug")}) | |
| 139 | + | |
| 140 | + def group_members(self, group_id: str) -> list[str]: | |
| 141 | + def load() -> dict[str, list[str]]: | |
| 142 | + out: dict[str, list[str]] = {} | |
| 143 | + for gid, cid in self.query_rows("SELECT group_id, country_id FROM group_members"): | |
| 144 | + out.setdefault(gid, []).append(cid) | |
| 145 | + if not out: | |
| 146 | + from countryatlas.registry import groups as reg_groups | |
| 147 | + | |
| 148 | + out = {g.id: list(g.members) for g in reg_groups()} | |
| 149 | + return out | |
| 150 | + | |
| 151 | + return self._cached("group_members", load).get(group_id, []) | |
| 152 | + | |
| 153 | + def sources(self) -> dict[str, dict[str, Any]]: | |
| 154 | + return self._cached("sources", lambda: {r["id"]: r for r in self.query("SELECT * FROM sources ORDER BY name")}) | |
| 155 | + | |
| 156 | + def indicator_sources(self) -> dict[tuple[str, str, str], dict[str, Any]]: | |
| 157 | + """(indicator_id, source_id, series_code) → indicator_sources row (first by priority).""" | |
| 158 | + | |
| 159 | + def load() -> dict[tuple[str, str, str], dict[str, Any]]: | |
| 160 | + out: dict[tuple[str, str, str], dict[str, Any]] = {} | |
| 161 | + rows = self.query("SELECT * FROM indicator_sources ORDER BY indicator_id, priority NULLS LAST") | |
| 162 | + for r in rows: | |
| 163 | + key = (r["indicator_id"], r["source_id"], r["series_code"] or "") | |
| 164 | + out.setdefault(key, r) | |
| 165 | + out.setdefault((r["indicator_id"], r["source_id"], "*"), r) | |
| 166 | + return out | |
| 167 | + | |
| 168 | + return self._cached("indicator_sources", load) | |
| 169 | + | |
| 170 | + def indicator_sources_for(self, indicator_id: str) -> list[dict[str, Any]]: | |
| 171 | + return [r for (iid, _sid, code), r in self.indicator_sources().items() if iid == indicator_id and code != "*"] | |
| 172 | + | |
| 173 | + def max_years(self) -> dict[str, int]: | |
| 174 | + """indicator_id → last year present in `latest` (used to flag stale ranks).""" | |
| 175 | + return self._cached( | |
| 176 | + "max_years", | |
| 177 | + lambda: {r[0]: r[1] for r in self.query_rows("SELECT indicator_id, max(year) FROM latest GROUP BY indicator_id")}, | |
| 178 | + ) | |
| 179 | + | |
| 180 | + | |
| 181 | +class Database: | |
| 182 | + """Process-wide handle. `current()` performs the inode/mtime check and reopens when the file was swapped.""" | |
| 183 | + | |
| 184 | + def __init__(self, path: Path | None = None) -> None: | |
| 185 | + self.path = Path(path) if path else settings.db_path | |
| 186 | + self._snap: Snapshot | None = None | |
| 187 | + self._lock = threading.Lock() | |
| 188 | + | |
| 189 | + def current(self) -> Snapshot | None: | |
| 190 | + try: | |
| 191 | + st = os.stat(self.path) | |
| 192 | + except FileNotFoundError: | |
| 193 | + with self._lock: | |
| 194 | + if self._snap is not None: | |
| 195 | + log.warning("database file vanished: %s", self.path) | |
| 196 | + old, self._snap = self._snap, None | |
| 197 | + old.close() | |
| 198 | + return None | |
| 199 | + snap = self._snap | |
| 200 | + if snap is not None and snap.ino == st.st_ino and snap.mtime_ns == st.st_mtime_ns: | |
| 201 | + return snap | |
| 202 | + with self._lock: | |
| 203 | + snap = self._snap | |
| 204 | + if snap is not None and snap.ino == st.st_ino and snap.mtime_ns == st.st_mtime_ns: | |
| 205 | + return snap | |
| 206 | + # DuckDB caches database instances per path inside the process: the old connection MUST be closed before | |
| 207 | + # reconnecting, otherwise connect() hands back the instance bound to the replaced (old inode) file. | |
| 208 | + old, self._snap = self._snap, None | |
| 209 | + if old is not None: | |
| 210 | + old.close() | |
| 211 | + try: | |
| 212 | + new = Snapshot(self.path, st) | |
| 213 | + except duckdb.Error as e: | |
| 214 | + log.error("cannot open %s: %s", self.path, e) | |
| 215 | + return None | |
| 216 | + self._snap = new | |
| 217 | + log.info("opened snapshot run_id=%s built_at=%s (%s)", new.run_id, new.built_at, self.path) | |
| 218 | + return new | |
| 219 | + | |
| 220 | + def require(self) -> Snapshot: | |
| 221 | + snap = self.current() | |
| 222 | + if snap is None: | |
| 223 | + raise DataNotBuilt() | |
| 224 | + return snap | |
| 225 | + | |
| 226 | + def close(self) -> None: | |
| 227 | + with self._lock: | |
| 228 | + if self._snap is not None: | |
| 229 | + self._snap.close() | |
| 230 | + self._snap = None | |
| 231 | + | |
| 232 | + | |
| 233 | +_database: Database | None = None | |
| 234 | + | |
| 235 | + | |
| 236 | +def get_database() -> Database: | |
| 237 | + global _database | |
| 238 | + if _database is None: | |
| 239 | + _database = Database() | |
| 240 | + return _database | |
| 241 | + | |
| 242 | + | |
| 243 | +def set_database(db: Database) -> None: | |
| 244 | + global _database | |
| 245 | + _database = db | |
| 246 | + | |
| 247 | + | |
| 248 | +def get_snapshot() -> Snapshot: | |
| 249 | + """FastAPI dependency: the current snapshot or 503.""" | |
| 250 | + return get_database().require() | |
added
src/countryatlas/api/errors.py
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +"""RFC 7807 problem+json errors.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import HTTPException | |
| 7 | + | |
| 8 | + | |
| 9 | +class Problem(HTTPException): | |
| 10 | + """HTTPException rendered as application/problem+json by the handlers in main.py.""" | |
| 11 | + | |
| 12 | + def __init__(self, status_code: int, title: str, detail: str | None = None, **extra: Any) -> None: | |
| 13 | + super().__init__(status_code=status_code, detail=detail or title) | |
| 14 | + self.title = title | |
| 15 | + self.extra = extra | |
| 16 | + | |
| 17 | + | |
| 18 | +def not_found(what: str, ident: str, hint: str | None = None) -> Problem: | |
| 19 | + detail = f"Unknown {what} '{ident}'." | |
| 20 | + if hint: | |
| 21 | + detail += " " + hint | |
| 22 | + return Problem(404, f"{what.capitalize()} not found", detail, resource=what, id=ident) | |
| 23 | + | |
| 24 | + | |
| 25 | +def bad_request(detail: str, **extra: Any) -> Problem: | |
| 26 | + return Problem(400, "Bad request", detail, **extra) | |
| 27 | + | |
| 28 | + | |
| 29 | +def problem_body(status: int, title: str, detail: str | None = None, instance: str | None = None, **extra: Any) -> dict: | |
| 30 | + body: dict[str, Any] = {"type": "about:blank", "title": title, "status": status} | |
| 31 | + if detail: | |
| 32 | + body["detail"] = detail | |
| 33 | + if instance: | |
| 34 | + body["instance"] = instance | |
| 35 | + body.update({k: v for k, v in extra.items() if v is not None}) | |
| 36 | + return body | |
added
src/countryatlas/api/formatting.py
+117 −0
@@ -0,0 +1,117 @@ | ||
| 1 | +"""Server-side number formatting (search hints, OpenGraph, headlines). The web app formats too — keep both in sync. | |
| 2 | + | |
| 3 | +`format_value(53372.1, gdp_per_capita)` → "53.4k"; `format_value(1.23e12, gdp)` → "1.2T"; percent → "3.4 %"; | |
| 4 | +years → "82.1 yrs"; tonnes → "5.2 t"; per_1000 → "3.2 per 1,000"; per_100k → "1.2 per 100k". | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import math | |
| 9 | +from typing import Any | |
| 10 | + | |
| 11 | +_SUFFIXES = [(1e12, "T"), (1e9, "B"), (1e6, "M"), (1e3, "k")] | |
| 12 | + | |
| 13 | + | |
| 14 | +def _get(indicator: Any, key: str, default: Any = None) -> Any: | |
| 15 | + if indicator is None: | |
| 16 | + return default | |
| 17 | + if isinstance(indicator, dict): | |
| 18 | + return indicator.get(key, default) if indicator.get(key) is not None else default | |
| 19 | + return getattr(indicator, key, default) if getattr(indicator, key, None) is not None else default | |
| 20 | + | |
| 21 | + | |
| 22 | +def compact_number(value: float, precision: int = 1, min_compact: float = 1e4) -> str: | |
| 23 | + """1234567 → '1.2M'; 53372 → '53.4k'; 812 → '812'; keeps sign.""" | |
| 24 | + if value is None or (isinstance(value, float) and (math.isnan(value) or math.isinf(value))): | |
| 25 | + return "—" | |
| 26 | + sign = "-" if value < 0 else "" | |
| 27 | + v = abs(float(value)) | |
| 28 | + if v >= min_compact: | |
| 29 | + for cut, suffix in _SUFFIXES: | |
| 30 | + if v >= cut: | |
| 31 | + s = f"{v / cut:.{precision}f}" | |
| 32 | + if "." in s: | |
| 33 | + s = s.rstrip("0").rstrip(".") | |
| 34 | + return f"{sign}{s}{suffix}" | |
| 35 | + if v >= 1000: | |
| 36 | + return f"{sign}{v:,.0f}" | |
| 37 | + if v == int(v) and v >= 10: | |
| 38 | + return f"{sign}{int(v)}" | |
| 39 | + return f"{sign}{v:.{precision}f}" | |
| 40 | + | |
| 41 | + | |
| 42 | +def plain_number(value: float, precision: int = 1) -> str: | |
| 43 | + if value is None or (isinstance(value, float) and (math.isnan(value) or math.isinf(value))): | |
| 44 | + return "—" | |
| 45 | + v = float(value) | |
| 46 | + if abs(v) >= 1000: | |
| 47 | + return f"{v:,.0f}" if precision == 0 or abs(v) >= 1e5 else f"{v:,.{precision}f}" | |
| 48 | + return f"{v:.{precision}f}" | |
| 49 | + | |
| 50 | + | |
| 51 | +def format_value(value: float | None, indicator: Any = None, *, with_unit: bool = True) -> str: | |
| 52 | + """Format a value according to the indicator's `format`/`precision`/`unit_short` (registry or DB row).""" | |
| 53 | + if value is None or (isinstance(value, float) and (math.isnan(value) or math.isinf(value))): | |
| 54 | + return "—" | |
| 55 | + fmt = _get(indicator, "format", "number") | |
| 56 | + precision = int(_get(indicator, "precision", 1)) | |
| 57 | + unit_short = _get(indicator, "unit_short", "") or "" | |
| 58 | + v = float(value) | |
| 59 | + | |
| 60 | + if fmt == "currency": | |
| 61 | + return compact_number(v, precision=1) | |
| 62 | + if fmt == "percent": | |
| 63 | + s = f"{v:.{max(precision, 1)}f}" | |
| 64 | + return f"{s} %" if with_unit else s | |
| 65 | + if fmt == "years": | |
| 66 | + s = f"{v:.{max(precision, 1)}f}" | |
| 67 | + return f"{s} yrs" if with_unit else s | |
| 68 | + if fmt == "index": | |
| 69 | + return f"{v:.{precision}f}" | |
| 70 | + if fmt == "ratio": | |
| 71 | + return f"{v:.{max(precision, 2)}f}" | |
| 72 | + if fmt == "celsius": | |
| 73 | + return f"{v:.{max(precision, 2)}f} °C" if with_unit else f"{v:.{max(precision, 2)}f}" | |
| 74 | + if fmt == "tonnes": | |
| 75 | + s = compact_number(v, precision=max(precision, 1)) if abs(v) >= 1e4 else f"{v:.{max(precision, 1)}f}" | |
| 76 | + return f"{s} t" if with_unit else s | |
| 77 | + if fmt == "per_1000": | |
| 78 | + return f"{v:.{max(precision, 1)}f} per 1,000" if with_unit else f"{v:.{max(precision, 1)}f}" | |
| 79 | + if fmt == "per_100k": | |
| 80 | + return f"{v:.{max(precision, 1)}f} per 100k" if with_unit else f"{v:.{max(precision, 1)}f}" | |
| 81 | + if fmt == "per_million": | |
| 82 | + return f"{compact_number(v, 1)} per M" if with_unit else compact_number(v, 1) | |
| 83 | + if fmt in ("kwh", "km", "ha"): | |
| 84 | + s = compact_number(v, precision=1) | |
| 85 | + unit = {"kwh": "kWh", "km": "km", "ha": "ha"}[fmt] | |
| 86 | + return f"{s} {unit}" if with_unit else s | |
| 87 | + # number (default) | |
| 88 | + s = compact_number(v, precision=1 if abs(v) >= 1e4 else precision) | |
| 89 | + if with_unit and unit_short and unit_short not in ("people", "US$", "intl $", "#", "number") and abs(v) < 1e4: | |
| 90 | + return f"{s} {unit_short}" | |
| 91 | + return s | |
| 92 | + | |
| 93 | + | |
| 94 | +def format_change(change_abs: float | None, change_pct: float | None, indicator: Any = None) -> str | None: | |
| 95 | + """Human change: percent-like indicators → '+1.2 pts', otherwise '+3.4 %'.""" | |
| 96 | + fmt = _get(indicator, "format", "number") | |
| 97 | + if fmt in ("percent", "index", "years", "ratio", "celsius") and change_abs is not None: | |
| 98 | + sign = "+" if change_abs >= 0 else "−" | |
| 99 | + unit = "pts" if fmt in ("percent", "index") else ("yrs" if fmt == "years" else "") | |
| 100 | + return f"{sign}{abs(change_abs):.1f} {unit}".strip() | |
| 101 | + if change_pct is not None: | |
| 102 | + sign = "+" if change_pct >= 0 else "−" | |
| 103 | + return f"{sign}{abs(change_pct):.1f} %" | |
| 104 | + if change_abs is not None: | |
| 105 | + sign = "+" if change_abs >= 0 else "−" | |
| 106 | + return f"{sign}{compact_number(abs(change_abs))}" | |
| 107 | + return None | |
| 108 | + | |
| 109 | + | |
| 110 | +def ordinal(n: int | None) -> str | None: | |
| 111 | + if n is None: | |
| 112 | + return None | |
| 113 | + if 10 <= n % 100 <= 20: | |
| 114 | + suf = "th" | |
| 115 | + else: | |
| 116 | + suf = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th") | |
| 117 | + return f"{n}{suf}" | |
added
src/countryatlas/api/main.py
+207 −0
@@ -0,0 +1,207 @@ | ||
| 1 | +"""CountryAtlas public API — FastAPI application factory. | |
| 2 | + | |
| 3 | +Run: `ca-api` (uvicorn, 1 worker, proxy headers) or | |
| 4 | +`CA_DATA_DIR=~/countryatlas-data python -m uvicorn countryatlas.api.main:app --port 8291`. | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import logging | |
| 9 | +import os | |
| 10 | +import time | |
| 11 | +from contextlib import asynccontextmanager | |
| 12 | +from pathlib import Path | |
| 13 | +from typing import Any | |
| 14 | + | |
| 15 | +import orjson | |
| 16 | +from fastapi import FastAPI, Request, Response | |
| 17 | +from fastapi.exceptions import RequestValidationError | |
| 18 | +from fastapi.middleware.cors import CORSMiddleware | |
| 19 | +from fastapi.middleware.gzip import GZipMiddleware | |
| 20 | +from fastapi.responses import JSONResponse | |
| 21 | +from starlette.exceptions import HTTPException as StarletteHTTPException | |
| 22 | + | |
| 23 | +from countryatlas.api.cache import ResponseCache, response_cache | |
| 24 | +from countryatlas.api.db import Database, DataNotBuilt, get_database, set_database | |
| 25 | +from countryatlas.api.errors import Problem, problem_body | |
| 26 | +from countryatlas.api.ratelimit import TokenBucketLimiter, client_ip | |
| 27 | +from countryatlas.api.routers import ( | |
| 28 | + admin, | |
| 29 | + changes, | |
| 30 | + compare, | |
| 31 | + countries, | |
| 32 | + download, | |
| 33 | + health, | |
| 34 | + home, | |
| 35 | + indicators, | |
| 36 | + methodology, | |
| 37 | + rankings, | |
| 38 | + regions, | |
| 39 | + search, | |
| 40 | + series, | |
| 41 | + sources, | |
| 42 | +) | |
| 43 | +from countryatlas.config import settings | |
| 44 | + | |
| 45 | +log = logging.getLogger("countryatlas.api") | |
| 46 | + | |
| 47 | +API_PREFIX = "/api/v1" | |
| 48 | +PROBLEM = "application/problem+json" | |
| 49 | + | |
| 50 | + | |
| 51 | +class ORJSONResponse(JSONResponse): | |
| 52 | + media_type = "application/json" | |
| 53 | + | |
| 54 | + def render(self, content: Any) -> bytes: | |
| 55 | + return orjson.dumps(content, default=str, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY) | |
| 56 | + | |
| 57 | + | |
| 58 | +def _problem(request: Request, status: int, title: str, detail: str | None = None, headers: dict | None = None, **extra: Any) -> JSONResponse: | |
| 59 | + body = problem_body(status, title, detail, instance=str(request.url.path), **extra) | |
| 60 | + return ORJSONResponse(body, status_code=status, media_type=PROBLEM, headers=headers) | |
| 61 | + | |
| 62 | + | |
| 63 | +def create_app(db_path: str | Path | None = None, *, rate_limit_per_minute: int | None = None, | |
| 64 | + cache: ResponseCache | None = None) -> FastAPI: | |
| 65 | + if db_path is not None: | |
| 66 | + set_database(Database(Path(db_path))) | |
| 67 | + db = get_database() | |
| 68 | + cache = cache or response_cache | |
| 69 | + rpm = rate_limit_per_minute if rate_limit_per_minute is not None else int(os.environ.get("CA_RATE_LIMIT_PER_MIN", "120")) | |
| 70 | + limiter = TokenBucketLimiter(rpm) if rpm > 0 else None | |
| 71 | + | |
| 72 | + @asynccontextmanager | |
| 73 | + async def lifespan(_app: FastAPI): # type: ignore[no-untyped-def] | |
| 74 | + snap = db.current() | |
| 75 | + log.info("CountryAtlas API starting — db=%s run_id=%s", db.path, snap.run_id if snap else "EMPTY") | |
| 76 | + yield | |
| 77 | + db.close() | |
| 78 | + | |
| 79 | + app = FastAPI( | |
| 80 | + lifespan=lifespan, | |
| 81 | + title="CountryAtlas API", | |
| 82 | + version="1.0.0", | |
| 83 | + summary="Country statistics with provenance on every value — www.countryatlas.co", | |
| 84 | + description=( | |
| 85 | + "Public data API of CountryAtlas. Countries (ISO3 or slug), indicators (slug), rankings, comparisons, groups, " | |
| 86 | + "search and downloads. Every value carries a `provenance` object (source, dataset, series code, retrieval date, URL, " | |
| 87 | + "licence). Errors use RFC 7807 problem+json. Rate limit: 120 requests / minute / IP." | |
| 88 | + ), | |
| 89 | + openapi_url=f"{API_PREFIX}/openapi.json", | |
| 90 | + docs_url=f"{API_PREFIX}/docs", | |
| 91 | + redoc_url=f"{API_PREFIX}/redoc", | |
| 92 | + default_response_class=ORJSONResponse, | |
| 93 | + contact={"name": "CountryAtlas", "url": settings.site_url}, | |
| 94 | + license_info={"name": "Data: see each source's licence (provenance.licence)"}, | |
| 95 | + ) | |
| 96 | + app.state.db = db | |
| 97 | + app.state.cache = cache | |
| 98 | + app.state.limiter = limiter | |
| 99 | + | |
| 100 | + # ------------------------------------------------------------------ middleware: rate limit, cache, run header | |
| 101 | + # Registered FIRST so it is the innermost middleware: GZip and CORS (added below) wrap it, hence the cache stores | |
| 102 | + # uncompressed bodies and the gzip layer compresses both hits and misses. | |
| 103 | + @app.middleware("http") | |
| 104 | + async def atlas_middleware(request: Request, call_next): # type: ignore[no-untyped-def] | |
| 105 | + path = request.url.path | |
| 106 | + public = path.startswith(API_PREFIX) and not path.startswith(f"{API_PREFIX}/admin") and path != f"{API_PREFIX}/health" | |
| 107 | + if limiter is not None and public: | |
| 108 | + ok, retry = limiter.allow(client_ip(request.headers, request.client.host if request.client else None)) | |
| 109 | + if not ok: | |
| 110 | + return _problem(request, 429, "Too many requests", f"Rate limit is {rpm} requests per minute per IP.", | |
| 111 | + headers={"Retry-After": str(max(1, int(retry + 0.999)))}) | |
| 112 | + snap = db.current() | |
| 113 | + run_id = snap.run_id if snap else None | |
| 114 | + cacheable = (public and request.method == "GET" and snap is not None and "/download." not in path) | |
| 115 | + key = cache.key(run_id or "", path, str(request.url.query)) if cacheable else None | |
| 116 | + if key is not None: | |
| 117 | + hit = cache.get(key) | |
| 118 | + if hit is not None: | |
| 119 | + status, body, media = hit | |
| 120 | + resp = Response(content=body, status_code=status, media_type=media) | |
| 121 | + resp.headers["X-Cache"] = "HIT" | |
| 122 | + if run_id: | |
| 123 | + resp.headers["X-CountryAtlas-Run"] = run_id | |
| 124 | + return resp | |
| 125 | + t0 = time.perf_counter() | |
| 126 | + response = await call_next(request) | |
| 127 | + if run_id: | |
| 128 | + response.headers["X-CountryAtlas-Run"] = run_id | |
| 129 | + response.headers["X-Response-Time"] = f"{(time.perf_counter() - t0) * 1000:.1f}ms" | |
| 130 | + if key is not None and response.status_code == 200 and "text/event-stream" not in (response.media_type or ""): | |
| 131 | + body = b"" | |
| 132 | + async for chunk in response.body_iterator: # type: ignore[attr-defined] | |
| 133 | + body += chunk if isinstance(chunk, bytes) else chunk.encode() | |
| 134 | + if len(body) < 4_000_000: | |
| 135 | + cache.set(key, (response.status_code, body, response.media_type or response.headers.get("content-type"))) | |
| 136 | + headers = dict(response.headers) | |
| 137 | + headers.pop("content-length", None) | |
| 138 | + new = Response(content=body, status_code=response.status_code, headers=headers, media_type=response.media_type) | |
| 139 | + new.headers["X-Cache"] = "MISS" | |
| 140 | + return new | |
| 141 | + return response | |
| 142 | + | |
| 143 | + app.add_middleware(GZipMiddleware, minimum_size=1024) | |
| 144 | + origins = {settings.site_url, "https://countryatlas.co", "https://www.countryatlas.co", "http://localhost:8290", "http://127.0.0.1:8290", | |
| 145 | + "http://localhost:3000"} | |
| 146 | + app.add_middleware(CORSMiddleware, allow_origins=sorted(origins), allow_methods=["GET", "POST", "OPTIONS"], | |
| 147 | + allow_headers=["*"], expose_headers=["X-CountryAtlas-Run", "X-Cache", "Retry-After"], max_age=3600) | |
| 148 | + | |
| 149 | + # ------------------------------------------------------------------ error handlers (problem+json) | |
| 150 | + @app.exception_handler(DataNotBuilt) | |
| 151 | + async def _not_built(request: Request, exc: DataNotBuilt) -> JSONResponse: | |
| 152 | + return _problem(request, 503, "Data not built yet", | |
| 153 | + f"The snapshot {db.path} does not exist yet. Run the pipeline (`ca refresh`) and retry.", | |
| 154 | + headers={"Retry-After": "300"}) | |
| 155 | + | |
| 156 | + @app.exception_handler(Problem) | |
| 157 | + async def _problem_handler(request: Request, exc: Problem) -> JSONResponse: | |
| 158 | + return _problem(request, exc.status_code, exc.title, exc.detail, headers=exc.headers, **exc.extra) | |
| 159 | + | |
| 160 | + @app.exception_handler(StarletteHTTPException) | |
| 161 | + async def _http_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: | |
| 162 | + title = {404: "Not found", 405: "Method not allowed", 403: "Forbidden", 401: "Unauthorized"}.get(exc.status_code, "Error") | |
| 163 | + detail = exc.detail if isinstance(exc.detail, str) else None | |
| 164 | + if exc.status_code == 404 and (detail in (None, "Not Found")): | |
| 165 | + detail = f"No route for {request.url.path}. See {API_PREFIX}/docs." | |
| 166 | + return _problem(request, exc.status_code, title, detail, headers=getattr(exc, "headers", None)) | |
| 167 | + | |
| 168 | + @app.exception_handler(RequestValidationError) | |
| 169 | + async def _validation_handler(request: Request, exc: RequestValidationError) -> JSONResponse: | |
| 170 | + errors = [{"loc": [str(x) for x in e.get("loc", [])], "msg": e.get("msg"), "type": e.get("type")} for e in exc.errors()] | |
| 171 | + return _problem(request, 422, "Validation error", "One or more parameters are invalid.", errors=errors) | |
| 172 | + | |
| 173 | + @app.exception_handler(Exception) | |
| 174 | + async def _unhandled(request: Request, exc: Exception) -> JSONResponse: | |
| 175 | + log.exception("unhandled error on %s", request.url.path) | |
| 176 | + return _problem(request, 500, "Internal server error", f"{type(exc).__name__}: {exc}") | |
| 177 | + | |
| 178 | + # ------------------------------------------------------------------ routers | |
| 179 | + for r in (health.router, countries.router, indicators.router, series.router, rankings.router, compare.router, regions.router, | |
| 180 | + search.router, home.router, changes.router, sources.router, methodology.router, download.router, admin.router): | |
| 181 | + app.include_router(r, prefix=API_PREFIX) | |
| 182 | + | |
| 183 | + @app.get("/", include_in_schema=False) | |
| 184 | + def root() -> dict[str, Any]: | |
| 185 | + return {"name": "CountryAtlas API", "docs": f"{API_PREFIX}/docs", "openapi": f"{API_PREFIX}/openapi.json", "health": f"{API_PREFIX}/health"} | |
| 186 | + | |
| 187 | + @app.get("/health", include_in_schema=False) | |
| 188 | + def root_health() -> dict[str, Any]: | |
| 189 | + return health.health() | |
| 190 | + | |
| 191 | + return app | |
| 192 | + | |
| 193 | + | |
| 194 | +app = create_app() | |
| 195 | + | |
| 196 | + | |
| 197 | +def run() -> None: | |
| 198 | + """Console entry point `ca-api`.""" | |
| 199 | + import uvicorn | |
| 200 | + | |
| 201 | + logging.basicConfig(level=os.environ.get("CA_LOG_LEVEL", "INFO")) | |
| 202 | + uvicorn.run("countryatlas.api.main:app", host=settings.api_host, port=settings.api_port, workers=1, proxy_headers=True, | |
| 203 | + forwarded_allow_ips="*", log_level=os.environ.get("CA_LOG_LEVEL", "info").lower(), access_log=True) | |
| 204 | + | |
| 205 | + | |
| 206 | +if __name__ == "__main__": | |
| 207 | + run() | |
added
src/countryatlas/api/provenance.py
+97 −0
@@ -0,0 +1,97 @@ | ||
| 1 | +"""Provenance object (docs/ARCHITECTURE.md §8) built from an observation row + indicator_sources + sources.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from datetime import date, datetime | |
| 5 | +from typing import Any | |
| 6 | + | |
| 7 | +from countryatlas.api.db import Snapshot | |
| 8 | + | |
| 9 | +OWID_DATASET_URLS = { | |
| 10 | + "co2": "https://github.com/owid/co2-data", | |
| 11 | + "energy": "https://github.com/owid/energy-data", | |
| 12 | +} | |
| 13 | + | |
| 14 | + | |
| 15 | +def source_url(source_id: str | None, dataset: str | None, code: str | None, iso2: str | None = None) -> str | None: | |
| 16 | + """Deep link into the source for this series (best effort, per connector).""" | |
| 17 | + sid = (source_id or "").lower() | |
| 18 | + code = code or "" | |
| 19 | + dataset = dataset or "" | |
| 20 | + if sid == "worldbank": | |
| 21 | + url = f"https://data.worldbank.org/indicator/{code}" if code else "https://data.worldbank.org/" | |
| 22 | + if code and iso2: | |
| 23 | + url += f"?locations={iso2}" | |
| 24 | + return url | |
| 25 | + if sid == "owid": | |
| 26 | + if dataset in OWID_DATASET_URLS: | |
| 27 | + return OWID_DATASET_URLS[dataset] | |
| 28 | + return f"https://ourworldindata.org/grapher/{code}" if code else "https://ourworldindata.org/" | |
| 29 | + if sid == "imf": | |
| 30 | + return "https://data.imf.org/" | |
| 31 | + if sid == "oecd": | |
| 32 | + return "https://data-explorer.oecd.org/" | |
| 33 | + if sid == "eurostat": | |
| 34 | + return f"https://ec.europa.eu/eurostat/databrowser/view/{dataset}/default/table" if dataset else "https://ec.europa.eu/eurostat/databrowser/" | |
| 35 | + if sid == "who": | |
| 36 | + return f"https://www.who.int/data/gho/data/indicators/indicator-details/GHO/{code}" if code else "https://www.who.int/data/gho" | |
| 37 | + if sid == "fred": | |
| 38 | + return f"https://fred.stlouisfed.org/series/{code}" if code else "https://fred.stlouisfed.org/" | |
| 39 | + if sid == "bis": | |
| 40 | + return "https://data.bis.org/" | |
| 41 | + if sid == "ilo": | |
| 42 | + return "https://ilostat.ilo.org/" | |
| 43 | + return None | |
| 44 | + | |
| 45 | + | |
| 46 | +def _iso(v: Any) -> str | None: | |
| 47 | + if v is None: | |
| 48 | + return None | |
| 49 | + if isinstance(v, datetime): | |
| 50 | + return v.isoformat(timespec="seconds") + ("Z" if v.tzinfo is None else "") | |
| 51 | + if isinstance(v, date): | |
| 52 | + return v.isoformat() | |
| 53 | + return str(v) | |
| 54 | + | |
| 55 | + | |
| 56 | +def build_provenance( | |
| 57 | + snap: Snapshot, | |
| 58 | + indicator_id: str, | |
| 59 | + source_id: str | None, | |
| 60 | + dataset: str | None, | |
| 61 | + series_code: str | None, | |
| 62 | + retrieved_at: Any = None, | |
| 63 | + source_updated_at: Any = None, | |
| 64 | + iso2: str | None = None, | |
| 65 | +) -> dict[str, Any]: | |
| 66 | + src = snap.sources().get(source_id or "", {}) if source_id else {} | |
| 67 | + isrc = snap.indicator_sources().get((indicator_id, source_id or "", series_code or "")) | |
| 68 | + if isrc is None: | |
| 69 | + isrc = snap.indicator_sources().get((indicator_id, source_id or "", "*"), {}) | |
| 70 | + url = source_url(source_id, dataset or isrc.get("dataset"), series_code or isrc.get("series_code"), iso2) | |
| 71 | + if url is None: | |
| 72 | + url = isrc.get("source_url") or src.get("url") | |
| 73 | + return { | |
| 74 | + "source": source_id, | |
| 75 | + "source_name": src.get("name") or (source_id or "").upper() or None, | |
| 76 | + "dataset": dataset or isrc.get("dataset"), | |
| 77 | + "series_code": series_code or isrc.get("series_code"), | |
| 78 | + "retrieved_at": _iso(retrieved_at), | |
| 79 | + "source_updated_at": _iso(source_updated_at), | |
| 80 | + "url": url, | |
| 81 | + "transform": isrc.get("transform"), | |
| 82 | + "licence": src.get("licence"), | |
| 83 | + } | |
| 84 | + | |
| 85 | + | |
| 86 | +def provenance_from_row(snap: Snapshot, row: dict[str, Any], indicator_id: str | None = None, iso2: str | None = None) -> dict[str, Any]: | |
| 87 | + """Row must carry source_id, source_dataset, source_series_code, retrieved_at, source_updated_at (observations shape).""" | |
| 88 | + return build_provenance( | |
| 89 | + snap, | |
| 90 | + indicator_id or row.get("indicator_id"), | |
| 91 | + row.get("source_id"), | |
| 92 | + row.get("source_dataset"), | |
| 93 | + row.get("source_series_code"), | |
| 94 | + row.get("retrieved_at"), | |
| 95 | + row.get("source_updated_at"), | |
| 96 | + iso2, | |
| 97 | + ) | |
added
src/countryatlas/api/ratelimit.py
+42 −0
@@ -0,0 +1,42 @@ | ||
| 1 | +"""In-process token bucket rate limiter (per client IP, X-Forwarded-For aware).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import threading | |
| 5 | +import time | |
| 6 | + | |
| 7 | + | |
| 8 | +class TokenBucketLimiter: | |
| 9 | + def __init__(self, rate_per_minute: int = 120, burst: int | None = None) -> None: | |
| 10 | + self.rate = float(rate_per_minute) | |
| 11 | + self.capacity = float(burst or rate_per_minute) | |
| 12 | + self._buckets: dict[str, tuple[float, float]] = {} # ip -> (tokens, last_ts) | |
| 13 | + self._lock = threading.Lock() | |
| 14 | + self._last_sweep = time.monotonic() | |
| 15 | + | |
| 16 | + def allow(self, key: str) -> tuple[bool, float]: | |
| 17 | + """Return (allowed, retry_after_seconds).""" | |
| 18 | + now = time.monotonic() | |
| 19 | + with self._lock: | |
| 20 | + tokens, last = self._buckets.get(key, (self.capacity, now)) | |
| 21 | + tokens = min(self.capacity, tokens + (now - last) * self.rate / 60.0) | |
| 22 | + if tokens >= 1.0: | |
| 23 | + self._buckets[key] = (tokens - 1.0, now) | |
| 24 | + allowed, retry = True, 0.0 | |
| 25 | + else: | |
| 26 | + self._buckets[key] = (tokens, now) | |
| 27 | + allowed, retry = False, (1.0 - tokens) * 60.0 / self.rate | |
| 28 | + if now - self._last_sweep > 300 and len(self._buckets) > 10_000: | |
| 29 | + cutoff = now - 120 | |
| 30 | + self._buckets = {k: v for k, v in self._buckets.items() if v[1] > cutoff} | |
| 31 | + self._last_sweep = now | |
| 32 | + return allowed, retry | |
| 33 | + | |
| 34 | + | |
| 35 | +def client_ip(headers, fallback: str | None) -> str: # type: ignore[no-untyped-def] | |
| 36 | + xff = headers.get("x-forwarded-for") | |
| 37 | + if xff: | |
| 38 | + return xff.split(",")[0].strip() | |
| 39 | + real = headers.get("x-real-ip") | |
| 40 | + if real: | |
| 41 | + return real.strip() | |
| 42 | + return fallback or "unknown" | |
added
src/countryatlas/api/routers/__init__.py
+0 −0
added
src/countryatlas/api/routers/admin.py
+186 −0
@@ -0,0 +1,186 @@ | ||
| 1 | +"""/admin/* — operations console, guarded by `X-Admin-Token` (403 when wrong, 503 when no token configured).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import json | |
| 5 | +import os | |
| 6 | +import signal | |
| 7 | +from datetime import UTC, datetime | |
| 8 | +from pathlib import Path | |
| 9 | +from typing import Any | |
| 10 | + | |
| 11 | +from fastapi import APIRouter, Depends, Header, Query | |
| 12 | + | |
| 13 | +from countryatlas.api.cache import response_cache | |
| 14 | +from countryatlas.api.common import meta_block | |
| 15 | +from countryatlas.api.db import Snapshot, get_database | |
| 16 | +from countryatlas.api.errors import Problem | |
| 17 | +from countryatlas.api.provenance import _iso | |
| 18 | +from countryatlas.config import settings | |
| 19 | + | |
| 20 | +router = APIRouter(prefix="/admin", tags=["admin"]) | |
| 21 | + | |
| 22 | + | |
| 23 | +def require_admin(x_admin_token: str | None = Header(None, alias="X-Admin-Token")) -> None: | |
| 24 | + if not settings.admin_token: | |
| 25 | + raise Problem(503, "Admin disabled", "CA_ADMIN_TOKEN is not configured on this server.") | |
| 26 | + if x_admin_token != settings.admin_token: | |
| 27 | + raise Problem(403, "Forbidden", "Invalid or missing X-Admin-Token header.") | |
| 28 | + | |
| 29 | + | |
| 30 | +def _snap() -> Snapshot: | |
| 31 | + return get_database().require() | |
| 32 | + | |
| 33 | + | |
| 34 | +def _fix_ts(rows: list[dict[str, Any]], *keys: str) -> list[dict[str, Any]]: | |
| 35 | + for r in rows: | |
| 36 | + for k in keys: | |
| 37 | + if k in r: | |
| 38 | + r[k] = _iso(r[k]) | |
| 39 | + return rows | |
| 40 | + | |
| 41 | + | |
| 42 | +def _read_json(path: Path) -> dict[str, Any] | None: | |
| 43 | + try: | |
| 44 | + return json.loads(path.read_text()) | |
| 45 | + except (OSError, ValueError): | |
| 46 | + return None | |
| 47 | + | |
| 48 | + | |
| 49 | +@router.get("/overview", dependencies=[Depends(require_admin)], summary="Meta, counts, connector health, stale sources, scheduler") | |
| 50 | +def overview() -> dict[str, Any]: | |
| 51 | + db = get_database() | |
| 52 | + snap = db.current() | |
| 53 | + sched = _read_json(settings.data_dir / "scheduler.json") | |
| 54 | + pid_file = settings.data_dir / "scheduler.pid" | |
| 55 | + pid = None | |
| 56 | + if pid_file.exists(): | |
| 57 | + try: | |
| 58 | + pid = int(pid_file.read_text().strip()) | |
| 59 | + except ValueError: | |
| 60 | + pid = None | |
| 61 | + alive = None | |
| 62 | + if pid: | |
| 63 | + try: | |
| 64 | + os.kill(pid, 0) | |
| 65 | + alive = True | |
| 66 | + except OSError: | |
| 67 | + alive = False | |
| 68 | + out: dict[str, Any] = { | |
| 69 | + "db": {"path": str(db.path), "exists": snap is not None, "size_bytes": db.path.stat().st_size if db.path.exists() else None}, | |
| 70 | + "scheduler": {"heartbeat": sched, "pid": pid, "alive": alive}, | |
| 71 | + "cache": response_cache.stats(), | |
| 72 | + "settings": {"data_dir": str(settings.data_dir), "api_host": settings.api_host, "api_port": settings.api_port, | |
| 73 | + "refresh_at": f"{settings.refresh_hour:02d}:{settings.refresh_minute:02d} {settings.timezone}"}, | |
| 74 | + } | |
| 75 | + if snap is None: | |
| 76 | + out["status"] = "empty" | |
| 77 | + return out | |
| 78 | + counts = {t: snap.table_count(t) for t in ("countries", "indicators", "observations", "observations_alt", "latest", "rankings", "changes", | |
| 79 | + "events", "similarity", "insights", "coverage", "import_runs", "validation_issues", "search_index")} | |
| 80 | + connectors = _fix_ts(snap.query( | |
| 81 | + """SELECT connector, count(*) AS n_runs, | |
| 82 | + max(finished_at) AS last_finished_at, | |
| 83 | + arg_max(status, coalesce(finished_at, started_at)) AS last_status, | |
| 84 | + sum(CASE WHEN status = 'ok' THEN 1 ELSE 0 END) AS n_ok, | |
| 85 | + sum(CASE WHEN status IN ('failed', 'quarantined') THEN 1 ELSE 0 END) AS n_failed, | |
| 86 | + sum(rows_valid) AS rows_valid, sum(warnings) AS warnings, sum(errors) AS errors | |
| 87 | + FROM import_runs GROUP BY connector ORDER BY connector"""), "last_finished_at") | |
| 88 | + stale = _fix_ts(snap.query( | |
| 89 | + """SELECT source_id, count(*) AS n_observations, max(source_updated_at) AS source_updated_at, max(retrieved_at) AS retrieved_at, | |
| 90 | + sum(CASE WHEN status = 'stale' THEN 1 ELSE 0 END) AS n_stale | |
| 91 | + FROM observations GROUP BY source_id ORDER BY source_updated_at NULLS FIRST"""), "source_updated_at", "retrieved_at") | |
| 92 | + statuses = {r[0]: r[1] for r in snap.query_rows("SELECT status, count(*) FROM observations GROUP BY status")} | |
| 93 | + out.update({"status": "ok", "meta": {**snap.meta, **meta_block(snap)}, "counts": counts, "connectors": connectors, "sources": stale, | |
| 94 | + "observation_statuses": statuses}) | |
| 95 | + return out | |
| 96 | + | |
| 97 | + | |
| 98 | +@router.get("/runs", dependencies=[Depends(require_admin)], summary="Import runs") | |
| 99 | +def runs(limit: int = Query(100, ge=1, le=2000), connector: str | None = None, status: str | None = None) -> dict[str, Any]: | |
| 100 | + snap = _snap() | |
| 101 | + where, params = [], [] | |
| 102 | + if connector: | |
| 103 | + where.append("connector = ?") | |
| 104 | + params.append(connector) | |
| 105 | + if status: | |
| 106 | + where.append("status = ?") | |
| 107 | + params.append(status) | |
| 108 | + rows = snap.query("SELECT * FROM import_runs" + (f" WHERE {' AND '.join(where)}" if where else "") | |
| 109 | + + " ORDER BY started_at DESC NULLS LAST LIMIT ?", [*params, limit]) | |
| 110 | + return {"meta": meta_block(snap), "n": len(rows), "items": _fix_ts(rows, "started_at", "finished_at")} | |
| 111 | + | |
| 112 | + | |
| 113 | +@router.get("/issues", dependencies=[Depends(require_admin)], summary="Validation issues") | |
| 114 | +def issues(limit: int = Query(200, ge=1, le=5000), severity: str | None = None, connector: str | None = None, | |
| 115 | + indicator: str | None = None, code: str | None = None, run_id: str | None = None) -> dict[str, Any]: | |
| 116 | + snap = _snap() | |
| 117 | + where, params = [], [] | |
| 118 | + for col, val in (("severity", severity), ("connector", connector), ("indicator_id", indicator), ("code", code), ("run_id", run_id)): | |
| 119 | + if val: | |
| 120 | + where.append(f"{col} = ?") | |
| 121 | + params.append(val) | |
| 122 | + w = f" WHERE {' AND '.join(where)}" if where else "" | |
| 123 | + rows = snap.query(f"SELECT * FROM validation_issues{w} ORDER BY severity, connector, indicator_id LIMIT ?", [*params, limit]) | |
| 124 | + summary = snap.query(f"SELECT severity, code, count(*) AS n FROM validation_issues{w} GROUP BY 1, 2 ORDER BY n DESC", params) | |
| 125 | + return {"meta": meta_block(snap), "n": len(rows), "summary": summary, "items": rows} | |
| 126 | + | |
| 127 | + | |
| 128 | +@router.get("/coverage", dependencies=[Depends(require_admin)], summary="Coverage matrix indicator × countries + last year") | |
| 129 | +def coverage() -> dict[str, Any]: | |
| 130 | + snap = _snap() | |
| 131 | + rows = _fix_ts(snap.query( | |
| 132 | + """SELECT i.id AS indicator_id, i.topic, i.primary_source_id, i.n_countries, i.n_observations, i.first_year, i.last_year, | |
| 133 | + i.latest_source_updated_at, | |
| 134 | + (SELECT count(DISTINCT country_id) FROM latest l WHERE l.indicator_id = i.id) AS n_latest, | |
| 135 | + (SELECT max(year) FROM latest l WHERE l.indicator_id = i.id) AS latest_year | |
| 136 | + FROM indicators i ORDER BY i.topic, i.id"""), "latest_source_updated_at") | |
| 137 | + countries = snap.query("SELECT c.id, c.short_name, cov.n_indicators, cov.n_observations, cov.latest_year, cov.coverage_pct " | |
| 138 | + "FROM countries c LEFT JOIN coverage cov ON cov.country_id = c.id ORDER BY cov.coverage_pct DESC NULLS LAST") | |
| 139 | + return {"meta": meta_block(snap), "n_indicators": len(rows), "indicators": rows, "countries": countries} | |
| 140 | + | |
| 141 | + | |
| 142 | +@router.get("/raw", dependencies=[Depends(require_admin)], summary="Raw files stored for a run") | |
| 143 | +def raw(run_id: str = Query(...)) -> dict[str, Any]: | |
| 144 | + snap = _snap() | |
| 145 | + runs = snap.query("SELECT connector, dataset, raw_path, status, rows_raw FROM import_runs WHERE run_id = ?", [run_id]) | |
| 146 | + files: list[dict[str, Any]] = [] | |
| 147 | + for r in runs: | |
| 148 | + p = r.get("raw_path") | |
| 149 | + if not p: | |
| 150 | + continue | |
| 151 | + path = Path(p) | |
| 152 | + if not path.is_absolute(): | |
| 153 | + path = settings.data_dir / path | |
| 154 | + if path.is_dir(): | |
| 155 | + for f in sorted(path.rglob("*")): | |
| 156 | + if f.is_file(): | |
| 157 | + st = f.stat() | |
| 158 | + files.append({"connector": r["connector"], "dataset": r["dataset"], "path": str(f), "size": st.st_size, | |
| 159 | + "modified_at": datetime.fromtimestamp(st.st_mtime, tz=UTC).isoformat(timespec="seconds")}) | |
| 160 | + elif path.is_file(): | |
| 161 | + st = path.stat() | |
| 162 | + files.append({"connector": r["connector"], "dataset": r["dataset"], "path": str(path), "size": st.st_size, | |
| 163 | + "modified_at": datetime.fromtimestamp(st.st_mtime, tz=UTC).isoformat(timespec="seconds")}) | |
| 164 | + else: | |
| 165 | + files.append({"connector": r["connector"], "dataset": r["dataset"], "path": str(path), "missing": True}) | |
| 166 | + return {"meta": meta_block(snap), "run_id": run_id, "n_runs": len(runs), "runs": runs, "n_files": len(files), "files": files} | |
| 167 | + | |
| 168 | + | |
| 169 | +@router.post("/refresh", dependencies=[Depends(require_admin)], summary="Ask the scheduler to refresh now (SIGUSR1)") | |
| 170 | +def refresh() -> dict[str, Any]: | |
| 171 | + pid_file = settings.data_dir / "scheduler.pid" | |
| 172 | + if not pid_file.exists(): | |
| 173 | + raise Problem(409, "Scheduler not running", f"No pid file at {pid_file}.") | |
| 174 | + try: | |
| 175 | + pid = int(pid_file.read_text().strip()) | |
| 176 | + os.kill(pid, signal.SIGUSR1) | |
| 177 | + except (ValueError, ProcessLookupError, PermissionError) as e: | |
| 178 | + raise Problem(409, "Scheduler not reachable", f"Could not signal scheduler: {e}") from e | |
| 179 | + return {"ok": True, "pid": pid, "signal": "SIGUSR1", "sent_at": datetime.now(UTC).isoformat(timespec="seconds")} | |
| 180 | + | |
| 181 | + | |
| 182 | +@router.post("/cache/clear", dependencies=[Depends(require_admin)], summary="Clear the in-process response cache") | |
| 183 | +def clear_cache() -> dict[str, Any]: | |
| 184 | + before = response_cache.stats() | |
| 185 | + response_cache.clear() | |
| 186 | + return {"ok": True, "before": before} | |
added
src/countryatlas/api/routers/changes.py
+42 −0
@@ -0,0 +1,42 @@ | ||
| 1 | +"""/changes — global feed of recent detected changes.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends, Query | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import meta_block, resolve_country, resolve_indicator | |
| 10 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 11 | +from countryatlas.api.routers.countries import CHANGES_SQL, change_item | |
| 12 | + | |
| 13 | +router = APIRouter(tags=["changes"]) | |
| 14 | + | |
| 15 | + | |
| 16 | +@router.get("/changes", response_model=schemas.ChangesResponse, summary="Global recent changes feed") | |
| 17 | +def list_changes(limit: int = Query(50, ge=1, le=500), offset: int = Query(0, ge=0), kind: str | None = Query(None), | |
| 18 | + indicator: str | None = Query(None), country: str | None = Query(None), topic: str | None = Query(None), | |
| 19 | + min_severity: float | None = Query(None, ge=0, le=1), snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 20 | + where, params = ["coalesce(c.kind, 'country') = 'country'"], [] | |
| 21 | + if kind: | |
| 22 | + where.append("x.kind = ?") | |
| 23 | + params.append(kind) | |
| 24 | + if indicator: | |
| 25 | + where.append("x.indicator_id = ?") | |
| 26 | + params.append(resolve_indicator(snap, indicator)["id"]) | |
| 27 | + if country: | |
| 28 | + where.append("x.country_id = ?") | |
| 29 | + params.append(resolve_country(snap, country)["id"]) | |
| 30 | + if topic: | |
| 31 | + where.append("x.indicator_id IN (SELECT id FROM indicators WHERE topic = ?)") | |
| 32 | + params.append(topic) | |
| 33 | + if min_severity is not None: | |
| 34 | + where.append("x.severity >= ?") | |
| 35 | + params.append(min_severity) | |
| 36 | + rows = snap.query( | |
| 37 | + CHANGES_SQL.format(table="changes") + " JOIN countries c ON c.id = x.country_id " | |
| 38 | + + f"WHERE {' AND '.join(where)} ORDER BY x.severity DESC NULLS LAST, x.period DESC, x.country_id LIMIT ? OFFSET ?", | |
| 39 | + [*params, limit, offset], | |
| 40 | + ) | |
| 41 | + kinds = [r[0] for r in snap.query_rows("SELECT DISTINCT kind FROM changes ORDER BY kind")] | |
| 42 | + return {"meta": meta_block(snap), "n": len(rows), "kinds": kinds, "items": [change_item(snap, r) for r in rows]} | |
added
src/countryatlas/api/routers/compare.py
+142 −0
@@ -0,0 +1,142 @@ | ||
| 1 | +"""/compare — series bundle with transformation modes, and a latest-values snapshot table.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends, Query | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import ( | |
| 10 | + country_card, | |
| 11 | + empty_metric, | |
| 12 | + indicator_card, | |
| 13 | + latest_rows, | |
| 14 | + merged_indicator, | |
| 15 | + meta_block, | |
| 16 | + metric_from_latest, | |
| 17 | + parse_csv, | |
| 18 | + resolve_country, | |
| 19 | + resolve_indicator, | |
| 20 | + resolve_topic, | |
| 21 | + series_stats, | |
| 22 | +) | |
| 23 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 24 | +from countryatlas.api.errors import bad_request | |
| 25 | +from countryatlas.api.routers.countries import fetch_series | |
| 26 | +from countryatlas.registry import topics as registry_topics | |
| 27 | + | |
| 28 | +router = APIRouter(prefix="/compare", tags=["compare"]) | |
| 29 | + | |
| 30 | +MODES = ("absolute", "per-capita", "index100", "pct") | |
| 31 | + | |
| 32 | + | |
| 33 | +def _population_by_year(snap: Snapshot, country_id: str) -> dict[int, float]: | |
| 34 | + return {int(y): float(v) for y, v in snap.query_rows( | |
| 35 | + "SELECT year, value FROM observations WHERE indicator_id = 'population' AND country_id = ? AND frequency = 'A' AND value IS NOT NULL", | |
| 36 | + [country_id])} | |
| 37 | + | |
| 38 | + | |
| 39 | +def transform_series(snap: Snapshot, s: dict[str, Any], ind: dict[str, Any], mode: str, base_year: int | None) -> dict[str, Any]: | |
| 40 | + """Apply compare mode in place. Returns the (possibly) transformed series with `transform` metadata.""" | |
| 41 | + values = s["values"] | |
| 42 | + info: dict[str, Any] = {"mode": mode} | |
| 43 | + if mode == "per-capita": | |
| 44 | + already = ind.get("per_capita_of") or "per-capita" in ind["id"] or (ind.get("aggregation") or "none") != "sum" | |
| 45 | + if already or ind["id"] == "population": | |
| 46 | + info["applied"] = False | |
| 47 | + info["note"] = "Indicator is already per-capita / a share; left unchanged." | |
| 48 | + else: | |
| 49 | + pop = _population_by_year(snap, s["country"]["id"]) | |
| 50 | + for v in values: | |
| 51 | + p = pop.get(v["year"]) if v.get("year") is not None else None | |
| 52 | + v["value"] = (v["value"] / p) if (v.get("value") is not None and p) else None | |
| 53 | + info["applied"] = True | |
| 54 | + info["unit"] = f"{s.get('unit') or ind.get('unit')} per person" | |
| 55 | + s["unit"] = info["unit"] | |
| 56 | + elif mode == "index100": | |
| 57 | + base = None | |
| 58 | + for v in values: | |
| 59 | + if v.get("value") is not None and not v.get("is_forecast") and (base_year is None or v["year"] >= base_year): | |
| 60 | + base = v["value"] | |
| 61 | + info["base_year"] = v["year"] | |
| 62 | + break | |
| 63 | + if base: | |
| 64 | + for v in values: | |
| 65 | + v["value"] = (v["value"] / base * 100.0) if v.get("value") is not None else None | |
| 66 | + info["applied"] = True | |
| 67 | + s["unit"] = f"index ({info['base_year']} = 100)" | |
| 68 | + else: | |
| 69 | + info["applied"] = False | |
| 70 | + elif mode == "pct": | |
| 71 | + prev: float | None = None | |
| 72 | + for v in values: | |
| 73 | + cur = v.get("value") | |
| 74 | + v["value"] = ((cur - prev) / abs(prev) * 100.0) if (cur is not None and prev not in (None, 0)) else None | |
| 75 | + prev = cur | |
| 76 | + info["applied"] = True | |
| 77 | + s["unit"] = "% change vs previous period" | |
| 78 | + else: | |
| 79 | + info["applied"] = False | |
| 80 | + s["transform"] = info | |
| 81 | + if mode != "absolute": | |
| 82 | + s["stats"] = series_stats(values) | |
| 83 | + return s | |
| 84 | + | |
| 85 | + | |
| 86 | +@router.get("", response_model=schemas.CompareResponse, summary="Compare countries on indicators (absolute, per-capita, index100, pct)") | |
| 87 | +def compare( | |
| 88 | + countries: str = Query(..., description="Comma-separated ISO3/slugs (2–8)"), | |
| 89 | + indicators: str = Query(..., description="Comma-separated indicator slugs (1–8)"), | |
| 90 | + from_: int | None = Query(None, alias="from"), | |
| 91 | + to: int | None = Query(None), | |
| 92 | + mode: str = Query("absolute", pattern="^(absolute|per-capita|index100|pct)$"), | |
| 93 | + include_forecast: bool = Query(True), | |
| 94 | + snap: Snapshot = Depends(get_snapshot), | |
| 95 | +) -> dict[str, Any]: | |
| 96 | + cs = [resolve_country(snap, c) for c in parse_csv(countries, limit=8)] | |
| 97 | + inds = [resolve_indicator(snap, i) for i in parse_csv(indicators, limit=8)] | |
| 98 | + if not cs or not inds: | |
| 99 | + raise bad_request("Provide at least one country and one indicator.") | |
| 100 | + series = [] | |
| 101 | + for ind in inds: | |
| 102 | + for c in cs: | |
| 103 | + s = fetch_series(snap, c, ind, year_from=from_, year_to=to, freq="A" if ind.get("frequency") == "A" else None, | |
| 104 | + include_forecast=include_forecast) | |
| 105 | + series.append(transform_series(snap, s, ind, mode, from_)) | |
| 106 | + return {"meta": meta_block(snap), "mode": mode, "base_year": from_, "countries": [country_card(c) for c in cs], | |
| 107 | + "indicators": [indicator_card(i) for i in inds], "series": series} | |
| 108 | + | |
| 109 | + | |
| 110 | +@router.get("/snapshot", response_model=schemas.CompareSnapshotResponse, summary="Latest values table for a topic") | |
| 111 | +def compare_snapshot( | |
| 112 | + countries: str = Query(..., description="Comma-separated ISO3/slugs (1–8)"), | |
| 113 | + topic: str | None = Query(None, description="Topic id; default = headline indicators"), | |
| 114 | + indicators: str | None = Query(None, description="Explicit indicator list (overrides topic)"), | |
| 115 | + snap: Snapshot = Depends(get_snapshot), | |
| 116 | +) -> dict[str, Any]: | |
| 117 | + cs = [resolve_country(snap, c) for c in parse_csv(countries, limit=8)] | |
| 118 | + t = None | |
| 119 | + if indicators: | |
| 120 | + ids = [resolve_indicator(snap, i)["id"] for i in parse_csv(indicators, limit=40)] | |
| 121 | + elif topic: | |
| 122 | + t = resolve_topic(topic) | |
| 123 | + ids = [s for s in t["indicators"] if s in snap.indicators()] | |
| 124 | + else: | |
| 125 | + ids = [s for s in registry_topics()["headline"] if s in snap.indicators()] | |
| 126 | + rows_by = {} | |
| 127 | + for r in latest_rows(snap, country_ids=[c["id"] for c in cs], indicator_ids=ids): | |
| 128 | + rows_by[(r["country_id"], r["indicator_id"])] = r | |
| 129 | + rows = [] | |
| 130 | + for iid in ids: | |
| 131 | + ind = merged_indicator(snap.indicators()[iid]) | |
| 132 | + cells = {} | |
| 133 | + for c in cs: | |
| 134 | + r = rows_by.get((c["id"], iid)) | |
| 135 | + cells[c["id"]] = metric_from_latest(snap, r, ind, c) if r and r.get("value") is not None else empty_metric(ind) | |
| 136 | + best = None | |
| 137 | + vals = [(cid, m["value"]) for cid, m in cells.items() if m.get("value") is not None] | |
| 138 | + if vals and ind.get("higher_is_better") is not None: | |
| 139 | + best = (max if ind["higher_is_better"] else min)(vals, key=lambda kv: kv[1])[0] | |
| 140 | + rows.append({"indicator": indicator_card(ind), "values": cells, "best": best}) | |
| 141 | + return {"meta": meta_block(snap), "topic": {k: t.get(k) for k in ("id", "name", "short", "blurb")} if t else None, | |
| 142 | + "countries": [country_card(c) for c in cs], "rows": rows} | |
added
src/countryatlas/api/routers/countries.py
+425 −0
@@ -0,0 +1,425 @@ | ||
| 1 | +"""/countries — list, country overview, topic pages, series, changes, events, similar, insights, DNA.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends, Query | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import ( | |
| 10 | + clean_float, | |
| 11 | + country_card, | |
| 12 | + empty_metric, | |
| 13 | + group_card, | |
| 14 | + indicator_card, | |
| 15 | + indicator_meta, | |
| 16 | + latest_rows, | |
| 17 | + merged_indicator, | |
| 18 | + meta_block, | |
| 19 | + metric_from_latest, | |
| 20 | + observation_value, | |
| 21 | + resolve_country, | |
| 22 | + resolve_group, | |
| 23 | + resolve_indicator, | |
| 24 | + resolve_topic, | |
| 25 | + series_stats, | |
| 26 | + sparklines, | |
| 27 | +) | |
| 28 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 29 | +from countryatlas.api.errors import bad_request | |
| 30 | +from countryatlas.api.formatting import format_value | |
| 31 | +from countryatlas.api.provenance import _iso, build_provenance, provenance_from_row | |
| 32 | +from countryatlas.registry import topics as registry_topics | |
| 33 | + | |
| 34 | +router = APIRouter(prefix="/countries", tags=["countries"]) | |
| 35 | + | |
| 36 | +HEADLINE_PIVOT = ("population", "gdp", "gdp-per-capita") | |
| 37 | + | |
| 38 | + | |
| 39 | +def _country_full(c: dict[str, Any]) -> dict[str, Any]: | |
| 40 | + out = country_card(c) | |
| 41 | + for k in ("official_name", "iso3", "iso_numeric", "capital", "continent", "subregion", "currency_code", "currency_name", | |
| 42 | + "area_km2", "latitude", "longitude", "un_member", "independent", "landlocked", "borders", "languages", | |
| 43 | + "demonym", "status"): | |
| 44 | + out[k] = c.get(k) | |
| 45 | + return out | |
| 46 | + | |
| 47 | + | |
| 48 | +@router.get("", response_model=schemas.CountriesResponse, summary="List countries") | |
| 49 | +def list_countries( | |
| 50 | + region: str | None = Query(None, description="Group id or slug (e.g. ecs, europe-central-asia, oecd, g7)"), | |
| 51 | + income: str | None = Query(None, description="Income group id: HIC | UMC | LMC | LIC (or slug high-income…)"), | |
| 52 | + q: str | None = Query(None, description="Substring on name / slug / ISO codes"), | |
| 53 | + sort: str = Query("name", pattern="^(name|population|gdp|gdp_per_capita|coverage)$"), | |
| 54 | + kind: str | None = Query(None, description="country | territory | all (default: countries and territories)"), | |
| 55 | + limit: int = Query(500, ge=1, le=1000), | |
| 56 | + offset: int = Query(0, ge=0), | |
| 57 | + snap: Snapshot = Depends(get_snapshot), | |
| 58 | +) -> dict[str, Any]: | |
| 59 | + where, params = [], [] | |
| 60 | + if kind and kind != "all": | |
| 61 | + where.append("(c.kind = ? OR c.status = ?)") | |
| 62 | + params += [kind, kind] | |
| 63 | + else: | |
| 64 | + where.append("coalesce(c.kind, 'country') <> 'aggregate'") | |
| 65 | + if region: | |
| 66 | + g = resolve_group(snap, region) | |
| 67 | + members = snap.group_members(g["id"]) | |
| 68 | + if g.get("kind") == "region" and g.get("wb_code") and not members: | |
| 69 | + where.append("c.region_wb = ?") | |
| 70 | + params.append(g["wb_code"]) | |
| 71 | + else: | |
| 72 | + where.append(f"c.id IN ({','.join('?' * len(members))})" if members else "FALSE") | |
| 73 | + params += members | |
| 74 | + if income: | |
| 75 | + inc = income.strip() | |
| 76 | + g = snap.groups().get(inc.lower()) or snap.groups_by_slug().get(inc.lower()) | |
| 77 | + code = (g or {}).get("wb_code") or inc.upper() | |
| 78 | + where.append("upper(c.income_group) = ?") | |
| 79 | + params.append(code.upper()) | |
| 80 | + if q: | |
| 81 | + like = f"%{q.strip().lower()}%" | |
| 82 | + where.append("(lower(c.short_name) LIKE ? OR lower(c.official_name) LIKE ? OR lower(c.slug) LIKE ? OR lower(c.id) LIKE ? OR lower(c.iso2) = ?)") | |
| 83 | + params += [like, like, like, like, q.strip().lower()] | |
| 84 | + order = { | |
| 85 | + "name": "c.short_name ASC", | |
| 86 | + "population": "population_latest DESC NULLS LAST, c.short_name", | |
| 87 | + "gdp": "gdp_latest DESC NULLS LAST, c.short_name", | |
| 88 | + "gdp_per_capita": "gdp_per_capita_latest DESC NULLS LAST, c.short_name", | |
| 89 | + "coverage": "coverage_pct DESC NULLS LAST, c.short_name", | |
| 90 | + }[sort] | |
| 91 | + sql = f""" | |
| 92 | + SELECT c.id, c.iso2, c.slug, c.short_name, c.flag_emoji, c.region_wb, c.region_wb_name, c.income_group, c.income_group_name, | |
| 93 | + c.kind, c.capital, c.continent, c.subregion, | |
| 94 | + cov.coverage_pct, cov.n_indicators, | |
| 95 | + max(CASE WHEN l.indicator_id = 'population' THEN l.value END) AS population_latest, | |
| 96 | + max(CASE WHEN l.indicator_id = 'population' THEN l.year END) AS population_year, | |
| 97 | + max(CASE WHEN l.indicator_id = 'gdp' THEN l.value END) AS gdp_latest, | |
| 98 | + max(CASE WHEN l.indicator_id = 'gdp' THEN l.year END) AS gdp_year, | |
| 99 | + max(CASE WHEN l.indicator_id = 'gdp-per-capita' THEN l.value END) AS gdp_per_capita_latest, | |
| 100 | + max(CASE WHEN l.indicator_id = 'gdp-per-capita' THEN l.year END) AS gdp_per_capita_year | |
| 101 | + FROM countries c | |
| 102 | + LEFT JOIN coverage cov ON cov.country_id = c.id | |
| 103 | + LEFT JOIN latest l ON l.country_id = c.id AND l.indicator_id IN ('population', 'gdp', 'gdp-per-capita') | |
| 104 | + WHERE {' AND '.join(where)} | |
| 105 | + GROUP BY ALL | |
| 106 | + ORDER BY {order} | |
| 107 | + """ | |
| 108 | + rows = snap.query(sql, params) | |
| 109 | + items = [] | |
| 110 | + for r in rows[offset: offset + limit]: | |
| 111 | + item = country_card(r) | |
| 112 | + item.update({ | |
| 113 | + "capital": r.get("capital"), "continent": r.get("continent"), "subregion": r.get("subregion"), | |
| 114 | + "population_latest": clean_float(r.get("population_latest")), "population_year": r.get("population_year"), | |
| 115 | + "gdp_latest": clean_float(r.get("gdp_latest")), "gdp_year": r.get("gdp_year"), | |
| 116 | + "gdp_per_capita_latest": clean_float(r.get("gdp_per_capita_latest")), "gdp_per_capita_year": r.get("gdp_per_capita_year"), | |
| 117 | + "coverage_pct": clean_float(r.get("coverage_pct")), "n_indicators": r.get("n_indicators"), | |
| 118 | + }) | |
| 119 | + items.append(item) | |
| 120 | + return {"meta": meta_block(snap), "n": len(rows), "filters": {"region": region, "income": income, "q": q, "sort": sort}, | |
| 121 | + "items": items} | |
| 122 | + | |
| 123 | + | |
| 124 | +def _groups_of(snap: Snapshot, country_id: str) -> list[dict[str, Any]]: | |
| 125 | + out = [] | |
| 126 | + for g in snap.groups().values(): | |
| 127 | + if country_id in snap.group_members(g["id"]): | |
| 128 | + out.append(group_card(g)) | |
| 129 | + order = {"world": 0, "region": 1, "continent": 2, "income": 3, "org": 4} | |
| 130 | + out.sort(key=lambda g: (order.get(g.get("kind") or "", 9), g.get("name") or "")) | |
| 131 | + return out | |
| 132 | + | |
| 133 | + | |
| 134 | +def _freshness(snap: Snapshot, country_id: str) -> dict[str, Any]: | |
| 135 | + row = snap.one( | |
| 136 | + """SELECT max(o.source_updated_at) AS su, max(o.retrieved_at) AS ra | |
| 137 | + FROM latest l JOIN observations o ON o.country_id = l.country_id AND o.indicator_id = l.indicator_id | |
| 138 | + AND o.period = l.period AND o.frequency = l.frequency | |
| 139 | + WHERE l.country_id = ?""", | |
| 140 | + [country_id], | |
| 141 | + ) or {} | |
| 142 | + return {"source_updated_at": _iso(row.get("su")), "retrieved_at": _iso(row.get("ra")), "built_at": snap.built_at} | |
| 143 | + | |
| 144 | + | |
| 145 | +def _coverage(snap: Snapshot, country_id: str) -> dict[str, Any] | None: | |
| 146 | + row = snap.one("SELECT * FROM coverage WHERE country_id = ?", [country_id]) | |
| 147 | + if row is None: | |
| 148 | + return None | |
| 149 | + row["updated_at"] = _iso(row.get("updated_at")) | |
| 150 | + row.pop("country_id", None) | |
| 151 | + return row | |
| 152 | + | |
| 153 | + | |
| 154 | +@router.get("/{id}", response_model=schemas.CountryResponse, summary="Country overview (header + headline metrics)") | |
| 155 | +def get_country(id: str, snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 156 | + c = resolve_country(snap, id) | |
| 157 | + t = registry_topics() | |
| 158 | + headline_ids = list(t["headline"]) | |
| 159 | + rows = {r["indicator_id"]: r for r in latest_rows(snap, country_id=c["id"], indicator_ids=[s for s in headline_ids if s in snap.indicators()])} | |
| 160 | + sparks = sparklines(snap, c["id"], [i for i in headline_ids if i in rows]) | |
| 161 | + headline = [] | |
| 162 | + for iid in headline_ids: | |
| 163 | + ind = indicator_meta(snap, iid) | |
| 164 | + if ind is None: | |
| 165 | + continue | |
| 166 | + if iid in rows: | |
| 167 | + headline.append(metric_from_latest(snap, rows[iid], ind, c, sparks.get(iid))) | |
| 168 | + else: | |
| 169 | + headline.append(empty_metric(ind)) | |
| 170 | + with_data = {r[0] for r in snap.query_rows("SELECT indicator_id FROM latest WHERE country_id = ? AND value IS NOT NULL", [c["id"]])} | |
| 171 | + topics = [] | |
| 172 | + for topic in sorted(t["topics"], key=lambda x: x.get("order", 99)): | |
| 173 | + ids = topic["indicators"] | |
| 174 | + topics.append({"id": topic["id"], "name": topic["name"], "short": topic.get("short"), "order": topic.get("order"), | |
| 175 | + "blurb": topic.get("blurb"), "n_indicators": len(ids), "n_with_data": sum(1 for i in ids if i in with_data)}) | |
| 176 | + neighbours = [country_card(snap.countries()[b]) for b in (c.get("borders") or []) if b in snap.countries()] | |
| 177 | + return { | |
| 178 | + "meta": meta_block(snap), | |
| 179 | + "country": _country_full(c), | |
| 180 | + "groups": _groups_of(snap, c["id"]), | |
| 181 | + "coverage": _coverage(snap, c["id"]), | |
| 182 | + "freshness": _freshness(snap, c["id"]), | |
| 183 | + "headline": headline, | |
| 184 | + "topics": topics, | |
| 185 | + "neighbours": neighbours, | |
| 186 | + } | |
| 187 | + | |
| 188 | + | |
| 189 | +@router.get("/{id}/topics/{topic}", response_model=schemas.CountryTopicResponse, summary="All indicators of a topic for a country") | |
| 190 | +def get_country_topic(id: str, topic: str, snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 191 | + c = resolve_country(snap, id) | |
| 192 | + t = resolve_topic(topic) | |
| 193 | + ids = list(t["indicators"]) # registry order; indicators absent from the snapshot are still listed with has_data=false | |
| 194 | + in_db = [s for s in ids if s in snap.indicators()] | |
| 195 | + rows = {r["indicator_id"]: r for r in latest_rows(snap, country_id=c["id"], indicator_ids=in_db)} | |
| 196 | + sparks = sparklines(snap, c["id"], [i for i in in_db if i in rows]) | |
| 197 | + blocks: dict[str, list[dict[str, Any]]] = {} | |
| 198 | + order: list[str] = [] | |
| 199 | + n_with = 0 | |
| 200 | + for iid in ids: | |
| 201 | + ind = indicator_meta(snap, iid) | |
| 202 | + if ind is None: | |
| 203 | + continue | |
| 204 | + sub = ind.get("subtopic") or "Other" | |
| 205 | + if sub not in blocks: | |
| 206 | + blocks[sub] = [] | |
| 207 | + order.append(sub) | |
| 208 | + if iid in rows and rows[iid].get("value") is not None: | |
| 209 | + blocks[sub].append(metric_from_latest(snap, rows[iid], ind, c, sparks.get(iid))) | |
| 210 | + n_with += 1 | |
| 211 | + else: | |
| 212 | + blocks[sub].append(empty_metric(ind)) | |
| 213 | + return { | |
| 214 | + "meta": meta_block(snap), | |
| 215 | + "country": country_card(c), | |
| 216 | + "topic": {k: t.get(k) for k in ("id", "name", "short", "order", "blurb")}, | |
| 217 | + "n_with_data": n_with, | |
| 218 | + "n_indicators": len(ids), | |
| 219 | + "subtopics": [{"subtopic": s, "indicators": blocks[s]} for s in order], | |
| 220 | + } | |
| 221 | + | |
| 222 | + | |
| 223 | +def fetch_series(snap: Snapshot, c: dict[str, Any], ind: dict[str, Any], *, year_from: int | None = None, year_to: int | None = None, | |
| 224 | + freq: str | None = None, include_forecast: bool = True, include_alt: bool = False) -> dict[str, Any]: | |
| 225 | + where = ["country_id = ?", "indicator_id = ?"] | |
| 226 | + params: list[Any] = [c["id"], ind["id"]] | |
| 227 | + if freq: | |
| 228 | + where.append("frequency = ?") | |
| 229 | + params.append(freq.upper()) | |
| 230 | + if year_from is not None: | |
| 231 | + where.append("year >= ?") | |
| 232 | + params.append(year_from) | |
| 233 | + if year_to is not None: | |
| 234 | + where.append("year <= ?") | |
| 235 | + params.append(year_to) | |
| 236 | + if not include_forecast: | |
| 237 | + where.append("NOT is_forecast") | |
| 238 | + sql_where = " AND ".join(where) | |
| 239 | + rows = snap.query(f"SELECT * FROM observations WHERE {sql_where} ORDER BY frequency, period", params) | |
| 240 | + if not freq and rows: | |
| 241 | + # keep one frequency: prefer the indicator's own frequency, else the most populated | |
| 242 | + freqs: dict[str, int] = {} | |
| 243 | + for r in rows: | |
| 244 | + freqs[r["frequency"]] = freqs.get(r["frequency"], 0) + 1 | |
| 245 | + pref = ind.get("frequency") if ind.get("frequency") in freqs else max(freqs, key=lambda k: freqs[k]) | |
| 246 | + rows = [r for r in rows if r["frequency"] == pref] | |
| 247 | + values = [observation_value(snap, r, ind, c.get("iso2")) for r in rows] | |
| 248 | + # dominant source + all sources used | |
| 249 | + by_source: dict[tuple, int] = {} | |
| 250 | + prov_by_source: dict[tuple, dict[str, Any]] = {} | |
| 251 | + for r in rows: | |
| 252 | + k = (r.get("source_id"), r.get("source_dataset"), r.get("source_series_code")) | |
| 253 | + by_source[k] = by_source.get(k, 0) + 1 | |
| 254 | + if k not in prov_by_source: | |
| 255 | + prov_by_source[k] = provenance_from_row(snap, r, ind["id"], c.get("iso2")) | |
| 256 | + dominant = max(by_source, key=lambda k: by_source[k]) if by_source else None | |
| 257 | + sources = [] | |
| 258 | + for k, n in sorted(by_source.items(), key=lambda kv: -kv[1]): | |
| 259 | + p = dict(prov_by_source[k]) | |
| 260 | + p["n_values"] = n | |
| 261 | + sources.append(p) | |
| 262 | + alternatives = None | |
| 263 | + if include_alt: | |
| 264 | + alt_rows = snap.query(f"SELECT * FROM observations_alt WHERE {sql_where} ORDER BY source_id, period", params) | |
| 265 | + alternatives = [observation_value(snap, r, ind, c.get("iso2")) for r in alt_rows] | |
| 266 | + freq_used = rows[0]["frequency"] if rows else (freq.upper() if freq else ind.get("frequency")) | |
| 267 | + return { | |
| 268 | + "indicator": indicator_card(ind), | |
| 269 | + "country": country_card(c), | |
| 270 | + "unit": rows[0].get("unit") if rows else ind.get("unit"), | |
| 271 | + "frequency": freq_used, | |
| 272 | + "values": values, | |
| 273 | + "alternatives": alternatives, | |
| 274 | + "provenance": prov_by_source.get(dominant) if dominant else None, | |
| 275 | + "sources": sources, | |
| 276 | + "stats": series_stats(values), | |
| 277 | + } | |
| 278 | + | |
| 279 | + | |
| 280 | +@router.get("/{id}/series/{indicator}", response_model=schemas.SeriesResponse, summary="Full history of one indicator for a country") | |
| 281 | +def get_country_series( | |
| 282 | + id: str, | |
| 283 | + indicator: str, | |
| 284 | + from_: int | None = Query(None, alias="from", ge=1800, le=2100), | |
| 285 | + to: int | None = Query(None, ge=1800, le=2100), | |
| 286 | + freq: str | None = Query(None, pattern="^(?i)(A|Q|M)$"), | |
| 287 | + include_forecast: bool = Query(True), | |
| 288 | + include_alt: bool = Query(False, description="Also return values from lower-priority sources (observations_alt)"), | |
| 289 | + snap: Snapshot = Depends(get_snapshot), | |
| 290 | +) -> dict[str, Any]: | |
| 291 | + c = resolve_country(snap, id) | |
| 292 | + ind = resolve_indicator(snap, indicator) | |
| 293 | + if from_ is not None and to is not None and from_ > to: | |
| 294 | + raise bad_request("`from` must be <= `to`.") | |
| 295 | + out = fetch_series(snap, c, ind, year_from=from_, year_to=to, freq=freq, include_forecast=include_forecast, include_alt=include_alt) | |
| 296 | + out["meta"] = meta_block(snap) | |
| 297 | + return out | |
| 298 | + | |
| 299 | + | |
| 300 | +def change_item(snap: Snapshot, r: dict[str, Any], with_country: bool = True) -> dict[str, Any]: | |
| 301 | + ind_row = snap.indicators().get(r["indicator_id"]) | |
| 302 | + ind = merged_indicator(ind_row) if ind_row else {"id": r["indicator_id"]} | |
| 303 | + c = snap.countries().get(r["country_id"]) | |
| 304 | + prov = None | |
| 305 | + if r.get("source_id") is not None: | |
| 306 | + prov = build_provenance(snap, ind["id"], r.get("source_id"), r.get("source_dataset"), r.get("source_series_code"), | |
| 307 | + r.get("retrieved_at"), r.get("source_updated_at"), (c or {}).get("iso2")) | |
| 308 | + return { | |
| 309 | + "id": r.get("id"), | |
| 310 | + "country": country_card(c) if (c and with_country) else None, | |
| 311 | + "indicator": indicator_card(ind) if ind_row else {"id": r["indicator_id"], "slug": r["indicator_id"]}, | |
| 312 | + "kind": r.get("kind"), | |
| 313 | + "period": r.get("period"), | |
| 314 | + "year": r.get("year"), | |
| 315 | + "value": clean_float(r.get("value")), | |
| 316 | + "ref_value": clean_float(r.get("ref_value")), | |
| 317 | + "delta": clean_float(r.get("delta")), | |
| 318 | + "delta_pct": clean_float(r.get("delta_pct")), | |
| 319 | + "window_years": r.get("window_years"), | |
| 320 | + "severity": clean_float(r.get("severity")), | |
| 321 | + "headline": r.get("headline"), | |
| 322 | + "detail": r.get("detail"), | |
| 323 | + "detected_at": _iso(r.get("detected_at")), | |
| 324 | + "formatted": format_value(clean_float(r.get("value")), ind) if ind_row else None, | |
| 325 | + "provenance": prov, | |
| 326 | + } | |
| 327 | + | |
| 328 | + | |
| 329 | +CHANGES_SQL = """ | |
| 330 | +SELECT x.*, o.source_id, o.source_dataset, o.source_series_code, o.retrieved_at, o.source_updated_at | |
| 331 | +FROM {table} x | |
| 332 | +LEFT JOIN observations o ON o.country_id = x.country_id AND o.indicator_id = x.indicator_id AND o.period = x.period | |
| 333 | + AND o.frequency = 'A' | |
| 334 | +""" | |
| 335 | + | |
| 336 | + | |
| 337 | +@router.get("/{id}/changes", response_model=schemas.ChangesResponse, summary="What changed recently for this country") | |
| 338 | +def get_country_changes(id: str, limit: int = Query(50, ge=1, le=500), kind: str | None = None, | |
| 339 | + snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 340 | + c = resolve_country(snap, id) | |
| 341 | + where, params = ["x.country_id = ?"], [c["id"]] | |
| 342 | + if kind: | |
| 343 | + where.append("x.kind = ?") | |
| 344 | + params.append(kind) | |
| 345 | + rows = snap.query(CHANGES_SQL.format(table="changes") + f" WHERE {' AND '.join(where)} ORDER BY x.severity DESC NULLS LAST, x.period DESC LIMIT ?", | |
| 346 | + [*params, limit]) | |
| 347 | + return {"meta": meta_block(snap), "n": len(rows), "items": [change_item(snap, r, with_country=False) for r in rows]} | |
| 348 | + | |
| 349 | + | |
| 350 | +@router.get("/{id}/events", response_model=schemas.ChangesResponse, summary="Timeline of notable events (whole history)") | |
| 351 | +def get_country_events(id: str, limit: int = Query(100, ge=1, le=1000), kind: str | None = None, indicator: str | None = None, | |
| 352 | + snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 353 | + c = resolve_country(snap, id) | |
| 354 | + where, params = ["x.country_id = ?"], [c["id"]] | |
| 355 | + if kind: | |
| 356 | + where.append("x.kind = ?") | |
| 357 | + params.append(kind) | |
| 358 | + if indicator: | |
| 359 | + where.append("x.indicator_id = ?") | |
| 360 | + params.append(resolve_indicator(snap, indicator)["id"]) | |
| 361 | + rows = snap.query(CHANGES_SQL.format(table="events") + f" WHERE {' AND '.join(where)} ORDER BY x.period DESC, x.severity DESC NULLS LAST LIMIT ?", | |
| 362 | + [*params, limit]) | |
| 363 | + return {"meta": meta_block(snap), "n": len(rows), "items": [change_item(snap, r, with_country=False) for r in rows]} | |
| 364 | + | |
| 365 | + | |
| 366 | +@router.get("/{id}/similar", response_model=schemas.SimilarResponse, summary="Most similar countries (explainable)") | |
| 367 | +def get_country_similar(id: str, mode: str = Query("overall"), limit: int = Query(12, ge=1, le=50), | |
| 368 | + snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 369 | + c = resolve_country(snap, id) | |
| 370 | + modes = [r[0] for r in snap.query_rows("SELECT DISTINCT mode FROM similarity ORDER BY mode")] | |
| 371 | + rows = snap.query("SELECT * FROM similarity WHERE country_id = ? AND mode = ? ORDER BY rank, score DESC LIMIT ?", [c["id"], mode, limit]) | |
| 372 | + peers = [] | |
| 373 | + for r in rows: | |
| 374 | + p = snap.countries().get(r["peer_id"]) | |
| 375 | + if p is None: | |
| 376 | + continue | |
| 377 | + peers.append({"country": country_card(p), "score": clean_float(r.get("score")), "rank": r.get("rank"), | |
| 378 | + "contributions": r.get("contributions")}) | |
| 379 | + return {"meta": meta_block(snap), "country": country_card(c), "mode": mode, "modes": modes or ["overall"], "peers": peers} | |
| 380 | + | |
| 381 | + | |
| 382 | +@router.get("/{id}/insights", response_model=schemas.InsightsResponse, summary="Computed insights (templated, no LLM)") | |
| 383 | +def get_country_insights(id: str, snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 384 | + c = resolve_country(snap, id) | |
| 385 | + rows = snap.query("SELECT * FROM insights WHERE country_id = ? ORDER BY computed_at DESC NULLS LAST, id", [c["id"]]) | |
| 386 | + items = [] | |
| 387 | + if rows: | |
| 388 | + all_ids = sorted({i for r in rows for i in (r.get("indicators") or [])}) | |
| 389 | + lat = {r["indicator_id"]: r for r in latest_rows(snap, country_id=c["id"], indicator_ids=all_ids)} if all_ids else {} | |
| 390 | + for r in rows: | |
| 391 | + provs = [ | |
| 392 | + build_provenance(snap, i, lat[i].get("source_id"), lat[i].get("source_dataset"), lat[i].get("source_series_code"), | |
| 393 | + lat[i].get("retrieved_at"), lat[i].get("source_updated_at"), c.get("iso2")) | |
| 394 | + for i in (r.get("indicators") or []) if i in lat | |
| 395 | + ] | |
| 396 | + items.append({"id": r.get("id"), "template_id": r.get("template_id"), "text": r.get("text") or "", "values": r.get("values"), | |
| 397 | + "indicators": list(r.get("indicators") or []), "computed_at": _iso(r.get("computed_at")), "provenance": provs}) | |
| 398 | + return {"meta": meta_block(snap), "country": country_card(c), "items": items} | |
| 399 | + | |
| 400 | + | |
| 401 | +DNA_LABELS = { | |
| 402 | + "income": ("Income", "gdp-per-capita-ppp"), "demographics": ("Demographics", "median-age"), | |
| 403 | + "urbanization": ("Urbanisation", "urban-population-share"), "trade": ("Trade openness", "trade-pct-gdp"), | |
| 404 | + "energy": ("Energy use", "energy-use-per-capita"), "emissions": ("Emissions", "co2-per-capita"), | |
| 405 | + "innovation": ("Innovation", "rd-expenditure-pct-gdp"), "education": ("Education", "tertiary-enrollment"), | |
| 406 | + "public_spending": ("Public spending", "government-expenditure-pct-gdp"), | |
| 407 | +} | |
| 408 | + | |
| 409 | + | |
| 410 | +@router.get("/{id}/dna", response_model=schemas.DNAResponse, summary="Country DNA (9 percentile dimensions)") | |
| 411 | +def get_country_dna(id: str, snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 412 | + import json | |
| 413 | + | |
| 414 | + c = resolve_country(snap, id) | |
| 415 | + row = snap.one("SELECT * FROM country_dna WHERE country_id = ?", [c["id"]]) | |
| 416 | + dims: dict[str, float | None] = {} | |
| 417 | + if row and row.get("dims") is not None: | |
| 418 | + raw = row["dims"] | |
| 419 | + if isinstance(raw, str): | |
| 420 | + raw = json.loads(raw) | |
| 421 | + dims = {k: clean_float(v) for k, v in dict(raw).items()} | |
| 422 | + dimensions = [{"id": k, "label": DNA_LABELS.get(k, (k.replace("_", " ").title(), None))[0], | |
| 423 | + "indicator": DNA_LABELS.get(k, (None, None))[1], "value": v} for k, v in dims.items()] | |
| 424 | + return {"meta": meta_block(snap), "country": country_card(c), "dims": dims, "year_ref": row.get("year_ref") if row else None, | |
| 425 | + "dimensions": dimensions} | |
added
src/countryatlas/api/routers/download.py
+144 −0
@@ -0,0 +1,144 @@ | ||
| 1 | +"""Downloads (CSV streamed / JSON) with provenance columns: indicator, country, compare bundle.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import csv | |
| 5 | +import io | |
| 6 | +from collections.abc import Iterable, Iterator | |
| 7 | +from typing import Any | |
| 8 | + | |
| 9 | +from fastapi import APIRouter, Depends, Query | |
| 10 | +from fastapi.responses import StreamingResponse | |
| 11 | + | |
| 12 | +from countryatlas.api.common import ( | |
| 13 | + country_card, | |
| 14 | + indicator_card, | |
| 15 | + meta_block, | |
| 16 | + parse_csv, | |
| 17 | + resolve_country, | |
| 18 | + resolve_indicator, | |
| 19 | +) | |
| 20 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 21 | +from countryatlas.api.errors import bad_request | |
| 22 | +from countryatlas.api.provenance import _iso, provenance_from_row | |
| 23 | + | |
| 24 | +router = APIRouter(tags=["download"]) | |
| 25 | + | |
| 26 | +COLUMNS = ["country_id", "country_name", "indicator_id", "indicator_name", "period", "year", "frequency", "value", "unit", | |
| 27 | + "is_estimate", "is_forecast", "status", "source", "source_name", "dataset", "series_code", "retrieved_at", | |
| 28 | + "source_updated_at", "url", "licence"] | |
| 29 | + | |
| 30 | +OBS_SQL = """ | |
| 31 | +SELECT o.country_id, c.short_name AS country_name, c.iso2, o.indicator_id, i.name AS indicator_name, o.period, o.year, o.frequency, | |
| 32 | + o.value, o.unit, o.is_estimate, o.is_forecast, o.status, o.source_id, o.source_dataset, o.source_series_code, o.retrieved_at, | |
| 33 | + o.source_updated_at | |
| 34 | +FROM observations o | |
| 35 | +LEFT JOIN countries c ON c.id = o.country_id | |
| 36 | +LEFT JOIN indicators i ON i.id = o.indicator_id | |
| 37 | +WHERE {where} | |
| 38 | +ORDER BY o.indicator_id, o.country_id, o.frequency, o.period | |
| 39 | +""" | |
| 40 | + | |
| 41 | + | |
| 42 | +def _rows(snap: Snapshot, where: str, params: list[Any], include_forecast: bool = True) -> Iterator[dict[str, Any]]: | |
| 43 | + if not include_forecast: | |
| 44 | + where += " AND NOT o.is_forecast" | |
| 45 | + for r in snap.query(OBS_SQL.format(where=where), params): | |
| 46 | + p = provenance_from_row(snap, r, r["indicator_id"], r.get("iso2")) | |
| 47 | + yield { | |
| 48 | + "country_id": r["country_id"], "country_name": r["country_name"], "indicator_id": r["indicator_id"], "indicator_name": r["indicator_name"], | |
| 49 | + "period": r["period"].isoformat() if r.get("period") else None, "year": r["year"], "frequency": r["frequency"], "value": r["value"], | |
| 50 | + "unit": r["unit"], "is_estimate": r["is_estimate"], "is_forecast": r["is_forecast"], "status": r["status"], | |
| 51 | + "source": p["source"], "source_name": p["source_name"], "dataset": p["dataset"], "series_code": p["series_code"], | |
| 52 | + "retrieved_at": p["retrieved_at"], "source_updated_at": p["source_updated_at"], "url": p["url"], "licence": p["licence"], | |
| 53 | + } | |
| 54 | + | |
| 55 | + | |
| 56 | +def _csv_stream(rows: Iterable[dict[str, Any]], header_comment: str | None = None) -> Iterator[bytes]: | |
| 57 | + buf = io.StringIO() | |
| 58 | + w = csv.DictWriter(buf, fieldnames=COLUMNS, extrasaction="ignore") | |
| 59 | + if header_comment: | |
| 60 | + buf.write(f"# {header_comment}\n") | |
| 61 | + w.writeheader() | |
| 62 | + yield buf.getvalue().encode() | |
| 63 | + buf.seek(0) | |
| 64 | + buf.truncate() | |
| 65 | + for n, r in enumerate(rows, 1): | |
| 66 | + w.writerow(r) | |
| 67 | + if n % 500 == 0: | |
| 68 | + yield buf.getvalue().encode() | |
| 69 | + buf.seek(0) | |
| 70 | + buf.truncate() | |
| 71 | + if buf.tell(): | |
| 72 | + yield buf.getvalue().encode() | |
| 73 | + | |
| 74 | + | |
| 75 | +def _csv_response(rows: Iterable[dict[str, Any]], filename: str, snap: Snapshot) -> StreamingResponse: | |
| 76 | + comment = f"CountryAtlas export · run {snap.run_id} · built {snap.built_at} · https://www.countryatlas.co · see /api/v1/methodology for licences" | |
| 77 | + return StreamingResponse(_csv_stream(rows, comment), media_type="text/csv; charset=utf-8", | |
| 78 | + headers={"Content-Disposition": f'attachment; filename="{filename}"'}) | |
| 79 | + | |
| 80 | + | |
| 81 | +def _json_payload(snap: Snapshot, rows: Iterable[dict[str, Any]], **extra: Any) -> dict[str, Any]: | |
| 82 | + data = list(rows) | |
| 83 | + return {"meta": meta_block(snap), **extra, "n": len(data), "columns": COLUMNS, "rows": data} | |
| 84 | + | |
| 85 | + | |
| 86 | +@router.get("/indicators/{slug}/download.{fmt}", summary="Download all observations of an indicator (csv|json)") | |
| 87 | +def download_indicator(slug: str, fmt: str, include_forecast: bool = Query(True), from_: int | None = Query(None, alias="from"), | |
| 88 | + to: int | None = Query(None), snap: Snapshot = Depends(get_snapshot)): | |
| 89 | + ind = resolve_indicator(snap, slug) | |
| 90 | + if fmt not in ("csv", "json"): | |
| 91 | + raise bad_request("Format must be csv or json.") | |
| 92 | + where, params = ["o.indicator_id = ?"], [ind["id"]] | |
| 93 | + if from_ is not None: | |
| 94 | + where.append("o.year >= ?") | |
| 95 | + params.append(from_) | |
| 96 | + if to is not None: | |
| 97 | + where.append("o.year <= ?") | |
| 98 | + params.append(to) | |
| 99 | + rows = _rows(snap, " AND ".join(where), params, include_forecast) | |
| 100 | + if fmt == "csv": | |
| 101 | + return _csv_response(rows, f"countryatlas-{ind['id']}.csv", snap) | |
| 102 | + return _json_payload(snap, rows, indicator=indicator_card(ind)) | |
| 103 | + | |
| 104 | + | |
| 105 | +@router.get("/countries/{id}/download.{fmt}", summary="Download the full dataset of a country (csv|json)") | |
| 106 | +def download_country(id: str, fmt: str, include_forecast: bool = Query(True), topic: str | None = Query(None), | |
| 107 | + snap: Snapshot = Depends(get_snapshot)): | |
| 108 | + c = resolve_country(snap, id) | |
| 109 | + if fmt not in ("csv", "json"): | |
| 110 | + raise bad_request("Format must be csv or json.") | |
| 111 | + where, params = ["o.country_id = ?"], [c["id"]] | |
| 112 | + if topic: | |
| 113 | + where.append("i.topic = ?") | |
| 114 | + params.append(topic) | |
| 115 | + rows = _rows(snap, " AND ".join(where), params, include_forecast) | |
| 116 | + if fmt == "csv": | |
| 117 | + return _csv_response(rows, f"countryatlas-{c['slug'] or c['id']}.csv", snap) | |
| 118 | + return _json_payload(snap, rows, country=country_card(c)) | |
| 119 | + | |
| 120 | + | |
| 121 | +@router.get("/compare/download.{fmt}", summary="Download a compare bundle (csv|json)") | |
| 122 | +def download_compare(fmt: str, countries: str = Query(...), indicators: str = Query(...), from_: int | None = Query(None, alias="from"), | |
| 123 | + to: int | None = Query(None), include_forecast: bool = Query(True), snap: Snapshot = Depends(get_snapshot)): | |
| 124 | + if fmt not in ("csv", "json"): | |
| 125 | + raise bad_request("Format must be csv or json.") | |
| 126 | + cs = [resolve_country(snap, x)["id"] for x in parse_csv(countries, limit=20)] | |
| 127 | + inds = [resolve_indicator(snap, x)["id"] for x in parse_csv(indicators, limit=20)] | |
| 128 | + if not cs or not inds: | |
| 129 | + raise bad_request("Provide at least one country and one indicator.") | |
| 130 | + where = [f"o.country_id IN ({','.join('?' * len(cs))})", f"o.indicator_id IN ({','.join('?' * len(inds))})"] | |
| 131 | + params: list[Any] = [*cs, *inds] | |
| 132 | + if from_ is not None: | |
| 133 | + where.append("o.year >= ?") | |
| 134 | + params.append(from_) | |
| 135 | + if to is not None: | |
| 136 | + where.append("o.year <= ?") | |
| 137 | + params.append(to) | |
| 138 | + rows = _rows(snap, " AND ".join(where), params, include_forecast) | |
| 139 | + if fmt == "csv": | |
| 140 | + return _csv_response(rows, "countryatlas-compare.csv", snap) | |
| 141 | + return _json_payload(snap, rows, countries=cs, indicators=inds) | |
| 142 | + | |
| 143 | + | |
| 144 | +__all__ = ["_iso", "router"] | |
added
src/countryatlas/api/routers/health.py
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +"""/health — liveness + snapshot status (never 503: reports `status: empty` when no DB yet).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.cache import response_cache | |
| 10 | +from countryatlas.api.db import get_database | |
| 11 | + | |
| 12 | +router = APIRouter(tags=["health"]) | |
| 13 | + | |
| 14 | +API_VERSION = "0.1.0" | |
| 15 | + | |
| 16 | + | |
| 17 | +@router.get("/health", response_model=schemas.HealthResponse, summary="Health / snapshot status") | |
| 18 | +def health() -> dict[str, Any]: | |
| 19 | + db = get_database() | |
| 20 | + snap = db.current() | |
| 21 | + if snap is None: | |
| 22 | + return {"status": "empty", "run_id": None, "built_at": None, "observations": 0, "db_path": str(db.path), "version": API_VERSION, | |
| 23 | + "cache": response_cache.stats()} | |
| 24 | + n_obs = snap.meta.get("observation_count") | |
| 25 | + return { | |
| 26 | + "status": "ok", | |
| 27 | + "run_id": snap.run_id, | |
| 28 | + "built_at": snap.built_at, | |
| 29 | + "observations": int(n_obs) if n_obs is not None else snap.table_count("observations"), | |
| 30 | + "countries": len(snap.countries()), | |
| 31 | + "indicators": len(snap.indicators()), | |
| 32 | + "db_path": str(db.path), | |
| 33 | + "version": API_VERSION, | |
| 34 | + "schema_version": snap.meta.get("schema_version"), | |
| 35 | + "cache": response_cache.stats(), | |
| 36 | + } | |
added
src/countryatlas/api/routers/home.py
+104 −0
@@ -0,0 +1,104 @@ | ||
| 1 | +"""/home — global snapshot + curated lists + recent changes + recently updated indicators.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import clean_float, country_card, indicator_card, merged_indicator, meta_block | |
| 10 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 11 | +from countryatlas.api.formatting import format_value | |
| 12 | +from countryatlas.api.provenance import _iso, build_provenance | |
| 13 | +from countryatlas.api.routers.countries import CHANGES_SQL, change_item | |
| 14 | +from countryatlas.api.routers.indicators import indicator_summary | |
| 15 | + | |
| 16 | +router = APIRouter(tags=["home"]) | |
| 17 | + | |
| 18 | +# (key, title, indicator, order, min_population, extra description) | |
| 19 | +CURATED = [ | |
| 20 | + ("largest_economies", "Largest economies", "gdp", "desc", None, "GDP, current US$"), | |
| 21 | + ("fastest_population_growth", "Fastest population growth", "population-growth", "desc", 1_000_000, "Countries above 1 M inhabitants"), | |
| 22 | + ("highest_life_expectancy", "Highest life expectancy", "life-expectancy", "desc", None, "Years at birth"), | |
| 23 | + ("energy_transition_leaders", "Energy transition leaders", "renewable-electricity-share", "desc", None, "Share of electricity from renewables"), | |
| 24 | + ("highest_gdp_per_capita_ppp", "Highest GDP per capita (PPP)", "gdp-per-capita-ppp", "desc", None, "International $"), | |
| 25 | + ("lowest_unemployment", "Lowest unemployment", "unemployment-rate", "asc", 5_000_000, "Countries above 5 M inhabitants"), | |
| 26 | +] | |
| 27 | + | |
| 28 | + | |
| 29 | +def curated_list(snap: Snapshot, indicator_id: str, order: str, min_pop: int | None, n: int = 8) -> dict[str, Any] | None: | |
| 30 | + ind_row = snap.indicators().get(indicator_id) | |
| 31 | + if ind_row is None: | |
| 32 | + return None | |
| 33 | + ind = merged_indicator(ind_row) | |
| 34 | + pop_join = "JOIN latest p ON p.country_id = l.country_id AND p.indicator_id = 'population' AND p.value >= ?" if min_pop else "" | |
| 35 | + rows = snap.query( | |
| 36 | + f"""SELECT l.country_id, l.value, l.year, l.change_pct, l.change_abs, l.source_id, l.rank_world, l.n_world, | |
| 37 | + o.source_dataset, o.source_series_code, o.retrieved_at, o.source_updated_at | |
| 38 | + FROM latest l | |
| 39 | + JOIN countries c ON c.id = l.country_id AND coalesce(c.kind, 'country') = 'country' | |
| 40 | + {pop_join} | |
| 41 | + LEFT JOIN observations o ON o.country_id = l.country_id AND o.indicator_id = l.indicator_id AND o.period = l.period AND o.frequency = l.frequency | |
| 42 | + WHERE l.indicator_id = ? AND l.value IS NOT NULL AND l.year >= (SELECT max(year) - 3 FROM latest WHERE indicator_id = ?) | |
| 43 | + ORDER BY l.value {'DESC' if order == 'desc' else 'ASC'} LIMIT ?""", | |
| 44 | + ([min_pop] if min_pop else []) + [indicator_id, indicator_id, n], | |
| 45 | + ) | |
| 46 | + items = [] | |
| 47 | + for i, r in enumerate(rows): | |
| 48 | + c = snap.countries().get(r["country_id"], {"id": r["country_id"]}) | |
| 49 | + v = clean_float(r["value"]) | |
| 50 | + items.append({"rank": i + 1, "country": country_card(c), "value": v, "formatted": format_value(v, ind), "year": r["year"], | |
| 51 | + "change_pct": clean_float(r.get("change_pct")), "change_abs": clean_float(r.get("change_abs")), | |
| 52 | + "rank_world": r.get("rank_world"), "n_world": r.get("n_world"), | |
| 53 | + "provenance": build_provenance(snap, indicator_id, r["source_id"], r.get("source_dataset"), r.get("source_series_code"), | |
| 54 | + r.get("retrieved_at"), r.get("source_updated_at"), c.get("iso2"))}) | |
| 55 | + return {"indicator": indicator_card(ind), "sort": order, "rows": items} | |
| 56 | + | |
| 57 | + | |
| 58 | +def global_snapshot(snap: Snapshot) -> dict[str, Any]: | |
| 59 | + row = snap.one( | |
| 60 | + """SELECT sum(CASE WHEN l.indicator_id = 'population' THEN l.value END) AS pop, | |
| 61 | + sum(CASE WHEN l.indicator_id = 'gdp' THEN l.value END) AS gdp, | |
| 62 | + median(CASE WHEN l.indicator_id = 'life-expectancy' THEN l.value END) AS le, | |
| 63 | + max(CASE WHEN l.indicator_id = 'population' THEN l.year END) AS pop_year, | |
| 64 | + max(CASE WHEN l.indicator_id = 'gdp' THEN l.year END) AS gdp_year, | |
| 65 | + max(CASE WHEN l.indicator_id = 'life-expectancy' THEN l.year END) AS le_year | |
| 66 | + FROM latest l JOIN countries c ON c.id = l.country_id AND coalesce(c.kind, 'country') = 'country' | |
| 67 | + WHERE l.indicator_id IN ('population', 'gdp', 'life-expectancy')""") or {} | |
| 68 | + n_countries = sum(1 for c in snap.countries().values() if (c.get("kind") or "country") == "country") | |
| 69 | + n_ind_with_data = int(snap.scalar("SELECT count(DISTINCT indicator_id) FROM latest") or 0) | |
| 70 | + n_obs = snap.meta.get("observation_count") | |
| 71 | + if n_obs is None: | |
| 72 | + n_obs = snap.table_count("observations") | |
| 73 | + return { | |
| 74 | + "world_population": clean_float(row.get("pop")), "world_population_formatted": format_value(clean_float(row.get("pop")), {"format": "number"}), | |
| 75 | + "world_population_year": row.get("pop_year"), | |
| 76 | + "world_gdp": clean_float(row.get("gdp")), "world_gdp_formatted": format_value(clean_float(row.get("gdp")), {"format": "currency"}), | |
| 77 | + "world_gdp_year": row.get("gdp_year"), | |
| 78 | + "median_life_expectancy": clean_float(row.get("le")), "median_life_expectancy_year": row.get("le_year"), | |
| 79 | + "n_countries": n_countries, "n_territories": len(snap.countries()) - n_countries, | |
| 80 | + "n_indicators": len(snap.indicators()), "n_indicators_with_data": n_ind_with_data, | |
| 81 | + "n_observations": int(n_obs), "n_sources": len(snap.sources()), | |
| 82 | + "built_at": snap.built_at, "run_id": snap.run_id, | |
| 83 | + "note": "World totals are sums/medians across countries in this snapshot (latest available year per country).", | |
| 84 | + } | |
| 85 | + | |
| 86 | + | |
| 87 | +@router.get("/home", response_model=schemas.HomeResponse, summary="Home page payload") | |
| 88 | +def home(snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 89 | + lists: dict[str, Any] = {} | |
| 90 | + for key, title, iid, order, min_pop, desc in CURATED: | |
| 91 | + lst = curated_list(snap, iid, order, min_pop) | |
| 92 | + if lst: | |
| 93 | + lists[key] = {"title": title, "description": desc, **lst} | |
| 94 | + changes = snap.query( | |
| 95 | + CHANGES_SQL.format(table="changes") + " JOIN countries c ON c.id = x.country_id AND coalesce(c.kind, 'country') = 'country'" | |
| 96 | + " ORDER BY x.severity DESC NULLS LAST, x.period DESC LIMIT 12") | |
| 97 | + ind_rows = [merged_indicator(i) for i in snap.indicators().values() if i.get("latest_source_updated_at") is not None] | |
| 98 | + ind_rows.sort(key=lambda i: (i.get("latest_source_updated_at") is None, _iso(i.get("latest_source_updated_at")) or ""), reverse=True) | |
| 99 | + recently = [indicator_summary(snap, i) for i in ind_rows[:12]] | |
| 100 | + featured = [indicator_summary(snap, merged_indicator(i)) for i in snap.indicators().values() if merged_indicator(i).get("featured")] | |
| 101 | + featured.sort(key=lambda i: (i.get("topic") or "", i.get("name") or "")) | |
| 102 | + return {"meta": meta_block(snap), "snapshot": global_snapshot(snap), "lists": lists, | |
| 103 | + "recent_changes": [change_item(snap, r) for r in changes], "recently_updated": recently, | |
| 104 | + "featured_indicators": featured, "trending": featured[:12]} | |
added
src/countryatlas/api/routers/indicators.py
+339 −0
@@ -0,0 +1,339 @@ | ||
| 1 | +"""/indicators — list, definition + coverage + world latest, map values, group trend.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends, Query | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import ( | |
| 10 | + clean_float, | |
| 11 | + country_card, | |
| 12 | + indicator_card, | |
| 13 | + is_per_capita_or_share, | |
| 14 | + merged_indicator, | |
| 15 | + meta_block, | |
| 16 | + quantile_breaks, | |
| 17 | + resolve_group, | |
| 18 | + resolve_indicator, | |
| 19 | +) | |
| 20 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 21 | +from countryatlas.api.formatting import format_value | |
| 22 | +from countryatlas.api.provenance import _iso, build_provenance, provenance_from_row, source_url | |
| 23 | +from countryatlas.registry import topics as registry_topics | |
| 24 | + | |
| 25 | +router = APIRouter(prefix="/indicators", tags=["indicators"]) | |
| 26 | + | |
| 27 | + | |
| 28 | +def indicator_summary(snap: Snapshot, ind: dict[str, Any], n_countries_total: int | None = None) -> dict[str, Any]: | |
| 29 | + ind = merged_indicator(ind) | |
| 30 | + out = indicator_card(ind) | |
| 31 | + total = n_countries_total or max(1, sum(1 for c in snap.countries().values() if (c.get("kind") or "country") == "country")) | |
| 32 | + out.update({ | |
| 33 | + "description": ind.get("description"), | |
| 34 | + "n_countries": ind.get("n_countries"), | |
| 35 | + "n_observations": ind.get("n_observations"), | |
| 36 | + "first_year": ind.get("first_year"), | |
| 37 | + "last_year": ind.get("last_year"), | |
| 38 | + "latest_source_updated_at": _iso(ind.get("latest_source_updated_at")), | |
| 39 | + "primary_source_id": ind.get("primary_source_id"), | |
| 40 | + "coverage_pct": round(100.0 * (ind.get("n_countries") or 0) / total, 1) if ind.get("n_countries") is not None else None, | |
| 41 | + "tags": list(ind.get("tags") or []), | |
| 42 | + }) | |
| 43 | + return out | |
| 44 | + | |
| 45 | + | |
| 46 | +def topics_of(indicator_id: str) -> list[str]: | |
| 47 | + return [t["id"] for t in registry_topics()["topics"] if indicator_id in t["indicators"]] | |
| 48 | + | |
| 49 | + | |
| 50 | +@router.get("", response_model=schemas.IndicatorsResponse, summary="List indicators") | |
| 51 | +def list_indicators( | |
| 52 | + topic: str | None = Query(None), | |
| 53 | + q: str | None = Query(None), | |
| 54 | + featured: bool | None = Query(None), | |
| 55 | + source: str | None = Query(None, description="Filter by primary source id"), | |
| 56 | + with_data: bool = Query(False, description="Only indicators that have observations in this snapshot"), | |
| 57 | + snap: Snapshot = Depends(get_snapshot), | |
| 58 | +) -> dict[str, Any]: | |
| 59 | + total = sum(1 for c in snap.countries().values() if (c.get("kind") or "country") == "country") | |
| 60 | + items = [] | |
| 61 | + topic_order: dict[str, int] = {} | |
| 62 | + if topic: | |
| 63 | + for t in registry_topics()["topics"]: | |
| 64 | + if t["id"] == topic: | |
| 65 | + topic_order = {s: i for i, s in enumerate(t["indicators"])} | |
| 66 | + ql = (q or "").strip().lower() | |
| 67 | + for ind in snap.indicators().values(): | |
| 68 | + m = merged_indicator(ind) | |
| 69 | + if topic and topic not in (topics_of(m["id"]) + [m.get("topic")]): | |
| 70 | + continue | |
| 71 | + if featured is not None and bool(m.get("featured")) != featured: | |
| 72 | + continue | |
| 73 | + if source and (m.get("primary_source_id") or "") != source: | |
| 74 | + continue | |
| 75 | + if with_data and not (m.get("n_observations") or 0): | |
| 76 | + continue | |
| 77 | + if ql and ql not in " ".join(str(m.get(k) or "") for k in ("name", "short_name", "slug", "description", "subtopic")).lower() \ | |
| 78 | + and not any(ql in str(t).lower() for t in (m.get("tags") or [])): | |
| 79 | + continue | |
| 80 | + items.append(indicator_summary(snap, m, total)) | |
| 81 | + if topic and topic_order: | |
| 82 | + items.sort(key=lambda i: (topic_order.get(i["id"], 999), i["name"] or "")) | |
| 83 | + else: | |
| 84 | + items.sort(key=lambda i: (not i.get("featured"), i.get("topic") or "", i.get("name") or "")) | |
| 85 | + return {"meta": meta_block(snap), "n": len(items), "filters": {"topic": topic, "q": q, "featured": featured, "source": source}, | |
| 86 | + "items": items} | |
| 87 | + | |
| 88 | + | |
| 89 | +def latest_common_year(snap: Snapshot, indicator_id: str, min_countries: int = 50) -> tuple[int | None, int]: | |
| 90 | + """Latest year with >= min_countries values (fallback: the year with the most values, then max year).""" | |
| 91 | + rows = snap.query_rows( | |
| 92 | + """SELECT o.year, count(*) AS n FROM observations o JOIN countries c ON c.id = o.country_id | |
| 93 | + WHERE o.indicator_id = ? AND o.frequency = 'A' AND NOT o.is_forecast AND o.value IS NOT NULL | |
| 94 | + AND coalesce(c.kind, 'country') = 'country' | |
| 95 | + GROUP BY o.year ORDER BY o.year DESC""", | |
| 96 | + [indicator_id], | |
| 97 | + ) | |
| 98 | + if not rows: | |
| 99 | + return None, 0 | |
| 100 | + max_n = max(n for _, n in rows) | |
| 101 | + threshold = min(min_countries, max(1, int(0.6 * max_n))) | |
| 102 | + for year, n in rows: | |
| 103 | + if n >= threshold: | |
| 104 | + return int(year), int(n) | |
| 105 | + return int(rows[0][0]), int(rows[0][1]) | |
| 106 | + | |
| 107 | + | |
| 108 | +def world_latest(snap: Snapshot, ind: dict[str, Any]) -> dict[str, Any] | None: | |
| 109 | + year, _n = latest_common_year(snap, ind["id"]) | |
| 110 | + if year is None: | |
| 111 | + return None | |
| 112 | + row = snap.one( | |
| 113 | + """SELECT median(o.value) AS med, avg(o.value) AS mean, sum(o.value) AS total, count(*) AS n, | |
| 114 | + sum(o.value * p.value) / nullif(sum(CASE WHEN o.value IS NOT NULL THEN p.value END), 0) AS wmean | |
| 115 | + FROM observations o | |
| 116 | + JOIN countries c ON c.id = o.country_id AND coalesce(c.kind, 'country') = 'country' | |
| 117 | + LEFT JOIN observations p ON p.indicator_id = 'population' AND p.country_id = o.country_id AND p.year = o.year | |
| 118 | + AND p.frequency = 'A' AND NOT p.is_forecast | |
| 119 | + WHERE o.indicator_id = ? AND o.year = ? AND o.frequency = 'A' AND NOT o.is_forecast AND o.value IS NOT NULL""", | |
| 120 | + [ind["id"], year], | |
| 121 | + ) or {} | |
| 122 | + agg = ind.get("aggregation") or "none" | |
| 123 | + if agg == "sum": | |
| 124 | + kind, value = "sum", clean_float(row.get("total")) | |
| 125 | + elif agg == "weighted_mean" or is_per_capita_or_share(ind): | |
| 126 | + kind, value = "weighted_mean", clean_float(row.get("wmean")) | |
| 127 | + if value is None: | |
| 128 | + kind, value = "median", clean_float(row.get("med")) | |
| 129 | + else: | |
| 130 | + kind, value = "median", clean_float(row.get("med")) | |
| 131 | + return { | |
| 132 | + "kind": kind, | |
| 133 | + "value": value, | |
| 134 | + "formatted": format_value(value, ind), | |
| 135 | + "year": year, | |
| 136 | + "n": row.get("n"), | |
| 137 | + "median": clean_float(row.get("med")), | |
| 138 | + "mean": clean_float(row.get("mean")), | |
| 139 | + "weighted_mean": clean_float(row.get("wmean")), | |
| 140 | + "sum": clean_float(row.get("total")) if agg == "sum" else None, | |
| 141 | + "weights": "population" if kind == "weighted_mean" else None, | |
| 142 | + "note": "Computed across countries in this snapshot (World Bank aggregates are not stored); " | |
| 143 | + + {"sum": "sum of country values", "weighted_mean": "population-weighted mean", "median": "median of country values"}[kind] | |
| 144 | + + f" for {year}.", | |
| 145 | + } | |
| 146 | + | |
| 147 | + | |
| 148 | +def ranked_extremes(snap: Snapshot, ind: dict[str, Any], year: int | None, n: int = 5) -> tuple[list[dict], list[dict]]: | |
| 149 | + if year is None: | |
| 150 | + return [], [] | |
| 151 | + rows = snap.query( | |
| 152 | + """SELECT o.country_id, o.value, o.year, o.source_id, o.source_dataset, o.source_series_code, o.retrieved_at, o.source_updated_at | |
| 153 | + FROM observations o JOIN countries c ON c.id = o.country_id AND coalesce(c.kind, 'country') = 'country' | |
| 154 | + WHERE o.indicator_id = ? AND o.year = ? AND o.frequency = 'A' AND NOT o.is_forecast AND o.value IS NOT NULL | |
| 155 | + ORDER BY o.value DESC""", | |
| 156 | + [ind["id"], year], | |
| 157 | + ) | |
| 158 | + hib = ind.get("higher_is_better") | |
| 159 | + ordered = rows if hib is not False else list(reversed(rows)) | |
| 160 | + | |
| 161 | + def pack(r: dict[str, Any], rank: int) -> dict[str, Any]: | |
| 162 | + c = snap.countries().get(r["country_id"], {"id": r["country_id"]}) | |
| 163 | + return {"country": country_card(c), "value": clean_float(r["value"]), "formatted": format_value(clean_float(r["value"]), ind), | |
| 164 | + "year": r["year"], "rank": rank, "provenance": provenance_from_row(snap, r, ind["id"], c.get("iso2"))} | |
| 165 | + | |
| 166 | + top = [pack(r, i + 1) for i, r in enumerate(ordered[:n])] | |
| 167 | + total = len(ordered) | |
| 168 | + bottom = [pack(r, total - n + 1 + i if total >= n else i + 1) for i, r in enumerate(ordered[-n:])] if total > n else [] | |
| 169 | + return top, bottom | |
| 170 | + | |
| 171 | + | |
| 172 | +@router.get("/{slug}", response_model=schemas.IndicatorResponse, summary="Indicator definition, sources, coverage, world value") | |
| 173 | +def get_indicator(slug: str, snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 174 | + ind = resolve_indicator(snap, slug) | |
| 175 | + total = sum(1 for c in snap.countries().values() if (c.get("kind") or "country") == "country") | |
| 176 | + srcs = [] | |
| 177 | + for r in sorted(snap.indicator_sources_for(ind["id"]), key=lambda r: (r.get("priority") or 99)): | |
| 178 | + s = snap.sources().get(r["source_id"], {}) | |
| 179 | + srcs.append({ | |
| 180 | + "source_id": r["source_id"], "source_name": s.get("name"), "dataset": r.get("dataset"), "series_code": r.get("series_code"), | |
| 181 | + "priority": r.get("priority"), "transform": r.get("transform"), "countries": list(r["countries"]) if r.get("countries") else None, | |
| 182 | + "notes": r.get("notes"), "url": source_url(r["source_id"], r.get("dataset"), r.get("series_code")) or r.get("source_url") or s.get("url"), | |
| 183 | + "licence": s.get("licence"), "n_observations": r.get("n_observations"), "n_countries": r.get("n_countries"), | |
| 184 | + "last_year": r.get("last_year"), "last_status": r.get("last_status"), "params": r.get("params"), | |
| 185 | + }) | |
| 186 | + fresh = snap.one( | |
| 187 | + "SELECT max(source_updated_at) AS su, max(retrieved_at) AS ra, min(year) AS y0, max(year) AS y1, " | |
| 188 | + "max(CASE WHEN NOT is_forecast THEN year END) AS y1_actual, count(*) AS n, count(DISTINCT country_id) AS nc " | |
| 189 | + "FROM observations WHERE indicator_id = ?", | |
| 190 | + [ind["id"]], | |
| 191 | + ) or {} | |
| 192 | + years_rows = snap.query_rows( | |
| 193 | + "SELECT year, count(*) FROM observations WHERE indicator_id = ? AND NOT is_forecast AND frequency = 'A' GROUP BY year ORDER BY year", | |
| 194 | + [ind["id"]], | |
| 195 | + ) | |
| 196 | + wl = world_latest(snap, ind) | |
| 197 | + year_used = wl["year"] if wl else (fresh.get("y1_actual")) | |
| 198 | + top5, bottom5 = ranked_extremes(snap, ind, year_used) | |
| 199 | + return { | |
| 200 | + "meta": meta_block(snap), | |
| 201 | + "indicator": {**indicator_summary(snap, ind, total), "description": ind.get("description"), "methodology": ind.get("methodology"), | |
| 202 | + "bounds": [ind.get("bounds_min"), ind.get("bounds_max")], "scale": ind.get("scale"), "per_capita_of": ind.get("per_capita_of"), | |
| 203 | + "source_priority": ind.get("source_priority")}, | |
| 204 | + "sources": srcs, | |
| 205 | + "coverage": {"n_countries": fresh.get("nc") or ind.get("n_countries"), "n_countries_total": total, | |
| 206 | + "coverage_pct": round(100.0 * (fresh.get("nc") or 0) / max(total, 1), 1), | |
| 207 | + "n_observations": fresh.get("n") or ind.get("n_observations"), | |
| 208 | + "by_year": [{"year": int(y), "n": int(n)} for y, n in years_rows]}, | |
| 209 | + "world_latest": wl, | |
| 210 | + "freshness": {"source_updated_at": _iso(fresh.get("su")), "retrieved_at": _iso(fresh.get("ra")), "built_at": snap.built_at}, | |
| 211 | + "top5": top5, | |
| 212 | + "bottom5": bottom5, | |
| 213 | + "years": {"first": fresh.get("y0"), "last": fresh.get("y1"), "last_actual": fresh.get("y1_actual"), "latest_common": year_used}, | |
| 214 | + "topics": topics_of(ind["id"]) or [ind.get("topic")], | |
| 215 | + } | |
| 216 | + | |
| 217 | + | |
| 218 | +@router.get("/{slug}/map", response_model=schemas.MapResponse, summary="Choropleth values for one year") | |
| 219 | +def get_indicator_map( | |
| 220 | + slug: str, | |
| 221 | + year: int | None = Query(None, ge=1800, le=2100), | |
| 222 | + nearest: bool = Query(False, description="Use each country's latest value within 3 years of the reference year"), | |
| 223 | + classes: int = Query(6, ge=5, le=7), | |
| 224 | + snap: Snapshot = Depends(get_snapshot), | |
| 225 | +) -> dict[str, Any]: | |
| 226 | + ind = resolve_indicator(snap, slug) | |
| 227 | + year_used = year | |
| 228 | + if year_used is None: | |
| 229 | + year_used, _ = latest_common_year(snap, ind["id"]) | |
| 230 | + values: dict[str, float | None] = {} | |
| 231 | + years: dict[str, int] = {} | |
| 232 | + provs: dict[tuple, dict[str, Any]] = {} | |
| 233 | + counts: dict[tuple, int] = {} | |
| 234 | + if year_used is not None: | |
| 235 | + if nearest: | |
| 236 | + rows = snap.query( | |
| 237 | + """SELECT country_id, arg_max(value, year) AS value, max(year) AS year, | |
| 238 | + arg_max(source_id, year) AS source_id, arg_max(source_dataset, year) AS source_dataset, | |
| 239 | + arg_max(source_series_code, year) AS source_series_code, arg_max(retrieved_at, year) AS retrieved_at, | |
| 240 | + arg_max(source_updated_at, year) AS source_updated_at | |
| 241 | + FROM observations o | |
| 242 | + WHERE indicator_id = ? AND frequency = 'A' AND NOT is_forecast AND value IS NOT NULL AND year BETWEEN ? AND ? | |
| 243 | + AND country_id IN (SELECT id FROM countries WHERE coalesce(kind, 'country') = 'country') | |
| 244 | + GROUP BY country_id""", | |
| 245 | + [ind["id"], year_used - 3, year_used], | |
| 246 | + ) | |
| 247 | + else: | |
| 248 | + rows = snap.query( | |
| 249 | + """SELECT country_id, value, year, source_id, source_dataset, source_series_code, retrieved_at, source_updated_at | |
| 250 | + FROM observations | |
| 251 | + WHERE indicator_id = ? AND year = ? AND frequency = 'A' AND NOT is_forecast AND value IS NOT NULL | |
| 252 | + AND country_id IN (SELECT id FROM countries WHERE coalesce(kind, 'country') = 'country')""", | |
| 253 | + [ind["id"], year_used], | |
| 254 | + ) | |
| 255 | + for r in rows: | |
| 256 | + v = clean_float(r["value"]) | |
| 257 | + if v is None: | |
| 258 | + continue | |
| 259 | + values[r["country_id"]] = v | |
| 260 | + years[r["country_id"]] = int(r["year"]) | |
| 261 | + k = (r.get("source_id"), r.get("source_dataset"), r.get("source_series_code")) | |
| 262 | + counts[k] = counts.get(k, 0) + 1 | |
| 263 | + if k not in provs: | |
| 264 | + provs[k] = provenance_from_row(snap, r, ind["id"]) | |
| 265 | + vals = list(values.values()) | |
| 266 | + breaks = quantile_breaks(vals, classes) | |
| 267 | + dominant = max(counts, key=lambda k: counts[k]) if counts else None | |
| 268 | + return { | |
| 269 | + "meta": meta_block(snap), | |
| 270 | + "indicator": indicator_card(ind), | |
| 271 | + "year": year, | |
| 272 | + "year_used": year_used, | |
| 273 | + "nearest": nearest, | |
| 274 | + "values": values, | |
| 275 | + "years": years if nearest else None, | |
| 276 | + "formatted": {k: format_value(v, ind) for k, v in values.items()}, | |
| 277 | + "legend": {"min": min(vals) if vals else None, "max": max(vals) if vals else None, "breaks": breaks, "n_classes": len(breaks) + 1}, | |
| 278 | + "n": len(values), | |
| 279 | + "provenance": provs.get(dominant) if dominant else None, | |
| 280 | + "sources": [dict(provs[k], n_values=n) for k, n in sorted(counts.items(), key=lambda kv: -kv[1])], | |
| 281 | + } | |
| 282 | + | |
| 283 | + | |
| 284 | +@router.get("/{slug}/trend", response_model=schemas.TrendResponse, summary="Aggregate trend for a group (median / mean / weighted / sum)") | |
| 285 | +def get_indicator_trend( | |
| 286 | + slug: str, | |
| 287 | + group: str = Query("world"), | |
| 288 | + from_: int | None = Query(None, alias="from"), | |
| 289 | + to: int | None = Query(None), | |
| 290 | + min_n: int = Query(5, ge=1), | |
| 291 | + snap: Snapshot = Depends(get_snapshot), | |
| 292 | +) -> dict[str, Any]: | |
| 293 | + ind = resolve_indicator(snap, slug) | |
| 294 | + g = resolve_group(snap, group) | |
| 295 | + members = snap.group_members(g["id"]) | |
| 296 | + where = ["o.indicator_id = ?", "o.frequency = 'A'", "NOT o.is_forecast", "o.value IS NOT NULL", | |
| 297 | + "coalesce(c.kind, 'country') = 'country'"] | |
| 298 | + params: list[Any] = [ind["id"]] | |
| 299 | + if g["id"] != "world": | |
| 300 | + if not members: | |
| 301 | + where.append("FALSE") | |
| 302 | + else: | |
| 303 | + where.append(f"o.country_id IN ({','.join('?' * len(members))})") | |
| 304 | + params += members | |
| 305 | + if from_ is not None: | |
| 306 | + where.append("o.year >= ?") | |
| 307 | + params.append(from_) | |
| 308 | + if to is not None: | |
| 309 | + where.append("o.year <= ?") | |
| 310 | + params.append(to) | |
| 311 | + rows = snap.query( | |
| 312 | + f"""SELECT o.year, median(o.value) AS med, avg(o.value) AS mean, sum(o.value) AS total, count(*) AS n, | |
| 313 | + sum(o.value * p.value) / nullif(sum(CASE WHEN o.value IS NOT NULL THEN p.value END), 0) AS wmean | |
| 314 | + FROM observations o | |
| 315 | + JOIN countries c ON c.id = o.country_id | |
| 316 | + LEFT JOIN observations p ON p.indicator_id = 'population' AND p.country_id = o.country_id AND p.year = o.year | |
| 317 | + AND p.frequency = 'A' AND NOT p.is_forecast | |
| 318 | + WHERE {' AND '.join(where)} | |
| 319 | + GROUP BY o.year HAVING count(*) >= ? ORDER BY o.year""", | |
| 320 | + [*params, min_n], | |
| 321 | + ) | |
| 322 | + agg = ind.get("aggregation") or "none" | |
| 323 | + weighted = is_per_capita_or_share(ind) | |
| 324 | + preferred = "sum" if agg == "sum" else ("weighted_mean" if weighted else "median") | |
| 325 | + points = [{"year": int(r["year"]), "median": clean_float(r["med"]), "mean": clean_float(r["mean"]), | |
| 326 | + "weighted_mean": clean_float(r["wmean"]) if weighted else None, | |
| 327 | + "sum": clean_float(r["total"]) if agg == "sum" else None, "n": int(r["n"])} for r in rows] | |
| 328 | + prov_rows = snap.query( | |
| 329 | + "SELECT source_id, source_dataset, source_series_code, max(retrieved_at) AS retrieved_at, max(source_updated_at) AS source_updated_at, count(*) AS n " | |
| 330 | + "FROM observations WHERE indicator_id = ? AND NOT is_forecast GROUP BY 1, 2, 3 ORDER BY n DESC", [ind["id"]]) | |
| 331 | + return { | |
| 332 | + "meta": meta_block(snap), "indicator": indicator_card(ind), | |
| 333 | + "group": {"id": g["id"], "slug": g.get("slug"), "name": g.get("name"), "kind": g.get("kind"), "wb_code": g.get("wb_code"), | |
| 334 | + "n_members": len(members) if members else g.get("n_members")}, | |
| 335 | + "preferred": preferred, "weights": "population" if weighted else None, "points": points, | |
| 336 | + "provenance": [dict(build_provenance(snap, ind["id"], r["source_id"], r["source_dataset"], r["source_series_code"], | |
| 337 | + r["retrieved_at"], r["source_updated_at"]), n_values=r["n"]) for r in prov_rows], | |
| 338 | + "note": "Aggregates are computed across member countries present in the snapshot (no World Bank aggregate series is stored).", | |
| 339 | + } | |
added
src/countryatlas/api/routers/methodology.py
+101 −0
@@ -0,0 +1,101 @@ | ||
| 1 | +"""/methodology — registry-derived description of units, priorities, validation rules, similarity and rankings.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from collections import Counter | |
| 5 | +from typing import Any | |
| 6 | + | |
| 7 | +from fastapi import APIRouter | |
| 8 | + | |
| 9 | +from countryatlas.api.db import get_database | |
| 10 | +from countryatlas.registry import CONNECTORS, STALE_DAYS_BY_FREQ | |
| 11 | +from countryatlas.registry import indicators as registry_indicators | |
| 12 | +from countryatlas.registry import topics as registry_topics | |
| 13 | + | |
| 14 | +router = APIRouter(tags=["methodology"]) | |
| 15 | + | |
| 16 | +VALIDATION_RULES = [ | |
| 17 | + {"code": "duplicate", "severity": "error", "text": "Duplicate key inside one dataset → the dataset is quarantined."}, | |
| 18 | + {"code": "out_of_bounds", "severity": "error", "text": "Impossible values (negative counts, shares outside [-5, 105], life expectancy outside [20, 100]; registry `bounds`) → row quarantined."}, | |
| 19 | + {"code": "unit_mismatch", "severity": "error", "text": "Unit differs from the registry unit → dataset quarantined."}, | |
| 20 | + {"code": "extreme_jump", "severity": "warning", "text": "|Δ| > jump_threshold × robust std (MAD) of the country series → row flagged `warning`, kept."}, | |
| 21 | + {"code": "partial_download", "severity": "error", "text": "Fewer than 30 % of the rows of the previous run for the same dataset → dataset quarantined, previous kept."}, | |
| 22 | + {"code": "stale", "severity": "info", "text": "Source not updated / latest period older than stale_after_days (annual 800 d, quarterly 200 d, monthly 75 d) → status `stale`."}, | |
| 23 | + {"code": "unknown_country", "severity": "info", "text": "Unknown country code → row dropped and logged (aggregates are dropped unless mapped to a group)."}, | |
| 24 | +] | |
| 25 | + | |
| 26 | +SIMILARITY_MODES = { | |
| 27 | + "overall": "Economic, demographic, energy and social features together.", | |
| 28 | + "economic": "GDP per capita (log), growth, inflation, trade openness, sector shares, government size.", | |
| 29 | + "demographic": "Population (log), median age, fertility, urbanisation, life expectancy, migration.", | |
| 30 | + "energy": "Energy use per capita, electricity mix shares, CO₂ per capita, energy intensity.", | |
| 31 | + "social": "Education, health, inequality, digital adoption, quality-of-life indicators.", | |
| 32 | +} | |
| 33 | + | |
| 34 | +DNA_DIMENSIONS = [ | |
| 35 | + {"id": "income", "label": "Income", "indicators": ["gdp-per-capita-ppp"]}, | |
| 36 | + {"id": "demographics", "label": "Demographics", "indicators": ["median-age", "fertility-rate"]}, | |
| 37 | + {"id": "urbanization", "label": "Urbanisation", "indicators": ["urban-population-share"]}, | |
| 38 | + {"id": "trade", "label": "Trade openness", "indicators": ["trade-pct-gdp"]}, | |
| 39 | + {"id": "energy", "label": "Energy use", "indicators": ["energy-use-per-capita"]}, | |
| 40 | + {"id": "emissions", "label": "Emissions", "indicators": ["co2-per-capita"]}, | |
| 41 | + {"id": "innovation", "label": "Innovation", "indicators": ["rd-expenditure-pct-gdp", "patent-applications-residents"]}, | |
| 42 | + {"id": "education", "label": "Education", "indicators": ["tertiary-enrollment", "expected-years-of-schooling"]}, | |
| 43 | + {"id": "public_spending", "label": "Public spending", "indicators": ["government-expenditure-pct-gdp"]}, | |
| 44 | +] | |
| 45 | + | |
| 46 | + | |
| 47 | +@router.get("/methodology", summary="Methodology (registry-derived)") | |
| 48 | +def methodology() -> dict[str, Any]: | |
| 49 | + inds = registry_indicators() | |
| 50 | + t = registry_topics() | |
| 51 | + units = Counter(i.unit for i in inds) | |
| 52 | + formats = Counter(i.format for i in inds) | |
| 53 | + connectors_used = Counter(s.connector for i in inds for s in i.sources) | |
| 54 | + snap = get_database().current() | |
| 55 | + meta = {"built_at": snap.built_at if snap else None, "run_id": snap.run_id if snap else None} | |
| 56 | + return { | |
| 57 | + "meta": meta, | |
| 58 | + "principles": [ | |
| 59 | + "The country is the primary unit; every visible number is traceable to its source (provenance object on every value).", | |
| 60 | + "Exactly one source per (indicator, country, period) is kept, chosen by the registry source priority; alternatives are stored separately.", | |
| 61 | + "Forecasts (IMF WEO projections) are flagged and never enter latest values, rankings or change detection.", | |
| 62 | + "Unusual values are never deleted, only flagged. Revisions are recorded, never silently overwritten.", | |
| 63 | + "Headlines, insights and change descriptions are template strings computed from data — no generative model.", | |
| 64 | + ], | |
| 65 | + "topics": [{"id": x["id"], "name": x["name"], "order": x.get("order"), "blurb": x.get("blurb"), "n_indicators": len(x["indicators"])} | |
| 66 | + for x in sorted(t["topics"], key=lambda x: x.get("order", 99))], | |
| 67 | + "headline_indicators": t["headline"], | |
| 68 | + "indicators": {"n": len(inds), "units": [{"unit": u, "n": n} for u, n in units.most_common()], | |
| 69 | + "formats": [{"format": f, "n": n} for f, n in formats.most_common()], | |
| 70 | + "frequencies": dict(Counter(i.frequency for i in inds)), | |
| 71 | + "aggregations": dict(Counter(i.aggregation for i in inds))}, | |
| 72 | + "sources": {"connectors": CONNECTORS, "series_mapped_per_connector": dict(connectors_used), | |
| 73 | + "priority_rule": "Each indicator lists its sources in priority order (1 = preferred). For each country × period the highest-priority " | |
| 74 | + "source with a value is kept in `observations`; the others go to `observations_alt`.", | |
| 75 | + "urls": {"worldbank": "https://data.worldbank.org/indicator/{code}?locations={iso2}", "owid": "https://ourworldindata.org/grapher/{slug}", | |
| 76 | + "imf": "https://data.imf.org/", "oecd": "https://data-explorer.oecd.org/", "eurostat": "https://ec.europa.eu/eurostat/databrowser/view/{dataset}/default/table", | |
| 77 | + "who": "https://www.who.int/data/gho/data/indicators/indicator-details/GHO/{code}", "fred": "https://fred.stlouisfed.org/series/{code}", | |
| 78 | + "bis": "https://data.bis.org/", "ilo": "https://ilostat.ilo.org/"}}, | |
| 79 | + "periods": {"annual": "YYYY-01-01", "quarterly": "YYYY-{01,04,07,10}-01", "monthly": "YYYY-MM-01"}, | |
| 80 | + "validation": {"rules": VALIDATION_RULES, "stale_after_days": STALE_DAYS_BY_FREQ, | |
| 81 | + "statuses": ["verified", "imported", "warning", "stale", "quarantined"]}, | |
| 82 | + "derived": { | |
| 83 | + "latest": "Last non-forecast observation per country × indicator; previous = previous period at the same frequency; ranks among countries " | |
| 84 | + "(kind = country) for the same year. A rank computed on a year older than the global max year by more than 2 is flagged `rank_is_stale`.", | |
| 85 | + "rankings": "Computed for every ranking-eligible indicator and every year with at least 20 countries. Rank 1 = highest value, or lowest when " | |
| 86 | + "`higher_is_better` is false. Group rankings recompute ranks within the group.", | |
| 87 | + "changes": "Detectors per country × indicator: YoY beyond ±(2 × MAD of yearly diffs) and an indicator-specific floor (inflation ±2 pts, " | |
| 88 | + "unemployment ±1 pt, GDP growth ±3 pts, population growth ±0.5 pt), record high/low, N-year high/low (10/20/30), sign flip, " | |
| 89 | + "acceleration/deceleration. severity = min(1, |z| / 4) blended with the floor ratio.", | |
| 90 | + "similarity": {"modes": SIMILARITY_MODES, "method": "Heavy-tailed features log-transformed, z-scored across countries (latest values; countries " | |
| 91 | + "with ≥ 70 % of a mode's features), weighted Euclidean distance d → score = 100 × exp(−d / d₀). Top 12 peers with per-feature " | |
| 92 | + "contributions."}, | |
| 93 | + "insights": "Templates computed from data (e.g. \"{country}'s population grew {pct}% since {y0}\").", | |
| 94 | + "country_dna": {"dimensions": DNA_DIMENSIONS, "method": "Percentile rank of the country among all countries for a representative indicator " | |
| 95 | + "(or the mean of 2–3), 0–100. Descriptive, not a score."}, | |
| 96 | + "world_aggregates": "The API does not store World Bank aggregates (WLD, OED…). Group values are computed across member countries: sum for " | |
| 97 | + "additive indicators, population-weighted mean for per-capita/share indicators, median otherwise; the response says which.", | |
| 98 | + }, | |
| 99 | + "formatting": {"currency": "compact scale (53.4k, 1.2B, 1.2T)", "percent": "1 decimal + ' %'", "years": "1 decimal + ' yrs'", | |
| 100 | + "number": "thousand separators, compact ≥ 10k", "tonnes": "' t'", "per_1000": "' per 1,000'", "per_100k": "' per 100k'"}, | |
| 101 | + } | |
added
src/countryatlas/api/routers/rankings.py
+184 −0
@@ -0,0 +1,184 @@ | ||
| 1 | +"""/rankings — rankable indicators, ranking table for a year/group, rank history.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends, Query | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import ( | |
| 10 | + clean_float, | |
| 11 | + country_card, | |
| 12 | + group_card, | |
| 13 | + indicator_card, | |
| 14 | + merged_indicator, | |
| 15 | + meta_block, | |
| 16 | + parse_csv, | |
| 17 | + resolve_country, | |
| 18 | + resolve_group, | |
| 19 | + resolve_indicator, | |
| 20 | + sparklines_for_countries, | |
| 21 | +) | |
| 22 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 23 | +from countryatlas.api.formatting import format_change, format_value | |
| 24 | +from countryatlas.api.provenance import provenance_from_row | |
| 25 | +from countryatlas.api.routers.indicators import indicator_summary | |
| 26 | + | |
| 27 | +router = APIRouter(prefix="/rankings", tags=["rankings"]) | |
| 28 | + | |
| 29 | + | |
| 30 | +@router.get("", response_model=schemas.RankingsListResponse, summary="Rankable indicators (featured first)") | |
| 31 | +def list_rankings(topic: str | None = None, snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 32 | + ranked = {r[0]: (r[1], r[2]) for r in snap.query_rows("SELECT indicator_id, max(year), max(n) FROM rankings GROUP BY indicator_id")} | |
| 33 | + items = [] | |
| 34 | + for ind in snap.indicators().values(): | |
| 35 | + m = merged_indicator(ind) | |
| 36 | + if not m.get("ranking_eligible", True): | |
| 37 | + continue | |
| 38 | + if topic and m.get("topic") != topic: | |
| 39 | + continue | |
| 40 | + if m["id"] not in ranked and not (m.get("n_countries") or 0): | |
| 41 | + continue | |
| 42 | + s = indicator_summary(snap, m) | |
| 43 | + s["ranking_year"], s["ranking_n"] = ranked.get(m["id"], (m.get("last_year"), m.get("n_countries"))) | |
| 44 | + items.append(s) | |
| 45 | + items.sort(key=lambda i: (not i.get("featured"), i.get("topic") or "", i.get("name") or "")) | |
| 46 | + return {"meta": meta_block(snap), "n": len(items), "items": items} | |
| 47 | + | |
| 48 | + | |
| 49 | +def _years_available(snap: Snapshot, indicator_id: str) -> list[int]: | |
| 50 | + ys = [int(r[0]) for r in snap.query_rows("SELECT DISTINCT year FROM rankings WHERE indicator_id = ? ORDER BY year", [indicator_id])] | |
| 51 | + if not ys: | |
| 52 | + ys = [int(r[0]) for r in snap.query_rows( | |
| 53 | + "SELECT year FROM observations WHERE indicator_id = ? AND frequency = 'A' AND NOT is_forecast AND value IS NOT NULL " | |
| 54 | + "GROUP BY year HAVING count(*) >= 5 ORDER BY year", [indicator_id])] | |
| 55 | + return ys | |
| 56 | + | |
| 57 | + | |
| 58 | +def default_sort(ind: dict[str, Any]) -> str: | |
| 59 | + return "asc" if ind.get("higher_is_better") is False else "desc" | |
| 60 | + | |
| 61 | + | |
| 62 | +@router.get("/{indicator}", response_model=schemas.RankingResponse, summary="Ranking table for one indicator") | |
| 63 | +def get_ranking( | |
| 64 | + indicator: str, | |
| 65 | + year: int | None = Query(None), | |
| 66 | + group: str = Query("world"), | |
| 67 | + sort: str | None = Query(None, pattern="^(asc|desc)$"), | |
| 68 | + limit: int = Query(50, ge=1, le=300), | |
| 69 | + offset: int = Query(0, ge=0), | |
| 70 | + sparkline: bool = Query(True), | |
| 71 | + snap: Snapshot = Depends(get_snapshot), | |
| 72 | +) -> dict[str, Any]: | |
| 73 | + ind = resolve_indicator(snap, indicator) | |
| 74 | + g = resolve_group(snap, group) | |
| 75 | + years = _years_available(snap, ind["id"]) | |
| 76 | + year_used = year if (year is not None and year in years) else (max(years) if years else year) | |
| 77 | + if year is not None and years and year not in years: | |
| 78 | + # nearest available year | |
| 79 | + year_used = min(years, key=lambda y: (abs(y - year), -y)) | |
| 80 | + sort_used = sort or default_sort(ind) | |
| 81 | + members = snap.group_members(g["id"]) if g["id"] != "world" else None | |
| 82 | + rows: list[dict[str, Any]] = [] | |
| 83 | + if year_used is not None: | |
| 84 | + where = ["o.indicator_id = ?", "o.year = ?", "o.frequency = 'A'", "NOT o.is_forecast", "o.value IS NOT NULL", | |
| 85 | + "coalesce(c.kind, 'country') = 'country'"] | |
| 86 | + params: list[Any] = [ind["id"], year_used] | |
| 87 | + if members is not None: | |
| 88 | + if not members: | |
| 89 | + where.append("FALSE") | |
| 90 | + else: | |
| 91 | + where.append(f"o.country_id IN ({','.join('?' * len(members))})") | |
| 92 | + params += members | |
| 93 | + direction = "DESC" if sort_used == "desc" else "ASC" | |
| 94 | + rows = snap.query( | |
| 95 | + f""" | |
| 96 | + WITH base AS ( | |
| 97 | + SELECT o.country_id, o.value, o.year, o.source_id, o.source_dataset, o.source_series_code, o.retrieved_at, o.source_updated_at | |
| 98 | + FROM observations o JOIN countries c ON c.id = o.country_id | |
| 99 | + WHERE {' AND '.join(where)} | |
| 100 | + ), | |
| 101 | + prev1 AS (SELECT country_id, value FROM observations WHERE indicator_id = ? AND year = ? AND frequency = 'A' AND NOT is_forecast), | |
| 102 | + prev10 AS (SELECT country_id, value FROM observations WHERE indicator_id = ? AND year = ? AND frequency = 'A' AND NOT is_forecast), | |
| 103 | + rk AS (SELECT country_id, rank, n, pct_rank FROM rankings WHERE indicator_id = ? AND year = ?) | |
| 104 | + SELECT b.*, row_number() OVER (ORDER BY b.value {direction}, b.country_id) AS rank_in_group, count(*) OVER () AS n_in_group, | |
| 105 | + rk.rank AS rank_world, rk.n AS n_world, rk.pct_rank, p1.value AS v1, p10.value AS v10 | |
| 106 | + FROM base b | |
| 107 | + LEFT JOIN prev1 p1 ON p1.country_id = b.country_id | |
| 108 | + LEFT JOIN prev10 p10 ON p10.country_id = b.country_id | |
| 109 | + LEFT JOIN rk ON rk.country_id = b.country_id | |
| 110 | + ORDER BY rank_in_group | |
| 111 | + LIMIT ? OFFSET ? | |
| 112 | + """, | |
| 113 | + [*params, ind["id"], year_used - 1, ind["id"], year_used - 10, ind["id"], year_used, limit, offset], | |
| 114 | + ) | |
| 115 | + n_total = int(rows[0]["n_in_group"]) if rows else 0 | |
| 116 | + if not rows and year_used is not None: | |
| 117 | + cnt = snap.scalar( | |
| 118 | + "SELECT count(*) FROM observations o JOIN countries c ON c.id = o.country_id WHERE o.indicator_id = ? AND o.year = ? " | |
| 119 | + "AND o.frequency = 'A' AND NOT o.is_forecast AND o.value IS NOT NULL AND coalesce(c.kind,'country') = 'country'" | |
| 120 | + + (f" AND o.country_id IN ({','.join('?' * len(members))})" if members else ""), | |
| 121 | + [ind["id"], year_used, *(members or [])], | |
| 122 | + ) | |
| 123 | + n_total = int(cnt or 0) | |
| 124 | + sparks = sparklines_for_countries(snap, ind["id"], [r["country_id"] for r in rows], max_year=year_used) if (sparkline and rows) else {} | |
| 125 | + out_rows = [] | |
| 126 | + for r in rows: | |
| 127 | + c = snap.countries().get(r["country_id"], {"id": r["country_id"]}) | |
| 128 | + v, v1, v10 = clean_float(r["value"]), clean_float(r.get("v1")), clean_float(r.get("v10")) | |
| 129 | + ch1 = {"abs": v - v1, "pct": (v - v1) / abs(v1) * 100 if v1 else None} if (v is not None and v1 is not None) else None | |
| 130 | + ch10 = {"abs": v - v10, "pct": (v - v10) / abs(v10) * 100 if v10 else None} if (v is not None and v10 is not None) else None | |
| 131 | + if ch1: | |
| 132 | + ch1["formatted"] = format_change(ch1["abs"], ch1["pct"], ind) | |
| 133 | + if ch10: | |
| 134 | + ch10["formatted"] = format_change(ch10["abs"], ch10["pct"], ind) | |
| 135 | + out_rows.append({ | |
| 136 | + "rank": int(r["rank_in_group"]), "rank_world": r.get("rank_world"), "n_world": r.get("n_world"), "pct_rank": clean_float(r.get("pct_rank")), | |
| 137 | + "country": country_card(c), "value": v, "formatted": format_value(v, ind), "year": r["year"], | |
| 138 | + "change_1y": ch1, "change_10y": ch10, "sparkline": sparks.get(r["country_id"], []), | |
| 139 | + "provenance": provenance_from_row(snap, r, ind["id"], c.get("iso2")), | |
| 140 | + }) | |
| 141 | + return { | |
| 142 | + "meta": meta_block(snap), "indicator": indicator_card(ind), "group": group_card(g), "year": year, "year_used": year_used, | |
| 143 | + "years_available": years, "sort": sort_used, "n": n_total, "limit": limit, "offset": offset, "rows": out_rows, | |
| 144 | + "label": ("Lowest" if sort_used == "asc" else "Highest") if ind.get("higher_is_better") is None else ("Best" if sort_used == default_sort(ind) else "Worst"), | |
| 145 | + } | |
| 146 | + | |
| 147 | + | |
| 148 | +@router.get("/{indicator}/history", response_model=schemas.RankHistoryResponse, summary="Rank by year for selected countries") | |
| 149 | +def get_ranking_history(indicator: str, countries: str = Query(..., description="Comma-separated ISO3/slugs"), | |
| 150 | + from_: int | None = Query(None, alias="from"), to: int | None = Query(None), | |
| 151 | + snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 152 | + ind = resolve_indicator(snap, indicator) | |
| 153 | + cs = [resolve_country(snap, c) for c in parse_csv(countries, limit=20)] | |
| 154 | + ids = [c["id"] for c in cs] | |
| 155 | + where = ["indicator_id = ?", f"country_id IN ({','.join('?' * len(ids))})"] | |
| 156 | + params: list[Any] = [ind["id"], *ids] | |
| 157 | + if from_ is not None: | |
| 158 | + where.append("year >= ?") | |
| 159 | + params.append(from_) | |
| 160 | + if to is not None: | |
| 161 | + where.append("year <= ?") | |
| 162 | + params.append(to) | |
| 163 | + rows = snap.query(f"SELECT country_id, year, rank, n, value, pct_rank FROM rankings WHERE {' AND '.join(where)} ORDER BY country_id, year", params) | |
| 164 | + if not rows: | |
| 165 | + # fall back to computing ranks from observations | |
| 166 | + rows = snap.query( | |
| 167 | + f"""WITH all_rows AS ( | |
| 168 | + SELECT o.country_id, o.year, o.value, | |
| 169 | + row_number() OVER (PARTITION BY o.year ORDER BY o.value {'ASC' if ind.get('higher_is_better') is False else 'DESC'}) AS rank, | |
| 170 | + count(*) OVER (PARTITION BY o.year) AS n | |
| 171 | + FROM observations o JOIN countries c ON c.id = o.country_id | |
| 172 | + WHERE o.indicator_id = ? AND o.frequency = 'A' AND NOT o.is_forecast AND o.value IS NOT NULL AND coalesce(c.kind,'country') = 'country') | |
| 173 | + SELECT country_id, year, rank, n, value, NULL AS pct_rank FROM all_rows WHERE {' AND '.join(where[1:]) if len(where) > 1 else 'TRUE'} | |
| 174 | + ORDER BY country_id, year""", | |
| 175 | + [ind["id"], *params[1:]], | |
| 176 | + ) | |
| 177 | + series: dict[str, list[dict[str, Any]]] = {c["id"]: [] for c in cs} | |
| 178 | + years: set[int] = set() | |
| 179 | + for r in rows: | |
| 180 | + years.add(int(r["year"])) | |
| 181 | + series.setdefault(r["country_id"], []).append({"year": int(r["year"]), "rank": r["rank"], "n": r["n"], "value": clean_float(r["value"]), | |
| 182 | + "pct_rank": clean_float(r.get("pct_rank"))}) | |
| 183 | + return {"meta": meta_block(snap), "indicator": indicator_card(ind), "countries": [country_card(c) for c in cs], | |
| 184 | + "years": sorted(years), "series": series} | |
added
src/countryatlas/api/routers/regions.py
+153 −0
@@ -0,0 +1,153 @@ | ||
| 1 | +"""/regions — country groups (WB regions, income groups, organisations, continents).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends, Query | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import ( | |
| 10 | + clean_float, | |
| 11 | + country_card, | |
| 12 | + group_card, | |
| 13 | + indicator_card, | |
| 14 | + merged_indicator, | |
| 15 | + meta_block, | |
| 16 | + resolve_group, | |
| 17 | + resolve_indicator, | |
| 18 | +) | |
| 19 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 20 | +from countryatlas.api.formatting import format_value | |
| 21 | +from countryatlas.api.provenance import build_provenance | |
| 22 | +from countryatlas.registry import topics as registry_topics | |
| 23 | + | |
| 24 | +router = APIRouter(prefix="/regions", tags=["regions"]) | |
| 25 | + | |
| 26 | +AGG_SPECS = [ # (indicator, aggregation, label) | |
| 27 | + ("population", "sum", "Total population"), | |
| 28 | + ("gdp", "sum", "Total GDP"), | |
| 29 | + ("gdp-per-capita", "weighted_mean", "GDP per capita (population-weighted)"), | |
| 30 | + ("life-expectancy", "median", "Median life expectancy"), | |
| 31 | + ("gdp-growth", "median", "Median GDP growth"), | |
| 32 | + ("inflation", "median", "Median inflation"), | |
| 33 | + ("unemployment-rate", "median", "Median unemployment rate"), | |
| 34 | + ("co2-per-capita", "weighted_mean", "CO₂ per capita (population-weighted)"), | |
| 35 | + ("internet-users", "weighted_mean", "Internet users (population-weighted)"), | |
| 36 | +] | |
| 37 | + | |
| 38 | + | |
| 39 | +def _group_stats(snap: Snapshot, members: list[str]) -> dict[str, Any]: | |
| 40 | + if not members: | |
| 41 | + return {} | |
| 42 | + ph = ",".join("?" * len(members)) | |
| 43 | + rows = snap.query( | |
| 44 | + f""" | |
| 45 | + WITH l AS (SELECT * FROM latest WHERE country_id IN ({ph}) AND value IS NOT NULL), | |
| 46 | + pop AS (SELECT country_id, value AS pop FROM latest WHERE indicator_id = 'population' AND country_id IN ({ph})) | |
| 47 | + SELECT l.indicator_id, sum(l.value) AS total, median(l.value) AS med, avg(l.value) AS mean, count(*) AS n, max(l.year) AS year, | |
| 48 | + sum(l.value * pop.pop) / nullif(sum(CASE WHEN l.value IS NOT NULL THEN pop.pop END), 0) AS wmean | |
| 49 | + FROM l LEFT JOIN pop ON pop.country_id = l.country_id | |
| 50 | + WHERE l.indicator_id IN ({','.join('?' * len(AGG_SPECS))}) | |
| 51 | + GROUP BY l.indicator_id | |
| 52 | + """, | |
| 53 | + [*members, *members, *[s[0] for s in AGG_SPECS]], | |
| 54 | + ) | |
| 55 | + by = {r["indicator_id"]: r for r in rows} | |
| 56 | + out: dict[str, Any] = {} | |
| 57 | + for iid, agg, label in AGG_SPECS: | |
| 58 | + r = by.get(iid) | |
| 59 | + ind_row = snap.indicators().get(iid) | |
| 60 | + if not r or not ind_row: | |
| 61 | + continue | |
| 62 | + ind = merged_indicator(ind_row) | |
| 63 | + val = clean_float({"sum": r["total"], "median": r["med"], "weighted_mean": r["wmean"] or r["med"]}[agg]) | |
| 64 | + out[iid] = {"indicator": indicator_card(ind), "kind": agg, "label": label, "value": val, "formatted": format_value(val, ind), | |
| 65 | + "n": r["n"], "year": r["year"]} | |
| 66 | + return out | |
| 67 | + | |
| 68 | + | |
| 69 | +@router.get("", response_model=schemas.RegionsResponse, summary="List groups (regions, income, organisations)") | |
| 70 | +def list_regions(kind: str | None = Query(None), snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 71 | + items = [] | |
| 72 | + # one pass: latest population / gdp per country, then sum per group in Python (32 groups × ≤ 218 members) | |
| 73 | + lat = {(r[0], r[1]): r[2] for r in snap.query_rows("SELECT country_id, indicator_id, value FROM latest WHERE indicator_id IN ('population', 'gdp')")} | |
| 74 | + for g in snap.groups().values(): | |
| 75 | + if kind and g.get("kind") != kind: | |
| 76 | + continue | |
| 77 | + members = snap.group_members(g["id"]) | |
| 78 | + pop = sum(v for v in (lat.get((m, "population")) for m in members) if v is not None) if members else None | |
| 79 | + gdp = sum(v for v in (lat.get((m, "gdp")) for m in members) if v is not None) if members else None | |
| 80 | + item = group_card(g) | |
| 81 | + item.update({"description": g.get("description"), "n_members": len(members) or g.get("n_members"), | |
| 82 | + "population_latest": clean_float(pop) if pop else None, "gdp_latest": clean_float(gdp) if gdp else None}) | |
| 83 | + items.append(item) | |
| 84 | + order = {"world": 0, "region": 1, "continent": 2, "income": 3, "org": 4} | |
| 85 | + items.sort(key=lambda g: (order.get(g.get("kind") or "", 9), g.get("name") or "")) | |
| 86 | + return {"meta": meta_block(snap), "n": len(items), "items": items} | |
| 87 | + | |
| 88 | + | |
| 89 | +@router.get("/{slug}", response_model=schemas.RegionResponse, summary="Group page: members, aggregates, member ranking") | |
| 90 | +def get_region(slug: str, indicator: str = Query("gdp-per-capita", description="Indicator for the member ranking"), | |
| 91 | + sort: str | None = Query(None, pattern="^(asc|desc)$"), snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 92 | + g = resolve_group(snap, slug) | |
| 93 | + members = snap.group_members(g["id"]) | |
| 94 | + headline = [s for s in registry_topics()["headline"] if s in snap.indicators()][:6] | |
| 95 | + members_out = [] | |
| 96 | + if members: | |
| 97 | + ph = ",".join("?" * len(members)) | |
| 98 | + rows = snap.query( | |
| 99 | + f"""SELECT l.country_id, l.indicator_id, l.value, l.year, l.rank_world, l.n_world, l.source_id, | |
| 100 | + o.source_dataset, o.source_series_code, o.retrieved_at, o.source_updated_at | |
| 101 | + FROM latest l LEFT JOIN observations o ON o.country_id = l.country_id AND o.indicator_id = l.indicator_id | |
| 102 | + AND o.period = l.period AND o.frequency = l.frequency | |
| 103 | + WHERE l.country_id IN ({ph}) AND l.indicator_id IN ({','.join('?' * len(headline))})""", | |
| 104 | + [*members, *headline], | |
| 105 | + ) | |
| 106 | + by: dict[str, dict[str, Any]] = {} | |
| 107 | + for r in rows: | |
| 108 | + by.setdefault(r["country_id"], {})[r["indicator_id"]] = r | |
| 109 | + for cid in members: | |
| 110 | + c = snap.countries().get(cid) | |
| 111 | + if c is None: | |
| 112 | + continue | |
| 113 | + vals = {} | |
| 114 | + for iid in headline: | |
| 115 | + r = by.get(cid, {}).get(iid) | |
| 116 | + ind = merged_indicator(snap.indicators()[iid]) | |
| 117 | + if r is None or r.get("value") is None: | |
| 118 | + vals[iid] = None | |
| 119 | + continue | |
| 120 | + vals[iid] = {"value": clean_float(r["value"]), "formatted": format_value(clean_float(r["value"]), ind), "year": r["year"], | |
| 121 | + "rank_world": r.get("rank_world"), "n_world": r.get("n_world"), | |
| 122 | + "provenance": build_provenance(snap, iid, r["source_id"], r.get("source_dataset"), r.get("source_series_code"), | |
| 123 | + r.get("retrieved_at"), r.get("source_updated_at"), c.get("iso2"))} | |
| 124 | + members_out.append({**country_card(c), "values": vals}) | |
| 125 | + members_out.sort(key=lambda m: -(m["values"].get("population") or {}).get("value", 0) if m["values"].get("population") else 0) | |
| 126 | + # member ranking on the chosen indicator | |
| 127 | + ind = resolve_indicator(snap, indicator) | |
| 128 | + sort_used = sort or ("asc" if ind.get("higher_is_better") is False else "desc") | |
| 129 | + ranking_rows = [] | |
| 130 | + if members: | |
| 131 | + ph = ",".join("?" * len(members)) | |
| 132 | + rows = snap.query( | |
| 133 | + f"""SELECT l.country_id, l.value, l.year, l.rank_world, l.n_world, l.change_pct, l.change_abs, l.source_id, | |
| 134 | + o.source_dataset, o.source_series_code, o.retrieved_at, o.source_updated_at | |
| 135 | + FROM latest l LEFT JOIN observations o ON o.country_id = l.country_id AND o.indicator_id = l.indicator_id | |
| 136 | + AND o.period = l.period AND o.frequency = l.frequency | |
| 137 | + WHERE l.indicator_id = ? AND l.country_id IN ({ph}) AND l.value IS NOT NULL | |
| 138 | + ORDER BY l.value {'DESC' if sort_used == 'desc' else 'ASC'}""", | |
| 139 | + [ind["id"], *members], | |
| 140 | + ) | |
| 141 | + for i, r in enumerate(rows): | |
| 142 | + c = snap.countries().get(r["country_id"], {"id": r["country_id"]}) | |
| 143 | + ranking_rows.append({"rank": i + 1, "country": country_card(c), "value": clean_float(r["value"]), | |
| 144 | + "formatted": format_value(clean_float(r["value"]), ind), "year": r["year"], "rank_world": r.get("rank_world"), | |
| 145 | + "n_world": r.get("n_world"), "change_pct": clean_float(r.get("change_pct")), "change_abs": clean_float(r.get("change_abs")), | |
| 146 | + "provenance": build_provenance(snap, ind["id"], r["source_id"], r.get("source_dataset"), r.get("source_series_code"), | |
| 147 | + r.get("retrieved_at"), r.get("source_updated_at"), c.get("iso2"))}) | |
| 148 | + return { | |
| 149 | + "meta": meta_block(snap), "group": group_card(g), "description": g.get("description"), "n_members": len(members), | |
| 150 | + "aggregates": _group_stats(snap, members), "headline_indicators": [indicator_card(merged_indicator(snap.indicators()[i])) for i in headline], | |
| 151 | + "members": members_out, | |
| 152 | + "ranking": {"indicator": indicator_card(ind), "sort": sort_used, "n": len(ranking_rows), "rows": ranking_rows}, | |
| 153 | + } | |
added
src/countryatlas/api/routers/search.py
+186 −0
@@ -0,0 +1,186 @@ | ||
| 1 | +"""/search — typed hits from `search_index` (prefix / substring / fuzzy) plus "country + topic" combos. | |
| 2 | + | |
| 3 | +If the snapshot's `search_index` is empty, an equivalent in-memory index is built from countries/indicators/groups/ | |
| 4 | +sources and the topics registry so search always works. | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import re | |
| 9 | +from typing import Any | |
| 10 | + | |
| 11 | +import duckdb | |
| 12 | +from fastapi import APIRouter, Depends, Query | |
| 13 | + | |
| 14 | +from countryatlas.api import schemas | |
| 15 | +from countryatlas.api.common import country_card, merged_indicator, meta_block | |
| 16 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 17 | +from countryatlas.registry import topics as registry_topics | |
| 18 | + | |
| 19 | +router = APIRouter(tags=["search"]) | |
| 20 | + | |
| 21 | +TYPE_LABEL = {"country": "Country", "indicator": "Indicator", "topic": "Topic", "region": "Region", "source": "Source"} | |
| 22 | +URL_PREFIX = {"country": "/countries/", "indicator": "/indicators/", "topic": "/topics/", "region": "/regions/", "source": "/sources/"} | |
| 23 | + | |
| 24 | +SEARCH_SQL = """ | |
| 25 | +SELECT type, id, slug, name, hint, weight, | |
| 26 | + CASE | |
| 27 | + WHEN lower(name) = $q OR lower(slug) = $q OR lower(id) = $q THEN 1.0 | |
| 28 | + WHEN lower(name) LIKE $q || '%' THEN 0.92 | |
| 29 | + WHEN ' ' || lower(name) LIKE '% ' || $q || '%' THEN 0.85 | |
| 30 | + WHEN lower(coalesce(alt_names, '')) LIKE '%' || $q || '%' THEN 0.8 | |
| 31 | + WHEN lower(name) LIKE '%' || $q || '%' THEN 0.75 | |
| 32 | + ELSE jaro_winkler_similarity(lower(name), $q) | |
| 33 | + END AS sim | |
| 34 | +FROM {table} | |
| 35 | +WHERE sim >= $minsim | |
| 36 | +ORDER BY sim * (0.6 + 0.4 * coalesce(weight, 0.5)) DESC, length(name) | |
| 37 | +LIMIT $limit | |
| 38 | +""" | |
| 39 | + | |
| 40 | + | |
| 41 | +def _build_memory_index(snap: Snapshot) -> duckdb.DuckDBPyConnection: | |
| 42 | + rows: list[tuple] = [] | |
| 43 | + for c in snap.countries().values(): | |
| 44 | + if (c.get("kind") or "country") == "aggregate": | |
| 45 | + continue | |
| 46 | + alt = " ".join(filter(None, [c.get("official_name"), c.get("iso2"), c.get("id"), c.get("capital"), c.get("demonym")])) | |
| 47 | + rows.append(("country", c["id"], c.get("slug"), c.get("short_name"), alt, f"Country · {c.get('region_wb_name') or ''}".rstrip(" ·"), 1.0)) | |
| 48 | + for i in snap.indicators().values(): | |
| 49 | + m = merged_indicator(i) | |
| 50 | + alt = " ".join(filter(None, [m.get("short_name"), *(m.get("tags") or []), m.get("subtopic")])) | |
| 51 | + topic_name = next((t["name"] for t in registry_topics()["topics"] if t["id"] == m.get("topic")), m.get("topic")) | |
| 52 | + hint = " · ".join(filter(None, ["Indicator", topic_name, m.get("unit")])) | |
| 53 | + rows.append(("indicator", m["id"], m.get("slug"), m.get("name"), alt, hint, 0.9 if m.get("featured") else 0.7)) | |
| 54 | + for t in registry_topics()["topics"]: | |
| 55 | + rows.append(("topic", t["id"], t["id"], t["name"], t.get("short") or "", f"Topic · {len(t['indicators'])} indicators", 0.6)) | |
| 56 | + for g in snap.groups().values(): | |
| 57 | + rows.append(("region", g["id"], g.get("slug"), g.get("name"), g.get("wb_code") or "", f"Region · {g.get('kind') or ''}", 0.6)) | |
| 58 | + for s in snap.sources().values(): | |
| 59 | + rows.append(("source", s["id"], s["id"], s.get("name"), s.get("organization") or "", "Source", 0.4)) | |
| 60 | + con = duckdb.connect(":memory:") | |
| 61 | + con.execute("CREATE TABLE search_index (type TEXT, id TEXT, slug TEXT, name TEXT, alt_names TEXT, hint TEXT, weight DOUBLE)") | |
| 62 | + con.executemany("INSERT INTO search_index VALUES (?, ?, ?, ?, ?, ?, ?)", rows) | |
| 63 | + return con | |
| 64 | + | |
| 65 | + | |
| 66 | +def _index_conn(snap: Snapshot) -> tuple[Any, bool]: | |
| 67 | + """(connection-like with .query, is_snapshot).""" | |
| 68 | + if snap.table_count("search_index") > 0: | |
| 69 | + return snap, True | |
| 70 | + return snap._cached("memory_search_index", lambda: _build_memory_index(snap)), False | |
| 71 | + | |
| 72 | + | |
| 73 | +def run_search(snap: Snapshot, q: str, limit: int, types: set[str] | None = None, minsim: float = 0.82) -> list[dict[str, Any]]: | |
| 74 | + ql = q.strip().lower() | |
| 75 | + if not ql: | |
| 76 | + return [] | |
| 77 | + conn, is_snap = _index_conn(snap) | |
| 78 | + params = {"q": ql, "minsim": minsim, "limit": limit * 3} | |
| 79 | + if is_snap: | |
| 80 | + rows = snap.query(SEARCH_SQL.format(table="search_index"), params) | |
| 81 | + else: | |
| 82 | + cur = conn.execute(SEARCH_SQL.format(table="search_index"), params) | |
| 83 | + cols = [d[0] for d in cur.description] | |
| 84 | + rows = [dict(zip(cols, r)) for r in cur.fetchall()] | |
| 85 | + hits = [] | |
| 86 | + for r in rows: | |
| 87 | + if types and r["type"] not in types: | |
| 88 | + continue | |
| 89 | + hit = {"type": r["type"], "id": r["id"], "slug": r.get("slug") or r["id"], "name": r["name"], "hint": None, | |
| 90 | + "score": round(min(1.0, float(r["sim"]) * (0.6 + 0.4 * min(1.0, float(r.get("weight") or 0.5)))), 4), | |
| 91 | + "url": URL_PREFIX.get(r["type"], "/") + (r.get("slug") or r["id"])} | |
| 92 | + hit["hint"] = typed_hint(snap, r) | |
| 93 | + if r["type"] == "country": | |
| 94 | + c = snap.countries().get(r["id"]) | |
| 95 | + if c: | |
| 96 | + hit["country"] = country_card(c) | |
| 97 | + if r["type"] == "indicator": | |
| 98 | + hit["indicator"] = r["id"] | |
| 99 | + if r["type"] == "topic": | |
| 100 | + hit["topic"] = r["id"] | |
| 101 | + hits.append(hit) | |
| 102 | + if len(hits) >= limit: | |
| 103 | + break | |
| 104 | + return hits | |
| 105 | + | |
| 106 | + | |
| 107 | +def typed_hint(snap: Snapshot, r: dict[str, Any]) -> str: | |
| 108 | + """'Country · North America', 'Indicator · Economy · % of GDP', 'Topic · 27 indicators', 'Region · OECD', 'Source · World Bank Group'.""" | |
| 109 | + t = r["type"] | |
| 110 | + label = TYPE_LABEL.get(t, t.capitalize()) | |
| 111 | + raw_hint = (r.get("hint") or "").strip() | |
| 112 | + if raw_hint.startswith(label): | |
| 113 | + return raw_hint | |
| 114 | + if t == "country": | |
| 115 | + c = snap.countries().get(r["id"], {}) | |
| 116 | + return " · ".join(filter(None, [label, c.get("region_wb_name") or raw_hint])) | |
| 117 | + if t == "indicator": | |
| 118 | + m = snap.indicators().get(r["id"]) | |
| 119 | + if m: | |
| 120 | + mm = merged_indicator(m) | |
| 121 | + topic_name = next((x["name"] for x in registry_topics()["topics"] if x["id"] == mm.get("topic")), (mm.get("topic") or "").capitalize()) | |
| 122 | + return " · ".join(filter(None, [label, topic_name, mm.get("unit")])) | |
| 123 | + return " · ".join(filter(None, [label, raw_hint])) | |
| 124 | + if t == "topic": | |
| 125 | + tp = next((x for x in registry_topics()["topics"] if x["id"] == r["id"]), None) | |
| 126 | + return f"{label} · {len(tp['indicators'])} indicators" if tp else " · ".join(filter(None, [label, raw_hint])) | |
| 127 | + if t == "region": | |
| 128 | + g = snap.groups().get(r["id"], {}) | |
| 129 | + kind = {"region": "World Bank region", "income": "Income group", "org": "Organisation", "continent": "Continent", "world": "World"}.get(g.get("kind") or "", g.get("kind")) | |
| 130 | + return " · ".join(filter(None, [label, kind or raw_hint, f"{g.get('n_members')} members" if g.get("n_members") else None])) | |
| 131 | + if t == "source": | |
| 132 | + s = snap.sources().get(r["id"], {}) | |
| 133 | + return " · ".join(filter(None, [label, s.get("organization") or raw_hint])) | |
| 134 | + return " · ".join(filter(None, [label, raw_hint])) | |
| 135 | + | |
| 136 | + | |
| 137 | +@router.get("/search", response_model=schemas.SearchResponse, summary="Search countries, indicators, topics, regions, sources") | |
| 138 | +def search(q: str = Query(..., min_length=1, max_length=120), limit: int = Query(10, ge=1, le=50), | |
| 139 | + type: str | None = Query(None, description="Restrict to a type: country|indicator|topic|region|source"), | |
| 140 | + snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 141 | + types = {type} if type else None | |
| 142 | + hits = run_search(snap, q, limit, types) | |
| 143 | + # combos: "<topic|indicator> <country>" or "<country> <topic|indicator>" → country + topic/indicator + combined hit | |
| 144 | + tokens = [t for t in re.split(r"[\s,/]+", q.strip().lower()) if t] | |
| 145 | + if len(tokens) >= 2 and not type: | |
| 146 | + combos: list[dict[str, Any]] = [] | |
| 147 | + for split in range(1, len(tokens)): | |
| 148 | + for left, right in ((tokens[:split], tokens[split:]), (tokens[split:], tokens[:split])): | |
| 149 | + c_hits = run_search(snap, " ".join(left), 1, {"country"}, minsim=0.9) | |
| 150 | + if not c_hits: | |
| 151 | + continue | |
| 152 | + other = run_search(snap, " ".join(right), 3, {"topic", "indicator"}, minsim=0.86) | |
| 153 | + if not other: | |
| 154 | + continue | |
| 155 | + country = c_hits[0] | |
| 156 | + for o in other: | |
| 157 | + if o["type"] == "topic": | |
| 158 | + url = f"/countries/{country['slug']}/{o['id']}" | |
| 159 | + name = f"{country['name']} · {o['name']}" | |
| 160 | + hint = f"Country topic · {o['name']}" | |
| 161 | + else: | |
| 162 | + url = f"/countries/{country['slug']}?indicator={o['slug']}" | |
| 163 | + name = f"{country['name']} · {o['name']}" | |
| 164 | + hint = f"Country indicator · {o.get('hint') or o['name']}" | |
| 165 | + combos.append({"type": "country_topic" if o["type"] == "topic" else "country_indicator", "id": f"{country['id']}:{o['id']}", | |
| 166 | + "slug": url.strip("/"), "name": name, "hint": hint, "score": round(min(1.0, (country["score"] + o["score"]) / 2 + 0.05), 4), | |
| 167 | + "url": url, "country": country.get("country"), "topic": o["id"] if o["type"] == "topic" else None, | |
| 168 | + "indicator": o["id"] if o["type"] == "indicator" else None}) | |
| 169 | + if combos: | |
| 170 | + # also surface the country and the topic/indicator themselves | |
| 171 | + for h in [country, *other]: | |
| 172 | + if not any(x["type"] == h["type"] and x["id"] == h["id"] for x in hits): | |
| 173 | + hits.append(h) | |
| 174 | + break | |
| 175 | + if combos: | |
| 176 | + break | |
| 177 | + seen = set() | |
| 178 | + merged = [] | |
| 179 | + for h in sorted(combos + hits, key=lambda h: -h["score"]): | |
| 180 | + k = (h["type"], h["id"]) | |
| 181 | + if k in seen: | |
| 182 | + continue | |
| 183 | + seen.add(k) | |
| 184 | + merged.append(h) | |
| 185 | + hits = merged[:limit] | |
| 186 | + return {"meta": meta_block(snap), "q": q, "n": len(hits), "hits": hits} | |
added
src/countryatlas/api/routers/series.py
+33 −0
@@ -0,0 +1,33 @@ | ||
| 1 | +"""/series — multi-country × multi-indicator bundle.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends, Query | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import meta_block, parse_csv, resolve_country, resolve_indicator | |
| 10 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 11 | +from countryatlas.api.errors import bad_request | |
| 12 | +from countryatlas.api.routers.countries import fetch_series | |
| 13 | + | |
| 14 | +router = APIRouter(tags=["series"]) | |
| 15 | + | |
| 16 | + | |
| 17 | +@router.get("/series", response_model=schemas.MultiSeriesResponse, summary="Series bundle for several countries and indicators") | |
| 18 | +def get_series( | |
| 19 | + country: str = Query(..., description="Comma-separated ISO3 or slugs (max 20)"), | |
| 20 | + indicator: str = Query(..., description="Comma-separated indicator slugs (max 12)"), | |
| 21 | + from_: int | None = Query(None, alias="from"), | |
| 22 | + to: int | None = Query(None), | |
| 23 | + freq: str | None = Query(None, pattern="^(?i)(A|Q|M)$"), | |
| 24 | + include_forecast: bool = Query(True), | |
| 25 | + snap: Snapshot = Depends(get_snapshot), | |
| 26 | +) -> dict[str, Any]: | |
| 27 | + countries = [resolve_country(snap, c) for c in parse_csv(country, limit=20)] | |
| 28 | + indicators = [resolve_indicator(snap, i) for i in parse_csv(indicator, limit=12)] | |
| 29 | + if not countries or not indicators: | |
| 30 | + raise bad_request("Provide at least one country and one indicator.") | |
| 31 | + series = [fetch_series(snap, c, ind, year_from=from_, year_to=to, freq=freq, include_forecast=include_forecast) | |
| 32 | + for ind in indicators for c in countries] | |
| 33 | + return {"meta": meta_block(snap), "n": len(series), "series": series} | |
added
src/countryatlas/api/routers/sources.py
+68 −0
@@ -0,0 +1,68 @@ | ||
| 1 | +"""/sources — data sources, their datasets/indicators and import runs.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from typing import Any | |
| 5 | + | |
| 6 | +from fastapi import APIRouter, Depends, Query | |
| 7 | + | |
| 8 | +from countryatlas.api import schemas | |
| 9 | +from countryatlas.api.common import indicator_card, merged_indicator, meta_block | |
| 10 | +from countryatlas.api.db import Snapshot, get_snapshot | |
| 11 | +from countryatlas.api.errors import not_found | |
| 12 | +from countryatlas.api.provenance import _iso, source_url | |
| 13 | + | |
| 14 | +router = APIRouter(prefix="/sources", tags=["sources"]) | |
| 15 | + | |
| 16 | + | |
| 17 | +def _source(s: dict[str, Any]) -> dict[str, Any]: | |
| 18 | + out = dict(s) | |
| 19 | + out["last_success_at"] = _iso(s.get("last_success_at")) | |
| 20 | + return out | |
| 21 | + | |
| 22 | + | |
| 23 | +@router.get("", response_model=schemas.SourcesResponse, summary="List data sources") | |
| 24 | +def list_sources(snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 25 | + counts = {r[0]: (r[1], r[2], r[3]) for r in snap.query_rows( | |
| 26 | + "SELECT source_id, count(*), count(DISTINCT indicator_id), max(retrieved_at) FROM observations GROUP BY source_id")} | |
| 27 | + items = [] | |
| 28 | + for s in snap.sources().values(): | |
| 29 | + item = _source(s) | |
| 30 | + n_obs, n_ind, last = counts.get(s["id"], (None, None, None)) | |
| 31 | + item["n_observations"] = s.get("n_observations") or n_obs | |
| 32 | + item["n_indicators"] = s.get("n_indicators") or n_ind | |
| 33 | + item["last_retrieved_at"] = _iso(last) | |
| 34 | + items.append(item) | |
| 35 | + items.sort(key=lambda s: -(s.get("n_observations") or 0)) | |
| 36 | + return {"meta": meta_block(snap), "n": len(items), "items": items} | |
| 37 | + | |
| 38 | + | |
| 39 | +@router.get("/{id}", response_model=schemas.SourceResponse, summary="Source detail: indicators, datasets, import runs, freshness") | |
| 40 | +def get_source(id: str, runs_limit: int = Query(30, ge=1, le=500), snap: Snapshot = Depends(get_snapshot)) -> dict[str, Any]: | |
| 41 | + s = snap.sources().get(id.lower()) | |
| 42 | + if s is None: | |
| 43 | + raise not_found("source", id, "See /api/v1/sources.") | |
| 44 | + isrc = [r for (iid, sid, code), r in snap.indicator_sources().items() if sid == s["id"] and code != "*"] | |
| 45 | + inds = [] | |
| 46 | + for r in sorted(isrc, key=lambda r: (r["indicator_id"], r.get("priority") or 99)): | |
| 47 | + ind_row = snap.indicators().get(r["indicator_id"]) | |
| 48 | + if ind_row is None: | |
| 49 | + continue | |
| 50 | + inds.append({**indicator_card(merged_indicator(ind_row)), "dataset": r.get("dataset"), "series_code": r.get("series_code"), | |
| 51 | + "priority": r.get("priority"), "transform": r.get("transform"), "n_observations": r.get("n_observations"), | |
| 52 | + "n_countries": r.get("n_countries"), "last_year": r.get("last_year"), "last_status": r.get("last_status"), | |
| 53 | + "url": source_url(s["id"], r.get("dataset"), r.get("series_code")) or r.get("source_url")}) | |
| 54 | + runs = snap.query("SELECT * FROM import_runs WHERE connector = ? ORDER BY started_at DESC NULLS LAST LIMIT ?", [s["id"], runs_limit]) | |
| 55 | + for r in runs: | |
| 56 | + r["started_at"], r["finished_at"] = _iso(r.get("started_at")), _iso(r.get("finished_at")) | |
| 57 | + fresh = snap.one("SELECT max(source_updated_at) AS su, max(retrieved_at) AS ra, count(*) AS n, count(DISTINCT indicator_id) AS ni " | |
| 58 | + "FROM observations WHERE source_id = ?", [s["id"]]) or {} | |
| 59 | + datasets = snap.query("SELECT source_dataset AS dataset, count(*) AS n_observations, count(DISTINCT indicator_id) AS n_indicators, " | |
| 60 | + "count(DISTINCT country_id) AS n_countries, min(year) AS first_year, max(year) AS last_year, max(retrieved_at) AS retrieved_at " | |
| 61 | + "FROM observations WHERE source_id = ? GROUP BY 1 ORDER BY n_observations DESC", [s["id"]]) | |
| 62 | + for d in datasets: | |
| 63 | + d["retrieved_at"] = _iso(d.get("retrieved_at")) | |
| 64 | + out = _source(s) | |
| 65 | + out["n_observations"] = out.get("n_observations") or fresh.get("n") | |
| 66 | + out["n_indicators"] = out.get("n_indicators") or fresh.get("ni") | |
| 67 | + return {"meta": meta_block(snap), "source": out, "indicators": inds, "datasets": datasets, "import_runs": runs, | |
| 68 | + "freshness": {"source_updated_at": _iso(fresh.get("su")), "retrieved_at": _iso(fresh.get("ra")), "built_at": snap.built_at}} | |
added
src/countryatlas/api/schemas.py
+570 −0
@@ -0,0 +1,570 @@ | ||
| 1 | +"""Pydantic response models (OpenAPI documentation + light validation). | |
| 2 | + | |
| 3 | +Every value object carries a `Provenance` (docs/ARCHITECTURE.md §8). Every top-level response carries `Meta`. | |
| 4 | +Models are lenient (`extra="allow"`) so routers can enrich payloads without breaking clients. | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +from datetime import date | |
| 9 | +from typing import Any | |
| 10 | + | |
| 11 | +from pydantic import BaseModel, ConfigDict, Field | |
| 12 | + | |
| 13 | + | |
| 14 | +class Base(BaseModel): | |
| 15 | + model_config = ConfigDict(extra="allow") | |
| 16 | + | |
| 17 | + | |
| 18 | +class Meta(Base): | |
| 19 | + built_at: str | None = None | |
| 20 | + run_id: str | None = None | |
| 21 | + generated_at: str | |
| 22 | + | |
| 23 | + | |
| 24 | +class Provenance(Base): | |
| 25 | + source: str | None = None | |
| 26 | + source_name: str | None = None | |
| 27 | + dataset: str | None = None | |
| 28 | + series_code: str | None = None | |
| 29 | + retrieved_at: str | None = None | |
| 30 | + source_updated_at: str | None = None | |
| 31 | + url: str | None = None | |
| 32 | + transform: str | None = None | |
| 33 | + licence: str | None = None | |
| 34 | + | |
| 35 | + | |
| 36 | +class Change(Base): | |
| 37 | + abs: float | None = None | |
| 38 | + pct: float | None = None | |
| 39 | + formatted: str | None = None | |
| 40 | + | |
| 41 | + | |
| 42 | +class Prev(Base): | |
| 43 | + period: date | None = None | |
| 44 | + value: float | None = None | |
| 45 | + | |
| 46 | + | |
| 47 | +class MetricValue(Base): | |
| 48 | + indicator: str | |
| 49 | + indicator_name: str | None = None | |
| 50 | + has_data: bool = True | |
| 51 | + value: float | None = None | |
| 52 | + formatted: str | None = None | |
| 53 | + period: date | None = None | |
| 54 | + year: int | None = None | |
| 55 | + frequency: str | None = None | |
| 56 | + unit: str | None = None | |
| 57 | + unit_short: str | None = None | |
| 58 | + format: str | None = None | |
| 59 | + is_estimate: bool = False | |
| 60 | + is_forecast: bool = False | |
| 61 | + status: str | None = None | |
| 62 | + prev: Prev | None = None | |
| 63 | + change: Change | None = None | |
| 64 | + change_10y: Change | None = None | |
| 65 | + rank_world: int | None = None | |
| 66 | + n_world: int | None = None | |
| 67 | + rank_region: int | None = None | |
| 68 | + n_region: int | None = None | |
| 69 | + rank_income: int | None = None | |
| 70 | + n_income: int | None = None | |
| 71 | + rank_year: int | None = None | |
| 72 | + rank_is_stale: bool = False | |
| 73 | + higher_is_better: bool | None = None | |
| 74 | + sparkline: list[list[int | float | None]] = Field(default_factory=list) | |
| 75 | + provenance: Provenance | None = None | |
| 76 | + | |
| 77 | + | |
| 78 | +class CountryCard(Base): | |
| 79 | + id: str | |
| 80 | + iso2: str | None = None | |
| 81 | + slug: str | None = None | |
| 82 | + name: str | None = None | |
| 83 | + flag: str | None = None | |
| 84 | + region: str | None = None | |
| 85 | + region_name: str | None = None | |
| 86 | + income: str | None = None | |
| 87 | + income_name: str | None = None | |
| 88 | + kind: str | None = None | |
| 89 | + | |
| 90 | + | |
| 91 | +class IndicatorCard(Base): | |
| 92 | + id: str | |
| 93 | + slug: str | |
| 94 | + name: str | None = None | |
| 95 | + short_name: str | None = None | |
| 96 | + topic: str | None = None | |
| 97 | + subtopic: str | None = None | |
| 98 | + unit: str | None = None | |
| 99 | + unit_short: str | None = None | |
| 100 | + format: str | None = None | |
| 101 | + precision: int | None = None | |
| 102 | + frequency: str | None = None | |
| 103 | + aggregation: str | None = None | |
| 104 | + higher_is_better: bool | None = None | |
| 105 | + ranking_eligible: bool | None = None | |
| 106 | + featured: bool | None = None | |
| 107 | + | |
| 108 | + | |
| 109 | +class GroupCard(Base): | |
| 110 | + id: str | |
| 111 | + slug: str | None = None | |
| 112 | + name: str | None = None | |
| 113 | + kind: str | None = None | |
| 114 | + wb_code: str | None = None | |
| 115 | + n_members: int | None = None | |
| 116 | + | |
| 117 | + | |
| 118 | +class CountrySummary(CountryCard): | |
| 119 | + capital: str | None = None | |
| 120 | + continent: str | None = None | |
| 121 | + subregion: str | None = None | |
| 122 | + population_latest: float | None = None | |
| 123 | + population_year: int | None = None | |
| 124 | + gdp_latest: float | None = None | |
| 125 | + gdp_year: int | None = None | |
| 126 | + gdp_per_capita_latest: float | None = None | |
| 127 | + gdp_per_capita_year: int | None = None | |
| 128 | + coverage_pct: float | None = None | |
| 129 | + n_indicators: int | None = None | |
| 130 | + | |
| 131 | + | |
| 132 | +class CountriesResponse(Base): | |
| 133 | + meta: Meta | |
| 134 | + n: int | |
| 135 | + filters: dict[str, Any] | |
| 136 | + items: list[CountrySummary] | |
| 137 | + | |
| 138 | + | |
| 139 | +class Coverage(Base): | |
| 140 | + n_indicators: int | None = None | |
| 141 | + n_observations: int | None = None | |
| 142 | + latest_year: int | None = None | |
| 143 | + coverage_pct: float | None = None | |
| 144 | + updated_at: str | None = None | |
| 145 | + | |
| 146 | + | |
| 147 | +class Freshness(Base): | |
| 148 | + source_updated_at: str | None = None | |
| 149 | + retrieved_at: str | None = None | |
| 150 | + built_at: str | None = None | |
| 151 | + | |
| 152 | + | |
| 153 | +class TopicSummary(Base): | |
| 154 | + id: str | |
| 155 | + name: str | |
| 156 | + short: str | None = None | |
| 157 | + order: int | None = None | |
| 158 | + blurb: str | None = None | |
| 159 | + n_indicators: int | |
| 160 | + n_with_data: int | |
| 161 | + | |
| 162 | + | |
| 163 | +class Country(CountryCard): | |
| 164 | + official_name: str | None = None | |
| 165 | + iso3: str | None = None | |
| 166 | + iso_numeric: str | None = None | |
| 167 | + capital: str | None = None | |
| 168 | + continent: str | None = None | |
| 169 | + subregion: str | None = None | |
| 170 | + currency_code: str | None = None | |
| 171 | + currency_name: str | None = None | |
| 172 | + area_km2: float | None = None | |
| 173 | + latitude: float | None = None | |
| 174 | + longitude: float | None = None | |
| 175 | + un_member: bool | None = None | |
| 176 | + independent: bool | None = None | |
| 177 | + landlocked: bool | None = None | |
| 178 | + borders: list[str] | None = None | |
| 179 | + languages: list[str] | None = None | |
| 180 | + demonym: str | None = None | |
| 181 | + status: str | None = None | |
| 182 | + | |
| 183 | + | |
| 184 | +class CountryResponse(Base): | |
| 185 | + meta: Meta | |
| 186 | + country: Country | |
| 187 | + groups: list[GroupCard] | |
| 188 | + coverage: Coverage | None = None | |
| 189 | + freshness: Freshness | |
| 190 | + headline: list[MetricValue] | |
| 191 | + topics: list[TopicSummary] | |
| 192 | + neighbours: list[CountryCard] = Field(default_factory=list) | |
| 193 | + | |
| 194 | + | |
| 195 | +class SubtopicBlock(Base): | |
| 196 | + subtopic: str | |
| 197 | + indicators: list[MetricValue] | |
| 198 | + | |
| 199 | + | |
| 200 | +class CountryTopicResponse(Base): | |
| 201 | + meta: Meta | |
| 202 | + country: CountryCard | |
| 203 | + topic: dict[str, Any] | |
| 204 | + n_with_data: int | |
| 205 | + n_indicators: int | |
| 206 | + subtopics: list[SubtopicBlock] | |
| 207 | + | |
| 208 | + | |
| 209 | +class SeriesValue(Base): | |
| 210 | + period: date | None = None | |
| 211 | + year: int | None = None | |
| 212 | + frequency: str | None = None | |
| 213 | + value: float | None = None | |
| 214 | + is_forecast: bool = False | |
| 215 | + is_estimate: bool = False | |
| 216 | + status: str | None = None | |
| 217 | + source_id: str | None = None | |
| 218 | + provenance: Provenance | None = None | |
| 219 | + | |
| 220 | + | |
| 221 | +class SeriesStats(Base): | |
| 222 | + min: dict[str, Any] | None = None | |
| 223 | + max: dict[str, Any] | None = None | |
| 224 | + first: dict[str, Any] | None = None | |
| 225 | + last: dict[str, Any] | None = None | |
| 226 | + cagr: float | None = None | |
| 227 | + n: int = 0 | |
| 228 | + | |
| 229 | + | |
| 230 | +class Series(Base): | |
| 231 | + indicator: IndicatorCard | |
| 232 | + country: CountryCard | |
| 233 | + unit: str | None = None | |
| 234 | + frequency: str | None = None | |
| 235 | + values: list[SeriesValue] | |
| 236 | + alternatives: list[SeriesValue] | None = None | |
| 237 | + provenance: Provenance | None = None | |
| 238 | + sources: list[Provenance] = Field(default_factory=list) | |
| 239 | + stats: SeriesStats | |
| 240 | + | |
| 241 | + | |
| 242 | +class SeriesResponse(Series): | |
| 243 | + meta: Meta | |
| 244 | + | |
| 245 | + | |
| 246 | +class MultiSeriesResponse(Base): | |
| 247 | + meta: Meta | |
| 248 | + n: int | |
| 249 | + series: list[Series] | |
| 250 | + | |
| 251 | + | |
| 252 | +class ChangeItem(Base): | |
| 253 | + id: str | None = None | |
| 254 | + country: CountryCard | None = None | |
| 255 | + indicator: IndicatorCard | None = None | |
| 256 | + kind: str | None = None | |
| 257 | + period: date | None = None | |
| 258 | + year: int | None = None | |
| 259 | + value: float | None = None | |
| 260 | + ref_value: float | None = None | |
| 261 | + delta: float | None = None | |
| 262 | + delta_pct: float | None = None | |
| 263 | + window_years: int | None = None | |
| 264 | + severity: float | None = None | |
| 265 | + headline: str | None = None | |
| 266 | + detail: Any = None | |
| 267 | + detected_at: str | None = None | |
| 268 | + formatted: str | None = None | |
| 269 | + provenance: Provenance | None = None | |
| 270 | + | |
| 271 | + | |
| 272 | +class ChangesResponse(Base): | |
| 273 | + meta: Meta | |
| 274 | + n: int | |
| 275 | + items: list[ChangeItem] | |
| 276 | + | |
| 277 | + | |
| 278 | +class SimilarPeer(Base): | |
| 279 | + country: CountryCard | |
| 280 | + score: float | None = None | |
| 281 | + rank: int | None = None | |
| 282 | + contributions: Any = None | |
| 283 | + | |
| 284 | + | |
| 285 | +class SimilarResponse(Base): | |
| 286 | + meta: Meta | |
| 287 | + country: CountryCard | |
| 288 | + mode: str | |
| 289 | + modes: list[str] | |
| 290 | + peers: list[SimilarPeer] | |
| 291 | + | |
| 292 | + | |
| 293 | +class Insight(Base): | |
| 294 | + id: str | None = None | |
| 295 | + template_id: str | None = None | |
| 296 | + text: str | |
| 297 | + values: Any = None | |
| 298 | + indicators: list[str] = Field(default_factory=list) | |
| 299 | + computed_at: str | None = None | |
| 300 | + provenance: list[Provenance] = Field(default_factory=list) | |
| 301 | + | |
| 302 | + | |
| 303 | +class InsightsResponse(Base): | |
| 304 | + meta: Meta | |
| 305 | + country: CountryCard | |
| 306 | + items: list[Insight] | |
| 307 | + | |
| 308 | + | |
| 309 | +class DNAResponse(Base): | |
| 310 | + meta: Meta | |
| 311 | + country: CountryCard | |
| 312 | + dims: dict[str, float | None] | |
| 313 | + year_ref: int | None = None | |
| 314 | + dimensions: list[dict[str, Any]] = Field(default_factory=list) | |
| 315 | + | |
| 316 | + | |
| 317 | +class IndicatorSummary(IndicatorCard): | |
| 318 | + description: str | None = None | |
| 319 | + n_countries: int | None = None | |
| 320 | + n_observations: int | None = None | |
| 321 | + first_year: int | None = None | |
| 322 | + last_year: int | None = None | |
| 323 | + latest_source_updated_at: str | None = None | |
| 324 | + primary_source_id: str | None = None | |
| 325 | + coverage_pct: float | None = None | |
| 326 | + | |
| 327 | + | |
| 328 | +class IndicatorsResponse(Base): | |
| 329 | + meta: Meta | |
| 330 | + n: int | |
| 331 | + filters: dict[str, Any] | |
| 332 | + items: list[IndicatorSummary] | |
| 333 | + | |
| 334 | + | |
| 335 | +class IndicatorSource(Base): | |
| 336 | + source_id: str | |
| 337 | + source_name: str | None = None | |
| 338 | + dataset: str | None = None | |
| 339 | + series_code: str | None = None | |
| 340 | + priority: int | None = None | |
| 341 | + transform: str | None = None | |
| 342 | + countries: list[str] | None = None | |
| 343 | + notes: str | None = None | |
| 344 | + url: str | None = None | |
| 345 | + licence: str | None = None | |
| 346 | + n_observations: int | None = None | |
| 347 | + n_countries: int | None = None | |
| 348 | + last_year: int | None = None | |
| 349 | + last_status: str | None = None | |
| 350 | + | |
| 351 | + | |
| 352 | +class WorldLatest(Base): | |
| 353 | + kind: str | |
| 354 | + value: float | None = None | |
| 355 | + formatted: str | None = None | |
| 356 | + year: int | None = None | |
| 357 | + n: int | None = None | |
| 358 | + median: float | None = None | |
| 359 | + weighted_mean: float | None = None | |
| 360 | + mean: float | None = None | |
| 361 | + sum: float | None = None | |
| 362 | + weights: str | None = None | |
| 363 | + | |
| 364 | + | |
| 365 | +class RankedValue(Base): | |
| 366 | + country: CountryCard | |
| 367 | + value: float | None = None | |
| 368 | + formatted: str | None = None | |
| 369 | + year: int | None = None | |
| 370 | + rank: int | None = None | |
| 371 | + provenance: Provenance | None = None | |
| 372 | + | |
| 373 | + | |
| 374 | +class IndicatorResponse(Base): | |
| 375 | + meta: Meta | |
| 376 | + indicator: IndicatorSummary | |
| 377 | + sources: list[IndicatorSource] | |
| 378 | + coverage: dict[str, Any] | |
| 379 | + world_latest: WorldLatest | None = None | |
| 380 | + freshness: Freshness | |
| 381 | + top5: list[RankedValue] | |
| 382 | + bottom5: list[RankedValue] | |
| 383 | + years: dict[str, Any] | |
| 384 | + topics: list[str] = Field(default_factory=list) | |
| 385 | + | |
| 386 | + | |
| 387 | +class MapLegend(Base): | |
| 388 | + min: float | None = None | |
| 389 | + max: float | None = None | |
| 390 | + breaks: list[float] | |
| 391 | + n_classes: int | |
| 392 | + | |
| 393 | + | |
| 394 | +class MapResponse(Base): | |
| 395 | + meta: Meta | |
| 396 | + indicator: IndicatorCard | |
| 397 | + year: int | None = None | |
| 398 | + year_used: int | None = None | |
| 399 | + nearest: bool = False | |
| 400 | + values: dict[str, float | None] | |
| 401 | + years: dict[str, int] | None = None | |
| 402 | + formatted: dict[str, str] | None = None | |
| 403 | + legend: MapLegend | |
| 404 | + n: int | |
| 405 | + provenance: Provenance | None = None | |
| 406 | + sources: list[Provenance] = Field(default_factory=list) | |
| 407 | + | |
| 408 | + | |
| 409 | +class TrendPoint(Base): | |
| 410 | + year: int | |
| 411 | + median: float | None = None | |
| 412 | + mean: float | None = None | |
| 413 | + weighted_mean: float | None = None | |
| 414 | + sum: float | None = None | |
| 415 | + n: int | |
| 416 | + | |
| 417 | + | |
| 418 | +class TrendResponse(Base): | |
| 419 | + meta: Meta | |
| 420 | + indicator: IndicatorCard | |
| 421 | + group: GroupCard | |
| 422 | + preferred: str | |
| 423 | + weights: str | None = None | |
| 424 | + points: list[TrendPoint] | |
| 425 | + provenance: list[Provenance] = Field(default_factory=list) | |
| 426 | + | |
| 427 | + | |
| 428 | +class RankingRow(Base): | |
| 429 | + rank: int | |
| 430 | + rank_world: int | None = None | |
| 431 | + n_world: int | None = None | |
| 432 | + pct_rank: float | None = None | |
| 433 | + country: CountryCard | |
| 434 | + value: float | None = None | |
| 435 | + formatted: str | None = None | |
| 436 | + year: int | None = None | |
| 437 | + change_1y: Change | None = None | |
| 438 | + change_10y: Change | None = None | |
| 439 | + sparkline: list[list[int | float | None]] = Field(default_factory=list) | |
| 440 | + provenance: Provenance | None = None | |
| 441 | + | |
| 442 | + | |
| 443 | +class RankingResponse(Base): | |
| 444 | + meta: Meta | |
| 445 | + indicator: IndicatorCard | |
| 446 | + group: GroupCard | |
| 447 | + year: int | None = None | |
| 448 | + year_used: int | None = None | |
| 449 | + years_available: list[int] | |
| 450 | + sort: str | |
| 451 | + n: int | |
| 452 | + limit: int | |
| 453 | + offset: int | |
| 454 | + rows: list[RankingRow] | |
| 455 | + | |
| 456 | + | |
| 457 | +class RankingsListResponse(Base): | |
| 458 | + meta: Meta | |
| 459 | + n: int | |
| 460 | + items: list[IndicatorSummary] | |
| 461 | + | |
| 462 | + | |
| 463 | +class RankHistoryResponse(Base): | |
| 464 | + meta: Meta | |
| 465 | + indicator: IndicatorCard | |
| 466 | + countries: list[CountryCard] | |
| 467 | + years: list[int] | |
| 468 | + series: dict[str, list[dict[str, Any]]] | |
| 469 | + | |
| 470 | + | |
| 471 | +class CompareResponse(Base): | |
| 472 | + meta: Meta | |
| 473 | + mode: str | |
| 474 | + base_year: int | None = None | |
| 475 | + countries: list[CountryCard] | |
| 476 | + indicators: list[IndicatorCard] | |
| 477 | + series: list[Series] | |
| 478 | + | |
| 479 | + | |
| 480 | +class CompareSnapshotResponse(Base): | |
| 481 | + meta: Meta | |
| 482 | + topic: dict[str, Any] | None = None | |
| 483 | + countries: list[CountryCard] | |
| 484 | + rows: list[dict[str, Any]] | |
| 485 | + | |
| 486 | + | |
| 487 | +class RegionsResponse(Base): | |
| 488 | + meta: Meta | |
| 489 | + n: int | |
| 490 | + items: list[dict[str, Any]] | |
| 491 | + | |
| 492 | + | |
| 493 | +class RegionResponse(Base): | |
| 494 | + meta: Meta | |
| 495 | + group: GroupCard | |
| 496 | + description: str | None = None | |
| 497 | + n_members: int | |
| 498 | + aggregates: dict[str, Any] | |
| 499 | + members: list[dict[str, Any]] | |
| 500 | + ranking: dict[str, Any] | |
| 501 | + | |
| 502 | + | |
| 503 | +class SearchHit(Base): | |
| 504 | + type: str | |
| 505 | + id: str | |
| 506 | + slug: str | None = None | |
| 507 | + name: str | |
| 508 | + hint: str | None = None | |
| 509 | + score: float | |
| 510 | + url: str | None = None | |
| 511 | + country: CountryCard | None = None | |
| 512 | + topic: str | None = None | |
| 513 | + indicator: str | None = None | |
| 514 | + | |
| 515 | + | |
| 516 | +class SearchResponse(Base): | |
| 517 | + meta: Meta | |
| 518 | + q: str | |
| 519 | + n: int | |
| 520 | + hits: list[SearchHit] | |
| 521 | + | |
| 522 | + | |
| 523 | +class HomeResponse(Base): | |
| 524 | + meta: Meta | |
| 525 | + snapshot: dict[str, Any] | |
| 526 | + lists: dict[str, Any] | |
| 527 | + recent_changes: list[ChangeItem] | |
| 528 | + recently_updated: list[IndicatorSummary] | |
| 529 | + featured_indicators: list[IndicatorSummary] | |
| 530 | + trending: list[IndicatorSummary] | |
| 531 | + | |
| 532 | + | |
| 533 | +class Source(Base): | |
| 534 | + id: str | |
| 535 | + name: str | None = None | |
| 536 | + organization: str | None = None | |
| 537 | + url: str | None = None | |
| 538 | + licence: str | None = None | |
| 539 | + attribution: str | None = None | |
| 540 | + api_base: str | None = None | |
| 541 | + last_success_at: str | None = None | |
| 542 | + n_indicators: int | None = None | |
| 543 | + n_observations: int | None = None | |
| 544 | + notes: str | None = None | |
| 545 | + | |
| 546 | + | |
| 547 | +class SourcesResponse(Base): | |
| 548 | + meta: Meta | |
| 549 | + n: int | |
| 550 | + items: list[Source] | |
| 551 | + | |
| 552 | + | |
| 553 | +class SourceResponse(Base): | |
| 554 | + meta: Meta | |
| 555 | + source: Source | |
| 556 | + indicators: list[dict[str, Any]] | |
| 557 | + import_runs: list[dict[str, Any]] | |
| 558 | + freshness: Freshness | |
| 559 | + | |
| 560 | + | |
| 561 | +class HealthResponse(Base): | |
| 562 | + status: str | |
| 563 | + run_id: str | None = None | |
| 564 | + built_at: str | None = None | |
| 565 | + observations: int | None = None | |
| 566 | + countries: int | None = None | |
| 567 | + indicators: int | None = None | |
| 568 | + db_path: str | None = None | |
| 569 | + version: str | None = None | |
| 570 | + cache: dict[str, int] | None = None | |
added
src/countryatlas/cli.py
+295 −0
@@ -0,0 +1,295 @@ | ||
| 1 | +"""`ca` — CountryAtlas pipeline CLI (typer). See docs/PIPELINE.md.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import json | |
| 5 | +import logging | |
| 6 | +import sys | |
| 7 | +from pathlib import Path | |
| 8 | +from typing import Annotated | |
| 9 | + | |
| 10 | +import typer | |
| 11 | +from rich.console import Console | |
| 12 | +from rich.table import Table | |
| 13 | + | |
| 14 | +from countryatlas.config import settings | |
| 15 | +from countryatlas.pipeline import setup_logging | |
| 16 | + | |
| 17 | +app = typer.Typer(name="ca", help="CountryAtlas data pipeline.", no_args_is_help=True, add_completion=False) | |
| 18 | +registry_app = typer.Typer(help="Registry (YAML) commands.") | |
| 19 | +export_app = typer.Typer(help="Export observations as CSV/JSON/Parquet.") | |
| 20 | +app.add_typer(registry_app, name="registry") | |
| 21 | +app.add_typer(export_app, name="export") | |
| 22 | +console = Console(stderr=True) | |
| 23 | +out = Console() | |
| 24 | + | |
| 25 | +ConnectorOpt = Annotated[list[str] | None, typer.Option("--connector", "-c", help="Connector id (repeatable).")] | |
| 26 | +IndicatorOpt = Annotated[str | None, typer.Option("--indicator", "-i", help="Restrict to one indicator slug.")] | |
| 27 | + | |
| 28 | + | |
| 29 | +@app.callback() | |
| 30 | +def _main(verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Debug logging.")] = False) -> None: | |
| 31 | + setup_logging(level=logging.DEBUG if verbose else logging.INFO) | |
| 32 | + | |
| 33 | + | |
| 34 | +# ------------------------------------------------------------------------------------------------------- registry | |
| 35 | +@registry_app.command("validate") | |
| 36 | +def registry_validate() -> None: | |
| 37 | + """YAML sanity: unique slugs, known connectors/topics, indicator references, similarity/insights files.""" | |
| 38 | + from countryatlas import registry | |
| 39 | + from countryatlas.pipeline.insights import insight_templates | |
| 40 | + from countryatlas.pipeline.similarity import similarity_config | |
| 41 | + | |
| 42 | + problems = registry.validate_registry() | |
| 43 | + known = set(registry.indicators_by_id()) | |
| 44 | + for mode, spec in (similarity_config().get("modes") or {}).items(): | |
| 45 | + for f in spec.get("features", []): | |
| 46 | + for key in ("indicator", "per"): | |
| 47 | + if f.get(key) and f[key] not in known: | |
| 48 | + problems.append(f"similarity.yaml[{mode}]: unknown indicator {f[key]}") | |
| 49 | + for dim, parts in (similarity_config().get("dna") or {}).items(): | |
| 50 | + for p in parts: | |
| 51 | + if p["indicator"] not in known: | |
| 52 | + problems.append(f"similarity.yaml[dna.{dim}]: unknown indicator {p['indicator']}") | |
| 53 | + for t in insight_templates(): | |
| 54 | + if t["indicator"] not in known: | |
| 55 | + problems.append(f"insights.yaml[{t['id']}]: unknown indicator {t['indicator']}") | |
| 56 | + n_c, n_i, n_g = len(registry.countries()), len(registry.indicators()), len(registry.groups()) | |
| 57 | + n_s = len(registry.source_specs()) | |
| 58 | + out.print(f"countries={n_c} indicators={n_i} groups={n_g} source specs={n_s}") | |
| 59 | + hard = [p for p in problems if "without sources" not in p] | |
| 60 | + for p in problems: | |
| 61 | + out.print(("[yellow]warn[/] " if p not in hard else "[red]error[/] ") + p) | |
| 62 | + if hard: | |
| 63 | + raise typer.Exit(code=1) | |
| 64 | + out.print("[green]registry OK[/]") | |
| 65 | + | |
| 66 | + | |
| 67 | +# ------------------------------------------------------------------------------------------------------- pipeline | |
| 68 | +@app.command() | |
| 69 | +def fetch(connector: ConnectorOpt = None, indicator: IndicatorOpt = None, | |
| 70 | + concurrency: Annotated[int | None, typer.Option(help="HTTP workers (default settings.http_concurrency).")] = None) -> None: | |
| 71 | + """Download raw payloads, normalize, validate and write staging parquet (one file per source spec).""" | |
| 72 | + from countryatlas.pipeline.fetch import run_fetch | |
| 73 | + | |
| 74 | + s = run_fetch(connectors=connector, indicator=indicator, concurrency=concurrency) | |
| 75 | + _print_fetch_summary(s) | |
| 76 | + if not s.runs and not s.skipped_connectors: | |
| 77 | + raise typer.Exit(code=1) | |
| 78 | + | |
| 79 | + | |
| 80 | +@app.command() | |
| 81 | +def normalize(connector: ConnectorOpt = None, indicator: IndicatorOpt = None) -> None: | |
| 82 | + """Re-normalize from the latest raw files (no download) and rewrite staging.""" | |
| 83 | + from countryatlas.pipeline.fetch import run_fetch | |
| 84 | + | |
| 85 | + s = run_fetch(connectors=connector, indicator=indicator, mode="normalize") | |
| 86 | + _print_fetch_summary(s) | |
| 87 | + | |
| 88 | + | |
| 89 | +@app.command() | |
| 90 | +def validate(connector: ConnectorOpt = None) -> None: | |
| 91 | + """Re-run the generic validation rules on every staging file (statuses + issues sidecars rewritten).""" | |
| 92 | + import polars as pl | |
| 93 | + | |
| 94 | + from countryatlas import registry | |
| 95 | + from countryatlas.pipeline.staging import list_staging_files, write_issues, write_parquet_atomic | |
| 96 | + from countryatlas.pipeline.validate import validate_frame | |
| 97 | + | |
| 98 | + specs = {(s.connector, s.dataset, s.code, s.indicator_id): s for s in registry.source_specs()} | |
| 99 | + from countryatlas.pipeline.staging import spec_stem | |
| 100 | + | |
| 101 | + by_stem = {(s.connector, spec_stem(s)): s for s in specs.values()} | |
| 102 | + table = Table(title="validation", show_lines=False) | |
| 103 | + for col in ("connector", "spec", "rows", "quarantined", "warnings", "stale", "dataset"): | |
| 104 | + table.add_column(col) | |
| 105 | + files = [f for f in list_staging_files() if not connector or f.parent.name in connector] | |
| 106 | + for f in files: | |
| 107 | + spec = by_stem.get((f.parent.name, f.stem)) | |
| 108 | + if spec is None: | |
| 109 | + console.print(f"[yellow]orphan staging file (spec no longer in registry): {f}[/]") | |
| 110 | + continue | |
| 111 | + ind = registry.indicators_by_id()[spec.indicator_id] | |
| 112 | + gv = validate_frame(pl.read_parquet(f), ind, check_partial=False) | |
| 113 | + write_parquet_atomic(gv.frame, f) | |
| 114 | + write_issues(spec, gv.issues) | |
| 115 | + table.add_row(spec.connector, f.stem, str(gv.frame.height), str(gv.n_quarantined), str(gv.n_warning), str(gv.n_stale), | |
| 116 | + "[red]QUARANTINE[/]" if gv.quarantine_dataset else "ok") | |
| 117 | + out.print(table) | |
| 118 | + | |
| 119 | + | |
| 120 | +@app.command() | |
| 121 | +def build(no_strict: Annotated[bool, typer.Option("--no-strict", help="Integrity problems become warnings.")] = False, | |
| 122 | + no_swap: Annotated[bool, typer.Option("--no-swap", help="Build only; do not replace the live DB.")] = False) -> None: | |
| 123 | + """Merge staging by source priority into a new DuckDB snapshot, compute derived tables, swap atomically.""" | |
| 124 | + from countryatlas.pipeline.build import build as _build | |
| 125 | + | |
| 126 | + r = _build(strict=not no_strict, swap=not no_swap) | |
| 127 | + out.print(json.dumps({"run_id": r.run_id, "db": str(r.db_path), "duration_s": round(r.duration_s, 1), **r.counts}, indent=1)) | |
| 128 | + for w in r.warnings: | |
| 129 | + console.print(f"[yellow]integrity warning:[/] {w}") | |
| 130 | + | |
| 131 | + | |
| 132 | +@app.command() | |
| 133 | +def refresh(connector: ConnectorOpt = None, indicator: IndicatorOpt = None, | |
| 134 | + no_strict: Annotated[bool, typer.Option("--no-strict")] = False) -> None: | |
| 135 | + """fetch + normalize + validate + build (idempotent).""" | |
| 136 | + from countryatlas.pipeline.build import refresh as _refresh | |
| 137 | + | |
| 138 | + summary = _refresh(connectors=connector, indicator=indicator, strict=not no_strict) | |
| 139 | + out.print(json.dumps(summary, indent=1, default=str)) | |
| 140 | + | |
| 141 | + | |
| 142 | +@app.command() | |
| 143 | +def schedule(connector: ConnectorOpt = None, | |
| 144 | + now: Annotated[bool, typer.Option("--now", help="Run a refresh immediately, then keep the schedule.")] = False) -> None: | |
| 145 | + """Long-running loop: refresh daily at CA_REFRESH_HOUR:CA_REFRESH_MINUTE (America/Toronto); SIGUSR1 = refresh now.""" | |
| 146 | + from countryatlas.pipeline.scheduler import serve | |
| 147 | + | |
| 148 | + serve(connectors=connector, run_immediately=now) | |
| 149 | + | |
| 150 | + | |
| 151 | +# --------------------------------------------------------------------------------------------------------- status | |
| 152 | +@app.command() | |
| 153 | +def status() -> None: | |
| 154 | + """Connectors, last runs, rows, freshness and DB meta.""" | |
| 155 | + from countryatlas.connectors import available_ids | |
| 156 | + from countryatlas.pipeline.staging import list_runs, list_staging_files | |
| 157 | + from countryatlas.registry import CONNECTORS, source_specs | |
| 158 | + | |
| 159 | + runs = list_runs() | |
| 160 | + files = {f.parent.name: [] for f in list_staging_files()} | |
| 161 | + for f in list_staging_files(): | |
| 162 | + files[f.parent.name].append(f) | |
| 163 | + impl = set(available_ids()) | |
| 164 | + specs_by_conn: dict[str, int] = {} | |
| 165 | + for s in source_specs(): | |
| 166 | + specs_by_conn[s.connector] = specs_by_conn.get(s.connector, 0) + 1 | |
| 167 | + t = Table(title=f"connectors (data dir {settings.data_dir})") | |
| 168 | + for col in ("connector", "implemented", "specs", "staging files", "last run", "ok", "failed/quar.", "rows norm"): | |
| 169 | + t.add_column(col) | |
| 170 | + for cid in CONNECTORS: | |
| 171 | + rs = [r for r in runs if r.connector == cid] | |
| 172 | + last = max((r.finished_at or r.started_at for r in rs), default=None) | |
| 173 | + ok = sum(1 for r in rs if r.status in ("ok", "partial")) | |
| 174 | + bad = sum(1 for r in rs if r.status in ("failed", "quarantined")) | |
| 175 | + rows = sum(r.rows_norm for r in rs if r.status in ("ok", "partial")) | |
| 176 | + t.add_row(cid, "yes" if cid in impl else "[dim]no[/]", str(specs_by_conn.get(cid, 0)), str(len(files.get(cid, []))), | |
| 177 | + last.strftime("%Y-%m-%d %H:%M") if last else "-", str(ok), f"[red]{bad}[/]" if bad else "0", f"{rows:,}") | |
| 178 | + out.print(t) | |
| 179 | + bad_runs = [r for r in runs if r.status in ("failed", "quarantined")] | |
| 180 | + if bad_runs: | |
| 181 | + tb = Table(title="failed / quarantined specs") | |
| 182 | + tb.add_column("connector"); tb.add_column("spec"); tb.add_column("status"); tb.add_column("message") | |
| 183 | + for r in bad_runs[:60]: | |
| 184 | + tb.add_row(r.connector, r.dataset, r.status, (r.message or "")[:110]) | |
| 185 | + out.print(tb) | |
| 186 | + if settings.db_path.exists(): | |
| 187 | + from countryatlas.pipeline.export import meta | |
| 188 | + | |
| 189 | + m = meta() | |
| 190 | + tm = Table(title=f"snapshot {settings.db_path} ({settings.db_path.stat().st_size / 1e6:.1f} MB)") | |
| 191 | + tm.add_column("key"); tm.add_column("value") | |
| 192 | + for k in ("build_run_id", "built_at", "schema_version", "observation_count", "observations_alt_count", "indicator_count", | |
| 193 | + "country_count", "latest_count", "rankings_count", "changes_count", "events_count", "insights_count", | |
| 194 | + "similarity_count", "connectors", "build_duration_s", "integrity_warnings"): | |
| 195 | + if k in m: | |
| 196 | + tm.add_row(k, m[k]) | |
| 197 | + out.print(tm) | |
| 198 | + else: | |
| 199 | + console.print(f"[yellow]no snapshot yet at {settings.db_path}[/]") | |
| 200 | + hb = settings.data_dir / "scheduler.json" | |
| 201 | + if hb.exists(): | |
| 202 | + out.print("scheduler:", hb.read_text()[:600]) | |
| 203 | + | |
| 204 | + | |
| 205 | +def _print_fetch_summary(s) -> None: | |
| 206 | + t = Table(title=f"run {s.run_id} ({s.duration_s:.0f}s)") | |
| 207 | + for col in ("connector", "ok", "failed/quarantined", "rows norm"): | |
| 208 | + t.add_column(col) | |
| 209 | + for cid, rs in sorted(s.by_connector().items()): | |
| 210 | + ok = [r for r in rs if r.status in ("ok", "partial")] | |
| 211 | + bad = [r for r in rs if r.status in ("failed", "quarantined")] | |
| 212 | + t.add_row(cid, str(len(ok)), f"[red]{len(bad)}[/]" if bad else "0", f"{sum(r.rows_norm for r in ok):,}") | |
| 213 | + out.print(t) | |
| 214 | + for cid in s.skipped_connectors: | |
| 215 | + console.print(f"[yellow]skipped connector {cid} (not implemented yet)[/]") | |
| 216 | + for r in s.failed: | |
| 217 | + console.print(f"[red]✗[/] {r.connector} {r.dataset}: {r.message}") | |
| 218 | + | |
| 219 | + | |
| 220 | +# --------------------------------------------------------------------------------------------------------- export | |
| 221 | +@export_app.command("indicator") | |
| 222 | +def export_indicator_cmd(slug: str, | |
| 223 | + fmt: Annotated[str, typer.Option("--format", "-f", help="csv|json|parquet")] = "csv", | |
| 224 | + out_dir: Annotated[Path | None, typer.Option("--out")] = None) -> None: | |
| 225 | + """Export all observations of one indicator.""" | |
| 226 | + from countryatlas.pipeline.export import export_indicator | |
| 227 | + | |
| 228 | + p = export_indicator(slug, fmt, out_dir) # type: ignore[arg-type] | |
| 229 | + out.print(str(p)) | |
| 230 | + | |
| 231 | + | |
| 232 | +@export_app.command("country") | |
| 233 | +def export_country_cmd(iso3: str, | |
| 234 | + fmt: Annotated[str, typer.Option("--format", "-f", help="csv|json|parquet")] = "csv", | |
| 235 | + out_dir: Annotated[Path | None, typer.Option("--out")] = None) -> None: | |
| 236 | + """Export the full dataset of one country.""" | |
| 237 | + from countryatlas.pipeline.export import export_country | |
| 238 | + | |
| 239 | + p = export_country(iso3, fmt, out_dir) # type: ignore[arg-type] | |
| 240 | + out.print(str(p)) | |
| 241 | + | |
| 242 | + | |
| 243 | +# ------------------------------------------------------------------------------------------------------------ sql | |
| 244 | +@app.command() | |
| 245 | +def sql(query: Annotated[str | None, typer.Argument(help="SQL to run (omit for an interactive prompt).")] = None, | |
| 246 | + limit: Annotated[int, typer.Option(help="Max rows printed.")] = 50) -> None: | |
| 247 | + """Query the live snapshot read-only (DuckDB SQL).""" | |
| 248 | + from countryatlas.pipeline.export import connect_readonly | |
| 249 | + | |
| 250 | + con = connect_readonly() | |
| 251 | + try: | |
| 252 | + if query: | |
| 253 | + _run_sql(con, query, limit) | |
| 254 | + return | |
| 255 | + out.print("[dim]ca sql — DuckDB read-only. Enter SQL, end with ';'. Ctrl-D to quit.[/]") | |
| 256 | + buf: list[str] = [] | |
| 257 | + while True: | |
| 258 | + try: | |
| 259 | + line = input("atlas> " if not buf else " ...> ") | |
| 260 | + except EOFError: | |
| 261 | + break | |
| 262 | + buf.append(line) | |
| 263 | + if line.strip().endswith(";"): | |
| 264 | + _run_sql(con, "\n".join(buf), limit) | |
| 265 | + buf = [] | |
| 266 | + finally: | |
| 267 | + con.close() | |
| 268 | + | |
| 269 | + | |
| 270 | +@app.command("db-shell", hidden=True) | |
| 271 | +def db_shell() -> None: | |
| 272 | + """Alias of `ca sql` without a query.""" | |
| 273 | + sql(None) | |
| 274 | + | |
| 275 | + | |
| 276 | +def _run_sql(con, query: str, limit: int) -> None: | |
| 277 | + try: | |
| 278 | + rel = con.execute(query) | |
| 279 | + rows = rel.fetchmany(limit + 1) | |
| 280 | + cols = [d[0] for d in rel.description] if rel.description else [] | |
| 281 | + except Exception as e: # noqa: BLE001 | |
| 282 | + console.print(f"[red]{type(e).__name__}[/]: {e}") | |
| 283 | + return | |
| 284 | + t = Table(show_lines=False) | |
| 285 | + for c in cols: | |
| 286 | + t.add_column(str(c)) | |
| 287 | + for r in rows[:limit]: | |
| 288 | + t.add_row(*[("" if v is None else str(v)) for v in r]) | |
| 289 | + out.print(t) | |
| 290 | + if len(rows) > limit: | |
| 291 | + console.print(f"[dim]… more rows (limit {limit})[/]") | |
| 292 | + | |
| 293 | + | |
| 294 | +if __name__ == "__main__": # pragma: no cover | |
| 295 | + sys.exit(app()) | |
modified
src/countryatlas/connectors/__init__.py
+90 −0
@@ -0,0 +1,90 @@ | ||
| 1 | +"""Connector registry. | |
| 2 | + | |
| 3 | +Adding a connector = (1) create `countryatlas/connectors/<id>.py` with a `Connector` subclass whose `id` ClassVar equals | |
| 4 | +`<id>`, (2) add one entry to `CONNECTORS` below (module path + class name), (3) map indicators to it in | |
| 5 | +`registry/indicators.yaml`. Connectors whose module does not exist yet are tolerated: `get_connector()` raises | |
| 6 | +`ConnectorNotAvailable`, and the pipeline skips their specs with a logged warning. | |
| 7 | + | |
| 8 | +`discover()` additionally scans the package for `Connector` subclasses so that a module dropped in the directory without a | |
| 9 | +dict entry is still found (the dict entry only makes the mapping explicit and lets us pick the class name). | |
| 10 | +""" | |
| 11 | +from __future__ import annotations | |
| 12 | + | |
| 13 | +import importlib | |
| 14 | +import inspect | |
| 15 | +import logging | |
| 16 | +import pkgutil | |
| 17 | +from functools import lru_cache | |
| 18 | + | |
| 19 | +from countryatlas.connectors.base import Connector | |
| 20 | + | |
| 21 | +log = logging.getLogger(__name__) | |
| 22 | + | |
| 23 | +# id → "module:Class". Modules may not exist yet (written by other agents); that is fine. | |
| 24 | +CONNECTORS: dict[str, str] = { | |
| 25 | + "worldbank": "countryatlas.connectors.worldbank:WorldBankConnector", | |
| 26 | + "owid": "countryatlas.connectors.owid:OWIDConnector", | |
| 27 | + "imf": "countryatlas.connectors.imf:IMFConnector", | |
| 28 | + "oecd": "countryatlas.connectors.oecd:OECDConnector", | |
| 29 | + "eurostat": "countryatlas.connectors.eurostat:EurostatConnector", | |
| 30 | + "who": "countryatlas.connectors.who:WHOConnector", | |
| 31 | + "fred": "countryatlas.connectors.fred:FREDConnector", | |
| 32 | + "bis": "countryatlas.connectors.bis:BISConnector", | |
| 33 | + "ilo": "countryatlas.connectors.ilo:ILOConnector", | |
| 34 | +} | |
| 35 | + | |
| 36 | + | |
| 37 | +class ConnectorNotAvailable(Exception): | |
| 38 | + """The connector is declared in the registry but its module/class is not implemented yet.""" | |
| 39 | + | |
| 40 | + | |
| 41 | +@lru_cache(maxsize=1) | |
| 42 | +def discover() -> dict[str, type[Connector]]: | |
| 43 | + """Return {id: class} for every importable connector (explicit dict entries + package scan).""" | |
| 44 | + found: dict[str, type[Connector]] = {} | |
| 45 | + import countryatlas.connectors as pkg | |
| 46 | + | |
| 47 | + for modinfo in pkgutil.iter_modules(pkg.__path__): | |
| 48 | + if modinfo.name in ("base", "__init__") or modinfo.name.startswith("_"): | |
| 49 | + continue | |
| 50 | + try: | |
| 51 | + mod = importlib.import_module(f"{pkg.__name__}.{modinfo.name}") | |
| 52 | + except Exception as e: # noqa: BLE001 — a broken connector must not break the others | |
| 53 | + log.warning("connector module %s failed to import: %s", modinfo.name, e) | |
| 54 | + continue | |
| 55 | + for _, cls in inspect.getmembers(mod, inspect.isclass): | |
| 56 | + if issubclass(cls, Connector) and cls is not Connector and getattr(cls, "id", ""): | |
| 57 | + found[cls.id] = cls | |
| 58 | + # explicit entries take precedence when both exist | |
| 59 | + for cid, target in CONNECTORS.items(): | |
| 60 | + modname, _, clsname = target.partition(":") | |
| 61 | + try: | |
| 62 | + mod = importlib.import_module(modname) | |
| 63 | + except ModuleNotFoundError: | |
| 64 | + continue | |
| 65 | + cls = getattr(mod, clsname, None) | |
| 66 | + if cls is not None and issubclass(cls, Connector): | |
| 67 | + found[cid] = cls | |
| 68 | + return found | |
| 69 | + | |
| 70 | + | |
| 71 | +def available_ids() -> list[str]: | |
| 72 | + return sorted(discover().keys()) | |
| 73 | + | |
| 74 | + | |
| 75 | +def connector_class(connector_id: str) -> type[Connector]: | |
| 76 | + classes = discover() | |
| 77 | + if connector_id not in classes: | |
| 78 | + raise ConnectorNotAvailable( | |
| 79 | + f"connector '{connector_id}' is declared but not implemented yet " | |
| 80 | + f"(expected {CONNECTORS.get(connector_id, '<module>:<Class>')})" | |
| 81 | + ) | |
| 82 | + return classes[connector_id] | |
| 83 | + | |
| 84 | + | |
| 85 | +def get_connector(connector_id: str) -> Connector: | |
| 86 | + """Instantiate a connector (each call = fresh instance with its own HTTP client and caches).""" | |
| 87 | + return connector_class(connector_id)() | |
| 88 | + | |
| 89 | + | |
| 90 | +__all__ = ["CONNECTORS", "Connector", "ConnectorNotAvailable", "available_ids", "connector_class", "discover", "get_connector"] | |
added
src/countryatlas/connectors/_series.py
+104 −0
@@ -0,0 +1,104 @@ | ||
| 1 | +"""Series-level helpers shared by the WHO / FRED / BIS / ILO connectors (not a connector: `_` prefix → skipped by discover). | |
| 2 | + | |
| 3 | +* `parse_sdmx_period` — '2026-08' → monthly, '2026-Q1' → quarterly, '2026' → annual (SDMX time formats used by BIS/ILO). | |
| 4 | +* `apply_series_transform` — registry transforms that need the whole per-country series rather than a scalar: | |
| 5 | + - `yoy_pct` year-on-year % change (12 months / 4 quarters / 1 year earlier, exact period match) | |
| 6 | + - `rebase:YYYY` re-index so that the mean of year YYYY = 100 (countries without any YYYY observation are dropped) | |
| 7 | + Any other transform string is treated as a scalar python expression on `x` (Connector.apply_transform semantics). | |
| 8 | +* `Point` — the minimal (period, year, frequency, value) tuple these helpers operate on. | |
| 9 | +""" | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import math | |
| 13 | +import re | |
| 14 | +from collections.abc import Iterable | |
| 15 | +from dataclasses import dataclass, replace | |
| 16 | +from datetime import date | |
| 17 | + | |
| 18 | +from countryatlas.connectors._util import parse_period | |
| 19 | +from countryatlas.models import Frequency | |
| 20 | + | |
| 21 | +_REBASE = re.compile(r"^rebase:(\d{4})$") | |
| 22 | +_SDMX_M = re.compile(r"^(\d{4})-(\d{2})$") | |
| 23 | +_SDMX_Q = re.compile(r"^(\d{4})-Q([1-4])$") | |
| 24 | + | |
| 25 | + | |
| 26 | +@dataclass(frozen=True) | |
| 27 | +class Point: | |
| 28 | + period: date | |
| 29 | + year: int | |
| 30 | + frequency: Frequency | |
| 31 | + value: float | |
| 32 | + | |
| 33 | + | |
| 34 | +def parse_sdmx_period(text: str) -> tuple[date, int, Frequency] | None: | |
| 35 | + """SDMX TIME_PERIOD → (first day of period, year, frequency). Falls back to `_util.parse_period`.""" | |
| 36 | + s = text.strip() | |
| 37 | + m = _SDMX_M.match(s) | |
| 38 | + if m: | |
| 39 | + y, mo = int(m.group(1)), int(m.group(2)) | |
| 40 | + if 1 <= mo <= 12: | |
| 41 | + return date(y, mo, 1), y, "M" | |
| 42 | + return None | |
| 43 | + m = _SDMX_Q.match(s) | |
| 44 | + if m: | |
| 45 | + y, q = int(m.group(1)), int(m.group(2)) | |
| 46 | + return date(y, 3 * (q - 1) + 1, 1), y, "Q" | |
| 47 | + return parse_period(s) | |
| 48 | + | |
| 49 | + | |
| 50 | +def previous_year_period(p: date, frequency: Frequency) -> date: | |
| 51 | + """The period exactly one year earlier (same month/quarter).""" | |
| 52 | + return date(p.year - 1, p.month, 1) if frequency != "A" else date(p.year - 1, 1, 1) | |
| 53 | + | |
| 54 | + | |
| 55 | +def is_series_transform(transform: str | None) -> bool: | |
| 56 | + return bool(transform) and (transform == "yoy_pct" or _REBASE.match(transform) is not None) | |
| 57 | + | |
| 58 | + | |
| 59 | +def apply_series_transform(points: Iterable[Point], transform: str | None) -> list[Point]: | |
| 60 | + """Apply a series-level transform to ONE country's points (any frequency, unsorted OK). Returns new points. | |
| 61 | + | |
| 62 | + yoy_pct: value_t / value_{t-1y} × 100 − 100; periods without a predecessor exactly one year earlier are dropped. | |
| 63 | + rebase:YYYY: value / mean(values in YYYY) × 100; if the country has no YYYY observation → [] (cannot be rebased). | |
| 64 | + """ | |
| 65 | + pts = sorted(points, key=lambda p: p.period) | |
| 66 | + if not transform: | |
| 67 | + return pts | |
| 68 | + if transform == "yoy_pct": | |
| 69 | + by_period = {p.period: p.value for p in pts} | |
| 70 | + out: list[Point] = [] | |
| 71 | + for p in pts: | |
| 72 | + prev = by_period.get(previous_year_period(p.period, p.frequency)) | |
| 73 | + if prev is None or prev == 0: | |
| 74 | + continue | |
| 75 | + out.append(replace(p, value=(p.value / prev - 1.0) * 100.0)) | |
| 76 | + return out | |
| 77 | + m = _REBASE.match(transform) | |
| 78 | + if m: | |
| 79 | + base_year = int(m.group(1)) | |
| 80 | + base = [p.value for p in pts if p.year == base_year] | |
| 81 | + if not base: | |
| 82 | + return [] | |
| 83 | + mean = sum(base) / len(base) | |
| 84 | + if mean == 0: | |
| 85 | + return [] | |
| 86 | + return [replace(p, value=p.value / mean * 100.0) for p in pts] | |
| 87 | + # scalar expression on x | |
| 88 | + return [replace(p, value=float(eval(transform, {"__builtins__": {}}, {"x": p.value}))) for p in pts] | |
| 89 | + | |
| 90 | + | |
| 91 | +def to_float(text: str | None) -> float | None: | |
| 92 | + """Parse a numeric cell; '', '.', 'NaN', 'NA' → None.""" | |
| 93 | + if text is None: | |
| 94 | + return None | |
| 95 | + t = text.strip() | |
| 96 | + if t in ("", ".", "NaN", "nan", "NA", "N/A", "null"): | |
| 97 | + return None | |
| 98 | + try: | |
| 99 | + v = float(t) | |
| 100 | + except ValueError: | |
| 101 | + return None | |
| 102 | + if not math.isfinite(v): | |
| 103 | + return None | |
| 104 | + return v | |
added
src/countryatlas/connectors/_util.py
+65 −0
@@ -0,0 +1,65 @@ | ||
| 1 | +"""Small helpers shared by connectors (period parsing, vectorised transforms). Not a connector.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import re | |
| 5 | +from datetime import UTC, date, datetime | |
| 6 | + | |
| 7 | +import numpy as np | |
| 8 | + | |
| 9 | +from countryatlas.models import Frequency | |
| 10 | + | |
| 11 | +_Q = re.compile(r"^(\d{4})[Qq]([1-4])$") | |
| 12 | +_M = re.compile(r"^(\d{4})[Mm]?(\d{2})$") | |
| 13 | + | |
| 14 | + | |
| 15 | +class ConnectorError(Exception): | |
| 16 | + """A non-transient problem with one spec (retired code, empty dataset, unexpected payload).""" | |
| 17 | + | |
| 18 | + | |
| 19 | +def parse_period(text: str) -> tuple[date, int, Frequency] | None: | |
| 20 | + """'2023' → (2023-01-01, 2023, 'A'); '2023Q2' → (2023-04-01, 2023, 'Q'); '2023M07' / '202307' → (2023-07-01, 2023, 'M'). | |
| 21 | + | |
| 22 | + Returns None when the string is not a period we understand (caller skips the row). | |
| 23 | + """ | |
| 24 | + s = text.strip() | |
| 25 | + if len(s) == 4 and s.isdigit(): | |
| 26 | + y = int(s) | |
| 27 | + return date(y, 1, 1), y, "A" | |
| 28 | + m = _Q.match(s) | |
| 29 | + if m: | |
| 30 | + y, q = int(m.group(1)), int(m.group(2)) | |
| 31 | + return date(y, 3 * (q - 1) + 1, 1), y, "Q" | |
| 32 | + m = _M.match(s) | |
| 33 | + if m: | |
| 34 | + y, mo = int(m.group(1)), int(m.group(2)) | |
| 35 | + if 1 <= mo <= 12: | |
| 36 | + return date(y, mo, 1), y, "M" | |
| 37 | + if len(s) == 10 and s[4] == "-" and s[7] == "-": | |
| 38 | + try: | |
| 39 | + d = date.fromisoformat(s) | |
| 40 | + except ValueError: | |
| 41 | + return None | |
| 42 | + return date(d.year, d.month, 1), d.year, "M" | |
| 43 | + return None | |
| 44 | + | |
| 45 | + | |
| 46 | +def apply_transform_array(values: np.ndarray, transform: str | None) -> np.ndarray: | |
| 47 | + """Vectorised version of Connector.apply_transform (registry-controlled expressions on `x`).""" | |
| 48 | + if not transform: | |
| 49 | + return values | |
| 50 | + out = eval(transform, {"__builtins__": {}, "np": np, "log": np.log, "exp": np.exp}, {"x": values}) | |
| 51 | + return np.asarray(out, dtype="float64") | |
| 52 | + | |
| 53 | + | |
| 54 | +def parse_date_utc(text: str | None) -> datetime | None: | |
| 55 | + """Parse 'YYYY-MM-DD' or ISO timestamps into an aware UTC datetime (None if not parseable).""" | |
| 56 | + if not text: | |
| 57 | + return None | |
| 58 | + t = text.strip() | |
| 59 | + try: | |
| 60 | + if len(t) == 10: | |
| 61 | + return datetime.fromisoformat(t).replace(tzinfo=UTC) | |
| 62 | + dt = datetime.fromisoformat(t) | |
| 63 | + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) | |
| 64 | + except ValueError: | |
| 65 | + return None | |
added
src/countryatlas/connectors/bis.py
+186 −0
@@ -0,0 +1,186 @@ | ||
| 1 | +"""Bank for International Settlements — Data Portal SDMX v2 connector (CSV). | |
| 2 | + | |
| 3 | +URL form (verified in docs/sources-research.md §8): | |
| 4 | + https://stats.bis.org/api/v2/data/dataflow/BIS/{dataflow}/1.0/{key}?format=csv | |
| 5 | + | |
| 6 | +Datasets / keys (the spec `code` IS the SDMX key, so one spec = one HTTP call for every reference area): | |
| 7 | +* `WS_CBPOL` policy rates, key `FREQ.REF_AREA` → `M.` (all areas, monthly, end of period, unit 368 = % p.a.) | |
| 8 | +* `WS_SPP` selected residential property prices, key `FREQ.REF_AREA.VALUE.UNIT_MEASURE` | |
| 9 | + → `Q..R.628` real index 2010=100, `Q..N.628` nominal index, `Q..R.771` real y/y % change. | |
| 10 | + | |
| 11 | +CSV columns: FREQ, REF_AREA (ISO2; `XM` euro area, `4T`/`5R`/`XW` aggregates → dropped by the ISO2 lookup), | |
| 12 | +TIME_PERIOD (`2026-08`, `2026-Q1`), OBS_VALUE (`NaN` when OBS_STATUS = M), OBS_STATUS, TITLE/COMPILATION with commas | |
| 13 | +(parsed with the csv module). | |
| 14 | + | |
| 15 | +Transforms: scalar expressions on `x`, plus series-level `rebase:YYYY` (indices are re-based per country so that the mean | |
| 16 | +of YYYY = 100 — the registry indices are 2015 = 100 while BIS publishes 2010 = 100) and `yoy_pct`. | |
| 17 | + | |
| 18 | +Licence: BIS terms of use (free reuse with attribution) — attribution "Bank for International Settlements". | |
| 19 | +""" | |
| 20 | +from __future__ import annotations | |
| 21 | + | |
| 22 | +import csv | |
| 23 | +import io | |
| 24 | +import logging | |
| 25 | +from datetime import datetime | |
| 26 | +from typing import Any, ClassVar | |
| 27 | + | |
| 28 | +from countryatlas.connectors._series import ( | |
| 29 | + Point, | |
| 30 | + apply_series_transform, | |
| 31 | + is_series_transform, | |
| 32 | + parse_sdmx_period, | |
| 33 | + to_float, | |
| 34 | +) | |
| 35 | +from countryatlas.connectors._util import ConnectorError | |
| 36 | +from countryatlas.connectors.base import Connector | |
| 37 | +from countryatlas.models import DatasetDescriptor, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 38 | +from countryatlas.registry import indicators_by_id, lookup | |
| 39 | + | |
| 40 | +log = logging.getLogger(__name__) | |
| 41 | + | |
| 42 | +DATAFLOWS: dict[str, dict[str, str]] = { | |
| 43 | + "WS_CBPOL": { | |
| 44 | + "name": "Central bank policy rates", | |
| 45 | + "url": "https://data.bis.org/topics/CBPOL", | |
| 46 | + "notes": "Monthly end-of-period policy rates for ~34 economies (US = midpoint of the target range).", | |
| 47 | + }, | |
| 48 | + "WS_SPP": { | |
| 49 | + "name": "Selected residential property prices", | |
| 50 | + "url": "https://data.bis.org/topics/RPP", | |
| 51 | + "notes": "Quarterly nominal (N) and real (R) indices 2010=100 (unit 628) and y/y % changes (unit 771), ~59 areas.", | |
| 52 | + }, | |
| 53 | +} | |
| 54 | +SOURCE_URL_PATTERN = "https://data.bis.org/topics/{topic}/BIS,{dataflow},1.0/{key}" | |
| 55 | +TOPIC_BY_DATAFLOW = {"WS_CBPOL": "CBPOL", "WS_SPP": "RPP"} | |
| 56 | + | |
| 57 | + | |
| 58 | +class BISConnector(Connector): | |
| 59 | + id: ClassVar[str] = "bis" | |
| 60 | + name: ClassVar[str] = "Bank for International Settlements" | |
| 61 | + organization: ClassVar[str] = "Bank for International Settlements" | |
| 62 | + url: ClassVar[str] = "https://data.bis.org" | |
| 63 | + licence: ClassVar[str] = "BIS terms and conditions (free reuse with attribution)" | |
| 64 | + attribution: ClassVar[str] = "Bank for International Settlements" | |
| 65 | + api_base: ClassVar[str] = "https://stats.bis.org/api/v2" | |
| 66 | + rate_per_minute: ClassVar[int] = 30 | |
| 67 | + timeout: ClassVar[float] = 180.0 | |
| 68 | + country_codes: ClassVar[str] = "iso2" | |
| 69 | + | |
| 70 | + def discover(self) -> list[DatasetDescriptor]: | |
| 71 | + return [ | |
| 72 | + DatasetDescriptor(connector=self.id, dataset=df, name=d["name"], url=d["url"], licence=self.licence, notes=d["notes"]) | |
| 73 | + for df, d in DATAFLOWS.items() | |
| 74 | + ] | |
| 75 | + | |
| 76 | + # --------------------------------------------------------------------- fetch | |
| 77 | + def data_url(self, dataflow: str, key: str) -> str: | |
| 78 | + return f"{self.api_base}/data/dataflow/BIS/{dataflow}/1.0/{key}" | |
| 79 | + | |
| 80 | + def fetch(self, spec: IndicatorSourceSpec) -> RawPayload: | |
| 81 | + dataflow, key = spec.dataset, spec.code | |
| 82 | + if not dataflow or not key: | |
| 83 | + raise ConnectorError("bis: spec needs dataset (dataflow id) and code (SDMX key)") | |
| 84 | + params: dict[str, Any] = {"format": "csv"} | |
| 85 | + for k, v in (spec.params or {}).items(): # e.g. startPeriod | |
| 86 | + params[k] = v | |
| 87 | + r = self.get(self.data_url(dataflow, key), params=params, headers={"Accept": "text/csv, */*;q=0.5"}) | |
| 88 | + head = r.content[:400] | |
| 89 | + if not head.startswith(b"FREQ,") and b"TIME_PERIOD" not in head: | |
| 90 | + raise ConnectorError(f"bis {dataflow}/{key}: unexpected response (not SDMX-CSV): {head[:80]!r}") | |
| 91 | + p = self.payload( | |
| 92 | + r, | |
| 93 | + dataset=dataflow, | |
| 94 | + code=key, | |
| 95 | + source_url=SOURCE_URL_PATTERN.format(topic=TOPIC_BY_DATAFLOW.get(dataflow, dataflow), dataflow=dataflow, key=key), | |
| 96 | + notes=DATAFLOWS.get(dataflow, {}).get("notes"), | |
| 97 | + ) | |
| 98 | + p.content_type = "text/csv" | |
| 99 | + return p | |
| 100 | + | |
| 101 | + # ----------------------------------------------------------------- normalize | |
| 102 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 103 | + r = raw[0] if isinstance(raw, list) else raw | |
| 104 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 105 | + unit = ind.unit if ind else "" | |
| 106 | + lk = lookup() | |
| 107 | + text = r.body.decode("utf-8-sig", errors="replace") | |
| 108 | + reader = csv.DictReader(io.StringIO(text)) | |
| 109 | + if not reader.fieldnames or "REF_AREA" not in reader.fieldnames or "TIME_PERIOD" not in reader.fieldnames: | |
| 110 | + raise ConnectorError(f"bis {spec.dataset}/{spec.code}: CSV lacks REF_AREA/TIME_PERIOD columns") | |
| 111 | + only = set(spec.countries or []) | |
| 112 | + series: dict[str, list[Point]] = {} | |
| 113 | + status: dict[tuple[str, Any], str] = {} | |
| 114 | + titles: dict[str, str] = {} | |
| 115 | + n_drop = {"area": 0, "null": 0, "period": 0} | |
| 116 | + for row in reader: | |
| 117 | + area = (row.get("REF_AREA") or "").strip() | |
| 118 | + iso3 = lk.from_iso2(area) if len(area) == 2 else lk.from_iso3(area) | |
| 119 | + if iso3 is None or (only and iso3 not in only): | |
| 120 | + n_drop["area"] += 1 | |
| 121 | + continue | |
| 122 | + v = to_float(row.get("OBS_VALUE")) | |
| 123 | + if v is None: | |
| 124 | + n_drop["null"] += 1 | |
| 125 | + continue | |
| 126 | + per = parse_sdmx_period(row.get("TIME_PERIOD") or "") | |
| 127 | + if per is None: | |
| 128 | + n_drop["period"] += 1 | |
| 129 | + continue | |
| 130 | + period, year, freq = per | |
| 131 | + series.setdefault(iso3, []).append(Point(period, year, freq, v)) | |
| 132 | + st = (row.get("OBS_STATUS") or "").strip() | |
| 133 | + if st and st != "A": | |
| 134 | + status[(iso3, period)] = st | |
| 135 | + if iso3 not in titles and (row.get("TITLE") or row.get("TITLE_TS")): | |
| 136 | + titles[iso3] = (row.get("TITLE") or row.get("TITLE_TS") or "").strip() | |
| 137 | + out: list[NormalizedObservation] = [] | |
| 138 | + transform = spec.transform | |
| 139 | + for iso3, pts in sorted(series.items()): | |
| 140 | + # a country may appear under several frequencies in one dataflow → transform per frequency | |
| 141 | + by_freq: dict[str, list[Point]] = {} | |
| 142 | + for p in pts: | |
| 143 | + by_freq.setdefault(p.frequency, []).append(p) | |
| 144 | + for freq, fpts in by_freq.items(): | |
| 145 | + fpts = _dedupe(fpts) | |
| 146 | + if is_series_transform(transform): | |
| 147 | + fpts = apply_series_transform(fpts, transform) | |
| 148 | + if not fpts: | |
| 149 | + log.info("bis %s/%s: %s dropped by transform %s (no base-year data)", spec.dataset, spec.code, iso3, transform) | |
| 150 | + elif transform: | |
| 151 | + fpts = [Point(p.period, p.year, p.frequency, self.apply_transform(p.value, transform)) for p in fpts] | |
| 152 | + for p in fpts: | |
| 153 | + meta: dict[str, Any] = {} | |
| 154 | + st = status.get((iso3, p.period)) | |
| 155 | + if st: | |
| 156 | + meta["obs_status"] = st | |
| 157 | + if titles.get(iso3): | |
| 158 | + meta["title"] = titles[iso3] | |
| 159 | + out.append( | |
| 160 | + NormalizedObservation( | |
| 161 | + country_id=iso3, | |
| 162 | + indicator_id=spec.indicator_id, | |
| 163 | + period=p.period, | |
| 164 | + year=p.year, | |
| 165 | + frequency=spec.frequency or freq, # type: ignore[arg-type] | |
| 166 | + value=p.value, | |
| 167 | + unit=unit, | |
| 168 | + source_id=self.id, | |
| 169 | + source_dataset=spec.dataset, | |
| 170 | + source_series_code=spec.code, | |
| 171 | + is_estimate=st in ("E", "P"), | |
| 172 | + is_forecast=False, | |
| 173 | + retrieved_at=r.retrieved_at, | |
| 174 | + source_updated_at=r.source_updated_at, | |
| 175 | + metadata=meta, | |
| 176 | + ) | |
| 177 | + ) | |
| 178 | + log.debug("bis %s/%s: %d rows kept, dropped %s", spec.dataset, spec.code, len(out), n_drop) | |
| 179 | + return out | |
| 180 | + | |
| 181 | + | |
| 182 | +def _dedupe(points: list[Point]) -> list[Point]: | |
| 183 | + seen: dict[datetime | Any, Point] = {} | |
| 184 | + for p in points: | |
| 185 | + seen[p.period] = p # last wins | |
| 186 | + return [seen[k] for k in sorted(seen)] | |
added
src/countryatlas/connectors/eurostat.py
+253 −0
@@ -0,0 +1,253 @@ | ||
| 1 | +"""Eurostat connector — JSON-stat 2.0 dissemination API (no key, CC BY 4.0, "Source: Eurostat"). | |
| 2 | + | |
| 3 | +fetch: `GET https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/{dataset}?format=JSON&lang=EN&{dim}={code}…` | |
| 4 | + `spec.dataset` = Eurostat dataset code (`une_rt_a`), `spec.params` = the dimension filters (`{age: Y15-74, unit: PC_ACT, | |
| 5 | + sex: T}`; list values are repeated: `geo=[DE, FR]` → `geo=DE&geo=FR`). No `geo`/`time` filter = all areas, all years. | |
| 6 | + Unknown dimension → HTTP 400 with an `error` array → ConnectorError. Unknown code → 200 with an empty `value`. | |
| 7 | +normalize: a generic JSON-stat decoder unravels the sparse `value` object (flat row-major index over `size` in `id` order) | |
| 8 | + into one dict of codes per cell. Every dimension other than `geo`/`time` must have exactly ONE category after | |
| 9 | + filtering — otherwise the spec is under-specified and a ConnectorError lists the offending dimensions. | |
| 10 | + geo → `lookup().from_iso2()` (EL→GR, UK→GB); aggregates (EU27_2020, EA20, EA21, EU28 …) are dropped. | |
| 11 | + time `2025`, `2025-Q1`, `2025-M06` → first day of period + frequency. `status` flags: `p` provisional / `e` estimated / | |
| 12 | + `s` Eurostat estimate → is_estimate; `f` forecast → is_forecast; `b` break, `u` low reliability, `d` definition differs | |
| 13 | + are kept in `metadata.flags`. | |
| 14 | +`source_updated_at` = the dataset's `updated` timestamp. | |
| 15 | +""" | |
| 16 | +from __future__ import annotations | |
| 17 | + | |
| 18 | +import logging | |
| 19 | +import math | |
| 20 | +from typing import Any, ClassVar | |
| 21 | + | |
| 22 | +import httpx | |
| 23 | +import orjson | |
| 24 | + | |
| 25 | +from countryatlas.connectors._util import ConnectorError, apply_transform_array, parse_date_utc, parse_period | |
| 26 | +from countryatlas.connectors.base import Connector | |
| 27 | +from countryatlas.models import DatasetDescriptor, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 28 | +from countryatlas.registry import indicators_by_id, lookup | |
| 29 | + | |
| 30 | +log = logging.getLogger(__name__) | |
| 31 | + | |
| 32 | +DATA_BROWSER_URL = "https://ec.europa.eu/eurostat/databrowser/view/{dataset}/default/table?lang=en" | |
| 33 | +ESTIMATE_FLAGS = {"p", "e", "s"} | |
| 34 | +FORECAST_FLAGS = {"f"} | |
| 35 | + | |
| 36 | + | |
| 37 | +class EurostatConnector(Connector): | |
| 38 | + id: ClassVar[str] = "eurostat" | |
| 39 | + name: ClassVar[str] = "Eurostat" | |
| 40 | + organization: ClassVar[str] = "European Commission — Eurostat" | |
| 41 | + url: ClassVar[str] = "https://ec.europa.eu/eurostat" | |
| 42 | + licence: ClassVar[str] = "CC BY 4.0" | |
| 43 | + attribution: ClassVar[str] = "Source: Eurostat" | |
| 44 | + api_base: ClassVar[str] = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data" | |
| 45 | + rate_per_minute: ClassVar[int] = 30 | |
| 46 | + timeout: ClassVar[float] = 180.0 | |
| 47 | + country_codes: ClassVar[str] = "eurostat" | |
| 48 | + | |
| 49 | + # ------------------------------------------------------------------ discover | |
| 50 | + def discover(self) -> list[DatasetDescriptor]: | |
| 51 | + seen: dict[str, DatasetDescriptor] = {} | |
| 52 | + for ind in indicators_by_id().values(): | |
| 53 | + for s in ind.sources: | |
| 54 | + if s.connector == self.id and s.dataset not in seen: | |
| 55 | + seen[s.dataset] = DatasetDescriptor( | |
| 56 | + connector=self.id, dataset=s.dataset, name=s.dataset, | |
| 57 | + url=DATA_BROWSER_URL.format(dataset=s.dataset), licence=self.licence, | |
| 58 | + ) | |
| 59 | + return list(seen.values()) or [ | |
| 60 | + DatasetDescriptor(connector=self.id, dataset="default", name=self.name, url=self.url, licence=self.licence) | |
| 61 | + ] | |
| 62 | + | |
| 63 | + # --------------------------------------------------------------------- fetch | |
| 64 | + def fetch(self, spec: IndicatorSourceSpec) -> RawPayload: | |
| 65 | + if not spec.dataset: | |
| 66 | + raise ConnectorError(f"eurostat {spec.indicator_id}: spec.dataset (Eurostat dataset code) is required") | |
| 67 | + query: list[tuple[str, str]] = [("format", "JSON"), ("lang", "EN")] | |
| 68 | + for k, v in (spec.params or {}).items(): | |
| 69 | + if isinstance(v, (list, tuple)): | |
| 70 | + query.extend((k, str(x)) for x in v) | |
| 71 | + elif v is not None: | |
| 72 | + query.append((k, str(v))) | |
| 73 | + url = f"{self.api_base}/{spec.dataset}" | |
| 74 | + try: | |
| 75 | + r = self.get(url, params=query) | |
| 76 | + except httpx.HTTPStatusError as e: | |
| 77 | + detail = _error_text(e.response.content) or e.response.text[:200] | |
| 78 | + raise ConnectorError(f"eurostat {spec.dataset}: HTTP {e.response.status_code} — {detail}") from e | |
| 79 | + doc = _load(r.content, spec.dataset) | |
| 80 | + if "error" in doc: | |
| 81 | + raise ConnectorError(f"eurostat {spec.dataset}: {_error_text(r.content)}") | |
| 82 | + if not doc.get("value"): | |
| 83 | + raise ConnectorError( | |
| 84 | + f"eurostat {spec.dataset}: empty value set for filters {spec.params} (sizes {doc.get('size')}) — wrong codes?" | |
| 85 | + ) | |
| 86 | + p = self.payload( | |
| 87 | + r, | |
| 88 | + dataset=spec.dataset, | |
| 89 | + code=spec.code or _code_from_params(spec.params), | |
| 90 | + label=doc.get("label"), | |
| 91 | + updated=doc.get("updated"), | |
| 92 | + source_url=DATA_BROWSER_URL.format(dataset=spec.dataset), | |
| 93 | + notes=f"Eurostat {spec.dataset} — {doc.get('label')}" if doc.get("label") else None, | |
| 94 | + ) | |
| 95 | + upd = parse_date_utc(doc.get("updated")) | |
| 96 | + if upd: | |
| 97 | + p.source_updated_at = upd | |
| 98 | + return p | |
| 99 | + | |
| 100 | + # ----------------------------------------------------------------- normalize | |
| 101 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 102 | + r = raw[0] if isinstance(raw, list) else raw | |
| 103 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 104 | + unit = ind.unit if ind else "" | |
| 105 | + doc = _load(r.body, spec.dataset) | |
| 106 | + if "error" in doc: | |
| 107 | + raise ConnectorError(f"eurostat {spec.dataset}: {_error_text(r.body)}") | |
| 108 | + cells = decode_jsonstat(doc) | |
| 109 | + if not cells: | |
| 110 | + return [] | |
| 111 | + ids = list(doc["id"]) | |
| 112 | + ambiguous = { | |
| 113 | + d: list(doc["dimension"][d]["category"]["index"]) | |
| 114 | + for d in ids | |
| 115 | + if d not in ("geo", "time") and doc["size"][ids.index(d)] > 1 | |
| 116 | + } | |
| 117 | + if ambiguous: | |
| 118 | + raise ConnectorError( | |
| 119 | + f"eurostat {spec.dataset}: dimensions with several categories after filtering — add params: " | |
| 120 | + + "; ".join(f"{d}={codes[:8]}" for d, codes in ambiguous.items()) | |
| 121 | + ) | |
| 122 | + lk = lookup() | |
| 123 | + src_upd = r.source_updated_at or parse_date_utc((r.meta or {}).get("updated")) | |
| 124 | + fixed = {d: codes for d, codes in ((d, doc["dimension"][d]["category"]["index"]) for d in ids) | |
| 125 | + if d not in ("geo", "time", "freq")} | |
| 126 | + fixed_codes = {d: next(iter(c)) for d, c in fixed.items()} | |
| 127 | + out: list[NormalizedObservation] = [] | |
| 128 | + seen: set[tuple[str, str]] = set() | |
| 129 | + raw_vals = [c["value"] for c in cells] | |
| 130 | + vals = apply_transform_array(_as_array(raw_vals), spec.transform).tolist() | |
| 131 | + for cell, v in zip(cells, vals, strict=True): | |
| 132 | + if v is None or math.isnan(v): | |
| 133 | + continue | |
| 134 | + geo = cell["dims"].get("geo", "") | |
| 135 | + iso3 = lk.from_iso2(geo) if len(geo) == 2 else None | |
| 136 | + if iso3 is None: | |
| 137 | + continue | |
| 138 | + per = parse_period(str(cell["dims"].get("time", "")).replace("-", "")) | |
| 139 | + if per is None: | |
| 140 | + continue | |
| 141 | + period, year, freq = per | |
| 142 | + if spec.frequency: | |
| 143 | + freq = spec.frequency | |
| 144 | + key = (iso3, period.isoformat()) | |
| 145 | + if key in seen: | |
| 146 | + continue | |
| 147 | + seen.add(key) | |
| 148 | + flags = (cell.get("status") or "").strip().lower() | |
| 149 | + meta: dict[str, Any] = {} | |
| 150 | + if flags: | |
| 151 | + meta["flags"] = flags | |
| 152 | + if fixed_codes: | |
| 153 | + meta["filters"] = fixed_codes | |
| 154 | + out.append( | |
| 155 | + NormalizedObservation( | |
| 156 | + country_id=iso3, | |
| 157 | + indicator_id=spec.indicator_id, | |
| 158 | + period=period, | |
| 159 | + year=year, | |
| 160 | + frequency=freq, | |
| 161 | + value=float(v), | |
| 162 | + unit=unit, | |
| 163 | + source_id=self.id, | |
| 164 | + source_dataset=spec.dataset, | |
| 165 | + source_series_code=spec.code or _code_from_params(spec.params), | |
| 166 | + is_estimate=any(f in ESTIMATE_FLAGS for f in flags), | |
| 167 | + is_forecast=any(f in FORECAST_FLAGS for f in flags), | |
| 168 | + retrieved_at=r.retrieved_at, | |
| 169 | + source_updated_at=src_upd, | |
| 170 | + metadata=meta, | |
| 171 | + ) | |
| 172 | + ) | |
| 173 | + return out | |
| 174 | + | |
| 175 | + | |
| 176 | +# ------------------------------------------------------------------------------------------------- JSON-stat decoder | |
| 177 | +def decode_jsonstat(doc: dict[str, Any]) -> list[dict[str, Any]]: | |
| 178 | + """JSON-stat 2.0 dataset → [{dims: {dim: code}, value: float|None, status: str|None}] for every present cell. | |
| 179 | + | |
| 180 | + `value` may be a sparse object {flat_index: value} or a dense list; the flat index is row-major over `size` in `id` | |
| 181 | + order (last dimension varies fastest). `status` is an object {flat_index: flag} or a list. | |
| 182 | + """ | |
| 183 | + ids: list[str] = list(doc["id"]) | |
| 184 | + sizes: list[int] = [int(s) for s in doc["size"]] | |
| 185 | + if len(ids) != len(sizes): | |
| 186 | + raise ConnectorError("jsonstat: id/size length mismatch") | |
| 187 | + codes_by_dim: list[list[str]] = [] | |
| 188 | + for d, n in zip(ids, sizes, strict=True): | |
| 189 | + index = doc["dimension"][d]["category"].get("index") | |
| 190 | + if isinstance(index, dict): | |
| 191 | + ordered = [None] * len(index) | |
| 192 | + for code, pos in index.items(): | |
| 193 | + ordered[int(pos)] = code | |
| 194 | + elif isinstance(index, list): | |
| 195 | + ordered = list(index) | |
| 196 | + else: # single-category dimension may omit index → use label keys | |
| 197 | + ordered = list(doc["dimension"][d]["category"].get("label", {}).keys()) | |
| 198 | + if len(ordered) != n: | |
| 199 | + raise ConnectorError(f"jsonstat: dimension {d} has {len(ordered)} codes but size {n}") | |
| 200 | + codes_by_dim.append(ordered) # type: ignore[arg-type] | |
| 201 | + value = doc.get("value") or {} | |
| 202 | + status = doc.get("status") or {} | |
| 203 | + if isinstance(value, list): | |
| 204 | + value = {str(i): v for i, v in enumerate(value) if v is not None} | |
| 205 | + if isinstance(status, list): | |
| 206 | + status = {str(i): s for i, s in enumerate(status) if s} | |
| 207 | + strides: list[int] = [1] * len(sizes) | |
| 208 | + for i in range(len(sizes) - 2, -1, -1): | |
| 209 | + strides[i] = strides[i + 1] * sizes[i + 1] | |
| 210 | + out: list[dict[str, Any]] = [] | |
| 211 | + for k, v in value.items(): | |
| 212 | + idx = int(k) | |
| 213 | + dims: dict[str, str] = {} | |
| 214 | + rem = idx | |
| 215 | + for d, stride, codes in zip(ids, strides, codes_by_dim, strict=True): | |
| 216 | + pos, rem = divmod(rem, stride) | |
| 217 | + dims[d] = codes[pos] | |
| 218 | + st = status.get(k) if isinstance(status, dict) else None | |
| 219 | + out.append({"dims": dims, "value": v, "status": st}) | |
| 220 | + return out | |
| 221 | + | |
| 222 | + | |
| 223 | +def _load(body: bytes, dataset: str) -> dict[str, Any]: | |
| 224 | + try: | |
| 225 | + doc = orjson.loads(body) | |
| 226 | + except orjson.JSONDecodeError as e: | |
| 227 | + raise ConnectorError(f"eurostat {dataset}: response is not JSON ({body[:80]!r})") from e | |
| 228 | + if not isinstance(doc, dict): | |
| 229 | + raise ConnectorError(f"eurostat {dataset}: unexpected payload shape") | |
| 230 | + return doc | |
| 231 | + | |
| 232 | + | |
| 233 | +def _error_text(body: bytes) -> str | None: | |
| 234 | + try: | |
| 235 | + doc = orjson.loads(body) | |
| 236 | + errs = doc.get("error") if isinstance(doc, dict) else None | |
| 237 | + if isinstance(errs, list): | |
| 238 | + return "; ".join(f"{e.get('status')} {e.get('label')}" for e in errs) | |
| 239 | + if errs: | |
| 240 | + return str(errs) | |
| 241 | + except Exception: # noqa: BLE001 | |
| 242 | + return None | |
| 243 | + return None | |
| 244 | + | |
| 245 | + | |
| 246 | +def _code_from_params(params: dict[str, Any] | None) -> str: | |
| 247 | + return "&".join(f"{k}={v}" for k, v in sorted((params or {}).items())) | |
| 248 | + | |
| 249 | + | |
| 250 | +def _as_array(values: list[Any]): | |
| 251 | + import numpy as np | |
| 252 | + | |
| 253 | + return np.array([float(v) if isinstance(v, (int, float)) else np.nan for v in values], dtype="float64") | |
added
src/countryatlas/connectors/fred.py
+287 −0
@@ -0,0 +1,287 @@ | ||
| 1 | +"""FRED (Federal Reserve Bank of St. Louis) connector. | |
| 2 | + | |
| 3 | +* Auth: `api_key` from `settings.fred_api_key` (env FRED_API_KEY). The key is NEVER logged: it is sent as a query | |
| 4 | + parameter, stripped from the stored payload URL and from error messages, and the httpx request logger is silenced. | |
| 5 | +* Pacing: the documented limit is 120 req/min but bursts trigger HTTP 429 followed by a multi-minute Akamai 403 ban, | |
| 6 | + so calls are serialised with ≥ 1.1 s spacing (`rate_per_minute=60` token bucket on top). | |
| 7 | +* fetch (2 calls per spec): | |
| 8 | + 1. `GET /fred/series?series_id=X` → title, units, frequency_short (M/Q/A/W/D), seasonal_adjustment_short, last_updated | |
| 9 | + (→ `source_updated_at`), observation_end. Lenient JSON (notes may contain control characters). | |
| 10 | + 2. `GET /fred/series/observations?series_id=X&observation_start=1950-01-01[&frequency=m&aggregation_method=avg]` | |
| 11 | + — weekly/daily series are aggregated server-side to monthly means (MORTGAGE30US); `spec.params` may override | |
| 12 | + any request parameter (`frequency`, `aggregation_method`, `units`, `observation_start`). | |
| 13 | +* normalize: `observations[].{date,value}`; `value == "."` = missing (incl. the incomplete current month/quarter for | |
| 14 | + aggregated series) → skipped. Frequency = spec.frequency or the (aggregated) series frequency. Period = first day of | |
| 15 | + month/quarter/year (FRED already uses the first day of the period). Country = `spec.countries[0]` (default USA); | |
| 16 | + BIS-derived series `Q{ISO2}[RN]628BIS` derive the country from the code. Transforms: scalar expression on `x`, | |
| 17 | + `yoy_pct` (e.g. CPIAUCSL → inflation), `rebase:YYYY` (INDPRO 2017=100 → 2015=100). Series units/title/seasonal | |
| 18 | + adjustment are copied into `metadata` (e.g. MEHOINUSA672N "2024 C-CPI-U Dollars"). | |
| 19 | + | |
| 20 | +Licence: "Federal Reserve Bank of St. Louis, FRED; third-party series subject to their own terms". | |
| 21 | +""" | |
| 22 | +from __future__ import annotations | |
| 23 | + | |
| 24 | +import json | |
| 25 | +import logging | |
| 26 | +import re | |
| 27 | +import threading | |
| 28 | +import time | |
| 29 | +from datetime import UTC, date, datetime | |
| 30 | +from typing import Any, ClassVar | |
| 31 | + | |
| 32 | +import httpx | |
| 33 | + | |
| 34 | +from countryatlas.config import settings | |
| 35 | +from countryatlas.connectors._series import Point, apply_series_transform, is_series_transform, to_float | |
| 36 | +from countryatlas.connectors._util import ConnectorError | |
| 37 | +from countryatlas.connectors.base import Connector | |
| 38 | +from countryatlas.models import DatasetDescriptor, Frequency, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 39 | +from countryatlas.registry import indicators_by_id, lookup | |
| 40 | + | |
| 41 | +log = logging.getLogger(__name__) | |
| 42 | +logging.getLogger("httpx").setLevel(logging.WARNING) # httpx logs full request URLs (would include api_key) | |
| 43 | + | |
| 44 | +_BIS_CODE = re.compile(r"^Q([A-Z0-9]{2})[RN]628BIS$") | |
| 45 | +_KEY_RE = re.compile(r"(api_key=)[0-9a-fA-F]+") | |
| 46 | +SERIES_PAGE = "https://fred.stlouisfed.org/series/{code}" | |
| 47 | +FREQ_MAP: dict[str, Frequency] = {"M": "M", "Q": "Q", "A": "A"} | |
| 48 | +AGGREGATED_TO_MONTHLY = {"W", "BW", "D"} | |
| 49 | + | |
| 50 | + | |
| 51 | +def redact(text: str) -> str: | |
| 52 | + return _KEY_RE.sub(r"\1<redacted>", text) | |
| 53 | + | |
| 54 | + | |
| 55 | +def _lenient_json(body: bytes) -> Any: | |
| 56 | + return json.JSONDecoder(strict=False).decode(body.decode("utf-8", errors="replace")) | |
| 57 | + | |
| 58 | + | |
| 59 | +def parse_fred_datetime(text: str | None) -> datetime | None: | |
| 60 | + """'2026-09-01 15:16:43-05' → aware datetime (FRED writes the offset as ±HH).""" | |
| 61 | + if not text: | |
| 62 | + return None | |
| 63 | + t = text.strip() | |
| 64 | + if re.search(r"[+-]\d{2}$", t): | |
| 65 | + t += "00" | |
| 66 | + try: | |
| 67 | + return datetime.strptime(t, "%Y-%m-%d %H:%M:%S%z") | |
| 68 | + except ValueError: | |
| 69 | + pass | |
| 70 | + try: | |
| 71 | + return datetime.strptime(t[:10], "%Y-%m-%d").replace(tzinfo=UTC) | |
| 72 | + except ValueError: | |
| 73 | + return None | |
| 74 | + | |
| 75 | + | |
| 76 | +class FREDConnector(Connector): | |
| 77 | + id: ClassVar[str] = "fred" | |
| 78 | + name: ClassVar[str] = "FRED" | |
| 79 | + organization: ClassVar[str] = "Federal Reserve Bank of St. Louis" | |
| 80 | + url: ClassVar[str] = "https://fred.stlouisfed.org" | |
| 81 | + licence: ClassVar[str] = "Federal Reserve Bank of St. Louis, FRED; third-party series subject to their own terms" | |
| 82 | + attribution: ClassVar[str] = "Federal Reserve Bank of St. Louis, FRED" | |
| 83 | + api_base: ClassVar[str] = "https://api.stlouisfed.org/fred" | |
| 84 | + rate_per_minute: ClassVar[int] = 60 | |
| 85 | + timeout: ClassVar[float] = 60.0 | |
| 86 | + country_codes: ClassVar[str] = "iso3" | |
| 87 | + | |
| 88 | + MIN_SPACING_S: ClassVar[float] = 1.1 | |
| 89 | + OBSERVATION_START: ClassVar[str] = "1950-01-01" | |
| 90 | + | |
| 91 | + def __init__(self) -> None: | |
| 92 | + super().__init__() | |
| 93 | + self._key = settings.fred_api_key | |
| 94 | + self._space_lock = threading.Lock() | |
| 95 | + self._last_call = 0.0 | |
| 96 | + self._meta_cache: dict[str, dict[str, Any]] = {} | |
| 97 | + | |
| 98 | + def discover(self) -> list[DatasetDescriptor]: | |
| 99 | + return [ | |
| 100 | + DatasetDescriptor( | |
| 101 | + connector=self.id, | |
| 102 | + dataset="FRED", | |
| 103 | + name="FRED economic data", | |
| 104 | + url="https://fred.stlouisfed.org/docs/api/fred/", | |
| 105 | + licence=self.licence, | |
| 106 | + notes="US macro/housing/labour series + BIS residential property price indices republished by FRED.", | |
| 107 | + ) | |
| 108 | + ] | |
| 109 | + | |
| 110 | + # ----------------------------------------------------------------------- HTTP | |
| 111 | + def get(self, url: str, params: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> httpx.Response: # type: ignore[override] | |
| 112 | + if not self._key: | |
| 113 | + raise ConnectorError("fred: FRED_API_KEY is not set (settings.fred_api_key)") | |
| 114 | + p = dict(params or {}) | |
| 115 | + p.setdefault("api_key", self._key) | |
| 116 | + p.setdefault("file_type", "json") | |
| 117 | + with self._space_lock: # serialise FRED calls: bursts → 429 then a multi-minute 403 ban | |
| 118 | + wait = self.MIN_SPACING_S - (time.monotonic() - self._last_call) | |
| 119 | + if wait > 0: | |
| 120 | + time.sleep(wait) | |
| 121 | + try: | |
| 122 | + return super().get(url, params=p, headers=headers) | |
| 123 | + except httpx.HTTPStatusError as e: | |
| 124 | + detail = "" | |
| 125 | + try: | |
| 126 | + detail = str(_lenient_json(e.response.content).get("error_message") or "")[:200] | |
| 127 | + except Exception: # noqa: BLE001 | |
| 128 | + detail = "" | |
| 129 | + code = e.response.status_code | |
| 130 | + hint = " (Akamai rate-limit ban — wait several minutes)" if code == 403 else "" | |
| 131 | + raise ConnectorError(f"fred: HTTP {code} on {redact(url)}{hint} {detail}".strip()) from None | |
| 132 | + finally: | |
| 133 | + self._last_call = time.monotonic() | |
| 134 | + | |
| 135 | + def payload(self, r: httpx.Response, dataset: str, code: str, pages: int = 1, **meta: Any) -> RawPayload: # type: ignore[override] | |
| 136 | + p = super().payload(r, dataset, code, pages, **meta) | |
| 137 | + p.url = redact(p.url) | |
| 138 | + return p | |
| 139 | + | |
| 140 | + # --------------------------------------------------------------------- fetch | |
| 141 | + def series_metadata(self, code: str) -> dict[str, Any]: | |
| 142 | + if code in self._meta_cache: | |
| 143 | + return self._meta_cache[code] | |
| 144 | + r = self.get(f"{self.api_base}/series", params={"series_id": code}) | |
| 145 | + doc = _lenient_json(r.content) | |
| 146 | + ss = doc.get("seriess") or [] | |
| 147 | + if not ss: | |
| 148 | + raise ConnectorError(f"fred {code}: series not found") | |
| 149 | + s = ss[0] | |
| 150 | + meta = { | |
| 151 | + "id": s.get("id"), | |
| 152 | + "title": s.get("title"), | |
| 153 | + "units": s.get("units"), | |
| 154 | + "units_short": s.get("units_short"), | |
| 155 | + "frequency": s.get("frequency"), | |
| 156 | + "frequency_short": s.get("frequency_short"), | |
| 157 | + "seasonal_adjustment": s.get("seasonal_adjustment"), | |
| 158 | + "seasonal_adjustment_short": s.get("seasonal_adjustment_short"), | |
| 159 | + "observation_start": s.get("observation_start"), | |
| 160 | + "observation_end": s.get("observation_end"), | |
| 161 | + "last_updated": s.get("last_updated"), | |
| 162 | + "popularity": s.get("popularity"), | |
| 163 | + "notes": (s.get("notes") or "")[:600], | |
| 164 | + } | |
| 165 | + self._meta_cache[code] = meta | |
| 166 | + return meta | |
| 167 | + | |
| 168 | + @staticmethod | |
| 169 | + def request_params(meta: dict[str, Any], spec: IndicatorSourceSpec) -> dict[str, Any]: | |
| 170 | + """Observation request parameters: weekly/daily → monthly average unless the spec says otherwise.""" | |
| 171 | + params: dict[str, Any] = {"series_id": spec.code, "observation_start": FREDConnector.OBSERVATION_START} | |
| 172 | + fs = (meta.get("frequency_short") or "").upper() | |
| 173 | + if fs in AGGREGATED_TO_MONTHLY: | |
| 174 | + params["frequency"] = "m" | |
| 175 | + params["aggregation_method"] = "avg" | |
| 176 | + params.update(spec.params or {}) | |
| 177 | + return params | |
| 178 | + | |
| 179 | + def fetch(self, spec: IndicatorSourceSpec) -> RawPayload: | |
| 180 | + if not spec.code: | |
| 181 | + raise ConnectorError("fred: spec without series_id") | |
| 182 | + meta = self.series_metadata(spec.code) | |
| 183 | + params = self.request_params(meta, spec) | |
| 184 | + r = self.get(f"{self.api_base}/series/observations", params=params) | |
| 185 | + doc = _lenient_json(r.content) | |
| 186 | + if "observations" not in doc: | |
| 187 | + raise ConnectorError(f"fred {spec.code}: unexpected payload {str(doc)[:120]}") | |
| 188 | + p = self.payload( | |
| 189 | + r, | |
| 190 | + dataset=spec.dataset or "FRED", | |
| 191 | + code=spec.code, | |
| 192 | + series_meta=meta, | |
| 193 | + request={k: v for k, v in params.items() if k not in ("api_key",)}, | |
| 194 | + source_url=SERIES_PAGE.format(code=spec.code), | |
| 195 | + notes=_notes(meta), | |
| 196 | + ) | |
| 197 | + p.content_type = "application/json" | |
| 198 | + upd = parse_fred_datetime(meta.get("last_updated")) | |
| 199 | + if upd: | |
| 200 | + p.source_updated_at = upd | |
| 201 | + if not doc["observations"]: | |
| 202 | + raise ConnectorError(f"fred {spec.code}: 0 observations") | |
| 203 | + return p | |
| 204 | + | |
| 205 | + # ----------------------------------------------------------------- normalize | |
| 206 | + @staticmethod | |
| 207 | + def country_for(spec: IndicatorSourceSpec) -> str | None: | |
| 208 | + lk = lookup() | |
| 209 | + m = _BIS_CODE.match(spec.code or "") | |
| 210 | + if m: | |
| 211 | + return lk.from_iso2(m.group(1)) # XM / 4T / 5R → None → dropped | |
| 212 | + if spec.countries: | |
| 213 | + return lk.from_iso3(spec.countries[0]) | |
| 214 | + return "USA" | |
| 215 | + | |
| 216 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 217 | + r = raw[0] if isinstance(raw, list) else raw | |
| 218 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 219 | + unit = ind.unit if ind else "" | |
| 220 | + iso3 = self.country_for(spec) | |
| 221 | + if iso3 is None: | |
| 222 | + log.info("fred %s: no registry country for this series (aggregate?) — 0 rows", spec.code) | |
| 223 | + return [] | |
| 224 | + doc = _lenient_json(r.body) | |
| 225 | + smeta = (r.meta or {}).get("series_meta") or {} | |
| 226 | + req = (r.meta or {}).get("request") or {} | |
| 227 | + fs = (smeta.get("frequency_short") or "").upper() | |
| 228 | + req_freq = str(req.get("frequency") or "").upper() | |
| 229 | + freq: Frequency | None = spec.frequency or FREQ_MAP.get(req_freq) or FREQ_MAP.get(fs) | |
| 230 | + if freq is None: | |
| 231 | + raise ConnectorError(f"fred {spec.code}: unsupported frequency {fs!r} — set frequency=m|q|a in params") | |
| 232 | + points: list[Point] = [] | |
| 233 | + for o in doc.get("observations") or []: | |
| 234 | + v = to_float(o.get("value")) | |
| 235 | + if v is None: | |
| 236 | + continue | |
| 237 | + try: | |
| 238 | + d = date.fromisoformat(str(o.get("date"))[:10]) | |
| 239 | + except ValueError: | |
| 240 | + continue | |
| 241 | + if freq == "A": | |
| 242 | + period = date(d.year, 1, 1) | |
| 243 | + elif freq == "Q": | |
| 244 | + period = date(d.year, 3 * ((d.month - 1) // 3) + 1, 1) | |
| 245 | + else: | |
| 246 | + period = date(d.year, d.month, 1) | |
| 247 | + points.append(Point(period, d.year, freq, v)) | |
| 248 | + if is_series_transform(spec.transform): | |
| 249 | + points = apply_series_transform(points, spec.transform) | |
| 250 | + elif spec.transform: | |
| 251 | + points = [Point(p.period, p.year, p.frequency, self.apply_transform(p.value, spec.transform)) for p in points] | |
| 252 | + meta: dict[str, Any] = {k: smeta.get(k) for k in ("title", "units", "seasonal_adjustment_short") if smeta.get(k)} | |
| 253 | + if smeta.get("frequency_short") and req_freq: | |
| 254 | + meta["aggregated_from"] = smeta["frequency_short"] | |
| 255 | + meta["aggregation_method"] = req.get("aggregation_method") | |
| 256 | + if spec.transform: | |
| 257 | + meta["transform"] = spec.transform | |
| 258 | + src_upd = r.source_updated_at or parse_fred_datetime(smeta.get("last_updated")) | |
| 259 | + out: list[NormalizedObservation] = [] | |
| 260 | + seen: set[date] = set() | |
| 261 | + for p in points: | |
| 262 | + if p.period in seen: | |
| 263 | + continue | |
| 264 | + seen.add(p.period) | |
| 265 | + out.append( | |
| 266 | + NormalizedObservation( | |
| 267 | + country_id=iso3, | |
| 268 | + indicator_id=spec.indicator_id, | |
| 269 | + period=p.period, | |
| 270 | + year=p.year, | |
| 271 | + frequency=freq, | |
| 272 | + value=p.value, | |
| 273 | + unit=unit, | |
| 274 | + source_id=self.id, | |
| 275 | + source_dataset=spec.dataset or "FRED", | |
| 276 | + source_series_code=spec.code, | |
| 277 | + retrieved_at=r.retrieved_at, | |
| 278 | + source_updated_at=src_upd, | |
| 279 | + metadata=dict(meta), | |
| 280 | + ) | |
| 281 | + ) | |
| 282 | + return out | |
| 283 | + | |
| 284 | + | |
| 285 | +def _notes(meta: dict[str, Any]) -> str | None: | |
| 286 | + parts = [str(meta.get(k)) for k in ("title", "units", "seasonal_adjustment") if meta.get(k)] | |
| 287 | + return " — ".join(parts) or None | |
added
src/countryatlas/connectors/ilo.py
+176 −0
@@ -0,0 +1,176 @@ | ||
| 1 | +"""ILOSTAT SDMX connector (CSV). | |
| 2 | + | |
| 3 | +Verified form (docs/sources-research.md §8): | |
| 4 | + https://sdmx.ilo.org/rest/data/ILO,{dataflow}/{key}?format=csv key = REF_AREA.FREQ.MEASURE.SEX.AGE | |
| 5 | + | |
| 6 | +Spec convention: `dataset` = dataflow id (e.g. `DF_UNE_2EAP_SEX_AGE_RT`, ILO modelled estimates incl. projections), | |
| 7 | +`code` = SDMX key with REF_AREA left empty for all areas (e.g. `.A..SEX_T.AGE_YTHADULT_YGE15`). | |
| 8 | + | |
| 9 | +CSV columns: DATAFLOW, REF_AREA (ISO3 + `X01…X99` aggregates → dropped by the registry lookup), FREQ, MEASURE, SEX, AGE, | |
| 10 | +TIME_PERIOD, OBS_VALUE, OBS_STATUS (`R` real value; blank = imputed / projected), UPPER_BOUND, LOWER_BOUND, SOURCE. | |
| 11 | + | |
| 12 | +Forecast flag: the dataflow name carries the release ("… ILO modelled estimates, Nov. 2025"); years strictly greater than | |
| 13 | +that release year are projections → `is_forecast=true`. Fallback when the name has no year: years > current year. | |
| 14 | +`is_estimate` = OBS_STATUS != 'R' (imputed / model-based values — ILO warns they must not be used for rankings). | |
| 15 | + | |
| 16 | +Licence: CC BY 4.0 — attribution "ILOSTAT, International Labour Organization". | |
| 17 | +""" | |
| 18 | +from __future__ import annotations | |
| 19 | + | |
| 20 | +import csv | |
| 21 | +import io | |
| 22 | +import logging | |
| 23 | +import re | |
| 24 | +import threading | |
| 25 | +from datetime import UTC, datetime | |
| 26 | +from typing import Any, ClassVar | |
| 27 | + | |
| 28 | +import orjson | |
| 29 | + | |
| 30 | +from countryatlas.connectors._series import parse_sdmx_period, to_float | |
| 31 | +from countryatlas.connectors._util import ConnectorError | |
| 32 | +from countryatlas.connectors.base import Connector | |
| 33 | +from countryatlas.models import DatasetDescriptor, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 34 | +from countryatlas.registry import indicators_by_id, lookup | |
| 35 | + | |
| 36 | +log = logging.getLogger(__name__) | |
| 37 | + | |
| 38 | +SOURCE_URL = "https://ilostat.ilo.org/data/" | |
| 39 | +_YEAR = re.compile(r"(19|20)\d{2}") | |
| 40 | + | |
| 41 | + | |
| 42 | +class ILOConnector(Connector): | |
| 43 | + id: ClassVar[str] = "ilo" | |
| 44 | + name: ClassVar[str] = "ILOSTAT" | |
| 45 | + organization: ClassVar[str] = "International Labour Organization" | |
| 46 | + url: ClassVar[str] = "https://ilostat.ilo.org" | |
| 47 | + licence: ClassVar[str] = "CC BY 4.0" | |
| 48 | + attribution: ClassVar[str] = "ILOSTAT, International Labour Organization" | |
| 49 | + api_base: ClassVar[str] = "https://sdmx.ilo.org/rest" | |
| 50 | + rate_per_minute: ClassVar[int] = 30 | |
| 51 | + timeout: ClassVar[float] = 180.0 | |
| 52 | + country_codes: ClassVar[str] = "iso3" | |
| 53 | + | |
| 54 | + def __init__(self) -> None: | |
| 55 | + super().__init__() | |
| 56 | + self._flow_cache: dict[str, dict[str, Any]] = {} | |
| 57 | + self._lock = threading.Lock() | |
| 58 | + | |
| 59 | + def discover(self) -> list[DatasetDescriptor]: | |
| 60 | + return [ | |
| 61 | + DatasetDescriptor( | |
| 62 | + connector=self.id, | |
| 63 | + dataset="DF_UNE_2EAP_SEX_AGE_RT", | |
| 64 | + name="Unemployment rate by sex and age — ILO modelled estimates", | |
| 65 | + url="https://ilostat.ilo.org/methods/concepts-and-definitions/ilo-modelled-estimates/", | |
| 66 | + licence=self.licence, | |
| 67 | + notes="Annual, ~190 countries + aggregates, includes 2-year projections (flagged is_forecast).", | |
| 68 | + ) | |
| 69 | + ] | |
| 70 | + | |
| 71 | + # --------------------------------------------------------------------- fetch | |
| 72 | + def dataflow_metadata(self, dataflow: str) -> dict[str, Any]: | |
| 73 | + """`/dataflow/ILO/{id}` (SDMX-JSON structure) → {name, version, release_year}. Cached per instance; optional.""" | |
| 74 | + with self._lock: | |
| 75 | + if dataflow in self._flow_cache: | |
| 76 | + return self._flow_cache[dataflow] | |
| 77 | + meta: dict[str, Any] = {} | |
| 78 | + try: | |
| 79 | + r = self.get(f"{self.api_base}/dataflow/ILO/{dataflow}", | |
| 80 | + headers={"Accept": "application/vnd.sdmx.structure+json"}) | |
| 81 | + flows = (orjson.loads(r.content).get("data") or {}).get("dataflows") or [] | |
| 82 | + if flows: | |
| 83 | + f = flows[0] | |
| 84 | + name = f.get("name") or "" | |
| 85 | + years = [int(m.group(0)) for m in _YEAR.finditer(name)] | |
| 86 | + meta = {"name": name, "version": f.get("version"), "release_year": max(years) if years else None, | |
| 87 | + "description": re.sub(r"<[^>]+>", "", f.get("description") or "")[:400]} | |
| 88 | + except Exception as e: # noqa: BLE001 — metadata is optional | |
| 89 | + log.warning("ilo: dataflow metadata for %s unavailable: %s", dataflow, e) | |
| 90 | + with self._lock: | |
| 91 | + self._flow_cache[dataflow] = meta | |
| 92 | + return meta | |
| 93 | + | |
| 94 | + def fetch(self, spec: IndicatorSourceSpec) -> RawPayload: | |
| 95 | + dataflow, key = spec.dataset, spec.code | |
| 96 | + if not dataflow or not key: | |
| 97 | + raise ConnectorError("ilo: spec needs dataset (dataflow id) and code (SDMX key)") | |
| 98 | + meta = self.dataflow_metadata(dataflow) | |
| 99 | + params: dict[str, Any] = {"format": "csv"} | |
| 100 | + params.update(spec.params or {}) | |
| 101 | + r = self.get(f"{self.api_base}/data/ILO,{dataflow}/{key}", params=params, | |
| 102 | + headers={"Accept": "text/csv, */*;q=0.5"}) | |
| 103 | + head = r.content[:300] | |
| 104 | + if b"REF_AREA" not in head or b"TIME_PERIOD" not in head: | |
| 105 | + raise ConnectorError(f"ilo {dataflow}/{key}: unexpected response (not SDMX-CSV): {head[:80]!r}") | |
| 106 | + p = self.payload( | |
| 107 | + r, | |
| 108 | + dataset=dataflow, | |
| 109 | + code=key, | |
| 110 | + dataflow_meta=meta, | |
| 111 | + source_url=SOURCE_URL, | |
| 112 | + notes=meta.get("name"), | |
| 113 | + ) | |
| 114 | + p.content_type = "text/csv" | |
| 115 | + return p | |
| 116 | + | |
| 117 | + # ----------------------------------------------------------------- normalize | |
| 118 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 119 | + r = raw[0] if isinstance(raw, list) else raw | |
| 120 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 121 | + unit = ind.unit if ind else "" | |
| 122 | + lk = lookup() | |
| 123 | + flow_meta = (r.meta or {}).get("dataflow_meta") or {} | |
| 124 | + release_year = flow_meta.get("release_year") or datetime.now(UTC).year | |
| 125 | + text = r.body.decode("utf-8-sig", errors="replace") | |
| 126 | + reader = csv.DictReader(io.StringIO(text)) | |
| 127 | + if not reader.fieldnames or "REF_AREA" not in reader.fieldnames: | |
| 128 | + raise ConnectorError(f"ilo {spec.dataset}/{spec.code}: CSV lacks REF_AREA column") | |
| 129 | + only = set(spec.countries or []) | |
| 130 | + best: dict[tuple[str, Any], NormalizedObservation] = {} | |
| 131 | + n_drop = {"area": 0, "null": 0, "period": 0} | |
| 132 | + for row in reader: | |
| 133 | + iso3 = lk.from_iso3(row.get("REF_AREA")) | |
| 134 | + if iso3 is None or (only and iso3 not in only): | |
| 135 | + n_drop["area"] += 1 | |
| 136 | + continue | |
| 137 | + v = to_float(row.get("OBS_VALUE")) | |
| 138 | + if v is None: | |
| 139 | + n_drop["null"] += 1 | |
| 140 | + continue | |
| 141 | + per = parse_sdmx_period(row.get("TIME_PERIOD") or "") | |
| 142 | + if per is None: | |
| 143 | + n_drop["period"] += 1 | |
| 144 | + continue | |
| 145 | + period, year, freq = per | |
| 146 | + st = (row.get("OBS_STATUS") or "").strip() | |
| 147 | + meta: dict[str, Any] = {} | |
| 148 | + if st: | |
| 149 | + meta["obs_status"] = st | |
| 150 | + lo, hi = to_float(row.get("LOWER_BOUND")), to_float(row.get("UPPER_BOUND")) | |
| 151 | + if lo is not None: | |
| 152 | + meta["low"] = lo | |
| 153 | + if hi is not None: | |
| 154 | + meta["high"] = hi | |
| 155 | + if row.get("SOURCE"): | |
| 156 | + meta["source"] = row["SOURCE"].strip() | |
| 157 | + obs = NormalizedObservation( | |
| 158 | + country_id=iso3, | |
| 159 | + indicator_id=spec.indicator_id, | |
| 160 | + period=period, | |
| 161 | + year=year, | |
| 162 | + frequency=spec.frequency or freq, | |
| 163 | + value=self.apply_transform(v, spec.transform), | |
| 164 | + unit=unit, | |
| 165 | + source_id=self.id, | |
| 166 | + source_dataset=spec.dataset, | |
| 167 | + source_series_code=spec.code, | |
| 168 | + is_estimate=st != "R", | |
| 169 | + is_forecast=year > int(release_year), | |
| 170 | + retrieved_at=r.retrieved_at, | |
| 171 | + source_updated_at=r.source_updated_at, | |
| 172 | + metadata=meta, | |
| 173 | + ) | |
| 174 | + best[(iso3, period)] = obs | |
| 175 | + log.debug("ilo %s/%s: %d rows kept, dropped %s", spec.dataset, spec.code, len(best), n_drop) | |
| 176 | + return [best[k] for k in sorted(best)] | |
added
src/countryatlas/connectors/imf.py
+206 −0
@@ -0,0 +1,206 @@ | ||
| 1 | +"""IMF World Economic Outlook connector (SDMX 2.1 REST at api.imf.org, CSV output, no API key). | |
| 2 | + | |
| 3 | +* fetch: `GET https://api.imf.org/external/sdmx/2.1/data/IMF.RES,WEO/{COUNTRIES}.{INDICATOR}.A` with `Accept: text/csv`. | |
| 4 | + COUNTRIES is left empty (= every area, ISO3 countries plus IMF group codes `G001`… which normalize drops). One request per | |
| 5 | + spec = one WEO indicator for all countries and all years (1980–2031, ~4 MB because the CSV repeats ~60 metadata columns). | |
| 6 | + Identical (dataset, code, params) requests are served from a per-instance cache. | |
| 7 | +* normalize: columns `COUNTRY, TIME_PERIOD, OBS_VALUE, LATEST_ACTUAL_ANNUAL_DATA, PUBLICATION_DATE, SCALE, UNIT, SERIES_NAME`. | |
| 8 | + - `is_forecast = year > LATEST_ACTUAL_ANNUAL_DATA` (per country × indicator). When the flag is missing for a series the | |
| 9 | + publication year is used (`year >= publication_year` → forecast) and `metadata.forecast_rule` says so. | |
| 10 | + - `is_estimate` is always False (WEO does not flag estimates separately from actuals). | |
| 11 | + - `OBS_VALUE` is already expressed in base units (e.g. NGDPD 2023 CAN = 2 196 593 836 000 US$ while `SCALE=9` only | |
| 12 | + describes the display unit "billions"). Do NOT add `x*1e9` transforms in the registry; `metadata.scale` keeps the hint. | |
| 13 | + - `KOS` (IMF Kosovo) → `XKX`; Taiwan `TWN` is kept when present in the country registry. | |
| 14 | +* `source_updated_at` = `PUBLICATION_DATE` of the vintage (2026-04-14 = April 2026 WEO). | |
| 15 | + | |
| 16 | +Licence: "© International Monetary Fund, World Economic Outlook database (April 2026)" — free reuse with citation. | |
| 17 | +""" | |
| 18 | +from __future__ import annotations | |
| 19 | + | |
| 20 | +import io | |
| 21 | +import logging | |
| 22 | +import threading | |
| 23 | +from datetime import UTC, date, datetime | |
| 24 | +from typing import Any, ClassVar | |
| 25 | + | |
| 26 | +import httpx | |
| 27 | +import numpy as np | |
| 28 | +import polars as pl | |
| 29 | + | |
| 30 | +from countryatlas.connectors._util import ConnectorError, apply_transform_array, parse_date_utc | |
| 31 | +from countryatlas.connectors.base import Connector | |
| 32 | +from countryatlas.models import DatasetDescriptor, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 33 | +from countryatlas.registry import indicators_by_id, lookup | |
| 34 | + | |
| 35 | +log = logging.getLogger(__name__) | |
| 36 | + | |
| 37 | +# dataset (registry) → SDMX dataflow "{AGENCY},{ID}" | |
| 38 | +DATAFLOWS: dict[str, str] = {"WEO": "IMF.RES,WEO"} | |
| 39 | +SOURCE_URL = "https://data.imf.org/en/datasets/IMF.RES:WEO" | |
| 40 | +CSV_COLUMNS = [ | |
| 41 | + "COUNTRY", "INDICATOR", "TIME_PERIOD", "OBS_VALUE", "SCALE", "UNIT", "SERIES_NAME", | |
| 42 | + "LATEST_ACTUAL_ANNUAL_DATA", "PUBLICATION_DATE", "UPDATE_DATE", | |
| 43 | +] | |
| 44 | +# IMF-specific area codes that are not ISO3 | |
| 45 | +IMF_ALIASES: dict[str, str] = {"KOS": "XKX", "UVK": "XKX"} | |
| 46 | + | |
| 47 | + | |
| 48 | +class IMFConnector(Connector): | |
| 49 | + id: ClassVar[str] = "imf" | |
| 50 | + name: ClassVar[str] = "International Monetary Fund" | |
| 51 | + organization: ClassVar[str] = "International Monetary Fund" | |
| 52 | + url: ClassVar[str] = "https://data.imf.org" | |
| 53 | + licence: ClassVar[str] = "© International Monetary Fund, World Economic Outlook database (April 2026)" | |
| 54 | + attribution: ClassVar[str] = ( | |
| 55 | + "International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO" | |
| 56 | + ) | |
| 57 | + api_base: ClassVar[str] = "https://api.imf.org/external/sdmx/2.1" | |
| 58 | + rate_per_minute: ClassVar[int] = 30 | |
| 59 | + timeout: ClassVar[float] = 180.0 | |
| 60 | + country_codes: ClassVar[str] = "iso3" | |
| 61 | + | |
| 62 | + def __init__(self) -> None: | |
| 63 | + super().__init__() | |
| 64 | + self._cache: dict[str, RawPayload] = {} | |
| 65 | + self._lock = threading.Lock() | |
| 66 | + | |
| 67 | + # ------------------------------------------------------------------ discover | |
| 68 | + def discover(self) -> list[DatasetDescriptor]: | |
| 69 | + return [ | |
| 70 | + DatasetDescriptor( | |
| 71 | + connector=self.id, | |
| 72 | + dataset="WEO", | |
| 73 | + name="World Economic Outlook database", | |
| 74 | + url=SOURCE_URL, | |
| 75 | + licence=self.licence, | |
| 76 | + notes="SDMX 2.1 CSV, dataflow IMF.RES,WEO (COUNTRY.INDICATOR.FREQUENCY), annual 1980–2031 incl. projections.", | |
| 77 | + ) | |
| 78 | + ] | |
| 79 | + | |
| 80 | + # --------------------------------------------------------------------- fetch | |
| 81 | + def fetch(self, spec: IndicatorSourceSpec) -> RawPayload: | |
| 82 | + dataset = spec.dataset or "WEO" | |
| 83 | + dataflow = DATAFLOWS.get(dataset, f"IMF.RES,{dataset}") | |
| 84 | + params = dict(spec.params or {}) | |
| 85 | + countries = params.pop("countries", None) | |
| 86 | + area = "+".join(countries) if isinstance(countries, list) else (countries or "") | |
| 87 | + freq = params.pop("frequency", "A") | |
| 88 | + url = f"{self.api_base}/data/{dataflow}/{area}.{spec.code}.{freq}" | |
| 89 | + query = {k: v for k, v in params.items() if k in ("startPeriod", "endPeriod")} | |
| 90 | + cache_key = f"{url}?{sorted(query.items())}" | |
| 91 | + with self._lock: | |
| 92 | + if cache_key in self._cache: | |
| 93 | + return self._cache[cache_key] | |
| 94 | + try: | |
| 95 | + r = self.get(url, params=query or None, headers={"Accept": "text/csv"}) | |
| 96 | + except httpx.HTTPStatusError as e: | |
| 97 | + raise ConnectorError(f"imf {dataset}/{spec.code}: HTTP {e.response.status_code} — unknown indicator code?") from e | |
| 98 | + body = r.content | |
| 99 | + if r.status_code == 204 or not body.strip(): | |
| 100 | + raise ConnectorError(f"imf {dataset}/{spec.code}: empty response (no data for this key)") | |
| 101 | + if not body.lstrip().startswith(b"DATAFLOW"): | |
| 102 | + raise ConnectorError(f"imf {dataset}/{spec.code}: unexpected payload {body[:80]!r} (not SDMX-CSV)") | |
| 103 | + head = _read_csv(body).head(1) | |
| 104 | + pub = head["PUBLICATION_DATE"][0] if "PUBLICATION_DATE" in head.columns and head.height else None | |
| 105 | + upd = head["UPDATE_DATE"][0] if "UPDATE_DATE" in head.columns and head.height else None | |
| 106 | + series_name = head["SERIES_NAME"][0] if "SERIES_NAME" in head.columns and head.height else None | |
| 107 | + p = self.payload( | |
| 108 | + r, | |
| 109 | + dataset=dataset, | |
| 110 | + code=spec.code, | |
| 111 | + publication_date=pub, | |
| 112 | + update_date=upd, | |
| 113 | + series_name=series_name, | |
| 114 | + source_url=SOURCE_URL, | |
| 115 | + notes=f"IMF WEO — {series_name}" if series_name else None, | |
| 116 | + ) | |
| 117 | + p.content_type = "text/csv" | |
| 118 | + pub_dt = parse_date_utc(pub) or parse_date_utc(upd) | |
| 119 | + if pub_dt: | |
| 120 | + p.source_updated_at = pub_dt | |
| 121 | + with self._lock: | |
| 122 | + self._cache[cache_key] = p | |
| 123 | + return p | |
| 124 | + | |
| 125 | + # ----------------------------------------------------------------- normalize | |
| 126 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 127 | + r = raw[0] if isinstance(raw, list) else raw | |
| 128 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 129 | + unit = ind.unit if ind else "" | |
| 130 | + df = _read_csv(r.body) | |
| 131 | + missing = [c for c in ("COUNTRY", "TIME_PERIOD", "OBS_VALUE") if c not in df.columns] | |
| 132 | + if missing: | |
| 133 | + raise ConnectorError(f"imf {spec.code}: CSV lacks columns {missing}") | |
| 134 | + if "INDICATOR" in df.columns: | |
| 135 | + df = df.filter(pl.col("INDICATOR") == spec.code) if (df["INDICATOR"] == spec.code).any() else df | |
| 136 | + df = df.filter(pl.col("OBS_VALUE").is_not_null() & (pl.col("OBS_VALUE").str.strip_chars() != "")) | |
| 137 | + if df.is_empty(): | |
| 138 | + return [] | |
| 139 | + lk = lookup() | |
| 140 | + df = df.with_columns(pl.col("COUNTRY").str.to_uppercase().replace(IMF_ALIASES).alias("iso")) | |
| 141 | + df = df.filter(pl.col("iso").is_in(sorted(lk.iso3))) | |
| 142 | + if df.is_empty(): | |
| 143 | + return [] | |
| 144 | + pub = (r.meta or {}).get("publication_date") | |
| 145 | + src_upd = r.source_updated_at or parse_date_utc(pub) | |
| 146 | + pub_year = src_upd.year if src_upd else datetime.now(UTC).year | |
| 147 | + years = df["TIME_PERIOD"].str.strip_chars().cast(pl.Int32, strict=False) | |
| 148 | + latest_col = ( | |
| 149 | + df["LATEST_ACTUAL_ANNUAL_DATA"].cast(pl.Int32, strict=False) | |
| 150 | + if "LATEST_ACTUAL_ANNUAL_DATA" in df.columns | |
| 151 | + else pl.Series([None] * df.height, dtype=pl.Int32) | |
| 152 | + ) | |
| 153 | + values = apply_transform_array(df["OBS_VALUE"].cast(pl.Float64, strict=False).to_numpy().astype("float64"), spec.transform) | |
| 154 | + scale = df["SCALE"].to_list() if "SCALE" in df.columns else [None] * df.height | |
| 155 | + src_unit = df["UNIT"].to_list() if "UNIT" in df.columns else [None] * df.height | |
| 156 | + freq = spec.frequency or "A" | |
| 157 | + out: list[NormalizedObservation] = [] | |
| 158 | + seen: set[tuple[str, int]] = set() | |
| 159 | + for iso, y, la, v, sc, su in zip( | |
| 160 | + df["iso"].to_list(), years.to_list(), latest_col.to_list(), values.tolist(), scale, src_unit, strict=True | |
| 161 | + ): | |
| 162 | + if y is None or v is None or not np.isfinite(v): | |
| 163 | + continue | |
| 164 | + key = (iso, y) | |
| 165 | + if key in seen: | |
| 166 | + continue | |
| 167 | + seen.add(key) | |
| 168 | + meta: dict[str, Any] = {} | |
| 169 | + if la is not None: | |
| 170 | + is_forecast = y > la | |
| 171 | + meta["latest_actual_annual_data"] = la | |
| 172 | + else: | |
| 173 | + is_forecast = y >= pub_year | |
| 174 | + meta["forecast_rule"] = f"no LATEST_ACTUAL_ANNUAL_DATA; year >= publication year {pub_year}" | |
| 175 | + if sc not in (None, "", "0"): | |
| 176 | + meta["scale"] = sc | |
| 177 | + if su: | |
| 178 | + meta["source_unit"] = su | |
| 179 | + out.append( | |
| 180 | + NormalizedObservation( | |
| 181 | + country_id=iso, | |
| 182 | + indicator_id=spec.indicator_id, | |
| 183 | + period=date(y, 1, 1), | |
| 184 | + year=y, | |
| 185 | + frequency=freq, | |
| 186 | + value=float(v), | |
| 187 | + unit=unit, | |
| 188 | + source_id=self.id, | |
| 189 | + source_dataset=spec.dataset or "WEO", | |
| 190 | + source_series_code=spec.code, | |
| 191 | + is_estimate=False, | |
| 192 | + is_forecast=bool(is_forecast), | |
| 193 | + retrieved_at=r.retrieved_at, | |
| 194 | + source_updated_at=src_upd, | |
| 195 | + metadata=meta, | |
| 196 | + ) | |
| 197 | + ) | |
| 198 | + return out | |
| 199 | + | |
| 200 | + | |
| 201 | +# ---------------------------------------------------------------------------------------------------------- helpers | |
| 202 | +def _read_csv(body: bytes) -> pl.DataFrame: | |
| 203 | + """Read the SDMX-CSV as strings, keeping only the columns we use (quoted multi-line metadata fields are handled).""" | |
| 204 | + header = body.split(b"\n", 1)[0].decode("utf-8", "replace").strip().split(",") | |
| 205 | + cols = [c for c in CSV_COLUMNS if c in header] | |
| 206 | + return pl.read_csv(io.BytesIO(body), columns=cols, infer_schema_length=0, quote_char='"', low_memory=False) | |
added
src/countryatlas/connectors/oecd.py
+239 −0
@@ -0,0 +1,239 @@ | ||
| 1 | +"""OECD SDMX connector (`https://sdmx.oecd.org/public/rest/`, no key, CC BY 4.0). | |
| 2 | + | |
| 3 | +Spec convention (registry/sources/oecd.yaml): | |
| 4 | +* `dataset` = full dataflow id `{AGENCY},{DSD@DF}[,{version}]` (e.g. `OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0`); | |
| 5 | + a missing version segment means "latest" (a trailing comma is appended). | |
| 6 | +* `code` = the SDMX key with EVERY dimension in the dataflow's order, `*` or empty for REF_AREA (all countries), | |
| 7 | + e.g. `*.Q.RHP.IX` → `/data/{dataset}/.Q.RHP.IX`. A key with too few dimensions is refused by the API (HTTP 403). | |
| 8 | +* `params`: `startPeriod` / `endPeriod` are forwarded; `filter: {COLUMN: value}` applies a post-hoc equality filter on the | |
| 9 | + CSV (for flows whose key cannot isolate a single series). | |
| 10 | +* `transform: "yoy"` is handled in code: year-on-year % change per country (lag 4 for quarterly, 12 monthly, 1 annual). | |
| 11 | + | |
| 12 | +fetch: `?startPeriod=…&dimensionAtObservation=AllDimensions&format=csvfilewithlabels` → CSV with code+label columns | |
| 13 | +(`REF_AREA, …, TIME_PERIOD, OBS_VALUE, OBS_STATUS, UNIT_MEASURE …`). HTTP 404 `NoRecordsFound` → ConnectorError. | |
| 14 | +normalize: ISO3 via registry lookup (aggregates OECD, EU27_2020, EA20, G20 … are dropped), periods `2024`, `2026-Q1`, | |
| 15 | +`2026-06` → first day of period + frequency, `OBS_STATUS` E/P → is_estimate, F → is_forecast. Duplicate (country, period) | |
| 16 | +pairs mean the key is under-specified → ConnectorError naming the varying dimensions (nothing is silently averaged). | |
| 17 | + | |
| 18 | +Rate limit: OECD documents 60 anonymous data requests per hour per IP → `rate_per_minute = 12` and a per-instance cache | |
| 19 | +so specs sharing a query (RHP index + RHP growth) download once. | |
| 20 | +""" | |
| 21 | +from __future__ import annotations | |
| 22 | + | |
| 23 | +import io | |
| 24 | +import logging | |
| 25 | +import threading | |
| 26 | +from datetime import date | |
| 27 | +from typing import Any, ClassVar | |
| 28 | + | |
| 29 | +import httpx | |
| 30 | +import numpy as np | |
| 31 | +import polars as pl | |
| 32 | + | |
| 33 | +from countryatlas.connectors._util import ConnectorError, apply_transform_array, parse_period | |
| 34 | +from countryatlas.connectors.base import Connector | |
| 35 | +from countryatlas.models import DatasetDescriptor, Frequency, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 36 | +from countryatlas.registry import indicators_by_id, lookup | |
| 37 | + | |
| 38 | +log = logging.getLogger(__name__) | |
| 39 | + | |
| 40 | +EXPLORER_URL = "https://data-explorer.oecd.org/?df[ds]=dsDisseminateFinalDMZ&df[id]={df_id}&df[ag]={agency}" | |
| 41 | +NON_DIMENSION_COLUMNS = { | |
| 42 | + "STRUCTURE", "STRUCTURE_ID", "STRUCTURE_NAME", "ACTION", "REF_AREA", "TIME_PERIOD", "OBS_VALUE", "OBS_STATUS", | |
| 43 | + "UNIT_MULT", "DECIMALS", "BASE_PER", "CONF_STATUS", "REF_YEAR_PRICE", "CURRENCY", | |
| 44 | +} | |
| 45 | +YOY_LAG = {"A": 1, "Q": 4, "M": 12} | |
| 46 | + | |
| 47 | + | |
| 48 | +class OECDConnector(Connector): | |
| 49 | + id: ClassVar[str] = "oecd" | |
| 50 | + name: ClassVar[str] = "OECD" | |
| 51 | + organization: ClassVar[str] = "Organisation for Economic Co-operation and Development" | |
| 52 | + url: ClassVar[str] = "https://data-explorer.oecd.org" | |
| 53 | + licence: ClassVar[str] = "CC BY 4.0" | |
| 54 | + attribution: ClassVar[str] = "OECD (2026), OECD Data Explorer, https://data-explorer.oecd.org" | |
| 55 | + api_base: ClassVar[str] = "https://sdmx.oecd.org/public/rest" | |
| 56 | + rate_per_minute: ClassVar[int] = 12 | |
| 57 | + timeout: ClassVar[float] = 300.0 | |
| 58 | + country_codes: ClassVar[str] = "iso3" | |
| 59 | + | |
| 60 | + def __init__(self) -> None: | |
| 61 | + super().__init__() | |
| 62 | + self._cache: dict[str, RawPayload] = {} | |
| 63 | + self._lock = threading.Lock() | |
| 64 | + | |
| 65 | + # ------------------------------------------------------------------ discover | |
| 66 | + def discover(self) -> list[DatasetDescriptor]: | |
| 67 | + seen: dict[str, DatasetDescriptor] = {} | |
| 68 | + for ind in indicators_by_id().values(): | |
| 69 | + for s in ind.sources: | |
| 70 | + if s.connector == self.id and s.dataset not in seen: | |
| 71 | + seen[s.dataset] = DatasetDescriptor( | |
| 72 | + connector=self.id, dataset=s.dataset, name=_dataflow_name(s.dataset), url=_explorer_url(s.dataset), | |
| 73 | + licence=self.licence, | |
| 74 | + ) | |
| 75 | + return list(seen.values()) or [ | |
| 76 | + DatasetDescriptor(connector=self.id, dataset="default", name=self.name, url=self.url, licence=self.licence) | |
| 77 | + ] | |
| 78 | + | |
| 79 | + # --------------------------------------------------------------------- fetch | |
| 80 | + def fetch(self, spec: IndicatorSourceSpec) -> RawPayload: | |
| 81 | + dataset = _with_version(spec.dataset) | |
| 82 | + key = _clean_key(spec.code) | |
| 83 | + params: dict[str, Any] = {"dimensionAtObservation": "AllDimensions", "format": "csvfilewithlabels"} | |
| 84 | + for k in ("startPeriod", "endPeriod"): | |
| 85 | + if (spec.params or {}).get(k) is not None: | |
| 86 | + params[k] = spec.params[k] | |
| 87 | + url = f"{self.api_base}/data/{dataset}/{key}" | |
| 88 | + cache_key = f"{url}?{sorted(params.items())}" | |
| 89 | + with self._lock: | |
| 90 | + if cache_key in self._cache: | |
| 91 | + return self._cache[cache_key] | |
| 92 | + try: | |
| 93 | + r = self.get(url, params=params) | |
| 94 | + except httpx.HTTPStatusError as e: | |
| 95 | + text = e.response.text[:200].strip() | |
| 96 | + raise ConnectorError(f"oecd {spec.dataset} {spec.code}: HTTP {e.response.status_code} {text}") from e | |
| 97 | + body = r.content | |
| 98 | + if r.status_code == 204 or not body.strip(): | |
| 99 | + raise ConnectorError(f"oecd {spec.dataset} {spec.code}: empty response (NoRecordsFound?)") | |
| 100 | + if not body.lstrip().startswith(b"STRUCTURE"): | |
| 101 | + raise ConnectorError(f"oecd {spec.dataset} {spec.code}: unexpected payload {body[:80]!r} (not SDMX-CSV)") | |
| 102 | + head = pl.read_csv(io.BytesIO(body), n_rows=1, infer_schema_length=0) | |
| 103 | + structure_name = head["STRUCTURE_NAME"][0] if "STRUCTURE_NAME" in head.columns and head.height else None | |
| 104 | + p = self.payload( | |
| 105 | + r, | |
| 106 | + dataset=spec.dataset, | |
| 107 | + code=spec.code, | |
| 108 | + structure_name=structure_name, | |
| 109 | + source_url=_explorer_url(spec.dataset), | |
| 110 | + notes=f"OECD — {structure_name}" if structure_name else None, | |
| 111 | + ) | |
| 112 | + p.content_type = "text/csv" | |
| 113 | + with self._lock: | |
| 114 | + self._cache[cache_key] = p | |
| 115 | + return p | |
| 116 | + | |
| 117 | + # ----------------------------------------------------------------- normalize | |
| 118 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 119 | + r = raw[0] if isinstance(raw, list) else raw | |
| 120 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 121 | + unit = ind.unit if ind else "" | |
| 122 | + df = pl.read_csv(io.BytesIO(r.body), infer_schema_length=0, low_memory=False) | |
| 123 | + missing = [c for c in ("REF_AREA", "TIME_PERIOD", "OBS_VALUE") if c not in df.columns] | |
| 124 | + if missing: | |
| 125 | + raise ConnectorError(f"oecd {spec.code}: CSV lacks columns {missing}") | |
| 126 | + for col, val in ((spec.params or {}).get("filter") or {}).items(): | |
| 127 | + if col not in df.columns: | |
| 128 | + raise ConnectorError(f"oecd {spec.code}: filter column {col} not in CSV") | |
| 129 | + df = df.filter(pl.col(col) == str(val)) | |
| 130 | + df = df.filter(pl.col("OBS_VALUE").is_not_null() & (pl.col("OBS_VALUE").str.strip_chars() != "")) | |
| 131 | + if df.is_empty(): | |
| 132 | + return [] | |
| 133 | + lk = lookup() | |
| 134 | + df = df.with_columns(pl.col("REF_AREA").str.to_uppercase().alias("iso")).filter(pl.col("iso").is_in(sorted(lk.iso3))) | |
| 135 | + if df.is_empty(): | |
| 136 | + return [] | |
| 137 | + # under-specified key → several rows per (country, period) | |
| 138 | + dup = df.group_by(["iso", "TIME_PERIOD"]).len().filter(pl.col("len") > 1) | |
| 139 | + if dup.height: | |
| 140 | + varying = [ | |
| 141 | + c for c in df.columns | |
| 142 | + if c.isupper() and c not in NON_DIMENSION_COLUMNS and not c.startswith("OBS_STATUS") and df[c].n_unique() > 1 | |
| 143 | + ] | |
| 144 | + raise ConnectorError( | |
| 145 | + f"oecd {spec.dataset} {spec.code}: {dup.height} duplicate (country, period) pairs — key under-specified; " | |
| 146 | + f"varying dimensions: {varying} (values e.g. { {c: df[c].unique().head(6).to_list() for c in varying} })" | |
| 147 | + ) | |
| 148 | + values = df["OBS_VALUE"].cast(pl.Float64, strict=False).to_numpy().astype("float64") | |
| 149 | + if spec.transform and spec.transform.strip().lower() != "yoy": | |
| 150 | + values = apply_transform_array(values, spec.transform) | |
| 151 | + status = df["OBS_STATUS"].to_list() if "OBS_STATUS" in df.columns else [None] * df.height | |
| 152 | + unit_measure = df["UNIT_MEASURE"].to_list() if "UNIT_MEASURE" in df.columns else [None] * df.height | |
| 153 | + base_per = df["BASE_PER"].to_list() if "BASE_PER" in df.columns else [None] * df.height | |
| 154 | + rows: list[tuple[str, date, int, Frequency, float, str | None, str | None, str | None]] = [] | |
| 155 | + for iso, tp, v, st, um, bp in zip(df["iso"].to_list(), df["TIME_PERIOD"].to_list(), values.tolist(), status, | |
| 156 | + unit_measure, base_per, strict=True): | |
| 157 | + if v is None or not np.isfinite(v): | |
| 158 | + continue | |
| 159 | + per = parse_period(str(tp).replace("-", "")) | |
| 160 | + if per is None: | |
| 161 | + continue | |
| 162 | + period, year, freq = per | |
| 163 | + rows.append((iso, period, year, spec.frequency or freq, float(v), st, um, bp)) | |
| 164 | + if spec.transform and spec.transform.strip().lower() == "yoy": | |
| 165 | + rows = _yoy(rows) | |
| 166 | + out: list[NormalizedObservation] = [] | |
| 167 | + for iso, period, year, freq, v, st, um, bp in rows: | |
| 168 | + st = (st or "").strip().upper() | |
| 169 | + meta: dict[str, Any] = {} | |
| 170 | + if st and st != "A": | |
| 171 | + meta["obs_status"] = st | |
| 172 | + if um: | |
| 173 | + meta["source_unit"] = um | |
| 174 | + if bp: | |
| 175 | + meta["base_period"] = bp | |
| 176 | + if spec.transform and spec.transform.strip().lower() == "yoy": | |
| 177 | + meta["derived"] = "year-on-year % change computed from the source index" | |
| 178 | + out.append( | |
| 179 | + NormalizedObservation( | |
| 180 | + country_id=iso, | |
| 181 | + indicator_id=spec.indicator_id, | |
| 182 | + period=period, | |
| 183 | + year=year, | |
| 184 | + frequency=freq, | |
| 185 | + value=v, | |
| 186 | + unit=unit, | |
| 187 | + source_id=self.id, | |
| 188 | + source_dataset=spec.dataset, | |
| 189 | + source_series_code=spec.code, | |
| 190 | + is_estimate=st in ("E", "P"), | |
| 191 | + is_forecast=st == "F", | |
| 192 | + retrieved_at=r.retrieved_at, | |
| 193 | + source_updated_at=r.source_updated_at, | |
| 194 | + metadata=meta, | |
| 195 | + ) | |
| 196 | + ) | |
| 197 | + return out | |
| 198 | + | |
| 199 | + | |
| 200 | +# ---------------------------------------------------------------------------------------------------------- helpers | |
| 201 | +def _with_version(dataset: str) -> str: | |
| 202 | + """`AGENCY,DSD@DF` → `AGENCY,DSD@DF,` (latest version); `AGENCY,DSD@DF,1.0` unchanged.""" | |
| 203 | + parts = dataset.split(",") | |
| 204 | + if len(parts) == 2: | |
| 205 | + return dataset + "," | |
| 206 | + return dataset | |
| 207 | + | |
| 208 | + | |
| 209 | +def _clean_key(code: str) -> str: | |
| 210 | + return ".".join("" if seg.strip() == "*" else seg.strip() for seg in code.split(".")) | |
| 211 | + | |
| 212 | + | |
| 213 | +def _dataflow_name(dataset: str) -> str: | |
| 214 | + parts = dataset.split(",") | |
| 215 | + return parts[1] if len(parts) > 1 else dataset | |
| 216 | + | |
| 217 | + | |
| 218 | +def _explorer_url(dataset: str) -> str: | |
| 219 | + parts = dataset.split(",") | |
| 220 | + if len(parts) < 2: | |
| 221 | + return "https://data-explorer.oecd.org" | |
| 222 | + df_id = parts[1].split("@")[-1] | |
| 223 | + return EXPLORER_URL.format(df_id=df_id, agency=parts[0]) | |
| 224 | + | |
| 225 | + | |
| 226 | +def _shift_year(d: date, years: int) -> date: | |
| 227 | + return date(d.year - years, d.month, d.day) | |
| 228 | + | |
| 229 | + | |
| 230 | +def _yoy(rows: list[tuple[str, date, int, Frequency, float, str | None, str | None, str | None]]) -> list[tuple]: | |
| 231 | + """Year-on-year % change: value(t) / value(t − 1 year) − 1, only when the exact previous-year period exists.""" | |
| 232 | + by_key = {(iso, period): v for iso, period, _y, _f, v, *_ in rows} | |
| 233 | + out = [] | |
| 234 | + for iso, period, year, freq, v, st, um, bp in rows: | |
| 235 | + prev = by_key.get((iso, _shift_year(period, 1))) | |
| 236 | + if prev is None or prev == 0: | |
| 237 | + continue | |
| 238 | + out.append((iso, period, year, freq, (v / prev - 1.0) * 100.0, st, um, bp)) | |
| 239 | + return out | |
added
src/countryatlas/connectors/owid.py
+258 −0
@@ -0,0 +1,258 @@ | ||
| 1 | +"""Our World in Data connector (CC BY 4.0 — attribution "Our World in Data"). | |
| 2 | + | |
| 3 | +Datasets: | |
| 4 | +* `co2` → owid-co2-data.csv (GitHub raw; columns country, year, iso_code, <metrics…>) | |
| 5 | +* `energy` → owid-energy-data.csv (GitHub raw; same shape) | |
| 6 | +* `grapher` → https://ourworldindata.org/grapher/{slug}.csv (+ {slug}.metadata.json for citation/source) | |
| 7 | + | |
| 8 | +The co2/energy files are shared by dozens of specs: they are downloaded ONCE per connector instance (cache keyed by | |
| 9 | +dataset) and normalized per column. For grapher charts the value column is the first numeric column after | |
| 10 | +entity/code/year; an additional column whose short name contains "projected"/"projection" is used for years without an | |
| 11 | +estimate and flagged `is_forecast=true` (e.g. UN WPP medium-variant projections in `median-age`). | |
| 12 | + | |
| 13 | +Rows with an empty `iso_code` or an `OWID_*` pseudo-code (World, continents, income groups…) are aggregates and dropped. | |
| 14 | +""" | |
| 15 | +from __future__ import annotations | |
| 16 | + | |
| 17 | +import io | |
| 18 | +import logging | |
| 19 | +import threading | |
| 20 | +from datetime import UTC, date, datetime | |
| 21 | +from typing import Any, ClassVar | |
| 22 | + | |
| 23 | +import numpy as np | |
| 24 | +import orjson | |
| 25 | +import polars as pl | |
| 26 | + | |
| 27 | +from countryatlas.connectors._util import ConnectorError, apply_transform_array, parse_date_utc | |
| 28 | +from countryatlas.connectors.base import Connector | |
| 29 | +from countryatlas.models import DatasetDescriptor, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 30 | +from countryatlas.registry import indicators_by_id, lookup | |
| 31 | + | |
| 32 | +log = logging.getLogger(__name__) | |
| 33 | + | |
| 34 | +DATASET_URLS: dict[str, str] = { | |
| 35 | + "co2": "https://raw.githubusercontent.com/owid/co2-data/master/owid-co2-data.csv", | |
| 36 | + "energy": "https://raw.githubusercontent.com/owid/energy-data/master/owid-energy-data.csv", | |
| 37 | +} | |
| 38 | +DATASET_NAMES: dict[str, str] = { | |
| 39 | + "co2": "OWID CO₂ and Greenhouse Gas Emissions dataset", | |
| 40 | + "energy": "OWID Energy dataset", | |
| 41 | + "grapher": "OWID Grapher charts", | |
| 42 | +} | |
| 43 | +DATASET_HOME: dict[str, str] = { | |
| 44 | + "co2": "https://github.com/owid/co2-data", | |
| 45 | + "energy": "https://github.com/owid/energy-data", | |
| 46 | +} | |
| 47 | +GRAPHER_CSV = "https://ourworldindata.org/grapher/{code}.csv?v=1&csvType=full&useColumnShortNames=true" | |
| 48 | +GRAPHER_META = "https://ourworldindata.org/grapher/{code}.metadata.json" | |
| 49 | +GRAPHER_PAGE = "https://ourworldindata.org/grapher/{code}" | |
| 50 | +FORECAST_HORIZON_YEARS = 6 # projected values are kept only up to current year + 6 | |
| 51 | + | |
| 52 | + | |
| 53 | +class OWIDConnector(Connector): | |
| 54 | + id: ClassVar[str] = "owid" | |
| 55 | + name: ClassVar[str] = "Our World in Data" | |
| 56 | + organization: ClassVar[str] = "Global Change Data Lab / University of Oxford" | |
| 57 | + url: ClassVar[str] = "https://ourworldindata.org" | |
| 58 | + licence: ClassVar[str] = "CC BY 4.0" | |
| 59 | + attribution: ClassVar[str] = "Our World in Data" | |
| 60 | + api_base: ClassVar[str] = "https://ourworldindata.org/grapher" | |
| 61 | + rate_per_minute: ClassVar[int] = 30 | |
| 62 | + timeout: ClassVar[float] = 180.0 | |
| 63 | + country_codes: ClassVar[str] = "iso3" | |
| 64 | + | |
| 65 | + def __init__(self) -> None: | |
| 66 | + super().__init__() | |
| 67 | + self._cache: dict[str, RawPayload] = {} | |
| 68 | + self._frames: dict[str, pl.DataFrame] = {} | |
| 69 | + self._lock = threading.Lock() | |
| 70 | + | |
| 71 | + @staticmethod | |
| 72 | + def raw_code_for(spec: IndicatorSourceSpec) -> str | None: | |
| 73 | + """Shared CSVs are stored under one raw code per dataset (used by `ca normalize` to find the file).""" | |
| 74 | + return f"owid-{spec.dataset}-data" if spec.dataset in DATASET_URLS else None | |
| 75 | + | |
| 76 | + def discover(self) -> list[DatasetDescriptor]: | |
| 77 | + out = [ | |
| 78 | + DatasetDescriptor(connector=self.id, dataset=d, name=DATASET_NAMES[d], url=u, licence=self.licence) | |
| 79 | + for d, u in DATASET_URLS.items() | |
| 80 | + ] | |
| 81 | + out.append( | |
| 82 | + DatasetDescriptor(connector=self.id, dataset="grapher", name=DATASET_NAMES["grapher"], url=self.api_base, | |
| 83 | + licence=self.licence) | |
| 84 | + ) | |
| 85 | + return out | |
| 86 | + | |
| 87 | + # --------------------------------------------------------------------- fetch | |
| 88 | + def fetch(self, spec: IndicatorSourceSpec) -> RawPayload: | |
| 89 | + if spec.dataset in DATASET_URLS: | |
| 90 | + return self._fetch_shared(spec.dataset) | |
| 91 | + if spec.dataset == "grapher": | |
| 92 | + return self._fetch_grapher(spec.code) | |
| 93 | + raise ConnectorError(f"owid: unknown dataset '{spec.dataset}' (expected co2|energy|grapher)") | |
| 94 | + | |
| 95 | + def _fetch_shared(self, dataset: str) -> RawPayload: | |
| 96 | + with self._lock: | |
| 97 | + if dataset in self._cache: | |
| 98 | + return self._cache[dataset] | |
| 99 | + url = DATASET_URLS[dataset] | |
| 100 | + log.info("owid: downloading %s (%s)", dataset, url) | |
| 101 | + r = self.get(url) | |
| 102 | + if not r.content or not r.content.lstrip().startswith(b"country"): | |
| 103 | + raise ConnectorError(f"owid {dataset}: unexpected CSV header") | |
| 104 | + p = self.payload( | |
| 105 | + r, | |
| 106 | + dataset=dataset, | |
| 107 | + code=f"owid-{dataset}-data", | |
| 108 | + source_url=DATASET_HOME[dataset], | |
| 109 | + etag=r.headers.get("ETag"), | |
| 110 | + notes=f"{DATASET_NAMES[dataset]} — {DATASET_HOME[dataset]}", | |
| 111 | + ) | |
| 112 | + p.content_type = "text/csv" | |
| 113 | + with self._lock: | |
| 114 | + self._cache[dataset] = p | |
| 115 | + return p | |
| 116 | + | |
| 117 | + def _fetch_grapher(self, code: str) -> RawPayload: | |
| 118 | + with self._lock: | |
| 119 | + if f"grapher:{code}" in self._cache: | |
| 120 | + return self._cache[f"grapher:{code}"] | |
| 121 | + r = self.get(GRAPHER_CSV.format(code=code)) | |
| 122 | + head = r.content[:200].lower() | |
| 123 | + if not head.startswith(b"entity"): | |
| 124 | + raise ConnectorError(f"owid grapher {code}: unexpected CSV header {head[:60]!r} (chart renamed or removed?)") | |
| 125 | + meta: dict[str, Any] = {} | |
| 126 | + try: | |
| 127 | + m = self.get(GRAPHER_META.format(code=code)) | |
| 128 | + doc = orjson.loads(m.content) | |
| 129 | + chart = doc.get("chart", {}) | |
| 130 | + cols = doc.get("columns", {}) | |
| 131 | + first = next(iter(cols.values()), {}) if cols else {} | |
| 132 | + meta = { | |
| 133 | + "title": chart.get("title"), | |
| 134 | + "subtitle": chart.get("subtitle"), | |
| 135 | + "citation": chart.get("citation"), | |
| 136 | + "originalChartUrl": chart.get("originalChartUrl"), | |
| 137 | + "columns": {k: {kk: v.get(kk) for kk in ("titleShort", "unit", "shortUnit", "lastUpdated", "citationShort", | |
| 138 | + "citationLong", "descriptionShort", "shortName", "timespan")} | |
| 139 | + for k, v in cols.items()}, | |
| 140 | + "lastUpdated": first.get("lastUpdated"), | |
| 141 | + } | |
| 142 | + except Exception as e: # noqa: BLE001 — metadata is optional | |
| 143 | + log.warning("owid grapher %s: metadata unavailable: %s", code, e) | |
| 144 | + notes = " — ".join(x for x in (meta.get("title"), meta.get("citation")) if x) or None | |
| 145 | + p = self.payload( | |
| 146 | + r, | |
| 147 | + dataset="grapher", | |
| 148 | + code=code, | |
| 149 | + grapher_metadata=meta, | |
| 150 | + source_url=GRAPHER_PAGE.format(code=code), | |
| 151 | + notes=notes, | |
| 152 | + final_url=str(r.url), | |
| 153 | + ) | |
| 154 | + p.content_type = "text/csv" | |
| 155 | + upd = parse_date_utc(meta.get("lastUpdated")) | |
| 156 | + if upd: | |
| 157 | + p.source_updated_at = upd | |
| 158 | + with self._lock: | |
| 159 | + self._cache[f"grapher:{code}"] = p | |
| 160 | + return p | |
| 161 | + | |
| 162 | + # ----------------------------------------------------------------- normalize | |
| 163 | + def _frame(self, raw: RawPayload) -> pl.DataFrame: | |
| 164 | + """Parse the CSV once per payload (cached by dataset/code so 40 specs reuse one parse).""" | |
| 165 | + key = f"{raw.dataset}:{raw.code}:{len(raw.body)}" | |
| 166 | + with self._lock: | |
| 167 | + if key in self._frames: | |
| 168 | + return self._frames[key] | |
| 169 | + df = pl.read_csv(io.BytesIO(raw.body), infer_schema_length=10000, null_values=["", "NA"], low_memory=False) | |
| 170 | + with self._lock: | |
| 171 | + self._frames[key] = df | |
| 172 | + return df | |
| 173 | + | |
| 174 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 175 | + r = raw[0] if isinstance(raw, list) else raw | |
| 176 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 177 | + unit = ind.unit if ind else "" | |
| 178 | + df = self._frame(r) | |
| 179 | + if r.dataset in DATASET_URLS: | |
| 180 | + frame = self._select_shared(df, spec.code) | |
| 181 | + else: | |
| 182 | + frame = self._select_grapher(df, spec.code) | |
| 183 | + if frame is None: | |
| 184 | + raise ConnectorError(f"owid {r.dataset}/{spec.code}: column not found in CSV") | |
| 185 | + # drop aggregates & unknown codes, map to registry ISO3 | |
| 186 | + lk = lookup() | |
| 187 | + frame = ( | |
| 188 | + frame.filter(pl.col("iso").is_not_null() & ~pl.col("iso").str.starts_with("OWID_") & pl.col("value").is_not_null()) | |
| 189 | + .with_columns(pl.col("iso").str.to_uppercase().replace("UNK", "XKX")) | |
| 190 | + .filter(pl.col("iso").is_in(sorted(lk.iso3))) | |
| 191 | + .filter(pl.col("year").is_not_null()) | |
| 192 | + ) | |
| 193 | + if "is_forecast" in frame.columns: | |
| 194 | + # projections (UN WPP runs to 2100): keep only a short horizon, like IMF WEO forecasts | |
| 195 | + horizon = datetime.now(tz=UTC).year + FORECAST_HORIZON_YEARS | |
| 196 | + frame = frame.filter(~pl.col("is_forecast") | (pl.col("year") <= horizon)) | |
| 197 | + if frame.is_empty(): | |
| 198 | + return [] | |
| 199 | + values = apply_transform_array(frame["value"].cast(pl.Float64).to_numpy().astype("float64"), spec.transform) | |
| 200 | + years = frame["year"].cast(pl.Int32).to_numpy() | |
| 201 | + isos = frame["iso"].to_list() | |
| 202 | + forecasts = frame["is_forecast"].to_numpy() if "is_forecast" in frame.columns else np.zeros(len(isos), dtype=bool) | |
| 203 | + src_upd = r.source_updated_at | |
| 204 | + dataset = spec.dataset | |
| 205 | + code = spec.code | |
| 206 | + out: list[NormalizedObservation] = [] | |
| 207 | + for iso, y, v, fc in zip(isos, years.tolist(), values.tolist(), forecasts.tolist(), strict=True): | |
| 208 | + if v is None or not np.isfinite(v): | |
| 209 | + continue | |
| 210 | + out.append( | |
| 211 | + NormalizedObservation( | |
| 212 | + country_id=iso, | |
| 213 | + indicator_id=spec.indicator_id, | |
| 214 | + period=date(int(y), 1, 1), | |
| 215 | + year=int(y), | |
| 216 | + frequency=spec.frequency or "A", | |
| 217 | + value=float(v), | |
| 218 | + unit=unit, | |
| 219 | + source_id=self.id, | |
| 220 | + source_dataset=dataset, | |
| 221 | + source_series_code=code, | |
| 222 | + is_forecast=bool(fc), | |
| 223 | + retrieved_at=r.retrieved_at, | |
| 224 | + source_updated_at=src_upd, | |
| 225 | + ) | |
| 226 | + ) | |
| 227 | + return out | |
| 228 | + | |
| 229 | + @staticmethod | |
| 230 | + def _select_shared(df: pl.DataFrame, column: str) -> pl.DataFrame | None: | |
| 231 | + if column not in df.columns: | |
| 232 | + return None | |
| 233 | + return df.select( | |
| 234 | + pl.col("iso_code").alias("iso"), pl.col("year"), pl.col(column).cast(pl.Float64, strict=False).alias("value") | |
| 235 | + ) | |
| 236 | + | |
| 237 | + @staticmethod | |
| 238 | + def _select_grapher(df: pl.DataFrame, code: str) -> pl.DataFrame | None: | |
| 239 | + cols = {c.lower(): c for c in df.columns} | |
| 240 | + ent, iso, year = cols.get("entity"), cols.get("code"), cols.get("year") | |
| 241 | + if not (ent and iso and year): | |
| 242 | + return None | |
| 243 | + numeric = [c for c in df.columns if c not in (ent, iso, year) and df.schema[c].is_numeric()] | |
| 244 | + if not numeric: | |
| 245 | + return None | |
| 246 | + estimate_cols = [c for c in numeric if "project" not in c.lower()] | |
| 247 | + value_col = estimate_cols[0] if estimate_cols else numeric[0] | |
| 248 | + proj_cols = [c for c in numeric if "project" in c.lower() and c != value_col] | |
| 249 | + base = df.select(pl.col(iso).alias("iso"), pl.col(year).alias("year"), pl.col(value_col).cast(pl.Float64).alias("value")) | |
| 250 | + if not proj_cols: | |
| 251 | + return base.with_columns(pl.lit(False).alias("is_forecast")) | |
| 252 | + proj = pl.col(proj_cols[0]).cast(pl.Float64) | |
| 253 | + return df.select( | |
| 254 | + pl.col(iso).alias("iso"), | |
| 255 | + pl.col(year).alias("year"), | |
| 256 | + pl.coalesce([pl.col(value_col).cast(pl.Float64), proj]).alias("value"), | |
| 257 | + (pl.col(value_col).is_null() & proj.is_not_null()).alias("is_forecast"), | |
| 258 | + ) | |
added
src/countryatlas/connectors/who.py
+219 −0
@@ -0,0 +1,219 @@ | ||
| 1 | +"""WHO Global Health Observatory (GHO) OData connector. | |
| 2 | + | |
| 3 | +* fetch: `GET https://ghoapi.azureedge.net/api/{code}?$filter=SpatialDimType eq 'COUNTRY' [and Dim1 eq '…'] [and Dim2 …] | |
| 4 | + [and Dim3 …]` — every `params` key named Dim1/Dim2/Dim3 becomes a server-side filter; any other key is appended as an | |
| 5 | + additional `$filter` clause on that column (e.g. `{"TimeDim": "ge 2000"}` is NOT supported; keep to equality). | |
| 6 | + OData `@odata.nextLink` pages are followed. The indicator name is fetched once (`/Indicator?$filter=IndicatorCode eq …`) | |
| 7 | + and stored in `RawPayload.meta["indicator_meta"]`. | |
| 8 | +* normalize: rows → `SpatialDim` (ISO3, mapped through the registry), `TimeDim` (int year), `NumericValue`. | |
| 9 | + Rows whose Dim1/Dim2/Dim3 differ from the spec params are dropped (defensive — the filter already did it), as are | |
| 10 | + non-country rows and unknown ISO3. `Low`/`High` confidence bounds go to metadata. Years after the current year are | |
| 11 | + projections (`M_Est_tob_curr` carries 2030 values) → `is_forecast=true`. Duplicate (country, year) keys after | |
| 12 | + filtering keep the most recently modified row (`Date`) and are logged. | |
| 13 | + `Date` (row last-modified) → `source_updated_at` = max over the payload. | |
| 14 | + | |
| 15 | +Licence: CC BY-NC-SA 3.0 IGO — attribution "World Health Organization, Global Health Observatory". | |
| 16 | +""" | |
| 17 | +from __future__ import annotations | |
| 18 | + | |
| 19 | +import logging | |
| 20 | +import threading | |
| 21 | +from datetime import UTC, date, datetime | |
| 22 | +from typing import Any, ClassVar | |
| 23 | + | |
| 24 | +import orjson | |
| 25 | + | |
| 26 | +from countryatlas.connectors._util import ConnectorError, parse_date_utc | |
| 27 | +from countryatlas.connectors.base import Connector | |
| 28 | +from countryatlas.models import DatasetDescriptor, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 29 | +from countryatlas.registry import indicators_by_id, lookup | |
| 30 | + | |
| 31 | +log = logging.getLogger(__name__) | |
| 32 | + | |
| 33 | +DIM_KEYS = ("Dim1", "Dim2", "Dim3") | |
| 34 | +SOURCE_URL_PATTERN = "https://www.who.int/data/gho/data/indicators/indicator-details/GHO/{code}" | |
| 35 | + | |
| 36 | + | |
| 37 | +def _odata_quote(v: Any) -> str: | |
| 38 | + return "'" + str(v).replace("'", "''") + "'" | |
| 39 | + | |
| 40 | + | |
| 41 | +class WHOConnector(Connector): | |
| 42 | + id: ClassVar[str] = "who" | |
| 43 | + name: ClassVar[str] = "WHO Global Health Observatory" | |
| 44 | + organization: ClassVar[str] = "World Health Organization" | |
| 45 | + url: ClassVar[str] = "https://www.who.int/data/gho" | |
| 46 | + licence: ClassVar[str] = "CC BY-NC-SA 3.0 IGO" | |
| 47 | + attribution: ClassVar[str] = "World Health Organization, Global Health Observatory" | |
| 48 | + api_base: ClassVar[str] = "https://ghoapi.azureedge.net/api" | |
| 49 | + rate_per_minute: ClassVar[int] = 60 | |
| 50 | + timeout: ClassVar[float] = 180.0 | |
| 51 | + country_codes: ClassVar[str] = "iso3" | |
| 52 | + | |
| 53 | + def __init__(self) -> None: | |
| 54 | + super().__init__() | |
| 55 | + self._meta_cache: dict[str, dict[str, Any]] = {} | |
| 56 | + self._lock = threading.Lock() | |
| 57 | + | |
| 58 | + def discover(self) -> list[DatasetDescriptor]: | |
| 59 | + return [ | |
| 60 | + DatasetDescriptor( | |
| 61 | + connector=self.id, | |
| 62 | + dataset="GHO", | |
| 63 | + name="Global Health Observatory indicators", | |
| 64 | + url="https://www.who.int/data/gho/info/gho-odata-api", | |
| 65 | + licence=self.licence, | |
| 66 | + notes="OData v4 API, one call per indicator (SpatialDimType eq 'COUNTRY' + Dim filters).", | |
| 67 | + ) | |
| 68 | + ] | |
| 69 | + | |
| 70 | + # --------------------------------------------------------------------- fetch | |
| 71 | + def indicator_metadata(self, code: str) -> dict[str, Any]: | |
| 72 | + with self._lock: | |
| 73 | + if code in self._meta_cache: | |
| 74 | + return self._meta_cache[code] | |
| 75 | + meta: dict[str, Any] = {} | |
| 76 | + try: | |
| 77 | + r = self.get(f"{self.api_base}/Indicator", params={"$filter": f"IndicatorCode eq {_odata_quote(code)}"}) | |
| 78 | + vals = orjson.loads(r.content).get("value") or [] | |
| 79 | + if vals: | |
| 80 | + meta = {"name": vals[0].get("IndicatorName"), "language": vals[0].get("Language")} | |
| 81 | + except Exception as e: # noqa: BLE001 — metadata is optional | |
| 82 | + log.warning("who: metadata for %s unavailable: %s", code, e) | |
| 83 | + with self._lock: | |
| 84 | + self._meta_cache[code] = meta | |
| 85 | + return meta | |
| 86 | + | |
| 87 | + @staticmethod | |
| 88 | + def build_filter(params: dict[str, Any] | None) -> str: | |
| 89 | + clauses = ["SpatialDimType eq 'COUNTRY'"] | |
| 90 | + for k, v in (params or {}).items(): | |
| 91 | + if v is None: | |
| 92 | + continue | |
| 93 | + clauses.append(f"{k} eq {_odata_quote(v)}") | |
| 94 | + return " and ".join(clauses) | |
| 95 | + | |
| 96 | + def fetch(self, spec: IndicatorSourceSpec) -> list[RawPayload]: | |
| 97 | + code = spec.code | |
| 98 | + if not code: | |
| 99 | + raise ConnectorError("who: spec without indicator code") | |
| 100 | + meta = self.indicator_metadata(code) | |
| 101 | + url: str | None = f"{self.api_base}/{code}" | |
| 102 | + params: dict[str, Any] | None = {"$filter": self.build_filter(spec.params)} | |
| 103 | + payloads: list[RawPayload] = [] | |
| 104 | + page = 1 | |
| 105 | + while url: | |
| 106 | + r = self.get(url, params=params) | |
| 107 | + try: | |
| 108 | + doc = orjson.loads(r.content) | |
| 109 | + except orjson.JSONDecodeError as e: | |
| 110 | + raise ConnectorError(f"who {code}: response is not JSON ({e}) — unknown indicator code?") from e | |
| 111 | + if not isinstance(doc, dict) or "value" not in doc: | |
| 112 | + raise ConnectorError(f"who {code}: unexpected payload shape (no 'value' array) — unknown indicator code?") | |
| 113 | + p = self.payload( | |
| 114 | + r, | |
| 115 | + dataset=spec.dataset or "GHO", | |
| 116 | + code=code, | |
| 117 | + page=page, | |
| 118 | + indicator_meta=meta, | |
| 119 | + filter=params["$filter"] if params else None, | |
| 120 | + source_url=SOURCE_URL_PATTERN.format(code=code), | |
| 121 | + notes=meta.get("name"), | |
| 122 | + ) | |
| 123 | + payloads.append(p) | |
| 124 | + url = doc.get("@odata.nextLink") | |
| 125 | + params = None # nextLink already carries the query string | |
| 126 | + page += 1 | |
| 127 | + for p in payloads: | |
| 128 | + p.pages = len(payloads) | |
| 129 | + n = sum(len(orjson.loads(p.body).get("value") or []) for p in payloads) | |
| 130 | + if n == 0: | |
| 131 | + raise ConnectorError(f"who {code}: 0 rows for filter {self.build_filter(spec.params)!r}") | |
| 132 | + return payloads | |
| 133 | + | |
| 134 | + # ----------------------------------------------------------------- normalize | |
| 135 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 136 | + raws = raw if isinstance(raw, list) else [raw] | |
| 137 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 138 | + unit = ind.unit if ind else "" | |
| 139 | + lk = lookup() | |
| 140 | + want = {k: str(v) for k, v in (spec.params or {}).items() if k in DIM_KEYS and v is not None} | |
| 141 | + current_year = datetime.now(UTC).year | |
| 142 | + best: dict[tuple[str, int], tuple[str, dict[str, Any], datetime]] = {} | |
| 143 | + latest_date: datetime | None = None | |
| 144 | + n_dropped = {"non_country": 0, "unknown": 0, "dim_mismatch": 0, "null": 0, "dups": 0} | |
| 145 | + for r in raws: | |
| 146 | + doc = orjson.loads(r.body) | |
| 147 | + for row in doc.get("value") or []: | |
| 148 | + if (row.get("SpatialDimType") or "").upper() != "COUNTRY": | |
| 149 | + n_dropped["non_country"] += 1 | |
| 150 | + continue | |
| 151 | + if any(str(row.get(k)) != v for k, v in want.items()): | |
| 152 | + n_dropped["dim_mismatch"] += 1 | |
| 153 | + continue | |
| 154 | + val = row.get("NumericValue") | |
| 155 | + if val is None: | |
| 156 | + n_dropped["null"] += 1 | |
| 157 | + continue | |
| 158 | + iso3 = lk.from_iso3(row.get("SpatialDim")) | |
| 159 | + if iso3 is None: | |
| 160 | + n_dropped["unknown"] += 1 | |
| 161 | + continue | |
| 162 | + year = row.get("TimeDim") | |
| 163 | + if year is None: | |
| 164 | + tdv = row.get("TimeDimensionValue") | |
| 165 | + if not (isinstance(tdv, str) and tdv[:4].isdigit()): | |
| 166 | + continue | |
| 167 | + year = int(tdv[:4]) | |
| 168 | + year = int(year) | |
| 169 | + row_date = parse_date_utc(row.get("Date")) or r.retrieved_at | |
| 170 | + if latest_date is None or row_date > latest_date: | |
| 171 | + latest_date = row_date | |
| 172 | + key = (iso3, year) | |
| 173 | + if key in best: | |
| 174 | + n_dropped["dups"] += 1 | |
| 175 | + if row_date <= best[key][2]: | |
| 176 | + continue | |
| 177 | + best[key] = (iso3, row, row_date) | |
| 178 | + if n_dropped["dups"]: | |
| 179 | + log.warning("who %s: %d duplicate (country, year) rows after Dim filters — kept the most recent", spec.code, | |
| 180 | + n_dropped["dups"]) | |
| 181 | + first = raws[0] | |
| 182 | + src_upd = latest_date or first.source_updated_at | |
| 183 | + out: list[NormalizedObservation] = [] | |
| 184 | + for (iso3, year), (_, row, row_date) in sorted(best.items()): | |
| 185 | + try: | |
| 186 | + value = self.apply_transform(float(row["NumericValue"]), spec.transform) | |
| 187 | + except (TypeError, ValueError): | |
| 188 | + continue | |
| 189 | + meta: dict[str, Any] = {} | |
| 190 | + if row.get("Low") is not None: | |
| 191 | + meta["low"] = row["Low"] | |
| 192 | + if row.get("High") is not None: | |
| 193 | + meta["high"] = row["High"] | |
| 194 | + for k in DIM_KEYS: | |
| 195 | + if row.get(k) and k not in want: | |
| 196 | + meta[k.lower()] = row[k] | |
| 197 | + if row.get("Comments"): | |
| 198 | + meta["comments"] = str(row["Comments"])[:300] | |
| 199 | + out.append( | |
| 200 | + NormalizedObservation( | |
| 201 | + country_id=iso3, | |
| 202 | + indicator_id=spec.indicator_id, | |
| 203 | + period=date(year, 1, 1), | |
| 204 | + year=year, | |
| 205 | + frequency=spec.frequency or "A", | |
| 206 | + value=value, | |
| 207 | + unit=unit, | |
| 208 | + source_id=self.id, | |
| 209 | + source_dataset=spec.dataset or "GHO", | |
| 210 | + source_series_code=spec.code, | |
| 211 | + is_estimate=False, | |
| 212 | + is_forecast=year > current_year, | |
| 213 | + retrieved_at=first.retrieved_at, | |
| 214 | + source_updated_at=src_upd, | |
| 215 | + metadata=meta, | |
| 216 | + ) | |
| 217 | + ) | |
| 218 | + log.debug("who %s: %d rows kept, dropped %s", spec.code, len(out), n_dropped) | |
| 219 | + return out | |
added
src/countryatlas/connectors/worldbank.py
+232 −0
@@ -0,0 +1,232 @@ | ||
| 1 | +"""World Bank Indicators API v2 connector (World Development Indicators and other WB databases). | |
| 2 | + | |
| 3 | +* fetch: `GET /v2/country/all/indicator/{code}?format=json&per_page=20000&date=1960:2026`, following every page | |
| 4 | + announced in the header object. Indicator metadata (`/v2/indicator/{code}`) is fetched once per code and stored in | |
| 5 | + `RawPayload.meta["indicator_meta"]` (sourceNote, sourceOrganization, lastupdated). | |
| 6 | +* normalize: rows carry `countryiso3code` (empty for some aggregates), `date`, `value` (null → skipped), `obs_status`. | |
| 7 | + Countries are mapped with `registry.lookup().from_iso3()`; aggregates (not in the registry) are dropped. | |
| 8 | +* Retired / invalid codes come back as HTTP 200 with `[{"message": [...]}]` → raised as ConnectorError (spec failed). | |
| 9 | + | |
| 10 | +Licence: CC BY 4.0 — attribution "World Bank, World Development Indicators". | |
| 11 | +""" | |
| 12 | +from __future__ import annotations | |
| 13 | + | |
| 14 | +import json | |
| 15 | +import logging | |
| 16 | +import threading | |
| 17 | +from typing import Any, ClassVar | |
| 18 | + | |
| 19 | +import orjson | |
| 20 | + | |
| 21 | +from countryatlas.connectors._util import ConnectorError, parse_date_utc, parse_period | |
| 22 | +from countryatlas.connectors.base import Connector | |
| 23 | +from countryatlas.models import DatasetDescriptor, IndicatorSourceSpec, NormalizedObservation, RawPayload | |
| 24 | +from countryatlas.registry import indicators_by_id, lookup | |
| 25 | + | |
| 26 | +log = logging.getLogger(__name__) | |
| 27 | + | |
| 28 | +SOURCE_URL_PATTERN = "https://data.worldbank.org/indicator/{code}" | |
| 29 | +WDI_SOURCE_ID = "2" | |
| 30 | +ARCHIVE_SOURCE_ID = "57" # "WDI Database Archives": indicator retired from WDI | |
| 31 | + | |
| 32 | + | |
| 33 | +class WorldBankConnector(Connector): | |
| 34 | + id: ClassVar[str] = "worldbank" | |
| 35 | + name: ClassVar[str] = "World Bank" | |
| 36 | + organization: ClassVar[str] = "World Bank Group" | |
| 37 | + url: ClassVar[str] = "https://data.worldbank.org" | |
| 38 | + licence: ClassVar[str] = "CC BY 4.0" | |
| 39 | + attribution: ClassVar[str] = "World Bank, World Development Indicators" | |
| 40 | + api_base: ClassVar[str] = "https://api.worldbank.org/v2" | |
| 41 | + rate_per_minute: ClassVar[int] = 100 | |
| 42 | + timeout: ClassVar[float] = 120.0 | |
| 43 | + country_codes: ClassVar[str] = "wb" | |
| 44 | + | |
| 45 | + PER_PAGE: ClassVar[int] = 20000 | |
| 46 | + DATE_RANGE: ClassVar[str] = "1960:2026" | |
| 47 | + | |
| 48 | + def __init__(self) -> None: | |
| 49 | + super().__init__() | |
| 50 | + self._meta_cache: dict[str, dict[str, Any]] = {} | |
| 51 | + self._meta_lock = threading.Lock() | |
| 52 | + | |
| 53 | + # ------------------------------------------------------------------ discover | |
| 54 | + def discover(self) -> list[DatasetDescriptor]: | |
| 55 | + return [ | |
| 56 | + DatasetDescriptor( | |
| 57 | + connector=self.id, | |
| 58 | + dataset="WDI", | |
| 59 | + name="World Development Indicators", | |
| 60 | + url="https://datatopics.worldbank.org/world-development-indicators/", | |
| 61 | + licence=self.licence, | |
| 62 | + notes="Indicators API v2, JSON, all countries and aggregates, 1960 onwards.", | |
| 63 | + ) | |
| 64 | + ] | |
| 65 | + | |
| 66 | + # --------------------------------------------------------------------- fetch | |
| 67 | + def indicator_metadata(self, code: str) -> dict[str, Any]: | |
| 68 | + """`/indicator/{code}` → {name, sourceNote, sourceOrganization, source, unit, lastupdated}. Cached per instance.""" | |
| 69 | + with self._meta_lock: | |
| 70 | + if code in self._meta_cache: | |
| 71 | + return self._meta_cache[code] | |
| 72 | + meta: dict[str, Any] = {} | |
| 73 | + try: | |
| 74 | + r = self.get(f"{self.api_base}/indicator/{code}", params={"format": "json"}) | |
| 75 | + doc = orjson.loads(r.content) | |
| 76 | + if isinstance(doc, list) and len(doc) == 2 and doc[1]: | |
| 77 | + item = doc[1][0] | |
| 78 | + meta = { | |
| 79 | + "name": item.get("name"), | |
| 80 | + "unit": item.get("unit") or None, | |
| 81 | + "source": (item.get("source") or {}).get("value"), | |
| 82 | + "source_id": (item.get("source") or {}).get("id"), | |
| 83 | + "sourceNote": item.get("sourceNote"), | |
| 84 | + "sourceOrganization": item.get("sourceOrganization"), | |
| 85 | + "topics": [t.get("value") for t in item.get("topics", []) if t.get("value")], | |
| 86 | + "lastupdated": doc[0].get("lastupdated"), | |
| 87 | + } | |
| 88 | + except Exception as e: # noqa: BLE001 — metadata is optional | |
| 89 | + log.warning("worldbank: metadata for %s unavailable: %s", code, e) | |
| 90 | + with self._meta_lock: | |
| 91 | + self._meta_cache[code] = meta | |
| 92 | + return meta | |
| 93 | + | |
| 94 | + def fetch(self, spec: IndicatorSourceSpec) -> list[RawPayload]: | |
| 95 | + code = spec.code | |
| 96 | + url = f"{self.api_base}/country/all/indicator/{code}" | |
| 97 | + params: dict[str, Any] = {"format": "json", "per_page": self.PER_PAGE, "date": self.DATE_RANGE, "page": 1} | |
| 98 | + params.update(spec.params or {}) | |
| 99 | + payloads: list[RawPayload] = [] | |
| 100 | + page, pages = 1, 1 | |
| 101 | + meta = self.indicator_metadata(code) | |
| 102 | + # Series living in another WB database (e.g. WGI = source 3) need `source=<id>`; the default endpoint only knows | |
| 103 | + # WDI (2). Codes moved to "WDI Database Archives" (57) are retired: the data endpoint no longer serves them. | |
| 104 | + src_id = str(meta.get("source_id") or "") | |
| 105 | + if "source" not in params and src_id and src_id not in (WDI_SOURCE_ID, ARCHIVE_SOURCE_ID): | |
| 106 | + params["source"] = src_id | |
| 107 | + while page <= pages: | |
| 108 | + params["page"] = page | |
| 109 | + r = self.get(url, params=params) | |
| 110 | + doc = orjson.loads(r.content) | |
| 111 | + header = _check_header(doc, code, archived=src_id == ARCHIVE_SOURCE_ID) | |
| 112 | + pages = int(header.get("pages") or 1) | |
| 113 | + lastupdated = header.get("lastupdated") | |
| 114 | + p = self.payload( | |
| 115 | + r, | |
| 116 | + dataset=spec.dataset or "WDI", | |
| 117 | + code=code, | |
| 118 | + pages=pages, | |
| 119 | + page=page, | |
| 120 | + total=header.get("total"), | |
| 121 | + lastupdated=lastupdated, | |
| 122 | + indicator_meta=meta, | |
| 123 | + source_url=SOURCE_URL_PATTERN.format(code=code), | |
| 124 | + notes=_notes(meta), | |
| 125 | + ) | |
| 126 | + upd = parse_date_utc(lastupdated) or parse_date_utc(meta.get("lastupdated")) | |
| 127 | + if upd: | |
| 128 | + p.source_updated_at = upd | |
| 129 | + payloads.append(p) | |
| 130 | + page += 1 | |
| 131 | + total_rows = sum(len(x) for x in (_rows(orjson.loads(pl.body)) for pl in payloads)) | |
| 132 | + if total_rows == 0: | |
| 133 | + raise ConnectorError(f"worldbank {code}: no data returned (empty series — possibly retired)") | |
| 134 | + return payloads | |
| 135 | + | |
| 136 | + # ----------------------------------------------------------------- normalize | |
| 137 | + def normalize(self, raw: RawPayload | list[RawPayload], spec: IndicatorSourceSpec) -> list[NormalizedObservation]: | |
| 138 | + raws = raw if isinstance(raw, list) else [raw] | |
| 139 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 140 | + unit = ind.unit if ind else "" | |
| 141 | + lk = lookup() | |
| 142 | + out: list[NormalizedObservation] = [] | |
| 143 | + seen: set[tuple[str, str]] = set() | |
| 144 | + n_dropped_aggregates = 0 | |
| 145 | + n_unknown = 0 | |
| 146 | + for r in raws: | |
| 147 | + doc = orjson.loads(r.body) | |
| 148 | + src_upd = r.source_updated_at or parse_date_utc((r.meta or {}).get("lastupdated")) | |
| 149 | + for row in _rows(doc): | |
| 150 | + val = row.get("value") | |
| 151 | + if val is None: | |
| 152 | + continue | |
| 153 | + iso3 = lk.from_iso3(row.get("countryiso3code") or None) | |
| 154 | + if iso3 is None: | |
| 155 | + # WB aggregates (regions, income groups) either have no ISO3 or one not in the registry | |
| 156 | + if (row.get("countryiso3code") or "") and len(row["countryiso3code"]) == 3: | |
| 157 | + n_unknown += 1 | |
| 158 | + else: | |
| 159 | + n_dropped_aggregates += 1 | |
| 160 | + continue | |
| 161 | + per = parse_period(str(row.get("date", ""))) | |
| 162 | + if per is None: | |
| 163 | + continue | |
| 164 | + period, year, freq = per | |
| 165 | + if spec.frequency: | |
| 166 | + freq = spec.frequency | |
| 167 | + key = (iso3, period.isoformat()) | |
| 168 | + if key in seen: # WB occasionally repeats a row across pages | |
| 169 | + continue | |
| 170 | + seen.add(key) | |
| 171 | + try: | |
| 172 | + value = self.apply_transform(float(val), spec.transform) | |
| 173 | + except (TypeError, ValueError): | |
| 174 | + continue | |
| 175 | + obs_status = (row.get("obs_status") or "").strip() | |
| 176 | + meta: dict[str, Any] = {} | |
| 177 | + if obs_status: | |
| 178 | + meta["obs_status"] = obs_status | |
| 179 | + if row.get("unit"): | |
| 180 | + meta["source_unit"] = row["unit"] | |
| 181 | + out.append( | |
| 182 | + NormalizedObservation( | |
| 183 | + country_id=iso3, | |
| 184 | + indicator_id=spec.indicator_id, | |
| 185 | + period=period, | |
| 186 | + year=year, | |
| 187 | + frequency=freq, | |
| 188 | + value=value, | |
| 189 | + unit=unit, | |
| 190 | + source_id=self.id, | |
| 191 | + source_dataset=spec.dataset or "WDI", | |
| 192 | + source_series_code=spec.code, | |
| 193 | + is_estimate=obs_status.upper() == "E", | |
| 194 | + is_forecast=obs_status.upper() == "F", | |
| 195 | + retrieved_at=r.retrieved_at, | |
| 196 | + source_updated_at=src_upd, | |
| 197 | + metadata=meta, | |
| 198 | + ) | |
| 199 | + ) | |
| 200 | + if n_unknown: | |
| 201 | + log.debug("worldbank %s: %d rows with unknown ISO3 dropped", spec.code, n_unknown) | |
| 202 | + return out | |
| 203 | + | |
| 204 | + | |
| 205 | +# ---------------------------------------------------------------------------------------------------------- helpers | |
| 206 | +def _check_header(doc: Any, code: str, archived: bool = False) -> dict[str, Any]: | |
| 207 | + """Return the header object of a WB v2 response or raise ConnectorError for retired/invalid codes.""" | |
| 208 | + if isinstance(doc, list) and doc and isinstance(doc[0], dict) and "message" in doc[0]: | |
| 209 | + msgs = doc[0]["message"] | |
| 210 | + text = "; ".join(f"{m.get('key')}: {m.get('value')}" for m in msgs) if isinstance(msgs, list) else str(msgs) | |
| 211 | + hint = " (code retired or invalid?)" | |
| 212 | + if archived: | |
| 213 | + hint = " — RETIRED: the code now lives in 'WDI Database Archives' (source 57); fix the registry mapping" | |
| 214 | + raise ConnectorError(f"worldbank {code}: API error — {text}{hint}") | |
| 215 | + if not isinstance(doc, list) or len(doc) < 2 or not isinstance(doc[0], dict): | |
| 216 | + raise ConnectorError(f"worldbank {code}: unexpected payload shape {json.dumps(doc)[:200]}") | |
| 217 | + return doc[0] | |
| 218 | + | |
| 219 | + | |
| 220 | +def _rows(doc: Any) -> list[dict[str, Any]]: | |
| 221 | + if isinstance(doc, list) and len(doc) >= 2 and isinstance(doc[1], list): | |
| 222 | + return doc[1] | |
| 223 | + return [] | |
| 224 | + | |
| 225 | + | |
| 226 | +def _notes(meta: dict[str, Any]) -> str | None: | |
| 227 | + parts = [] | |
| 228 | + if meta.get("sourceNote"): | |
| 229 | + parts.append(str(meta["sourceNote"]).strip()) | |
| 230 | + if meta.get("sourceOrganization"): | |
| 231 | + parts.append("Source: " + " ".join(str(meta["sourceOrganization"]).split())) | |
| 232 | + return "\n".join(parts) or None | |
modified
src/countryatlas/pipeline/__init__.py
+66 −0
@@ -0,0 +1,66 @@ | ||
| 1 | +"""CountryAtlas data pipeline: fetch → normalize → validate → staging parquet → build DuckDB snapshot → atomic swap. | |
| 2 | + | |
| 3 | +See docs/PIPELINE.md. Modules: | |
| 4 | + fetch.py per-spec fetch/normalize/validate with error isolation, staging writer | |
| 5 | + validate.py generic deterministic validation rules (ARCHITECTURE §6) | |
| 6 | + build.py snapshot build (registry tables, merge by priority, revisions, derived tables, integrity, swap) | |
| 7 | + derived.py SQL-based derived tables (latest, rankings, coverage, search_index, meta) | |
| 8 | + changes.py deterministic change/event detectors + headline templates | |
| 9 | + similarity.py peers (5 modes) + country DNA | |
| 10 | + insights.py templated insights (registry/insights.yaml) | |
| 11 | + scheduler.py daily loop (03:15 America/Toronto, SIGUSR1) | |
| 12 | + export.py CSV/JSON/Parquet exports | |
| 13 | +""" | |
| 14 | +from __future__ import annotations | |
| 15 | + | |
| 16 | +import logging | |
| 17 | +import sys | |
| 18 | +from datetime import UTC, datetime | |
| 19 | +from pathlib import Path | |
| 20 | + | |
| 21 | +_LOG_FORMAT = "%(asctime)s %(levelname)-7s %(name)s: %(message)s" | |
| 22 | + | |
| 23 | + | |
| 24 | +def new_run_id(now: datetime | None = None) -> str: | |
| 25 | + """Run identifiers look like 20260911T031500Z (UTC, sortable).""" | |
| 26 | + now = now or datetime.now(UTC) | |
| 27 | + return now.strftime("%Y%m%dT%H%M%SZ") | |
| 28 | + | |
| 29 | + | |
| 30 | +def setup_logging(level: int = logging.INFO, logfile: Path | None = None, rich: bool = True) -> None: | |
| 31 | + """Log to stderr (rich when available) and optionally to a file. Idempotent.""" | |
| 32 | + root = logging.getLogger() | |
| 33 | + if getattr(root, "_ca_configured", False): | |
| 34 | + if logfile is not None and not any(getattr(h, "_ca_file", None) == str(logfile) for h in root.handlers): | |
| 35 | + _add_file_handler(root, logfile) | |
| 36 | + return | |
| 37 | + root.setLevel(level) | |
| 38 | + for h in list(root.handlers): | |
| 39 | + root.removeHandler(h) | |
| 40 | + handler: logging.Handler | |
| 41 | + if rich: | |
| 42 | + try: | |
| 43 | + from rich.logging import RichHandler | |
| 44 | + | |
| 45 | + handler = RichHandler(rich_tracebacks=False, show_path=False, markup=False, log_time_format="%H:%M:%S") | |
| 46 | + handler.setFormatter(logging.Formatter("%(name)s: %(message)s")) | |
| 47 | + except Exception: # noqa: BLE001 | |
| 48 | + handler = logging.StreamHandler(sys.stderr) | |
| 49 | + handler.setFormatter(logging.Formatter(_LOG_FORMAT)) | |
| 50 | + else: | |
| 51 | + handler = logging.StreamHandler(sys.stderr) | |
| 52 | + handler.setFormatter(logging.Formatter(_LOG_FORMAT)) | |
| 53 | + root.addHandler(handler) | |
| 54 | + if logfile is not None: | |
| 55 | + _add_file_handler(root, logfile) | |
| 56 | + for noisy in ("httpx", "httpcore", "urllib3"): | |
| 57 | + logging.getLogger(noisy).setLevel(logging.WARNING) | |
| 58 | + root._ca_configured = True # type: ignore[attr-defined] | |
| 59 | + | |
| 60 | + | |
| 61 | +def _add_file_handler(root: logging.Logger, logfile: Path) -> None: | |
| 62 | + logfile.parent.mkdir(parents=True, exist_ok=True) | |
| 63 | + fh = logging.FileHandler(logfile, encoding="utf-8") | |
| 64 | + fh.setFormatter(logging.Formatter(_LOG_FORMAT)) | |
| 65 | + fh._ca_file = str(logfile) # type: ignore[attr-defined] | |
| 66 | + root.addHandler(fh) | |
added
src/countryatlas/pipeline/build.py
+448 −0
@@ -0,0 +1,448 @@ | ||
| 1 | +"""Snapshot build: staging parquet → brand-new DuckDB file → derived tables → integrity checks → atomic swap. | |
| 2 | + | |
| 3 | +The live database (`settings.db_path`) is never opened for writing. A failure anywhere leaves it untouched. | |
| 4 | +""" | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import json | |
| 8 | +import logging | |
| 9 | +import os | |
| 10 | +import shutil | |
| 11 | +import time | |
| 12 | +from dataclasses import dataclass, field | |
| 13 | +from datetime import UTC, datetime | |
| 14 | +from pathlib import Path | |
| 15 | +from typing import Any | |
| 16 | + | |
| 17 | +import duckdb | |
| 18 | +import polars as pl | |
| 19 | + | |
| 20 | +from countryatlas import registry | |
| 21 | +from countryatlas.config import ROOT, settings | |
| 22 | +from countryatlas.connectors import discover | |
| 23 | +from countryatlas.pipeline import derived, new_run_id | |
| 24 | +from countryatlas.pipeline.changes import compute_changes_and_events | |
| 25 | +from countryatlas.pipeline.insights import InsightContext, compute_insights | |
| 26 | +from countryatlas.pipeline.similarity import compute_country_dna, compute_similarity | |
| 27 | +from countryatlas.pipeline.staging import list_runs, list_staging_files, read_json | |
| 28 | + | |
| 29 | +log = logging.getLogger(__name__) | |
| 30 | + | |
| 31 | +SCHEMA_VERSION = 1 | |
| 32 | +SCHEMA_SQL = Path(__file__).resolve().parents[1] / "storage" / "schema.sql" | |
| 33 | +MIN_OBSERVATIONS_WITH_WB = 100_000 | |
| 34 | +HEADLINE_MIN_COUNTRIES = 100 | |
| 35 | + | |
| 36 | +# Public facts about sources whose connector module is not implemented yet (so `sources` is complete for the API). | |
| 37 | +SOURCE_DEFAULTS: dict[str, dict[str, str]] = { | |
| 38 | + "worldbank": {"name": "World Bank", "organization": "World Bank Group", "url": "https://data.worldbank.org"}, | |
| 39 | + "owid": {"name": "Our World in Data", "organization": "Global Change Data Lab", "url": "https://ourworldindata.org"}, | |
| 40 | + "imf": {"name": "IMF", "organization": "International Monetary Fund", "url": "https://www.imf.org/en/Data"}, | |
| 41 | + "oecd": {"name": "OECD", "organization": "Organisation for Economic Co-operation and Development", "url": "https://data.oecd.org"}, | |
| 42 | + "eurostat": {"name": "Eurostat", "organization": "European Commission", "url": "https://ec.europa.eu/eurostat"}, | |
| 43 | + "who": {"name": "WHO", "organization": "World Health Organization", "url": "https://www.who.int/data/gho"}, | |
| 44 | + "fred": {"name": "FRED", "organization": "Federal Reserve Bank of St. Louis", "url": "https://fred.stlouisfed.org"}, | |
| 45 | + "bis": {"name": "BIS", "organization": "Bank for International Settlements", "url": "https://data.bis.org"}, | |
| 46 | + "ilo": {"name": "ILOSTAT", "organization": "International Labour Organization", "url": "https://ilostat.ilo.org"}, | |
| 47 | +} | |
| 48 | + | |
| 49 | + | |
| 50 | +class IntegrityError(Exception): | |
| 51 | + pass | |
| 52 | + | |
| 53 | + | |
| 54 | +@dataclass | |
| 55 | +class BuildResult: | |
| 56 | + run_id: str | |
| 57 | + db_path: Path | |
| 58 | + snapshot_path: Path | None | |
| 59 | + counts: dict[str, int] = field(default_factory=dict) | |
| 60 | + duration_s: float = 0.0 | |
| 61 | + warnings: list[str] = field(default_factory=list) | |
| 62 | + | |
| 63 | + | |
| 64 | +def _timer(log_label: str, t0: list[float]) -> None: | |
| 65 | + now = time.monotonic() | |
| 66 | + log.info(" %-28s %6.1fs", log_label, now - t0[0]) | |
| 67 | + t0[0] = now | |
| 68 | + | |
| 69 | + | |
| 70 | +# ------------------------------------------------------------------------------------------------ registry tables | |
| 71 | +def _load_registry_tables(con: duckdb.DuckDBPyConnection, source_meta: dict[str, dict[str, Any]], | |
| 72 | + spec_meta: dict[tuple[str, str, str, str], dict[str, Any]]) -> None: | |
| 73 | + countries = registry.countries() | |
| 74 | + con.register("df_countries", pl.DataFrame([c.as_row() for c in countries])) | |
| 75 | + con.execute( | |
| 76 | + """INSERT INTO countries SELECT id, iso2, iso3, iso_numeric, slug, short_name, official_name, capital, continent, | |
| 77 | + region_wb, region_wb_name, subregion, income_group, income_group_name, currency_code, currency_name, area_km2, | |
| 78 | + latitude, longitude, flag_emoji, un_member, independent, landlocked, borders, languages, demonym, status, kind | |
| 79 | + FROM df_countries""" | |
| 80 | + ) | |
| 81 | + groups = registry.groups() | |
| 82 | + con.register("df_groups", pl.DataFrame( | |
| 83 | + [{"id": g.id, "slug": g.slug, "name": g.name, "kind": g.kind, "description": g.description, "wb_code": g.wb_code, | |
| 84 | + "n_members": len(g.members)} for g in groups], | |
| 85 | + schema={"id": pl.Utf8, "slug": pl.Utf8, "name": pl.Utf8, "kind": pl.Utf8, "description": pl.Utf8, "wb_code": pl.Utf8, | |
| 86 | + "n_members": pl.Int32})) | |
| 87 | + con.execute("INSERT INTO groups SELECT * FROM df_groups") | |
| 88 | + con.register("df_gm", pl.DataFrame({"group_id": [g.id for g in groups for _ in g.members], | |
| 89 | + "country_id": [m for g in groups for m in g.members]}, | |
| 90 | + schema={"group_id": pl.Utf8, "country_id": pl.Utf8})) | |
| 91 | + con.execute("INSERT INTO group_members SELECT * FROM df_gm") | |
| 92 | + | |
| 93 | + src_rows = [] | |
| 94 | + for sid in registry.CONNECTORS: | |
| 95 | + m = {**SOURCE_DEFAULTS.get(sid, {}), **source_meta.get(sid, {})} | |
| 96 | + src_rows.append({"id": sid, "name": m.get("name") or sid, "organization": m.get("organization"), "url": m.get("url"), | |
| 97 | + "licence": m.get("licence"), "attribution": m.get("attribution"), "api_base": m.get("api_base"), | |
| 98 | + "notes": m.get("notes")}) | |
| 99 | + con.register("df_sources", pl.DataFrame(src_rows, schema={k: pl.Utf8 for k in src_rows[0]})) | |
| 100 | + con.execute( | |
| 101 | + """INSERT INTO sources (id, name, organization, url, licence, attribution, api_base, notes) | |
| 102 | + SELECT id, name, organization, url, licence, attribution, api_base, notes FROM df_sources""" | |
| 103 | + ) | |
| 104 | + | |
| 105 | + inds = registry.indicators() | |
| 106 | + ind_rows = [] | |
| 107 | + for i in inds: | |
| 108 | + lo, hi = (i.bounds or [None, None])[:2] | |
| 109 | + ind_rows.append({ | |
| 110 | + "id": i.id, "slug": i.slug, "name": i.name, "short_name": i.short_name, "description": i.description, | |
| 111 | + "topic": i.topic, "subtopic": i.subtopic, "unit": i.unit, "unit_short": i.unit_short, "frequency": i.frequency, | |
| 112 | + "precision": int(i.precision), "aggregation": i.aggregation, "higher_is_better": i.higher_is_better, | |
| 113 | + "ranking_eligible": bool(i.ranking_eligible), "featured": bool(i.featured), "format": i.format, "scale": i.scale, | |
| 114 | + "bounds_min": None if lo is None else float(lo), "bounds_max": None if hi is None else float(hi), | |
| 115 | + "methodology": i.methodology, "tags": list(i.tags or []), "per_capita_of": i.per_capita_of, | |
| 116 | + }) | |
| 117 | + con.register("df_ind", pl.DataFrame(ind_rows, schema={ | |
| 118 | + "id": pl.Utf8, "slug": pl.Utf8, "name": pl.Utf8, "short_name": pl.Utf8, "description": pl.Utf8, "topic": pl.Utf8, | |
| 119 | + "subtopic": pl.Utf8, "unit": pl.Utf8, "unit_short": pl.Utf8, "frequency": pl.Utf8, "precision": pl.Int32, | |
| 120 | + "aggregation": pl.Utf8, "higher_is_better": pl.Boolean, "ranking_eligible": pl.Boolean, "featured": pl.Boolean, | |
| 121 | + "format": pl.Utf8, "scale": pl.Utf8, "bounds_min": pl.Float64, "bounds_max": pl.Float64, "methodology": pl.Utf8, | |
| 122 | + "tags": pl.List(pl.Utf8), "per_capita_of": pl.Utf8})) | |
| 123 | + con.execute( | |
| 124 | + """INSERT INTO indicators (id, slug, name, short_name, description, topic, subtopic, unit, unit_short, frequency, | |
| 125 | + precision, aggregation, higher_is_better, ranking_eligible, featured, format, scale, bounds_min, bounds_max, | |
| 126 | + methodology, tags, per_capita_of) SELECT * FROM df_ind""" | |
| 127 | + ) | |
| 128 | + | |
| 129 | + is_rows = [] | |
| 130 | + for i in inds: | |
| 131 | + for s in i.sources: | |
| 132 | + m = spec_meta.get((s.connector, s.dataset, s.code, s.indicator_id), {}) | |
| 133 | + is_rows.append({ | |
| 134 | + "indicator_id": s.indicator_id, "source_id": s.connector, "dataset": s.dataset, "series_code": s.code, | |
| 135 | + "params": json.dumps(s.params or {}), "priority": int(s.priority), "transform": s.transform, | |
| 136 | + "countries": list(s.countries) if s.countries else None, "notes": m.get("notes") or s.notes, | |
| 137 | + "source_url": m.get("source_url"), "last_run_id": m.get("run_id"), "last_status": m.get("status"), | |
| 138 | + }) | |
| 139 | + con.register("df_is", pl.DataFrame(is_rows, schema={ | |
| 140 | + "indicator_id": pl.Utf8, "source_id": pl.Utf8, "dataset": pl.Utf8, "series_code": pl.Utf8, "params": pl.Utf8, | |
| 141 | + "priority": pl.Int32, "transform": pl.Utf8, "countries": pl.List(pl.Utf8), "notes": pl.Utf8, "source_url": pl.Utf8, | |
| 142 | + "last_run_id": pl.Utf8, "last_status": pl.Utf8})) | |
| 143 | + con.execute( | |
| 144 | + """INSERT INTO indicator_sources (indicator_id, source_id, dataset, series_code, params, priority, transform, | |
| 145 | + countries, notes, source_url, last_run_id, last_status) | |
| 146 | + SELECT indicator_id, source_id, dataset, series_code, params::JSON, priority, transform, countries, notes, | |
| 147 | + source_url, last_run_id, last_status FROM df_is""" | |
| 148 | + ) | |
| 149 | + for v in ("df_countries", "df_groups", "df_gm", "df_sources", "df_ind", "df_is"): | |
| 150 | + con.unregister(v) | |
| 151 | + | |
| 152 | + | |
| 153 | +def _connector_meta() -> dict[str, dict[str, Any]]: | |
| 154 | + out: dict[str, dict[str, Any]] = {} | |
| 155 | + for cid, cls in discover().items(): | |
| 156 | + out[cid] = {"name": cls.name, "organization": cls.organization, "url": cls.url, "licence": cls.licence, | |
| 157 | + "attribution": cls.attribution, "api_base": cls.api_base} | |
| 158 | + return out | |
| 159 | + | |
| 160 | + | |
| 161 | +def _spec_meta() -> dict[tuple[str, str, str, str], dict[str, Any]]: | |
| 162 | + """Sidecar meta/run info per spec key (connector, dataset, code, indicator).""" | |
| 163 | + out: dict[tuple[str, str, str, str], dict[str, Any]] = {} | |
| 164 | + for s in registry.source_specs(): | |
| 165 | + from countryatlas.pipeline.staging import spec_paths | |
| 166 | + | |
| 167 | + p = spec_paths(s) | |
| 168 | + m = read_json(p["meta"]) or {} | |
| 169 | + r = read_json(p["run"]) or {} | |
| 170 | + if m or r: | |
| 171 | + out[(s.connector, s.dataset, s.code, s.indicator_id)] = {**m, "status": r.get("status"), "run_id": r.get("run_id")} | |
| 172 | + return out | |
| 173 | + | |
| 174 | + | |
| 175 | +# ------------------------------------------------------------------------------------------------ observations | |
| 176 | +def _load_staging(con: duckdb.DuckDBPyConnection, files: list[Path]) -> int: | |
| 177 | + if not files: | |
| 178 | + con.execute("CREATE TEMP TABLE staging_all AS SELECT * FROM observations LIMIT 0") | |
| 179 | + con.execute("ALTER TABLE staging_all ADD COLUMN priority INTEGER") | |
| 180 | + return 0 | |
| 181 | + paths = [str(p) for p in files] | |
| 182 | + con.execute( | |
| 183 | + """ | |
| 184 | + CREATE TEMP TABLE staging_all AS | |
| 185 | + SELECT s.country_id, s.indicator_id, s.period, s.year, s.frequency, s.value, s.unit, s.source_id, s.source_dataset, | |
| 186 | + s.source_series_code, s.is_estimate, s.is_forecast, 1 AS revision, s.retrieved_at, s.source_updated_at, | |
| 187 | + s.status, s.metadata, | |
| 188 | + coalesce(isrc.priority, 99) AS priority | |
| 189 | + FROM read_parquet($paths, union_by_name = true) s | |
| 190 | + LEFT JOIN indicator_sources isrc | |
| 191 | + ON isrc.indicator_id = s.indicator_id AND isrc.source_id = s.source_id | |
| 192 | + AND isrc.dataset = s.source_dataset AND isrc.series_code = s.source_series_code | |
| 193 | + WHERE s.value IS NOT NULL | |
| 194 | + """, | |
| 195 | + {"paths": paths}, | |
| 196 | + ) | |
| 197 | + return con.execute("SELECT count(*) FROM staging_all").fetchone()[0] | |
| 198 | + | |
| 199 | + | |
| 200 | +def _merge_observations(con: duckdb.DuckDBPyConnection) -> tuple[int, int]: | |
| 201 | + cols = ("country_id, indicator_id, period, year, frequency, value, unit, source_id, source_dataset, source_series_code, " | |
| 202 | + "is_estimate, is_forecast, revision, retrieved_at, source_updated_at, status, metadata") | |
| 203 | + con.execute( | |
| 204 | + f""" | |
| 205 | + CREATE TEMP TABLE ranked AS | |
| 206 | + SELECT *, row_number() OVER (PARTITION BY country_id, indicator_id, period, frequency | |
| 207 | + ORDER BY priority, source_id, source_dataset, source_series_code) AS rn | |
| 208 | + FROM staging_all; | |
| 209 | + INSERT INTO observations SELECT {cols.replace('metadata', 'metadata::JSON')} FROM ranked WHERE rn = 1; | |
| 210 | + INSERT INTO observations_alt SELECT {cols.replace('metadata', 'metadata::JSON')} FROM ranked WHERE rn > 1; | |
| 211 | + DROP TABLE ranked; | |
| 212 | + """ | |
| 213 | + ) | |
| 214 | + n = con.execute("SELECT count(*) FROM observations").fetchone()[0] | |
| 215 | + n_alt = con.execute("SELECT count(*) FROM observations_alt").fetchone()[0] | |
| 216 | + return n, n_alt | |
| 217 | + | |
| 218 | + | |
| 219 | +def _carry_revisions(con: duckdb.DuckDBPyConnection, previous_db: Path, run_id: str) -> tuple[int, int]: | |
| 220 | + """Compare with the previous live snapshot; record changed values and copy the old revisions table.""" | |
| 221 | + if not previous_db.exists(): | |
| 222 | + return 0, 0 | |
| 223 | + try: | |
| 224 | + con.execute(f"ATTACH '{previous_db}' AS prev (READ_ONLY)") | |
| 225 | + except Exception as e: # noqa: BLE001 | |
| 226 | + log.warning("previous snapshot could not be attached (%s) — revisions not carried forward", e) | |
| 227 | + return 0, 0 | |
| 228 | + try: | |
| 229 | + tables = {r[0] for r in con.execute("SELECT table_name FROM information_schema.tables WHERE table_catalog = 'prev'").fetchall()} | |
| 230 | + n_new = n_old = 0 | |
| 231 | + if "observations" in tables: | |
| 232 | + con.execute( | |
| 233 | + """ | |
| 234 | + INSERT INTO observation_revisions | |
| 235 | + SELECT n.country_id, n.indicator_id, n.period, n.frequency, p.value, n.value, p.source_id, n.source_id, | |
| 236 | + now()::TIMESTAMP, $run_id | |
| 237 | + FROM observations n | |
| 238 | + JOIN prev.observations p | |
| 239 | + ON p.country_id = n.country_id AND p.indicator_id = n.indicator_id AND p.period = n.period | |
| 240 | + AND p.frequency = n.frequency | |
| 241 | + WHERE abs(coalesce(p.value, 0) - coalesce(n.value, 0)) > 1e-9 * greatest(1, abs(coalesce(p.value, 0))) | |
| 242 | + OR p.source_id IS DISTINCT FROM n.source_id | |
| 243 | + """, | |
| 244 | + {"run_id": run_id}, | |
| 245 | + ) | |
| 246 | + n_new = con.execute("SELECT count(*) FROM observation_revisions").fetchone()[0] | |
| 247 | + if "observation_revisions" in tables: | |
| 248 | + con.execute("INSERT INTO observation_revisions SELECT * FROM prev.observation_revisions") | |
| 249 | + n_old = con.execute("SELECT count(*) FROM observation_revisions").fetchone()[0] - n_new | |
| 250 | + return n_new, n_old | |
| 251 | + finally: | |
| 252 | + con.execute("DETACH prev") | |
| 253 | + | |
| 254 | + | |
| 255 | +def _load_runs_and_issues(con: duckdb.DuckDBPyConnection) -> tuple[int, int]: | |
| 256 | + runs = list_runs() | |
| 257 | + if runs: | |
| 258 | + df = pl.DataFrame([r.model_dump() for r in runs]).with_columns( | |
| 259 | + pl.col("started_at").dt.replace_time_zone(None), pl.col("finished_at").dt.replace_time_zone(None) | |
| 260 | + ) | |
| 261 | + con.register("df_runs", df) | |
| 262 | + con.execute( | |
| 263 | + """INSERT INTO import_runs SELECT run_id, connector, dataset, started_at, finished_at, status, rows_raw, rows_norm, | |
| 264 | + rows_valid, warnings, errors, message, raw_path FROM df_runs""" | |
| 265 | + ) | |
| 266 | + con.unregister("df_runs") | |
| 267 | + issues: list[dict[str, Any]] = [] | |
| 268 | + for p in settings.staging_dir.glob("*/*.issues.json") if settings.staging_dir.exists() else []: | |
| 269 | + doc = read_json(p) or {} | |
| 270 | + connector = p.parent.name | |
| 271 | + run = read_json(p.with_name(p.name.replace(".issues.json", ".run.json"))) or {} | |
| 272 | + for i in doc.get("issues", []): | |
| 273 | + issues.append({"run_id": run.get("run_id"), "connector": connector, "indicator_id": i.get("indicator_id"), | |
| 274 | + "country_id": i.get("country_id"), "period": i.get("period"), "severity": i.get("severity"), | |
| 275 | + "code": i.get("code"), "message": i.get("message")}) | |
| 276 | + if issues: | |
| 277 | + df = pl.DataFrame(issues, schema={"run_id": pl.Utf8, "connector": pl.Utf8, "indicator_id": pl.Utf8, "country_id": pl.Utf8, | |
| 278 | + "period": pl.Utf8, "severity": pl.Utf8, "code": pl.Utf8, "message": pl.Utf8}) | |
| 279 | + df = df.with_columns(pl.col("period").str.to_date(strict=False)) | |
| 280 | + con.register("df_issues", df) | |
| 281 | + con.execute("INSERT INTO validation_issues SELECT * FROM df_issues") | |
| 282 | + con.unregister("df_issues") | |
| 283 | + return len(runs), len(issues) | |
| 284 | + | |
| 285 | + | |
| 286 | +# ------------------------------------------------------------------------------------------------ integrity + swap | |
| 287 | +def _integrity(con: duckdb.DuckDBPyConnection, files: list[Path], strict: bool) -> list[str]: | |
| 288 | + problems: list[str] = [] | |
| 289 | + n_obs = con.execute("SELECT count(*) FROM observations").fetchone()[0] | |
| 290 | + has_wb = any(p.parent.name == "worldbank" for p in files) | |
| 291 | + if has_wb and n_obs < MIN_OBSERVATIONS_WITH_WB: | |
| 292 | + problems.append(f"only {n_obs} observations although World Bank staging is present (< {MIN_OBSERVATIONS_WITH_WB})") | |
| 293 | + staged_indicators = {r[0] for r in con.execute("SELECT DISTINCT indicator_id FROM staging_all").fetchall()} | |
| 294 | + cov = dict(con.execute("SELECT indicator_id, count(*) FROM latest GROUP BY indicator_id").fetchall()) | |
| 295 | + for slug in registry.topics()["headline"]: | |
| 296 | + if slug not in staged_indicators: | |
| 297 | + continue # no connector has produced this indicator yet — cannot judge | |
| 298 | + n = cov.get(slug, 0) | |
| 299 | + if n < HEADLINE_MIN_COUNTRIES: | |
| 300 | + problems.append(f"headline indicator {slug} has only {n} countries in latest (< {HEADLINE_MIN_COUNTRIES})") | |
| 301 | + if problems and strict: | |
| 302 | + raise IntegrityError("; ".join(problems)) | |
| 303 | + return problems | |
| 304 | + | |
| 305 | + | |
| 306 | +def _swap(build_path: Path, run_id: str) -> Path: | |
| 307 | + settings.ensure_dirs() | |
| 308 | + os.replace(build_path, settings.db_path) | |
| 309 | + wal = build_path.with_suffix(".duckdb.wal") | |
| 310 | + if wal.exists(): | |
| 311 | + wal.unlink() | |
| 312 | + snap = settings.snapshots_dir / f"atlas-{run_id}.duckdb" | |
| 313 | + shutil.copy2(settings.db_path, snap) | |
| 314 | + snaps = sorted(settings.snapshots_dir.glob("atlas-*.duckdb")) | |
| 315 | + for old in snaps[: max(0, len(snaps) - settings.keep_snapshots)]: | |
| 316 | + old.unlink(missing_ok=True) | |
| 317 | + return snap | |
| 318 | + | |
| 319 | + | |
| 320 | +# ------------------------------------------------------------------------------------------------ main entry point | |
| 321 | +def build(run_id: str | None = None, strict: bool = True, swap: bool = True) -> BuildResult: | |
| 322 | + run_id = run_id or new_run_id() | |
| 323 | + settings.ensure_dirs() | |
| 324 | + t_start = time.monotonic() | |
| 325 | + t0 = [t_start] | |
| 326 | + build_path = settings.build_dir / f"atlas-{run_id}.duckdb" | |
| 327 | + for stale in settings.build_dir.glob("atlas-*.duckdb*"): | |
| 328 | + stale.unlink(missing_ok=True) | |
| 329 | + files = list_staging_files() | |
| 330 | + log.info("build %s: %d staging files → %s", run_id, len(files), build_path) | |
| 331 | + counts: dict[str, int] = {} | |
| 332 | + warnings: list[str] = [] | |
| 333 | + con = duckdb.connect(str(build_path)) | |
| 334 | + try: | |
| 335 | + con.execute(SCHEMA_SQL.read_text()) | |
| 336 | + _load_registry_tables(con, _connector_meta(), _spec_meta()) | |
| 337 | + _timer("registry tables", t0) | |
| 338 | + | |
| 339 | + counts["staging_rows"] = _load_staging(con, files) | |
| 340 | + counts["observations"], counts["observations_alt"] = _merge_observations(con) | |
| 341 | + _timer("merge observations", t0) | |
| 342 | + | |
| 343 | + counts["revisions_new"], counts["revisions_carried"] = _carry_revisions(con, settings.db_path, run_id) | |
| 344 | + counts["import_runs"], counts["validation_issues"] = _load_runs_and_issues(con) | |
| 345 | + _timer("revisions + runs", t0) | |
| 346 | + | |
| 347 | + derived.create_base_views(con) | |
| 348 | + derived.build_ranks(con) | |
| 349 | + counts["latest"] = derived.build_latest(con) | |
| 350 | + counts["rankings"] = derived.build_rankings(con) | |
| 351 | + derived.build_coverage(con) | |
| 352 | + derived.update_indicator_coverage(con) | |
| 353 | + derived.update_sources_stats(con) | |
| 354 | + _timer("latest/rankings/coverage", t0) | |
| 355 | + | |
| 356 | + ind_by_id = registry.indicators_by_id() | |
| 357 | + series = con.execute( | |
| 358 | + "SELECT country_id, indicator_id, period, year, value FROM obs_ok ORDER BY country_id, indicator_id, period" | |
| 359 | + ).pl() | |
| 360 | + ch, ev = compute_changes_and_events(series, ind_by_id) | |
| 361 | + con.register("df_changes", ch) | |
| 362 | + con.register("df_events", ev) | |
| 363 | + con.execute("INSERT INTO changes SELECT id, country_id, indicator_id, kind, period, year, value, ref_value, delta, " | |
| 364 | + "delta_pct, window_years, severity, headline, detail::JSON, detected_at FROM df_changes") | |
| 365 | + con.execute("INSERT INTO events SELECT id, country_id, indicator_id, kind, period, year, value, ref_value, delta, " | |
| 366 | + "delta_pct, window_years, severity, headline, detail::JSON FROM df_events") | |
| 367 | + counts["changes"], counts["events"] = ch.height, ev.height | |
| 368 | + _timer("changes + events", t0) | |
| 369 | + | |
| 370 | + countries = registry.countries() | |
| 371 | + country_ids = [c.id for c in countries if c.kind == "country"] | |
| 372 | + latest_df = con.execute("SELECT country_id, indicator_id, year, value FROM latest").pl() | |
| 373 | + sim = compute_similarity(latest_df, country_ids) | |
| 374 | + con.register("df_sim", sim) | |
| 375 | + con.execute("INSERT INTO similarity SELECT country_id, mode, peer_id, score, rank, contributions::JSON FROM df_sim") | |
| 376 | + dna = compute_country_dna(latest_df, country_ids) | |
| 377 | + con.register("df_dna", dna) | |
| 378 | + con.execute("INSERT INTO country_dna SELECT country_id, dims::JSON, year_ref FROM df_dna") | |
| 379 | + counts["similarity"], counts["country_dna"] = sim.height, dna.height | |
| 380 | + _timer("similarity + dna", t0) | |
| 381 | + | |
| 382 | + annual = con.execute("SELECT country_id, indicator_id, year, value FROM obs_ok WHERE frequency = 'A'").pl() | |
| 383 | + ctx = InsightContext(annual, latest_df, registry.countries_by_id(), registry.groups_by_id(), ind_by_id) | |
| 384 | + ins = compute_insights(ctx) | |
| 385 | + con.register("df_ins", ins) | |
| 386 | + con.execute("INSERT INTO insights SELECT id, country_id, template_id, text, values::JSON, indicators, computed_at FROM df_ins") | |
| 387 | + counts["insights"] = ins.height | |
| 388 | + _timer("insights", t0) | |
| 389 | + | |
| 390 | + counts["search_index"] = derived.build_search_index(con, countries, registry.groups(), registry.indicators(), | |
| 391 | + _connector_meta()) | |
| 392 | + counts["indicators_with_data"] = con.execute("SELECT count(DISTINCT indicator_id) FROM observations").fetchone()[0] | |
| 393 | + counts["countries_with_data"] = con.execute("SELECT count(DISTINCT country_id) FROM observations").fetchone()[0] | |
| 394 | + warnings = _integrity(con, files, strict) | |
| 395 | + for w in warnings: | |
| 396 | + log.warning("integrity: %s", w) | |
| 397 | + derived.write_meta(con, { | |
| 398 | + "schema_version": SCHEMA_VERSION, "build_run_id": run_id, "observation_count": counts["observations"], | |
| 399 | + "observations_alt_count": counts["observations_alt"], "indicator_count": counts["indicators_with_data"], | |
| 400 | + "country_count": counts["countries_with_data"], "latest_count": counts["latest"], | |
| 401 | + "rankings_count": counts["rankings"], "changes_count": counts["changes"], "events_count": counts["events"], | |
| 402 | + "insights_count": counts["insights"], "similarity_count": counts["similarity"], | |
| 403 | + "staging_files": len(files), "connectors": ",".join(sorted({p.parent.name for p in files})), | |
| 404 | + "registry_dir": str(settings.registry_dir), "repo": str(ROOT), | |
| 405 | + "build_duration_s": f"{time.monotonic() - t_start:.1f}", | |
| 406 | + "integrity_warnings": " | ".join(warnings), | |
| 407 | + }) | |
| 408 | + con.execute("CHECKPOINT") | |
| 409 | + con.close() | |
| 410 | + _timer("meta + checkpoint", t0) | |
| 411 | + except Exception: | |
| 412 | + try: | |
| 413 | + con.close() | |
| 414 | + except Exception as e: # noqa: BLE001 | |
| 415 | + log.debug("closing failed build connection: %s", e) | |
| 416 | + build_path.unlink(missing_ok=True) | |
| 417 | + build_path.with_suffix(".duckdb.wal").unlink(missing_ok=True) | |
| 418 | + log.exception("build %s FAILED — live database untouched", run_id) | |
| 419 | + raise | |
| 420 | + | |
| 421 | + snap: Path | None = None | |
| 422 | + if swap: | |
| 423 | + snap = _swap(build_path, run_id) | |
| 424 | + log.info("build %s swapped into %s (snapshot %s)", run_id, settings.db_path, snap.name) | |
| 425 | + dur = time.monotonic() - t_start | |
| 426 | + log.info("build %s done in %.1fs: %s", run_id, dur, json.dumps(counts)) | |
| 427 | + return BuildResult(run_id=run_id, db_path=settings.db_path if swap else build_path, snapshot_path=snap, counts=counts, | |
| 428 | + duration_s=dur, warnings=warnings) | |
| 429 | + | |
| 430 | + | |
| 431 | +def refresh(connectors: list[str] | None = None, indicator: str | None = None, strict: bool = True) -> dict[str, Any]: | |
| 432 | + """fetch + normalize + validate + build, one run_id. Returns a summary dict (also used by the scheduler).""" | |
| 433 | + from countryatlas.pipeline.fetch import run_fetch | |
| 434 | + | |
| 435 | + run_id = new_run_id() | |
| 436 | + started = datetime.now(UTC) | |
| 437 | + summary = run_fetch(connectors=connectors, indicator=indicator, run_id=run_id) | |
| 438 | + result = build(run_id=run_id, strict=strict) | |
| 439 | + return { | |
| 440 | + "run_id": run_id, | |
| 441 | + "started_at": started.isoformat(), | |
| 442 | + "finished_at": datetime.now(UTC).isoformat(), | |
| 443 | + "fetch": {"ok": len(summary.ok), "failed": len(summary.failed), "skipped_connectors": summary.skipped_connectors, | |
| 444 | + "duration_s": round(summary.duration_s, 1), | |
| 445 | + "failed_specs": [{"connector": r.connector, "dataset": r.dataset, "message": r.message} for r in summary.failed]}, | |
| 446 | + "build": {"counts": result.counts, "duration_s": round(result.duration_s, 1), "warnings": result.warnings, | |
| 447 | + "snapshot": str(result.snapshot_path) if result.snapshot_path else None}, | |
| 448 | + } | |
added
src/countryatlas/pipeline/changes.py
+366 −0
@@ -0,0 +1,366 @@ | ||
| 1 | +"""Deterministic change / event detectors (ARCHITECTURE §7) with templated English headlines — no LLM anywhere. | |
| 2 | + | |
| 3 | +`changes` = detectors evaluated at the LATEST period of each country×indicator series | |
| 4 | +`events` = detectors evaluated over the WHOLE history (yoy jumps/drops, records after a ≥5-year gap, sign flips) | |
| 5 | + | |
| 6 | +Working scale: percent-like indicators are analysed on absolute differences ("points"); positive level series | |
| 7 | +(GDP, population, emissions…) on log-differences (reported as % change); everything else on absolute differences. | |
| 8 | +The robust dispersion is 1.4826·MAD of the differences; z = (Δ − median) / that. A change must also clear the | |
| 9 | +indicator's `change_floor` (registry) — default floor: 5 % relative change for level series, 2 % of the series range | |
| 10 | +otherwise. severity ∈ [0, 1] blends the z-score and the floor ratio. | |
| 11 | +""" | |
| 12 | +from __future__ import annotations | |
| 13 | + | |
| 14 | +import hashlib | |
| 15 | +import json | |
| 16 | +import logging | |
| 17 | +from dataclasses import dataclass | |
| 18 | +from datetime import UTC, date, datetime | |
| 19 | +from typing import Any | |
| 20 | + | |
| 21 | +import numpy as np | |
| 22 | +import polars as pl | |
| 23 | + | |
| 24 | +from countryatlas.pipeline.format import delta_mode, fmt_delta, fmt_value | |
| 25 | +from countryatlas.registry import Indicator | |
| 26 | + | |
| 27 | +log = logging.getLogger(__name__) | |
| 28 | + | |
| 29 | +MAD_SCALE = 1.4826 | |
| 30 | +Z_THRESHOLD = 2.0 | |
| 31 | +MIN_POINTS = 5 | |
| 32 | +RECORD_MIN_POINTS = 10 | |
| 33 | +N_YEAR_WINDOWS = (30, 20, 10) | |
| 34 | +DEFAULT_REL_FLOOR = 0.05 # 5 % for level series without change_floor | |
| 35 | +DEFAULT_RANGE_FLOOR = 0.02 # 2 % of the series range for other series without change_floor | |
| 36 | +MAX_EVENTS_PER_SERIES = 30 | |
| 37 | +MIN_SCALE_RATIO = 0.25 # lower bound of the robust scale, as a fraction of the floor (see _scale) | |
| 38 | +RECORD_GAP_YEARS = 5 | |
| 39 | +SIGN_FLIP_HINTS = ("growth", "balance", "net-migration", "inflation", "change") | |
| 40 | + | |
| 41 | +EVENT_COLUMNS = [ | |
| 42 | + "id", "country_id", "indicator_id", "kind", "period", "year", "value", "ref_value", "delta", "delta_pct", | |
| 43 | + "window_years", "severity", "headline", "detail", | |
| 44 | +] | |
| 45 | + | |
| 46 | + | |
| 47 | +@dataclass | |
| 48 | +class Series: | |
| 49 | + country_id: str | |
| 50 | + indicator_id: str | |
| 51 | + periods: list[date] | |
| 52 | + years: np.ndarray | |
| 53 | + values: np.ndarray | |
| 54 | + | |
| 55 | + | |
| 56 | +def _id(country: str, indicator: str, kind: str, period: date, scope: str) -> str: | |
| 57 | + return hashlib.sha1(f"{scope}|{country}|{indicator}|{kind}|{period.isoformat()}".encode()).hexdigest()[:16] | |
| 58 | + | |
| 59 | + | |
| 60 | +def _working(values: np.ndarray, mode: str) -> np.ndarray: | |
| 61 | + if mode == "relative": | |
| 62 | + with np.errstate(divide="ignore", invalid="ignore"): | |
| 63 | + return np.where(values > 0, np.log(values), np.nan) | |
| 64 | + return values | |
| 65 | + | |
| 66 | + | |
| 67 | +def _floor(ind: Indicator, values: np.ndarray, mode: str) -> tuple[float, bool]: | |
| 68 | + """(floor, absolute): the floor a move must clear; `absolute` = compare the raw Δ (registry change_floor is | |
| 69 | + always expressed in the indicator's own unit), else compare the working-scale Δ (log-diff or absolute).""" | |
| 70 | + if ind.change_floor is not None: | |
| 71 | + return float(ind.change_floor), True | |
| 72 | + if mode == "relative": | |
| 73 | + return DEFAULT_REL_FLOOR, False | |
| 74 | + rng = float(np.nanmax(values) - np.nanmin(values)) if len(values) else 0.0 | |
| 75 | + return (DEFAULT_RANGE_FLOOR * rng if rng > 0 else 0.0), False | |
| 76 | + | |
| 77 | + | |
| 78 | +def _robust(d: np.ndarray) -> tuple[float, float]: | |
| 79 | + """median and 1.4826·MAD of finite diffs (nan-safe).""" | |
| 80 | + dd = d[np.isfinite(d)] | |
| 81 | + if len(dd) < 2: | |
| 82 | + return 0.0, 0.0 | |
| 83 | + med = float(np.median(dd)) | |
| 84 | + mad = float(np.median(np.abs(dd - med))) * MAD_SCALE | |
| 85 | + return med, mad | |
| 86 | + | |
| 87 | + | |
| 88 | +def _scale(mad: float, floor: float, floor_abs: bool, mode: str, ref: float) -> float: | |
| 89 | + """Robust dispersion with a lower bound of MIN_SCALE_RATIO × floor (in working units), so that perfectly smooth | |
| 90 | + series (MAD ≈ 0, e.g. constant growth) still produce finite z-scores instead of being skipped.""" | |
| 91 | + if floor_abs and mode == "relative": | |
| 92 | + floor_w = floor / abs(ref) if ref else 0.0 | |
| 93 | + else: | |
| 94 | + floor_w = floor | |
| 95 | + return max(mad, MIN_SCALE_RATIO * floor_w) | |
| 96 | + | |
| 97 | + | |
| 98 | +def _sign_flip_applicable(ind: Indicator) -> bool: | |
| 99 | + lo = (ind.bounds or [None, None])[0] | |
| 100 | + if lo is not None and float(lo) >= 0: | |
| 101 | + return False | |
| 102 | + return any(h in ind.slug for h in SIGN_FLIP_HINTS) | |
| 103 | + | |
| 104 | + | |
| 105 | +def _pct(v: float, ref: float) -> float | None: | |
| 106 | + if ref is None or ref == 0 or not np.isfinite(ref): | |
| 107 | + return None | |
| 108 | + return float((v - ref) / abs(ref) * 100.0) | |
| 109 | + | |
| 110 | + | |
| 111 | +def _row( | |
| 112 | + s: Series, ind: Indicator, kind: str, i: int, ref_value: float | None, severity: float, headline: str, | |
| 113 | + detail: dict[str, Any], window: int | None, scope: str, | |
| 114 | +) -> dict[str, Any]: | |
| 115 | + v = float(s.values[i]) | |
| 116 | + delta = None if ref_value is None else float(v - ref_value) | |
| 117 | + return { | |
| 118 | + "id": _id(s.country_id, s.indicator_id, kind, s.periods[i], scope), | |
| 119 | + "country_id": s.country_id, | |
| 120 | + "indicator_id": s.indicator_id, | |
| 121 | + "kind": kind, | |
| 122 | + "period": s.periods[i], | |
| 123 | + "year": int(s.years[i]), | |
| 124 | + "value": v, | |
| 125 | + "ref_value": None if ref_value is None else float(ref_value), | |
| 126 | + "delta": delta, | |
| 127 | + "delta_pct": None if ref_value is None else _pct(v, ref_value), | |
| 128 | + "window_years": window, | |
| 129 | + "severity": float(min(1.0, max(0.0, severity))), | |
| 130 | + "headline": headline, | |
| 131 | + "detail": json.dumps(detail, default=str), | |
| 132 | + } | |
| 133 | + | |
| 134 | + | |
| 135 | +# ------------------------------------------------------------------------------------------------ changes (latest) | |
| 136 | +def detect_changes(s: Series, ind: Indicator) -> list[dict[str, Any]]: | |
| 137 | + n = len(s.values) | |
| 138 | + if n < 2: | |
| 139 | + return [] | |
| 140 | + name = ind.display_name | |
| 141 | + mode = delta_mode(ind) | |
| 142 | + v = s.values | |
| 143 | + w = _working(v, mode) | |
| 144 | + d = np.diff(w) | |
| 145 | + out: list[dict[str, Any]] = [] | |
| 146 | + i = n - 1 | |
| 147 | + year = int(s.years[i]) | |
| 148 | + prev = float(v[i - 1]) | |
| 149 | + cur = float(v[i]) | |
| 150 | + dl = float(d[-1]) if np.isfinite(d[-1]) else None | |
| 151 | + med, mad = _robust(d[:-1]) if n > MIN_POINTS else (0.0, 0.0) | |
| 152 | + floor, floor_abs = _floor(ind, v, mode) | |
| 153 | + scale = _scale(mad, floor, floor_abs, mode, prev) | |
| 154 | + since_txt = "" | |
| 155 | + | |
| 156 | + # --- YoY jump / drop ----------------------------------------------------------------------------------------- | |
| 157 | + if dl is not None and n >= MIN_POINTS and scale > 0: | |
| 158 | + z = (dl - med) / scale | |
| 159 | + magnitude = abs(cur - prev) if floor_abs else abs(dl) | |
| 160 | + if abs(z) > Z_THRESHOLD and magnitude >= floor > 0: | |
| 161 | + kind = "yoy_jump" if dl > 0 else "yoy_drop" | |
| 162 | + verb = "rose" if dl > 0 else "fell" | |
| 163 | + # largest since: last earlier year whose move was at least as large in the same direction | |
| 164 | + earlier = d[:-1] | |
| 165 | + if dl > 0: | |
| 166 | + idx = np.where(earlier >= dl)[0] | |
| 167 | + else: | |
| 168 | + idx = np.where(earlier <= dl)[0] | |
| 169 | + if len(idx): | |
| 170 | + since_year = int(s.years[idx[-1] + 1]) | |
| 171 | + since_txt = f"largest {'rise' if dl > 0 else 'drop'} since {since_year}" | |
| 172 | + else: | |
| 173 | + since_txt = f"largest {'rise' if dl > 0 else 'drop'} on record" | |
| 174 | + sev = 0.6 * min(1.0, abs(z) / 4.0) + 0.4 * min(1.0, magnitude / (2 * floor)) | |
| 175 | + delta_txt = fmt_delta(cur - prev, _pct(cur, prev), ind) | |
| 176 | + headline = f"{name} {verb} {delta_txt} to {fmt_value(cur, ind)} in {year} ({since_txt})." | |
| 177 | + out.append( | |
| 178 | + _row(s, ind, kind, i, prev, sev, headline, | |
| 179 | + {"z": round(float(z), 2), "mad": mad, "scale": scale, "floor": floor, "mode": mode, "since": since_txt, | |
| 180 | + "n_points": n, "prev_year": int(s.years[i - 1])}, 1, "changes") | |
| 181 | + ) | |
| 182 | + | |
| 183 | + # --- record high / low, N-year high / low ----------------------------------------------------------------- | |
| 184 | + if n >= RECORD_MIN_POINTS: | |
| 185 | + past = v[:-1] | |
| 186 | + pmax, pmin = float(np.nanmax(past)), float(np.nanmin(past)) | |
| 187 | + first_year = int(s.years[0]) | |
| 188 | + if cur > pmax: | |
| 189 | + sev = 0.6 + min(0.4, n / 150.0) | |
| 190 | + out.append( | |
| 191 | + _row(s, ind, "record_high", i, pmax, sev, | |
| 192 | + f"{name} reached a record high of {fmt_value(cur, ind)} in {year} (series since {first_year}).", | |
| 193 | + {"previous_max": pmax, "n_points": n, "first_year": first_year}, n, "changes") | |
| 194 | + ) | |
| 195 | + elif cur < pmin: | |
| 196 | + sev = 0.6 + min(0.4, n / 150.0) | |
| 197 | + out.append( | |
| 198 | + _row(s, ind, "record_low", i, pmin, sev, | |
| 199 | + f"{name} fell to a record low of {fmt_value(cur, ind)} in {year} (series since {first_year}).", | |
| 200 | + {"previous_min": pmin, "n_points": n, "first_year": first_year}, n, "changes") | |
| 201 | + ) | |
| 202 | + else: | |
| 203 | + for N in N_YEAR_WINDOWS: | |
| 204 | + mask = (s.years[:-1] > year - N) & (s.years[:-1] <= year - 1) | |
| 205 | + if mask.sum() < max(5, int(N * 0.6)) or s.years[0] > year - N: | |
| 206 | + continue | |
| 207 | + win = past[mask] | |
| 208 | + if cur > float(np.nanmax(win)): | |
| 209 | + out.append( | |
| 210 | + _row(s, ind, "n_year_high", i, float(np.nanmax(win)), 0.3 + N / 100.0, | |
| 211 | + f"{name} hit a {N}-year high of {fmt_value(cur, ind)} in {year}.", | |
| 212 | + {"window": N, "window_max": float(np.nanmax(win))}, N, "changes") | |
| 213 | + ) | |
| 214 | + break | |
| 215 | + if cur < float(np.nanmin(win)): | |
| 216 | + out.append( | |
| 217 | + _row(s, ind, "n_year_low", i, float(np.nanmin(win)), 0.3 + N / 100.0, | |
| 218 | + f"{name} hit a {N}-year low of {fmt_value(cur, ind)} in {year}.", | |
| 219 | + {"window": N, "window_min": float(np.nanmin(win))}, N, "changes") | |
| 220 | + ) | |
| 221 | + break | |
| 222 | + | |
| 223 | + # --- sign flip ----------------------------------------------------------------------------------------------- | |
| 224 | + if _sign_flip_applicable(ind) and prev * cur < 0: | |
| 225 | + direction = "negative" if cur < 0 else "positive" | |
| 226 | + out.append( | |
| 227 | + _row(s, ind, "sign_flip", i, prev, 0.7, | |
| 228 | + f"{name} turned {direction} in {year} ({fmt_value(cur, ind)}, after {fmt_value(prev, ind)} in " | |
| 229 | + f"{int(s.years[i - 1])}).", | |
| 230 | + {"direction": direction, "prev_year": int(s.years[i - 1])}, 1, "changes") | |
| 231 | + ) | |
| 232 | + | |
| 233 | + # --- acceleration / deceleration (3 consecutive increases/decreases of the difference) ------------------- | |
| 234 | + if n >= 5 and np.all(np.isfinite(d[-4:])): | |
| 235 | + dd = d[-4:] | |
| 236 | + last_mag = abs(cur - prev) if floor_abs else abs(d[-1]) | |
| 237 | + if dd[3] > dd[2] > dd[1] > dd[0] and last_mag >= floor > 0 and d[-1] > 0: | |
| 238 | + out.append( | |
| 239 | + _row(s, ind, "accelerating", i, prev, 0.4, | |
| 240 | + f"{name} has accelerated for three consecutive years, reaching {fmt_value(cur, ind)} in {year}.", | |
| 241 | + {"diffs": [float(x) for x in dd], "mode": mode}, 3, "changes") | |
| 242 | + ) | |
| 243 | + elif dd[3] < dd[2] < dd[1] < dd[0] and last_mag >= floor > 0 and d[-1] < 0: | |
| 244 | + out.append( | |
| 245 | + _row(s, ind, "decelerating", i, prev, 0.4, | |
| 246 | + f"{name} has fallen faster for three consecutive years, reaching {fmt_value(cur, ind)} in {year}.", | |
| 247 | + {"diffs": [float(x) for x in dd], "mode": mode}, 3, "changes") | |
| 248 | + ) | |
| 249 | + return out | |
| 250 | + | |
| 251 | + | |
| 252 | +# ------------------------------------------------------------------------------------------------ events (history) | |
| 253 | +def detect_events(s: Series, ind: Indicator) -> list[dict[str, Any]]: | |
| 254 | + n = len(s.values) | |
| 255 | + if n < MIN_POINTS: | |
| 256 | + return [] | |
| 257 | + name = ind.display_name | |
| 258 | + mode = delta_mode(ind) | |
| 259 | + v = s.values | |
| 260 | + w = _working(v, mode) | |
| 261 | + d = np.diff(w) | |
| 262 | + med, mad = _robust(d) | |
| 263 | + floor, floor_abs = _floor(ind, v, mode) | |
| 264 | + scale = _scale(mad, floor, floor_abs, mode, float(np.nanmedian(np.abs(v)))) | |
| 265 | + out: list[dict[str, Any]] = [] | |
| 266 | + | |
| 267 | + # YoY jumps/drops anywhere in the history | |
| 268 | + if scale > 0 and floor > 0: | |
| 269 | + z = (d - med) / scale | |
| 270 | + magnitude = np.abs(np.diff(v)) if floor_abs else np.abs(d) | |
| 271 | + hits = np.where((np.abs(z) > Z_THRESHOLD) & (magnitude >= floor) & np.isfinite(z))[0] | |
| 272 | + for j in hits: | |
| 273 | + i = int(j) + 1 | |
| 274 | + prev, cur = float(v[i - 1]), float(v[i]) | |
| 275 | + kind = "yoy_jump" if d[j] > 0 else "yoy_drop" | |
| 276 | + verb = "rose" if d[j] > 0 else "fell" | |
| 277 | + sev = 0.6 * min(1.0, abs(float(z[j])) / 4.0) + 0.4 * min(1.0, float(magnitude[j]) / (2 * floor)) | |
| 278 | + headline = f"{name} {verb} {fmt_delta(cur - prev, _pct(cur, prev), ind)} to {fmt_value(cur, ind)} in {int(s.years[i])}." | |
| 279 | + out.append(_row(s, ind, kind, i, prev, sev, headline, | |
| 280 | + {"z": round(float(z[j]), 2), "mode": mode, "prev_year": int(s.years[i - 1])}, 1, "events")) | |
| 281 | + | |
| 282 | + # Records reached after a gap of ≥ RECORD_GAP_YEARS years since the previous record (monotone series stay quiet) | |
| 283 | + if n >= RECORD_MIN_POINTS: | |
| 284 | + run_max = np.maximum.accumulate(v) | |
| 285 | + run_min = np.minimum.accumulate(v) | |
| 286 | + last_max_year = int(s.years[0]) | |
| 287 | + last_min_year = int(s.years[0]) | |
| 288 | + for i in range(1, n): | |
| 289 | + y = int(s.years[i]) | |
| 290 | + if v[i] > run_max[i - 1]: | |
| 291 | + if i >= RECORD_MIN_POINTS and y - last_max_year >= RECORD_GAP_YEARS: | |
| 292 | + out.append(_row(s, ind, "record_high", i, float(run_max[i - 1]), 0.6, | |
| 293 | + f"{name} set a new high of {fmt_value(float(v[i]), ind)} in {y}, the first since " | |
| 294 | + f"{last_max_year}.", {"previous_record_year": last_max_year}, y - last_max_year, | |
| 295 | + "events")) | |
| 296 | + last_max_year = y | |
| 297 | + if v[i] < run_min[i - 1]: | |
| 298 | + if i >= RECORD_MIN_POINTS and y - last_min_year >= RECORD_GAP_YEARS: | |
| 299 | + out.append(_row(s, ind, "record_low", i, float(run_min[i - 1]), 0.6, | |
| 300 | + f"{name} fell to a new low of {fmt_value(float(v[i]), ind)} in {y}, the first since " | |
| 301 | + f"{last_min_year}.", {"previous_record_year": last_min_year}, y - last_min_year, | |
| 302 | + "events")) | |
| 303 | + last_min_year = y | |
| 304 | + | |
| 305 | + # Sign flips | |
| 306 | + if _sign_flip_applicable(ind): | |
| 307 | + sg = np.sign(v) | |
| 308 | + flips = np.where(sg[1:] * sg[:-1] < 0)[0] | |
| 309 | + for j in flips: | |
| 310 | + i = int(j) + 1 | |
| 311 | + direction = "negative" if v[i] < 0 else "positive" | |
| 312 | + out.append(_row(s, ind, "sign_flip", i, float(v[i - 1]), 0.7, | |
| 313 | + f"{name} turned {direction} in {int(s.years[i])} ({fmt_value(float(v[i]), ind)}).", | |
| 314 | + {"direction": direction}, 1, "events")) | |
| 315 | + | |
| 316 | + if len(out) > MAX_EVENTS_PER_SERIES: | |
| 317 | + out.sort(key=lambda r: r["severity"], reverse=True) | |
| 318 | + out = out[:MAX_EVENTS_PER_SERIES] | |
| 319 | + return out | |
| 320 | + | |
| 321 | + | |
| 322 | +# ------------------------------------------------------------------------------------------------ driver | |
| 323 | +def iter_series(df: pl.DataFrame): | |
| 324 | + """Yield Series objects from a frame sorted by (country_id, indicator_id, period).""" | |
| 325 | + if df.is_empty(): | |
| 326 | + return | |
| 327 | + df = df.sort(["country_id", "indicator_id", "period"]) | |
| 328 | + countries = df.get_column("country_id").to_numpy() | |
| 329 | + indicators = df.get_column("indicator_id").to_numpy() | |
| 330 | + years = df.get_column("year").to_numpy().astype(np.int32) | |
| 331 | + values = df.get_column("value").to_numpy().astype(np.float64) | |
| 332 | + periods = df.get_column("period").to_list() | |
| 333 | + key = np.char.add(np.char.add(countries.astype(str), "|"), indicators.astype(str)) | |
| 334 | + change = np.flatnonzero(key[1:] != key[:-1]) + 1 | |
| 335 | + starts = np.concatenate(([0], change)) | |
| 336 | + ends = np.concatenate((change, [len(key)])) | |
| 337 | + for a, b in zip(starts.tolist(), ends.tolist(), strict=True): | |
| 338 | + yield Series(str(countries[a]), str(indicators[a]), periods[a:b], years[a:b], values[a:b]) | |
| 339 | + | |
| 340 | + | |
| 341 | +def compute_changes_and_events(df: pl.DataFrame, indicators: dict[str, Indicator]) -> tuple[pl.DataFrame, pl.DataFrame]: | |
| 342 | + """df: non-forecast, non-quarantined observations (country_id, indicator_id, period, year, value).""" | |
| 343 | + changes: list[dict[str, Any]] = [] | |
| 344 | + events: list[dict[str, Any]] = [] | |
| 345 | + n_series = 0 | |
| 346 | + for s in iter_series(df): | |
| 347 | + ind = indicators.get(s.indicator_id) | |
| 348 | + if ind is None: | |
| 349 | + continue | |
| 350 | + n_series += 1 | |
| 351 | + try: | |
| 352 | + changes.extend(detect_changes(s, ind)) | |
| 353 | + events.extend(detect_events(s, ind)) | |
| 354 | + except Exception: | |
| 355 | + log.exception("detector failed for %s/%s", s.country_id, s.indicator_id) | |
| 356 | + log.info("detectors: %d series → %d changes, %d events", n_series, len(changes), len(events)) | |
| 357 | + detected_at = datetime.now(UTC).replace(tzinfo=None) | |
| 358 | + schema = { | |
| 359 | + "id": pl.Utf8, "country_id": pl.Utf8, "indicator_id": pl.Utf8, "kind": pl.Utf8, "period": pl.Date, | |
| 360 | + "year": pl.Int32, "value": pl.Float64, "ref_value": pl.Float64, "delta": pl.Float64, "delta_pct": pl.Float64, | |
| 361 | + "window_years": pl.Int32, "severity": pl.Float64, "headline": pl.Utf8, "detail": pl.Utf8, | |
| 362 | + } | |
| 363 | + ch = pl.DataFrame(changes, schema=schema) if changes else pl.DataFrame(schema=schema) | |
| 364 | + ch = ch.with_columns(pl.lit(detected_at).cast(pl.Datetime("us")).alias("detected_at")) | |
| 365 | + ev = pl.DataFrame(events, schema=schema) if events else pl.DataFrame(schema=schema) | |
| 366 | + return ch, ev | |
added
src/countryatlas/pipeline/derived.py
+236 −0
@@ -0,0 +1,236 @@ | ||
| 1 | +"""SQL-based derived tables computed inside the new snapshot: latest, rankings, coverage, indicator coverage, | |
| 2 | +search_index, sources stats, meta. Everything here is set-based DuckDB SQL (no Python loops over observations). | |
| 3 | + | |
| 4 | +Ranking direction (documented contract): rank 1 = "best" when `higher_is_better` is set (true → highest value, | |
| 5 | +false → lowest value); when `higher_is_better` is NULL rank 1 = highest value ("highest"). Ranks are computed among | |
| 6 | +`kind='country'` rows only, per (indicator, year), using the last period of the year for Q/M series. | |
| 7 | +pct_rank ∈ [0, 1]: 1 − (rank − 1)/(n − 1) (1.0 = rank 1). | |
| 8 | +""" | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import logging | |
| 12 | +import math | |
| 13 | +from datetime import UTC, datetime | |
| 14 | +from typing import Any | |
| 15 | + | |
| 16 | +import duckdb | |
| 17 | +import polars as pl | |
| 18 | + | |
| 19 | +from countryatlas.registry import Country, Group, Indicator, lookup, topics | |
| 20 | + | |
| 21 | +log = logging.getLogger(__name__) | |
| 22 | + | |
| 23 | +MIN_COUNTRIES_FOR_RANKING = 20 | |
| 24 | + | |
| 25 | +_ORDER = ( | |
| 26 | + "ORDER BY CASE WHEN higher_is_better = false THEN value END ASC NULLS LAST, " | |
| 27 | + "CASE WHEN higher_is_better IS DISTINCT FROM false THEN value END DESC NULLS LAST" | |
| 28 | +) | |
| 29 | + | |
| 30 | + | |
| 31 | +def create_base_views(con: duckdb.DuckDBPyConnection) -> None: | |
| 32 | + """obs_ok = observations usable for derived tables (non-forecast, non-quarantined, finite value).""" | |
| 33 | + con.execute( | |
| 34 | + """ | |
| 35 | + CREATE OR REPLACE TEMP VIEW obs_ok AS | |
| 36 | + SELECT o.* | |
| 37 | + FROM observations o | |
| 38 | + WHERE NOT coalesce(o.is_forecast, false) | |
| 39 | + AND coalesce(o.status, 'imported') <> 'quarantined' | |
| 40 | + AND o.value IS NOT NULL AND isfinite(o.value) | |
| 41 | + """ | |
| 42 | + ) | |
| 43 | + | |
| 44 | + | |
| 45 | +def build_ranks(con: duckdb.DuckDBPyConnection) -> None: | |
| 46 | + con.execute( | |
| 47 | + f""" | |
| 48 | + CREATE OR REPLACE TEMP TABLE yearly AS | |
| 49 | + SELECT o.country_id, o.indicator_id, o.year, o.value, c.region_wb, c.income_group, i.higher_is_better | |
| 50 | + FROM ( | |
| 51 | + SELECT *, row_number() OVER (PARTITION BY country_id, indicator_id, year ORDER BY period DESC) AS rn | |
| 52 | + FROM obs_ok | |
| 53 | + ) o | |
| 54 | + JOIN countries c ON c.id = o.country_id | |
| 55 | + JOIN indicators i ON i.id = o.indicator_id | |
| 56 | + WHERE o.rn = 1 AND c.kind = 'country'; | |
| 57 | + | |
| 58 | + CREATE OR REPLACE TEMP TABLE ranks_all AS | |
| 59 | + SELECT country_id, indicator_id, year, value, | |
| 60 | + rank() OVER (PARTITION BY indicator_id, year {_ORDER}) AS rank_world, | |
| 61 | + count(*) OVER (PARTITION BY indicator_id, year) AS n_world, | |
| 62 | + CASE WHEN region_wb IS NULL THEN NULL ELSE | |
| 63 | + rank() OVER (PARTITION BY indicator_id, year, region_wb {_ORDER}) END AS rank_region, | |
| 64 | + CASE WHEN region_wb IS NULL THEN NULL ELSE | |
| 65 | + count(*) OVER (PARTITION BY indicator_id, year, region_wb) END AS n_region, | |
| 66 | + CASE WHEN income_group IS NULL THEN NULL ELSE | |
| 67 | + rank() OVER (PARTITION BY indicator_id, year, income_group {_ORDER}) END AS rank_income, | |
| 68 | + CASE WHEN income_group IS NULL THEN NULL ELSE | |
| 69 | + count(*) OVER (PARTITION BY indicator_id, year, income_group) END AS n_income | |
| 70 | + FROM yearly; | |
| 71 | + """ | |
| 72 | + ) | |
| 73 | + | |
| 74 | + | |
| 75 | +def build_latest(con: duckdb.DuckDBPyConnection) -> int: | |
| 76 | + con.execute( | |
| 77 | + """ | |
| 78 | + CREATE OR REPLACE TEMP TABLE lat AS | |
| 79 | + WITH ordered AS ( | |
| 80 | + SELECT *, | |
| 81 | + row_number() OVER (PARTITION BY country_id, indicator_id ORDER BY period DESC) AS rn, | |
| 82 | + lag(period) OVER (PARTITION BY country_id, indicator_id ORDER BY period) AS prev_period, | |
| 83 | + lag(value) OVER (PARTITION BY country_id, indicator_id ORDER BY period) AS prev_value | |
| 84 | + FROM obs_ok | |
| 85 | + ) | |
| 86 | + SELECT * FROM ordered WHERE rn = 1; | |
| 87 | + | |
| 88 | + INSERT INTO latest | |
| 89 | + SELECT l.country_id, l.indicator_id, l.period, l.year, l.frequency, l.value, | |
| 90 | + l.prev_period, l.prev_value, | |
| 91 | + l.value - l.prev_value AS change_abs, | |
| 92 | + CASE WHEN l.prev_value IS NULL OR l.prev_value = 0 THEN NULL | |
| 93 | + ELSE (l.value - l.prev_value) / abs(l.prev_value) * 100 END AS change_pct, | |
| 94 | + r.rank_world, r.n_world, r.rank_region, r.n_region, r.rank_income, r.n_income, | |
| 95 | + l.year AS rank_year, l.source_id, l.is_forecast, l.is_estimate, l.status, | |
| 96 | + t.value AS value_10y_ago, | |
| 97 | + l.value - t.value AS change_10y_abs, | |
| 98 | + CASE WHEN t.value IS NULL OR t.value = 0 THEN NULL ELSE (l.value - t.value) / abs(t.value) * 100 END | |
| 99 | + FROM lat l | |
| 100 | + LEFT JOIN ranks_all r ON r.country_id = l.country_id AND r.indicator_id = l.indicator_id AND r.year = l.year | |
| 101 | + LEFT JOIN obs_ok t ON t.country_id = l.country_id AND t.indicator_id = l.indicator_id | |
| 102 | + AND t.period = CAST(l.period - INTERVAL 10 YEAR AS DATE) AND t.frequency = l.frequency; | |
| 103 | + """ | |
| 104 | + ) | |
| 105 | + return con.execute("SELECT count(*) FROM latest").fetchone()[0] | |
| 106 | + | |
| 107 | + | |
| 108 | +def build_rankings(con: duckdb.DuckDBPyConnection) -> int: | |
| 109 | + con.execute( | |
| 110 | + f""" | |
| 111 | + INSERT INTO rankings | |
| 112 | + SELECT r.indicator_id, r.year, r.country_id, r.value, r.rank_world, r.n_world, | |
| 113 | + CASE WHEN r.n_world > 1 THEN 1.0 - (r.rank_world - 1.0) / (r.n_world - 1.0) ELSE 1.0 END | |
| 114 | + FROM ranks_all r JOIN indicators i ON i.id = r.indicator_id | |
| 115 | + WHERE coalesce(i.ranking_eligible, true) AND r.n_world >= {MIN_COUNTRIES_FOR_RANKING} | |
| 116 | + """ | |
| 117 | + ) | |
| 118 | + return con.execute("SELECT count(*) FROM rankings").fetchone()[0] | |
| 119 | + | |
| 120 | + | |
| 121 | +def build_coverage(con: duckdb.DuckDBPyConnection) -> None: | |
| 122 | + con.execute( | |
| 123 | + """ | |
| 124 | + INSERT INTO coverage | |
| 125 | + WITH tot AS (SELECT count(DISTINCT indicator_id) AS n FROM obs_ok) | |
| 126 | + SELECT c.id, | |
| 127 | + count(DISTINCT o.indicator_id), | |
| 128 | + count(o.value), | |
| 129 | + max(o.year), | |
| 130 | + CASE WHEN (SELECT n FROM tot) > 0 THEN 100.0 * count(DISTINCT o.indicator_id) / (SELECT n FROM tot) ELSE 0 END, | |
| 131 | + now()::TIMESTAMP | |
| 132 | + FROM countries c LEFT JOIN obs_ok o ON o.country_id = c.id | |
| 133 | + GROUP BY c.id | |
| 134 | + """ | |
| 135 | + ) | |
| 136 | + | |
| 137 | + | |
| 138 | +def update_indicator_coverage(con: duckdb.DuckDBPyConnection) -> None: | |
| 139 | + con.execute( | |
| 140 | + """ | |
| 141 | + UPDATE indicators SET | |
| 142 | + n_countries = s.n_countries, n_observations = s.n_obs, first_year = s.first_year, last_year = s.last_year, | |
| 143 | + latest_source_updated_at = s.upd, primary_source_id = s.primary_source | |
| 144 | + FROM ( | |
| 145 | + -- first_year/last_year describe actual data (forecasts excluded); n_observations counts everything | |
| 146 | + SELECT indicator_id, | |
| 147 | + count(DISTINCT country_id) AS n_countries, count(*) AS n_obs, | |
| 148 | + min(CASE WHEN NOT coalesce(is_forecast, false) THEN year END) AS first_year, | |
| 149 | + max(CASE WHEN NOT coalesce(is_forecast, false) THEN year END) AS last_year, | |
| 150 | + max(source_updated_at) AS upd, | |
| 151 | + arg_max(source_id, cnt) AS primary_source | |
| 152 | + FROM (SELECT *, count(*) OVER (PARTITION BY indicator_id, source_id) AS cnt FROM observations) | |
| 153 | + GROUP BY indicator_id | |
| 154 | + ) s | |
| 155 | + WHERE indicators.id = s.indicator_id; | |
| 156 | + | |
| 157 | + UPDATE indicator_sources SET | |
| 158 | + n_observations = s.n_obs, n_countries = s.n_countries, last_year = s.last_year | |
| 159 | + FROM ( | |
| 160 | + SELECT indicator_id, source_id, source_dataset, source_series_code, | |
| 161 | + count(*) AS n_obs, count(DISTINCT country_id) AS n_countries, max(year) AS last_year | |
| 162 | + FROM staging_all GROUP BY ALL | |
| 163 | + ) s | |
| 164 | + WHERE indicator_sources.indicator_id = s.indicator_id AND indicator_sources.source_id = s.source_id | |
| 165 | + AND indicator_sources.dataset = s.source_dataset AND indicator_sources.series_code = s.source_series_code; | |
| 166 | + """ | |
| 167 | + ) | |
| 168 | + | |
| 169 | + | |
| 170 | +def update_sources_stats(con: duckdb.DuckDBPyConnection) -> None: | |
| 171 | + con.execute( | |
| 172 | + """ | |
| 173 | + UPDATE sources SET | |
| 174 | + n_indicators = s.n_ind, n_observations = s.n_obs | |
| 175 | + FROM (SELECT source_id, count(DISTINCT indicator_id) AS n_ind, count(*) AS n_obs FROM observations GROUP BY source_id) s | |
| 176 | + WHERE sources.id = s.source_id; | |
| 177 | + | |
| 178 | + UPDATE sources SET last_success_at = r.t | |
| 179 | + FROM (SELECT connector, max(finished_at) AS t FROM import_runs WHERE status IN ('ok', 'partial') GROUP BY connector) r | |
| 180 | + WHERE sources.id = r.connector; | |
| 181 | + """ | |
| 182 | + ) | |
| 183 | + | |
| 184 | + | |
| 185 | +def build_search_index( | |
| 186 | + con: duckdb.DuckDBPyConnection, | |
| 187 | + countries: list[Country], | |
| 188 | + groups: list[Group], | |
| 189 | + indicators: list[Indicator], | |
| 190 | + connectors_meta: dict[str, dict[str, Any]], | |
| 191 | +) -> int: | |
| 192 | + """Denormalised search rows. Country weight grows with log10(population) so large countries rank first on ties.""" | |
| 193 | + pop = { | |
| 194 | + r[0]: r[1] | |
| 195 | + for r in con.execute("SELECT country_id, value FROM latest WHERE indicator_id = 'population'").fetchall() | |
| 196 | + } | |
| 197 | + alt_by_iso: dict[str, set[str]] = {} | |
| 198 | + for name, iso in lookup().name_to_iso3.items(): | |
| 199 | + alt_by_iso.setdefault(iso, set()).add(name) | |
| 200 | + rows: list[dict[str, Any]] = [] | |
| 201 | + for c in countries: | |
| 202 | + alts = {c.official_name, c.capital or "", c.iso2, c.iso3, c.demonym or ""} | {a.title() for a in alt_by_iso.get(c.iso3, set())} | |
| 203 | + alts.discard(c.short_name) | |
| 204 | + alts.discard("") | |
| 205 | + p = pop.get(c.id) | |
| 206 | + w = 1.0 + (math.log10(p) / 10.0 if p and p > 0 else 0.0) | |
| 207 | + rows.append({"type": "country", "id": c.id, "slug": c.slug, "name": c.short_name, | |
| 208 | + "alt_names": " | ".join(sorted(alts)), "hint": c.region_wb_name, "weight": w}) | |
| 209 | + t = topics() | |
| 210 | + topic_names = {x["id"]: x["name"] for x in t["topics"]} | |
| 211 | + for i in indicators: | |
| 212 | + alts = {i.short_name or "", i.unit or "", *(i.tags or []), i.subtopic or ""} | |
| 213 | + alts.discard("") | |
| 214 | + rows.append({"type": "indicator", "id": i.id, "slug": i.slug, "name": i.name, "alt_names": " | ".join(sorted(alts)), | |
| 215 | + "hint": topic_names.get(i.topic, i.topic), "weight": 1.3 if i.featured else 1.0}) | |
| 216 | + for x in t["topics"]: | |
| 217 | + rows.append({"type": "topic", "id": x["id"], "slug": x["id"], "name": x["name"], "alt_names": x.get("short", ""), | |
| 218 | + "hint": x.get("blurb", ""), "weight": 0.9}) | |
| 219 | + for g in groups: | |
| 220 | + rows.append({"type": "region", "id": g.id, "slug": g.slug, "name": g.name, "alt_names": g.wb_code or "", | |
| 221 | + "hint": f"{g.kind} · {len(g.members)} members", "weight": 0.9}) | |
| 222 | + for sid, m in connectors_meta.items(): | |
| 223 | + rows.append({"type": "source", "id": sid, "slug": sid, "name": m.get("name") or sid, | |
| 224 | + "alt_names": m.get("organization") or "", "hint": m.get("url") or "", "weight": 0.6}) | |
| 225 | + df = pl.DataFrame(rows, schema={"type": pl.Utf8, "id": pl.Utf8, "slug": pl.Utf8, "name": pl.Utf8, "alt_names": pl.Utf8, | |
| 226 | + "hint": pl.Utf8, "weight": pl.Float64}) | |
| 227 | + con.register("df_search", df) | |
| 228 | + con.execute("INSERT INTO search_index SELECT * FROM df_search") | |
| 229 | + con.unregister("df_search") | |
| 230 | + return len(rows) | |
| 231 | + | |
| 232 | + | |
| 233 | +def write_meta(con: duckdb.DuckDBPyConnection, entries: dict[str, Any]) -> None: | |
| 234 | + now = datetime.now(UTC).isoformat() | |
| 235 | + entries = {"built_at": now, **entries} | |
| 236 | + con.executemany("INSERT OR REPLACE INTO meta VALUES (?, ?)", [(k, str(v)) for k, v in entries.items()]) | |
added
src/countryatlas/pipeline/export.py
+106 −0
@@ -0,0 +1,106 @@ | ||
| 1 | +"""CSV / JSON / Parquet exports of an indicator or a country from the live snapshot (also reusable by the API).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import logging | |
| 5 | +from datetime import UTC, datetime | |
| 6 | +from pathlib import Path | |
| 7 | +from typing import Literal | |
| 8 | + | |
| 9 | +import duckdb | |
| 10 | +import orjson | |
| 11 | +import polars as pl | |
| 12 | + | |
| 13 | +from countryatlas.config import settings | |
| 14 | + | |
| 15 | +log = logging.getLogger(__name__) | |
| 16 | + | |
| 17 | +Format = Literal["csv", "json", "parquet"] | |
| 18 | + | |
| 19 | + | |
| 20 | +def connect_readonly(db_path: Path | None = None) -> duckdb.DuckDBPyConnection: | |
| 21 | + p = db_path or settings.db_path | |
| 22 | + if not p.exists(): | |
| 23 | + raise FileNotFoundError(f"no snapshot at {p} — run `ca build` first") | |
| 24 | + return duckdb.connect(str(p), read_only=True) | |
| 25 | + | |
| 26 | + | |
| 27 | +def query_frame(sql: str, params: dict | list | None = None, db_path: Path | None = None) -> pl.DataFrame: | |
| 28 | + con = connect_readonly(db_path) | |
| 29 | + try: | |
| 30 | + return con.execute(sql, params or {}).pl() | |
| 31 | + finally: | |
| 32 | + con.close() | |
| 33 | + | |
| 34 | + | |
| 35 | +def meta(db_path: Path | None = None) -> dict[str, str]: | |
| 36 | + con = connect_readonly(db_path) | |
| 37 | + try: | |
| 38 | + return dict(con.execute("SELECT key, value FROM meta").fetchall()) | |
| 39 | + finally: | |
| 40 | + con.close() | |
| 41 | + | |
| 42 | + | |
| 43 | +def indicator_frame(slug: str, db_path: Path | None = None) -> pl.DataFrame: | |
| 44 | + return query_frame( | |
| 45 | + """ | |
| 46 | + SELECT o.indicator_id, o.country_id, c.short_name AS country, o.year, o.period, o.frequency, o.value, o.unit, | |
| 47 | + o.source_id, o.source_dataset, o.source_series_code, o.is_estimate, o.is_forecast, o.status, | |
| 48 | + o.source_updated_at, o.retrieved_at | |
| 49 | + FROM observations o JOIN countries c ON c.id = o.country_id | |
| 50 | + WHERE o.indicator_id = $slug ORDER BY o.country_id, o.period | |
| 51 | + """, | |
| 52 | + {"slug": slug}, | |
| 53 | + db_path, | |
| 54 | + ) | |
| 55 | + | |
| 56 | + | |
| 57 | +def country_frame(iso3: str, db_path: Path | None = None) -> pl.DataFrame: | |
| 58 | + return query_frame( | |
| 59 | + """ | |
| 60 | + SELECT o.country_id, o.indicator_id, i.name AS indicator, i.topic, o.year, o.period, o.frequency, o.value, o.unit, | |
| 61 | + o.source_id, o.source_dataset, o.source_series_code, o.is_estimate, o.is_forecast, o.status, | |
| 62 | + o.source_updated_at, o.retrieved_at | |
| 63 | + FROM observations o JOIN indicators i ON i.id = o.indicator_id | |
| 64 | + WHERE o.country_id = $iso ORDER BY i.topic, o.indicator_id, o.period | |
| 65 | + """, | |
| 66 | + {"iso": iso3.upper()}, | |
| 67 | + db_path, | |
| 68 | + ) | |
| 69 | + | |
| 70 | + | |
| 71 | +def write_frame(df: pl.DataFrame, path: Path, fmt: Format, header: dict | None = None) -> Path: | |
| 72 | + path.parent.mkdir(parents=True, exist_ok=True) | |
| 73 | + if fmt == "csv": | |
| 74 | + df.write_csv(path) | |
| 75 | + elif fmt == "parquet": | |
| 76 | + df.write_parquet(path, compression="zstd") | |
| 77 | + elif fmt == "json": | |
| 78 | + rows = df.with_columns(pl.col(pl.Date).cast(pl.Utf8), pl.col(pl.Datetime).cast(pl.Utf8)).to_dicts() | |
| 79 | + doc = {"meta": header or {}, "rows": rows} | |
| 80 | + path.write_bytes(orjson.dumps(doc, option=orjson.OPT_NON_STR_KEYS)) | |
| 81 | + else: | |
| 82 | + raise ValueError(f"unknown format {fmt}") | |
| 83 | + return path | |
| 84 | + | |
| 85 | + | |
| 86 | +def _header(kind: str, key: str, n: int) -> dict: | |
| 87 | + m = meta() | |
| 88 | + return {"type": kind, "key": key, "rows": n, "generated_at": datetime.now(UTC).isoformat(), | |
| 89 | + "run_id": m.get("build_run_id"), "built_at": m.get("built_at"), "site": settings.site_url, | |
| 90 | + "licence_note": "Values are redistributed under the licence of each source (see sources table / provenance)."} | |
| 91 | + | |
| 92 | + | |
| 93 | +def export_indicator(slug: str, fmt: Format = "csv", out_dir: Path | None = None) -> Path: | |
| 94 | + df = indicator_frame(slug) | |
| 95 | + if df.is_empty(): | |
| 96 | + raise ValueError(f"indicator '{slug}' has no observations in the snapshot") | |
| 97 | + out = (out_dir or settings.exports_dir / "indicators") / f"{slug}.{fmt}" | |
| 98 | + return write_frame(df, out, fmt, _header("indicator", slug, df.height)) | |
| 99 | + | |
| 100 | + | |
| 101 | +def export_country(iso3: str, fmt: Format = "csv", out_dir: Path | None = None) -> Path: | |
| 102 | + df = country_frame(iso3) | |
| 103 | + if df.is_empty(): | |
| 104 | + raise ValueError(f"country '{iso3}' has no observations in the snapshot") | |
| 105 | + out = (out_dir or settings.exports_dir / "countries") / f"{iso3.upper()}.{fmt}" | |
| 106 | + return write_frame(df, out, fmt, _header("country", iso3.upper(), df.height)) | |
added
src/countryatlas/pipeline/fetch.py
+228 −0
@@ -0,0 +1,228 @@ | ||
| 1 | +"""Fetch / normalize / validate every indicator-source spec of the requested connectors into staging parquet files. | |
| 2 | + | |
| 3 | +One unit of work = one `IndicatorSourceSpec`. Each unit runs in its own try/except: a failure writes a `failed` | |
| 4 | +ImportRun sidecar and leaves the previous staging parquet untouched. Units run concurrently in a ThreadPoolExecutor | |
| 5 | +(`settings.http_concurrency`); connectors keep their own rate limiters. | |
| 6 | + | |
| 7 | +`mode="fetch"` download → store raw → normalize → validate → staging | |
| 8 | +`mode="normalize"` re-normalize from the latest raw files (no HTTP) | |
| 9 | +""" | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import logging | |
| 13 | +import threading | |
| 14 | +import time | |
| 15 | +from concurrent.futures import ThreadPoolExecutor, as_completed | |
| 16 | +from dataclasses import dataclass, field | |
| 17 | +from datetime import UTC, datetime | |
| 18 | +from typing import Literal | |
| 19 | + | |
| 20 | +from countryatlas.config import settings | |
| 21 | +from countryatlas.connectors import ConnectorNotAvailable, get_connector | |
| 22 | +from countryatlas.connectors._util import ConnectorError | |
| 23 | +from countryatlas.connectors.base import Connector | |
| 24 | +from countryatlas.models import ImportRun, IndicatorSourceSpec, RawPayload | |
| 25 | +from countryatlas.pipeline import new_run_id | |
| 26 | +from countryatlas.pipeline.staging import ( | |
| 27 | + latest_raw_payloads, | |
| 28 | + read_previous_rows, | |
| 29 | + rows_to_frame, | |
| 30 | + spec_paths, | |
| 31 | + write_issues, | |
| 32 | + write_meta, | |
| 33 | + write_parquet_atomic, | |
| 34 | + write_run, | |
| 35 | +) | |
| 36 | +from countryatlas.pipeline.validate import validate_frame | |
| 37 | +from countryatlas.registry import indicators_by_id, source_specs | |
| 38 | + | |
| 39 | +log = logging.getLogger(__name__) | |
| 40 | + | |
| 41 | +Mode = Literal["fetch", "normalize"] | |
| 42 | + | |
| 43 | + | |
| 44 | +@dataclass | |
| 45 | +class FetchSummary: | |
| 46 | + run_id: str | |
| 47 | + runs: list[ImportRun] = field(default_factory=list) | |
| 48 | + skipped_connectors: list[str] = field(default_factory=list) | |
| 49 | + duration_s: float = 0.0 | |
| 50 | + | |
| 51 | + @property | |
| 52 | + def ok(self) -> list[ImportRun]: | |
| 53 | + return [r for r in self.runs if r.status == "ok"] | |
| 54 | + | |
| 55 | + @property | |
| 56 | + def failed(self) -> list[ImportRun]: | |
| 57 | + return [r for r in self.runs if r.status in ("failed", "quarantined")] | |
| 58 | + | |
| 59 | + def by_connector(self) -> dict[str, list[ImportRun]]: | |
| 60 | + out: dict[str, list[ImportRun]] = {} | |
| 61 | + for r in self.runs: | |
| 62 | + out.setdefault(r.connector, []).append(r) | |
| 63 | + return out | |
| 64 | + | |
| 65 | + | |
| 66 | +def _raw_code_for(connector: Connector, spec: IndicatorSourceSpec) -> str | None: | |
| 67 | + """Connectors that store one shared raw file for many specs expose the raw code via a class hook.""" | |
| 68 | + hook = getattr(connector, "raw_code_for", None) | |
| 69 | + if callable(hook): | |
| 70 | + return hook(spec) | |
| 71 | + return None | |
| 72 | + | |
| 73 | + | |
| 74 | +def process_spec(connector: Connector, spec: IndicatorSourceSpec, run_id: str, mode: Mode = "fetch") -> ImportRun: | |
| 75 | + """The unit of work. Never raises: any problem becomes a failed/quarantined ImportRun.""" | |
| 76 | + started = datetime.now(UTC) | |
| 77 | + dataset_label = f"{spec.dataset}:{spec.code}→{spec.indicator_id}" | |
| 78 | + run = ImportRun(run_id=run_id, connector=connector.id, dataset=dataset_label, started_at=started) | |
| 79 | + paths = spec_paths(spec) | |
| 80 | + try: | |
| 81 | + ind = indicators_by_id().get(spec.indicator_id) | |
| 82 | + if ind is None: | |
| 83 | + raise ConnectorError(f"indicator {spec.indicator_id} not in registry") | |
| 84 | + # 1. raw | |
| 85 | + if mode == "fetch": | |
| 86 | + raw = connector.fetch(spec) | |
| 87 | + raws: list[RawPayload] = raw if isinstance(raw, list) else [raw] | |
| 88 | + raw_paths = [connector.store_raw(p) for p in raws] | |
| 89 | + run.raw_path = str(raw_paths[0].parent) if raw_paths else None | |
| 90 | + else: | |
| 91 | + raws = latest_raw_payloads(connector.id, spec, raw_code=_raw_code_for(connector, spec)) | |
| 92 | + if not raws: | |
| 93 | + raise ConnectorError("no raw payload found for this spec — run `ca fetch` first") | |
| 94 | + run.raw_path = str(settings.raw_dir / connector.id / (spec.dataset or "default")) | |
| 95 | + run.rows_raw = sum(len(p.body) for p in raws) # bytes; rows are unknown before parsing | |
| 96 | + # 2. normalize | |
| 97 | + rows = connector.normalize(raws if len(raws) > 1 else raws[0], spec) | |
| 98 | + run.rows_norm = len(rows) | |
| 99 | + if not rows: | |
| 100 | + raise ConnectorError("normalization produced 0 rows (all aggregates/nulls, or column missing)") | |
| 101 | + # 3. connector-specific validation (duplicates …) | |
| 102 | + report = connector.validate(rows) | |
| 103 | + if report.quarantine_dataset: | |
| 104 | + run.status = "quarantined" | |
| 105 | + run.errors = report.errors | |
| 106 | + run.warnings = report.warnings | |
| 107 | + run.message = f"connector validation: {report.errors} errors — " + "; ".join( | |
| 108 | + i.message for i in report.issues[:3] | |
| 109 | + ) | |
| 110 | + write_issues(spec, report.issues) | |
| 111 | + return _finish(spec, run) | |
| 112 | + # 4. generic validation | |
| 113 | + frame = rows_to_frame(rows) | |
| 114 | + prev_rows = read_previous_rows(spec) if mode == "fetch" else None | |
| 115 | + gv = validate_frame(frame, ind, previous_rows=prev_rows) | |
| 116 | + issues = list(report.issues) + gv.issues | |
| 117 | + run.warnings = gv.warnings + report.warnings | |
| 118 | + run.errors = gv.errors + report.errors | |
| 119 | + if gv.quarantine_dataset: | |
| 120 | + run.status = "quarantined" | |
| 121 | + run.message = gv.message | |
| 122 | + write_issues(spec, issues) | |
| 123 | + log.warning("%s %s QUARANTINED: %s (previous staging kept)", connector.id, dataset_label, gv.message) | |
| 124 | + return _finish(spec, run) | |
| 125 | + # 5. staging | |
| 126 | + write_parquet_atomic(gv.frame, paths["parquet"]) | |
| 127 | + write_issues(spec, issues) | |
| 128 | + first = raws[0] | |
| 129 | + write_meta( | |
| 130 | + spec, | |
| 131 | + { | |
| 132 | + "source_url": (first.meta or {}).get("source_url"), | |
| 133 | + "notes": (first.meta or {}).get("notes") or spec.notes, | |
| 134 | + "source_updated_at": first.source_updated_at.isoformat() if first.source_updated_at else None, | |
| 135 | + "retrieved_at": first.retrieved_at.isoformat(), | |
| 136 | + "url": first.url, | |
| 137 | + "pages": first.pages, | |
| 138 | + "run_id": run_id, | |
| 139 | + "licence": connector.licence, | |
| 140 | + "attribution": connector.attribution, | |
| 141 | + }, | |
| 142 | + ) | |
| 143 | + run.rows_valid = int(gv.frame.filter(gv.frame["status"] != "quarantined").height) | |
| 144 | + run.status = "ok" if gv.n_quarantined == 0 else "partial" | |
| 145 | + run.message = ( | |
| 146 | + f"{run.rows_norm} rows; {gv.n_quarantined} quarantined, {gv.n_warning} warnings, {gv.n_stale} stale" | |
| 147 | + ) | |
| 148 | + return _finish(spec, run) | |
| 149 | + except ConnectorError as e: | |
| 150 | + run.status = "failed" | |
| 151 | + run.errors = 1 | |
| 152 | + run.message = str(e) | |
| 153 | + log.error("%s %s FAILED: %s", connector.id, dataset_label, e) | |
| 154 | + return _finish(spec, run) | |
| 155 | + except Exception as e: | |
| 156 | + run.status = "failed" | |
| 157 | + run.errors = 1 | |
| 158 | + run.message = f"{type(e).__name__}: {e}" | |
| 159 | + log.exception("%s %s FAILED", connector.id, dataset_label) | |
| 160 | + return _finish(spec, run) | |
| 161 | + | |
| 162 | + | |
| 163 | +def _finish(spec: IndicatorSourceSpec, run: ImportRun) -> ImportRun: | |
| 164 | + run.finished_at = datetime.now(UTC) | |
| 165 | + try: | |
| 166 | + write_run(spec, run) | |
| 167 | + except Exception: | |
| 168 | + log.exception("could not write run sidecar for %s", spec.key) | |
| 169 | + return run | |
| 170 | + | |
| 171 | + | |
| 172 | +def run_fetch( | |
| 173 | + connectors: list[str] | None = None, | |
| 174 | + indicator: str | None = None, | |
| 175 | + run_id: str | None = None, | |
| 176 | + mode: Mode = "fetch", | |
| 177 | + concurrency: int | None = None, | |
| 178 | +) -> FetchSummary: | |
| 179 | + """Process every spec of the requested connectors (default: all connectors with specs in the registry).""" | |
| 180 | + t0 = time.monotonic() | |
| 181 | + run_id = run_id or new_run_id() | |
| 182 | + settings.ensure_dirs() | |
| 183 | + specs = source_specs(indicator=indicator) | |
| 184 | + wanted = set(connectors) if connectors else {s.connector for s in specs} | |
| 185 | + summary = FetchSummary(run_id=run_id) | |
| 186 | + by_conn: dict[str, list[IndicatorSourceSpec]] = {} | |
| 187 | + for s in specs: | |
| 188 | + if s.connector in wanted: | |
| 189 | + by_conn.setdefault(s.connector, []).append(s) | |
| 190 | + for cid in sorted(wanted - set(by_conn)): | |
| 191 | + log.warning("connector %s: no specs in the registry match the request", cid) | |
| 192 | + | |
| 193 | + instances: dict[str, Connector] = {} | |
| 194 | + for cid in sorted(by_conn): | |
| 195 | + try: | |
| 196 | + instances[cid] = get_connector(cid) | |
| 197 | + except ConnectorNotAvailable as e: | |
| 198 | + log.warning("skipping %d specs — %s", len(by_conn[cid]), e) | |
| 199 | + summary.skipped_connectors.append(cid) | |
| 200 | + work = [(instances[cid], s) for cid, ss in by_conn.items() if cid in instances for s in ss] | |
| 201 | + log.info("run %s: %d specs across %d connectors (mode=%s)", run_id, len(work), len(instances), mode) | |
| 202 | + | |
| 203 | + n_workers = max(1, concurrency or settings.http_concurrency) | |
| 204 | + done = 0 | |
| 205 | + lock = threading.Lock() | |
| 206 | + with ThreadPoolExecutor(max_workers=n_workers, thread_name_prefix="ca-fetch") as ex: | |
| 207 | + futures = {ex.submit(process_spec, conn, spec, run_id, mode): spec for conn, spec in work} | |
| 208 | + for fut in as_completed(futures): | |
| 209 | + run = fut.result() | |
| 210 | + with lock: | |
| 211 | + done += 1 | |
| 212 | + summary.runs.append(run) | |
| 213 | + if run.status in ("ok", "partial"): | |
| 214 | + log.info("[%d/%d] %s %s: %s", done, len(work), run.connector, run.dataset, run.message) | |
| 215 | + for conn in instances.values(): | |
| 216 | + try: | |
| 217 | + conn.close() | |
| 218 | + except Exception as e: # noqa: BLE001 | |
| 219 | + log.debug("closing %s: %s", conn.id, e) | |
| 220 | + summary.duration_s = time.monotonic() - t0 | |
| 221 | + n_ok = len([r for r in summary.runs if r.status in ("ok", "partial")]) | |
| 222 | + log.info( | |
| 223 | + "run %s finished in %.0fs: %d ok, %d failed/quarantined, skipped connectors: %s", | |
| 224 | + run_id, summary.duration_s, n_ok, len(summary.failed), summary.skipped_connectors or "none", | |
| 225 | + ) | |
| 226 | + for r in summary.failed: | |
| 227 | + log.warning(" ✗ %s %s — %s", r.connector, r.dataset, r.message) | |
| 228 | + return summary | |
added
src/countryatlas/pipeline/format.py
+90 −0
@@ -0,0 +1,90 @@ | ||
| 1 | +"""Number formatting for headlines/insights, driven by the indicator's format/unit/precision (no locale, English).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import math | |
| 5 | + | |
| 6 | +from countryatlas.registry import Indicator | |
| 7 | + | |
| 8 | +_SCALES = [(1e12, "T"), (1e9, "B"), (1e6, "M")] | |
| 9 | +PERCENT_FORMATS = {"percent"} | |
| 10 | +RELATIVE_FORMATS = {"currency", "number", "tonnes", "kwh", "per_1000", "per_100k", "per_million", "km", "ha"} | |
| 11 | + | |
| 12 | + | |
| 13 | +def _num(v: float, precision: int) -> str: | |
| 14 | + if v is None or not math.isfinite(v): | |
| 15 | + return "n/a" | |
| 16 | + if precision <= 0: | |
| 17 | + return f"{v:,.0f}" | |
| 18 | + return f"{v:,.{precision}f}" | |
| 19 | + | |
| 20 | + | |
| 21 | +def _scaled(v: float, precision: int) -> str: | |
| 22 | + a = abs(v) | |
| 23 | + for k, suffix in _SCALES: | |
| 24 | + if a >= k: | |
| 25 | + return f"{v / k:,.{max(precision, 1)}f}{suffix}" | |
| 26 | + if a >= 1e4: | |
| 27 | + return f"{v:,.0f}" | |
| 28 | + return _num(v, precision) | |
| 29 | + | |
| 30 | + | |
| 31 | +def fmt_value(v: float, ind: Indicator) -> str: | |
| 32 | + """53372.1 (currency US$) → 'US$ 53,372'; 3.42 (percent) → '3.4 %'; 39.1e6 (number people) → '39.1M people'.""" | |
| 33 | + p = int(ind.precision if ind.precision is not None else 1) | |
| 34 | + f = ind.format | |
| 35 | + us = (ind.unit_short or "").strip() | |
| 36 | + if f == "percent": | |
| 37 | + return f"{_num(v, p)} %" | |
| 38 | + if f == "currency": | |
| 39 | + return f"{us + ' ' if us else ''}{_scaled(v, p)}".strip() | |
| 40 | + if f == "number": | |
| 41 | + s = _scaled(v, p) | |
| 42 | + return f"{s} {us}".strip() if us and us not in ("#",) else s | |
| 43 | + if f == "years": | |
| 44 | + return f"{_num(v, p)} years" | |
| 45 | + if f == "celsius": | |
| 46 | + return f"{_num(v, max(p, 2))} °C" | |
| 47 | + if f == "per_1000": | |
| 48 | + return f"{_num(v, p)} per 1,000" | |
| 49 | + if f == "per_100k": | |
| 50 | + return f"{_num(v, p)} per 100,000" | |
| 51 | + if f == "per_million": | |
| 52 | + return f"{_num(v, p)} per million" | |
| 53 | + if f == "index": | |
| 54 | + return _num(v, p) | |
| 55 | + if f == "ratio": | |
| 56 | + return _num(v, max(p, 2)) | |
| 57 | + if f == "tonnes": | |
| 58 | + return f"{_scaled(v, p)} {us or 't'}".strip() | |
| 59 | + return f"{_scaled(v, p)} {us}".strip() | |
| 60 | + | |
| 61 | + | |
| 62 | +def delta_mode(ind: Indicator) -> str: | |
| 63 | + """'points' for percent-like indicators, 'relative' for positive level series, else 'absolute'.""" | |
| 64 | + if ind.format in PERCENT_FORMATS or (ind.unit or "").startswith("%"): | |
| 65 | + return "points" | |
| 66 | + lo = (ind.bounds or [None, None])[0] | |
| 67 | + if ind.format in RELATIVE_FORMATS and lo is not None and float(lo) >= 0: | |
| 68 | + return "relative" | |
| 69 | + return "absolute" | |
| 70 | + | |
| 71 | + | |
| 72 | +def fmt_delta(delta: float, delta_pct: float | None, ind: Indicator) -> str: | |
| 73 | + """'2.1 points' | '8.3 %' | '1.4 years' according to delta_mode.""" | |
| 74 | + mode = delta_mode(ind) | |
| 75 | + p = int(ind.precision if ind.precision is not None else 1) | |
| 76 | + if mode == "points": | |
| 77 | + return f"{_num(abs(delta), max(p, 1))} points" | |
| 78 | + if mode == "relative" and delta_pct is not None and math.isfinite(delta_pct): | |
| 79 | + return f"{_num(abs(delta_pct), 1)} %" | |
| 80 | + if ind.format == "years": | |
| 81 | + return f"{_num(abs(delta), max(p, 1))} years" | |
| 82 | + return fmt_value(abs(delta), ind) | |
| 83 | + | |
| 84 | + | |
| 85 | +def fmt_pct(v: float, precision: int = 1) -> str: | |
| 86 | + return f"{_num(v, precision)} %" | |
| 87 | + | |
| 88 | + | |
| 89 | +def fmt_int(v: float) -> str: | |
| 90 | + return f"{round(v):,}" | |
added
src/countryatlas/pipeline/insights.py
+222 −0
@@ -0,0 +1,222 @@ | ||
| 1 | +"""Templated insights (registry/insights.yaml). Every number is computed from the snapshot; text is a format string.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import hashlib | |
| 5 | +import json | |
| 6 | +import logging | |
| 7 | +from datetime import UTC, datetime | |
| 8 | +from functools import lru_cache | |
| 9 | +from typing import Any | |
| 10 | + | |
| 11 | +import numpy as np | |
| 12 | +import polars as pl | |
| 13 | +import yaml | |
| 14 | + | |
| 15 | +from countryatlas.config import settings | |
| 16 | +from countryatlas.pipeline.format import fmt_pct, fmt_value | |
| 17 | +from countryatlas.registry import Country, Group, Indicator | |
| 18 | + | |
| 19 | +log = logging.getLogger(__name__) | |
| 20 | + | |
| 21 | +NEAREST_YEAR_TOLERANCE = 3 | |
| 22 | + | |
| 23 | +VERBS = { | |
| 24 | + "grow": ("grew", "shrank"), | |
| 25 | + "rise": ("rose", "fell"), | |
| 26 | + "gain": ("gained", "lost"), | |
| 27 | + "updown": ("went up", "went down"), | |
| 28 | +} | |
| 29 | + | |
| 30 | + | |
| 31 | +@lru_cache(maxsize=1) | |
| 32 | +def insight_templates() -> list[dict[str, Any]]: | |
| 33 | + p = settings.registry_dir / "insights.yaml" | |
| 34 | + return (yaml.safe_load(p.read_text()) or {}).get("templates", []) if p.exists() else [] | |
| 35 | + | |
| 36 | + | |
| 37 | +class InsightContext: | |
| 38 | + """Fast lookups over the snapshot: series per (country, indicator), latest values, group membership.""" | |
| 39 | + | |
| 40 | + def __init__( | |
| 41 | + self, | |
| 42 | + obs: pl.DataFrame, # country_id, indicator_id, year, value — annual, non-forecast, non-quarantined | |
| 43 | + latest: pl.DataFrame, # country_id, indicator_id, year, value | |
| 44 | + countries: dict[str, Country], | |
| 45 | + groups: dict[str, Group], | |
| 46 | + indicators: dict[str, Indicator], | |
| 47 | + ) -> None: | |
| 48 | + self.countries = countries | |
| 49 | + self.groups = groups | |
| 50 | + self.indicators = indicators | |
| 51 | + self.series: dict[tuple[str, str], dict[int, float]] = {} | |
| 52 | + for c, i, y, v in obs.select("country_id", "indicator_id", "year", "value").iter_rows(): | |
| 53 | + self.series.setdefault((c, i), {})[int(y)] = float(v) | |
| 54 | + self.latest: dict[tuple[str, str], tuple[int, float]] = { | |
| 55 | + (c, i): (int(y), float(v)) for c, i, y, v in latest.select("country_id", "indicator_id", "year", "value").iter_rows() | |
| 56 | + } | |
| 57 | + # values by (indicator, year) → {country: value} for group ranks/medians (built lazily) | |
| 58 | + self._by_year: dict[tuple[str, int], dict[str, float]] = {} | |
| 59 | + self.member_of: dict[str, set[str]] = {} | |
| 60 | + for g in groups.values(): | |
| 61 | + for m in g.members: | |
| 62 | + self.member_of.setdefault(m, set()).add(g.id) | |
| 63 | + | |
| 64 | + def value_near(self, c: str, ind: str, year: int) -> tuple[int, float] | None: | |
| 65 | + s = self.series.get((c, ind)) | |
| 66 | + if not s: | |
| 67 | + return None | |
| 68 | + if year in s: | |
| 69 | + return year, s[year] | |
| 70 | + for k in range(1, NEAREST_YEAR_TOLERANCE + 1): | |
| 71 | + for y in (year + k, year - k): | |
| 72 | + if y in s: | |
| 73 | + return y, s[y] | |
| 74 | + return None | |
| 75 | + | |
| 76 | + def values_for_year(self, ind: str, year: int) -> dict[str, float]: | |
| 77 | + key = (ind, year) | |
| 78 | + if key not in self._by_year: | |
| 79 | + self._by_year[key] = {c: s[year] for (c, i), s in self.series.items() if i == ind and year in s} | |
| 80 | + return self._by_year[key] | |
| 81 | + | |
| 82 | + def resolve_group(self, c: str, group: str) -> Group | None: | |
| 83 | + country = self.countries.get(c) | |
| 84 | + if country is None: | |
| 85 | + return None | |
| 86 | + if group == "world": | |
| 87 | + return self.groups.get("world") | |
| 88 | + if group == "region": | |
| 89 | + return next((g for g in self.groups.values() if g.kind == "region" and g.wb_code == country.region_wb), None) | |
| 90 | + if group == "income": | |
| 91 | + return next((g for g in self.groups.values() if g.kind == "income" and g.id.upper() == (country.income_group or "")), None) | |
| 92 | + g = self.groups.get(group) | |
| 93 | + return g if g and c in g.members else None | |
| 94 | + | |
| 95 | + | |
| 96 | +def _ordinal(n: int) -> str: | |
| 97 | + return f"{n}{'th' if 10 <= n % 100 <= 20 else {1: 'st', 2: 'nd', 3: 'rd'}.get(n % 10, 'th')}" | |
| 98 | + | |
| 99 | + | |
| 100 | +def _signed(txt: str, positive: bool) -> str: | |
| 101 | + return ("+" if positive else "−") + txt | |
| 102 | + | |
| 103 | + | |
| 104 | +def render_template(t: dict[str, Any], c: str, ctx: InsightContext) -> tuple[str, dict[str, Any], list[str]] | None: | |
| 105 | + """Return (text, values, indicators) or None when the template does not apply to this country.""" | |
| 106 | + ind_id = t["indicator"] | |
| 107 | + ind = ctx.indicators.get(ind_id) | |
| 108 | + country = ctx.countries.get(c) | |
| 109 | + if ind is None or country is None: | |
| 110 | + return None | |
| 111 | + kind = t["kind"] | |
| 112 | + base = {"country": country.short_name, "indicator": ind.display_name} | |
| 113 | + lat = ctx.latest.get((c, ind_id)) | |
| 114 | + if lat is None: | |
| 115 | + return None | |
| 116 | + y1, v1 = lat | |
| 117 | + vals: dict[str, Any] = {"y1": y1, "v1": v1} | |
| 118 | + | |
| 119 | + if kind == "change_since": | |
| 120 | + near = ctx.value_near(c, ind_id, int(t["since"])) | |
| 121 | + if near is None or near[0] >= y1: | |
| 122 | + return None | |
| 123 | + y0, v0 = near | |
| 124 | + delta = v1 - v0 | |
| 125 | + pct = (delta / abs(v0) * 100.0) if v0 else None | |
| 126 | + up, down = VERBS.get(t.get("verb_style", "rise"), VERBS["rise"]) | |
| 127 | + p = max(int(ind.precision or 0), 1) | |
| 128 | + vals.update({"y0": y0, "v0": v0, "delta": delta, "pct": pct}) | |
| 129 | + fmt = { | |
| 130 | + **base, "y0": y0, "y1": y1, "v0": fmt_value(v0, ind), "v1": fmt_value(v1, ind), | |
| 131 | + "delta": f"{delta:,.{p}f}", "abs_delta": (f"{abs(delta):,.{p}f} years" if ind.format == "years" else f"{abs(delta):,.{p}f}"), | |
| 132 | + "signed_delta": _signed(f"{abs(delta):,.{p}f}", delta >= 0), | |
| 133 | + "pct": fmt_pct(pct) if pct is not None else "n/a", | |
| 134 | + "abs_pct": fmt_pct(abs(pct)) if pct is not None else "n/a", | |
| 135 | + "signed_pct": _signed(fmt_pct(abs(pct)), pct >= 0) if pct is not None else "n/a", | |
| 136 | + "verb": up if delta >= 0 else down, | |
| 137 | + } | |
| 138 | + if abs(delta) < 1e-9: | |
| 139 | + return None | |
| 140 | + return t["text"].format(**fmt), vals, [ind_id] | |
| 141 | + | |
| 142 | + if kind == "rank_in_group": | |
| 143 | + g = ctx.resolve_group(c, t["group"]) | |
| 144 | + if g is None: | |
| 145 | + return None | |
| 146 | + pool = ctx.values_for_year(ind_id, y1) | |
| 147 | + members = [m for m in g.members if m in pool and ctx.countries.get(m) is not None] | |
| 148 | + n = len(members) | |
| 149 | + if n < int(t.get("min_n", 5)) or c not in pool: | |
| 150 | + return None | |
| 151 | + desc = ind.higher_is_better is not False | |
| 152 | + ordered = sorted(members, key=lambda m: pool[m], reverse=desc) | |
| 153 | + rank = ordered.index(c) + 1 | |
| 154 | + vals.update({"rank": rank, "n": n, "group": g.id}) | |
| 155 | + fmt = {**base, "rank": _ordinal(rank), "n": n, "group": g.name, "v1": fmt_value(v1, ind), "y1": y1} | |
| 156 | + return t["text"].format(**fmt), vals, [ind_id] | |
| 157 | + | |
| 158 | + if kind == "vs_median": | |
| 159 | + g = ctx.resolve_group(c, t["group"]) | |
| 160 | + if g is None: | |
| 161 | + return None | |
| 162 | + pool = ctx.values_for_year(ind_id, y1) | |
| 163 | + arr = np.array([pool[m] for m in g.members if m in pool and m != c], dtype=float) | |
| 164 | + if arr.size < 5: | |
| 165 | + return None | |
| 166 | + median = float(np.median(arr)) | |
| 167 | + ratio = v1 / median if median else None | |
| 168 | + vals.update({"median": median, "ratio": ratio, "diff": v1 - median, "group": g.id, "n": int(arr.size)}) | |
| 169 | + if ratio is None: | |
| 170 | + return None | |
| 171 | + ratio_txt = f"{ratio:.1f}×" if ratio >= 1.05 else ("close to" if 0.95 < ratio < 1.05 else f"{ratio:.2f}×") | |
| 172 | + fmt = { | |
| 173 | + **base, "v1": fmt_value(v1, ind), "y1": y1, "median": fmt_value(median, ind), "ratio": ratio_txt, | |
| 174 | + "diff": fmt_value(abs(v1 - median), ind), "group": g.name, | |
| 175 | + "above_below": "above" if v1 > median * 1.02 else ("below" if v1 < median * 0.98 else "in line with"), | |
| 176 | + } | |
| 177 | + return t["text"].format(**fmt), vals, [ind_id] | |
| 178 | + | |
| 179 | + if kind == "avg_growth": | |
| 180 | + s = ctx.series.get((c, ind_id)) or {} | |
| 181 | + y0 = int(t["from"]) | |
| 182 | + ys = sorted(y for y in s if y0 <= y <= y1) | |
| 183 | + if len(ys) < 5: | |
| 184 | + return None | |
| 185 | + avg = float(np.mean([s[y] for y in ys])) | |
| 186 | + vals.update({"avg": avg, "y0": ys[0], "n_years": len(ys)}) | |
| 187 | + fmt = {**base, "avg": fmt_pct(avg), "y0": ys[0], "y1": y1, "n_years": len(ys)} | |
| 188 | + return t["text"].format(**fmt), vals, [ind_id] | |
| 189 | + | |
| 190 | + log.warning("insights: unknown template kind %s", kind) | |
| 191 | + return None | |
| 192 | + | |
| 193 | + | |
| 194 | +def compute_insights(ctx: InsightContext) -> pl.DataFrame: | |
| 195 | + templates = insight_templates() | |
| 196 | + computed_at = datetime.now(UTC).replace(tzinfo=None) | |
| 197 | + rows: list[dict[str, Any]] = [] | |
| 198 | + for c in ctx.countries: | |
| 199 | + for t in templates: | |
| 200 | + try: | |
| 201 | + r = render_template(t, c, ctx) | |
| 202 | + except Exception: | |
| 203 | + log.exception("insight %s failed for %s", t.get("id"), c) | |
| 204 | + continue | |
| 205 | + if r is None: | |
| 206 | + continue | |
| 207 | + text, vals, inds = r | |
| 208 | + rows.append( | |
| 209 | + { | |
| 210 | + "id": hashlib.sha1(f"{c}|{t['id']}".encode()).hexdigest()[:16], | |
| 211 | + "country_id": c, | |
| 212 | + "template_id": t["id"], | |
| 213 | + "text": text, | |
| 214 | + "values": json.dumps(vals, default=str), | |
| 215 | + "indicators": inds, | |
| 216 | + "computed_at": computed_at, | |
| 217 | + } | |
| 218 | + ) | |
| 219 | + schema = {"id": pl.Utf8, "country_id": pl.Utf8, "template_id": pl.Utf8, "text": pl.Utf8, "values": pl.Utf8, | |
| 220 | + "indicators": pl.List(pl.Utf8), "computed_at": pl.Datetime("us")} | |
| 221 | + log.info("insights: %d rows from %d templates", len(rows), len(templates)) | |
| 222 | + return pl.DataFrame(rows, schema=schema) if rows else pl.DataFrame(schema=schema) | |
added
src/countryatlas/pipeline/scheduler.py
+116 −0
@@ -0,0 +1,116 @@ | ||
| 1 | +"""`ca schedule`: long-running loop. Refreshes daily at settings.refresh_hour:refresh_minute (America/Toronto by | |
| 2 | +default), immediately on SIGUSR1, logs to logs/refresh-<date>.log and writes a heartbeat to data_dir/scheduler.json. | |
| 3 | +""" | |
| 4 | +from __future__ import annotations | |
| 5 | + | |
| 6 | +import json | |
| 7 | +import logging | |
| 8 | +import os | |
| 9 | +import signal | |
| 10 | +import threading | |
| 11 | +import time | |
| 12 | +import traceback | |
| 13 | +from datetime import UTC, datetime, timedelta | |
| 14 | +from pathlib import Path | |
| 15 | +from typing import Any | |
| 16 | +from zoneinfo import ZoneInfo | |
| 17 | + | |
| 18 | +from countryatlas.config import settings | |
| 19 | +from countryatlas.pipeline import setup_logging | |
| 20 | + | |
| 21 | +log = logging.getLogger(__name__) | |
| 22 | + | |
| 23 | +_wake = threading.Event() | |
| 24 | +_stop = threading.Event() | |
| 25 | + | |
| 26 | + | |
| 27 | +def heartbeat_path() -> Path: | |
| 28 | + return settings.data_dir / "scheduler.json" | |
| 29 | + | |
| 30 | + | |
| 31 | +def next_run_at(now: datetime | None = None) -> datetime: | |
| 32 | + tz = ZoneInfo(settings.timezone) | |
| 33 | + now = (now or datetime.now(UTC)).astimezone(tz) | |
| 34 | + candidate = now.replace(hour=settings.refresh_hour, minute=settings.refresh_minute, second=0, microsecond=0) | |
| 35 | + if candidate <= now: | |
| 36 | + candidate += timedelta(days=1) | |
| 37 | + return candidate | |
| 38 | + | |
| 39 | + | |
| 40 | +def write_heartbeat(**fields: Any) -> None: | |
| 41 | + p = heartbeat_path() | |
| 42 | + p.parent.mkdir(parents=True, exist_ok=True) | |
| 43 | + cur: dict[str, Any] = {} | |
| 44 | + if p.exists(): | |
| 45 | + try: | |
| 46 | + cur = json.loads(p.read_text()) | |
| 47 | + except Exception: # noqa: BLE001 | |
| 48 | + cur = {} | |
| 49 | + cur.update(fields) | |
| 50 | + cur["updated_at"] = datetime.now(UTC).isoformat() | |
| 51 | + cur["pid"] = os.getpid() | |
| 52 | + tmp = p.with_suffix(".json.tmp") | |
| 53 | + tmp.write_text(json.dumps(cur, indent=1, default=str)) | |
| 54 | + os.replace(tmp, p) | |
| 55 | + | |
| 56 | + | |
| 57 | +def _on_usr1(signum: int, frame: Any) -> None: | |
| 58 | + log.info("SIGUSR1 received — refreshing now") | |
| 59 | + _wake.set() | |
| 60 | + | |
| 61 | + | |
| 62 | +def _on_term(signum: int, frame: Any) -> None: | |
| 63 | + log.info("signal %s received — stopping after the current step", signum) | |
| 64 | + _stop.set() | |
| 65 | + _wake.set() | |
| 66 | + | |
| 67 | + | |
| 68 | +def run_once(connectors: list[str] | None = None, strict: bool = True) -> dict[str, Any]: | |
| 69 | + """One refresh with its own dated log file. Never raises (status recorded in the heartbeat).""" | |
| 70 | + from countryatlas.pipeline.build import refresh | |
| 71 | + | |
| 72 | + day = datetime.now(ZoneInfo(settings.timezone)).strftime("%Y-%m-%d") | |
| 73 | + setup_logging(logfile=settings.logs_dir / f"refresh-{day}.log") | |
| 74 | + started = datetime.now(UTC) | |
| 75 | + write_heartbeat(status="running", last_started=started.isoformat()) | |
| 76 | + try: | |
| 77 | + summary = refresh(connectors=connectors, strict=strict) | |
| 78 | + write_heartbeat(status="idle", last_run=started.isoformat(), last_status="ok", last_summary=summary, | |
| 79 | + last_error=None, next_run=next_run_at().isoformat()) | |
| 80 | + return {"status": "ok", **summary} | |
| 81 | + except Exception as e: | |
| 82 | + log.exception("refresh failed") | |
| 83 | + write_heartbeat(status="idle", last_run=started.isoformat(), last_status="failed", | |
| 84 | + last_error=f"{type(e).__name__}: {e}", last_traceback=traceback.format_exc()[-4000:], | |
| 85 | + next_run=next_run_at().isoformat()) | |
| 86 | + return {"status": "failed", "error": str(e)} | |
| 87 | + | |
| 88 | + | |
| 89 | +def serve(connectors: list[str] | None = None, run_immediately: bool = False, strict: bool = True) -> None: | |
| 90 | + signal.signal(signal.SIGUSR1, _on_usr1) | |
| 91 | + signal.signal(signal.SIGTERM, _on_term) | |
| 92 | + signal.signal(signal.SIGINT, _on_term) | |
| 93 | + settings.ensure_dirs() | |
| 94 | + nxt = next_run_at() | |
| 95 | + write_heartbeat(status="idle", next_run=nxt.isoformat(), timezone=settings.timezone, | |
| 96 | + schedule=f"{settings.refresh_hour:02d}:{settings.refresh_minute:02d}") | |
| 97 | + log.info("scheduler started (pid %d): next refresh at %s (%s); SIGUSR1 = refresh now", os.getpid(), nxt, settings.timezone) | |
| 98 | + if run_immediately: | |
| 99 | + _wake.set() | |
| 100 | + while not _stop.is_set(): | |
| 101 | + now = datetime.now(UTC) | |
| 102 | + wait = max(0.0, (nxt - now).total_seconds()) | |
| 103 | + _wake.wait(timeout=min(wait, 60.0)) | |
| 104 | + if _stop.is_set(): | |
| 105 | + break | |
| 106 | + now = datetime.now(UTC) | |
| 107 | + if _wake.is_set() or now >= nxt: | |
| 108 | + _wake.clear() | |
| 109 | + result = run_once(connectors=connectors, strict=strict) | |
| 110 | + log.info("refresh finished: %s", result.get("status")) | |
| 111 | + nxt = next_run_at() | |
| 112 | + write_heartbeat(next_run=nxt.isoformat()) | |
| 113 | + log.info("next refresh at %s", nxt) | |
| 114 | + time.sleep(1) | |
| 115 | + write_heartbeat(status="stopped") | |
| 116 | + log.info("scheduler stopped") | |
added
src/countryatlas/pipeline/similarity.py
+195 −0
@@ -0,0 +1,195 @@ | ||
| 1 | +"""Similarity (5 modes, registry/similarity.yaml) and Country DNA (9 percentile dimensions). | |
| 2 | + | |
| 3 | +Inputs are the `latest` values (non-forecast, non-quarantined latest observation per country×indicator). Features are | |
| 4 | +transformed (log / log1p), z-scored across countries (nan-aware), and compared with a weighted Euclidean distance over | |
| 5 | +the features both countries have (rescaled to the mode's full weight; a pair needs ≥ 50 % of the weight in common). | |
| 6 | +score = 100·exp(−d/d0), d0 = median of all pairwise distances of the mode. Top 12 peers stored, with per-feature | |
| 7 | +contributions (share of the squared distance) so every score is explainable. | |
| 8 | +""" | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import json | |
| 12 | +import logging | |
| 13 | +from functools import lru_cache | |
| 14 | +from typing import Any | |
| 15 | + | |
| 16 | +import numpy as np | |
| 17 | +import polars as pl | |
| 18 | +import yaml | |
| 19 | + | |
| 20 | +from countryatlas.config import settings | |
| 21 | + | |
| 22 | +log = logging.getLogger(__name__) | |
| 23 | + | |
| 24 | +TOP_PEERS = 12 | |
| 25 | +MIN_FEATURE_COVERAGE = 0.70 | |
| 26 | +MIN_SHARED_WEIGHT = 0.50 | |
| 27 | + | |
| 28 | + | |
| 29 | +@lru_cache(maxsize=1) | |
| 30 | +def similarity_config() -> dict[str, Any]: | |
| 31 | + p = settings.registry_dir / "similarity.yaml" | |
| 32 | + return yaml.safe_load(p.read_text()) if p.exists() else {"modes": {}, "dna": {}} | |
| 33 | + | |
| 34 | + | |
| 35 | +def _feature_vector(latest: dict[tuple[str, str], float], countries: list[str], f: dict[str, Any]) -> np.ndarray: | |
| 36 | + """Latest value per country for one feature definition (with optional `per` ratio and transform).""" | |
| 37 | + ind = f["indicator"] | |
| 38 | + per = f.get("per") | |
| 39 | + scale = float(f.get("scale", 1.0)) | |
| 40 | + out = np.full(len(countries), np.nan) | |
| 41 | + for i, c in enumerate(countries): | |
| 42 | + v = latest.get((c, ind)) | |
| 43 | + if v is None: | |
| 44 | + continue | |
| 45 | + if per: | |
| 46 | + den = latest.get((c, per)) | |
| 47 | + if den is None or den == 0: | |
| 48 | + continue | |
| 49 | + v = v / den * scale | |
| 50 | + out[i] = v | |
| 51 | + t = f.get("transform", "none") | |
| 52 | + with np.errstate(divide="ignore", invalid="ignore"): | |
| 53 | + if t == "log": | |
| 54 | + out = np.where(out > 0, np.log(out), np.nan) | |
| 55 | + elif t == "log1p": | |
| 56 | + out = np.where(out >= 0, np.log1p(out), np.nan) | |
| 57 | + return out | |
| 58 | + | |
| 59 | + | |
| 60 | +def _zscore(m: np.ndarray) -> np.ndarray: | |
| 61 | + mu = np.nanmean(m, axis=0) | |
| 62 | + sd = np.nanstd(m, axis=0) | |
| 63 | + sd = np.where(sd > 0, sd, 1.0) | |
| 64 | + return (m - mu) / sd | |
| 65 | + | |
| 66 | + | |
| 67 | +def compute_similarity(latest_df: pl.DataFrame, countries: list[str]) -> pl.DataFrame: | |
| 68 | + """latest_df: columns country_id, indicator_id, value (latest non-forecast). Returns the `similarity` rows.""" | |
| 69 | + latest = {(r[0], r[1]): float(r[2]) for r in latest_df.select("country_id", "indicator_id", "value").iter_rows()} | |
| 70 | + cfg = similarity_config() | |
| 71 | + rows: list[dict[str, Any]] = [] | |
| 72 | + for mode, spec in (cfg.get("modes") or {}).items(): | |
| 73 | + feats = spec.get("features") or [] | |
| 74 | + if not feats: | |
| 75 | + continue | |
| 76 | + names = [f["indicator"] if not f.get("per") else f"{f['indicator']}/{f['per']}" for f in feats] | |
| 77 | + weights = np.array([float(f.get("weight", 1.0)) for f in feats]) | |
| 78 | + m = np.column_stack([_feature_vector(latest, countries, f) for f in feats]) # (n_countries, n_features) | |
| 79 | + present = np.isfinite(m) | |
| 80 | + coverage = (present * weights).sum(axis=1) / weights.sum() | |
| 81 | + eligible = coverage >= MIN_FEATURE_COVERAGE | |
| 82 | + if eligible.sum() < 3: | |
| 83 | + log.warning("similarity[%s]: only %d eligible countries — skipped", mode, int(eligible.sum())) | |
| 84 | + continue | |
| 85 | + z = _zscore(np.where(present, m, np.nan)) | |
| 86 | + idx = np.flatnonzero(eligible) | |
| 87 | + Z = np.nan_to_num(z[idx], nan=0.0) | |
| 88 | + P = present[idx].astype(float) | |
| 89 | + W = weights[None, :] | |
| 90 | + # squared weighted differences on shared features, rescaled to full weight | |
| 91 | + diff2 = (Z[:, None, :] - Z[None, :, :]) ** 2 # (k, k, f) | |
| 92 | + shared = P[:, None, :] * P[None, :, :] | |
| 93 | + w_shared = (shared * W).sum(axis=2) | |
| 94 | + d2 = (diff2 * shared * W).sum(axis=2) | |
| 95 | + with np.errstate(divide="ignore", invalid="ignore"): | |
| 96 | + d = np.sqrt(d2 * (weights.sum() / w_shared)) | |
| 97 | + d[w_shared < MIN_SHARED_WEIGHT * weights.sum()] = np.nan | |
| 98 | + np.fill_diagonal(d, np.nan) | |
| 99 | + finite = d[np.isfinite(d)] | |
| 100 | + if finite.size == 0: | |
| 101 | + continue | |
| 102 | + d0 = float(np.median(finite)) or 1.0 | |
| 103 | + score = 100.0 * np.exp(-d / d0) | |
| 104 | + for a_pos, a in enumerate(idx): | |
| 105 | + order = np.argsort(-np.nan_to_num(score[a_pos], nan=-1.0)) | |
| 106 | + rank = 0 | |
| 107 | + for b_pos in order: | |
| 108 | + if b_pos == a_pos or not np.isfinite(score[a_pos, b_pos]): | |
| 109 | + continue | |
| 110 | + rank += 1 | |
| 111 | + if rank > TOP_PEERS: | |
| 112 | + break | |
| 113 | + b = idx[b_pos] | |
| 114 | + contrib_raw = diff2[a_pos, b_pos] * shared[a_pos, b_pos] * weights | |
| 115 | + total = contrib_raw.sum() or 1.0 | |
| 116 | + contributions = { | |
| 117 | + names[k]: { | |
| 118 | + "z_a": round(float(z[a, k]), 3) if present[a, k] else None, | |
| 119 | + "z_b": round(float(z[b, k]), 3) if present[b, k] else None, | |
| 120 | + "weight": float(weights[k]), | |
| 121 | + "contribution": round(float(contrib_raw[k] / total), 4) if shared[a_pos, b_pos, k] else None, | |
| 122 | + } | |
| 123 | + for k in range(len(feats)) | |
| 124 | + } | |
| 125 | + rows.append( | |
| 126 | + { | |
| 127 | + "country_id": countries[a], | |
| 128 | + "mode": mode, | |
| 129 | + "peer_id": countries[b], | |
| 130 | + "score": round(float(score[a_pos, b_pos]), 2), | |
| 131 | + "rank": rank, | |
| 132 | + "contributions": json.dumps(contributions), | |
| 133 | + } | |
| 134 | + ) | |
| 135 | + log.info("similarity[%s]: %d countries scored (d0=%.3f)", mode, len(idx), d0) | |
| 136 | + schema = {"country_id": pl.Utf8, "mode": pl.Utf8, "peer_id": pl.Utf8, "score": pl.Float64, "rank": pl.Int32, | |
| 137 | + "contributions": pl.Utf8} | |
| 138 | + return pl.DataFrame(rows, schema=schema) if rows else pl.DataFrame(schema=schema) | |
| 139 | + | |
| 140 | + | |
| 141 | +# ------------------------------------------------------------------------------------------------------ Country DNA | |
| 142 | +def _percentile(v: np.ndarray) -> np.ndarray: | |
| 143 | + """Percentile rank 0–100 among finite values (average ranks for ties); nan stays nan.""" | |
| 144 | + out = np.full(len(v), np.nan) | |
| 145 | + ok = np.isfinite(v) | |
| 146 | + n = int(ok.sum()) | |
| 147 | + if n < 2: | |
| 148 | + return out | |
| 149 | + x = v[ok] | |
| 150 | + order = np.argsort(x, kind="mergesort") | |
| 151 | + ranks = np.empty(n) | |
| 152 | + ranks[order] = np.arange(n) | |
| 153 | + # average ties | |
| 154 | + sorted_x = x[order] | |
| 155 | + i = 0 | |
| 156 | + while i < n: | |
| 157 | + j = i | |
| 158 | + while j + 1 < n and sorted_x[j + 1] == sorted_x[i]: | |
| 159 | + j += 1 | |
| 160 | + if j > i: | |
| 161 | + ranks[order[i:j + 1]] = (i + j) / 2.0 | |
| 162 | + i = j + 1 | |
| 163 | + out[ok] = ranks / (n - 1) * 100.0 | |
| 164 | + return out | |
| 165 | + | |
| 166 | + | |
| 167 | +def compute_country_dna(latest_df: pl.DataFrame, countries: list[str]) -> pl.DataFrame: | |
| 168 | + latest = {(r[0], r[1]): float(r[2]) for r in latest_df.select("country_id", "indicator_id", "value").iter_rows()} | |
| 169 | + years = {(r[0], r[1]): int(r[2]) for r in latest_df.select("country_id", "indicator_id", "year").iter_rows()} | |
| 170 | + cfg = similarity_config().get("dna") or {} | |
| 171 | + dims_pct: dict[str, np.ndarray] = {} | |
| 172 | + used_indicators: list[str] = [] | |
| 173 | + for dim, parts in cfg.items(): | |
| 174 | + acc = np.full(len(countries), np.nan) | |
| 175 | + cnt = np.zeros(len(countries)) | |
| 176 | + for part in parts: | |
| 177 | + vec = _feature_vector(latest, countries, part) | |
| 178 | + pct = _percentile(vec) | |
| 179 | + if part.get("invert"): | |
| 180 | + pct = 100.0 - pct | |
| 181 | + ok = np.isfinite(pct) | |
| 182 | + acc[ok] = np.nan_to_num(acc[ok], nan=0.0) + pct[ok] | |
| 183 | + cnt[ok] += 1 | |
| 184 | + used_indicators.append(part["indicator"]) | |
| 185 | + with np.errstate(invalid="ignore", divide="ignore"): | |
| 186 | + dims_pct[dim] = np.where(cnt > 0, acc / np.where(cnt > 0, cnt, 1), np.nan) | |
| 187 | + rows = [] | |
| 188 | + for i, c in enumerate(countries): | |
| 189 | + dims = {dim: (round(float(arr[i]), 1) if np.isfinite(arr[i]) else None) for dim, arr in dims_pct.items()} | |
| 190 | + if all(v is None for v in dims.values()): | |
| 191 | + continue | |
| 192 | + ys = [years[(c, ind)] for ind in used_indicators if (c, ind) in years] | |
| 193 | + rows.append({"country_id": c, "dims": json.dumps(dims), "year_ref": max(ys) if ys else None}) | |
| 194 | + schema = {"country_id": pl.Utf8, "dims": pl.Utf8, "year_ref": pl.Int32} | |
| 195 | + return pl.DataFrame(rows, schema=schema) if rows else pl.DataFrame(schema=schema) | |
added
src/countryatlas/pipeline/staging.py
+221 −0
@@ -0,0 +1,221 @@ | ||
| 1 | +"""Staging area: one parquet per (connector, dataset, code, indicator) spec + sidecars. | |
| 2 | + | |
| 3 | +staging/<connector>/<dataset>__<code>__<indicator>.parquet NormalizedObservation columns (metadata as JSON text) | |
| 4 | +staging/<connector>/<dataset>__<code>__<indicator>.run.json ImportRun of the last attempt (ok|failed|quarantined|partial) | |
| 5 | +staging/<connector>/<dataset>__<code>__<indicator>.issues.json validation issues of the last successful write | |
| 6 | +staging/<connector>/<dataset>__<code>__<indicator>.meta.json source_url / notes / source_updated_at for indicator_sources | |
| 7 | + | |
| 8 | +Also: reconstruction of RawPayload objects from raw/ (for `ca normalize` without re-download). | |
| 9 | +""" | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import gzip | |
| 13 | +import json | |
| 14 | +import logging | |
| 15 | +import os | |
| 16 | +from datetime import UTC, datetime | |
| 17 | +from pathlib import Path | |
| 18 | +from typing import Any | |
| 19 | + | |
| 20 | +import polars as pl | |
| 21 | + | |
| 22 | +from countryatlas.config import settings | |
| 23 | +from countryatlas.models import ImportRun, IndicatorSourceSpec, NormalizedObservation, RawPayload, ValidationIssue | |
| 24 | + | |
| 25 | +log = logging.getLogger(__name__) | |
| 26 | + | |
| 27 | +STAGING_SCHEMA: dict[str, pl.DataType] = { | |
| 28 | + "country_id": pl.Utf8, | |
| 29 | + "indicator_id": pl.Utf8, | |
| 30 | + "period": pl.Date, | |
| 31 | + "year": pl.Int32, | |
| 32 | + "frequency": pl.Utf8, | |
| 33 | + "value": pl.Float64, | |
| 34 | + "unit": pl.Utf8, | |
| 35 | + "source_id": pl.Utf8, | |
| 36 | + "source_dataset": pl.Utf8, | |
| 37 | + "source_series_code": pl.Utf8, | |
| 38 | + "is_estimate": pl.Boolean, | |
| 39 | + "is_forecast": pl.Boolean, | |
| 40 | + "retrieved_at": pl.Datetime("us"), | |
| 41 | + "source_updated_at": pl.Datetime("us"), | |
| 42 | + "status": pl.Utf8, | |
| 43 | + "metadata": pl.Utf8, | |
| 44 | +} | |
| 45 | + | |
| 46 | + | |
| 47 | +def safe_name(s: str) -> str: | |
| 48 | + return "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in s) or "x" | |
| 49 | + | |
| 50 | + | |
| 51 | +def spec_stem(spec: IndicatorSourceSpec) -> str: | |
| 52 | + return f"{safe_name(spec.dataset or 'default')}__{safe_name(spec.code)}__{safe_name(spec.indicator_id)}" | |
| 53 | + | |
| 54 | + | |
| 55 | +def spec_paths(spec: IndicatorSourceSpec) -> dict[str, Path]: | |
| 56 | + d = settings.staging_dir / spec.connector | |
| 57 | + stem = spec_stem(spec) | |
| 58 | + return { | |
| 59 | + "parquet": d / f"{stem}.parquet", | |
| 60 | + "run": d / f"{stem}.run.json", | |
| 61 | + "issues": d / f"{stem}.issues.json", | |
| 62 | + "meta": d / f"{stem}.meta.json", | |
| 63 | + } | |
| 64 | + | |
| 65 | + | |
| 66 | +def _naive_utc(dt: datetime | None) -> datetime | None: | |
| 67 | + if dt is None: | |
| 68 | + return None | |
| 69 | + if dt.tzinfo is not None: | |
| 70 | + dt = dt.astimezone(UTC).replace(tzinfo=None) | |
| 71 | + return dt | |
| 72 | + | |
| 73 | + | |
| 74 | +def rows_to_frame(rows: list[NormalizedObservation]) -> pl.DataFrame: | |
| 75 | + """NormalizedObservation list → polars frame with the staging schema (metadata serialised as JSON text).""" | |
| 76 | + if not rows: | |
| 77 | + return pl.DataFrame(schema=STAGING_SCHEMA) | |
| 78 | + data = { | |
| 79 | + "country_id": [r.country_id for r in rows], | |
| 80 | + "indicator_id": [r.indicator_id for r in rows], | |
| 81 | + "period": [r.period for r in rows], | |
| 82 | + "year": [r.year for r in rows], | |
| 83 | + "frequency": [r.frequency for r in rows], | |
| 84 | + "value": [r.value for r in rows], | |
| 85 | + "unit": [r.unit for r in rows], | |
| 86 | + "source_id": [r.source_id for r in rows], | |
| 87 | + "source_dataset": [r.source_dataset for r in rows], | |
| 88 | + "source_series_code": [r.source_series_code for r in rows], | |
| 89 | + "is_estimate": [r.is_estimate for r in rows], | |
| 90 | + "is_forecast": [r.is_forecast for r in rows], | |
| 91 | + "retrieved_at": [_naive_utc(r.retrieved_at) for r in rows], | |
| 92 | + "source_updated_at": [_naive_utc(r.source_updated_at) for r in rows], | |
| 93 | + "status": [r.status for r in rows], | |
| 94 | + "metadata": [json.dumps(r.metadata, default=str) if r.metadata else None for r in rows], | |
| 95 | + } | |
| 96 | + return pl.DataFrame(data, schema=STAGING_SCHEMA) | |
| 97 | + | |
| 98 | + | |
| 99 | +def write_parquet_atomic(df: pl.DataFrame, path: Path) -> None: | |
| 100 | + path.parent.mkdir(parents=True, exist_ok=True) | |
| 101 | + tmp = path.with_suffix(path.suffix + ".tmp") | |
| 102 | + df.write_parquet(tmp, compression="zstd") | |
| 103 | + os.replace(tmp, path) | |
| 104 | + | |
| 105 | + | |
| 106 | +def write_json(path: Path, obj: Any) -> None: | |
| 107 | + path.parent.mkdir(parents=True, exist_ok=True) | |
| 108 | + tmp = path.with_suffix(path.suffix + ".tmp") | |
| 109 | + tmp.write_text(json.dumps(obj, default=str, ensure_ascii=False, indent=1)) | |
| 110 | + os.replace(tmp, path) | |
| 111 | + | |
| 112 | + | |
| 113 | +def write_run(spec: IndicatorSourceSpec, run: ImportRun) -> None: | |
| 114 | + write_json(spec_paths(spec)["run"], run.model_dump(mode="json")) | |
| 115 | + | |
| 116 | + | |
| 117 | +def write_issues(spec: IndicatorSourceSpec, issues: list[ValidationIssue], cap: int = 5000) -> None: | |
| 118 | + payload = { | |
| 119 | + "n_issues": len(issues), | |
| 120 | + "truncated": len(issues) > cap, | |
| 121 | + "issues": [i.model_dump(mode="json") for i in issues[:cap]], | |
| 122 | + } | |
| 123 | + write_json(spec_paths(spec)["issues"], payload) | |
| 124 | + | |
| 125 | + | |
| 126 | +def write_meta(spec: IndicatorSourceSpec, meta: dict[str, Any]) -> None: | |
| 127 | + write_json(spec_paths(spec)["meta"], meta) | |
| 128 | + | |
| 129 | + | |
| 130 | +def read_previous_rows(spec: IndicatorSourceSpec) -> int | None: | |
| 131 | + p = spec_paths(spec)["parquet"] | |
| 132 | + if not p.exists(): | |
| 133 | + return None | |
| 134 | + try: | |
| 135 | + return pl.scan_parquet(p).select(pl.len()).collect().item() | |
| 136 | + except Exception: # noqa: BLE001 | |
| 137 | + return None | |
| 138 | + | |
| 139 | + | |
| 140 | +def list_staging_files(connector: str | None = None) -> list[Path]: | |
| 141 | + base = settings.staging_dir | |
| 142 | + if not base.exists(): | |
| 143 | + return [] | |
| 144 | + pattern = f"{connector}/*.parquet" if connector else "*/*.parquet" | |
| 145 | + return sorted(p for p in base.glob(pattern) if not p.name.endswith(".tmp")) | |
| 146 | + | |
| 147 | + | |
| 148 | +def read_json(path: Path) -> Any | None: | |
| 149 | + if not path.exists(): | |
| 150 | + return None | |
| 151 | + try: | |
| 152 | + return json.loads(path.read_text()) | |
| 153 | + except Exception: # noqa: BLE001 | |
| 154 | + return None | |
| 155 | + | |
| 156 | + | |
| 157 | +def list_runs(connector: str | None = None) -> list[ImportRun]: | |
| 158 | + base = settings.staging_dir | |
| 159 | + if not base.exists(): | |
| 160 | + return [] | |
| 161 | + pattern = f"{connector}/*.run.json" if connector else "*/*.run.json" | |
| 162 | + out = [] | |
| 163 | + for p in sorted(base.glob(pattern)): | |
| 164 | + doc = read_json(p) | |
| 165 | + if doc: | |
| 166 | + try: | |
| 167 | + out.append(ImportRun(**doc)) | |
| 168 | + except Exception as e: # noqa: BLE001 | |
| 169 | + log.warning("unreadable run sidecar %s: %s", p, e) | |
| 170 | + return out | |
| 171 | + | |
| 172 | + | |
| 173 | +# --------------------------------------------------------------------------------------------- raw reconstruction | |
| 174 | +def _is_hash(s: str) -> bool: | |
| 175 | + return len(s) == 10 and all(c in "0123456789abcdef" for c in s) | |
| 176 | + | |
| 177 | + | |
| 178 | +def latest_raw_payloads(connector_id: str, spec: IndicatorSourceSpec, raw_code: str | None = None) -> list[RawPayload]: | |
| 179 | + """Rebuild the RawPayload list of the most recent day for this spec from raw/<connector>/<dataset>/<day>/… | |
| 180 | + | |
| 181 | + `raw_code` overrides the filename prefix when the connector stores a shared file under another code (OWID co2/energy). | |
| 182 | + Pages are ordered by `meta.page` when present. | |
| 183 | + """ | |
| 184 | + d = settings.raw_dir / connector_id / (spec.dataset or "default") | |
| 185 | + if not d.exists(): | |
| 186 | + return [] | |
| 187 | + prefix = safe_name(raw_code or spec.code) + "-" | |
| 188 | + days = sorted((x for x in d.iterdir() if x.is_dir()), reverse=True) | |
| 189 | + for day in days: | |
| 190 | + # `prefix` may also be a prefix of another code (e.g. "median-age-" vs "median-age-projected-"): the hash part | |
| 191 | + # written by store_raw is exactly 10 hex chars followed by the extension, so keep only files of that shape. | |
| 192 | + files = sorted(x for x in day.glob(f"{prefix}*.gz") if _is_hash(x.name[len(prefix):].split(".")[0])) | |
| 193 | + if not files: | |
| 194 | + continue | |
| 195 | + out: list[RawPayload] = [] | |
| 196 | + for f in files: | |
| 197 | + sidecar = f.with_suffix(".meta.json") | |
| 198 | + info = read_json(sidecar) or {} | |
| 199 | + with gzip.open(f, "rb") as fh: | |
| 200 | + body = fh.read() | |
| 201 | + ra = info.get("retrieved_at") | |
| 202 | + retrieved = datetime.fromisoformat(ra) if ra else datetime.fromtimestamp(f.stat().st_mtime, tz=UTC) | |
| 203 | + su = info.get("source_updated_at") | |
| 204 | + out.append( | |
| 205 | + RawPayload( | |
| 206 | + connector=connector_id, | |
| 207 | + dataset=spec.dataset or "default", | |
| 208 | + code=raw_code or spec.code, | |
| 209 | + url=info.get("url", ""), | |
| 210 | + retrieved_at=retrieved, | |
| 211 | + status_code=int(info.get("status_code") or 200), | |
| 212 | + content_type=info.get("content_type") or ("text/csv" if f.name.endswith(".csv.gz") else "application/json"), | |
| 213 | + body=body, | |
| 214 | + source_updated_at=datetime.fromisoformat(su) if su else None, | |
| 215 | + pages=int(info.get("pages") or 1), | |
| 216 | + meta=info.get("meta") or {}, | |
| 217 | + ) | |
| 218 | + ) | |
| 219 | + out.sort(key=lambda p: int((p.meta or {}).get("page") or 1)) | |
| 220 | + return out | |
| 221 | + return [] | |
added
src/countryatlas/pipeline/validate.py
+200 −0
@@ -0,0 +1,200 @@ | ||
| 1 | +"""Generic, deterministic validation rules (ARCHITECTURE §6) applied to one staging frame (one spec). | |
| 2 | + | |
| 3 | +Row-level outcomes only ever *flag* (status column); nothing is deleted: | |
| 4 | + out_of_bounds value outside registry `bounds` → status quarantined | |
| 5 | + extreme_jump |Δ| > jump_threshold × 1.4826·MAD(country diffs), ≥5 points → status warning | |
| 6 | + stale the country's latest period ended > stale_after_days ago → status stale (that latest row only) | |
| 7 | +Dataset-level outcomes (quarantine_dataset=True → the pipeline keeps the previous staging file): | |
| 8 | + duplicate same key twice (connector.validate already errors on this) | |
| 9 | + unit_mismatch rows whose `unit` differs from the registry unit | |
| 10 | + partial_download rows < 30 % of the previous staging file for the same spec | |
| 11 | +""" | |
| 12 | +from __future__ import annotations | |
| 13 | + | |
| 14 | +import logging | |
| 15 | +from dataclasses import dataclass, field | |
| 16 | +from datetime import UTC, date, datetime, timedelta | |
| 17 | + | |
| 18 | +import polars as pl | |
| 19 | + | |
| 20 | +from countryatlas.models import ValidationIssue | |
| 21 | +from countryatlas.registry import Indicator | |
| 22 | + | |
| 23 | +log = logging.getLogger(__name__) | |
| 24 | + | |
| 25 | +PARTIAL_RATIO = 0.30 | |
| 26 | +MAD_SCALE = 1.4826 | |
| 27 | +MIN_POINTS_FOR_JUMP = 5 | |
| 28 | +MAX_ROW_ISSUES = 2000 # per code per spec, to keep validation_issues manageable | |
| 29 | +LOG_FLOOR = 0.10 # log-diff floor ≈ 10 % relative change | |
| 30 | +RANGE_FLOOR = 0.02 # absolute-diff floor = 2 % of the country's series range | |
| 31 | +# formats whose level series grow multiplicatively → compare on log-differences when the registry lower bound is ≥ 0 | |
| 32 | +LOG_FORMATS = {"currency", "number", "tonnes", "kwh", "per_1000", "per_100k", "per_million", "km", "ha"} | |
| 33 | + | |
| 34 | + | |
| 35 | +def uses_log_diffs(indicator: Indicator) -> bool: | |
| 36 | + lo = (indicator.bounds or [None, None])[0] | |
| 37 | + return indicator.format in LOG_FORMATS and lo is not None and float(lo) >= 0 | |
| 38 | + | |
| 39 | + | |
| 40 | +@dataclass | |
| 41 | +class GenericValidation: | |
| 42 | + frame: pl.DataFrame | |
| 43 | + issues: list[ValidationIssue] = field(default_factory=list) | |
| 44 | + quarantine_dataset: bool = False | |
| 45 | + message: str | None = None | |
| 46 | + n_quarantined: int = 0 | |
| 47 | + n_warning: int = 0 | |
| 48 | + n_stale: int = 0 | |
| 49 | + | |
| 50 | + @property | |
| 51 | + def errors(self) -> int: | |
| 52 | + return sum(1 for i in self.issues if i.severity == "error") | |
| 53 | + | |
| 54 | + @property | |
| 55 | + def warnings(self) -> int: | |
| 56 | + return sum(1 for i in self.issues if i.severity == "warning") | |
| 57 | + | |
| 58 | + | |
| 59 | +def _period_end(freq: str) -> pl.Expr: | |
| 60 | + """Last day covered by the period (annual 2024-01-01 → 2024-12-31).""" | |
| 61 | + p = pl.col("period") | |
| 62 | + if freq == "M": | |
| 63 | + return p.dt.offset_by("1mo") - pl.duration(days=1) | |
| 64 | + if freq == "Q": | |
| 65 | + return p.dt.offset_by("3mo") - pl.duration(days=1) | |
| 66 | + return p.dt.offset_by("1y") - pl.duration(days=1) | |
| 67 | + | |
| 68 | + | |
| 69 | +def validate_frame( | |
| 70 | + df: pl.DataFrame, | |
| 71 | + indicator: Indicator, | |
| 72 | + previous_rows: int | None = None, | |
| 73 | + now: datetime | None = None, | |
| 74 | + check_partial: bool = True, | |
| 75 | +) -> GenericValidation: | |
| 76 | + """Apply the generic rules and return the frame with updated `status` + issues.""" | |
| 77 | + now = now or datetime.now(UTC) | |
| 78 | + issues: list[ValidationIssue] = [] | |
| 79 | + quarantine = False | |
| 80 | + message: str | None = None | |
| 81 | + ind_id = indicator.slug | |
| 82 | + n = df.height | |
| 83 | + | |
| 84 | + if n == 0: | |
| 85 | + return GenericValidation(frame=df, issues=issues, quarantine_dataset=False, message="no rows") | |
| 86 | + | |
| 87 | + # --- dataset-level: unit mismatch ------------------------------------------------------------------------- | |
| 88 | + units = df.get_column("unit").drop_nulls().unique().to_list() | |
| 89 | + bad_units = [u for u in units if u != indicator.unit] | |
| 90 | + if bad_units: | |
| 91 | + quarantine = True | |
| 92 | + message = f"unit mismatch: {bad_units} ≠ registry '{indicator.unit}'" | |
| 93 | + issues.append(ValidationIssue(severity="error", code="unit_mismatch", message=message, indicator_id=ind_id)) | |
| 94 | + | |
| 95 | + # --- dataset-level: partial download ---------------------------------------------------------------------- | |
| 96 | + if check_partial and previous_rows and previous_rows > 0 and n < PARTIAL_RATIO * previous_rows: | |
| 97 | + quarantine = True | |
| 98 | + message = f"partial download: {n} rows < {PARTIAL_RATIO:.0%} of previous {previous_rows}" | |
| 99 | + issues.append(ValidationIssue(severity="error", code="partial_download", message=message, indicator_id=ind_id)) | |
| 100 | + | |
| 101 | + # --- dataset-level: duplicates (defensive; connector.validate already checks) ------------------------------ | |
| 102 | + dup = df.group_by(["country_id", "period", "frequency"]).len().filter(pl.col("len") > 1) | |
| 103 | + if dup.height: | |
| 104 | + quarantine = True | |
| 105 | + message = f"{dup.height} duplicate keys" | |
| 106 | + for r in dup.head(MAX_ROW_ISSUES).iter_rows(named=True): | |
| 107 | + issues.append( | |
| 108 | + ValidationIssue(severity="error", code="duplicate", message="duplicate key", indicator_id=ind_id, | |
| 109 | + country_id=r["country_id"], period=r["period"]) | |
| 110 | + ) | |
| 111 | + | |
| 112 | + # --- row-level: bounds ------------------------------------------------------------------------------------- | |
| 113 | + lo, hi = (indicator.bounds or [None, None])[:2] | |
| 114 | + oob = pl.lit(False) | |
| 115 | + if lo is not None: | |
| 116 | + oob = oob | (pl.col("value") < float(lo)) | |
| 117 | + if hi is not None: | |
| 118 | + oob = oob | (pl.col("value") > float(hi)) | |
| 119 | + oob = oob | ~pl.col("value").is_finite() | |
| 120 | + df = df.with_columns(oob.alias("_oob")) | |
| 121 | + | |
| 122 | + # --- row-level: extreme jumps (per country series, robust MAD of first differences) ----------------------- | |
| 123 | + # Level series that grow multiplicatively (GDP, population, emissions…) are compared on log-differences so that a | |
| 124 | + # growing series is not flagged just because recent absolute steps dwarf early ones. Rates/shares use absolute | |
| 125 | + # differences. A minimum floor (10 % relative, or 2 % of the series range) avoids flagging smooth series whose | |
| 126 | + # MAD is tiny (e.g. median age moving by 0.3 year instead of 0.2). | |
| 127 | + df = df.sort(["country_id", "period"]) | |
| 128 | + grp = "country_id" | |
| 129 | + log_mode = uses_log_diffs(indicator) | |
| 130 | + working = pl.when(pl.col("value") > 0).then(pl.col("value").log()) if log_mode else pl.col("value") | |
| 131 | + k = float(indicator.jump_threshold or 4.0) * MAD_SCALE | |
| 132 | + df = ( | |
| 133 | + df.with_columns(working.alias("_w"), pl.col("value").count().over(grp).alias("_n")) | |
| 134 | + .with_columns(pl.col("_w").diff().over(grp).alias("_diff")) | |
| 135 | + .with_columns(pl.col("_diff").median().over(grp).alias("_med")) | |
| 136 | + .with_columns( | |
| 137 | + (pl.col("_diff") - pl.col("_med")).abs().median().over(grp).alias("_mad"), | |
| 138 | + (pl.col("_w").max().over(grp) - pl.col("_w").min().over(grp)).alias("_range"), | |
| 139 | + ) | |
| 140 | + .with_columns( | |
| 141 | + pl.max_horizontal(k * pl.col("_mad"), pl.lit(LOG_FLOOR) if log_mode else RANGE_FLOOR * pl.col("_range")).alias("_thr") | |
| 142 | + ) | |
| 143 | + ) | |
| 144 | + jump = ( | |
| 145 | + (pl.col("_n") >= MIN_POINTS_FOR_JUMP) | |
| 146 | + & (pl.col("_mad") > 0) | |
| 147 | + & pl.col("_diff").is_not_null() | |
| 148 | + & (pl.col("_diff").abs() > pl.col("_thr")) | |
| 149 | + ) | |
| 150 | + df = df.with_columns(jump.fill_null(False).alias("_jump")).drop(["_n", "_med", "_mad", "_w", "_range"]) | |
| 151 | + | |
| 152 | + # --- row-level: stale (country's latest period ended too long ago) ---------------------------------------- | |
| 153 | + freq = indicator.frequency | |
| 154 | + stale_days = int(indicator.stale_after_days or 800) | |
| 155 | + cutoff = (now - timedelta(days=stale_days)).date() | |
| 156 | + is_latest = pl.col("period") == pl.col("period").max().over(grp) | |
| 157 | + stale = is_latest & (_period_end(freq) < pl.lit(cutoff)) | |
| 158 | + df = df.with_columns(stale.alias("_stale")) | |
| 159 | + | |
| 160 | + # --- statuses: quarantined > warning > stale > imported ----------------------------------------------------- | |
| 161 | + df = df.with_columns( | |
| 162 | + pl.when(pl.col("_oob")) | |
| 163 | + .then(pl.lit("quarantined")) | |
| 164 | + .when(pl.col("_jump")) | |
| 165 | + .then(pl.lit("warning")) | |
| 166 | + .when(pl.col("_stale")) | |
| 167 | + .then(pl.lit("stale")) | |
| 168 | + .otherwise(pl.lit("imported")) | |
| 169 | + .alias("status") | |
| 170 | + ) | |
| 171 | + n_q = int(df.get_column("_oob").sum()) | |
| 172 | + n_w = int((df.get_column("_jump") & ~df.get_column("_oob")).sum()) | |
| 173 | + n_s = int((df.get_column("_stale") & ~df.get_column("_oob") & ~df.get_column("_jump")).sum()) | |
| 174 | + | |
| 175 | + for r in df.filter(pl.col("_oob")).head(MAX_ROW_ISSUES).iter_rows(named=True): | |
| 176 | + issues.append( | |
| 177 | + ValidationIssue(severity="warning", code="out_of_bounds", indicator_id=ind_id, country_id=r["country_id"], | |
| 178 | + period=r["period"], message=f"value {r['value']!r} outside bounds [{lo}, {hi}]") | |
| 179 | + ) | |
| 180 | + for r in df.filter(pl.col("_jump") & ~pl.col("_oob")).head(MAX_ROW_ISSUES).iter_rows(named=True): | |
| 181 | + issues.append( | |
| 182 | + ValidationIssue(severity="warning", code="extreme_jump", indicator_id=ind_id, country_id=r["country_id"], | |
| 183 | + period=r["period"], | |
| 184 | + message=f"Δ={r['_diff']:.4g} exceeds {indicator.jump_threshold}×MAD threshold {r['_thr']:.4g}") | |
| 185 | + ) | |
| 186 | + if n_s: | |
| 187 | + issues.append( | |
| 188 | + ValidationIssue(severity="info", code="stale", indicator_id=ind_id, | |
| 189 | + message=f"{n_s} countries whose latest period ended before {cutoff} ({stale_days} d)") | |
| 190 | + ) | |
| 191 | + latest_period: date = df.get_column("period").max() # type: ignore[assignment] | |
| 192 | + if latest_period is not None and (latest_period + timedelta(days=365)) < cutoff: | |
| 193 | + issues.append( | |
| 194 | + ValidationIssue(severity="warning", code="stale", indicator_id=ind_id, | |
| 195 | + message=f"dataset latest period {latest_period} is older than {stale_days} days") | |
| 196 | + ) | |
| 197 | + | |
| 198 | + out = df.drop(["_oob", "_jump", "_diff", "_thr", "_stale"]) | |
| 199 | + return GenericValidation(frame=out, issues=issues, quarantine_dataset=quarantine, message=message, | |
| 200 | + n_quarantined=n_q, n_warning=n_w, n_stale=n_s) | |
modified
src/countryatlas/registry/__init__.py
+35 −0
@@ -209,11 +209,46 @@ def indicators() -> list[Indicator]: | ||
| 209 | 209 | if s["connector"] not in CONNECTORS: |
| 210 | 210 | raise ValueError(f"indicator {i['slug']}: unknown connector {s['connector']}") |
| 211 | 211 | specs.append(IndicatorSourceSpec(indicator_id=i["slug"], **{k: v for k, v in s.items()})) |
| 212 | + for s in _extra_sources().get(i["slug"], []): | |
| 213 | + specs.append(IndicatorSourceSpec(indicator_id=i["slug"], **s)) | |
| 212 | 214 | specs.sort(key=lambda s: s.priority) |
| 213 | 215 | out.append(Indicator(**d, sources=specs)) |
| 214 | 216 | return out |
| 215 | 217 | |
| 216 | 218 | |
| 219 | +@lru_cache(maxsize=1) | |
| 220 | +def _extra_sources() -> dict[str, list[dict[str, Any]]]: | |
| 221 | + """Per-connector mapping files `registry/sources/<connector>.yaml`: | |
| 222 | + | |
| 223 | + sources: | |
| 224 | + - {indicator: gdp, dataset: WEO, code: NGDPD, priority: 2, transform: "x*1e9"} | |
| 225 | + | |
| 226 | + The connector id defaults to the file name. Lets each connector own its mappings without editing indicators.yaml. | |
| 227 | + Unknown indicator slugs are ignored with a warning printed to stderr. | |
| 228 | + """ | |
| 229 | + import sys | |
| 230 | + | |
| 231 | + out: dict[str, list[dict[str, Any]]] = {} | |
| 232 | + d = settings.registry_dir / "sources" | |
| 233 | + if not d.exists(): | |
| 234 | + return out | |
| 235 | + known = {i["slug"] for i in _read("indicators.yaml")["indicators"]} | |
| 236 | + for f in sorted(d.glob("*.yaml")): | |
| 237 | + connector = f.stem | |
| 238 | + if connector not in CONNECTORS: | |
| 239 | + raise ValueError(f"registry/sources/{f.name}: unknown connector {connector}") | |
| 240 | + data = yaml.safe_load(f.read_text()) or {} | |
| 241 | + for s in data.get("sources", []) or []: | |
| 242 | + s = dict(s) | |
| 243 | + slug = s.pop("indicator") | |
| 244 | + if slug not in known: | |
| 245 | + print(f"[registry] sources/{f.name}: unknown indicator {slug} (ignored)", file=sys.stderr) | |
| 246 | + continue | |
| 247 | + s.setdefault("connector", connector) | |
| 248 | + out.setdefault(slug, []).append(s) | |
| 249 | + return out | |
| 250 | + | |
| 251 | + | |
| 217 | 252 | @lru_cache(maxsize=1) |
| 218 | 253 | def indicators_by_id() -> dict[str, Indicator]: |
| 219 | 254 | return {i.slug: i for i in indicators()} |
added
tests/api/__init__.py
+0 −0
added
tests/api/conftest.py
+66 −0
@@ -0,0 +1,66 @@ | ||
| 1 | +"""Shared fixtures: a synthetic DuckDB snapshot + FastAPI TestClient (rate limit disabled, fresh cache).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import os | |
| 5 | +import warnings | |
| 6 | +from pathlib import Path | |
| 7 | + | |
| 8 | +import pytest | |
| 9 | + | |
| 10 | +warnings.filterwarnings("ignore", message=".*httpx2.*") | |
| 11 | +os.environ.setdefault("CA_ADMIN_TOKEN", "test-admin-token") | |
| 12 | + | |
| 13 | +from fastapi.testclient import TestClient | |
| 14 | + | |
| 15 | +from countryatlas.api.cache import ResponseCache | |
| 16 | +from countryatlas.api.main import create_app | |
| 17 | +from countryatlas.config import settings | |
| 18 | +from tests.fixtures.make_fixture_db import COUNTRIES, INDICATORS, RUN_ID, build, value | |
| 19 | + | |
| 20 | +settings.admin_token = "test-admin-token" | |
| 21 | + | |
| 22 | +PREFIX = "/api/v1" | |
| 23 | + | |
| 24 | + | |
| 25 | +@pytest.fixture(scope="session") | |
| 26 | +def fixture_db(tmp_path_factory: pytest.TempPathFactory) -> Path: | |
| 27 | + return build(tmp_path_factory.mktemp("atlas") / "atlas.duckdb") | |
| 28 | + | |
| 29 | + | |
| 30 | +@pytest.fixture(scope="session") | |
| 31 | +def app(fixture_db: Path): | |
| 32 | + return create_app(fixture_db, rate_limit_per_minute=0, cache=ResponseCache()) | |
| 33 | + | |
| 34 | + | |
| 35 | +@pytest.fixture(scope="session") | |
| 36 | +def client(app) -> TestClient: | |
| 37 | + with TestClient(app, raise_server_exceptions=False) as c: | |
| 38 | + yield c | |
| 39 | + | |
| 40 | + | |
| 41 | +@pytest.fixture | |
| 42 | +def get(client: TestClient): | |
| 43 | + def _get(path: str, status: int = 200, **kw): | |
| 44 | + r = client.get(PREFIX + path, **kw) | |
| 45 | + assert r.status_code == status, f"{path} → {r.status_code}: {r.text[:400]}" | |
| 46 | + return r | |
| 47 | + | |
| 48 | + return _get | |
| 49 | + | |
| 50 | + | |
| 51 | +def assert_provenance(p: dict) -> None: | |
| 52 | + assert p is not None, "missing provenance" | |
| 53 | + for k in ("source", "source_name", "dataset", "series_code", "retrieved_at", "source_updated_at", "url", "transform", "licence"): | |
| 54 | + assert k in p, f"provenance missing {k}" | |
| 55 | + assert p["source"] in ("worldbank", "imf", "owid", "who") | |
| 56 | + assert p["url"].startswith("http") | |
| 57 | + assert p["retrieved_at"] | |
| 58 | + | |
| 59 | + | |
| 60 | +def assert_meta(body: dict) -> None: | |
| 61 | + assert body["meta"]["run_id"] == RUN_ID | |
| 62 | + assert body["meta"]["built_at"] == "2026-09-11T00:00:00Z" | |
| 63 | + assert body["meta"]["generated_at"].endswith("Z") | |
| 64 | + | |
| 65 | + | |
| 66 | +__all__ = ["COUNTRIES", "INDICATORS", "PREFIX", "RUN_ID", "assert_meta", "assert_provenance", "value"] | |
added
tests/api/test_countries.py
+153 −0
@@ -0,0 +1,153 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +import csv | |
| 4 | +import io | |
| 5 | + | |
| 6 | +import pytest | |
| 7 | + | |
| 8 | +from tests.api.conftest import COUNTRIES, assert_meta, assert_provenance, value | |
| 9 | + | |
| 10 | + | |
| 11 | +def test_list_countries(get): | |
| 12 | + body = get("/countries").json() | |
| 13 | + assert_meta(body) | |
| 14 | + assert body["n"] == len(COUNTRIES) | |
| 15 | + ids = [c["id"] for c in body["items"]] | |
| 16 | + assert ids == sorted(ids, key=lambda i: next(c["name"] for c in body["items"] if c["id"] == i)) | |
| 17 | + can = next(c for c in body["items"] if c["id"] == "CAN") | |
| 18 | + assert can["slug"] == "canada" and can["flag"] and can["region_name"] == "North America" and can["income"] == "HIC" | |
| 19 | + assert can["population_latest"] == pytest.approx(value("CAN", "population", 2024)) | |
| 20 | + assert can["gdp_per_capita_latest"] == pytest.approx(value("CAN", "gdp-per-capita", 2024)) | |
| 21 | + assert can["coverage_pct"] == 100.0 | |
| 22 | + | |
| 23 | + | |
| 24 | +def test_list_countries_filters_and_sort(get): | |
| 25 | + g7 = get("/countries?region=g7&sort=gdp").json() | |
| 26 | + assert {c["id"] for c in g7["items"]} == {"CAN", "USA", "FRA", "DEU", "JPN"} | |
| 27 | + vals = [c["gdp_latest"] for c in g7["items"]] | |
| 28 | + assert vals == sorted(vals, reverse=True) | |
| 29 | + assert get("/countries?region=north-america").json()["n"] == 2 | |
| 30 | + assert {c["id"] for c in get("/countries?income=LMC").json()["items"]} == {"IND", "NGA"} | |
| 31 | + assert get("/countries?q=jap").json()["items"][0]["id"] == "JPN" | |
| 32 | + assert get("/countries?region=nowhere", status=404) | |
| 33 | + assert get("/countries?sort=bogus", status=422) | |
| 34 | + | |
| 35 | + | |
| 36 | +def test_country_overview(get): | |
| 37 | + body = get("/countries/canada").json() | |
| 38 | + assert_meta(body) | |
| 39 | + c = body["country"] | |
| 40 | + assert c["id"] == "CAN" and c["iso2"] == "CA" and c["capital"] == "Ottawa" and c["borders"] == ["USA"] | |
| 41 | + assert "g7" in {g["id"] for g in body["groups"]} and "world" in {g["id"] for g in body["groups"]} | |
| 42 | + assert body["coverage"]["n_indicators"] == 14 | |
| 43 | + assert body["freshness"]["source_updated_at"].startswith("2026-07-01") and body["freshness"]["built_at"] | |
| 44 | + headline = {m["indicator"]: m for m in body["headline"]} | |
| 45 | + assert list(headline) == ["population", "gdp", "gdp-per-capita", "gdp-growth", "inflation", "unemployment-rate", "life-expectancy", | |
| 46 | + "median-age", "government-debt-pct-gdp", "co2-per-capita", "renewable-electricity-share", "internet-users"] | |
| 47 | + gpc = headline["gdp-per-capita"] | |
| 48 | + assert gpc["has_data"] and gpc["year"] == 2024 and gpc["value"] == pytest.approx(value("CAN", "gdp-per-capita", 2024)) | |
| 49 | + assert gpc["change"]["pct"] == pytest.approx(3.0) and gpc["change"]["formatted"] == "+3.0 %" | |
| 50 | + assert gpc["rank_world"] and gpc["n_world"] == 8 and gpc["rank_region"] and gpc["n_region"] == 2 | |
| 51 | + assert gpc["formatted"].endswith("k") | |
| 52 | + assert len(gpc["sparkline"]) == 30 and gpc["sparkline"][-1][0] == 2024 and isinstance(gpc["sparkline"][-1][0], int) | |
| 53 | + assert_provenance(gpc["provenance"]) | |
| 54 | + assert gpc["provenance"]["url"] == "https://data.worldbank.org/indicator/NY.GDP.PCAP.CD?locations=CA" | |
| 55 | + assert gpc["provenance"]["series_code"] == "NY.GDP.PCAP.CD" and gpc["provenance"]["licence"] == "CC BY 4.0" | |
| 56 | + # OWID provenance for co2 | |
| 57 | + assert headline["co2-per-capita"]["provenance"]["url"] == "https://github.com/owid/co2-data" | |
| 58 | + assert headline["median-age"]["provenance"]["url"] == "https://ourworldindata.org/grapher/median-age" | |
| 59 | + topics = {t["id"]: t for t in body["topics"]} | |
| 60 | + assert topics["economy"]["n_with_data"] >= 5 and topics["income"]["n_with_data"] == 0 | |
| 61 | + assert [t["order"] for t in body["topics"]] == sorted(t["order"] for t in body["topics"]) | |
| 62 | + assert body["neighbours"][0]["id"] == "USA" | |
| 63 | + | |
| 64 | + | |
| 65 | +def test_country_case_insensitive_and_iso2(get): | |
| 66 | + assert get("/countries/can").json()["country"]["id"] == "CAN" | |
| 67 | + assert get("/countries/CANADA").json()["country"]["id"] == "CAN" | |
| 68 | + assert get("/countries/ca").json()["country"]["id"] == "CAN" | |
| 69 | + | |
| 70 | + | |
| 71 | +def test_country_404_problem_json(client): | |
| 72 | + r = client.get("/api/v1/countries/atlantis") | |
| 73 | + assert r.status_code == 404 | |
| 74 | + assert r.headers["content-type"].startswith("application/problem+json") | |
| 75 | + body = r.json() | |
| 76 | + assert body["title"] == "Country not found" and body["status"] == 404 and "atlantis" in body["detail"] | |
| 77 | + assert body["instance"] == "/api/v1/countries/atlantis" | |
| 78 | + | |
| 79 | + | |
| 80 | +def test_country_topic(get): | |
| 81 | + body = get("/countries/CAN/topics/economy").json() | |
| 82 | + assert body["topic"]["id"] == "economy" and body["n_indicators"] > body["n_with_data"] > 0 | |
| 83 | + subs = [s["subtopic"] for s in body["subtopics"]] | |
| 84 | + assert subs[0] == "Output" # registry order | |
| 85 | + all_inds = [i for s in body["subtopics"] for i in s["indicators"]] | |
| 86 | + with_data = [i for i in all_inds if i["has_data"]] | |
| 87 | + without = [i for i in all_inds if not i["has_data"]] | |
| 88 | + assert with_data and without | |
| 89 | + assert without[0]["provenance"] is None and without[0]["value"] is None | |
| 90 | + for m in with_data: | |
| 91 | + assert_provenance(m["provenance"]) | |
| 92 | + assert m["sparkline"] | |
| 93 | + get("/countries/CAN/topics/astrology", status=404) | |
| 94 | + | |
| 95 | + | |
| 96 | +def test_country_series(get): | |
| 97 | + body = get("/countries/CAN/series/gdp?from=2010&to=2026&include_alt=true").json() | |
| 98 | + assert_meta(body) | |
| 99 | + assert body["indicator"]["id"] == "gdp" and body["country"]["id"] == "CAN" and body["unit"] == "current US$" | |
| 100 | + years = [v["year"] for v in body["values"]] | |
| 101 | + assert years == list(range(2010, 2027)) | |
| 102 | + fc = [v for v in body["values"] if v["is_forecast"]] | |
| 103 | + assert [v["year"] for v in fc] == [2025, 2026] and fc[0]["source_id"] == "imf" | |
| 104 | + assert body["values"][0]["value"] == pytest.approx(value("CAN", "gdp", 2010)) | |
| 105 | + for v in body["values"]: | |
| 106 | + assert_provenance(v["provenance"]) | |
| 107 | + assert body["provenance"]["source"] == "worldbank" | |
| 108 | + assert {s["source"] for s in body["sources"]} == {"worldbank", "imf"} | |
| 109 | + assert body["alternatives"] and body["alternatives"][0]["source_id"] == "imf" | |
| 110 | + assert body["stats"]["last"]["year"] == 2024 and body["stats"]["cagr"] == pytest.approx(4.03, abs=0.01) | |
| 111 | + no_fc = get("/countries/CAN/series/gdp?include_forecast=false").json() | |
| 112 | + assert all(not v["is_forecast"] for v in no_fc["values"]) | |
| 113 | + get("/countries/CAN/series/not-an-indicator", status=404) | |
| 114 | + get("/countries/CAN/series/gdp?from=2020&to=2010", status=400) | |
| 115 | + | |
| 116 | + | |
| 117 | +def test_country_changes_events(get): | |
| 118 | + ch = get("/countries/BRA/changes").json() | |
| 119 | + assert ch["n"] > 0 | |
| 120 | + sev = [c["severity"] for c in ch["items"]] | |
| 121 | + assert sev == sorted(sev, reverse=True) | |
| 122 | + assert ch["items"][0]["headline"] and ch["items"][0]["indicator"]["slug"] | |
| 123 | + assert_provenance(ch["items"][0]["provenance"]) | |
| 124 | + ev = get("/countries/USA/events?limit=5").json() | |
| 125 | + assert 0 < ev["n"] <= 5 and ev["items"][0]["kind"] in ("sign_flip", "yoy_jump") | |
| 126 | + get("/countries/ZZZ/changes", status=404) | |
| 127 | + | |
| 128 | + | |
| 129 | +def test_country_similar_insights_dna(get): | |
| 130 | + sim = get("/countries/CAN/similar?mode=overall").json() | |
| 131 | + assert sim["mode"] == "overall" and "economic" in sim["modes"] | |
| 132 | + assert sim["peers"][0]["country"]["id"] in ("USA", "DEU", "FRA", "JPN") and sim["peers"][0]["score"] > 50 | |
| 133 | + assert sim["peers"][0]["contributions"] | |
| 134 | + ins = get("/countries/CAN/insights").json() | |
| 135 | + assert ins["items"] and "population grew" in ins["items"][0]["text"] or "Life expectancy" in ins["items"][0]["text"] | |
| 136 | + assert any(i["provenance"] for i in ins["items"]) | |
| 137 | + dna = get("/countries/CAN/dna").json() | |
| 138 | + assert dna["year_ref"] == 2024 and 0 <= dna["dims"]["income"] <= 100 and dna["dims"]["trade"] is None | |
| 139 | + assert any(d["id"] == "income" and d["label"] == "Income" for d in dna["dimensions"]) | |
| 140 | + | |
| 141 | + | |
| 142 | +def test_country_download(client): | |
| 143 | + r = client.get("/api/v1/countries/CAN/download.csv") | |
| 144 | + assert r.status_code == 200 and r.headers["content-type"].startswith("text/csv") | |
| 145 | + assert 'filename="countryatlas-canada.csv"' in r.headers["content-disposition"] | |
| 146 | + lines = r.text.splitlines() | |
| 147 | + assert lines[0].startswith("# CountryAtlas export") | |
| 148 | + rows = list(csv.DictReader(io.StringIO("\n".join(lines[1:])))) | |
| 149 | + assert len(rows) > 400 and {"source", "series_code", "retrieved_at", "url", "licence"} <= set(rows[0]) | |
| 150 | + assert all(r["country_id"] == "CAN" for r in rows) | |
| 151 | + j = client.get("/api/v1/countries/CAN/download.json?include_forecast=false").json() | |
| 152 | + assert j["n"] == sum(1 for r in rows if r["is_forecast"] == "False") and j["rows"][0]["source"] | |
| 153 | + assert client.get("/api/v1/countries/CAN/download.xml").status_code == 400 | |
added
tests/api/test_indicators.py
+96 −0
@@ -0,0 +1,96 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +import pytest | |
| 4 | + | |
| 5 | +from tests.api.conftest import COUNTRIES, assert_meta, assert_provenance, value | |
| 6 | + | |
| 7 | + | |
| 8 | +def test_list_indicators(get): | |
| 9 | + body = get("/indicators").json() | |
| 10 | + assert_meta(body) | |
| 11 | + assert body["n"] == 14 | |
| 12 | + assert body["items"][0]["featured"] is True # featured first | |
| 13 | + gdp = next(i for i in body["items"] if i["id"] == "gdp") | |
| 14 | + assert gdp["n_countries"] == 8 and gdp["coverage_pct"] == 100.0 and gdp["last_year"] == 2024 and gdp["primary_source_id"] == "worldbank" | |
| 15 | + eco = get("/indicators?topic=economy").json() | |
| 16 | + assert [i["id"] for i in eco["items"]][:3] == ["gdp", "gdp-per-capita", "gdp-per-capita-ppp"] # topics.yaml order | |
| 17 | + assert all(i["featured"] for i in get("/indicators?featured=true").json()["items"]) | |
| 18 | + assert get("/indicators?q=infl").json()["items"][0]["id"] == "inflation" | |
| 19 | + | |
| 20 | + | |
| 21 | +def test_indicator_detail(get): | |
| 22 | + body = get("/indicators/gdp-per-capita").json() | |
| 23 | + ind = body["indicator"] | |
| 24 | + assert ind["slug"] == "gdp-per-capita" and ind["unit"] == "current US$" and ind["format"] == "currency" and ind["higher_is_better"] is True | |
| 25 | + assert ind["source_priority"] == ["worldbank", "imf"] | |
| 26 | + srcs = body["sources"] | |
| 27 | + assert [s["source_id"] for s in srcs] == ["worldbank", "imf"] and srcs[0]["series_code"] == "NY.GDP.PCAP.CD" | |
| 28 | + assert srcs[0]["url"].startswith("https://data.worldbank.org/indicator/NY.GDP.PCAP.CD") and srcs[0]["licence"] == "CC BY 4.0" | |
| 29 | + assert body["coverage"]["n_countries"] == 8 and body["coverage"]["by_year"][0]["year"] == 1990 | |
| 30 | + wl = body["world_latest"] | |
| 31 | + assert wl["kind"] == "weighted_mean" and wl["weights"] == "population" and wl["year"] == 2024 and wl["n"] == 8 | |
| 32 | + pops = {c: value(c, "population", 2024) for c in COUNTRIES} | |
| 33 | + expected = sum(value(c, "gdp-per-capita", 2024) * pops[c] for c in COUNTRIES) / sum(pops.values()) | |
| 34 | + assert wl["value"] == pytest.approx(expected, rel=1e-6) | |
| 35 | + assert body["top5"][0]["country"]["id"] == "JPN" and body["top5"][0]["rank"] == 1 | |
| 36 | + assert body["bottom5"][-1]["country"]["id"] == "IND" and body["bottom5"][-1]["rank"] == 8 # IND has the lowest synthetic GDP pc | |
| 37 | + assert_provenance(body["top5"][0]["provenance"]) | |
| 38 | + assert body["freshness"]["retrieved_at"] and body["years"] == {"first": 1990, "last": 2026, "last_actual": 2024, "latest_common": 2024} | |
| 39 | + assert "economy" in body["topics"] | |
| 40 | + | |
| 41 | + | |
| 42 | +def test_indicator_world_latest_kinds(get): | |
| 43 | + assert get("/indicators/gdp").json()["world_latest"]["kind"] == "sum" | |
| 44 | + le = get("/indicators/life-expectancy").json()["world_latest"] | |
| 45 | + assert le["kind"] == "weighted_mean" and le["median"] is not None | |
| 46 | + # lower-is-better indicator → top5 = lowest values | |
| 47 | + infl = get("/indicators/inflation").json() | |
| 48 | + assert infl["top5"][0]["value"] <= infl["top5"][-1]["value"] | |
| 49 | + | |
| 50 | + | |
| 51 | +def test_indicator_404(client): | |
| 52 | + r = client.get("/api/v1/indicators/happiness-of-cats") | |
| 53 | + assert r.status_code == 404 and r.json()["title"] == "Indicator not found" and "gdp-per-capita" in r.json()["detail"] | |
| 54 | + | |
| 55 | + | |
| 56 | +def test_indicator_map(get): | |
| 57 | + body = get("/indicators/life-expectancy/map").json() | |
| 58 | + assert body["year"] is None and body["year_used"] == 2024 and body["n"] == 8 and body["nearest"] is False | |
| 59 | + assert body["values"]["CAN"] == pytest.approx(value("CAN", "life-expectancy", 2024)) | |
| 60 | + assert body["formatted"]["CAN"].endswith("yrs") | |
| 61 | + lg = body["legend"] | |
| 62 | + assert lg["min"] < lg["max"] and lg["breaks"] == sorted(lg["breaks"]) and 3 <= lg["n_classes"] <= 7 | |
| 63 | + assert_provenance(body["provenance"]) | |
| 64 | + y23 = get("/indicators/life-expectancy/map?year=2023").json() | |
| 65 | + assert y23["year_used"] == 2023 and y23["values"]["CAN"] == pytest.approx(value("CAN", "life-expectancy", 2023)) | |
| 66 | + # internet-users has no values for NGA before 2005 → nearest within 3 years | |
| 67 | + m = get("/indicators/internet-users/map?year=2006&nearest=true").json() | |
| 68 | + assert m["years"]["NGA"] == 2006 and m["years"]["CAN"] == 2006 | |
| 69 | + m2 = get("/indicators/internet-users/map?year=2004&nearest=true").json() | |
| 70 | + assert "NGA" not in m2["values"] and m2["years"]["CAN"] == 2004 | |
| 71 | + empty = get("/indicators/internet-users/map?year=1990").json() | |
| 72 | + assert empty["n"] == 0 and empty["legend"]["breaks"] == [] | |
| 73 | + | |
| 74 | + | |
| 75 | +def test_indicator_trend(get): | |
| 76 | + body = get("/indicators/gdp/trend?group=world").json() | |
| 77 | + assert body["group"]["id"] == "world" and body["preferred"] == "sum" | |
| 78 | + p = body["points"] | |
| 79 | + assert p[0]["year"] == 1990 and p[-1]["year"] == 2024 and p[-1]["n"] == 8 | |
| 80 | + assert p[-1]["sum"] == pytest.approx(sum(value(c, "gdp", 2024) for c in COUNTRIES)) | |
| 81 | + assert p[-1]["median"] is not None and p[-1]["mean"] is not None | |
| 82 | + g7 = get("/indicators/life-expectancy/trend?group=g7&from=2020").json() | |
| 83 | + assert g7["preferred"] == "weighted_mean" and g7["weights"] == "population" and g7["points"][0]["year"] == 2020 and g7["points"][0]["n"] == 5 | |
| 84 | + assert g7["points"][-1]["weighted_mean"] is not None | |
| 85 | + assert body["provenance"] and body["provenance"][0]["source"] == "worldbank" | |
| 86 | + get("/indicators/gdp/trend?group=narnia", status=404) | |
| 87 | + | |
| 88 | + | |
| 89 | +def test_indicator_download(client): | |
| 90 | + r = client.get("/api/v1/indicators/gdp/download.csv?from=2020&include_forecast=false") | |
| 91 | + assert r.status_code == 200 and r.headers["content-type"].startswith("text/csv") | |
| 92 | + lines = [l for l in r.text.splitlines() if l and not l.startswith("#")] | |
| 93 | + assert lines[0].split(",")[:4] == ["country_id", "country_name", "indicator_id", "indicator_name"] | |
| 94 | + assert len(lines) - 1 == 8 * 5 | |
| 95 | + j = client.get("/api/v1/indicators/gdp/download.json?from=2024&to=2024").json() | |
| 96 | + assert j["n"] == 8 and j["rows"][0]["series_code"] == "NY.GDP.MKTP.CD" | |
added
tests/api/test_infra.py
+199 −0
@@ -0,0 +1,199 @@ | ||
| 1 | +"""Infrastructure behaviour: run header, cache invalidation on snapshot swap, empty DB (503), rate limit, admin guard, OpenAPI, formatting.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import os | |
| 5 | +import shutil | |
| 6 | +import time | |
| 7 | +from pathlib import Path | |
| 8 | + | |
| 9 | +import duckdb | |
| 10 | +from fastapi.testclient import TestClient | |
| 11 | + | |
| 12 | +from countryatlas.api.cache import ResponseCache | |
| 13 | +from countryatlas.api.formatting import compact_number, format_change, format_value | |
| 14 | +from countryatlas.api.main import create_app | |
| 15 | +from countryatlas.api.provenance import source_url | |
| 16 | +from countryatlas.config import settings | |
| 17 | +from tests.api.conftest import RUN_ID | |
| 18 | + | |
| 19 | + | |
| 20 | +def test_run_header_and_cache(client): | |
| 21 | + r1 = client.get("/api/v1/countries/FRA") | |
| 22 | + assert r1.headers["x-countryatlas-run"] == RUN_ID and r1.headers["x-cache"] == "MISS" | |
| 23 | + r2 = client.get("/api/v1/countries/FRA") | |
| 24 | + assert r2.headers["x-cache"] == "HIT" and r2.json() == r1.json() | |
| 25 | + assert "x-cache" not in {k.lower() for k in client.get("/api/v1/countries/FRA/download.csv").headers} # downloads not cached | |
| 26 | + assert client.get("/api/v1/countries/FRA?x=1").headers["x-cache"] == "MISS" # different query → different key | |
| 27 | + | |
| 28 | + | |
| 29 | +def test_cache_key_includes_run_id(): | |
| 30 | + c = ResponseCache(maxsize=2) | |
| 31 | + c.set(ResponseCache.key("run1", "/x", "a=1"), 1) | |
| 32 | + assert c.get(ResponseCache.key("run1", "/x", "a=1")) == 1 | |
| 33 | + assert c.get(ResponseCache.key("run2", "/x", "a=1")) is None | |
| 34 | + assert ResponseCache.key("r", "/x", {"b": 2, "a": 1}) == ("r", "/x", "a=1&b=2") | |
| 35 | + c.set(("k2",), 2) | |
| 36 | + c.set(("k3",), 3) | |
| 37 | + assert c.get(ResponseCache.key("run1", "/x", "a=1")) is None # LRU evicted | |
| 38 | + assert c.stats()["size"] == 2 | |
| 39 | + | |
| 40 | + | |
| 41 | +def test_snapshot_swap_reopens(tmp_path: Path, fixture_db: Path): | |
| 42 | + db_path = tmp_path / "atlas.duckdb" | |
| 43 | + shutil.copy(fixture_db, db_path) | |
| 44 | + app = create_app(db_path, rate_limit_per_minute=0, cache=ResponseCache()) | |
| 45 | + with TestClient(app) as c: | |
| 46 | + assert c.get("/api/v1/health").json()["run_id"] == RUN_ID | |
| 47 | + assert c.get("/api/v1/countries/CAN").headers["x-cache"] == "MISS" | |
| 48 | + assert c.get("/api/v1/countries/CAN").headers["x-cache"] == "HIT" | |
| 49 | + # build a new snapshot with a different run_id and swap it atomically (new inode) | |
| 50 | + new = tmp_path / "build.duckdb" | |
| 51 | + shutil.copy(fixture_db, new) | |
| 52 | + con = duckdb.connect(str(new)) | |
| 53 | + con.execute("UPDATE meta SET value = 'fixture-NEXT' WHERE key = 'build_run_id'") | |
| 54 | + con.execute("UPDATE observations SET value = 12345 WHERE country_id = 'CAN' AND indicator_id = 'population' AND year = 2024") | |
| 55 | + con.execute("UPDATE latest SET value = 12345 WHERE country_id = 'CAN' AND indicator_id = 'population'") | |
| 56 | + con.close() | |
| 57 | + os.replace(new, db_path) | |
| 58 | + h = c.get("/api/v1/health").json() | |
| 59 | + assert h["run_id"] == "fixture-NEXT" | |
| 60 | + r = c.get("/api/v1/countries/CAN") | |
| 61 | + assert r.headers["x-countryatlas-run"] == "fixture-NEXT" and r.headers["x-cache"] == "MISS" | |
| 62 | + pop = next(m for m in r.json()["headline"] if m["indicator"] == "population") | |
| 63 | + assert pop["value"] == 12345 | |
| 64 | + # file removed → 503 problem+json, health = empty; file back → recovers | |
| 65 | + os.remove(db_path) | |
| 66 | + assert c.get("/api/v1/health").json()["status"] == "empty" | |
| 67 | + r = c.get("/api/v1/countries/CAN") | |
| 68 | + assert r.status_code == 503 and r.json()["title"] == "Data not built yet" and r.headers["content-type"].startswith("application/problem+json") | |
| 69 | + shutil.copy(fixture_db, db_path) | |
| 70 | + assert c.get("/api/v1/countries/CAN").status_code == 200 | |
| 71 | + | |
| 72 | + | |
| 73 | +def test_empty_database(tmp_path: Path): | |
| 74 | + app = create_app(tmp_path / "missing.duckdb", rate_limit_per_minute=0, cache=ResponseCache()) | |
| 75 | + with TestClient(app, raise_server_exceptions=False) as c: | |
| 76 | + h = c.get("/api/v1/health") | |
| 77 | + assert h.status_code == 200 and h.json()["status"] == "empty" and h.json()["observations"] == 0 | |
| 78 | + for p in ("/api/v1/countries", "/api/v1/home", "/api/v1/search?q=x", "/api/v1/indicators/gdp", "/api/v1/rankings/gdp"): | |
| 79 | + r = c.get(p) | |
| 80 | + assert r.status_code == 503, p | |
| 81 | + assert r.json()["title"] == "Data not built yet" and r.headers.get("retry-after") | |
| 82 | + assert c.get("/api/v1/methodology").status_code == 200 # registry-only | |
| 83 | + assert c.get("/api/v1/openapi.json").status_code == 200 | |
| 84 | + | |
| 85 | + | |
| 86 | +def test_rate_limit(fixture_db: Path): | |
| 87 | + app = create_app(fixture_db, rate_limit_per_minute=3, cache=ResponseCache()) | |
| 88 | + with TestClient(app, raise_server_exceptions=False) as c: | |
| 89 | + codes = [c.get("/api/v1/health").status_code for _ in range(5)] | |
| 90 | + assert codes == [200] * 5 # health exempt | |
| 91 | + codes = [c.get("/api/v1/countries/CAN", headers={"X-Forwarded-For": "203.0.113.9"}).status_code for _ in range(5)] | |
| 92 | + assert codes[:3] == [200, 200, 200] and codes[3] == 429 | |
| 93 | + r = c.get("/api/v1/countries/CAN", headers={"X-Forwarded-For": "203.0.113.9"}) | |
| 94 | + assert r.headers["retry-after"] and r.json()["title"] == "Too many requests" | |
| 95 | + assert c.get("/api/v1/countries/CAN", headers={"X-Forwarded-For": "203.0.113.10"}).status_code == 200 # other IP | |
| 96 | + | |
| 97 | + | |
| 98 | +def test_admin_guard(client): | |
| 99 | + assert client.get("/api/v1/admin/overview").status_code == 403 | |
| 100 | + assert client.get("/api/v1/admin/overview", headers={"X-Admin-Token": "wrong"}).status_code == 403 | |
| 101 | + ok = client.get("/api/v1/admin/overview", headers={"X-Admin-Token": "test-admin-token"}) | |
| 102 | + assert ok.status_code == 200 | |
| 103 | + body = ok.json() | |
| 104 | + assert body["status"] == "ok" and body["counts"]["observations"] > 3000 and body["db"]["exists"] | |
| 105 | + conns = {c["connector"]: c for c in body["connectors"]} | |
| 106 | + assert conns["who"]["last_status"] == "failed" and conns["worldbank"]["n_ok"] == 2 | |
| 107 | + assert body["sources"] and body["cache"]["maxsize"] and body["scheduler"]["alive"] is None | |
| 108 | + saved = settings.admin_token | |
| 109 | + try: | |
| 110 | + settings.admin_token = None | |
| 111 | + r = client.get("/api/v1/admin/overview", headers={"X-Admin-Token": "test-admin-token"}) | |
| 112 | + assert r.status_code == 503 and r.json()["title"] == "Admin disabled" | |
| 113 | + finally: | |
| 114 | + settings.admin_token = saved | |
| 115 | + | |
| 116 | + | |
| 117 | +def test_admin_endpoints(client, tmp_path: Path): | |
| 118 | + h = {"X-Admin-Token": "test-admin-token"} | |
| 119 | + runs = client.get("/api/v1/admin/runs?limit=3", headers=h).json() | |
| 120 | + assert runs["n"] == 3 and runs["items"][0]["run_id"] == RUN_ID | |
| 121 | + assert client.get("/api/v1/admin/runs?connector=who", headers=h).json()["items"][0]["status"] == "failed" | |
| 122 | + issues = client.get("/api/v1/admin/issues?severity=warning", headers=h).json() | |
| 123 | + assert issues["n"] == 2 and issues["summary"][0]["code"] == "extreme_jump" | |
| 124 | + cov = client.get("/api/v1/admin/coverage", headers=h).json() | |
| 125 | + assert cov["n_indicators"] == 14 and cov["indicators"][0]["n_countries"] == 8 and cov["countries"][0]["coverage_pct"] == 100.0 | |
| 126 | + raw = client.get(f"/api/v1/admin/raw?run_id={RUN_ID}", headers=h).json() | |
| 127 | + assert raw["n_runs"] == 5 and any(f.get("missing") for f in raw["files"]) | |
| 128 | + # refresh: no pid file → 409 ; pid file with own pid → SIGUSR1 delivered (handler installed) | |
| 129 | + saved = settings.data_dir | |
| 130 | + settings.data_dir = tmp_path | |
| 131 | + try: | |
| 132 | + r = client.post("/api/v1/admin/refresh", headers=h) | |
| 133 | + assert r.status_code == 409 | |
| 134 | + import signal | |
| 135 | + | |
| 136 | + got = [] | |
| 137 | + old = signal.signal(signal.SIGUSR1, lambda *a: got.append(1)) | |
| 138 | + (tmp_path / "scheduler.pid").write_text(str(os.getpid())) | |
| 139 | + r = client.post("/api/v1/admin/refresh", headers=h) | |
| 140 | + time.sleep(0.05) | |
| 141 | + signal.signal(signal.SIGUSR1, old) | |
| 142 | + assert r.status_code == 200 and r.json()["signal"] == "SIGUSR1" and got | |
| 143 | + assert client.post("/api/v1/admin/cache/clear", headers=h).json()["ok"] | |
| 144 | + finally: | |
| 145 | + settings.data_dir = saved | |
| 146 | + | |
| 147 | + | |
| 148 | +def test_openapi_and_docs(client): | |
| 149 | + spec = client.get("/api/v1/openapi.json").json() | |
| 150 | + paths = set(spec["paths"]) | |
| 151 | + for p in ("/api/v1/health", "/api/v1/countries", "/api/v1/countries/{id}", "/api/v1/countries/{id}/topics/{topic}", | |
| 152 | + "/api/v1/countries/{id}/series/{indicator}", "/api/v1/countries/{id}/changes", "/api/v1/countries/{id}/events", | |
| 153 | + "/api/v1/countries/{id}/similar", "/api/v1/countries/{id}/insights", "/api/v1/countries/{id}/dna", | |
| 154 | + "/api/v1/countries/{id}/download.{fmt}", "/api/v1/indicators", "/api/v1/indicators/{slug}", "/api/v1/indicators/{slug}/map", | |
| 155 | + "/api/v1/indicators/{slug}/trend", "/api/v1/indicators/{slug}/download.{fmt}", "/api/v1/series", "/api/v1/rankings", | |
| 156 | + "/api/v1/rankings/{indicator}", "/api/v1/rankings/{indicator}/history", "/api/v1/compare", "/api/v1/compare/snapshot", | |
| 157 | + "/api/v1/compare/download.{fmt}", "/api/v1/regions", "/api/v1/regions/{slug}", "/api/v1/search", "/api/v1/home", | |
| 158 | + "/api/v1/changes", "/api/v1/sources", "/api/v1/sources/{id}", "/api/v1/methodology", "/api/v1/admin/overview", | |
| 159 | + "/api/v1/admin/runs", "/api/v1/admin/issues", "/api/v1/admin/coverage", "/api/v1/admin/raw", "/api/v1/admin/refresh"): | |
| 160 | + assert p in paths, p | |
| 161 | + assert "Provenance" in spec["components"]["schemas"] | |
| 162 | + assert client.get("/api/v1/docs").status_code == 200 and client.get("/api/v1/redoc").status_code == 200 | |
| 163 | + | |
| 164 | + | |
| 165 | +def test_unknown_route_and_validation(client): | |
| 166 | + r = client.get("/api/v1/nothing-here") | |
| 167 | + assert r.status_code == 404 and r.headers["content-type"].startswith("application/problem+json") and "/api/v1/docs" in r.json()["detail"] | |
| 168 | + r = client.get("/api/v1/rankings/gdp?limit=0") | |
| 169 | + assert r.status_code == 422 and r.json()["errors"][0]["loc"] == ["query", "limit"] | |
| 170 | + | |
| 171 | + | |
| 172 | +def test_formatting(): | |
| 173 | + assert format_value(53372.1, {"format": "currency"}) == "53.4k" | |
| 174 | + assert format_value(1.23e12, {"format": "currency"}) == "1.2T" | |
| 175 | + assert format_value(45.3e9, {"format": "currency"}) == "45.3B" | |
| 176 | + assert format_value(3.44, {"format": "percent", "precision": 1}) == "3.4 %" | |
| 177 | + assert format_value(82.13, {"format": "years"}) == "82.1 yrs" | |
| 178 | + assert format_value(5.234, {"format": "tonnes", "precision": 2}) == "5.23 t" | |
| 179 | + assert format_value(3.2, {"format": "per_1000"}) == "3.2 per 1,000" | |
| 180 | + assert format_value(1_234_567, {"format": "number"}) == "1.2M" | |
| 181 | + assert format_value(812.4, {"format": "number", "precision": 0}) == "812" | |
| 182 | + assert format_value(None, {"format": "number"}) == "—" | |
| 183 | + assert format_value(float("nan"), {"format": "number"}) == "—" | |
| 184 | + assert compact_number(-2_500_000) == "-2.5M" | |
| 185 | + assert format_change(1.2, 30.0, {"format": "percent"}) == "+1.2 pts" | |
| 186 | + assert format_change(-100.0, -3.4, {"format": "currency"}) == "−3.4 %" | |
| 187 | + | |
| 188 | + | |
| 189 | +def test_source_urls(): | |
| 190 | + assert source_url("worldbank", "WDI", "NY.GDP.PCAP.CD", "CA") == "https://data.worldbank.org/indicator/NY.GDP.PCAP.CD?locations=CA" | |
| 191 | + assert source_url("owid", "grapher", "median-age") == "https://ourworldindata.org/grapher/median-age" | |
| 192 | + assert source_url("owid", "energy", "renewables_share_elec") == "https://github.com/owid/energy-data" | |
| 193 | + assert source_url("eurostat", "prc_hpi_a", "") == "https://ec.europa.eu/eurostat/databrowser/view/prc_hpi_a/default/table" | |
| 194 | + assert source_url("who", "GHO", "WHOSIS_000001") == "https://www.who.int/data/gho/data/indicators/indicator-details/GHO/WHOSIS_000001" | |
| 195 | + assert source_url("fred", "", "UNRATE") == "https://fred.stlouisfed.org/series/UNRATE" | |
| 196 | + assert source_url("imf", "WEO", "NGDPD") == "https://data.imf.org/" | |
| 197 | + assert source_url("bis", "", "") == "https://data.bis.org/" and source_url("ilo", "", "") == "https://ilostat.ilo.org/" | |
| 198 | + assert source_url("oecd", "X", "Y") == "https://data-explorer.oecd.org/" | |
| 199 | + assert source_url("unknown", "", "") is None | |
added
tests/api/test_misc_routers.py
+117 −0
@@ -0,0 +1,117 @@ | ||
| 1 | +"""regions, search, home, changes, sources, methodology, health.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import pytest | |
| 5 | + | |
| 6 | +from tests.api.conftest import COUNTRIES, RUN_ID, assert_meta, assert_provenance, value | |
| 7 | + | |
| 8 | + | |
| 9 | +def test_regions(get): | |
| 10 | + body = get("/regions").json() | |
| 11 | + assert_meta(body) | |
| 12 | + ids = [g["id"] for g in body["items"]] | |
| 13 | + assert ids[0] == "world" and "g7" in ids and "hic" in ids | |
| 14 | + world = body["items"][0] | |
| 15 | + assert world["n_members"] == 8 and world["population_latest"] == pytest.approx(sum(value(c, "population", 2024) for c in COUNTRIES)) | |
| 16 | + assert all(g["kind"] == "org" for g in get("/regions?kind=org").json()["items"]) | |
| 17 | + | |
| 18 | + | |
| 19 | +def test_region_detail(get): | |
| 20 | + body = get("/regions/g7?indicator=life-expectancy").json() | |
| 21 | + assert body["group"]["id"] == "g7" and body["n_members"] == 5 | |
| 22 | + agg = body["aggregates"] | |
| 23 | + assert agg["population"]["kind"] == "sum" and agg["gdp"]["kind"] == "sum" and agg["life-expectancy"]["kind"] == "median" | |
| 24 | + assert agg["gdp"]["value"] == pytest.approx(sum(value(c, "gdp", 2024) for c in ("CAN", "USA", "FRA", "DEU", "JPN"))) | |
| 25 | + m = body["members"][0] | |
| 26 | + assert m["id"] == "USA" and m["values"]["population"]["value"] and m["values"]["gdp"]["formatted"].endswith("T") | |
| 27 | + assert_provenance(m["values"]["gdp"]["provenance"]) | |
| 28 | + rk = body["ranking"] | |
| 29 | + assert rk["indicator"]["id"] == "life-expectancy" and rk["rows"][0]["rank"] == 1 and rk["rows"][0]["country"]["id"] == "JPN" | |
| 30 | + assert_provenance(rk["rows"][0]["provenance"]) | |
| 31 | + assert get("/regions/europe-central-asia").json()["n_members"] == 2 | |
| 32 | + get("/regions/mars", status=404) | |
| 33 | + | |
| 34 | + | |
| 35 | +def test_search(get): | |
| 36 | + body = get("/search?q=infl").json() | |
| 37 | + assert_meta(body) | |
| 38 | + assert body["hits"][0]["type"] == "indicator" and body["hits"][0]["id"] == "inflation" | |
| 39 | + assert body["hits"][0]["hint"].startswith("Indicator · Economy") | |
| 40 | + can = get("/search?q=can").json()["hits"] | |
| 41 | + assert can[0]["type"] == "country" and can[0]["id"] == "CAN" and can[0]["hint"] == "Country · North America" and can[0]["url"] == "/countries/canada" | |
| 42 | + fuzzy = get("/search?q=canda").json()["hits"] | |
| 43 | + assert fuzzy and fuzzy[0]["id"] == "CAN" | |
| 44 | + combo = get("/search?q=housing canada").json()["hits"] | |
| 45 | + types = {h["type"]: h for h in combo} | |
| 46 | + assert "country_topic" in types and types["country_topic"]["url"] == "/countries/canada/housing" and types["country_topic"]["topic"] == "housing" | |
| 47 | + assert "country" in types and "topic" in types | |
| 48 | + combo2 = get("/search?q=canada gdp").json()["hits"] | |
| 49 | + assert any(h["type"] == "country_indicator" and h["indicator"] == "gdp" for h in combo2) | |
| 50 | + assert get("/search?q=oecd").json()["hits"][0]["type"] == "region" | |
| 51 | + assert get("/search?q=zzzzqqq").json()["n"] == 0 | |
| 52 | + assert all(h["type"] == "indicator" for h in get("/search?q=a&type=indicator&limit=3").json()["hits"]) | |
| 53 | + get("/search", status=422) | |
| 54 | + | |
| 55 | + | |
| 56 | +def test_home(get): | |
| 57 | + body = get("/home").json() | |
| 58 | + assert_meta(body) | |
| 59 | + snap = body["snapshot"] | |
| 60 | + assert snap["n_countries"] == 8 and snap["n_indicators"] == 14 and snap["n_observations"] > 3000 and snap["built_at"] | |
| 61 | + assert snap["world_population"] == pytest.approx(sum(value(c, "population", 2024) for c in COUNTRIES)) | |
| 62 | + assert snap["world_gdp_formatted"].endswith("T") and snap["median_life_expectancy"] > 60 | |
| 63 | + lists = body["lists"] | |
| 64 | + assert set(lists) == {"largest_economies", "fastest_population_growth", "highest_life_expectancy", "energy_transition_leaders", | |
| 65 | + "highest_gdp_per_capita_ppp", "lowest_unemployment"} | |
| 66 | + le = lists["largest_economies"] | |
| 67 | + assert le["rows"][0]["country"]["id"] == "USA" and le["rows"][0]["rank"] == 1 and len(le["rows"]) == 8 | |
| 68 | + assert_provenance(le["rows"][0]["provenance"]) | |
| 69 | + assert lists["lowest_unemployment"]["rows"][0]["country"]["id"] == "JPN" | |
| 70 | + assert lists["energy_transition_leaders"]["rows"][0]["country"]["id"] == "BRA" | |
| 71 | + assert len(body["recent_changes"]) == 12 and body["recent_changes"][0]["country"]["id"] and body["recent_changes"][0]["headline"] | |
| 72 | + assert body["recently_updated"] and body["featured_indicators"] and body["trending"] | |
| 73 | + assert all(i["featured"] for i in body["featured_indicators"]) | |
| 74 | + | |
| 75 | + | |
| 76 | +def test_changes_feed(get): | |
| 77 | + body = get("/changes?limit=5").json() | |
| 78 | + assert body["n"] == 5 and body["kinds"] and body["items"][0]["country"]["slug"] | |
| 79 | + sev = [c["severity"] for c in body["items"]] | |
| 80 | + assert sev == sorted(sev, reverse=True) | |
| 81 | + assert_provenance(body["items"][0]["provenance"]) | |
| 82 | + rec = get("/changes?kind=record_high&indicator=population").json() | |
| 83 | + assert rec["n"] > 0 and all(c["kind"] == "record_high" and c["indicator"]["id"] == "population" for c in rec["items"]) | |
| 84 | + assert get("/changes?country=CAN").json()["items"][0]["country"]["id"] == "CAN" | |
| 85 | + get("/changes?indicator=nope", status=404) | |
| 86 | + | |
| 87 | + | |
| 88 | +def test_sources(get): | |
| 89 | + body = get("/sources").json() | |
| 90 | + assert_meta(body) | |
| 91 | + ids = [s["id"] for s in body["items"]] | |
| 92 | + assert ids[0] == "worldbank" and set(ids) == {"worldbank", "imf", "owid", "who"} | |
| 93 | + wb = body["items"][0] | |
| 94 | + assert wb["licence"] == "CC BY 4.0" and wb["n_observations"] > 2000 and wb["n_indicators"] >= 10 | |
| 95 | + det = get("/sources/worldbank").json() | |
| 96 | + assert det["source"]["id"] == "worldbank" and det["indicators"][0]["series_code"] and det["datasets"][0]["dataset"] == "WDI" | |
| 97 | + assert det["import_runs"] and det["import_runs"][0]["run_id"] == RUN_ID and det["import_runs"][0]["status"] == "ok" | |
| 98 | + assert det["freshness"]["retrieved_at"].startswith("2026-09-10") | |
| 99 | + get("/sources/nasa", status=404) | |
| 100 | + | |
| 101 | + | |
| 102 | +def test_methodology(get): | |
| 103 | + body = get("/methodology").json() | |
| 104 | + assert body["meta"]["run_id"] == RUN_ID | |
| 105 | + assert body["topics"][0]["id"] == "economy" and body["headline_indicators"][0] == "population" | |
| 106 | + assert body["indicators"]["n"] > 200 and body["validation"]["rules"] and body["validation"]["stale_after_days"]["A"] == 800 | |
| 107 | + assert "similarity" in body["derived"] and "world_aggregates" in body["derived"] | |
| 108 | + assert body["sources"]["urls"]["worldbank"].startswith("https://data.worldbank.org") | |
| 109 | + | |
| 110 | + | |
| 111 | +def test_health(client): | |
| 112 | + r = client.get("/api/v1/health") | |
| 113 | + assert r.status_code == 200 | |
| 114 | + body = r.json() | |
| 115 | + assert body["status"] == "ok" and body["run_id"] == RUN_ID and body["observations"] > 3000 and body["countries"] == 8 and body["indicators"] == 14 | |
| 116 | + assert client.get("/health").json()["status"] == "ok" | |
| 117 | + assert client.get("/").json()["docs"] == "/api/v1/docs" | |
added
tests/api/test_rankings_compare_series.py
+102 −0
@@ -0,0 +1,102 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +import pytest | |
| 4 | + | |
| 5 | +from tests.api.conftest import assert_meta, assert_provenance, value | |
| 6 | + | |
| 7 | + | |
| 8 | +def test_rankings_list(get): | |
| 9 | + body = get("/rankings").json() | |
| 10 | + assert_meta(body) | |
| 11 | + assert body["items"][0]["featured"] is True | |
| 12 | + assert {"gdp", "life-expectancy"} <= {i["id"] for i in body["items"]} | |
| 13 | + assert all(i["ranking_eligible"] for i in body["items"]) | |
| 14 | + | |
| 15 | + | |
| 16 | +def test_ranking_table(get): | |
| 17 | + body = get("/rankings/gdp-per-capita?limit=3").json() | |
| 18 | + assert body["year_used"] == 2024 and body["years_available"][0] == 1990 and body["n"] == 8 and body["sort"] == "desc" | |
| 19 | + rows = body["rows"] | |
| 20 | + assert [r["rank"] for r in rows] == [1, 2, 3] and rows[0]["country"]["id"] == "JPN" and rows[0]["rank_world"] == 1 | |
| 21 | + assert rows[0]["value"] == pytest.approx(value("JPN", "gdp-per-capita", 2024)) | |
| 22 | + assert rows[0]["change_1y"]["pct"] == pytest.approx(3.0) and rows[0]["change_10y"]["pct"] == pytest.approx(34.39, abs=0.01) | |
| 23 | + assert len(rows[0]["sparkline"]) == 30 and rows[0]["sparkline"][-1][0] == 2024 | |
| 24 | + assert_provenance(rows[0]["provenance"]) | |
| 25 | + page2 = get("/rankings/gdp-per-capita?limit=3&offset=3").json() | |
| 26 | + assert page2["rows"][0]["rank"] == 4 | |
| 27 | + asc = get("/rankings/gdp-per-capita?sort=asc&limit=1").json() | |
| 28 | + assert asc["rows"][0]["country"]["id"] == "IND" | |
| 29 | + # lower-is-better default sort ascending | |
| 30 | + infl = get("/rankings/inflation?limit=8").json() | |
| 31 | + assert infl["sort"] == "asc" and infl["rows"][0]["value"] <= infl["rows"][-1]["value"] | |
| 32 | + # nearest year fallback | |
| 33 | + assert get("/rankings/gdp-per-capita?year=2030").json()["year_used"] == 2024 | |
| 34 | + y2000 = get("/rankings/gdp-per-capita?year=2000&limit=1").json() | |
| 35 | + assert y2000["year_used"] == 2000 and y2000["rows"][0]["year"] == 2000 | |
| 36 | + get("/rankings/nothing", status=404) | |
| 37 | + | |
| 38 | + | |
| 39 | +def test_ranking_group(get): | |
| 40 | + body = get("/rankings/unemployment-rate?group=g7&limit=10").json() | |
| 41 | + assert body["group"]["id"] == "g7" and body["n"] == 5 and [r["rank"] for r in body["rows"]] == [1, 2, 3, 4, 5] | |
| 42 | + assert body["rows"][0]["country"]["id"] == "JPN" # lowest unemployment in fixture | |
| 43 | + assert body["rows"][0]["rank_world"] >= 1 and body["rows"][0]["n_world"] == 8 | |
| 44 | + | |
| 45 | + | |
| 46 | +def test_ranking_history(get): | |
| 47 | + body = get("/rankings/gdp-per-capita/history?countries=CAN,usa&from=2020").json() | |
| 48 | + assert [c["id"] for c in body["countries"]] == ["CAN", "USA"] and body["years"] == [2020, 2021, 2022, 2023, 2024] | |
| 49 | + assert body["series"]["CAN"][-1]["year"] == 2024 and 1 <= body["series"]["CAN"][-1]["rank"] <= 8 | |
| 50 | + get("/rankings/gdp-per-capita/history?countries=XXX", status=404) | |
| 51 | + | |
| 52 | + | |
| 53 | +def test_series_bundle(get): | |
| 54 | + body = get("/series?country=CAN,FRA&indicator=gdp,inflation&from=2020&to=2024").json() | |
| 55 | + assert_meta(body) | |
| 56 | + assert body["n"] == 4 | |
| 57 | + s = body["series"][0] | |
| 58 | + assert s["indicator"]["id"] == "gdp" and s["country"]["id"] == "CAN" and [v["year"] for v in s["values"]] == list(range(2020, 2025)) | |
| 59 | + assert_provenance(s["values"][0]["provenance"]) | |
| 60 | + get("/series?country=CAN&indicator=nope", status=404) | |
| 61 | + get("/series?country=CAN", status=422) | |
| 62 | + | |
| 63 | + | |
| 64 | +def test_compare_modes(get): | |
| 65 | + body = get("/compare?countries=CAN,USA,FRA&indicators=gdp,gdp-per-capita&from=2000&to=2024").json() | |
| 66 | + assert_meta(body) | |
| 67 | + assert body["mode"] == "absolute" and len(body["series"]) == 6 and [c["id"] for c in body["countries"]] == ["CAN", "USA", "FRA"] | |
| 68 | + idx = get("/compare?countries=CAN,USA&indicators=gdp&mode=index100&from=2000").json() | |
| 69 | + s = idx["series"][0] | |
| 70 | + assert s["transform"]["applied"] and s["transform"]["base_year"] == 2000 and s["values"][0]["value"] == pytest.approx(100.0) | |
| 71 | + assert s["values"][1]["value"] == pytest.approx(value("CAN", "gdp", 2001) / value("CAN", "gdp", 2000) * 100) | |
| 72 | + assert s["unit"].startswith("index") | |
| 73 | + pc = get("/compare?countries=CAN&indicators=gdp,gdp-per-capita&mode=per-capita&from=2024&to=2024").json() | |
| 74 | + gdp_pc, gpc = pc["series"] | |
| 75 | + assert gdp_pc["transform"]["applied"] and gdp_pc["values"][0]["value"] == pytest.approx(value("CAN", "gdp-per-capita", 2024)) | |
| 76 | + assert gpc["transform"]["applied"] is False # already per-capita | |
| 77 | + pct = get("/compare?countries=CAN&indicators=gdp&mode=pct&from=2020&to=2022").json()["series"][0] | |
| 78 | + assert pct["values"][0]["value"] is None and pct["values"][1]["value"] == pytest.approx( | |
| 79 | + (value("CAN", "gdp", 2021) / value("CAN", "gdp", 2020) - 1) * 100) | |
| 80 | + get("/compare?countries=CAN&indicators=gdp&mode=weird", status=422) | |
| 81 | + get("/compare?countries=CAN,XXX&indicators=gdp", status=404) | |
| 82 | + | |
| 83 | + | |
| 84 | +def test_compare_snapshot(get): | |
| 85 | + body = get("/compare/snapshot?countries=CAN,USA&topic=economy").json() | |
| 86 | + assert body["topic"]["id"] == "economy" and [c["id"] for c in body["countries"]] == ["CAN", "USA"] | |
| 87 | + row = next(r for r in body["rows"] if r["indicator"]["id"] == "gdp-per-capita") | |
| 88 | + assert row["values"]["CAN"]["has_data"] and row["best"] == "USA" | |
| 89 | + assert_provenance(row["values"]["CAN"]["provenance"]) | |
| 90 | + missing = next(r for r in body["rows"] if r["indicator"]["id"] == "gdp-growth") | |
| 91 | + assert missing["values"]["CAN"]["has_data"] | |
| 92 | + default = get("/compare/snapshot?countries=CAN").json() | |
| 93 | + assert default["topic"] is None and default["rows"][0]["indicator"]["id"] == "population" | |
| 94 | + | |
| 95 | + | |
| 96 | +def test_compare_download(client): | |
| 97 | + r = client.get("/api/v1/compare/download.csv?countries=CAN,USA&indicators=gdp&from=2023&include_forecast=false") | |
| 98 | + assert r.status_code == 200 | |
| 99 | + lines = [l for l in r.text.splitlines() if l and not l.startswith("#")] | |
| 100 | + assert len(lines) - 1 == 4 and "series_code" in lines[0] | |
| 101 | + j = client.get("/api/v1/compare/download.json?countries=CAN&indicators=gdp&from=2024&to=2024&include_forecast=false").json() | |
| 102 | + assert j["n"] == 1 and j["rows"][0]["value"] == pytest.approx(value("CAN", "gdp", 2024)) | |
added
tests/conftest.py
+23 −0
@@ -0,0 +1,23 @@ | ||
| 1 | +"""Shared fixtures: every test gets an isolated CA data dir (settings is a process-wide singleton → patched).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from pathlib import Path | |
| 5 | + | |
| 6 | +import pytest | |
| 7 | + | |
| 8 | +from countryatlas.config import settings | |
| 9 | + | |
| 10 | +FIXTURES = Path(__file__).parent / "fixtures" | |
| 11 | + | |
| 12 | + | |
| 13 | +@pytest.fixture() | |
| 14 | +def data_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: | |
| 15 | + d = tmp_path / "ca-data" | |
| 16 | + monkeypatch.setattr(settings, "data_dir", d) | |
| 17 | + settings.ensure_dirs() | |
| 18 | + return d | |
| 19 | + | |
| 20 | + | |
| 21 | +@pytest.fixture() | |
| 22 | +def fixtures() -> Path: | |
| 23 | + return FIXTURES | |
added
tests/connectors/__init__.py
+0 −0
added
tests/connectors/_helpers.py
+56 −0
@@ -0,0 +1,56 @@ | ||
| 1 | +"""Offline helpers for the WHO / FRED / BIS / ILO connector tests (kept out of the shared conftest). | |
| 2 | + | |
| 3 | +* `make_response` — build an httpx.Response as if returned by `Connector.get`. | |
| 4 | +* `install_fake_get` — body of a `fake_get` fixture: patch `connector.get` with a router(url, params) → (body, content_type); | |
| 5 | + returns the call log. Each test module wraps it in its own `@pytest.fixture def fake_get(monkeypatch)`. | |
| 6 | +* `raw_from_file` — RawPayload from a recorded fixture file. | |
| 7 | +""" | |
| 8 | +from __future__ import annotations | |
| 9 | + | |
| 10 | +from collections.abc import Callable | |
| 11 | +from datetime import UTC, datetime | |
| 12 | +from pathlib import Path | |
| 13 | +from typing import Any | |
| 14 | + | |
| 15 | +import httpx | |
| 16 | +import pytest | |
| 17 | + | |
| 18 | +from countryatlas.models import RawPayload | |
| 19 | + | |
| 20 | +Router = Callable[[str, dict[str, Any] | None], tuple[bytes, str]] | |
| 21 | + | |
| 22 | + | |
| 23 | +def make_response(url: str, body: bytes, content_type: str, params: dict[str, Any] | None = None) -> httpx.Response: | |
| 24 | + req = httpx.Request("GET", httpx.URL(url, params=params or {})) | |
| 25 | + return httpx.Response(200, content=body, headers={"Content-Type": content_type}, request=req) | |
| 26 | + | |
| 27 | + | |
| 28 | +def install_fake_get(monkeypatch: pytest.MonkeyPatch) -> Callable[[Any, Router], list[dict[str, Any]]]: | |
| 29 | + """Body of the `fake_get` fixture; each test module declares `fake_get = fixture(lambda monkeypatch: install_fake_get(…))`.""" | |
| 30 | + | |
| 31 | + def _install(connector: Any, router: Router) -> list[dict[str, Any]]: | |
| 32 | + calls: list[dict[str, Any]] = [] | |
| 33 | + | |
| 34 | + def _get(url: str, params: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> httpx.Response: | |
| 35 | + calls.append({"url": url, "params": dict(params or {}), "headers": dict(headers or {})}) | |
| 36 | + body, ct = router(url, params) | |
| 37 | + return make_response(url, body, ct, params) | |
| 38 | + | |
| 39 | + monkeypatch.setattr(connector, "get", _get) | |
| 40 | + return calls | |
| 41 | + | |
| 42 | + return _install | |
| 43 | + | |
| 44 | + | |
| 45 | +def raw_from_file(connector: str, dataset: str, code: str, path: Path, content_type: str, **meta: Any) -> RawPayload: | |
| 46 | + return RawPayload( | |
| 47 | + connector=connector, | |
| 48 | + dataset=dataset, | |
| 49 | + code=code, | |
| 50 | + url=f"fixture://{path.name}", | |
| 51 | + retrieved_at=datetime.now(UTC), | |
| 52 | + status_code=200, | |
| 53 | + content_type=content_type, | |
| 54 | + body=path.read_bytes(), | |
| 55 | + meta=meta, | |
| 56 | + ) | |
added
tests/connectors/conftest.py
+32 −0
@@ -0,0 +1,32 @@ | ||
| 1 | +"""Connector-test fixtures (scoped to tests/connectors so the root conftest stays untouched). | |
| 2 | + | |
| 3 | +Tests marked `live` hit the real IMF/OECD/Eurostat endpoints; they are skipped unless selected with `pytest -m live`. | |
| 4 | +""" | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +from datetime import UTC, datetime | |
| 8 | +from pathlib import Path | |
| 9 | + | |
| 10 | +import pytest | |
| 11 | + | |
| 12 | +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" | |
| 13 | + | |
| 14 | + | |
| 15 | +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: | |
| 16 | + markexpr = config.getoption("-m", default="") or "" | |
| 17 | + if "live" in markexpr: | |
| 18 | + return | |
| 19 | + skip = pytest.mark.skip(reason="live test — select explicitly with `pytest -m live`") | |
| 20 | + for item in items: | |
| 21 | + if "live" in item.keywords: | |
| 22 | + item.add_marker(skip) | |
| 23 | + | |
| 24 | + | |
| 25 | +@pytest.fixture | |
| 26 | +def fixtures_dir() -> Path: | |
| 27 | + return FIXTURES | |
| 28 | + | |
| 29 | + | |
| 30 | +@pytest.fixture | |
| 31 | +def retrieved_at() -> datetime: | |
| 32 | + return datetime(2026, 9, 11, 12, 0, tzinfo=UTC) | |
added
tests/connectors/test_bis.py
+102 −0
@@ -0,0 +1,102 @@ | ||
| 1 | +"""BIS SDMX connector — offline tests on trimmed WS_CBPOL / WS_SPP CSVs, plus live smoke tests.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from datetime import date | |
| 5 | + | |
| 6 | +import pytest | |
| 7 | + | |
| 8 | +from countryatlas.connectors._series import parse_sdmx_period | |
| 9 | +from countryatlas.connectors.bis import BISConnector | |
| 10 | +from countryatlas.models import IndicatorSourceSpec | |
| 11 | +from tests.connectors._helpers import install_fake_get, raw_from_file | |
| 12 | + | |
| 13 | + | |
| 14 | +@pytest.fixture | |
| 15 | +def fake_get(monkeypatch): | |
| 16 | + return install_fake_get(monkeypatch) | |
| 17 | + | |
| 18 | + | |
| 19 | +CBPOL =IndicatorSourceSpec(indicator_id="policy-rate", connector="bis", dataset="WS_CBPOL", code="M.", frequency="M", priority=1) | |
| 20 | +SPP_R = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="bis", dataset="WS_SPP", code="Q..R.628", | |
| 21 | + frequency="Q", priority=1, transform="rebase:2015") | |
| 22 | +SPP_G = IndicatorSourceSpec(indicator_id="house-price-growth", connector="bis", dataset="WS_SPP", code="Q..R.771", | |
| 23 | + frequency="Q", priority=1) | |
| 24 | + | |
| 25 | + | |
| 26 | +def test_sdmx_period_parsing(): | |
| 27 | + assert parse_sdmx_period("2026-08") == (date(2026, 8, 1), 2026, "M") | |
| 28 | + assert parse_sdmx_period("2026-Q3") == (date(2026, 7, 1), 2026, "Q") | |
| 29 | + assert parse_sdmx_period("2024") == (date(2024, 1, 1), 2024, "A") | |
| 30 | + assert parse_sdmx_period("2026-13") is None | |
| 31 | + | |
| 32 | + | |
| 33 | +def test_policy_rate_iso2_mapping_and_nan(fixtures_dir): | |
| 34 | + conn = BISConnector() | |
| 35 | + raw = raw_from_file("bis", "WS_CBPOL", "M.", fixtures_dir / "bis" / "WS_CBPOL_M.csv", "text/csv") | |
| 36 | + rows = conn.normalize(raw, CBPOL) | |
| 37 | + # CA/US → CAN/USA; XM (euro area) dropped; AR rows are NaN with OBS_STATUS=M → dropped | |
| 38 | + assert {r.country_id for r in rows} == {"CAN", "USA"} | |
| 39 | + assert all(r.frequency == "M" and r.period.day == 1 and r.unit == "% per annum" for r in rows) | |
| 40 | + us = {r.period: r.value for r in rows if r.country_id == "USA"} | |
| 41 | + assert us[date(2026, 8, 1)] == pytest.approx(3.625) | |
| 42 | + assert conn.validate(rows).quarantine_dataset is False | |
| 43 | + | |
| 44 | + | |
| 45 | +def test_real_index_rebased_to_2015(fixtures_dir): | |
| 46 | + conn = BISConnector() | |
| 47 | + raw = raw_from_file("bis", "WS_SPP", "Q..R.628", fixtures_dir / "bis" / "WS_SPP_R_628.csv", "text/csv") | |
| 48 | + rows = conn.normalize(raw, SPP_R) | |
| 49 | + assert {r.country_id for r in rows} == {"CAN", "USA"} | |
| 50 | + for iso in ("CAN", "USA"): | |
| 51 | + base = [r.value for r in rows if r.country_id == iso and r.year == 2015] | |
| 52 | + assert len(base) == 4 and sum(base) / 4 == pytest.approx(100.0) | |
| 53 | + q = [r for r in rows if r.country_id == "CAN"] | |
| 54 | + assert q[0].period == date(2014, 1, 1) and q[0].frequency == "Q" and q[0].unit == "index (2015 = 100)" | |
| 55 | + assert len(q) == 12 | |
| 56 | + | |
| 57 | + | |
| 58 | +def test_growth_series_taken_as_published(fixtures_dir): | |
| 59 | + conn = BISConnector() | |
| 60 | + raw = raw_from_file("bis", "WS_SPP", "Q..R.771", fixtures_dir / "bis" / "WS_SPP_R_771.csv", "text/csv") | |
| 61 | + rows = conn.normalize(raw, SPP_G) | |
| 62 | + us = {r.period: r.value for r in rows if r.country_id == "USA"} | |
| 63 | + assert us[date(2025, 1, 1)] == pytest.approx(-0.0324) | |
| 64 | + assert all(r.unit == "annual %" for r in rows) | |
| 65 | + | |
| 66 | + | |
| 67 | +def test_rebase_drops_country_without_base_year(fixtures_dir): | |
| 68 | + conn = BISConnector() | |
| 69 | + text = (fixtures_dir / "bis" / "WS_SPP_R_628.csv").read_text() | |
| 70 | + lines = [ln for ln in text.splitlines() if not (",CA," in ln and ",2015-Q" in ln)] | |
| 71 | + raw = raw_from_file("bis", "WS_SPP", "Q..R.628", fixtures_dir / "bis" / "WS_SPP_R_628.csv", "text/csv") | |
| 72 | + raw.body = "\n".join(lines).encode() | |
| 73 | + rows = conn.normalize(raw, SPP_R) | |
| 74 | + assert {r.country_id for r in rows} == {"USA"} | |
| 75 | + | |
| 76 | + | |
| 77 | +def test_fetch_builds_verified_url(fixtures_dir, fake_get): | |
| 78 | + conn = BISConnector() | |
| 79 | + body = (fixtures_dir / "bis" / "WS_CBPOL_M.csv").read_bytes() | |
| 80 | + calls = fake_get(conn, lambda url, params: (body, "text/csv")) | |
| 81 | + p = conn.fetch(CBPOL) | |
| 82 | + assert calls[0]["url"] == "https://stats.bis.org/api/v2/data/dataflow/BIS/WS_CBPOL/1.0/M." | |
| 83 | + assert calls[0]["params"] == {"format": "csv"} | |
| 84 | + assert p.content_type == "text/csv" and p.dataset == "WS_CBPOL" and p.code == "M." | |
| 85 | + | |
| 86 | + | |
| 87 | +@pytest.mark.live | |
| 88 | +def test_live_policy_rates_all_areas(): | |
| 89 | + conn = BISConnector() | |
| 90 | + spec = CBPOL.model_copy(update={"params": {"startPeriod": "2026-01"}}) | |
| 91 | + rows = conn.normalize(conn.fetch(spec), spec) | |
| 92 | + isos = {r.country_id for r in rows} | |
| 93 | + assert len(isos) >= 30 and {"CAN", "USA", "GBR", "JPN"} <= isos | |
| 94 | + assert conn.validate(rows).quarantine_dataset is False | |
| 95 | + | |
| 96 | + | |
| 97 | +@pytest.mark.live | |
| 98 | +def test_live_property_prices_real_index(): | |
| 99 | + conn = BISConnector() | |
| 100 | + spec = SPP_R.model_copy(update={"params": {"startPeriod": "2015-Q1"}}) | |
| 101 | + rows = conn.normalize(conn.fetch(spec), spec) | |
| 102 | + assert len({r.country_id for r in rows}) >= 50 | |
added
tests/connectors/test_eurostat.py
+126 −0
@@ -0,0 +1,126 @@ | ||
| 1 | +"""Eurostat JSON-stat connector — recorded fixtures (une_rt_a with EL + EU27_2020; rd_e_gerdtot with `p` flags).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import json | |
| 5 | +from datetime import date | |
| 6 | + | |
| 7 | +import pytest | |
| 8 | + | |
| 9 | +from countryatlas.connectors._util import ConnectorError | |
| 10 | +from countryatlas.connectors.eurostat import EurostatConnector, decode_jsonstat | |
| 11 | +from countryatlas.models import IndicatorSourceSpec, RawPayload | |
| 12 | + | |
| 13 | + | |
| 14 | +def _raw(fixtures_dir, name: str, dataset: str, code: str, retrieved_at) -> RawPayload: | |
| 15 | + body = (fixtures_dir / "eurostat" / name).read_bytes() | |
| 16 | + doc = json.loads(body) | |
| 17 | + return RawPayload(connector="eurostat", dataset=dataset, code=code, url="fixture://eurostat", retrieved_at=retrieved_at, | |
| 18 | + status_code=200, content_type="application/json", body=body, meta={"updated": doc.get("updated")}) | |
| 19 | + | |
| 20 | + | |
| 21 | +@pytest.fixture | |
| 22 | +def es() -> EurostatConnector: | |
| 23 | + return EurostatConnector() | |
| 24 | + | |
| 25 | + | |
| 26 | +def test_decoder_sparse_row_major(): | |
| 27 | + doc = { | |
| 28 | + "id": ["freq", "geo", "time"], "size": [1, 2, 3], | |
| 29 | + "dimension": { | |
| 30 | + "freq": {"category": {"index": {"A": 0}}}, | |
| 31 | + "geo": {"category": {"index": {"DE": 0, "EL": 1}}}, | |
| 32 | + "time": {"category": {"index": {"2022": 0, "2023": 1, "2024": 2}}}, | |
| 33 | + }, | |
| 34 | + "value": {"0": 1.0, "2": 3.0, "4": 5.0}, | |
| 35 | + "status": {"4": "p"}, | |
| 36 | + } | |
| 37 | + cells = decode_jsonstat(doc) | |
| 38 | + assert [(c["dims"]["geo"], c["dims"]["time"], c["value"], c["status"]) for c in cells] == [ | |
| 39 | + ("DE", "2022", 1.0, None), ("DE", "2024", 3.0, None), ("EL", "2023", 5.0, "p"), | |
| 40 | + ] | |
| 41 | + | |
| 42 | + | |
| 43 | +def test_unemployment_maps_el_and_drops_eu_aggregate(es, fixtures_dir, retrieved_at): | |
| 44 | + spec = IndicatorSourceSpec(indicator_id="unemployment-rate", connector="eurostat", dataset="une_rt_a", code="Y15-74.PC_ACT.T", | |
| 45 | + params={"age": "Y15-74", "unit": "PC_ACT", "sex": "T"}, priority=3) | |
| 46 | + rows = es.normalize(_raw(fixtures_dir, "une_rt_a_sample.json", "une_rt_a", spec.code, retrieved_at), spec) | |
| 47 | + assert {r.country_id for r in rows} == {"DEU", "GRC"}, "EL → GRC, EU27_2020 dropped" | |
| 48 | + grc = sorted((r for r in rows if r.country_id == "GRC"), key=lambda r: r.year) | |
| 49 | + assert [r.year for r in grc] == [2022, 2023, 2024, 2025] | |
| 50 | + assert grc[0].period == date(2022, 1, 1) and grc[0].frequency == "A" | |
| 51 | + assert grc[0].value == pytest.approx(12.5) | |
| 52 | + assert grc[0].unit == "% of labour force" | |
| 53 | + assert grc[0].source_updated_at is not None and grc[0].source_updated_at.year == 2026 | |
| 54 | + assert grc[0].metadata["filters"] == {"age": "Y15-74", "unit": "PC_ACT", "sex": "T"} | |
| 55 | + assert es.validate(rows).errors == 0 | |
| 56 | + | |
| 57 | + | |
| 58 | +def test_status_flags_mark_estimates(es, fixtures_dir, retrieved_at): | |
| 59 | + spec = IndicatorSourceSpec(indicator_id="rd-expenditure-pct-gdp", connector="eurostat", dataset="rd_e_gerdtot", | |
| 60 | + code="TOTAL.PC_GDP", params={"sectperf": "TOTAL", "unit": "PC_GDP"}, priority=3) | |
| 61 | + rows = es.normalize(_raw(fixtures_dir, "rd_e_gerdtot_sample.json", "rd_e_gerdtot", spec.code, retrieved_at), spec) | |
| 62 | + assert {r.country_id for r in rows} == {"DEU", "FRA"} | |
| 63 | + flagged = [r for r in rows if r.is_estimate] | |
| 64 | + assert len(flagged) == 2 and all(r.year == 2024 for r in flagged) and all(r.metadata["flags"] == "p" for r in flagged) | |
| 65 | + assert not any(r.is_forecast for r in rows) | |
| 66 | + deu = {r.year: r.value for r in rows if r.country_id == "DEU"} | |
| 67 | + assert deu == {2021: pytest.approx(3.07), 2022: pytest.approx(3.04), 2023: pytest.approx(3.13), 2024: pytest.approx(3.13)} | |
| 68 | + | |
| 69 | + | |
| 70 | +def test_forecast_flag_and_quarterly_time(es, retrieved_at): | |
| 71 | + doc = { | |
| 72 | + "id": ["freq", "unit", "geo", "time"], "size": [1, 1, 2, 2], "updated": "2026-07-02T11:00:00+0200", | |
| 73 | + "dimension": { | |
| 74 | + "freq": {"category": {"index": {"Q": 0}}}, | |
| 75 | + "unit": {"category": {"index": {"PC": 0}}}, | |
| 76 | + "geo": {"category": {"index": {"UK": 0, "FR": 1}}}, | |
| 77 | + "time": {"category": {"index": {"2025-Q4": 0, "2026-Q1": 1}}}, | |
| 78 | + }, | |
| 79 | + "value": {"0": 1.5, "1": 1.7, "2": 2.0, "3": 2.2}, | |
| 80 | + "status": {"3": "f", "1": "bp"}, | |
| 81 | + } | |
| 82 | + raw = RawPayload(connector="eurostat", dataset="x_q", code="PC", url="fixture://", retrieved_at=retrieved_at, | |
| 83 | + status_code=200, body=json.dumps(doc).encode()) | |
| 84 | + spec = IndicatorSourceSpec(indicator_id="inflation", connector="eurostat", dataset="x_q", code="PC", params={"unit": "PC"}) | |
| 85 | + rows = es.normalize(raw, spec) | |
| 86 | + by = {(r.country_id, r.period): r for r in rows} | |
| 87 | + assert set(by) == {("GBR", date(2025, 10, 1)), ("GBR", date(2026, 1, 1)), ("FRA", date(2025, 10, 1)), ("FRA", date(2026, 1, 1))} | |
| 88 | + assert all(r.frequency == "Q" for r in rows) | |
| 89 | + assert by[("FRA", date(2026, 1, 1))].is_forecast is True | |
| 90 | + assert by[("GBR", date(2026, 1, 1))].is_estimate is True and by[("GBR", date(2026, 1, 1))].metadata["flags"] == "bp" | |
| 91 | + | |
| 92 | + | |
| 93 | +def test_ambiguous_dimension_raises(es, fixtures_dir, retrieved_at): | |
| 94 | + body = json.loads((fixtures_dir / "eurostat" / "une_rt_a_sample.json").read_text()) | |
| 95 | + # pretend the sex dimension was not filtered: 2 categories with the same cell count | |
| 96 | + body["id"] = ["freq", "age", "unit", "sex", "geo", "time"] | |
| 97 | + body["size"] = [1, 1, 1, 2, 3, len(body["dimension"]["time"]["category"]["index"])] | |
| 98 | + body["dimension"]["sex"]["category"]["index"] = {"T": 0, "M": 1} | |
| 99 | + raw = RawPayload(connector="eurostat", dataset="une_rt_a", code="x", url="fixture://", retrieved_at=retrieved_at, | |
| 100 | + status_code=200, body=json.dumps(body).encode()) | |
| 101 | + spec = IndicatorSourceSpec(indicator_id="unemployment-rate", connector="eurostat", dataset="une_rt_a", code="x") | |
| 102 | + with pytest.raises(ConnectorError, match="sex="): | |
| 103 | + es.normalize(raw, spec) | |
| 104 | + | |
| 105 | + | |
| 106 | +def test_registry_specs_for_eurostat(): | |
| 107 | + from countryatlas import registry | |
| 108 | + | |
| 109 | + specs = registry.source_specs("eurostat") | |
| 110 | + by_ind = {s.indicator_id: s for s in specs} | |
| 111 | + assert {"employment-rate", "median-household-income", "at-risk-of-poverty-rate", "homeownership-rate", | |
| 112 | + "housing-cost-overburden-rate", "inflation"} <= set(by_ind) | |
| 113 | + for s in specs: | |
| 114 | + assert s.dataset and s.params, s.indicator_id | |
| 115 | + assert "geo" not in s.params and "time" not in s.params | |
| 116 | + | |
| 117 | + | |
| 118 | +@pytest.mark.live | |
| 119 | +def test_live_small_query(): | |
| 120 | + es = EurostatConnector() | |
| 121 | + spec = IndicatorSourceSpec(indicator_id="unemployment-rate", connector="eurostat", dataset="une_rt_a", code="Y15-74.PC_ACT.T", | |
| 122 | + params={"age": "Y15-74", "unit": "PC_ACT", "sex": "T", "geo": ["DE", "EL"], "sinceTimePeriod": 2023}) | |
| 123 | + raw = es.fetch(spec) | |
| 124 | + rows = es.normalize(raw, spec) | |
| 125 | + assert {r.country_id for r in rows} == {"DEU", "GRC"} | |
| 126 | + es.close() | |
added
tests/connectors/test_fred.py
+148 −0
@@ -0,0 +1,148 @@ | ||
| 1 | +"""FRED connector — offline tests on recorded /series + /series/observations payloads, plus a live smoke test.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import json | |
| 5 | +from datetime import date | |
| 6 | + | |
| 7 | +import pytest | |
| 8 | + | |
| 9 | +from countryatlas.connectors import fred as fred_mod | |
| 10 | +from countryatlas.connectors._util import ConnectorError | |
| 11 | +from countryatlas.connectors.fred import FREDConnector, parse_fred_datetime, redact | |
| 12 | +from countryatlas.models import IndicatorSourceSpec | |
| 13 | +from tests.connectors._helpers import make_response, raw_from_file | |
| 14 | + | |
| 15 | +KEY = "0123456789abcdef0123456789abcdef" | |
| 16 | + | |
| 17 | + | |
| 18 | +def _series_meta(fixtures_dir, code): | |
| 19 | + return json.load(open(fixtures_dir / "fred" / f"series_{code}.json"))["seriess"][0] | |
| 20 | + | |
| 21 | + | |
| 22 | +@pytest.fixture | |
| 23 | +def conn(monkeypatch): | |
| 24 | + monkeypatch.setattr(fred_mod.settings, "fred_api_key", KEY) | |
| 25 | + c = FREDConnector() | |
| 26 | + c.MIN_SPACING_S = 0.0 # no pacing in unit tests | |
| 27 | + return c | |
| 28 | + | |
| 29 | + | |
| 30 | +def test_redaction_and_datetime_parsing(): | |
| 31 | + assert redact(f"https://x/fred/series?series_id=A&api_key={KEY}&file_type=json") == \ | |
| 32 | + "https://x/fred/series?series_id=A&api_key=<redacted>&file_type=json" | |
| 33 | + dt = parse_fred_datetime("2026-09-01 15:16:43-05") | |
| 34 | + assert dt is not None and dt.utcoffset().total_seconds() == -5 * 3600 and dt.year == 2026 | |
| 35 | + assert parse_fred_datetime("2026-09-01").tzinfo is not None | |
| 36 | + | |
| 37 | + | |
| 38 | +def test_missing_key_raises(monkeypatch): | |
| 39 | + monkeypatch.setattr(fred_mod.settings, "fred_api_key", None) | |
| 40 | + c = FREDConnector() | |
| 41 | + with pytest.raises(ConnectorError, match="FRED_API_KEY"): | |
| 42 | + c.get("https://api.stlouisfed.org/fred/series", params={"series_id": "FEDFUNDS"}) | |
| 43 | + | |
| 44 | + | |
| 45 | +def test_fetch_adds_key_aggregates_weekly_and_redacts_url(conn, fixtures_dir, monkeypatch): | |
| 46 | + series = (fixtures_dir / "fred" / "series_MORTGAGE30US.json").read_bytes() | |
| 47 | + obs = (fixtures_dir / "fred" / "obs_MORTGAGE30US_m.json").read_bytes() | |
| 48 | + calls: list[dict] = [] | |
| 49 | + | |
| 50 | + # patch the underlying Connector.get (the FRED override adds the key + pacing on top of it) | |
| 51 | + import countryatlas.connectors.base as base_mod | |
| 52 | + | |
| 53 | + def _base_get(self, url, params=None, headers=None): | |
| 54 | + calls.append({"url": url, "params": dict(params or {})}) | |
| 55 | + body = series if url.endswith("/series") else obs | |
| 56 | + return make_response(url, body, "application/json", params) | |
| 57 | + | |
| 58 | + monkeypatch.setattr(base_mod.Connector, "get", _base_get) | |
| 59 | + spec = IndicatorSourceSpec(indicator_id="mortgage-rate", connector="fred", dataset="FRED", code="MORTGAGE30US", | |
| 60 | + countries=["USA"], frequency="M", priority=1) | |
| 61 | + p = conn.fetch(spec) | |
| 62 | + assert calls[0]["params"]["api_key"] == KEY and calls[0]["params"]["file_type"] == "json" | |
| 63 | + assert calls[1]["params"]["frequency"] == "m" and calls[1]["params"]["aggregation_method"] == "avg" | |
| 64 | + assert calls[1]["params"]["observation_start"] == "1950-01-01" | |
| 65 | + assert KEY not in p.url and "api_key=<redacted>" in p.url | |
| 66 | + assert KEY not in json.dumps(p.meta) | |
| 67 | + assert p.source_updated_at is not None and p.source_updated_at.year == 2026 | |
| 68 | + assert p.meta["series_meta"]["frequency_short"] == "W" | |
| 69 | + rows = conn.normalize(p, spec) | |
| 70 | + assert [(r.period.isoformat(), r.value) for r in rows] == [("2026-05-01", 6.44), ("2026-06-01", 6.49), | |
| 71 | + ("2026-07-01", 6.54), ("2026-08-01", 6.67)] | |
| 72 | + assert all(r.frequency == "M" and r.country_id == "USA" for r in rows) # the "." (incomplete Sept) is skipped | |
| 73 | + assert rows[0].metadata["aggregated_from"] == "W" | |
| 74 | + | |
| 75 | + | |
| 76 | +def test_normalize_monthly_series_and_missing_dot(conn, fixtures_dir): | |
| 77 | + spec = IndicatorSourceSpec(indicator_id="policy-rate", connector="fred", dataset="FRED", code="FEDFUNDS", | |
| 78 | + countries=["USA"], frequency="M", priority=2) | |
| 79 | + raw = raw_from_file("fred", "FRED", "FEDFUNDS", fixtures_dir / "fred" / "obs_FEDFUNDS.json", "application/json", | |
| 80 | + series_meta=_series_meta(fixtures_dir, "FEDFUNDS"), request={"series_id": "FEDFUNDS"}) | |
| 81 | + rows = conn.normalize(raw, spec) | |
| 82 | + assert len(rows) == 6 and rows[0].period == date(2024, 1, 1) and rows[0].frequency == "M" | |
| 83 | + assert rows[0].unit == "% per annum" and rows[0].value == pytest.approx(5.33) | |
| 84 | + # inject a missing value | |
| 85 | + doc = json.loads(raw.body) | |
| 86 | + doc["observations"].append({"date": "2024-07-01", "value": "."}) | |
| 87 | + raw.body = json.dumps(doc).encode() | |
| 88 | + assert len(conn.normalize(raw, spec)) == 6 | |
| 89 | + | |
| 90 | + | |
| 91 | +def test_yoy_pct_transform_computes_inflation(conn, fixtures_dir): | |
| 92 | + spec = IndicatorSourceSpec(indicator_id="inflation", connector="fred", dataset="FRED", code="CPIAUCSL", | |
| 93 | + countries=["USA"], frequency="M", priority=3, transform="yoy_pct") | |
| 94 | + raw = raw_from_file("fred", "FRED", "CPIAUCSL", fixtures_dir / "fred" / "obs_CPIAUCSL.json", "application/json", | |
| 95 | + series_meta=_series_meta(fixtures_dir, "CPIAUCSL"), request={"series_id": "CPIAUCSL"}) | |
| 96 | + obs = {o["date"]: float(o["value"]) for o in json.loads(raw.body)["observations"]} | |
| 97 | + rows = conn.normalize(raw, spec) | |
| 98 | + # 2024 has no predecessor in the fixture → only 2025-01..03 remain | |
| 99 | + assert [r.period.isoformat() for r in rows] == ["2025-01-01", "2025-02-01", "2025-03-01"] | |
| 100 | + expected = (obs["2025-01-01"] / obs["2024-01-01"] - 1) * 100 | |
| 101 | + assert rows[0].value == pytest.approx(expected) | |
| 102 | + assert rows[0].unit == "annual %" and rows[0].metadata["transform"] == "yoy_pct" | |
| 103 | + | |
| 104 | + | |
| 105 | +def test_bis_series_derives_country_and_rebases(conn, fixtures_dir): | |
| 106 | + spec = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="fred", dataset="FRED", code="QCAR628BIS", | |
| 107 | + frequency="Q", priority=3, transform="rebase:2015") | |
| 108 | + assert FREDConnector.country_for(spec) == "CAN" | |
| 109 | + assert FREDConnector.country_for(spec.model_copy(update={"code": "QXMR628BIS"})) is None # euro area | |
| 110 | + raw = raw_from_file("fred", "FRED", "QCAR628BIS", fixtures_dir / "fred" / "obs_QCAR628BIS.json", "application/json", | |
| 111 | + series_meta=_series_meta(fixtures_dir, "QCAR628BIS"), request={"series_id": "QCAR628BIS"}) | |
| 112 | + rows = conn.normalize(raw, spec) | |
| 113 | + assert all(r.country_id == "CAN" and r.frequency == "Q" for r in rows) | |
| 114 | + y2015 = [r.value for r in rows if r.year == 2015] | |
| 115 | + assert len(y2015) == 4 and sum(y2015) / 4 == pytest.approx(100.0) | |
| 116 | + assert rows[0].period == date(2015, 1, 1) and rows[1].period == date(2015, 4, 1) | |
| 117 | + assert rows[0].unit == "index (2015 = 100)" | |
| 118 | + | |
| 119 | + | |
| 120 | +def test_quarterly_and_annual_periods(conn, fixtures_dir): | |
| 121 | + meta = _series_meta(fixtures_dir, "FEDFUNDS") | |
| 122 | + body = json.dumps({"observations": [{"date": "2025-04-01", "value": "65.1"}, {"date": "2025-07-01", "value": "65.3"}]}).encode() | |
| 123 | + raw = raw_from_file("fred", "FRED", "RHORUSQ156N", fixtures_dir / "fred" / "obs_FEDFUNDS.json", "application/json", | |
| 124 | + series_meta={**meta, "frequency_short": "Q"}, request={}) | |
| 125 | + raw.body = body | |
| 126 | + spec = IndicatorSourceSpec(indicator_id="homeownership-rate", connector="fred", dataset="FRED", code="RHORUSQ156N", | |
| 127 | + countries=["USA"], frequency="Q") | |
| 128 | + rows = conn.normalize(raw, spec) | |
| 129 | + assert [(r.period, r.frequency) for r in rows] == [(date(2025, 4, 1), "Q"), (date(2025, 7, 1), "Q")] | |
| 130 | + raw.meta["series_meta"]["frequency_short"] = "A" | |
| 131 | + raw.body = json.dumps({"observations": [{"date": "2024-01-01", "value": "83730"}]}).encode() | |
| 132 | + spec = spec.model_copy(update={"indicator_id": "median-household-income", "code": "MEHOINUSA672N", "frequency": "A"}) | |
| 133 | + rows = conn.normalize(raw, spec) | |
| 134 | + assert rows[0].period == date(2024, 1, 1) and rows[0].frequency == "A" and rows[0].year == 2024 | |
| 135 | + | |
| 136 | + | |
| 137 | +@pytest.mark.live | |
| 138 | +def test_live_fedfunds(monkeypatch): | |
| 139 | + import os | |
| 140 | + | |
| 141 | + if not os.environ.get("FRED_API_KEY"): | |
| 142 | + pytest.skip("FRED_API_KEY not set") | |
| 143 | + monkeypatch.setattr(fred_mod.settings, "fred_api_key", os.environ["FRED_API_KEY"]) | |
| 144 | + conn = FREDConnector() | |
| 145 | + spec = IndicatorSourceSpec(indicator_id="policy-rate", connector="fred", dataset="FRED", code="FEDFUNDS", | |
| 146 | + countries=["USA"], frequency="M", params={"observation_start": "2024-01-01"}) | |
| 147 | + rows = conn.normalize(conn.fetch(spec), spec) | |
| 148 | + assert len(rows) >= 20 and rows[0].country_id == "USA" | |
added
tests/connectors/test_ilo.py
+77 −0
@@ -0,0 +1,77 @@ | ||
| 1 | +"""ILOSTAT connector — offline tests on a trimmed DF_UNE_2EAP_SEX_AGE_RT CSV (CAN, FRA, SOM, X01), plus live smoke.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from datetime import date | |
| 5 | + | |
| 6 | +import pytest | |
| 7 | + | |
| 8 | +from countryatlas.connectors.ilo import ILOConnector | |
| 9 | +from countryatlas.models import IndicatorSourceSpec | |
| 10 | +from tests.connectors._helpers import install_fake_get, raw_from_file | |
| 11 | + | |
| 12 | + | |
| 13 | +@pytest.fixture | |
| 14 | +def fake_get(monkeypatch): | |
| 15 | + return install_fake_get(monkeypatch) | |
| 16 | + | |
| 17 | + | |
| 18 | +SPEC =IndicatorSourceSpec(indicator_id="unemployment-rate", connector="ilo", dataset="DF_UNE_2EAP_SEX_AGE_RT", | |
| 19 | + code=".A..SEX_T.AGE_YTHADULT_YGE15", frequency="A", priority=4) | |
| 20 | + | |
| 21 | + | |
| 22 | +def _raw(fixtures_dir, **meta): | |
| 23 | + return raw_from_file("ilo", SPEC.dataset, SPEC.code, fixtures_dir / "ilo" / "DF_UNE_2EAP_SEX_AGE_RT.csv", "text/csv", **meta) | |
| 24 | + | |
| 25 | + | |
| 26 | +def test_normalize_drops_aggregates_flags_projections_and_imputations(fixtures_dir): | |
| 27 | + conn = ILOConnector() | |
| 28 | + rows = conn.normalize(_raw(fixtures_dir, dataflow_meta={"release_year": 2025, "name": "… Nov. 2025"}), SPEC) | |
| 29 | + assert {r.country_id for r in rows} == {"CAN", "FRA", "SOM"} # X01 aggregate dropped | |
| 30 | + can = {r.year: r for r in rows if r.country_id == "CAN"} | |
| 31 | + assert can[2024].is_estimate is False and can[2024].is_forecast is False # OBS_STATUS = R | |
| 32 | + assert can[2025].is_estimate is True and can[2025].is_forecast is False # nowcast (blank status), release year | |
| 33 | + assert can[2026].is_forecast is True and can[2027].is_forecast is True | |
| 34 | + assert can[2025].value == pytest.approx(6.907) # = WB SL.UEM.TOTL.ZS 2025 | |
| 35 | + som = {r.year: r for r in rows if r.country_id == "SOM"} | |
| 36 | + assert som[2024].is_estimate is True and som[2024].metadata["low"] == pytest.approx(4.737) | |
| 37 | + assert all(r.frequency == "A" and r.period == date(r.year, 1, 1) and r.unit == "% of labour force" for r in rows) | |
| 38 | + assert conn.validate(rows).quarantine_dataset is False | |
| 39 | + | |
| 40 | + | |
| 41 | +def test_release_year_fallback_is_current_year(fixtures_dir): | |
| 42 | + from datetime import UTC, datetime | |
| 43 | + | |
| 44 | + rows = ILOConnector().normalize(_raw(fixtures_dir), SPEC) # no dataflow_meta → years > now.year are forecasts | |
| 45 | + y = datetime.now(UTC).year | |
| 46 | + assert all(r.is_forecast == (r.year > y) for r in rows) | |
| 47 | + | |
| 48 | + | |
| 49 | +def test_fetch_builds_verified_url_and_parses_release_year(fixtures_dir, fake_get): | |
| 50 | + conn = ILOConnector() | |
| 51 | + csv_body = (fixtures_dir / "ilo" / "DF_UNE_2EAP_SEX_AGE_RT.csv").read_bytes() | |
| 52 | + flow = (fixtures_dir / "ilo" / "dataflow.json").read_bytes() | |
| 53 | + | |
| 54 | + def router(url, params): | |
| 55 | + if "/dataflow/ILO/" in url: | |
| 56 | + return flow, "application/vnd.sdmx.structure+json" | |
| 57 | + return csv_body, "text/csv" | |
| 58 | + | |
| 59 | + calls = fake_get(conn, router) | |
| 60 | + p = conn.fetch(SPEC) | |
| 61 | + assert calls[0]["url"] == "https://sdmx.ilo.org/rest/dataflow/ILO/DF_UNE_2EAP_SEX_AGE_RT" | |
| 62 | + assert calls[1]["url"] == "https://sdmx.ilo.org/rest/data/ILO,DF_UNE_2EAP_SEX_AGE_RT/.A..SEX_T.AGE_YTHADULT_YGE15" | |
| 63 | + assert calls[1]["params"] == {"format": "csv"} | |
| 64 | + assert p.meta["dataflow_meta"]["release_year"] == 2025 | |
| 65 | + assert "Nov. 2025" in p.meta["dataflow_meta"]["name"] | |
| 66 | + rows = conn.normalize(p, SPEC) | |
| 67 | + assert {r.year for r in rows if r.is_forecast} == {2026, 2027} | |
| 68 | + | |
| 69 | + | |
| 70 | +@pytest.mark.live | |
| 71 | +def test_live_youth_unemployment(): | |
| 72 | + conn = ILOConnector() | |
| 73 | + spec = SPEC.model_copy(update={"indicator_id": "youth-unemployment-rate", "code": ".A..SEX_T.AGE_YTHADULT_Y15-24", | |
| 74 | + "params": {"startPeriod": "2023"}}) | |
| 75 | + rows = conn.normalize(conn.fetch(spec), spec) | |
| 76 | + assert len({r.country_id for r in rows}) >= 150 and "CAN" in {r.country_id for r in rows} | |
| 77 | + assert conn.validate(rows).quarantine_dataset is False | |
added
tests/connectors/test_imf.py
+88 −0
@@ -0,0 +1,88 @@ | ||
| 1 | +"""IMF WEO connector — recorded SDMX-CSV fixtures (CAN, USA, KOS, G001 for NGDP_RPCH; CAN, USA for NGDPD).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from datetime import date | |
| 5 | + | |
| 6 | +import pytest | |
| 7 | + | |
| 8 | +from countryatlas.connectors.imf import IMFConnector | |
| 9 | +from countryatlas.models import IndicatorSourceSpec, RawPayload | |
| 10 | + | |
| 11 | + | |
| 12 | +def _raw(fixtures_dir, name: str, code: str, retrieved_at) -> RawPayload: | |
| 13 | + body = (fixtures_dir / "imf" / name).read_bytes() | |
| 14 | + return RawPayload( | |
| 15 | + connector="imf", dataset="WEO", code=code, url="fixture://imf", retrieved_at=retrieved_at, status_code=200, | |
| 16 | + content_type="text/csv", body=body, meta={"publication_date": "2026-04-14T13:00:00Z"}, | |
| 17 | + ) | |
| 18 | + | |
| 19 | + | |
| 20 | +@pytest.fixture | |
| 21 | +def imf() -> IMFConnector: | |
| 22 | + return IMFConnector() | |
| 23 | + | |
| 24 | + | |
| 25 | +def test_normalize_growth_maps_iso3_and_flags_forecasts(imf, fixtures_dir, retrieved_at): | |
| 26 | + spec = IndicatorSourceSpec(indicator_id="gdp-growth", connector="imf", dataset="WEO", code="NGDP_RPCH", priority=2) | |
| 27 | + rows = imf.normalize(_raw(fixtures_dir, "weo_NGDP_RPCH_sample.csv", "NGDP_RPCH", retrieved_at), spec) | |
| 28 | + countries = {r.country_id for r in rows} | |
| 29 | + assert countries == {"CAN", "USA", "XKX"}, "G001 (World aggregate) dropped, KOS mapped to XKX" | |
| 30 | + can = sorted((r for r in rows if r.country_id == "CAN"), key=lambda r: r.year) | |
| 31 | + assert [r.year for r in can] == list(range(2022, 2029)) | |
| 32 | + assert can[0].period == date(2022, 1, 1) and can[0].frequency == "A" | |
| 33 | + # LATEST_ACTUAL_ANNUAL_DATA = 2025 → 2026+ are projections | |
| 34 | + assert {r.year: r.is_forecast for r in can} == {2022: False, 2023: False, 2024: False, 2025: False, 2026: True, | |
| 35 | + 2027: True, 2028: True} | |
| 36 | + assert all(r.is_estimate is False for r in rows) | |
| 37 | + assert can[0].value == pytest.approx(4.69542) | |
| 38 | + assert can[0].unit == "annual %" | |
| 39 | + assert can[0].source_updated_at is not None and can[0].source_updated_at.date() == date(2026, 4, 14) | |
| 40 | + assert can[0].metadata["latest_actual_annual_data"] == 2025 | |
| 41 | + assert can[0].source_dataset == "WEO" and can[0].source_series_code == "NGDP_RPCH" | |
| 42 | + | |
| 43 | + | |
| 44 | +def test_normalize_gdp_values_are_base_units_without_transform(imf, fixtures_dir, retrieved_at): | |
| 45 | + spec = IndicatorSourceSpec(indicator_id="gdp", connector="imf", dataset="WEO", code="NGDPD", priority=2) | |
| 46 | + rows = imf.normalize(_raw(fixtures_dir, "weo_NGDPD_sample.csv", "NGDPD", retrieved_at), spec) | |
| 47 | + can2023 = next(r for r in rows if r.country_id == "CAN" and r.year == 2023) | |
| 48 | + assert can2023.value == pytest.approx(2.196593836e12, rel=1e-6), "OBS_VALUE already in US$, SCALE=9 is display-only" | |
| 49 | + assert can2023.metadata.get("scale") == "9" | |
| 50 | + assert can2023.unit == "current US$" | |
| 51 | + | |
| 52 | + | |
| 53 | +def test_transform_is_applied_when_present(imf, fixtures_dir, retrieved_at): | |
| 54 | + spec = IndicatorSourceSpec(indicator_id="gdp", connector="imf", dataset="WEO", code="NGDPD", priority=2, transform="x/1e9") | |
| 55 | + rows = imf.normalize(_raw(fixtures_dir, "weo_NGDPD_sample.csv", "NGDPD", retrieved_at), spec) | |
| 56 | + can2023 = next(r for r in rows if r.country_id == "CAN" and r.year == 2023) | |
| 57 | + assert can2023.value == pytest.approx(2196.593836, rel=1e-6) | |
| 58 | + | |
| 59 | + | |
| 60 | +def test_validate_no_duplicates(imf, fixtures_dir, retrieved_at): | |
| 61 | + spec = IndicatorSourceSpec(indicator_id="gdp-growth", connector="imf", dataset="WEO", code="NGDP_RPCH", priority=2) | |
| 62 | + rows = imf.normalize(_raw(fixtures_dir, "weo_NGDP_RPCH_sample.csv", "NGDP_RPCH", retrieved_at), spec) | |
| 63 | + report = imf.validate(rows) | |
| 64 | + assert report.errors == 0 and not report.quarantine_dataset | |
| 65 | + | |
| 66 | + | |
| 67 | +def test_registry_specs_for_imf_include_inline_and_extra(): | |
| 68 | + from countryatlas import registry | |
| 69 | + | |
| 70 | + specs = registry.source_specs("imf") | |
| 71 | + codes = {s.code for s in specs} | |
| 72 | + assert {"NGDP_RPCH", "PCPIPCH", "LUR", "GGXWDG_NGDP", "NGSD_NGDP", "NID_NGDP", "LP"} <= codes | |
| 73 | + for s in specs: | |
| 74 | + assert s.dataset == "WEO" | |
| 75 | + assert s.transform is None or "1e9" not in s.transform, f"{s.code}: OBS_VALUE is already in base units" | |
| 76 | + | |
| 77 | + | |
| 78 | +@pytest.mark.live | |
| 79 | +def test_live_fetch_small_key(): | |
| 80 | + imf = IMFConnector() | |
| 81 | + spec = IndicatorSourceSpec(indicator_id="unemployment-rate", connector="imf", dataset="WEO", code="LUR", priority=2, | |
| 82 | + params={"countries": ["CAN", "USA"], "startPeriod": 2023, "endPeriod": 2027}) | |
| 83 | + raw = imf.fetch(spec) | |
| 84 | + assert raw.status_code == 200 and raw.body.startswith(b"DATAFLOW") | |
| 85 | + rows = imf.normalize(raw, spec) | |
| 86 | + assert {r.country_id for r in rows} == {"CAN", "USA"} | |
| 87 | + assert any(r.is_forecast for r in rows) and any(not r.is_forecast for r in rows) | |
| 88 | + imf.close() | |
added
tests/connectors/test_oecd.py
+127 −0
@@ -0,0 +1,127 @@ | ||
| 1 | +"""OECD SDMX connector — recorded csvfilewithlabels fixtures (house prices RHP quarterly; KEI production index monthly).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from datetime import date | |
| 5 | + | |
| 6 | +import pytest | |
| 7 | + | |
| 8 | +from countryatlas.connectors._util import ConnectorError | |
| 9 | +from countryatlas.connectors.oecd import OECDConnector, _clean_key, _with_version | |
| 10 | +from countryatlas.models import IndicatorSourceSpec, RawPayload | |
| 11 | + | |
| 12 | +HP = "OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0" | |
| 13 | +KEI = "OECD.SDD.STES,DSD_KEI@DF_KEI,4.0" | |
| 14 | + | |
| 15 | + | |
| 16 | +def _raw(fixtures_dir, name: str, dataset: str, code: str, retrieved_at) -> RawPayload: | |
| 17 | + body = (fixtures_dir / "oecd" / name).read_bytes() | |
| 18 | + return RawPayload(connector="oecd", dataset=dataset, code=code, url="fixture://oecd", retrieved_at=retrieved_at, | |
| 19 | + status_code=200, content_type="text/csv", body=body) | |
| 20 | + | |
| 21 | + | |
| 22 | +@pytest.fixture | |
| 23 | +def oecd() -> OECDConnector: | |
| 24 | + return OECDConnector() | |
| 25 | + | |
| 26 | + | |
| 27 | +def test_key_and_version_helpers(): | |
| 28 | + assert _clean_key("*.Q.RHP.IX") == ".Q.RHP.IX" | |
| 29 | + assert _clean_key(".Q.RHP.IX") == ".Q.RHP.IX" | |
| 30 | + assert _with_version("OECD.ELS.SPD,DSD_SOCX_AGG@DF_SOCX_AGG") == "OECD.ELS.SPD,DSD_SOCX_AGG@DF_SOCX_AGG," | |
| 31 | + assert _with_version(HP) == HP | |
| 32 | + | |
| 33 | + | |
| 34 | +def test_quarterly_index(oecd, fixtures_dir, retrieved_at): | |
| 35 | + spec = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="oecd", dataset=HP, code="*.Q.RHP.IX", | |
| 36 | + priority=1, frequency="Q") | |
| 37 | + rows = oecd.normalize(_raw(fixtures_dir, "house_prices_RHP_sample.csv", HP, "*.Q.RHP.IX", retrieved_at), spec) | |
| 38 | + assert {r.country_id for r in rows} == {"CAN", "USA"} | |
| 39 | + usa = sorted((r for r in rows if r.country_id == "USA"), key=lambda r: r.period) | |
| 40 | + assert usa[0].period == date(2023, 1, 1) and usa[0].frequency == "Q" and usa[0].year == 2023 | |
| 41 | + assert any(r.period == date(2025, 4, 1) for r in usa), "2025-Q2 → 2025-04-01" | |
| 42 | + q2 = next(r for r in usa if r.period == date(2025, 4, 1)) | |
| 43 | + assert q2.value == pytest.approx(154.108309038445) | |
| 44 | + assert q2.unit == "index (2015 = 100)" | |
| 45 | + assert q2.metadata["source_unit"] == "IX" and q2.metadata["base_period"] == "2015" | |
| 46 | + assert not q2.is_estimate and not q2.is_forecast | |
| 47 | + assert oecd.validate(rows).errors == 0 | |
| 48 | + | |
| 49 | + | |
| 50 | +def test_yoy_transform_derives_growth(oecd, fixtures_dir, retrieved_at): | |
| 51 | + spec = IndicatorSourceSpec(indicator_id="house-price-growth", connector="oecd", dataset=HP, code="*.Q.RHP.IX", | |
| 52 | + priority=1, frequency="Q", transform="yoy") | |
| 53 | + raw = _raw(fixtures_dir, "house_prices_RHP_sample.csv", HP, "*.Q.RHP.IX", retrieved_at) | |
| 54 | + idx = {(r.country_id, r.period): r.value for r in oecd.normalize( | |
| 55 | + raw, IndicatorSourceSpec(indicator_id="real-house-price-index", connector="oecd", dataset=HP, code="*.Q.RHP.IX"))} | |
| 56 | + rows = oecd.normalize(raw, spec) | |
| 57 | + assert rows, "growth rows exist once a full year of history is present" | |
| 58 | + # first four quarters (2023) have no previous-year value → dropped | |
| 59 | + assert min(r.period for r in rows) == date(2024, 1, 1) | |
| 60 | + r = next(r for r in rows if r.country_id == "CAN" and r.period == date(2025, 1, 1)) | |
| 61 | + expected = (idx[("CAN", date(2025, 1, 1))] / idx[("CAN", date(2024, 1, 1))] - 1) * 100 | |
| 62 | + assert r.value == pytest.approx(expected) | |
| 63 | + assert r.unit == "annual %" and r.metadata["derived"].startswith("year-on-year") | |
| 64 | + | |
| 65 | + | |
| 66 | +def test_monthly_kei_periods(oecd, fixtures_dir, retrieved_at): | |
| 67 | + spec = IndicatorSourceSpec(indicator_id="industrial-production-index", connector="oecd", dataset=KEI, | |
| 68 | + code="*.M.PRVM.IX.BTE.Y._Z", priority=1, frequency="M") | |
| 69 | + rows = oecd.normalize(_raw(fixtures_dir, "kei_PRVM_sample.csv", KEI, spec.code, retrieved_at), spec) | |
| 70 | + can = sorted((r for r in rows if r.country_id == "CAN"), key=lambda r: r.period) | |
| 71 | + assert can[0].frequency == "M" and can[0].period.day == 1 and can[0].period.year == 2026 | |
| 72 | + assert any(r.period == date(2026, 6, 1) for r in can) | |
| 73 | + assert next(r for r in can if r.period == date(2026, 6, 1)).value == pytest.approx(109.273561067984) | |
| 74 | + | |
| 75 | + | |
| 76 | +def test_post_hoc_filter_and_estimate_flag(oecd, fixtures_dir, retrieved_at): | |
| 77 | + body = (fixtures_dir / "oecd" / "kei_PRVM_sample.csv").read_text() | |
| 78 | + # mark one row as estimate and another as provisional | |
| 79 | + lines = body.splitlines() | |
| 80 | + lines[1] = lines[1].replace(",A,Normal value,", ",E,Estimated value,", 1) | |
| 81 | + lines[2] = lines[2].replace(",A,Normal value,", ",P,Provisional value,", 1) | |
| 82 | + raw = RawPayload(connector="oecd", dataset=KEI, code="*.M.PRVM.IX.BTE.Y._Z", url="fixture://oecd", | |
| 83 | + retrieved_at=retrieved_at, status_code=200, content_type="text/csv", body="\n".join(lines).encode()) | |
| 84 | + spec = IndicatorSourceSpec(indicator_id="industrial-production-index", connector="oecd", dataset=KEI, | |
| 85 | + code="*.M.PRVM.IX.BTE.Y._Z", priority=1, frequency="M", params={"filter": {"ACTIVITY": "BTE"}}) | |
| 86 | + rows = oecd.normalize(raw, spec) | |
| 87 | + assert sum(r.is_estimate for r in rows) == 2 | |
| 88 | + assert {r.metadata.get("obs_status") for r in rows if r.is_estimate} == {"E", "P"} | |
| 89 | + bad = IndicatorSourceSpec(indicator_id="industrial-production-index", connector="oecd", dataset=KEI, | |
| 90 | + code="*.M.PRVM.IX.BTE.Y._Z", params={"filter": {"ACTIVITY": "ZZZ"}}) | |
| 91 | + assert oecd.normalize(raw, bad) == [] | |
| 92 | + | |
| 93 | + | |
| 94 | +def test_underspecified_key_raises(oecd, fixtures_dir, retrieved_at): | |
| 95 | + body = (fixtures_dir / "oecd" / "house_prices_RHP_sample.csv").read_text() | |
| 96 | + dup = body + "\n" + body.splitlines()[1].replace(",RHP,Real house price indices,", ",HPI,Nominal house price indices,") | |
| 97 | + raw = RawPayload(connector="oecd", dataset=HP, code="*.Q..IX", url="fixture://oecd", retrieved_at=retrieved_at, | |
| 98 | + status_code=200, content_type="text/csv", body=dup.encode()) | |
| 99 | + spec = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="oecd", dataset=HP, code="*.Q..IX") | |
| 100 | + with pytest.raises(ConnectorError, match="under-specified"): | |
| 101 | + oecd.normalize(raw, spec) | |
| 102 | + | |
| 103 | + | |
| 104 | +def test_registry_specs_for_oecd(): | |
| 105 | + from countryatlas import registry | |
| 106 | + | |
| 107 | + specs = registry.source_specs("oecd") | |
| 108 | + by_ind = {s.indicator_id: s for s in specs} | |
| 109 | + assert {"real-house-price-index", "tax-revenue-pct-gdp", "average-annual-wages", "industrial-production-index", | |
| 110 | + "tertiary-attainment-25-64"} <= set(by_ind) | |
| 111 | + assert by_ind["industrial-production-index"].frequency == "M" | |
| 112 | + assert by_ind["house-price-growth"].transform == "yoy" | |
| 113 | + for s in specs: | |
| 114 | + assert s.dataset.count(",") in (1, 2), s.dataset | |
| 115 | + assert s.code.startswith("*."), s.code | |
| 116 | + | |
| 117 | + | |
| 118 | +@pytest.mark.live | |
| 119 | +def test_live_house_prices_two_countries(): | |
| 120 | + oecd = OECDConnector() | |
| 121 | + spec = IndicatorSourceSpec(indicator_id="real-house-price-index", connector="oecd", dataset=HP, code="CAN+USA.Q.RHP.IX", | |
| 122 | + priority=1, frequency="Q", params={"startPeriod": "2024-Q1"}) | |
| 123 | + raw = oecd.fetch(spec) | |
| 124 | + rows = oecd.normalize(raw, spec) | |
| 125 | + assert {r.country_id for r in rows} == {"CAN", "USA"} | |
| 126 | + assert all(r.frequency == "Q" for r in rows) | |
| 127 | + oecd.close() | |
added
tests/connectors/test_who.py
+89 −0
@@ -0,0 +1,89 @@ | ||
| 1 | +"""WHO GHO connector — offline tests on a trimmed recorded payload (M_Est_tob_curr, CAN + USA + decoys) + live smoke.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from datetime import UTC, datetime | |
| 5 | + | |
| 6 | +import orjson | |
| 7 | +import pytest | |
| 8 | + | |
| 9 | +from countryatlas.connectors.who import WHOConnector | |
| 10 | +from countryatlas.models import IndicatorSourceSpec | |
| 11 | +from tests.connectors._helpers import install_fake_get, raw_from_file | |
| 12 | + | |
| 13 | + | |
| 14 | +@pytest.fixture | |
| 15 | +def fake_get(monkeypatch): | |
| 16 | + return install_fake_get(monkeypatch) | |
| 17 | + | |
| 18 | + | |
| 19 | +SPEC =IndicatorSourceSpec(indicator_id="smoking-prevalence", connector="who", dataset="GHO", code="M_Est_tob_curr", | |
| 20 | + params={"Dim1": "SEX_BTSX"}, priority=2) | |
| 21 | + | |
| 22 | + | |
| 23 | +def _raw(fixtures_dir): | |
| 24 | + return raw_from_file("who", "GHO", "M_Est_tob_curr", fixtures_dir / "who" / "M_Est_tob_curr.json", "application/json", | |
| 25 | + indicator_meta={"name": "Estimate of current tobacco use prevalence (%)"}) | |
| 26 | + | |
| 27 | + | |
| 28 | +def test_build_filter_includes_country_type_and_dims(): | |
| 29 | + f = WHOConnector.build_filter({"Dim1": "SEX_BTSX", "Dim2": "AGEGROUP_YEARSALL"}) | |
| 30 | + assert f == "SpatialDimType eq 'COUNTRY' and Dim1 eq 'SEX_BTSX' and Dim2 eq 'AGEGROUP_YEARSALL'" | |
| 31 | + assert WHOConnector.build_filter(None) == "SpatialDimType eq 'COUNTRY'" | |
| 32 | + | |
| 33 | + | |
| 34 | +def test_normalize_maps_iso3_filters_dims_and_flags_projections(fixtures_dir): | |
| 35 | + conn = WHOConnector() | |
| 36 | + rows = conn.normalize(_raw(fixtures_dir), SPEC) | |
| 37 | + # decoys: REGION row (AMR), SEX_MLE row, unknown ISO3 'ZZZ' are all dropped | |
| 38 | + assert {r.country_id for r in rows} == {"CAN", "USA"} | |
| 39 | + assert all(r.frequency == "A" and r.period.month == 1 and r.period.day == 1 for r in rows) | |
| 40 | + assert all(r.unit == "% of adults" for r in rows) | |
| 41 | + can = {r.year: r for r in rows if r.country_id == "CAN"} | |
| 42 | + assert can[2030].is_forecast is True and can[2030].value == pytest.approx(8.3) | |
| 43 | + assert can[2024].is_forecast is False | |
| 44 | + assert all(r.year <= datetime.now(UTC).year or r.is_forecast for r in rows) | |
| 45 | + # confidence bounds go to metadata; the row `Date` feeds source_updated_at | |
| 46 | + assert "low" in rows[0].metadata and "high" in rows[0].metadata | |
| 47 | + assert rows[0].source_updated_at is not None and rows[0].source_updated_at.year >= 2024 | |
| 48 | + # no duplicate keys | |
| 49 | + keys = {(r.country_id, r.period) for r in rows} | |
| 50 | + assert len(keys) == len(rows) | |
| 51 | + assert conn.validate(rows).quarantine_dataset is False | |
| 52 | + | |
| 53 | + | |
| 54 | +def test_normalize_applies_scalar_transform(fixtures_dir): | |
| 55 | + spec = SPEC.model_copy(update={"transform": "x/10", "indicator_id": "nurses-per-1000"}) | |
| 56 | + rows = WHOConnector().normalize(_raw(fixtures_dir), spec) | |
| 57 | + can = {r.year: r for r in rows if r.country_id == "CAN"} | |
| 58 | + assert can[2030].value == pytest.approx(0.83) | |
| 59 | + assert can[2030].unit == "per 1,000 people" | |
| 60 | + | |
| 61 | + | |
| 62 | +def test_fetch_uses_filter_and_metadata(fixtures_dir, fake_get): | |
| 63 | + conn = WHOConnector() | |
| 64 | + data = (fixtures_dir / "who" / "M_Est_tob_curr.json").read_bytes() | |
| 65 | + meta = (fixtures_dir / "who" / "Indicator_M_Est_tob_curr.json").read_bytes() | |
| 66 | + | |
| 67 | + def router(url, params): | |
| 68 | + if url.endswith("/Indicator"): | |
| 69 | + return meta, "application/json" | |
| 70 | + return data, "application/json" | |
| 71 | + | |
| 72 | + calls = fake_get(conn, router) | |
| 73 | + payloads = conn.fetch(SPEC) | |
| 74 | + assert len(payloads) == 1 and payloads[0].pages == 1 | |
| 75 | + assert calls[0]["url"].endswith("/Indicator") and "M_Est_tob_curr" in calls[0]["params"]["$filter"] | |
| 76 | + assert calls[1]["url"].endswith("/M_Est_tob_curr") | |
| 77 | + assert calls[1]["params"]["$filter"] == "SpatialDimType eq 'COUNTRY' and Dim1 eq 'SEX_BTSX'" | |
| 78 | + assert payloads[0].meta["indicator_meta"]["name"].startswith("Estimate of current tobacco") | |
| 79 | + assert orjson.loads(payloads[0].body)["value"] | |
| 80 | + | |
| 81 | + | |
| 82 | +@pytest.mark.live | |
| 83 | +def test_live_suicide_rate_has_no_duplicates(): | |
| 84 | + conn = WHOConnector() | |
| 85 | + spec = IndicatorSourceSpec(indicator_id="suicide-rate", connector="who", dataset="GHO", code="SDGSUICIDE", | |
| 86 | + params={"Dim1": "SEX_BTSX", "Dim2": "AGEGROUP_YEARSALL"}) | |
| 87 | + rows = conn.normalize(conn.fetch(spec), spec) | |
| 88 | + assert len(rows) > 3000 and "CAN" in {r.country_id for r in rows} | |
| 89 | + assert conn.validate(rows).quarantine_dataset is False | |
added
tests/fixtures/__init__.py
+0 −0
added
tests/fixtures/bis/WS_CBPOL_M.csv
+62 −0
@@ -0,0 +1,62 @@ | ||
| 1 | +FREQ,REF_AREA,UNIT_MEASURE,UNIT_MULT,TIME_FORMAT,COMPILATION,DECIMALS,SOURCE_REF,SUPP_INFO_BREAKS,TITLE,TIME_PERIOD,OBS_VALUE,OBS_STATUS,OBS_CONF,OBS_PRE_BREAK | |
| 2 | +M,AR,368,0,,"From 10 July 2025 onwards: no policy rate adopted; from 22 July 2024 to 9 July 2025: liquidity absorption rate for treasury bills; from 18 December 2023 to 21 July 2024: Overnight Reverse Repos Interest Rate; from 6 January 2022 to 17 December 2023: 28-day Liquidity Bills (LELIQ) interest rate; from 21 January 2020 to 5 January 2022: weighted average interest rate of minimum term LELIQ issued at the last auction process; from 1 October 2018 to 20 January 2020, average interest rate of the accepted offers for the liquidity bills; from 8 August 2018 to 30 September 2018: 7 days liquidity bills interest rate; from 2 January 2017 to 7 August 2018, median of the repo rate corridor; from 16 December 2015 to 1 January 2017: interest rate in BCRA bills (LEBACs), 35 days LEBAC auction; from 29 Jan 2014 to 15 Dec 2015: CB issues, 3 months; from 11 Sep 2009 to 28 Jan 2014:7-day reverse repo operations; before 10 Sep 2009, refer to https://data.bis.org/topics/CBPOL",4,Central Bank of Argentina,"From 10 July 2025 onwards, the monetary policy of the Central Bank of Argentina is based on controlling monetary aggregates, eliminating the concept of a ""monetary policy interest rate"".", Central bank policy rates - Argentina - Monthly - End of period,2001-12,NaN,M,F, | |
| 3 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-01,3,A,F, | |
| 4 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-02,3,A,F, | |
| 5 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-03,2.75,A,F, | |
| 6 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-04,2.75,A,F, | |
| 7 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-05,2.75,A,F, | |
| 8 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-06,2.75,A,F, | |
| 9 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-07,2.75,A,F, | |
| 10 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-08,2.75,A,F, | |
| 11 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-09,2.5,A,F, | |
| 12 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-10,2.25,A,F, | |
| 13 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-11,2.25,A,F, | |
| 14 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2025-12,2.25,A,F, | |
| 15 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2026-01,2.25,A,F, | |
| 16 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2026-02,2.25,A,F, | |
| 17 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2026-03,2.25,A,F, | |
| 18 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2026-04,2.25,A,F, | |
| 19 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2026-05,2.25,A,F, | |
| 20 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2026-06,2.25,A,F, | |
| 21 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2026-07,2.25,A,F, | |
| 22 | +M,CA,368,0,,"From 1 Jun 1994 onwards: Central Bank target, overnight rate; from 27 Jul 1960 to 31 May 1994: official bank rate.",4,Bank of Canada,"In Jun 1994 the Bank of Canada (BoC) began shifting emphasis from the bank rate to the target for the overnight rate as its key monetary policy instrument. In Feb 1999 the target for the overnight rate was defined as the midpoint of the band, or 25 basis points below the bank rate. In May 2001 the BoC began emphasizing the target as its key interest rate in its communications with the public.", Central bank policy rates - Canada - Monthly - End of period,2026-08,2.25,A,F, | |
| 23 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-01,3,A,F, | |
| 24 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-02,2.75,A,F, | |
| 25 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-03,2.5,A,F, | |
| 26 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-04,2.25,A,F, | |
| 27 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-05,2.25,A,F, | |
| 28 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-06,2,A,F, | |
| 29 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-07,2,A,F, | |
| 30 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-08,2,A,F, | |
| 31 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-09,2,A,F, | |
| 32 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-10,2,A,F, | |
| 33 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-11,2,A,F, | |
| 34 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2025-12,2,A,F, | |
| 35 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2026-01,2,A,F, | |
| 36 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2026-02,2,A,F, | |
| 37 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2026-03,2,A,F, | |
| 38 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2026-04,2,A,F, | |
| 39 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2026-05,2,A,F, | |
| 40 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2026-06,2.25,A,F, | |
| 41 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2026-07,2.25,A,F, | |
| 42 | +M,XM,368,0,,"From 18 Sep 2024 onwards: official central bank steering rate is the deposit facility rate, fixed rate; from 15 Oct 2008 to 17 Sep 2024: official central bank liquidity providing, main refinancing operations, fixed rate; from 28 Jun 2000 to 14 Oct 2008: official central bank liquidity providing, main refinancing operations, minimum bid rate; from 5 Jan 1999 to 27 Jun 2000: official central bank liquidity providing, main refinancing operations, fixed rate.",4,European Central Bank,, Central bank policy rates - Euro area - Monthly - End of period,2026-08,2.25,A,F, | |
| 43 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-01,4.375,A,F, | |
| 44 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-02,4.375,A,F, | |
| 45 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-03,4.375,A,F, | |
| 46 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-04,4.375,A,F, | |
| 47 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-05,4.375,A,F, | |
| 48 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-06,4.375,A,F, | |
| 49 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-07,4.375,A,F, | |
| 50 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-08,4.375,A,F, | |
| 51 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-09,4.125,A,F, | |
| 52 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-10,3.875,A,F, | |
| 53 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-11,3.875,A,F, | |
| 54 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2025-12,3.625,A,F, | |
| 55 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2026-01,3.625,A,F, | |
| 56 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2026-02,3.625,A,F, | |
| 57 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2026-03,3.625,A,F, | |
| 58 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2026-04,3.625,A,F, | |
| 59 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2026-05,3.625,A,F, | |
| 60 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2026-06,3.625,A,F, | |
| 61 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2026-07,3.625,A,F, | |
| 62 | +M,US,368,0,,From 19 Dec 1985 onwards: mid-point of the Federal Reserve target rate; from 1 Jul 1954 to 18 Dec 1985: US Fed Funds effective rate.,4,US Federal Reserve System,, Central bank policy rates - United States - Monthly - End of period,2026-08,3.625,A,F, | |
added
tests/fixtures/bis/WS_SPP_R_628.csv
+37 −0
@@ -0,0 +1,37 @@ | ||
| 1 | +FREQ,REF_AREA,VALUE,UNIT_MEASURE,UNIT_MULT,BREAKS,COVERAGE,TITLE_TS,TIME_PERIOD,OBS_VALUE,OBS_STATUS,OBS_CONF,OBS_PRE_BREAK | |
| 2 | +Q,XM,R,628,0,,,,2014-Q1,90.5641,A,F, | |
| 3 | +Q,XM,R,628,0,,,,2014-Q2,90.6937,A,F, | |
| 4 | +Q,XM,R,628,0,,,,2014-Q3,91.595,A,F, | |
| 5 | +Q,XM,R,628,0,,,,2014-Q4,90.9369,A,F, | |
| 6 | +Q,XM,R,628,0,,,,2015-Q1,91.8444,A,F, | |
| 7 | +Q,XM,R,628,0,,,,2015-Q2,91.577,A,F, | |
| 8 | +Q,XM,R,628,0,,,,2015-Q3,92.842,A,F, | |
| 9 | +Q,XM,R,628,0,,,,2015-Q4,93.1932,A,F, | |
| 10 | +Q,XM,R,628,0,,,,2016-Q1,95.0339,A,F, | |
| 11 | +Q,XM,R,628,0,,,,2016-Q2,95.1252,A,F, | |
| 12 | +Q,XM,R,628,0,,,,2016-Q3,96.657,A,F, | |
| 13 | +Q,XM,R,628,0,,,,2016-Q4,96.7116,A,F, | |
| 14 | +Q,US,R,628,0,,,,2014-Q1,108.6448,A,F, | |
| 15 | +Q,US,R,628,0,,,,2014-Q2,108.0744,A,F, | |
| 16 | +Q,US,R,628,0,,,,2014-Q3,109.368,A,F, | |
| 17 | +Q,US,R,628,0,,,,2014-Q4,111.9405,A,F, | |
| 18 | +Q,US,R,628,0,,,,2015-Q1,113.9893,A,F, | |
| 19 | +Q,US,R,628,0,,,,2015-Q2,113.9327,A,F, | |
| 20 | +Q,US,R,628,0,,,,2015-Q3,115.2478,A,F, | |
| 21 | +Q,US,R,628,0,,,,2015-Q4,117.4922,A,F, | |
| 22 | +Q,US,R,628,0,,,,2016-Q1,118.837,A,F, | |
| 23 | +Q,US,R,628,0,,,,2016-Q2,118.7786,A,F, | |
| 24 | +Q,US,R,628,0,,,,2016-Q3,120.0298,A,F, | |
| 25 | +Q,US,R,628,0,,,,2016-Q4,121.6285,A,F, | |
| 26 | +Q,CA,R,628,0,,,,2014-Q1,109.3824,A,F, | |
| 27 | +Q,CA,R,628,0,,,,2014-Q2,110.4865,A,F, | |
| 28 | +Q,CA,R,628,0,,,,2014-Q3,111.0629,A,F, | |
| 29 | +Q,CA,R,628,0,,,,2014-Q4,111.8767,A,F, | |
| 30 | +Q,CA,R,628,0,,,,2015-Q1,114.8825,A,F, | |
| 31 | +Q,CA,R,628,0,,,,2015-Q2,117.4707,A,F, | |
| 32 | +Q,CA,R,628,0,,,,2015-Q3,119.206,A,F, | |
| 33 | +Q,CA,R,628,0,,,,2015-Q4,120.8037,A,F, | |
| 34 | +Q,CA,R,628,0,,,,2016-Q1,126.8529,A,F, | |
| 35 | +Q,CA,R,628,0,,,,2016-Q2,133.3875,A,F, | |
| 36 | +Q,CA,R,628,0,,,,2016-Q3,137.2739,A,F, | |
| 37 | +Q,CA,R,628,0,,,,2016-Q4,138.2891,A,F, | |
added
tests/fixtures/bis/WS_SPP_R_771.csv
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +FREQ,REF_AREA,VALUE,UNIT_MEASURE,UNIT_MULT,BREAKS,COVERAGE,TITLE_TS,TIME_PERIOD,OBS_VALUE,OBS_STATUS,OBS_CONF,OBS_PRE_BREAK | |
| 2 | +Q,US,R,771,0,,,,2025-Q1,-0.0324,A,F, | |
| 3 | +Q,US,R,771,0,,,,2025-Q2,-0.485,A,F, | |
| 4 | +Q,US,R,771,0,,,,2025-Q3,-1.4758,A,F, | |
| 5 | +Q,US,R,771,0,,,,2025-Q4,-1.714,A,F, | |
| 6 | +Q,US,R,771,0,,,,2026-Q1,-2.0736,A,F, | |
| 7 | +Q,CA,R,771,0,,,,2025-Q1,-3.6898,A,F, | |
| 8 | +Q,CA,R,771,0,,,,2025-Q2,-4.6015,A,F, | |
| 9 | +Q,CA,R,771,0,,,,2025-Q3,-5.0705,A,F, | |
| 10 | +Q,CA,R,771,0,,,,2025-Q4,-5.5052,A,F, | |
| 11 | +Q,CA,R,771,0,,,,2026-Q1,-6.8079,A,F, | |
added
tests/fixtures/eurostat/rd_e_gerdtot_sample.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"version":"2.0","class":"dataset","label":"GERD by sector of performance","source":"ESTAT","updated":"2026-03-18T23:00:00+0100","value":{"0":3.07,"1":3.04,"2":3.13,"3":3.13,"4":2.21,"5":2.22,"6":2.18,"7":2.18},"status":{"3":"p","7":"p"},"id":["freq","sectperf","unit","geo","time"],"size":[1,1,1,2,4],"dimension":{"freq":{"label":"Time frequency","category":{"index":{"A":0},"label":{"A":"Annual"}}},"sectperf":{"label":"Sector of performance","category":{"index":{"TOTAL":0},"label":{"TOTAL":"All sectors"}}},"unit":{"label":"Unit of measure","category":{"index":{"PC_GDP":0},"label":{"PC_GDP":"Percentage of gross domestic product (GDP)"}}},"geo":{"label":"Geopolitical entity (reporting)","category":{"index":{"DE":0,"FR":1},"label":{"DE":"Germany","FR":"France"}}},"time":{"label":"Time","category":{"index":{"2021":0,"2022":1,"2023":2,"2024":3},"label":{"2021":"2021","2022":"2022","2023":"2023","2024":"2024"}}}},"extension":{"lang":"EN","id":"RD_E_GERDTOT","agencyId":"ESTAT","version":"1.0","datastructure":{"id":"RD_E_GERDTOT","agencyId":"ESTAT","version":"39.0"},"annotation":[{"type":"CREATED","date":"2021-03-10T17:39:05+0100"},{"type":"DISSEMINATION_DOI_XML","title":"<adms:identifier xmlns:adms=\"http://www.w3.org/ns/adms#\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core.html\" xmlns:dct=\"http://purl.org/dc/terms/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\"><adms:Identifier rdf:about=\"https://doi.org/10.2908/RD_E_GERDTOT\"><skos:notation rdf:datatype=\"http://purl.org/spar/datacite/doi\">10.2908/RD_E_GERDTOT</skos:notation><dct:creator rdf:resource=\"http://publications.europa.eu/resource/authority/corporate-body/ESTAT\"/><dct:issued rdf:datatype=\"http://www.w3.org/2001/XMLSchema#date\">2023-01-19</dct:issued></adms:Identifier></adms:identifier>"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/documents/10186/6246844/RD-GBA-Information-note.pdf/55d1d61c-519f-f95d-db45-36f9f00351fc","text":"Information note"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/documents/10186/6246844/RD-GBA-Information-note.pdf/55d1d61c-519f-f95d-db45-36f9f00351fc"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/documents/10186/6246844/RD-GBA-Information-note.pdf/55d1d61c-519f-f95d-db45-36f9f00351fc"},{"type":"DISSEMINATION_OBJECT_TYPE","title":"DATASET"},{"type":"DISSEMINATION_TIMESTAMP_DATA","date":"2026-03-18T23:00:00+0100"},{"type":"DISSEMINATION_TIMESTAMP_GLOBAL","date":"2026-04-13T23:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_PLANNED","date":"2026-04-13T23:00:00+0200"},{"type":"ESMS_HTML","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/cache/metadata/en/rd_esms.htm"},{"type":"ESMS_SDMX","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/api/dissemination/files?file=metadata/rd_esms.sdmx.zip"},{"type":"OBS_COUNT","title":"43474"},{"type":"OBS_PERIOD_OVERALL_LATEST","title":"2024"},{"type":"OBS_PERIOD_OVERALL_OLDEST","title":"1980"},{"type":"SOURCE_INSTITUTIONS","text":"Eurostat; Organisation for Economic Cooperation and Development (OECD)"},{"type":"UPDATE_DATA","date":"2026-03-18T23:00:00+0100"},{"type":"UPDATE_STRUCTURE","date":"2026-03-18T23:00:00+0100"}],"status":{"label":{"p":"provisional"}},"positions-with-no-data":{"freq":[],"sectperf":[],"unit":[],"geo":[],"time":[]}}} | |
| \ No newline at end of file | ||
added
tests/fixtures/eurostat/une_rt_a_sample.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"version":"2.0","class":"dataset","label":"Unemployment by sex and age - annual data","source":"ESTAT","updated":"2026-09-10T23:00:00+0200","value":{"4":3.1,"5":3.1,"6":3.5,"7":3.8,"8":12.5,"9":11.1,"10":10.1,"11":8.9,"0":6.2,"1":6.1,"2":6.0,"3":6.0},"id":["freq","age","unit","sex","geo","time"],"size":[1,1,1,1,3,4],"dimension":{"freq":{"label":"Time frequency","category":{"index":{"A":0},"label":{"A":"Annual"}}},"age":{"label":"Age class","category":{"index":{"Y15-74":0},"label":{"Y15-74":"From 15 to 74 years"}}},"unit":{"label":"Unit of measure","category":{"index":{"PC_ACT":0},"label":{"PC_ACT":"Percentage of population in the labour force"}}},"sex":{"label":"Sex","category":{"index":{"T":0},"label":{"T":"Total"}}},"geo":{"label":"Geopolitical entity (reporting)","category":{"index":{"EU27_2020":0,"DE":1,"EL":2},"label":{"EU27_2020":"European Union - 27 countries (from 2020)","DE":"Germany","EL":"Greece"}}},"time":{"label":"Time","category":{"index":{"2022":0,"2023":1,"2024":2,"2025":3},"label":{"2022":"2022","2023":"2023","2024":"2024","2025":"2025"}}}},"extension":{"lang":"EN","id":"UNE_RT_A","agencyId":"ESTAT","version":"1.0","datastructure":{"id":"UNE_RT_A","agencyId":"ESTAT","version":"48.0"},"annotation":[{"type":"CREATED","date":"2021-07-14T12:27:40+0200"},{"type":"DISSEMINATION_DOI_XML","title":"<adms:identifier xmlns:adms=\"http://www.w3.org/ns/adms#\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core.html\" xmlns:dct=\"http://purl.org/dc/terms/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\"><adms:Identifier rdf:about=\"https://doi.org/10.2908/UNE_RT_A\"><skos:notation rdf:datatype=\"http://purl.org/spar/datacite/doi\">10.2908/UNE_RT_A</skos:notation><dct:creator rdf:resource=\"http://publications.europa.eu/resource/authority/corporate-body/ESTAT\"/><dct:issued rdf:datatype=\"http://www.w3.org/2001/XMLSchema#date\">2023-01-23</dct:issued></adms:Identifier></adms:identifier>"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_LFSQ_20240604.pdf","text":"Information note"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_LFSQ_20240604.pdf"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_LFSQ_20240604.pdf"},{"type":"DISSEMINATION_OBJECT_TYPE","title":"DATASET"},{"type":"DISSEMINATION_TIMESTAMP_DATA","date":"2026-09-10T23:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_GLOBAL","date":"2026-09-10T23:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_PLANNED","date":"2026-09-10T23:00:00+0200"},{"type":"ESMS_HTML","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/cache/metadata/en/lfsi_esms.htm"},{"type":"ESMS_SDMX","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/api/dissemination/files?file=metadata/lfsi_esms.sdmx.zip"},{"type":"OBS_COUNT","title":"39708"},{"type":"OBS_PERIOD_OVERALL_LATEST","title":"2025"},{"type":"OBS_PERIOD_OVERALL_OLDEST","title":"2003"},{"type":"SOURCE_INSTITUTIONS","text":"Eurostat"},{"type":"UPDATE_DATA","date":"2026-09-10T23:00:00+0200"},{"type":"UPDATE_STRUCTURE","date":"2026-03-12T23:00:00+0100"}],"positions-with-no-data":{"freq":[],"age":[],"unit":[],"sex":[],"geo":[],"time":[]}}} | |
| \ No newline at end of file | ||
added
tests/fixtures/fred/obs_CPIAUCSL.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","observation_start":"2024-01-01","observation_end":"2025-03-01","units":"lin","output_type":1,"file_type":"json","order_by":"observation_date","sort_order":"asc","count":15,"offset":0,"limit":100000,"observations":[{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-01-01","value":"309.698"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-02-01","value":"310.967"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-03-01","value":"312.345"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-04-01","value":"313.023"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-05-01","value":"313.175"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-06-01","value":"313.044"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-07-01","value":"313.569"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-08-01","value":"314.062"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-09-01","value":"314.732"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-10-01","value":"315.631"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-11-01","value":"316.528"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-12-01","value":"317.604"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2025-01-01","value":"318.961"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2025-02-01","value":"319.679"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2025-03-01","value":"319.785"}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/fred/obs_FEDFUNDS.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","observation_start":"2024-01-01","observation_end":"2024-06-01","units":"lin","output_type":1,"file_type":"json","order_by":"observation_date","sort_order":"asc","count":6,"offset":0,"limit":100000,"observations":[{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-01-01","value":"5.33"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-02-01","value":"5.33"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-03-01","value":"5.33"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-04-01","value":"5.33"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-05-01","value":"5.33"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2024-06-01","value":"5.33"}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/fred/obs_MORTGAGE30US_m.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","observation_start":"2026-05-01","observation_end":"9999-12-31","units":"lin","output_type":1,"file_type":"json","order_by":"observation_date","sort_order":"asc","count":5,"offset":0,"limit":100000,"observations":[{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2026-05-01","value":"6.44"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2026-06-01","value":"6.49"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2026-07-01","value":"6.54"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2026-08-01","value":"6.67"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2026-09-01","value":"."}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/fred/obs_QCAR628BIS.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","observation_start":"2015-01-01","observation_end":"2016-06-01","units":"lin","output_type":1,"file_type":"json","order_by":"observation_date","sort_order":"asc","count":6,"offset":0,"limit":100000,"observations":[{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2015-01-01","value":"114.8825"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2015-04-01","value":"117.4707"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2015-07-01","value":"119.206"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2015-10-01","value":"120.8037"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2016-01-01","value":"126.8529"},{"realtime_start":"2026-09-11","realtime_end":"2026-09-11","date":"2016-04-01","value":"133.3875"}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/fred/series_CPIAUCSL.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"realtime_start":"2026-08-12","realtime_end":"2026-08-12","seriess":[{"id":"CPIAUCSL","realtime_start":"2026-08-12","realtime_end":"2026-08-12","title":"Consumer Price Index for All Urban Consumers: All Items in U.S. City Average","observation_start":"1947-01-01","observation_end":"2026-07-01","frequency":"Monthly","frequency_short":"M","units":"Index 1982-1984=100","units_short":"Index 1982-1984=100","seasonal_adjustment":"Seasonally Adjusted","seasonal_adjustment_short":"SA","last_updated":"2026-08-12 09:10:19-05","popularity":96,"notes":"The Consumer Price Index for All Urban Consumers: All Items (CPIAUCSL) is a price index of a basket of goods and services paid by urban consumers. Percent changes in the price index measure the inflation rate between any two time periods. The most common inflation metric is the percent change from one year ago. It can also represent the buying habits of urban consumers. This particular index includes roughly 88 percent of the total population, accounting for wage earners, clerical workers, technical workers, self-employed, short-term workers, unemployed, retirees, and those not in the labor force.\r\n\r\nThe CPIs are based on prices for food, clothing, shelter, and fuels; transportation fares; service fees (e.g., water and sewer service); and sales taxes. Prices are collected monthly from about 4,000 housing units and approximately 26,000 retail establishments across 87 urban areas. To calculate the index, price changes are averaged with weights representing their importance in the spending of the particular group. The index measures price changes (as a percent change) from a predetermined reference date. In addition to the original unadjusted index distributed, the Bureau of Labor Statistics also releases a seasonally adjusted index. The unadjusted series reflects all factors that may influence a change in prices. However, it can be very useful to look at the seasonally adjusted CPI, which removes the effects of seasonal changes, such as weather, school year, production cycles, and holidays.\r\n\r\nThe CPI can be used to recognize periods of inflation and deflation. Significant increases in the CPI within a short time frame might indicate a period of inflation, and significant decreases in CPI within a short time frame might indicate a period of deflation. However, because the CPI includes volatile food and oil prices, it might not be a reliable measure of inflationary and deflationary periods. For a more accurate detection, the core CPI (CPILFESL (https:\/\/fred.stlouisfed.org\/series\/CPILFESL)) is often used. When using the CPI, please note that it is not applicable to all consumers and should not be used to determine relative living costs. Additionally, the CPI is a statistical measure vulnerable to sampling error since it is based on a sample of prices and not the complete average.\r\n\r\nFor more information on the CPI, see the Handbook of Methods (https:\/\/www.bls.gov\/opub\/hom\/cpi\/), the release notes and announcements (https:\/\/www.bls.gov\/cpi\/), and the Frequently Asked Questions (https:\/\/www.bls.gov\/cpi\/questions-and-answers.htm) (FAQs)."}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/fred/series_FEDFUNDS.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"realtime_start":"2026-09-01","realtime_end":"2026-09-01","seriess":[{"id":"FEDFUNDS","realtime_start":"2026-09-01","realtime_end":"2026-09-01","title":"Federal Funds Effective Rate","observation_start":"1954-07-01","observation_end":"2026-08-01","frequency":"Monthly","frequency_short":"M","units":"Percent","units_short":"%","seasonal_adjustment":"Not Seasonally Adjusted","seasonal_adjustment_short":"NSA","last_updated":"2026-09-01 15:16:43-05","popularity":96,"notes":" Daily Federal Funds Rate from 1928-1954 (https:\/\/fred.stlouisfed.org\/categories\/33951).\n\nThe federal funds rate is the interest rate at which depository institutions trade federal funds (balances held at Federal Reserve Banks) with each other overnight. When a depository institution has surplus balances in its reserve account, it lends to other banks in need of larger balances. In simpler terms, a bank with excess cash, which is often referred to as liquidity, will lend to another bank that needs to quickly raise liquidity. (1) The rate that the borrowing institution pays to the lending institution is determined between the two banks; the weighted average rate for all of these types of negotiations is called the effective federal funds rate.(2) The effective federal funds rate is essentially determined by the market but is influenced by the Federal Reserve as it uses the Interest on Reserve Balances rate to steer the federal funds rate toward the target range.(2)\n\nThe Federal Open Market Committee (FOMC) meets eight times a year to determine the federal funds target range. The Fed's primary tool for influencing the federal funds rate is the interest the Fed pays on the funds that banks hold as reserve balances at their Federal Reserve Bank, which is the Interest on Reserves Balances (IORB) rate. Because banks are unlikely to lend funds in the federal funds market for less than they get paid in their reserve balance account at the Federal Reserve, the Interest on Reserve Balances (IORB) is an effective tool for guiding the federal funds rate. (3) Whether the Federal Reserve raises or lowers the target range for the federal funds rate depends on the state of the economy. If the FOMC believes the economy is growing too fast and inflation pressures are inconsistent with the dual mandate of the Federal Reserve, the Committee may temper economic activity by raising the target range for federal funds rate, and increasing the IORB rate to steer the federal funds rate into the target range. In the opposing scenario, the FOMC may spur greater economic activity by lowering the target range for federal funds rate, and decreasing the IORB rate to steer the federal funds rate into the target range. (3) Therefore, the FOMC must observe the current state of the economy to determine the best course of monetary policy that will maximize economic growth while adhering to the dual mandate set forth by Congress. In making its monetary policy decisions, the FOMC considers a wealth of economic data, such as: trends in prices and wages, employment, consumer spending and income, business investments, and foreign exchange markets.\n\nThe federal funds rate is the central interest rate in the U.S. financial market. It influences other interest rates such as the prime rate, which is the rate banks charge their customers with higher credit ratings. Additionally, the federal funds rate indirectly influences longer- term interest rates such as mortgages, loans, and savings, all of which are very important to consumer wealth and confidence.(2)\n\nReferences\n(1) Federal Reserve Bank of New York. \"Federal funds.\" Fedpoints, August 2007.\n(2) Monetary Policy (https:\/\/www.federalreserve.gov\/monetarypolicy.htm), Board of Governors of the Federal Reserve System.\n(3) The Fed Explained (https:\/\/www.federalreserve.gov\/aboutthefed\/files\/the-fed-explained.pdf), Board of Governors of the Federal Reserve System\n\nFor further information, see The Fed's New Monetary Policy Tools (https:\/\/www.stlouisfed.org\/publications\/page-one-economics\/2020\/08\/03\/the-feds-new-monetary-policy-tools), Page One Economics, Federal Reserve Bank of St. Louis. \n\nFor questions on the data, please contact the data source (https:\/\/www.federalreserve.gov\/apps\/ContactUs\/feedback.aspx?refurl=\/releases\/h15\/%). For questions on FRED functionality, please contact us here (https:\/\/fred.stlouisfed.org\/contactus\/).<\/p>"}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/fred/series_MORTGAGE30US.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"realtime_start":"2026-09-10","realtime_end":"2026-09-10","seriess":[{"id":"MORTGAGE30US","realtime_start":"2026-09-10","realtime_end":"2026-09-10","title":"30-Year Fixed Rate Mortgage Average in the United States","observation_start":"1971-04-02","observation_end":"2026-09-10","frequency":"Weekly, Ending Thursday","frequency_short":"W","units":"Percent","units_short":"%","seasonal_adjustment":"Not Seasonally Adjusted","seasonal_adjustment_short":"NSA","last_updated":"2026-09-10 11:02:25-05","popularity":98,"notes":"On November 17, 2022, Freddie Mac changed the methodology of the Primary Mortgage Market Survey\u00ae (PMMS\u00ae). The weekly mortgage rate is now based on applications submitted to Freddie Mac from lenders across the country. For more information regarding Freddie Mac\u2019s enhancement, see their research note (https:\/\/www.freddiemac.com\/research\/insight\/20221103-freddie-macs-newly-enhanced-mortgage-rate-survey).\n\nData are provided \u201cas is\u201d by Freddie Mac\u00ae, with no warranties of any kind, express or implied, including but not limited to warranties of accuracy or implied warranties of merchantability or fitness for a particular purpose. Use of the data is at the user\u2019s sole risk. In no event will Freddie Mac be liable for any damages arising out of or related to the data, including but not limited to direct, indirect, incidental, special, consequential, or punitive damages, whether under a contract, tort, or any other theory of liability, even if Freddie Mac is aware of the possibility of such damages.\n\nCopyright, 2016, Freddie Mac. Reprinted with permission."}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/fred/series_QCAR628BIS.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"realtime_start":"2026-07-03","realtime_end":"2026-07-03","seriess":[{"id":"QCAR628BIS","realtime_start":"2026-07-03","realtime_end":"2026-07-03","title":"Real Residential Property Prices for Canada","observation_start":"1970-01-01","observation_end":"2026-01-01","frequency":"Quarterly","frequency_short":"Q","units":"Index 2010=100","units_short":"Index 2010=100","seasonal_adjustment":"Not Seasonally Adjusted","seasonal_adjustment_short":"NSA","last_updated":"2026-06-25 11:02:27-05","popularity":68,"notes":"Source Code: Q:CA:R:628\n\nCoverage includes national residential average. The series is deflated using CPI.\n\nFor more information, please see https:\/\/www.bis.org\/statistics\/pp_detailed.htm.\n\nAny use of the series shall be cited as follows: \"Sources: National sources, BIS Residential Property Price database, http:\/\/www.bis.org\/statistics\/pp.htm.\"\n\nCopyright, 2016, Bank for International Settlements (BIS). Terms and conditions of use are available at http:\/\/www.bis.org\/terms_conditions.htm#Copyright_and_Permissions."}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/ilo/DF_UNE_2EAP_SEX_AGE_RT.csv
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +DATAFLOW,REF_AREA,FREQ,MEASURE,SEX,AGE,TIME_PERIOD,OBS_VALUE,OBS_STATUS,UNIT_MEASURE_TYPE,UNIT_MEASURE,UNIT_MULT,SOURCE,NOTE_SOURCE,NOTE_INDICATOR,NOTE_CLASSIF,DECIMALS,UPPER_BOUND,LOWER_BOUND | |
| 2 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),CAN,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2022,5.279,R,RT,PT,0,ILO - Modelled Estimates,,,,1,5.279,5.279 | |
| 3 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),CAN,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2023,5.415,R,RT,PT,0,ILO - Modelled Estimates,,,,1,5.415,5.415 | |
| 4 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),CAN,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2024,6.351,R,RT,PT,0,ILO - Modelled Estimates,,,,1,6.351,6.351 | |
| 5 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),CAN,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2025,6.907,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 6 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),CAN,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2026,7.294,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 7 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),CAN,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2027,7.328,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 8 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),FRA,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2022,7.303,R,RT,PT,0,ILO - Modelled Estimates,,,,1,7.303,7.303 | |
| 9 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),FRA,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2023,7.335,R,RT,PT,0,ILO - Modelled Estimates,,,,1,7.335,7.335 | |
| 10 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),FRA,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2024,7.4,R,RT,PT,0,ILO - Modelled Estimates,,,,1,7.4,7.4 | |
| 11 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),FRA,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2025,7.542,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 12 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),FRA,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2026,7.654,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 13 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),FRA,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2027,7.558,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 14 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),SOM,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2022,19.112,,RT,PT,0,ILO - Modelled Estimates,,,,1,33.18,5.045 | |
| 15 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),SOM,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2023,18.96,,RT,PT,0,ILO - Modelled Estimates,,,,1,35.1,4.74 | |
| 16 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),SOM,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2024,18.949,,RT,PT,0,ILO - Modelled Estimates,,,,1,37.161,4.737 | |
| 17 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),SOM,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2025,18.948,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 18 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),SOM,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2026,18.933,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 19 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),SOM,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2027,18.884,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 20 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),X01,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2022,5.253,,RT,PT,0,ILO - Modelled Estimates,,,,1,5.926,4.688 | |
| 21 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),X01,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2023,4.896,,RT,PT,0,ILO - Modelled Estimates,,,,1,5.646,4.264 | |
| 22 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),X01,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2024,4.883,,RT,PT,0,ILO - Modelled Estimates,,,,1,5.839,4.057 | |
| 23 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),X01,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2025,4.869,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 24 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),X01,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2026,4.861,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
| 25 | +ILO:DF_UNE_2EAP_SEX_AGE_RT(1.0),X01,A,UNE_2EAP_RT,SEX_T,AGE_YTHADULT_YGE15,2027,4.842,,RT,PT,0,ILO - Modelled Estimates,,,,1,, | |
added
tests/fixtures/ilo/dataflow.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"data": {"dataflows": [{"id": "DF_UNE_2EAP_SEX_AGE_RT", "version": "1.0", "agencyID": "ILO", "isExternalReference": false, "isFinal": true, "name": "Unemployment rate by sex and age -- ILO modelled estimates, Nov. 2025", "names": {"en": "Unemployment rate by sex and age -- ILO modelled estimates, Nov. 2025"}, "description": "<p><strong>Imputed observations are not based on national data, are subject to high uncertainty and should not be used for country comparisons or rankings. </strong>This series is based on the 13th ICLS definitions. The unemployment rate conveys the number of persons who are unemployed as a percent ", "descriptions": {"en": "<p><strong>Imputed observations are not based on national data, are subject to high uncertainty and should not be used for country comparisons or rankings. </strong>This series is based on the 13th ICLS definitions. The unemployment rate conveys the number of persons who are unemployed as a percent of the labour force (i.e., the employed plus the unemployed). The unemployed comprise all persons of working age who were: a) without work during the reference period, i.e. were not in paid employment or self-employment; b) currently available for work, i.e. were available for paid employment or self-employment during the reference period; and c) seeking work, i.e. had taken specific steps in a specified recent period to seek paid employment or self-employment. For more information, refer to the <a href=\"https://ilostat.ilo.org/methods/concepts-and-definitions/ilo-modelled-estimates/\">ILO Modelled Estimates (ILOEST) database description</a>.</p>"}, "annotations": [{"title": "02/12/2025 13:33:47", "type": "LAST_UPDATE"}, {"title": "AGE", "type": "LAYOUT_COLUMN"}, {"title": "REF_AREA,SEX,TIME_PERIOD", "type": "LAYOUT_ROW"}, {"title": "FREQ=A,endPeriod=2027-12-31,AGE=AGE_YTHADULT_YGE15+AGE_YTHADULT_Y15-64+AGE_YTHADULT_Y15-24+AGE_YTHADULT_YGE25,SEX=SEX_T,LASTNOBSERVATIONS=1", "type": "DEFAULT"}, {"title": "647000", "type": "ORDER"}, {"title": "353000", "type": "SEARCH_WEIGHT"}], "structure": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=ILO:UNE_2EAP_SEX_AGE_RT(1.0)"}]}} | |
| \ No newline at end of file | ||
added
tests/fixtures/imf/weo_NGDPD_sample.csv
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +DATAFLOW,COUNTRY,INDICATOR,FREQUENCY,TIME_PERIOD,OBS_VALUE,SCALE,PRECISION,DECIMALS_DISPLAYED,FUNCTIONAL_CAT,INT_ACC_ITEM,NA_STO,GFS_STO,COICOP_1999,TRADE_FLOW,COMMODITY,SOC_CONCEPTS,SECTOR,ACCOUNTING_ENTRY,INDEX_TYPE,PRICES,STATISTICAL_MEASURES,EXRATE,TRANSFORMATION,UNIT,REPORTING_PERIOD_TYPE,DERIVATION_TYPE,OVERLAP,COUNTRY_UPDATE_DATE,DOI,FULL_DESCRIPTION,AUTHOR,PUBLISHER,DEPARTMENT,CONTACT_POINT,TOPIC,TOPIC_DATASET,KEYWORDS,KEYWORDS_DATASET,LANGUAGE,PUBLICATION_DATE,UPDATE_DATE,METHODOLOGY,METHODOLOGY_NOTES,ACCESS_SHARING_LEVEL,ACCESS_SHARING_NOTES,SECURITY_CLASSIFICATION,SOURCE,SHORT_SOURCE_CITATION,FULL_SOURCE_CITATION,LICENSE,SUGGESTED_CITATION,KEY_INDICATOR,SERIES_NAME,LATEST_ACTUAL_ANNUAL_DATA,HISTORICAL_DATA_SOURCE,BASE_YEAR,START_END_MONTHS_OF_REPORTING_YEAR,CHAIN_WEIGHTED,BASIS_OF_PROJECTIONS,VALUATION,PRICES_SECTOR_HARMONIZED_PRICES,LABOR_SECTOR_EMPLOYMENT_TYPE,FISCAL_SECTOR_GENERAL_GOVERNMENT_COMPOSITION,FISCAL_SECTOR_VALUATION_OF_DEBT,FISCAL_SECTOR_INSTRUMENTS_INCLUDED_IN_GROSS_AND_NET_DEBT,TRADE_SECTOR_OIL_COVERAGE,PRIMARY_DOMESTIC_CURRENCY | |
| 2 | +IMF.RES:WEO(9.0.0),CAN,NGDPD,A,2023,2196593836000,9,,3,,,B1GQ,,,,,,,,,V,,,,USD,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Nominal GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Current prices, US dollar",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 3 | +IMF.RES:WEO(9.0.0),CAN,NGDPD,A,2024,2270076190000,9,,3,,,B1GQ,,,,,,,,,V,,,,USD,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Nominal GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Current prices, US dollar",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 4 | +IMF.RES:WEO(9.0.0),CAN,NGDPD,A,2025,2319899772000,9,,3,,,B1GQ,,,,,,,,,V,,,,USD,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Nominal GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Current prices, US dollar",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 5 | +IMF.RES:WEO(9.0.0),CAN,NGDPD,A,2026,2507340482000,9,,3,,,B1GQ,,,,,,,,,V,,,,USD,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Nominal GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Current prices, US dollar",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 6 | +IMF.RES:WEO(9.0.0),USA,NGDPD,A,2023,27811500000000,9,,3,,,B1GQ,,,,,,,,,V,,,,USD,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Nominal GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Current prices, US dollar",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 7 | +IMF.RES:WEO(9.0.0),USA,NGDPD,A,2024,29298025000000,9,,3,,,B1GQ,,,,,,,,,V,,,,USD,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Nominal GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Current prices, US dollar",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 8 | +IMF.RES:WEO(9.0.0),USA,NGDPD,A,2025,30767075000000,9,,3,,,B1GQ,,,,,,,,,V,,,,USD,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Nominal GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Current prices, US dollar",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 9 | +IMF.RES:WEO(9.0.0),USA,NGDPD,A,2026,32383920433000,9,,3,,,B1GQ,,,,,,,,,V,,,,USD,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Nominal GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Current prices, US dollar",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
added
tests/fixtures/imf/weo_NGDP_RPCH_sample.csv
+29 −0
@@ -0,0 +1,29 @@ | ||
| 1 | +DATAFLOW,COUNTRY,INDICATOR,FREQUENCY,TIME_PERIOD,OBS_VALUE,SCALE,PRECISION,DECIMALS_DISPLAYED,FUNCTIONAL_CAT,INT_ACC_ITEM,NA_STO,GFS_STO,COICOP_1999,TRADE_FLOW,COMMODITY,SOC_CONCEPTS,SECTOR,ACCOUNTING_ENTRY,INDEX_TYPE,PRICES,STATISTICAL_MEASURES,EXRATE,TRANSFORMATION,UNIT,REPORTING_PERIOD_TYPE,DERIVATION_TYPE,OVERLAP,COUNTRY_UPDATE_DATE,DOI,FULL_DESCRIPTION,AUTHOR,PUBLISHER,DEPARTMENT,CONTACT_POINT,TOPIC,TOPIC_DATASET,KEYWORDS,KEYWORDS_DATASET,LANGUAGE,PUBLICATION_DATE,UPDATE_DATE,METHODOLOGY,METHODOLOGY_NOTES,ACCESS_SHARING_LEVEL,ACCESS_SHARING_NOTES,SECURITY_CLASSIFICATION,SOURCE,SHORT_SOURCE_CITATION,FULL_SOURCE_CITATION,LICENSE,SUGGESTED_CITATION,KEY_INDICATOR,SERIES_NAME,LATEST_ACTUAL_ANNUAL_DATA,HISTORICAL_DATA_SOURCE,BASE_YEAR,START_END_MONTHS_OF_REPORTING_YEAR,CHAIN_WEIGHTED,BASIS_OF_PROJECTIONS,VALUATION,PRICES_SECTOR_HARMONIZED_PRICES,LABOR_SECTOR_EMPLOYMENT_TYPE,FISCAL_SECTOR_GENERAL_GOVERNMENT_COMPOSITION,FISCAL_SECTOR_VALUATION_OF_DEBT,FISCAL_SECTOR_INSTRUMENTS_INCLUDED_IN_GROSS_AND_NET_DEBT,TRADE_SECTOR_OIL_COVERAGE,PRIMARY_DOMESTIC_CURRENCY | |
| 2 | +IMF.RES:WEO(9.0.0),CAN,NGDP_RPCH,A,2022,4.69542,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 3 | +IMF.RES:WEO(9.0.0),CAN,NGDP_RPCH,A,2023,1.953085,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 4 | +IMF.RES:WEO(9.0.0),CAN,NGDP_RPCH,A,2024,2.046266,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 5 | +IMF.RES:WEO(9.0.0),CAN,NGDP_RPCH,A,2025,1.742685,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 6 | +IMF.RES:WEO(9.0.0),CAN,NGDP_RPCH,A,2026,1.499992,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 7 | +IMF.RES:WEO(9.0.0),CAN,NGDP_RPCH,A,2027,1.895884,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 8 | +IMF.RES:WEO(9.0.0),CAN,NGDP_RPCH,A,2028,1.65465,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/25/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Data rebased to 2017,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,Canadian dollar | |
| 9 | +IMF.RES:WEO(9.0.0),G001,NGDP_RPCH,A,2022,3.773925,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,,,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",,,,,,,,,,,,,, | |
| 10 | +IMF.RES:WEO(9.0.0),G001,NGDP_RPCH,A,2023,3.327812,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,,,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",,,,,,,,,,,,,, | |
| 11 | +IMF.RES:WEO(9.0.0),G001,NGDP_RPCH,A,2024,3.416268,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,,,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",,,,,,,,,,,,,, | |
| 12 | +IMF.RES:WEO(9.0.0),G001,NGDP_RPCH,A,2025,3.441279,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,,,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",,,,,,,,,,,,,, | |
| 13 | +IMF.RES:WEO(9.0.0),G001,NGDP_RPCH,A,2026,3.055974,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,,,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",,,,,,,,,,,,,, | |
| 14 | +IMF.RES:WEO(9.0.0),G001,NGDP_RPCH,A,2027,3.223795,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,,,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",,,,,,,,,,,,,, | |
| 15 | +IMF.RES:WEO(9.0.0),G001,NGDP_RPCH,A,2028,3.239925,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,,,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",,,,,,,,,,,,,, | |
| 16 | +IMF.RES:WEO(9.0.0),KOS,NGDP_RPCH,A,2022,4.278013,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/23/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,European System of Accounts (ESA) 2010,Base year changes as new data is released - previous year's prices,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2016,January/December,No,,,,,,,,,Euro | |
| 17 | +IMF.RES:WEO(9.0.0),KOS,NGDP_RPCH,A,2023,4.068595,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/23/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,European System of Accounts (ESA) 2010,Base year changes as new data is released - previous year's prices,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2016,January/December,No,,,,,,,,,Euro | |
| 18 | +IMF.RES:WEO(9.0.0),KOS,NGDP_RPCH,A,2024,4.572267,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/23/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,European System of Accounts (ESA) 2010,Base year changes as new data is released - previous year's prices,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2016,January/December,No,,,,,,,,,Euro | |
| 19 | +IMF.RES:WEO(9.0.0),KOS,NGDP_RPCH,A,2025,3.591166,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/23/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,European System of Accounts (ESA) 2010,Base year changes as new data is released - previous year's prices,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2016,January/December,No,,,,,,,,,Euro | |
| 20 | +IMF.RES:WEO(9.0.0),KOS,NGDP_RPCH,A,2026,3.3,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/23/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,European System of Accounts (ESA) 2010,Base year changes as new data is released - previous year's prices,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2016,January/December,No,,,,,,,,,Euro | |
| 21 | +IMF.RES:WEO(9.0.0),KOS,NGDP_RPCH,A,2027,3.8,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/23/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,European System of Accounts (ESA) 2010,Base year changes as new data is released - previous year's prices,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2016,January/December,No,,,,,,,,,Euro | |
| 22 | +IMF.RES:WEO(9.0.0),KOS,NGDP_RPCH,A,2028,4,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/23/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,European System of Accounts (ESA) 2010,Base year changes as new data is released - previous year's prices,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2016,January/December,No,,,,,,,,,Euro | |
| 23 | +IMF.RES:WEO(9.0.0),USA,NGDP_RPCH,A,2022,2.524222,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 24 | +IMF.RES:WEO(9.0.0),USA,NGDP_RPCH,A,2023,2.934535,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 25 | +IMF.RES:WEO(9.0.0),USA,NGDP_RPCH,A,2024,2.793116,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 26 | +IMF.RES:WEO(9.0.0),USA,NGDP_RPCH,A,2025,2.11733,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 27 | +IMF.RES:WEO(9.0.0),USA,NGDP_RPCH,A,2026,2.323695,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 28 | +IMF.RES:WEO(9.0.0),USA,NGDP_RPCH,A,2027,2.1,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
| 29 | +IMF.RES:WEO(9.0.0),USA,NGDP_RPCH,A,2028,2.1,0,,3,,,B1GQ,,,,,,,,,Q,,,,PT,,,OL,9/30/2025,,"The World Economic Outlook (WEO) database contains selected macroeconomic data series from the statistical appendix of the World Economic Outlook report, which presents the IMF staff's analysis and projections of economic developments at the global level, in major country groups and in many individual countries. The WEO is released in April and September/October each year. Use this database to find data on national accounts, gross domestic product (GDP), inflation, unemployment rates, balance of payments, fiscal indicators, trade for countries and country groups (aggregates), and commodity prices whose data are reported by the IMF. Data are available from 1980 to the present, and projections are given for the next two years. Additionally, medium-term projections are available for selected indicators. For some countries, data are incomplete or unavailable for certain years.",,IMF,RES,datahelp@imf.org,E10_GDP,"C53,F32,F21_IIP,F34,H00,E10,E10_GDP,F10,WEO",Real GDP,"Demographics, Real sector, Price indexes, Gross domestic product, National accounts, Labor markets, Unemployment, International trade, Export prices, Import prices, International reserves, Monetary aggregates, Consumer prices, Exchange rates, External position, Financial account, Balance of payments, Capital account, Current account, External debt, International investment position, Public sector and general government, Debt service, Fiscal account",EN,2026-04-14T13:00:00Z,2026-04-15T13:00:00Z,System of National Accounts (SNA) 2008,Real GDP determined by chained Fisher quantity growth rates.,PUBLIC_OPEN,,PUB,,IMF staff calculations.,,© International Monetary Fund Copyright. All Rights Reserved. https://www.imf.org/external/terms.htm,"International Monetary Fund. World Economic Outlook (WEO), https://data.imf.org/en/datasets/IMF.RES:WEO. Accessed on [current date].",true,"Gross domestic product (GDP), Constant prices, Percent change",2025,National Statistics Office,2017,January/December,"Yes, from 1980",,,,,,,,,US dollar | |
added
tests/fixtures/make_fixture_db.py
+362 −0
@@ -0,0 +1,362 @@ | ||
| 1 | +"""Build a small synthetic DuckDB snapshot for API tests (NEVER shipped). | |
| 2 | + | |
| 3 | +8 countries × 12 indicators × 1990–2024, values follow obvious deterministic formulas (so any test can recompute them), | |
| 4 | +plus IMF forecast rows (2025–2026), alternative-source rows, derived tables (latest, rankings, changes, events, similarity, | |
| 5 | +insights, country_dna, coverage), import_runs, validation_issues, search_index and meta — all from `schema.sql`. | |
| 6 | + | |
| 7 | +Usage: `python tests/fixtures/make_fixture_db.py /tmp/atlas-fixture.duckdb` | |
| 8 | +""" | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import json | |
| 12 | +import math | |
| 13 | +import sys | |
| 14 | +from datetime import date, datetime | |
| 15 | +from pathlib import Path | |
| 16 | + | |
| 17 | +import duckdb | |
| 18 | + | |
| 19 | +ROOT = Path(__file__).resolve().parents[2] | |
| 20 | +SCHEMA = ROOT / "src" / "countryatlas" / "storage" / "schema.sql" | |
| 21 | + | |
| 22 | +COUNTRIES = ["CAN", "USA", "FRA", "DEU", "JPN", "IND", "BRA", "NGA"] | |
| 23 | +INDICATORS = ["population", "gdp", "gdp-per-capita", "gdp-growth", "inflation", "unemployment-rate", "life-expectancy", | |
| 24 | + "median-age", "government-debt-pct-gdp", "co2-per-capita", "renewable-electricity-share", "internet-users", | |
| 25 | + "gdp-per-capita-ppp", "population-growth"] | |
| 26 | +YEARS = list(range(1990, 2025)) | |
| 27 | +RUN_ID = "fixture-20260911T000000" | |
| 28 | +BUILT_AT = "2026-09-11T00:00:00Z" | |
| 29 | +RETRIEVED = datetime(2026, 9, 10, 3, 20, 11) | |
| 30 | +SOURCE_UPDATED = datetime(2026, 7, 1, 0, 0, 0) | |
| 31 | + | |
| 32 | +# synthetic anchors (1990 population in millions, gdp per capita 1990 in US$, life expectancy 1990) | |
| 33 | +ANCHOR = { | |
| 34 | + "CAN": dict(pop=27.7, gpc=21000, le=77.4, growth=1.0, base_unemp=8.0, ren=60, inet_year=1996), | |
| 35 | + "USA": dict(pop=249.6, gpc=23900, le=75.2, growth=1.0, base_unemp=5.6, ren=11, inet_year=1995), | |
| 36 | + "FRA": dict(pop=58.0, gpc=21800, le=76.7, growth=0.5, base_unemp=9.0, ren=15, inet_year=1997), | |
| 37 | + "DEU": dict(pop=79.4, gpc=22200, le=75.3, growth=0.2, base_unemp=6.5, ren=4, inet_year=1997), | |
| 38 | + "JPN": dict(pop=123.5, gpc=25400, le=78.8, growth=0.1, base_unemp=2.1, ren=12, inet_year=1996), | |
| 39 | + "IND": dict(pop=870.5, gpc=370, le=57.9, growth=1.8, base_unemp=5.5, ren=25, inet_year=2002), | |
| 40 | + "BRA": dict(pop=150.7, gpc=3100, le=65.3, growth=1.4, base_unemp=7.0, ren=93, inet_year=2000), | |
| 41 | + "NGA": dict(pop=95.2, gpc=560, le=45.8, growth=2.6, base_unemp=4.0, ren=20, inet_year=2005), | |
| 42 | +} | |
| 43 | + | |
| 44 | + | |
| 45 | +def value(country: str, indicator: str, year: int) -> float | None: | |
| 46 | + a = ANCHOR[country] | |
| 47 | + t = year - 1990 | |
| 48 | + pop = a["pop"] * 1e6 * (1 + a["growth"] / 100) ** t | |
| 49 | + gpc = a["gpc"] * (1.03 if country in ("CAN", "USA", "FRA", "DEU", "JPN") else 1.06) ** t | |
| 50 | + if indicator == "population": | |
| 51 | + return round(pop) | |
| 52 | + if indicator == "population-growth": | |
| 53 | + return a["growth"] - 0.01 * t | |
| 54 | + if indicator == "gdp": | |
| 55 | + return pop * gpc | |
| 56 | + if indicator == "gdp-per-capita": | |
| 57 | + return gpc | |
| 58 | + if indicator == "gdp-per-capita-ppp": | |
| 59 | + return gpc * (1.1 if country in ("CAN", "USA", "FRA", "DEU", "JPN") else 3.2) | |
| 60 | + if indicator == "gdp-growth": | |
| 61 | + base = 2.0 if country in ("CAN", "USA", "FRA", "DEU", "JPN") else 5.0 | |
| 62 | + if year == 2009: | |
| 63 | + return base - 5.0 # global recession → sign flip | |
| 64 | + if year == 2020: | |
| 65 | + return base - 8.0 | |
| 66 | + return base + 0.8 * math.sin(t / 2.0) | |
| 67 | + if indicator == "inflation": | |
| 68 | + if year == 2022: | |
| 69 | + return 8.0 if country != "JPN" else 2.5 | |
| 70 | + return 2.0 + 0.5 * math.cos(t / 3.0) + (3.0 if country in ("IND", "BRA", "NGA") else 0) | |
| 71 | + if indicator == "unemployment-rate": | |
| 72 | + return a["base_unemp"] + (3.0 if year == 2020 else 0) + 0.5 * math.sin(t / 4.0) | |
| 73 | + if indicator == "life-expectancy": | |
| 74 | + return a["le"] + 0.22 * t - (0.8 if year in (2020, 2021) else 0) | |
| 75 | + if indicator == "median-age": | |
| 76 | + return (28 if country in ("IND", "NGA", "BRA") else 33) + 0.3 * t - (8 if country == "NGA" else 0) | |
| 77 | + if indicator == "government-debt-pct-gdp": | |
| 78 | + if year < 1995: | |
| 79 | + return None # deliberately missing early years | |
| 80 | + return 40 + 1.2 * t + (60 if country == "JPN" else 0) + (10 if year >= 2020 else 0) | |
| 81 | + if indicator == "co2-per-capita": | |
| 82 | + base = {"CAN": 16, "USA": 20, "FRA": 6.5, "DEU": 11, "JPN": 9, "IND": 0.7, "BRA": 1.5, "NGA": 0.4}[country] | |
| 83 | + return base * (0.99 ** t if base > 5 else 1.02 ** t) | |
| 84 | + if indicator == "renewable-electricity-share": | |
| 85 | + return min(99.0, a["ren"] + 0.6 * t) | |
| 86 | + if indicator == "internet-users": | |
| 87 | + if year < a["inet_year"]: | |
| 88 | + return None | |
| 89 | + return min(98.0, 100 / (1 + math.exp(-(year - a["inet_year"] - 8) / 2.5))) | |
| 90 | + raise KeyError(indicator) | |
| 91 | + | |
| 92 | + | |
| 93 | +def build(path: Path) -> Path: | |
| 94 | + from countryatlas.registry import countries as reg_countries | |
| 95 | + from countryatlas.registry import groups as reg_groups | |
| 96 | + from countryatlas.registry import indicators as reg_indicators | |
| 97 | + from countryatlas.registry import topics as reg_topics | |
| 98 | + | |
| 99 | + path = Path(path) | |
| 100 | + if path.exists(): | |
| 101 | + path.unlink() | |
| 102 | + con = duckdb.connect(str(path)) | |
| 103 | + con.execute(SCHEMA.read_text()) | |
| 104 | + | |
| 105 | + # ---- countries | |
| 106 | + cs = [c for c in reg_countries() if c.id in COUNTRIES] | |
| 107 | + cols = ["id", "iso2", "iso3", "iso_numeric", "slug", "short_name", "official_name", "capital", "continent", "region_wb", "region_wb_name", | |
| 108 | + "subregion", "income_group", "income_group_name", "currency_code", "currency_name", "area_km2", "latitude", "longitude", | |
| 109 | + "flag_emoji", "un_member", "independent", "landlocked", "borders", "languages", "demonym", "status", "kind"] | |
| 110 | + con.executemany(f"INSERT INTO countries ({', '.join(cols)}) VALUES ({', '.join('?' * len(cols))})", | |
| 111 | + [[getattr(c, k) for k in cols] for c in cs]) | |
| 112 | + | |
| 113 | + # ---- groups + members (restricted to fixture countries) | |
| 114 | + members_rows = [] | |
| 115 | + for g in reg_groups(): | |
| 116 | + m = [x for x in g.members if x in COUNTRIES] | |
| 117 | + con.execute("INSERT INTO groups VALUES (?, ?, ?, ?, ?, ?, ?)", [g.id, g.slug, g.name, g.kind, g.description, g.wb_code, len(m)]) | |
| 118 | + members_rows += [[g.id, x] for x in m] | |
| 119 | + con.executemany("INSERT INTO group_members VALUES (?, ?)", members_rows) | |
| 120 | + | |
| 121 | + # ---- sources | |
| 122 | + sources = [ | |
| 123 | + ("worldbank", "World Bank", "World Bank Group", "https://data.worldbank.org/", "CC BY 4.0", "World Development Indicators, The World Bank", | |
| 124 | + "https://api.worldbank.org/v2"), | |
| 125 | + ("imf", "IMF", "International Monetary Fund", "https://data.imf.org/", "IMF Terms of Use", "World Economic Outlook, IMF", "https://www.imf.org/external/datamapper/api/v1"), | |
| 126 | + ("owid", "Our World in Data", "Global Change Data Lab", "https://ourworldindata.org/", "CC BY 4.0", "Our World in Data", "https://ourworldindata.org/grapher"), | |
| 127 | + ("who", "WHO", "World Health Organization", "https://www.who.int/data/gho", "CC BY-NC-SA 3.0 IGO", "WHO Global Health Observatory", "https://ghoapi.azureedge.net/api"), | |
| 128 | + ] | |
| 129 | + con.executemany("INSERT INTO sources (id, name, organization, url, licence, attribution, api_base, last_success_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", | |
| 130 | + [[*s, RETRIEVED] for s in sources]) | |
| 131 | + | |
| 132 | + # ---- indicators + indicator_sources | |
| 133 | + inds = {i.slug: i for i in reg_indicators() if i.slug in INDICATORS} | |
| 134 | + for slug in INDICATORS: | |
| 135 | + i = inds[slug] | |
| 136 | + con.execute( | |
| 137 | + """INSERT INTO indicators (id, slug, name, short_name, description, topic, subtopic, unit, unit_short, frequency, precision, aggregation, | |
| 138 | + higher_is_better, ranking_eligible, featured, format, scale, bounds_min, bounds_max, methodology, tags, per_capita_of) | |
| 139 | + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", | |
| 140 | + [i.slug, i.slug, i.name, i.short_name, i.description, i.topic, i.subtopic, i.unit, i.unit_short, i.frequency, i.precision, | |
| 141 | + i.aggregation, i.higher_is_better, i.ranking_eligible, i.featured, i.format, i.scale, i.bounds[0], i.bounds[1] if len(i.bounds) > 1 else None, | |
| 142 | + i.methodology, i.tags, i.per_capita_of]) | |
| 143 | + for s in i.sources: | |
| 144 | + if s.connector not in {x[0] for x in sources}: | |
| 145 | + continue | |
| 146 | + con.execute("INSERT INTO indicator_sources (indicator_id, source_id, dataset, series_code, params, priority, transform, countries, notes) " | |
| 147 | + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", | |
| 148 | + [i.slug, s.connector, s.dataset, s.code, json.dumps(s.params), s.priority, s.transform, s.countries, s.notes]) | |
| 149 | + | |
| 150 | + # ---- observations (primary source = first registry source; alt = second when present) | |
| 151 | + obs_rows, alt_rows = [], [] | |
| 152 | + for slug in INDICATORS: | |
| 153 | + i = inds[slug] | |
| 154 | + primary = i.sources[0] | |
| 155 | + alt = i.sources[1] if len(i.sources) > 1 and i.sources[1].connector in {x[0] for x in sources} else None | |
| 156 | + for c in COUNTRIES: | |
| 157 | + for y in YEARS: | |
| 158 | + v = value(c, slug, y) | |
| 159 | + if v is None: | |
| 160 | + continue | |
| 161 | + status = "verified" | |
| 162 | + if slug == "inflation" and y == 2022: | |
| 163 | + status = "warning" | |
| 164 | + obs_rows.append([c, slug, date(y, 1, 1), y, "A", v, i.unit, primary.connector, primary.dataset, primary.code, False, False, 0, | |
| 165 | + RETRIEVED, SOURCE_UPDATED, status, None]) | |
| 166 | + if alt is not None: | |
| 167 | + alt_rows.append([c, slug, date(y, 1, 1), y, "A", v * 1.01, i.unit, alt.connector, alt.dataset, alt.code, False, False, 0, | |
| 168 | + RETRIEVED, SOURCE_UPDATED, "imported", None]) | |
| 169 | + # IMF forecasts for gdp / gdp-per-capita / gdp-growth / inflation | |
| 170 | + if alt is not None and alt.connector == "imf": | |
| 171 | + for y in (2025, 2026): | |
| 172 | + v = value(c, slug, 2024) * (1.03 ** (y - 2024)) if slug != "gdp-growth" else 2.2 | |
| 173 | + obs_rows.append([c, slug, date(y, 1, 1), y, "A", v, i.unit, "imf", "WEO", alt.code, True, True, 0, RETRIEVED, SOURCE_UPDATED, | |
| 174 | + "imported", json.dumps({"forecast": True})]) | |
| 175 | + ins = "INSERT INTO {t} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" | |
| 176 | + con.executemany(ins.format(t="observations"), sorted(obs_rows, key=lambda r: (r[1], r[0], r[2]))) | |
| 177 | + con.executemany(ins.format(t="observations_alt"), alt_rows) | |
| 178 | + | |
| 179 | + # ---- latest (last non-forecast value; ranks within fixture countries) | |
| 180 | + con.execute(""" | |
| 181 | + INSERT INTO latest | |
| 182 | + WITH nf AS (SELECT * FROM observations WHERE NOT is_forecast AND value IS NOT NULL), | |
| 183 | + cur AS (SELECT * FROM nf QUALIFY row_number() OVER (PARTITION BY country_id, indicator_id ORDER BY period DESC) = 1), | |
| 184 | + prev AS (SELECT * FROM nf QUALIFY row_number() OVER (PARTITION BY country_id, indicator_id ORDER BY period DESC) = 2), | |
| 185 | + ten AS (SELECT n.country_id, n.indicator_id, n.value FROM nf n JOIN cur ON cur.country_id = n.country_id AND cur.indicator_id = n.indicator_id | |
| 186 | + AND n.year = cur.year - 10), | |
| 187 | + ranked AS ( | |
| 188 | + SELECT cur.*, c.region_wb, c.income_group, | |
| 189 | + rank() OVER (PARTITION BY cur.indicator_id, cur.year ORDER BY CASE WHEN i.higher_is_better = false THEN cur.value END ASC, | |
| 190 | + CASE WHEN i.higher_is_better = false THEN NULL ELSE cur.value END DESC) AS rw, | |
| 191 | + count(*) OVER (PARTITION BY cur.indicator_id, cur.year) AS nw, | |
| 192 | + rank() OVER (PARTITION BY cur.indicator_id, cur.year, c.region_wb ORDER BY CASE WHEN i.higher_is_better = false THEN cur.value END ASC, | |
| 193 | + CASE WHEN i.higher_is_better = false THEN NULL ELSE cur.value END DESC) AS rr, | |
| 194 | + count(*) OVER (PARTITION BY cur.indicator_id, cur.year, c.region_wb) AS nr, | |
| 195 | + rank() OVER (PARTITION BY cur.indicator_id, cur.year, c.income_group ORDER BY CASE WHEN i.higher_is_better = false THEN cur.value END ASC, | |
| 196 | + CASE WHEN i.higher_is_better = false THEN NULL ELSE cur.value END DESC) AS ri, | |
| 197 | + count(*) OVER (PARTITION BY cur.indicator_id, cur.year, c.income_group) AS ni | |
| 198 | + FROM cur JOIN countries c ON c.id = cur.country_id JOIN indicators i ON i.id = cur.indicator_id) | |
| 199 | + SELECT r.country_id, r.indicator_id, r.period, r.year, r.frequency, r.value, p.period, p.value, | |
| 200 | + r.value - p.value, CASE WHEN p.value <> 0 THEN (r.value - p.value) / abs(p.value) * 100 END, | |
| 201 | + r.rw, r.nw, r.rr, r.nr, r.ri, r.ni, r.year, r.source_id, r.is_forecast, r.is_estimate, r.status, | |
| 202 | + t.value, r.value - t.value, CASE WHEN t.value <> 0 THEN (r.value - t.value) / abs(t.value) * 100 END | |
| 203 | + FROM ranked r | |
| 204 | + LEFT JOIN prev p ON p.country_id = r.country_id AND p.indicator_id = r.indicator_id | |
| 205 | + LEFT JOIN ten t ON t.country_id = r.country_id AND t.indicator_id = r.indicator_id | |
| 206 | + """) | |
| 207 | + | |
| 208 | + # ---- rankings (every year, ranking-eligible indicators) | |
| 209 | + con.execute(""" | |
| 210 | + INSERT INTO rankings | |
| 211 | + SELECT o.indicator_id, o.year, o.country_id, o.value, | |
| 212 | + rank() OVER (PARTITION BY o.indicator_id, o.year ORDER BY CASE WHEN i.higher_is_better = false THEN o.value END ASC, | |
| 213 | + CASE WHEN i.higher_is_better = false THEN NULL ELSE o.value END DESC) AS rank, | |
| 214 | + count(*) OVER (PARTITION BY o.indicator_id, o.year) AS n, | |
| 215 | + percent_rank() OVER (PARTITION BY o.indicator_id, o.year ORDER BY o.value) AS pct_rank | |
| 216 | + FROM observations o JOIN indicators i ON i.id = o.indicator_id | |
| 217 | + WHERE NOT o.is_forecast AND o.value IS NOT NULL AND i.ranking_eligible | |
| 218 | + """) | |
| 219 | + | |
| 220 | + # ---- changes / events (simple deterministic detectors on the synthetic series) | |
| 221 | + now = datetime(2026, 9, 11, 0, 0, 0) | |
| 222 | + changes, events = [], [] | |
| 223 | + for c in COUNTRIES: | |
| 224 | + for slug in INDICATORS: | |
| 225 | + series = [(y, value(c, slug, y)) for y in YEARS if value(c, slug, y) is not None] | |
| 226 | + if len(series) < 3: | |
| 227 | + continue | |
| 228 | + vals = [v for _, v in series] | |
| 229 | + y, v = series[-1] | |
| 230 | + py, pv = series[-2] | |
| 231 | + d = v - pv | |
| 232 | + dp = d / abs(pv) * 100 if pv else None | |
| 233 | + if v == max(vals): | |
| 234 | + changes.append([f"{c}:{slug}:{y}:record_high", c, slug, "record_high", date(y, 1, 1), y, v, max(vals[:-1]), d, dp, len(vals), | |
| 235 | + 0.6, f"{c} reached a record high for {slug} in {y}.", json.dumps({"n_years": len(vals)}), now]) | |
| 236 | + if slug in ("inflation", "unemployment-rate", "gdp-growth") and abs(d) >= 1.0: | |
| 237 | + sev = min(1.0, abs(d) / 5.0) | |
| 238 | + changes.append([f"{c}:{slug}:{y}:yoy", c, slug, "yoy_jump" if d > 0 else "yoy_drop", date(y, 1, 1), y, v, pv, d, dp, 1, sev, | |
| 239 | + f"{slug} moved {d:+.1f} pts in {c} in {y}.", json.dumps({"floor": 1.0}), now]) | |
| 240 | + for (y0, v0), (y1, v1) in zip(series, series[1:]): | |
| 241 | + if slug == "gdp-growth" and (v0 > 0 > v1 or v0 < 0 < v1): | |
| 242 | + events.append([f"{c}:{slug}:{y1}:sign_flip", c, slug, "sign_flip", date(y1, 1, 1), y1, v1, v0, v1 - v0, None, 1, 0.8, | |
| 243 | + f"{c}'s GDP growth turned {'negative' if v1 < 0 else 'positive'} in {y1}.", None]) | |
| 244 | + if slug == "inflation" and v1 - v0 > 3: | |
| 245 | + events.append([f"{c}:{slug}:{y1}:yoy_jump", c, slug, "yoy_jump", date(y1, 1, 1), y1, v1, v0, v1 - v0, None, 1, 0.7, | |
| 246 | + f"Inflation jumped {v1 - v0:+.1f} pts in {c} in {y1}.", None]) | |
| 247 | + con.executemany("INSERT INTO changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", changes) | |
| 248 | + con.executemany("INSERT INTO events VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", events) | |
| 249 | + | |
| 250 | + # ---- similarity (distance on log gdp pc + life expectancy) | |
| 251 | + feats = {c: (math.log(value(c, "gdp-per-capita", 2024)), value(c, "life-expectancy", 2024) / 10) for c in COUNTRIES} | |
| 252 | + sim_rows = [] | |
| 253 | + for mode in ("overall", "economic", "demographic"): | |
| 254 | + for c in COUNTRIES: | |
| 255 | + peers = [] | |
| 256 | + for p in COUNTRIES: | |
| 257 | + if p == c: | |
| 258 | + continue | |
| 259 | + d = math.dist(feats[c], feats[p]) | |
| 260 | + peers.append((p, 100 * math.exp(-d / 1.5), d)) | |
| 261 | + peers.sort(key=lambda x: -x[1]) | |
| 262 | + for rank, (p, score, d) in enumerate(peers, 1): | |
| 263 | + contrib = {"gdp-per-capita": {"z_a": feats[c][0], "z_b": feats[p][0], "weight": 1, "contribution": abs(feats[c][0] - feats[p][0]) / d if d else 0}, | |
| 264 | + "life-expectancy": {"z_a": feats[c][1], "z_b": feats[p][1], "weight": 1, "contribution": abs(feats[c][1] - feats[p][1]) / d if d else 0}} | |
| 265 | + sim_rows.append([c, mode, p, score, rank, json.dumps(contrib)]) | |
| 266 | + con.executemany("INSERT INTO similarity VALUES (?, ?, ?, ?, ?, ?)", sim_rows) | |
| 267 | + | |
| 268 | + # ---- insights | |
| 269 | + ins_rows = [] | |
| 270 | + for c in COUNTRIES: | |
| 271 | + p0, p1 = value(c, "population", 1990), value(c, "population", 2024) | |
| 272 | + pct = (p1 / p0 - 1) * 100 | |
| 273 | + ins_rows.append([f"{c}:pop_growth_since", c, "pop_growth_since", f"{c}'s population grew {pct:.0f}% since 1990.", json.dumps({"pct": pct, "y0": 1990}), | |
| 274 | + ["population"], now]) | |
| 275 | + le = value(c, "life-expectancy", 2024) | |
| 276 | + ins_rows.append([f"{c}:life_expectancy", c, "life_expectancy_level", f"Life expectancy in {c} is {le:.1f} years.", json.dumps({"value": le}), | |
| 277 | + ["life-expectancy"], now]) | |
| 278 | + con.executemany("INSERT INTO insights VALUES (?, ?, ?, ?, ?, ?, ?)", ins_rows) | |
| 279 | + | |
| 280 | + # ---- country_dna (percentile ranks) | |
| 281 | + def pct_rank(slug: str, c: str) -> float: | |
| 282 | + vals = sorted(value(x, slug, 2024) for x in COUNTRIES) | |
| 283 | + return 100.0 * vals.index(value(c, slug, 2024)) / (len(vals) - 1) | |
| 284 | + | |
| 285 | + for c in COUNTRIES: | |
| 286 | + dims = {"income": pct_rank("gdp-per-capita-ppp", c), "demographics": pct_rank("median-age", c), "emissions": pct_rank("co2-per-capita", c), | |
| 287 | + "energy": pct_rank("renewable-electricity-share", c), "public_spending": pct_rank("government-debt-pct-gdp", c), | |
| 288 | + "urbanization": None, "trade": None, "innovation": None, "education": pct_rank("internet-users", c)} | |
| 289 | + con.execute("INSERT INTO country_dna VALUES (?, ?, ?)", [c, json.dumps(dims), 2024]) | |
| 290 | + | |
| 291 | + # ---- coverage | |
| 292 | + con.execute(""" | |
| 293 | + INSERT INTO coverage | |
| 294 | + SELECT country_id, count(DISTINCT indicator_id), count(*), max(year), 100.0 * count(DISTINCT indicator_id) / (SELECT count(*) FROM indicators), ? | |
| 295 | + FROM observations WHERE NOT is_forecast GROUP BY country_id""", [now]) | |
| 296 | + | |
| 297 | + # ---- indicator coverage columns + sources counts | |
| 298 | + con.execute(""" | |
| 299 | + UPDATE indicators SET | |
| 300 | + n_countries = (SELECT count(DISTINCT country_id) FROM observations o WHERE o.indicator_id = indicators.id AND NOT is_forecast), | |
| 301 | + n_observations = (SELECT count(*) FROM observations o WHERE o.indicator_id = indicators.id AND NOT is_forecast), | |
| 302 | + first_year = (SELECT min(year) FROM observations o WHERE o.indicator_id = indicators.id), | |
| 303 | + last_year = (SELECT max(year) FROM observations o WHERE o.indicator_id = indicators.id AND NOT is_forecast), | |
| 304 | + latest_source_updated_at = (SELECT max(source_updated_at) FROM observations o WHERE o.indicator_id = indicators.id), | |
| 305 | + primary_source_id = (SELECT source_id FROM observations o WHERE o.indicator_id = indicators.id GROUP BY source_id ORDER BY count(*) DESC LIMIT 1) | |
| 306 | + """) | |
| 307 | + con.execute(""" | |
| 308 | + UPDATE indicator_sources SET | |
| 309 | + n_observations = (SELECT count(*) FROM observations o WHERE o.indicator_id = indicator_sources.indicator_id AND o.source_id = indicator_sources.source_id), | |
| 310 | + n_countries = (SELECT count(DISTINCT country_id) FROM observations o WHERE o.indicator_id = indicator_sources.indicator_id AND o.source_id = indicator_sources.source_id), | |
| 311 | + last_year = (SELECT max(year) FROM observations o WHERE o.indicator_id = indicator_sources.indicator_id AND o.source_id = indicator_sources.source_id), | |
| 312 | + last_run_id = ?, last_status = 'ok'""", [RUN_ID]) | |
| 313 | + con.execute("UPDATE sources SET n_indicators = (SELECT count(DISTINCT indicator_id) FROM observations o WHERE o.source_id = sources.id), " | |
| 314 | + "n_observations = (SELECT count(*) FROM observations o WHERE o.source_id = sources.id)") | |
| 315 | + | |
| 316 | + # ---- import runs + validation issues | |
| 317 | + runs = [ | |
| 318 | + [RUN_ID, "worldbank", "WDI", datetime(2026, 9, 10, 3, 15), datetime(2026, 9, 10, 3, 20), "ok", 12000, 11800, 11790, 3, 0, None, "raw/worldbank/WDI/2026-09-10"], | |
| 319 | + [RUN_ID, "imf", "WEO", datetime(2026, 9, 10, 3, 20), datetime(2026, 9, 10, 3, 22), "ok", 4000, 3900, 3900, 0, 0, None, "raw/imf/WEO/2026-09-10"], | |
| 320 | + [RUN_ID, "owid", "co2", datetime(2026, 9, 10, 3, 22), datetime(2026, 9, 10, 3, 23), "ok", 3000, 2900, 2900, 0, 0, None, "raw/owid/co2/2026-09-10"], | |
| 321 | + [RUN_ID, "owid", "energy", datetime(2026, 9, 10, 3, 23), datetime(2026, 9, 10, 3, 24), "partial", 3000, 2000, 2000, 5, 0, "2 pages failed", "raw/owid/energy/2026-09-10"], | |
| 322 | + [RUN_ID, "who", "GHO", datetime(2026, 9, 10, 3, 24), datetime(2026, 9, 10, 3, 25), "failed", 0, 0, 0, 0, 1, "HTTP 503", None], | |
| 323 | + ["fixture-20260910T000000", "worldbank", "WDI", datetime(2026, 9, 9, 3, 15), datetime(2026, 9, 9, 3, 20), "ok", 12000, 11800, 11790, 1, 0, None, "raw/worldbank/WDI/2026-09-09"], | |
| 324 | + ] | |
| 325 | + con.executemany("INSERT INTO import_runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", runs) | |
| 326 | + con.executemany("INSERT INTO validation_issues VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ | |
| 327 | + [RUN_ID, "worldbank", "inflation", "NGA", date(2022, 1, 1), "warning", "extreme_jump", "|Δ| = 6.0 > 4 × MAD"], | |
| 328 | + [RUN_ID, "worldbank", "inflation", "BRA", date(2022, 1, 1), "warning", "extreme_jump", "|Δ| = 6.0 > 4 × MAD"], | |
| 329 | + [RUN_ID, "owid", "renewable-electricity-share", None, None, "info", "partial_download", "2 pages failed, previous kept"], | |
| 330 | + [RUN_ID, "who", "life-expectancy", None, None, "error", "schema_change", "HTTP 503"], | |
| 331 | + ]) | |
| 332 | + | |
| 333 | + # ---- search index | |
| 334 | + si = [] | |
| 335 | + for c in cs: | |
| 336 | + si.append(["country", c.id, c.slug, c.short_name, f"{c.official_name} {c.iso2} {c.iso3} {c.capital}", f"Country · {c.region_wb_name}", 1.0]) | |
| 337 | + tnames = {t["id"]: t["name"] for t in reg_topics()["topics"]} | |
| 338 | + for slug in INDICATORS: | |
| 339 | + i = inds[slug] | |
| 340 | + si.append(["indicator", slug, slug, i.name, f"{i.short_name or ''} {' '.join(i.tags)}", f"Indicator · {tnames.get(i.topic, i.topic)} · {i.unit}", | |
| 341 | + 0.9 if i.featured else 0.7]) | |
| 342 | + for t in reg_topics()["topics"]: | |
| 343 | + si.append(["topic", t["id"], t["id"], t["name"], t.get("short", ""), f"Topic · {len(t['indicators'])} indicators", 0.6]) | |
| 344 | + for g in reg_groups(): | |
| 345 | + si.append(["region", g.id, g.slug, g.name, g.wb_code or "", f"Region · {g.kind}", 0.6]) | |
| 346 | + for s in sources: | |
| 347 | + si.append(["source", s[0], s[0], s[1], s[2], "Source", 0.4]) | |
| 348 | + con.executemany("INSERT INTO search_index VALUES (?, ?, ?, ?, ?, ?, ?)", si) | |
| 349 | + | |
| 350 | + # ---- meta | |
| 351 | + n_obs = con.execute("SELECT count(*) FROM observations").fetchone()[0] | |
| 352 | + con.executemany("INSERT INTO meta VALUES (?, ?)", [ | |
| 353 | + ["build_run_id", RUN_ID], ["built_at", BUILT_AT], ["schema_version", "1"], ["indicator_count", str(len(INDICATORS))], | |
| 354 | + ["country_count", str(len(COUNTRIES))], ["observation_count", str(n_obs)], ["fixture", "true"], | |
| 355 | + ]) | |
| 356 | + con.close() | |
| 357 | + return path | |
| 358 | + | |
| 359 | + | |
| 360 | +if __name__ == "__main__": | |
| 361 | + out = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/tmp/atlas-fixture.duckdb") | |
| 362 | + print(build(out)) | |
added
tests/fixtures/oecd/house_prices_RHP_sample.csv
+29 −0
@@ -0,0 +1,29 @@ | ||
| 1 | +STRUCTURE,STRUCTURE_ID,STRUCTURE_NAME,ACTION,REF_AREA,Reference area,FREQ,Frequency of observation,MEASURE,Measure,UNIT_MEASURE,Unit of measure,TIME_PERIOD,Time period,OBS_VALUE,Observation value,OBS_STATUS,Observation status,UNIT_MULT,Unit multiplier,ADJUSTMENT,Adjustment,DECIMALS,Decimals,BASE_PER,Base period | |
| 2 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2025-Q2,,154.108309038445,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 3 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2025-Q3,,153.584464727063,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 4 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2025-Q4,,136.68289059965,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 5 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2024-Q1,,143.64645165129,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 6 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2024-Q2,,144.356998726433,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 7 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2024-Q3,,144.074050555108,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 8 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2023-Q1,,141.377594687013,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 9 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2023-Q2,,141.367635151329,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 10 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2025-Q4,,153.808521660016,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 11 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2026-Q1,,134.23397713458,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 12 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2026-Q2,,130.206163080039,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 13 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2023-Q1,,147.087172515903,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 14 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2023-Q2,,148.779760187957,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 15 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2023-Q3,,150.982874590119,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 16 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2023-Q4,,152.051917547031,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 17 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2024-Q1,,152.300754253783,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 18 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2024-Q2,,153.157909401831,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 19 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2024-Q3,,153.970981528031,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 20 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2024-Q4,,154.971633688811,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 21 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2025-Q1,,154.781356425008,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 22 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2025-Q2,,140.234103846876,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 23 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2025-Q3,,137.248613776868,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 24 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2023-Q3,,145.837592804845,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 25 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2023-Q4,,144.803826015121,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 26 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2026-Q1,,152.945855206175,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 27 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,USA,United States,Q,Quarterly,RHP,Real house price indices,IX,Index,2026-Q2,,151.525340187709,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 28 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2024-Q4,,144.550752556667,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
| 29 | +DATAFLOW,OECD.ECO.MPD:DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES(1.0),Analytical house prices indicators,I,CAN,Canada,Q,Quarterly,RHP,Real house price indices,IX,Index,2025-Q1,,143.937431533559,,A,Normal value,0,Units,S,"Seasonally adjusted, not calendar adjusted",1,One,2015, | |
added
tests/fixtures/oecd/kei_PRVM_sample.csv
+14 −0
@@ -0,0 +1,14 @@ | ||
| 1 | +STRUCTURE,STRUCTURE_ID,STRUCTURE_NAME,ACTION,REF_AREA,Reference area,FREQ,Frequency of observation,MEASURE,Measure,UNIT_MEASURE,Unit of measure,ACTIVITY,Economic activity,ADJUSTMENT,Adjustment,TRANSFORMATION,Transformation,TIME_PERIOD,Time period,OBS_VALUE,Observation value,OBS_STATUS,Observation status,UNIT_MULT,Unit multiplier,DECIMALS,Decimals,BASE_PER,Base period | |
| 2 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,USA,United States,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-04,,101.532958867368,,A,Normal value,0,Units,1,One,2015, | |
| 3 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,CAN,Canada,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-04,,108.557160864928,,A,Normal value,0,Units,1,One,2015, | |
| 4 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,CAN,Canada,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-05,,109.425282767039,,A,Normal value,0,Units,1,One,2015, | |
| 5 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,CAN,Canada,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-06,,109.273561067984,,A,Normal value,0,Units,1,One,2015, | |
| 6 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,CAN,Canada,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-01,,106.16469219317699,,A,Normal value,0,Units,1,One,2015, | |
| 7 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,CAN,Canada,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-02,,107.146034987252,,A,Normal value,0,Units,1,One,2015, | |
| 8 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,USA,United States,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-05,,101.52315416337201,,A,Normal value,0,Units,1,One,2015, | |
| 9 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,CAN,Canada,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-03,,107.03794753623399,,A,Normal value,0,Units,1,One,2015, | |
| 10 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,USA,United States,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-01,,100.06621476444799,,A,Normal value,0,Units,1,One,2015, | |
| 11 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,USA,United States,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-02,,100.924373957682,,A,Normal value,0,Units,1,One,2015, | |
| 12 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,USA,United States,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-03,,100.774827462381,,A,Normal value,0,Units,1,One,2015, | |
| 13 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,USA,United States,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-06,,101.79738876303298,,A,Normal value,0,Units,1,One,2015, | |
| 14 | +DATAFLOW,OECD.SDD.STES:DSD_KEI@DF_KEI(4.0),Key short-term economic indicators,I,USA,United States,M,Monthly,PRVM,Production volume,IX,Index,BTE,Industry (except construction),Y,Calendar and seasonally adjusted,_Z,Not applicable,2026-07,,102.002495247648,,A,Normal value,0,Units,1,One,2015, | |
added
tests/fixtures/owid_co2_subset.csv
+151 −0
@@ -0,0 +1,151 @@ | ||
| 1 | +country,year,iso_code,co2,co2_per_capita,population | |
| 2 | +Asia,2000,,9330.988,2.49,3746832777 | |
| 3 | +Asia,2001,,9518.094,2.506,3797954107 | |
| 4 | +Asia,2002,,10033.99,2.608,3847359674 | |
| 5 | +Asia,2003,,11036.212,2.833,3895799439 | |
| 6 | +Asia,2004,,11688.357,2.963,3944378395 | |
| 7 | +Asia,2005,,12547.688,3.143,3992890580 | |
| 8 | +Asia,2006,,13390.617,3.313,4041771416 | |
| 9 | +Asia,2007,,14118.133,3.451,4090564345 | |
| 10 | +Asia,2008,,14843.06,3.586,4139267468 | |
| 11 | +Asia,2009,,15377.568,3.671,4188829464 | |
| 12 | +Asia,2010,,16536.467,3.901,4238694946 | |
| 13 | +Asia,2011,,17768.818,4.144,4288350005 | |
| 14 | +Asia,2012,,18438.48,4.25,4338443538 | |
| 15 | +Asia,2013,,18685.477,4.259,4387757369 | |
| 16 | +Asia,2014,,18961.438,4.275,4435744576 | |
| 17 | +Asia,2015,,19070.406,4.255,4482088489 | |
| 18 | +Asia,2016,,19202.404,4.242,4527178153 | |
| 19 | +Asia,2017,,19775.838,4.326,4571224645 | |
| 20 | +Asia,2018,,20366.35,4.416,4612352852 | |
| 21 | +Asia,2019,,20926.584,4.499,4651098724 | |
| 22 | +Asia,2020,,20691.949,4.415,4686811214 | |
| 23 | +Asia,2021,,21520.502,4.562,4716999164 | |
| 24 | +Asia,2022,,22237.812,4.685,4746329727 | |
| 25 | +Asia,2023,,22997.092,4.814,4776659631 | |
| 26 | +Asia,2024,,23392.488,4.868,4805539800 | |
| 27 | +Canada,2000,CAN,566.68,18.344,30891800 | |
| 28 | +Canada,2001,CAN,558.981,17.908,31213578 | |
| 29 | +Canada,2002,CAN,564.256,17.893,31535579 | |
| 30 | +Canada,2003,CAN,581.19,18.255,31836483 | |
| 31 | +Canada,2004,CAN,576.286,17.935,32132680 | |
| 32 | +Canada,2005,CAN,570.396,17.583,32440172 | |
| 33 | +Canada,2006,CAN,566.315,17.287,32759174 | |
| 34 | +Canada,2007,CAN,589.956,17.828,33092171 | |
| 35 | +Canada,2008,CAN,574.214,17.167,33449088 | |
| 36 | +Canada,2009,CAN,541.13,15.997,33826370 | |
| 37 | +Canada,2010,CAN,554.307,16.209,34196900 | |
| 38 | +Canada,2011,CAN,562.832,16.288,34555449 | |
| 39 | +Canada,2012,CAN,561.77,16.086,34922513 | |
| 40 | +Canada,2013,CAN,568.59,16.11,35293421 | |
| 41 | +Canada,2014,CAN,564.938,15.854,35634265 | |
| 42 | +Canada,2015,CAN,563.003,15.655,35962236 | |
| 43 | +Canada,2016,CAN,553.707,15.231,36353345 | |
| 44 | +Canada,2017,CAN,566.66,15.395,36808498 | |
| 45 | +Canada,2018,CAN,575.091,15.42,37294996 | |
| 46 | +Canada,2019,CAN,579.007,15.325,37782934 | |
| 47 | +Canada,2020,CAN,524.208,13.733,38171903 | |
| 48 | +Canada,2021,CAN,537.479,13.977,38454058 | |
| 49 | +Canada,2022,CAN,547.658,14.107,38821259 | |
| 50 | +Canada,2023,CAN,545.479,13.88,39299098 | |
| 51 | +Canada,2024,CAN,533.34,13.42,39742429 | |
| 52 | +France,2000,FRA,407.445,6.85,59483716 | |
| 53 | +France,2001,FRA,411.592,6.871,59905131 | |
| 54 | +France,2002,FRA,407.027,6.747,60327247 | |
| 55 | +France,2003,FRA,412.989,6.799,60741119 | |
| 56 | +France,2004,FRA,414.073,6.769,61175244 | |
| 57 | +France,2005,FRA,416.343,6.756,61625031 | |
| 58 | +France,2006,FRA,406.669,6.554,62049831 | |
| 59 | +France,2007,FRA,396.461,6.35,62432438 | |
| 60 | +France,2008,FRA,389.794,6.209,62780187 | |
| 61 | +France,2009,FRA,371.404,5.885,63106460 | |
| 62 | +France,2010,FRA,377.079,5.946,63417366 | |
| 63 | +France,2011,FRA,355.127,5.572,63733776 | |
| 64 | +France,2012,FRA,358.1,5.59,64058465 | |
| 65 | +France,2013,FRA,360.013,5.591,64387736 | |
| 66 | +France,2014,FRA,327.809,5.067,64692495 | |
| 67 | +France,2015,FRA,332.131,5.116,64916337 | |
| 68 | +France,2016,FRA,335.496,5.155,65086853 | |
| 69 | +France,2017,FRA,338.368,5.183,65284775 | |
| 70 | +France,2018,FRA,323.102,4.931,65519537 | |
| 71 | +France,2019,FRA,316.321,4.812,65729460 | |
| 72 | +France,2020,FRA,281.515,4.272,65905273 | |
| 73 | +France,2021,FRA,307.272,4.65,66083547 | |
| 74 | +France,2022,FRA,295.304,4.456,66277412 | |
| 75 | +France,2023,FRA,270.263,4.068,66438828 | |
| 76 | +France,2024,FRA,264.156,3.969,66548532 | |
| 77 | +Kuwait,2000,KWT,54.98,28.127,1954713 | |
| 78 | +Kuwait,2001,KWT,59.119,29.44,2008127 | |
| 79 | +Kuwait,2002,KWT,60.114,29.182,2059983 | |
| 80 | +Kuwait,2003,KWT,62.491,29.618,2109886 | |
| 81 | +Kuwait,2004,KWT,65.855,30.526,2157329 | |
| 82 | +Kuwait,2005,KWT,74.615,33.358,2236797 | |
| 83 | +Kuwait,2006,KWT,76.222,32.233,2364741 | |
| 84 | +Kuwait,2007,KWT,77.295,30.821,2507890 | |
| 85 | +Kuwait,2008,KWT,84.442,31.848,2651433 | |
| 86 | +Kuwait,2009,KWT,88.832,31.777,2795489 | |
| 87 | +Kuwait,2010,KWT,89.721,30.482,2943374 | |
| 88 | +Kuwait,2011,KWT,87.151,27.819,3132815 | |
| 89 | +Kuwait,2012,KWT,101.201,30.327,3337025 | |
| 90 | +Kuwait,2013,KWT,83.047,23.675,3507755 | |
| 91 | +Kuwait,2014,KWT,74.994,20.458,3665769 | |
| 92 | +Kuwait,2015,KWT,93.317,24.337,3834465 | |
| 93 | +Kuwait,2016,KWT,104.019,25.979,4003976 | |
| 94 | +Kuwait,2017,KWT,100.01,24.072,4154686 | |
| 95 | +Kuwait,2018,KWT,102.453,23.697,4323389 | |
| 96 | +Kuwait,2019,KWT,104.654,23.559,4442201 | |
| 97 | +Kuwait,2020,KWT,90.103,20.477,4400146 | |
| 98 | +Kuwait,2021,KWT,87.869,20.15,4360748 | |
| 99 | +Kuwait,2022,KWT,115.849,25.242,4589514 | |
| 100 | +Kuwait,2023,KWT,123.169,25.454,4838781 | |
| 101 | +Kuwait,2024,KWT,129.519,26.248,4934508 | |
| 102 | +Nigeria,2000,NGA,96.831,0.766,126382490 | |
| 103 | +Nigeria,2001,NGA,100.218,0.772,129862594 | |
| 104 | +Nigeria,2002,NGA,89.906,0.674,133471995 | |
| 105 | +Nigeria,2003,NGA,100.03,0.729,137202646 | |
| 106 | +Nigeria,2004,NGA,94.962,0.673,141057045 | |
| 107 | +Nigeria,2005,NGA,101.666,0.701,145017256 | |
| 108 | +Nigeria,2006,NGA,90.097,0.604,149077335 | |
| 109 | +Nigeria,2007,NGA,82.17,0.536,153267254 | |
| 110 | +Nigeria,2008,NGA,86.489,0.549,157595014 | |
| 111 | +Nigeria,2009,NGA,77.031,0.475,162049466 | |
| 112 | +Nigeria,2010,NGA,112.05,0.672,166642888 | |
| 113 | +Nigeria,2011,NGA,126.115,0.736,171379602 | |
| 114 | +Nigeria,2012,NGA,110.125,0.625,176200627 | |
| 115 | +Nigeria,2013,NGA,117.401,0.648,181049440 | |
| 116 | +Nigeria,2014,NGA,123.971,0.667,185896917 | |
| 117 | +Nigeria,2015,NGA,110.525,0.58,190671883 | |
| 118 | +Nigeria,2016,NGA,116.396,0.596,195443698 | |
| 119 | +Nigeria,2017,NGA,112.54,0.562,200254580 | |
| 120 | +Nigeria,2018,NGA,106.786,0.521,204938752 | |
| 121 | +Nigeria,2019,NGA,127.859,0.61,209485637 | |
| 122 | +Nigeria,2020,NGA,124.899,0.584,213996185 | |
| 123 | +Nigeria,2021,NGA,147.777,0.676,218529286 | |
| 124 | +Nigeria,2022,NGA,131.441,0.589,223150906 | |
| 125 | +Nigeria,2023,NGA,129.376,0.568,227882949 | |
| 126 | +Nigeria,2024,NGA,135.824,0.584,232679482 | |
| 127 | +World,2000,,25511.482,4.134,6171702992 | |
| 128 | +World,2001,,25692.994,4.108,6254936464 | |
| 129 | +World,2002,,26265.332,4.144,6337730343 | |
| 130 | +World,2003,,27652.691,4.307,6420361633 | |
| 131 | +World,2004,,28610.023,4.399,6503377774 | |
| 132 | +World,2005,,29598.951,4.494,6586970132 | |
| 133 | +World,2006,,30594.152,4.586,6671452019 | |
| 134 | +World,2007,,31499.256,4.662,6757308776 | |
| 135 | +World,2008,,32049.66,4.683,6844457659 | |
| 136 | +World,2009,,31513.014,4.546,6932766416 | |
| 137 | +World,2010,,33317.672,4.745,7021732143 | |
| 138 | +World,2011,,34479.988,4.849,7110923773 | |
| 139 | +World,2012,,34954.734,4.854,7201202478 | |
| 140 | +World,2013,,35275.82,4.838,7291793585 | |
| 141 | +World,2014,,35465.695,4.805,7381616239 | |
| 142 | +World,2015,,35403.527,4.739,7470491876 | |
| 143 | +World,2016,,35392.832,4.682,7558554527 | |
| 144 | +World,2017,,35974.609,4.705,7645617952 | |
| 145 | +World,2018,,36734.004,4.752,7729902779 | |
| 146 | +World,2019,,37086.566,4.748,7811293699 | |
| 147 | +World,2020,,35158.23,4.458,7887001289 | |
| 148 | +World,2021,,36866.863,4.635,7954448387 | |
| 149 | +World,2022,,37527.773,4.678,8021407196 | |
| 150 | +World,2023,,38094.039,4.708,8091734933 | |
| 151 | +World,2024,,38598.578,4.729,8161972574 | |
added
tests/fixtures/owid_grapher_median-age.csv
+65 −0
@@ -0,0 +1,65 @@ | ||
| 1 | +entity,code,year,median_age__sex_all__age_all__variant_estimates,median_age__sex_all__age_all__variant_medium__projected | |
| 2 | +Canada,CAN,2015,39.629, | |
| 3 | +Canada,CAN,2016,39.733, | |
| 4 | +Canada,CAN,2017,39.77, | |
| 5 | +Canada,CAN,2018,39.769, | |
| 6 | +Canada,CAN,2019,39.788, | |
| 7 | +Canada,CAN,2020,39.901, | |
| 8 | +Canada,CAN,2021,40.105, | |
| 9 | +Canada,CAN,2022,40.253, | |
| 10 | +Canada,CAN,2023,40.343, | |
| 11 | +Canada,CAN,2024,,40.476 | |
| 12 | +Canada,CAN,2025,,40.643 | |
| 13 | +Canada,CAN,2026,,40.838 | |
| 14 | +Canada,CAN,2027,,41.048 | |
| 15 | +Canada,CAN,2028,,41.258 | |
| 16 | +Canada,CAN,2029,,41.466 | |
| 17 | +Canada,CAN,2030,,41.672 | |
| 18 | +Japan,JPN,2015,45.845, | |
| 19 | +Japan,JPN,2016,46.195, | |
| 20 | +Japan,JPN,2017,46.535, | |
| 21 | +Japan,JPN,2018,46.889, | |
| 22 | +Japan,JPN,2019,47.262, | |
| 23 | +Japan,JPN,2020,47.674, | |
| 24 | +Japan,JPN,2021,48.123, | |
| 25 | +Japan,JPN,2022,48.548, | |
| 26 | +Japan,JPN,2023,48.958, | |
| 27 | +Japan,JPN,2024,,49.383 | |
| 28 | +Japan,JPN,2025,,49.792 | |
| 29 | +Japan,JPN,2026,,50.181 | |
| 30 | +Japan,JPN,2027,,50.544 | |
| 31 | +Japan,JPN,2028,,50.881 | |
| 32 | +Japan,JPN,2029,,51.192 | |
| 33 | +Japan,JPN,2030,,51.48 | |
| 34 | +Niger,NER,2015,14.343, | |
| 35 | +Niger,NER,2016,14.417, | |
| 36 | +Niger,NER,2017,14.498, | |
| 37 | +Niger,NER,2018,14.591, | |
| 38 | +Niger,NER,2019,14.699, | |
| 39 | +Niger,NER,2020,14.821, | |
| 40 | +Niger,NER,2021,14.958, | |
| 41 | +Niger,NER,2022,15.102, | |
| 42 | +Niger,NER,2023,15.25, | |
| 43 | +Niger,NER,2024,,15.4 | |
| 44 | +Niger,NER,2025,,15.555 | |
| 45 | +Niger,NER,2026,,15.717 | |
| 46 | +Niger,NER,2027,,15.886 | |
| 47 | +Niger,NER,2028,,16.062 | |
| 48 | +Niger,NER,2029,,16.243 | |
| 49 | +Niger,NER,2030,,16.428 | |
| 50 | +World,OWID_WRL,2015,28.296, | |
| 51 | +World,OWID_WRL,2016,28.54, | |
| 52 | +World,OWID_WRL,2017,28.786, | |
| 53 | +World,OWID_WRL,2018,29.041, | |
| 54 | +World,OWID_WRL,2019,29.305, | |
| 55 | +World,OWID_WRL,2020,29.575, | |
| 56 | +World,OWID_WRL,2021,29.833, | |
| 57 | +World,OWID_WRL,2022,30.093, | |
| 58 | +World,OWID_WRL,2023,30.364, | |
| 59 | +World,OWID_WRL,2024,,30.621 | |
| 60 | +World,OWID_WRL,2025,,30.863 | |
| 61 | +World,OWID_WRL,2026,,31.099 | |
| 62 | +World,OWID_WRL,2027,,31.331 | |
| 63 | +World,OWID_WRL,2028,,31.558 | |
| 64 | +World,OWID_WRL,2029,,31.778 | |
| 65 | +World,OWID_WRL,2030,,31.992 | |
added
tests/fixtures/wb_NY.GDP.PCAP.CD.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +[{"page": 1, "pages": 1, "per_page": 20000, "total": 260, "sourceid": "2", "lastupdated": "2026-07-13"}, [{"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2025", "value": 53997.2579539931, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2024", "value": 50907.2761474674, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2023", "value": 49141.5400740302, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2022", "value": 47056.4874992383, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2021", "value": 45545.9474677756, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2020", "value": 40453.9686533276, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2019", "value": 41570.3760789982, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2018", "value": 41295.6262218527, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2017", "value": 38970.4660370017, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2016", "value": 37232.2355227355, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2015", "value": 36734.6310227088, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2014", "value": 39576.1363506592, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2013", "value": 39099.8824050102, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2012", "value": 38756.7592855469, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2011", "value": 38837.3278350951, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2010", "value": 35833.7115834135, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2009", "value": 34160.3443365118, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2008", "value": 36948.6937394895, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2007", "value": 34703.2215480124, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2006", "value": 31743.3108447684, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2005", "value": 30080.7834726251, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2004", "value": 28482.3042788075, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2003", "value": 25642.0998662756, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2002", "value": 22974.3592654579, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2001", "value": 22129.0189979774, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "2000", "value": 22428.1726072309, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1999", "value": 22013.5780601712, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1998", "value": 21128.265507888, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1997", "value": 21308.6065181459, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1996", "value": 21791.1991160631, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1995", "value": 21739.9319298831, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1994", "value": 19720.3175871797, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1993", "value": 18549.9752077281, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1992", "value": 18533.1621649982, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1991", "value": 17431.5724536724, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1990", "value": 16636.4892295512, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1989", "value": 14952.8323588626, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1988", "value": 14461.0324091605, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1987", "value": 12932.5461891018, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1986", "value": 11227.3834467229, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1985", "value": 9299.05109519506, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1984", "value": 8919.82922866066, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1983", "value": 8611.95690854169, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1982", "value": 8391.40363012421, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1981", "value": 8537.57496050414, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1980", "value": 8474.01441100501, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1979", "value": 7652.3565490127, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1978", "value": 6677.23314203314, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1977", "value": 5604.01744747437, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1976", "value": 4988.94295390305, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1975", "value": 4617.80429809259, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1974", "value": 4174.52891466863, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1973", "value": 3724.02876595768, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1972", "value": 3097.15802810705, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1971", "value": 2691.13957190777, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1970", "value": 2448.68072079549, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1969", "value": 2251.31103536381, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1968", "value": 2061.77922534071, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1967", "value": 1918.49117738741, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1966", "value": 1812.92731613899, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1965", "value": 1673.40567192883, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1964", "value": 1557.2422754082, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1963", "value": 1441.78056642276, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1962", "value": 1356.6339159265, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1961", "value": 1269.99482092477, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "XD", "value": "High income"}, "countryiso3code": "", "date": "1960", "value": 1205.22806652384, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2025", "value": 14405.8484595528, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2024", "value": 13717.0909956338, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2023", "value": 13313.8575826001, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2022", "value": 12876.2377124072, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2021", "value": 12472.9074157191, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2020", "value": 10999.4917581781, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2019", "value": 11417.5958587724, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2018", "value": 11344.728168656, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2017", "value": 10784.3950896297, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2016", "value": 10239.4656645738, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2015", "value": 10191.9752512662, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2014", "value": 10912.2447775638, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2013", "value": 10759.3457336689, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2012", "value": 10597.8460782595, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2011", "value": 10496.9310026996, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2010", "value": 9555.06543969498, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2009", "value": 8827.04322994899, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2008", "value": 9425.73781604823, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2007", "value": 8685.33108680793, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2006", "value": 7803.19467222601, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2005", "value": 7290.5661453297, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2004", "value": 6818.00375324836, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2003", "value": 6129.8663827398, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2002", "value": 5539.32465002692, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2001", "value": 5404.20269822848, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "2000", "value": 5512.56177163867, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1999", "value": 5401.2271886284, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1998", "value": 5302.33956683372, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1997", "value": 5408.5695352216, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1996", "value": 5507.22343967666, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1995", "value": 5463.19624240532, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1994", "value": 4967.01235983571, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1993", "value": 4685.29564260401, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1992", "value": 4679.98200500774, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1991", "value": 4453.89641554733, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1990", "value": 4338.94333025321, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1989", "value": 3930.50666963009, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1988", "value": 3829.99224771219, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1987", "value": 3487.87666375394, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1986", "value": 3118.57247275326, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1985", "value": 2690.59161836012, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1984", "value": 2614.3302826929, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1983", "value": 2561.16933615596, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1982", "value": 2554.07605312193, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1981", "value": 2628.61699908178, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1980", "value": 2587.00992594104, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1979", "value": 2335.18686803709, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1978", "value": 2055.4940430146, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1977", "value": 1766.27881654492, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1976", "value": 1589.73366543803, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1975", "value": 1488.93763061623, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1974", "value": 1362.18919097786, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1973", "value": 1201.53152954524, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1972", "value": 1002.00152690776, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1971", "value": 884.878986369128, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1970", "value": 817.683493933039, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1969", "value": 761.11140319217, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1968", "value": 703.462134148118, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1967", "value": 664.401307345425, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1966", "value": 638.970700706876, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1965", "value": 601.888592945637, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1964", "value": 563.850666935202, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1963", "value": 525.186416425983, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1962", "value": 496.105588960377, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1961", "value": 471.931933312191, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "1W", "value": "World"}, "countryiso3code": "WLD", "date": "1960", "value": 452.653355677478, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2025", "value": 10713.2856869511, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2024", "value": 10310.5486973693, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2023", "value": 10377.5892792557, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2022", "value": 9281.33334441234, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2021", "value": 7972.5366498646, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2020", "value": 7074.19378337644, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2019", "value": 9029.83326681073, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2018", "value": 9300.66164923219, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2017", "value": 10080.5092819305, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2016", "value": 8836.28652735657, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2015", "value": 8936.19661712113, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2014", "value": 12274.9939689363, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2013", "value": 12458.8912138813, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2012", "value": 12521.7213519966, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2011", "value": 13396.6243562986, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "BR", "value": "Brazil"}, "countryiso3code": "BRA", "date": "2010", "value": 11403.2821278747, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2025", "value": 55697.6639660837, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2024", "value": 55015.7066917734, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2023", "value": 54847.5370112652, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2022", "value": 56496.9275340663, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2021", "value": 52886.6407813788, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2020", "value": 43537.8981198851, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2019", "value": 46352.9695208927, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2018", "value": 46539.2177903178, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2017", "value": 45129.7335281051, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2016", "value": 42314.1033623905, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2015", "value": 43594.2379080706, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2014", "value": 50960.8942086685, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2013", "value": 52637.7313222622, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2012", "value": 52670.1429540952, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2011", "value": 52224.1237767406, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "CA", "value": "Canada"}, "countryiso3code": "CAN", "date": "2010", "value": 47560.8378831258, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2025", "value": 48985.7307807924, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2024", "value": 46103.0840855884, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2023", "value": 44700.1384177544, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2022", "value": 40988.6396406579, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2021", "value": 43725.0999521245, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2020", "value": 39169.8606000707, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2019", "value": 40408.2848574751, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2018", "value": 41418.1766484844, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2017", "value": 38687.1626407164, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2016", "value": 37024.2157133669, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2015", "value": 36702.4323733379, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2014", "value": 43148.0459288416, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2013", "value": 42669.1795111893, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2012", "value": 40863.5814412333, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2011", "value": 43929.7840873812, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "FR", "value": "France"}, "countryiso3code": "FRA", "date": "2010", "value": 40694.8211697025, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2025", "value": 60496.4350820414, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2024", "value": 56103.7323182554, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2023", "value": 54776.7668235491, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2022", "value": 50506.5179638543, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2021", "value": 52349.2459994422, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2020", "value": 47394.8734504469, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2019", "value": 47656.199739747, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2018", "value": 48916.1686612154, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2017", "value": 45553.9341495339, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2016", "value": 42948.9381932694, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2015", "value": 41929.7549110722, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2014", "value": 48959.5991203133, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2013", "value": 47206.8353514242, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2012", "value": 44718.0159214069, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2011", "value": 47630.9761883572, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "DE", "value": "Germany"}, "countryiso3code": "DEU", "date": "2010", "value": 42396.9665976293, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2025", "value": 2702.47987141553, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2024", "value": 2591.9916607122, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2023", "value": 2434.44826340989, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2022", "value": 2279.98145719369, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2021", "value": 2239.61384367482, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2020", "value": 1907.04251637669, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2019", "value": 2041.42863698585, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2018", "value": 1966.25455171679, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2017", "value": 1950.10468280866, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2016", "value": 1707.50892912243, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2015", "value": 1583.99815907985, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2014", "value": 1553.88396075118, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2013", "value": 1432.84397512195, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2012", "value": 1429.32199520032, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2011", "value": 1445.46127486037, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "IN", "value": "India"}, "countryiso3code": "IND", "date": "2010", "value": 1347.51939071367, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2025", "value": 35951.0449549304, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2024", "value": 33797.1014287876, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2023", "value": 35215.003535366, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2022", "value": 35548.2645222433, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2021", "value": 41580.739040705, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2020", "value": 41098.9739004553, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2019", "value": 41424.8655601543, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2018", "value": 40645.478086519, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2017", "value": 39679.8635262948, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2016", "value": 40214.9688631562, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2015", "value": 35664.6446110613, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2014", "value": 39172.8471161948, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2013", "value": 41369.1948479498, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2012", "value": 49626.6830327571, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2011", "value": 49122.077634247, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "JP", "value": "Japan"}, "countryiso3code": "JPN", "date": "2010", "value": 45378.1378109399, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2025", "value": 1224.25410237743, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2024", "value": 1084.16041805058, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2023", "value": 2138.76383718757, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2022", "value": 2899.16047469267, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2021", "value": 2787.48779220991, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2020", "value": 2797.184580679, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2019", "value": 3189.81286496183, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2018", "value": 2057.87944554011, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2017", "value": 1876.34027111773, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2016", "value": 2070.4127339595, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2015", "value": 2585.73360671797, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2014", "value": 3088.72131316169, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2013", "value": 2872.79083379398, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2012", "value": 2633.19734671361, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2011", "value": 2418.41316975875, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "NG", "value": "Nigeria"}, "countryiso3code": "NGA", "date": "2010", "value": 2202.25672957344, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2025", "value": 90026.5163005744, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2024", "value": 86169.6641581917, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2023", "value": 82586.7847707894, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2022", "value": 78008.6895811877, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2021", "value": 71441.2319805947, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2020", "value": 64465.2971415748, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2019", "value": 65227.9565911038, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2018", "value": 62875.6661382728, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2017", "value": 60047.7190728307, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2016", "value": 57976.628204291, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2015", "value": 56849.4697923159, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2014", "value": 55153.3940182967, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2013", "value": 53297.3862901595, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2012", "value": 51708.3940614082, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2011", "value": 50024.8812320773, "unit": "", "obs_status": "", "decimal": 1}, {"indicator": {"id": "NY.GDP.PCAP.CD", "value": "GDP per capita (current US$)"}, "country": {"id": "US", "value": "United States"}, "countryiso3code": "USA", "date": "2010", "value": 48642.6312088213, "unit": "", "obs_status": "", "decimal": 1}]] | |
| \ No newline at end of file | ||
added
tests/fixtures/wb_retired.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +[{"message": [{"id": "175", "key": "Invalid value", "value": "The provided parameter value is not valid"}]}] | |
| \ No newline at end of file | ||
added
tests/fixtures/who/Indicator_M_Est_tob_curr.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"@odata.context": "https://ghoapi.azureedge.net/api/$metadata#Indicator", "value": [{"IndicatorCode": "M_Est_tob_curr", "IndicatorName": "Estimate of current tobacco use prevalence (%)", "Language": "EN"}]} | |
| \ No newline at end of file | ||
added
tests/fixtures/who/M_Est_tob_curr.json
+680 −0
@@ -0,0 +1,680 @@ | ||
| 1 | +{ | |
| 2 | +"@odata.context": "https://ghoapi.azureedge.net/api/$metadata#M_Est_tob_curr", | |
| 3 | +"value": [ | |
| 4 | +{ | |
| 5 | +"Id": 1275112, | |
| 6 | +"IndicatorCode": "M_Est_tob_curr", | |
| 7 | +"SpatialDimType": "COUNTRY", | |
| 8 | +"SpatialDim": "USA", | |
| 9 | +"TimeDimType": "YEAR", | |
| 10 | +"ParentLocationCode": "AMR", | |
| 11 | +"ParentLocation": "Americas", | |
| 12 | +"Dim1Type": "SEX", | |
| 13 | +"TimeDim": 2030, | |
| 14 | +"Dim1": "SEX_BTSX", | |
| 15 | +"Dim2Type": null, | |
| 16 | +"Dim2": null, | |
| 17 | +"Dim3Type": null, | |
| 18 | +"Dim3": null, | |
| 19 | +"DataSourceDimType": null, | |
| 20 | +"DataSourceDim": null, | |
| 21 | +"Value": "12.5 [9.1-15.9]", | |
| 22 | +"NumericValue": 12.5, | |
| 23 | +"Low": 9.1, | |
| 24 | +"High": 15.9, | |
| 25 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 26 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 27 | +"TimeDimensionValue": "2030", | |
| 28 | +"TimeDimensionBegin": "2030-01-01T00:00:00+01:00", | |
| 29 | +"TimeDimensionEnd": "2030-12-31T00:00:00+01:00" | |
| 30 | +}, | |
| 31 | +{ | |
| 32 | +"Id": 1768400, | |
| 33 | +"IndicatorCode": "M_Est_tob_curr", | |
| 34 | +"SpatialDimType": "COUNTRY", | |
| 35 | +"SpatialDim": "CAN", | |
| 36 | +"TimeDimType": "YEAR", | |
| 37 | +"ParentLocationCode": "AMR", | |
| 38 | +"ParentLocation": "Americas", | |
| 39 | +"Dim1Type": "SEX", | |
| 40 | +"TimeDim": 2022, | |
| 41 | +"Dim1": "SEX_BTSX", | |
| 42 | +"Dim2Type": null, | |
| 43 | +"Dim2": null, | |
| 44 | +"Dim3Type": null, | |
| 45 | +"Dim3": null, | |
| 46 | +"DataSourceDimType": null, | |
| 47 | +"DataSourceDim": null, | |
| 48 | +"Value": "11.5 [9.4-13.6]", | |
| 49 | +"NumericValue": 11.5, | |
| 50 | +"Low": 9.4, | |
| 51 | +"High": 13.6, | |
| 52 | +"Comments": null, | |
| 53 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 54 | +"TimeDimensionValue": "2022", | |
| 55 | +"TimeDimensionBegin": "2022-01-01T00:00:00+01:00", | |
| 56 | +"TimeDimensionEnd": "2022-12-31T00:00:00+01:00" | |
| 57 | +}, | |
| 58 | +{ | |
| 59 | +"Id": 2130358, | |
| 60 | +"IndicatorCode": "M_Est_tob_curr", | |
| 61 | +"SpatialDimType": "COUNTRY", | |
| 62 | +"SpatialDim": "CAN", | |
| 63 | +"TimeDimType": "YEAR", | |
| 64 | +"ParentLocationCode": "AMR", | |
| 65 | +"ParentLocation": "Americas", | |
| 66 | +"Dim1Type": "SEX", | |
| 67 | +"TimeDim": 2000, | |
| 68 | +"Dim1": "SEX_BTSX", | |
| 69 | +"Dim2Type": null, | |
| 70 | +"Dim2": null, | |
| 71 | +"Dim3Type": null, | |
| 72 | +"Dim3": null, | |
| 73 | +"DataSourceDimType": null, | |
| 74 | +"DataSourceDim": null, | |
| 75 | +"Value": "28.5 [23.9-33.0]", | |
| 76 | +"NumericValue": 28.5, | |
| 77 | +"Low": 23.9, | |
| 78 | +"High": 33.0, | |
| 79 | +"Comments": null, | |
| 80 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 81 | +"TimeDimensionValue": "2000", | |
| 82 | +"TimeDimensionBegin": "2000-01-01T00:00:00+01:00", | |
| 83 | +"TimeDimensionEnd": "2000-12-31T00:00:00+01:00" | |
| 84 | +}, | |
| 85 | +{ | |
| 86 | +"Id": 2428897, | |
| 87 | +"IndicatorCode": "M_Est_tob_curr", | |
| 88 | +"SpatialDimType": "COUNTRY", | |
| 89 | +"SpatialDim": "CAN", | |
| 90 | +"TimeDimType": "YEAR", | |
| 91 | +"ParentLocationCode": "AMR", | |
| 92 | +"ParentLocation": "Americas", | |
| 93 | +"Dim1Type": "SEX", | |
| 94 | +"TimeDim": 2007, | |
| 95 | +"Dim1": "SEX_BTSX", | |
| 96 | +"Dim2Type": null, | |
| 97 | +"Dim2": null, | |
| 98 | +"Dim3Type": null, | |
| 99 | +"Dim3": null, | |
| 100 | +"DataSourceDimType": null, | |
| 101 | +"DataSourceDim": null, | |
| 102 | +"Value": "21.2 [17.8-24.6]", | |
| 103 | +"NumericValue": 21.2, | |
| 104 | +"Low": 17.8, | |
| 105 | +"High": 24.6, | |
| 106 | +"Comments": null, | |
| 107 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 108 | +"TimeDimensionValue": "2007", | |
| 109 | +"TimeDimensionBegin": "2007-01-01T00:00:00+01:00", | |
| 110 | +"TimeDimensionEnd": "2007-12-31T00:00:00+01:00" | |
| 111 | +}, | |
| 112 | +{ | |
| 113 | +"Id": 2656711, | |
| 114 | +"IndicatorCode": "M_Est_tob_curr", | |
| 115 | +"SpatialDimType": "COUNTRY", | |
| 116 | +"SpatialDim": "USA", | |
| 117 | +"TimeDimType": "YEAR", | |
| 118 | +"ParentLocationCode": "AMR", | |
| 119 | +"ParentLocation": "Americas", | |
| 120 | +"Dim1Type": "SEX", | |
| 121 | +"TimeDim": 2007, | |
| 122 | +"Dim1": "SEX_BTSX", | |
| 123 | +"Dim2Type": null, | |
| 124 | +"Dim2": null, | |
| 125 | +"Dim3Type": null, | |
| 126 | +"Dim3": null, | |
| 127 | +"DataSourceDimType": null, | |
| 128 | +"DataSourceDim": null, | |
| 129 | +"Value": "29.7 [21.8-37.6]", | |
| 130 | +"NumericValue": 29.7, | |
| 131 | +"Low": 21.8, | |
| 132 | +"High": 37.6, | |
| 133 | +"Comments": null, | |
| 134 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 135 | +"TimeDimensionValue": "2007", | |
| 136 | +"TimeDimensionBegin": "2007-01-01T00:00:00+01:00", | |
| 137 | +"TimeDimensionEnd": "2007-12-31T00:00:00+01:00" | |
| 138 | +}, | |
| 139 | +{ | |
| 140 | +"Id": 2763447, | |
| 141 | +"IndicatorCode": "M_Est_tob_curr", | |
| 142 | +"SpatialDimType": "COUNTRY", | |
| 143 | +"SpatialDim": "USA", | |
| 144 | +"TimeDimType": "YEAR", | |
| 145 | +"ParentLocationCode": "AMR", | |
| 146 | +"ParentLocation": "Americas", | |
| 147 | +"Dim1Type": "SEX", | |
| 148 | +"TimeDim": 2025, | |
| 149 | +"Dim1": "SEX_BTSX", | |
| 150 | +"Dim2Type": null, | |
| 151 | +"Dim2": null, | |
| 152 | +"Dim3Type": null, | |
| 153 | +"Dim3": null, | |
| 154 | +"DataSourceDimType": null, | |
| 155 | +"DataSourceDim": null, | |
| 156 | +"Value": "15.0 [12.1-18.0]", | |
| 157 | +"NumericValue": 15.0, | |
| 158 | +"Low": 12.1, | |
| 159 | +"High": 18.0, | |
| 160 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 161 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 162 | +"TimeDimensionValue": "2025", | |
| 163 | +"TimeDimensionBegin": "2025-01-01T00:00:00+01:00", | |
| 164 | +"TimeDimensionEnd": "2025-12-31T00:00:00+01:00" | |
| 165 | +}, | |
| 166 | +{ | |
| 167 | +"Id": 3393230, | |
| 168 | +"IndicatorCode": "M_Est_tob_curr", | |
| 169 | +"SpatialDimType": "COUNTRY", | |
| 170 | +"SpatialDim": "CAN", | |
| 171 | +"TimeDimType": "YEAR", | |
| 172 | +"ParentLocationCode": "AMR", | |
| 173 | +"ParentLocation": "Americas", | |
| 174 | +"Dim1Type": "SEX", | |
| 175 | +"TimeDim": 2030, | |
| 176 | +"Dim1": "SEX_BTSX", | |
| 177 | +"Dim2Type": null, | |
| 178 | +"Dim2": null, | |
| 179 | +"Dim3Type": null, | |
| 180 | +"Dim3": null, | |
| 181 | +"DataSourceDimType": null, | |
| 182 | +"DataSourceDim": null, | |
| 183 | +"Value": "8.3 [6.9-9.7]", | |
| 184 | +"NumericValue": 8.3, | |
| 185 | +"Low": 6.9, | |
| 186 | +"High": 9.7, | |
| 187 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 188 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 189 | +"TimeDimensionValue": "2030", | |
| 190 | +"TimeDimensionBegin": "2030-01-01T00:00:00+01:00", | |
| 191 | +"TimeDimensionEnd": "2030-12-31T00:00:00+01:00" | |
| 192 | +}, | |
| 193 | +{ | |
| 194 | +"Id": 3407770, | |
| 195 | +"IndicatorCode": "M_Est_tob_curr", | |
| 196 | +"SpatialDimType": "COUNTRY", | |
| 197 | +"SpatialDim": "USA", | |
| 198 | +"TimeDimType": "YEAR", | |
| 199 | +"ParentLocationCode": "AMR", | |
| 200 | +"ParentLocation": "Americas", | |
| 201 | +"Dim1Type": "SEX", | |
| 202 | +"TimeDim": 2005, | |
| 203 | +"Dim1": "SEX_BTSX", | |
| 204 | +"Dim2Type": null, | |
| 205 | +"Dim2": null, | |
| 206 | +"Dim3Type": null, | |
| 207 | +"Dim3": null, | |
| 208 | +"DataSourceDimType": null, | |
| 209 | +"DataSourceDim": null, | |
| 210 | +"Value": "31.9 [21.7-42.0]", | |
| 211 | +"NumericValue": 31.9, | |
| 212 | +"Low": 21.7, | |
| 213 | +"High": 42.0, | |
| 214 | +"Comments": null, | |
| 215 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 216 | +"TimeDimensionValue": "2005", | |
| 217 | +"TimeDimensionBegin": "2005-01-01T00:00:00+01:00", | |
| 218 | +"TimeDimensionEnd": "2005-12-31T00:00:00+01:00" | |
| 219 | +}, | |
| 220 | +{ | |
| 221 | +"Id": 3680876, | |
| 222 | +"IndicatorCode": "M_Est_tob_curr", | |
| 223 | +"SpatialDimType": "COUNTRY", | |
| 224 | +"SpatialDim": "USA", | |
| 225 | +"TimeDimType": "YEAR", | |
| 226 | +"ParentLocationCode": "AMR", | |
| 227 | +"ParentLocation": "Americas", | |
| 228 | +"Dim1Type": "SEX", | |
| 229 | +"TimeDim": 2010, | |
| 230 | +"Dim1": "SEX_BTSX", | |
| 231 | +"Dim2Type": null, | |
| 232 | +"Dim2": null, | |
| 233 | +"Dim3Type": null, | |
| 234 | +"Dim3": null, | |
| 235 | +"DataSourceDimType": null, | |
| 236 | +"DataSourceDim": null, | |
| 237 | +"Value": "26.5 [19.9-33.0]", | |
| 238 | +"NumericValue": 26.5, | |
| 239 | +"Low": 19.9, | |
| 240 | +"High": 33.0, | |
| 241 | +"Comments": null, | |
| 242 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 243 | +"TimeDimensionValue": "2010", | |
| 244 | +"TimeDimensionBegin": "2010-01-01T00:00:00+01:00", | |
| 245 | +"TimeDimensionEnd": "2010-12-31T00:00:00+01:00" | |
| 246 | +}, | |
| 247 | +{ | |
| 248 | +"Id": 4718535, | |
| 249 | +"IndicatorCode": "M_Est_tob_curr", | |
| 250 | +"SpatialDimType": "COUNTRY", | |
| 251 | +"SpatialDim": "CAN", | |
| 252 | +"TimeDimType": "YEAR", | |
| 253 | +"ParentLocationCode": "AMR", | |
| 254 | +"ParentLocation": "Americas", | |
| 255 | +"Dim1Type": "SEX", | |
| 256 | +"TimeDim": 2015, | |
| 257 | +"Dim1": "SEX_BTSX", | |
| 258 | +"Dim2Type": null, | |
| 259 | +"Dim2": null, | |
| 260 | +"Dim3Type": null, | |
| 261 | +"Dim3": null, | |
| 262 | +"DataSourceDimType": null, | |
| 263 | +"DataSourceDim": null, | |
| 264 | +"Value": "15.3 [13.0-17.7]", | |
| 265 | +"NumericValue": 15.3, | |
| 266 | +"Low": 13.0, | |
| 267 | +"High": 17.7, | |
| 268 | +"Comments": null, | |
| 269 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 270 | +"TimeDimensionValue": "2015", | |
| 271 | +"TimeDimensionBegin": "2015-01-01T00:00:00+01:00", | |
| 272 | +"TimeDimensionEnd": "2015-12-31T00:00:00+01:00" | |
| 273 | +}, | |
| 274 | +{ | |
| 275 | +"Id": 5155027, | |
| 276 | +"IndicatorCode": "M_Est_tob_curr", | |
| 277 | +"SpatialDimType": "COUNTRY", | |
| 278 | +"SpatialDim": "CAN", | |
| 279 | +"TimeDimType": "YEAR", | |
| 280 | +"ParentLocationCode": "AMR", | |
| 281 | +"ParentLocation": "Americas", | |
| 282 | +"Dim1Type": "SEX", | |
| 283 | +"TimeDim": 2005, | |
| 284 | +"Dim1": "SEX_BTSX", | |
| 285 | +"Dim2Type": null, | |
| 286 | +"Dim2": null, | |
| 287 | +"Dim3Type": null, | |
| 288 | +"Dim3": null, | |
| 289 | +"DataSourceDimType": null, | |
| 290 | +"DataSourceDim": null, | |
| 291 | +"Value": "23.1 [19.3-26.8]", | |
| 292 | +"NumericValue": 23.1, | |
| 293 | +"Low": 19.3, | |
| 294 | +"High": 26.8, | |
| 295 | +"Comments": null, | |
| 296 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 297 | +"TimeDimensionValue": "2005", | |
| 298 | +"TimeDimensionBegin": "2005-01-01T00:00:00+01:00", | |
| 299 | +"TimeDimensionEnd": "2005-12-31T00:00:00+01:00" | |
| 300 | +}, | |
| 301 | +{ | |
| 302 | +"Id": 5681201, | |
| 303 | +"IndicatorCode": "M_Est_tob_curr", | |
| 304 | +"SpatialDimType": "COUNTRY", | |
| 305 | +"SpatialDim": "CAN", | |
| 306 | +"TimeDimType": "YEAR", | |
| 307 | +"ParentLocationCode": "AMR", | |
| 308 | +"ParentLocation": "Americas", | |
| 309 | +"Dim1Type": "SEX", | |
| 310 | +"TimeDim": 2025, | |
| 311 | +"Dim1": "SEX_BTSX", | |
| 312 | +"Dim2Type": null, | |
| 313 | +"Dim2": null, | |
| 314 | +"Dim3Type": null, | |
| 315 | +"Dim3": null, | |
| 316 | +"DataSourceDimType": null, | |
| 317 | +"DataSourceDim": null, | |
| 318 | +"Value": "10.1 [8.4-11.9]", | |
| 319 | +"NumericValue": 10.1, | |
| 320 | +"Low": 8.4, | |
| 321 | +"High": 11.9, | |
| 322 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 323 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 324 | +"TimeDimensionValue": "2025", | |
| 325 | +"TimeDimensionBegin": "2025-01-01T00:00:00+01:00", | |
| 326 | +"TimeDimensionEnd": "2025-12-31T00:00:00+01:00" | |
| 327 | +}, | |
| 328 | +{ | |
| 329 | +"Id": 6111069, | |
| 330 | +"IndicatorCode": "M_Est_tob_curr", | |
| 331 | +"SpatialDimType": "COUNTRY", | |
| 332 | +"SpatialDim": "USA", | |
| 333 | +"TimeDimType": "YEAR", | |
| 334 | +"ParentLocationCode": "AMR", | |
| 335 | +"ParentLocation": "Americas", | |
| 336 | +"Dim1Type": "SEX", | |
| 337 | +"TimeDim": 2024, | |
| 338 | +"Dim1": "SEX_BTSX", | |
| 339 | +"Dim2Type": null, | |
| 340 | +"Dim2": null, | |
| 341 | +"Dim3Type": null, | |
| 342 | +"Dim3": null, | |
| 343 | +"DataSourceDimType": null, | |
| 344 | +"DataSourceDim": null, | |
| 345 | +"Value": "15.6 [12.4-18.9]", | |
| 346 | +"NumericValue": 15.6, | |
| 347 | +"Low": 12.4, | |
| 348 | +"High": 18.9, | |
| 349 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 350 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 351 | +"TimeDimensionValue": "2024", | |
| 352 | +"TimeDimensionBegin": "2024-01-01T00:00:00+01:00", | |
| 353 | +"TimeDimensionEnd": "2024-12-31T00:00:00+01:00" | |
| 354 | +}, | |
| 355 | +{ | |
| 356 | +"Id": 6120552, | |
| 357 | +"IndicatorCode": "M_Est_tob_curr", | |
| 358 | +"SpatialDimType": "COUNTRY", | |
| 359 | +"SpatialDim": "CAN", | |
| 360 | +"TimeDimType": "YEAR", | |
| 361 | +"ParentLocationCode": "AMR", | |
| 362 | +"ParentLocation": "Americas", | |
| 363 | +"Dim1Type": "SEX", | |
| 364 | +"TimeDim": 2023, | |
| 365 | +"Dim1": "SEX_BTSX", | |
| 366 | +"Dim2Type": null, | |
| 367 | +"Dim2": null, | |
| 368 | +"Dim3Type": null, | |
| 369 | +"Dim3": null, | |
| 370 | +"DataSourceDimType": null, | |
| 371 | +"DataSourceDim": null, | |
| 372 | +"Value": "11.0 [9.2-12.9]", | |
| 373 | +"NumericValue": 11.0, | |
| 374 | +"Low": 9.2, | |
| 375 | +"High": 12.9, | |
| 376 | +"Comments": null, | |
| 377 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 378 | +"TimeDimensionValue": "2023", | |
| 379 | +"TimeDimensionBegin": "2023-01-01T00:00:00+01:00", | |
| 380 | +"TimeDimensionEnd": "2023-12-31T00:00:00+01:00" | |
| 381 | +}, | |
| 382 | +{ | |
| 383 | +"Id": 7167141, | |
| 384 | +"IndicatorCode": "M_Est_tob_curr", | |
| 385 | +"SpatialDimType": "COUNTRY", | |
| 386 | +"SpatialDim": "USA", | |
| 387 | +"TimeDimType": "YEAR", | |
| 388 | +"ParentLocationCode": "AMR", | |
| 389 | +"ParentLocation": "Americas", | |
| 390 | +"Dim1Type": "SEX", | |
| 391 | +"TimeDim": 2023, | |
| 392 | +"Dim1": "SEX_BTSX", | |
| 393 | +"Dim2Type": null, | |
| 394 | +"Dim2": null, | |
| 395 | +"Dim3Type": null, | |
| 396 | +"Dim3": null, | |
| 397 | +"DataSourceDimType": null, | |
| 398 | +"DataSourceDim": null, | |
| 399 | +"Value": "16.2 [12.9-19.6]", | |
| 400 | +"NumericValue": 16.2, | |
| 401 | +"Low": 12.9, | |
| 402 | +"High": 19.6, | |
| 403 | +"Comments": null, | |
| 404 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 405 | +"TimeDimensionValue": "2023", | |
| 406 | +"TimeDimensionBegin": "2023-01-01T00:00:00+01:00", | |
| 407 | +"TimeDimensionEnd": "2023-12-31T00:00:00+01:00" | |
| 408 | +}, | |
| 409 | +{ | |
| 410 | +"Id": 7694696, | |
| 411 | +"IndicatorCode": "M_Est_tob_curr", | |
| 412 | +"SpatialDimType": "COUNTRY", | |
| 413 | +"SpatialDim": "USA", | |
| 414 | +"TimeDimType": "YEAR", | |
| 415 | +"ParentLocationCode": "AMR", | |
| 416 | +"ParentLocation": "Americas", | |
| 417 | +"Dim1Type": "SEX", | |
| 418 | +"TimeDim": 2015, | |
| 419 | +"Dim1": "SEX_BTSX", | |
| 420 | +"Dim2Type": null, | |
| 421 | +"Dim2": null, | |
| 422 | +"Dim3Type": null, | |
| 423 | +"Dim3": null, | |
| 424 | +"DataSourceDimType": null, | |
| 425 | +"DataSourceDim": null, | |
| 426 | +"Value": "21.9 [17.5-26.4]", | |
| 427 | +"NumericValue": 21.9, | |
| 428 | +"Low": 17.5, | |
| 429 | +"High": 26.4, | |
| 430 | +"Comments": null, | |
| 431 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 432 | +"TimeDimensionValue": "2015", | |
| 433 | +"TimeDimensionBegin": "2015-01-01T00:00:00+01:00", | |
| 434 | +"TimeDimensionEnd": "2015-12-31T00:00:00+01:00" | |
| 435 | +}, | |
| 436 | +{ | |
| 437 | +"Id": 7749854, | |
| 438 | +"IndicatorCode": "M_Est_tob_curr", | |
| 439 | +"SpatialDimType": "COUNTRY", | |
| 440 | +"SpatialDim": "USA", | |
| 441 | +"TimeDimType": "YEAR", | |
| 442 | +"ParentLocationCode": "AMR", | |
| 443 | +"ParentLocation": "Americas", | |
| 444 | +"Dim1Type": "SEX", | |
| 445 | +"TimeDim": 2022, | |
| 446 | +"Dim1": "SEX_BTSX", | |
| 447 | +"Dim2Type": null, | |
| 448 | +"Dim2": null, | |
| 449 | +"Dim3Type": null, | |
| 450 | +"Dim3": null, | |
| 451 | +"DataSourceDimType": null, | |
| 452 | +"DataSourceDim": null, | |
| 453 | +"Value": "16.8 [13.5-20.1]", | |
| 454 | +"NumericValue": 16.8, | |
| 455 | +"Low": 13.5, | |
| 456 | +"High": 20.1, | |
| 457 | +"Comments": null, | |
| 458 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 459 | +"TimeDimensionValue": "2022", | |
| 460 | +"TimeDimensionBegin": "2022-01-01T00:00:00+01:00", | |
| 461 | +"TimeDimensionEnd": "2022-12-31T00:00:00+01:00" | |
| 462 | +}, | |
| 463 | +{ | |
| 464 | +"Id": 7758837, | |
| 465 | +"IndicatorCode": "M_Est_tob_curr", | |
| 466 | +"SpatialDimType": "COUNTRY", | |
| 467 | +"SpatialDim": "USA", | |
| 468 | +"TimeDimType": "YEAR", | |
| 469 | +"ParentLocationCode": "AMR", | |
| 470 | +"ParentLocation": "Americas", | |
| 471 | +"Dim1Type": "SEX", | |
| 472 | +"TimeDim": 2000, | |
| 473 | +"Dim1": "SEX_BTSX", | |
| 474 | +"Dim2Type": null, | |
| 475 | +"Dim2": null, | |
| 476 | +"Dim3Type": null, | |
| 477 | +"Dim3": null, | |
| 478 | +"DataSourceDimType": null, | |
| 479 | +"DataSourceDim": null, | |
| 480 | +"Value": "38.5 [25.2-51.8]", | |
| 481 | +"NumericValue": 38.5, | |
| 482 | +"Low": 25.2, | |
| 483 | +"High": 51.8, | |
| 484 | +"Comments": null, | |
| 485 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 486 | +"TimeDimensionValue": "2000", | |
| 487 | +"TimeDimensionBegin": "2000-01-01T00:00:00+01:00", | |
| 488 | +"TimeDimensionEnd": "2000-12-31T00:00:00+01:00" | |
| 489 | +}, | |
| 490 | +{ | |
| 491 | +"Id": 9100464, | |
| 492 | +"IndicatorCode": "M_Est_tob_curr", | |
| 493 | +"SpatialDimType": "COUNTRY", | |
| 494 | +"SpatialDim": "USA", | |
| 495 | +"TimeDimType": "YEAR", | |
| 496 | +"ParentLocationCode": "AMR", | |
| 497 | +"ParentLocation": "Americas", | |
| 498 | +"Dim1Type": "SEX", | |
| 499 | +"TimeDim": 2020, | |
| 500 | +"Dim1": "SEX_BTSX", | |
| 501 | +"Dim2Type": null, | |
| 502 | +"Dim2": null, | |
| 503 | +"Dim3Type": null, | |
| 504 | +"Dim3": null, | |
| 505 | +"DataSourceDimType": null, | |
| 506 | +"DataSourceDim": null, | |
| 507 | +"Value": "18.2 [14.9-21.6]", | |
| 508 | +"NumericValue": 18.2, | |
| 509 | +"Low": 14.9, | |
| 510 | +"High": 21.6, | |
| 511 | +"Comments": null, | |
| 512 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 513 | +"TimeDimensionValue": "2020", | |
| 514 | +"TimeDimensionBegin": "2020-01-01T00:00:00+01:00", | |
| 515 | +"TimeDimensionEnd": "2020-12-31T00:00:00+01:00" | |
| 516 | +}, | |
| 517 | +{ | |
| 518 | +"Id": 9133475, | |
| 519 | +"IndicatorCode": "M_Est_tob_curr", | |
| 520 | +"SpatialDimType": "COUNTRY", | |
| 521 | +"SpatialDim": "CAN", | |
| 522 | +"TimeDimType": "YEAR", | |
| 523 | +"ParentLocationCode": "AMR", | |
| 524 | +"ParentLocation": "Americas", | |
| 525 | +"Dim1Type": "SEX", | |
| 526 | +"TimeDim": 2020, | |
| 527 | +"Dim1": "SEX_BTSX", | |
| 528 | +"Dim2Type": null, | |
| 529 | +"Dim2": null, | |
| 530 | +"Dim3Type": null, | |
| 531 | +"Dim3": null, | |
| 532 | +"DataSourceDimType": null, | |
| 533 | +"DataSourceDim": null, | |
| 534 | +"Value": "12.4 [10.5-14.4]", | |
| 535 | +"NumericValue": 12.4, | |
| 536 | +"Low": 10.5, | |
| 537 | +"High": 14.4, | |
| 538 | +"Comments": null, | |
| 539 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 540 | +"TimeDimensionValue": "2020", | |
| 541 | +"TimeDimensionBegin": "2020-01-01T00:00:00+01:00", | |
| 542 | +"TimeDimensionEnd": "2020-12-31T00:00:00+01:00" | |
| 543 | +}, | |
| 544 | +{ | |
| 545 | +"Id": 9531067, | |
| 546 | +"IndicatorCode": "M_Est_tob_curr", | |
| 547 | +"SpatialDimType": "COUNTRY", | |
| 548 | +"SpatialDim": "CAN", | |
| 549 | +"TimeDimType": "YEAR", | |
| 550 | +"ParentLocationCode": "AMR", | |
| 551 | +"ParentLocation": "Americas", | |
| 552 | +"Dim1Type": "SEX", | |
| 553 | +"TimeDim": 2024, | |
| 554 | +"Dim1": "SEX_BTSX", | |
| 555 | +"Dim2Type": null, | |
| 556 | +"Dim2": null, | |
| 557 | +"Dim3Type": null, | |
| 558 | +"Dim3": null, | |
| 559 | +"DataSourceDimType": null, | |
| 560 | +"DataSourceDim": null, | |
| 561 | +"Value": "10.6 [8.6-12.5]", | |
| 562 | +"NumericValue": 10.6, | |
| 563 | +"Low": 8.6, | |
| 564 | +"High": 12.5, | |
| 565 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 566 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 567 | +"TimeDimensionValue": "2024", | |
| 568 | +"TimeDimensionBegin": "2024-01-01T00:00:00+01:00", | |
| 569 | +"TimeDimensionEnd": "2024-12-31T00:00:00+01:00" | |
| 570 | +}, | |
| 571 | +{ | |
| 572 | +"Id": 9632763, | |
| 573 | +"IndicatorCode": "M_Est_tob_curr", | |
| 574 | +"SpatialDimType": "COUNTRY", | |
| 575 | +"SpatialDim": "CAN", | |
| 576 | +"TimeDimType": "YEAR", | |
| 577 | +"ParentLocationCode": "AMR", | |
| 578 | +"ParentLocation": "Americas", | |
| 579 | +"Dim1Type": "SEX", | |
| 580 | +"TimeDim": 2010, | |
| 581 | +"Dim1": "SEX_BTSX", | |
| 582 | +"Dim2Type": null, | |
| 583 | +"Dim2": null, | |
| 584 | +"Dim3Type": null, | |
| 585 | +"Dim3": null, | |
| 586 | +"DataSourceDimType": null, | |
| 587 | +"DataSourceDim": null, | |
| 588 | +"Value": "18.7 [16.0-21.5]", | |
| 589 | +"NumericValue": 18.7, | |
| 590 | +"Low": 16.0, | |
| 591 | +"High": 21.5, | |
| 592 | +"Comments": null, | |
| 593 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 594 | +"TimeDimensionValue": "2010", | |
| 595 | +"TimeDimensionBegin": "2010-01-01T00:00:00+01:00", | |
| 596 | +"TimeDimensionEnd": "2010-12-31T00:00:00+01:00" | |
| 597 | +}, | |
| 598 | +{ | |
| 599 | +"Id": 1275112, | |
| 600 | +"IndicatorCode": "M_Est_tob_curr", | |
| 601 | +"SpatialDimType": "REGION", | |
| 602 | +"SpatialDim": "AMR", | |
| 603 | +"TimeDimType": "YEAR", | |
| 604 | +"ParentLocationCode": "AMR", | |
| 605 | +"ParentLocation": "Americas", | |
| 606 | +"Dim1Type": "SEX", | |
| 607 | +"TimeDim": 2020, | |
| 608 | +"Dim1": "SEX_BTSX", | |
| 609 | +"Dim2Type": null, | |
| 610 | +"Dim2": null, | |
| 611 | +"Dim3Type": null, | |
| 612 | +"Dim3": null, | |
| 613 | +"DataSourceDimType": null, | |
| 614 | +"DataSourceDim": null, | |
| 615 | +"Value": "12.5 [9.1-15.9]", | |
| 616 | +"NumericValue": 15.0, | |
| 617 | +"Low": 9.1, | |
| 618 | +"High": 15.9, | |
| 619 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 620 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 621 | +"TimeDimensionValue": "2030", | |
| 622 | +"TimeDimensionBegin": "2030-01-01T00:00:00+01:00", | |
| 623 | +"TimeDimensionEnd": "2030-12-31T00:00:00+01:00" | |
| 624 | +}, | |
| 625 | +{ | |
| 626 | +"Id": 1275112, | |
| 627 | +"IndicatorCode": "M_Est_tob_curr", | |
| 628 | +"SpatialDimType": "COUNTRY", | |
| 629 | +"SpatialDim": "USA", | |
| 630 | +"TimeDimType": "YEAR", | |
| 631 | +"ParentLocationCode": "AMR", | |
| 632 | +"ParentLocation": "Americas", | |
| 633 | +"Dim1Type": "SEX", | |
| 634 | +"TimeDim": 2020, | |
| 635 | +"Dim1": "SEX_MLE", | |
| 636 | +"Dim2Type": null, | |
| 637 | +"Dim2": null, | |
| 638 | +"Dim3Type": null, | |
| 639 | +"Dim3": null, | |
| 640 | +"DataSourceDimType": null, | |
| 641 | +"DataSourceDim": null, | |
| 642 | +"Value": "12.5 [9.1-15.9]", | |
| 643 | +"NumericValue": 99.0, | |
| 644 | +"Low": 9.1, | |
| 645 | +"High": 15.9, | |
| 646 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 647 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 648 | +"TimeDimensionValue": "2030", | |
| 649 | +"TimeDimensionBegin": "2030-01-01T00:00:00+01:00", | |
| 650 | +"TimeDimensionEnd": "2030-12-31T00:00:00+01:00" | |
| 651 | +}, | |
| 652 | +{ | |
| 653 | +"Id": 1275112, | |
| 654 | +"IndicatorCode": "M_Est_tob_curr", | |
| 655 | +"SpatialDimType": "COUNTRY", | |
| 656 | +"SpatialDim": "ZZZ", | |
| 657 | +"TimeDimType": "YEAR", | |
| 658 | +"ParentLocationCode": "AMR", | |
| 659 | +"ParentLocation": "Americas", | |
| 660 | +"Dim1Type": "SEX", | |
| 661 | +"TimeDim": 2020, | |
| 662 | +"Dim1": "SEX_BTSX", | |
| 663 | +"Dim2Type": null, | |
| 664 | +"Dim2": null, | |
| 665 | +"Dim3Type": null, | |
| 666 | +"Dim3": null, | |
| 667 | +"DataSourceDimType": null, | |
| 668 | +"DataSourceDim": null, | |
| 669 | +"Value": "12.5 [9.1-15.9]", | |
| 670 | +"NumericValue": 50.0, | |
| 671 | +"Low": 9.1, | |
| 672 | +"High": 15.9, | |
| 673 | +"Comments": "The most recent national survey was conducted in 2023. This is a projection.", | |
| 674 | +"Date": "2026-01-15T18:01:21.65+01:00", | |
| 675 | +"TimeDimensionValue": "2030", | |
| 676 | +"TimeDimensionBegin": "2030-01-01T00:00:00+01:00", | |
| 677 | +"TimeDimensionEnd": "2030-12-31T00:00:00+01:00" | |
| 678 | +} | |
| 679 | +] | |
| 680 | +} | |
| \ No newline at end of file | ||
added
tests/test_build.py
+142 −0
@@ -0,0 +1,142 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +import json | |
| 4 | +from datetime import UTC, date, datetime | |
| 5 | +from pathlib import Path | |
| 6 | + | |
| 7 | +import duckdb | |
| 8 | +import pytest | |
| 9 | + | |
| 10 | +from countryatlas.config import settings | |
| 11 | +from countryatlas.models import ImportRun, IndicatorSourceSpec, NormalizedObservation | |
| 12 | +from countryatlas.pipeline import build as build_mod | |
| 13 | +from countryatlas.pipeline.staging import rows_to_frame, spec_paths, write_parquet_atomic, write_run | |
| 14 | + | |
| 15 | +COUNTRIES = ["CAN", "USA", "FRA", "DEU", "JPN", "BRA", "IND", "NGA", "AUS", "MEX", "KOR", "ITA", "ESP", "GBR", "CHN", | |
| 16 | + "ZAF", "EGY", "TUR", "ARG", "IDN", "SWE", "NOR", "CHL", "POL"] | |
| 17 | + | |
| 18 | + | |
| 19 | +def _stage(spec: IndicatorSourceSpec, unit: str, base: float, growth: float, years: range, forecast_from: int | None = None) -> None: | |
| 20 | + now = datetime.now(UTC) | |
| 21 | + rows = [] | |
| 22 | + for k, c in enumerate(COUNTRIES): | |
| 23 | + for y in years: | |
| 24 | + v = base * (1 + 0.05 * k) * (growth ** (y - years.start)) | |
| 25 | + rows.append(NormalizedObservation(country_id=c, indicator_id=spec.indicator_id, period=date(y, 1, 1), year=y, | |
| 26 | + frequency="A", value=v, unit=unit, source_id=spec.connector, | |
| 27 | + source_dataset=spec.dataset, source_series_code=spec.code, retrieved_at=now, | |
| 28 | + is_forecast=bool(forecast_from and y >= forecast_from))) | |
| 29 | + p = spec_paths(spec) | |
| 30 | + write_parquet_atomic(rows_to_frame(rows), p["parquet"]) | |
| 31 | + write_run(spec, ImportRun(run_id="T1", connector=spec.connector, dataset=spec.dataset, started_at=now, finished_at=now, | |
| 32 | + status="ok", rows_norm=len(rows), rows_valid=len(rows))) | |
| 33 | + | |
| 34 | + | |
| 35 | +def _tiny_staging() -> None: | |
| 36 | + wb = IndicatorSourceSpec(indicator_id="gdp-per-capita", connector="worldbank", dataset="WDI", code="NY.GDP.PCAP.CD") | |
| 37 | + _stage(wb, "current US$", 10_000, 1.03, range(2000, 2025)) | |
| 38 | + # a lower-priority alternative source for the same indicator → observations_alt | |
| 39 | + imf = IndicatorSourceSpec(indicator_id="gdp-per-capita", connector="imf", dataset="WEO", code="NGDPDPC", priority=2) | |
| 40 | + _stage(imf, "current US$", 10_100, 1.03, range(2000, 2027), forecast_from=2025) | |
| 41 | + pop = IndicatorSourceSpec(indicator_id="population", connector="worldbank", dataset="WDI", code="SP.POP.TOTL") | |
| 42 | + _stage(pop, "people", 5_000_000, 1.01, range(2000, 2025)) | |
| 43 | + le = IndicatorSourceSpec(indicator_id="life-expectancy", connector="worldbank", dataset="WDI", code="SP.DYN.LE00.IN") | |
| 44 | + _stage(le, "years", 70, 1.002, range(2000, 2024)) | |
| 45 | + ma = IndicatorSourceSpec(indicator_id="median-age", connector="owid", dataset="grapher", code="median-age") | |
| 46 | + _stage(ma, "years", 25, 1.005, range(2000, 2024)) | |
| 47 | + fr = IndicatorSourceSpec(indicator_id="fertility-rate", connector="worldbank", dataset="WDI", code="SP.DYN.TFRT.IN") | |
| 48 | + _stage(fr, "births per woman", 3.0, 0.99, range(2000, 2024)) | |
| 49 | + | |
| 50 | + | |
| 51 | +@pytest.mark.usefixtures("data_dir") | |
| 52 | +def test_build_tiny_staging_produces_all_tables() -> None: | |
| 53 | + _tiny_staging() | |
| 54 | + r = build_mod.build(run_id="20260101T000000Z", strict=False) | |
| 55 | + assert settings.db_path.exists() | |
| 56 | + assert r.snapshot_path is not None and r.snapshot_path.exists() | |
| 57 | + con = duckdb.connect(str(settings.db_path), read_only=True) | |
| 58 | + try: | |
| 59 | + tables = {t[0] for t in con.execute("SELECT table_name FROM information_schema.tables").fetchall()} | |
| 60 | + for t in ("countries", "groups", "group_members", "sources", "indicators", "indicator_sources", "observations", | |
| 61 | + "observations_alt", "observation_revisions", "latest", "rankings", "changes", "events", "similarity", | |
| 62 | + "insights", "country_dna", "coverage", "import_runs", "validation_issues", "search_index", "meta"): | |
| 63 | + assert t in tables, t | |
| 64 | + n_obs = con.execute("SELECT count(*) FROM observations").fetchone()[0] | |
| 65 | + n_alt = con.execute("SELECT count(*) FROM observations_alt").fetchone()[0] | |
| 66 | + assert n_obs > 0 and n_alt > 0 # IMF rows lose the priority race where WB has a value | |
| 67 | + # forecasts kept in observations but excluded from latest/rankings | |
| 68 | + assert con.execute("SELECT count(*) FROM observations WHERE is_forecast").fetchone()[0] > 0 | |
| 69 | + assert con.execute("SELECT count(*) FROM latest WHERE is_forecast").fetchone()[0] == 0 | |
| 70 | + assert con.execute("SELECT max(year) FROM latest WHERE indicator_id='gdp-per-capita'").fetchone()[0] == 2024 | |
| 71 | + lat = con.execute("SELECT rank_world, n_world, change_10y_pct FROM latest WHERE country_id='CAN' AND indicator_id='gdp-per-capita'").fetchone() | |
| 72 | + assert lat[0] is not None and lat[1] == len(COUNTRIES) and lat[2] is not None | |
| 73 | + assert con.execute("SELECT count(*) FROM rankings WHERE indicator_id='gdp-per-capita' AND year=2024").fetchone()[0] == len(COUNTRIES) | |
| 74 | + # rank 1 = highest value (higher_is_better true for gdp-per-capita) | |
| 75 | + top = con.execute("SELECT country_id FROM rankings WHERE indicator_id='gdp-per-capita' AND year=2024 AND rank=1").fetchone()[0] | |
| 76 | + assert top == COUNTRIES[-1] | |
| 77 | + assert con.execute("SELECT count(*) FROM coverage").fetchone()[0] == len(con.execute("SELECT * FROM countries").fetchall()) | |
| 78 | + assert con.execute("SELECT count(*) FROM search_index WHERE type='country'").fetchone()[0] >= 200 | |
| 79 | + assert con.execute("SELECT count(*) FROM insights").fetchone()[0] > 0 | |
| 80 | + assert con.execute("SELECT count(*) FROM country_dna").fetchone()[0] > 0 | |
| 81 | + assert con.execute("SELECT count(*) FROM import_runs").fetchone()[0] == 6 | |
| 82 | + meta = dict(con.execute("SELECT key, value FROM meta").fetchall()) | |
| 83 | + assert meta["schema_version"] == "1" and meta["build_run_id"] == "20260101T000000Z" | |
| 84 | + assert int(meta["observation_count"]) == n_obs | |
| 85 | + ind = con.execute("SELECT n_countries, last_year, primary_source_id FROM indicators WHERE id='gdp-per-capita'").fetchone() | |
| 86 | + assert ind == (len(COUNTRIES), 2024, "worldbank") | |
| 87 | + finally: | |
| 88 | + con.close() | |
| 89 | + | |
| 90 | + | |
| 91 | +@pytest.mark.usefixtures("data_dir") | |
| 92 | +def test_failed_build_keeps_live_db_and_revisions_carry_forward(monkeypatch: pytest.MonkeyPatch) -> None: | |
| 93 | + _tiny_staging() | |
| 94 | + first = build_mod.build(run_id="20260101T000000Z", strict=False) | |
| 95 | + before = settings.db_path.stat() | |
| 96 | + # 1) a failure in the derived step must not touch the live DB | |
| 97 | + def boom(con): | |
| 98 | + raise RuntimeError("synthetic failure") | |
| 99 | + | |
| 100 | + monkeypatch.setattr(build_mod.derived, "build_latest", boom) | |
| 101 | + with pytest.raises(RuntimeError, match="synthetic"): | |
| 102 | + build_mod.build(run_id="20260102T000000Z", strict=False) | |
| 103 | + after = settings.db_path.stat() | |
| 104 | + assert (after.st_ino, after.st_mtime_ns, after.st_size) == (before.st_ino, before.st_mtime_ns, before.st_size) | |
| 105 | + assert not list(settings.build_dir.glob("atlas-20260102*")) | |
| 106 | + monkeypatch.undo() | |
| 107 | + # 2) an integrity failure in strict mode also keeps the live DB | |
| 108 | + monkeypatch.setattr(build_mod, "HEADLINE_MIN_COUNTRIES", 10_000) | |
| 109 | + with pytest.raises(build_mod.IntegrityError): | |
| 110 | + build_mod.build(run_id="20260103T000000Z", strict=True) | |
| 111 | + assert settings.db_path.stat().st_ino == before.st_ino | |
| 112 | + monkeypatch.undo() | |
| 113 | + # 3) a changed value is recorded in observation_revisions on the next successful build | |
| 114 | + spec = IndicatorSourceSpec(indicator_id="gdp-per-capita", connector="worldbank", dataset="WDI", code="NY.GDP.PCAP.CD") | |
| 115 | + p = spec_paths(spec)["parquet"] | |
| 116 | + import polars as pl | |
| 117 | + | |
| 118 | + df = pl.read_parquet(p) | |
| 119 | + df = df.with_columns(pl.when((pl.col("country_id") == "CAN") & (pl.col("year") == 2020)).then(pl.col("value") * 1.5) | |
| 120 | + .otherwise(pl.col("value")).alias("value")) | |
| 121 | + write_parquet_atomic(df, p) | |
| 122 | + second = build_mod.build(run_id="20260104T000000Z", strict=False) | |
| 123 | + assert second.counts["revisions_new"] == 1 | |
| 124 | + con = duckdb.connect(str(settings.db_path), read_only=True) | |
| 125 | + rev = con.execute("SELECT country_id, old_value, new_value, run_id FROM observation_revisions").fetchall() | |
| 126 | + con.close() | |
| 127 | + assert rev[0][0] == "CAN" and rev[0][2] == pytest.approx(rev[0][1] * 1.5) and rev[0][3] == "20260104T000000Z" | |
| 128 | + snaps = sorted(settings.snapshots_dir.glob("atlas-*.duckdb")) | |
| 129 | + assert [s.name for s in snaps] == [f"atlas-{first.run_id}.duckdb", f"atlas-{second.run_id}.duckdb"] | |
| 130 | + | |
| 131 | + | |
| 132 | +@pytest.mark.usefixtures("data_dir") | |
| 133 | +def test_export_helpers(tmp_path: Path) -> None: | |
| 134 | + _tiny_staging() | |
| 135 | + build_mod.build(run_id="20260101T000000Z", strict=False) | |
| 136 | + from countryatlas.pipeline.export import export_country, export_indicator | |
| 137 | + | |
| 138 | + p = export_indicator("gdp-per-capita", "json", tmp_path) | |
| 139 | + doc = json.loads(p.read_bytes()) | |
| 140 | + assert doc["meta"]["run_id"] == "20260101T000000Z" and len(doc["rows"]) == len(COUNTRIES) * 27 | |
| 141 | + c = export_country("CAN", "csv", tmp_path) | |
| 142 | + assert c.exists() and c.read_text().count("\n") > 50 | |
added
tests/test_changes.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +from datetime import date | |
| 4 | + | |
| 5 | +import numpy as np | |
| 6 | +import polars as pl | |
| 7 | + | |
| 8 | +from countryatlas.pipeline.changes import Series, compute_changes_and_events, detect_changes, detect_events | |
| 9 | +from countryatlas.registry import indicators_by_id | |
| 10 | + | |
| 11 | + | |
| 12 | +def _series(ind: str, values: list[float], start: int = 2000, country: str = "CAN") -> Series: | |
| 13 | + years = np.arange(start, start + len(values), dtype=np.int32) | |
| 14 | + return Series(country, ind, [date(int(y), 1, 1) for y in years], years, np.array(values, dtype=float)) | |
| 15 | + | |
| 16 | + | |
| 17 | +def test_inflation_drop_headline_and_since() -> None: | |
| 18 | + ind = indicators_by_id()["inflation"] | |
| 19 | + vals = [2.0, 2.1, 1.9, 2.2, 2.0, 2.3, 8.1, 5.9, 5.5, 3.4] # 2015: 5.5 → 2016: 3.4 (−2.1 pts) hmm: last drop 2.1 | |
| 20 | + vals = [2.0, 2.1, 1.9, 2.2, 2.0, 2.3, 2.1, 2.0, 6.8, 3.4] | |
| 21 | + out = detect_changes(_series("inflation", vals, 2016), ind) | |
| 22 | + kinds = {r["kind"]: r for r in out} | |
| 23 | + assert "yoy_drop" in kinds | |
| 24 | + h = kinds["yoy_drop"]["headline"] | |
| 25 | + assert h.startswith("Inflation fell 3.4 points to 3.4 % in 2025") | |
| 26 | + assert "largest drop" in h | |
| 27 | + assert 0 < kinds["yoy_drop"]["severity"] <= 1 | |
| 28 | + | |
| 29 | + | |
| 30 | +def test_floor_blocks_small_moves() -> None: | |
| 31 | + ind = indicators_by_id()["inflation"] # change_floor 2 points | |
| 32 | + vals = [2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0] # +1 pt: huge z but below the floor | |
| 33 | + out = detect_changes(_series("inflation", vals), ind) | |
| 34 | + assert not any(r["kind"] in ("yoy_jump", "yoy_drop") for r in out) | |
| 35 | + | |
| 36 | + | |
| 37 | +def test_record_high_and_relative_headline() -> None: | |
| 38 | + ind = indicators_by_id()["gdp-per-capita"] | |
| 39 | + vals = [1000 * 1.03**k for k in range(15)] | |
| 40 | + vals[-1] = vals[-2] * 1.25 | |
| 41 | + out = detect_changes(_series("gdp-per-capita", vals), ind) | |
| 42 | + kinds = {r["kind"] for r in out} | |
| 43 | + assert "record_high" in kinds and "yoy_jump" in kinds | |
| 44 | + jump = next(r for r in out if r["kind"] == "yoy_jump") | |
| 45 | + assert "25.0 %" in jump["headline"] and "US$" in jump["headline"] | |
| 46 | + | |
| 47 | + | |
| 48 | +def test_sign_flip_and_events_history() -> None: | |
| 49 | + ind = indicators_by_id()["gdp-growth"] | |
| 50 | + vals = [3.0, 2.5, 3.1, 2.8, 3.0, 2.9, -4.5, 5.0, 2.0, -1.0] | |
| 51 | + ch = detect_changes(_series("gdp-growth", vals), ind) | |
| 52 | + assert any(r["kind"] == "sign_flip" for r in ch) | |
| 53 | + ev = detect_events(_series("gdp-growth", vals), ind) | |
| 54 | + flips = [r for r in ev if r["kind"] == "sign_flip"] | |
| 55 | + assert len(flips) == 3 # 2006, 2007, 2009 | |
| 56 | + assert any(r["kind"] == "yoy_drop" and r["year"] == 2006 for r in ev) | |
| 57 | + | |
| 58 | + | |
| 59 | +def test_driver_over_frame() -> None: | |
| 60 | + inds = indicators_by_id() | |
| 61 | + rows = [] | |
| 62 | + for c in ("CAN", "FRA"): | |
| 63 | + for k, v in enumerate([2.0, 2.1, 1.9, 2.2, 2.0, 2.3, 2.1, 2.0, 6.8, 3.4]): | |
| 64 | + rows.append({"country_id": c, "indicator_id": "inflation", "period": date(2016 + k, 1, 1), "year": 2016 + k, "value": v}) | |
| 65 | + df = pl.DataFrame(rows) | |
| 66 | + ch, ev = compute_changes_and_events(df, inds) | |
| 67 | + assert ch.filter(pl.col("kind") == "yoy_drop").height == 2 | |
| 68 | + assert set(ch.columns) >= {"id", "country_id", "indicator_id", "kind", "headline", "severity", "detected_at"} | |
| 69 | + assert ev.height >= 2 | |
added
tests/test_owid.py
+48 −0
@@ -0,0 +1,48 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +from datetime import UTC, datetime | |
| 4 | +from pathlib import Path | |
| 5 | + | |
| 6 | +from countryatlas.connectors.owid import OWIDConnector | |
| 7 | +from countryatlas.models import IndicatorSourceSpec, RawPayload | |
| 8 | + | |
| 9 | + | |
| 10 | +def _conn() -> OWIDConnector: | |
| 11 | + c = OWIDConnector.__new__(OWIDConnector) | |
| 12 | + c._cache, c._frames = {}, {} | |
| 13 | + import threading | |
| 14 | + | |
| 15 | + c._lock = threading.Lock() | |
| 16 | + return c | |
| 17 | + | |
| 18 | + | |
| 19 | +def _raw(body: bytes, dataset: str, code: str) -> RawPayload: | |
| 20 | + return RawPayload(connector="owid", dataset=dataset, code=code, url="http://test", retrieved_at=datetime.now(UTC), | |
| 21 | + status_code=200, content_type="text/csv", body=body) | |
| 22 | + | |
| 23 | + | |
| 24 | +def test_co2_subset_normalizes_per_column(fixtures: Path) -> None: | |
| 25 | + body = (fixtures / "owid_co2_subset.csv").read_bytes() | |
| 26 | + raw = _raw(body, "co2", "owid-co2-data") | |
| 27 | + conn = _conn() | |
| 28 | + rows = conn.normalize(raw, IndicatorSourceSpec(indicator_id="co2-per-capita", connector="owid", dataset="co2", code="co2_per_capita")) | |
| 29 | + isos = {r.country_id for r in rows} | |
| 30 | + assert isos == {"CAN", "FRA", "NGA", "KWT"} # World (OWID_WRL) and Asia (no code) dropped | |
| 31 | + assert all(r.unit == "tonnes per person" for r in rows) | |
| 32 | + assert all(r.frequency == "A" and r.period.month == 1 for r in rows) | |
| 33 | + # second spec on the same payload reuses the parsed frame | |
| 34 | + rows2 = conn.normalize(raw, IndicatorSourceSpec(indicator_id="co2-emissions", connector="owid", dataset="co2", code="co2")) | |
| 35 | + assert len(rows2) >= len(rows) - 5 | |
| 36 | + assert len(conn._frames) == 1 | |
| 37 | + | |
| 38 | + | |
| 39 | +def test_grapher_projection_column_becomes_forecast(fixtures: Path) -> None: | |
| 40 | + body = (fixtures / "owid_grapher_median-age.csv").read_bytes() | |
| 41 | + conn = _conn() | |
| 42 | + rows = conn.normalize(_raw(body, "grapher", "median-age"), | |
| 43 | + IndicatorSourceSpec(indicator_id="median-age", connector="owid", dataset="grapher", code="median-age")) | |
| 44 | + can = {r.year: r for r in rows if r.country_id == "CAN"} | |
| 45 | + assert can[2020].is_forecast is False | |
| 46 | + assert any(r.is_forecast for r in can.values()) | |
| 47 | + assert max(can) <= datetime.now(UTC).year + 6 | |
| 48 | + assert "WORLD" not in {r.country_id for r in rows} and "OWID_WRL" not in {r.country_id for r in rows} | |
added
tests/test_registry.py
+51 −0
@@ -0,0 +1,51 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +from countryatlas import registry | |
| 4 | + | |
| 5 | + | |
| 6 | +def test_registry_loads() -> None: | |
| 7 | + cs = registry.countries() | |
| 8 | + assert len(cs) >= 200 | |
| 9 | + assert all(c.iso3 == c.id for c in cs) | |
| 10 | + slugs = {c.slug for c in cs} | |
| 11 | + assert len(slugs) == len(cs) | |
| 12 | + inds = registry.indicators() | |
| 13 | + assert len(inds) >= 200 | |
| 14 | + assert all(i.topic in registry.TOPICS for i in inds) | |
| 15 | + gs = registry.groups_by_id() | |
| 16 | + assert "world" in gs and len(gs["world"].members) == len(cs) | |
| 17 | + assert "oecd" in gs and "CAN" in gs["oecd"].members | |
| 18 | + | |
| 19 | + | |
| 20 | +def test_lookup_maps_codes() -> None: | |
| 21 | + lk = registry.lookup() | |
| 22 | + assert lk.from_iso3("CAN") == "CAN" | |
| 23 | + assert lk.from_iso3("UNK") == "XKX" | |
| 24 | + assert lk.from_iso3("WLD") is None | |
| 25 | + assert lk.from_iso3("") is None | |
| 26 | + assert lk.from_iso2("GB") == "GBR" | |
| 27 | + assert lk.from_name("Viet Nam") == "VNM" | |
| 28 | + | |
| 29 | + | |
| 30 | +def test_source_specs_filter() -> None: | |
| 31 | + wb = registry.source_specs(connector="worldbank") | |
| 32 | + assert wb and all(s.connector == "worldbank" for s in wb) | |
| 33 | + one = registry.source_specs(indicator="gdp-per-capita") | |
| 34 | + assert {s.indicator_id for s in one} == {"gdp-per-capita"} | |
| 35 | + assert one[0].priority <= one[-1].priority | |
| 36 | + | |
| 37 | + | |
| 38 | +def test_similarity_and_insights_registries_reference_known_indicators() -> None: | |
| 39 | + from countryatlas.pipeline.insights import insight_templates | |
| 40 | + from countryatlas.pipeline.similarity import similarity_config | |
| 41 | + | |
| 42 | + known = set(registry.indicators_by_id()) | |
| 43 | + cfg = similarity_config() | |
| 44 | + assert set(cfg["modes"]) == {"overall", "economic", "demographic", "energy", "social"} | |
| 45 | + for spec in cfg["modes"].values(): | |
| 46 | + for f in spec["features"]: | |
| 47 | + assert f["indicator"] in known, f | |
| 48 | + assert len(cfg["dna"]) == 9 | |
| 49 | + templates = insight_templates() | |
| 50 | + assert len(templates) >= 12 | |
| 51 | + assert all(t["indicator"] in known for t in templates) | |
added
tests/test_validate.py
+55 −0
@@ -0,0 +1,55 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +from datetime import UTC, date, datetime | |
| 4 | + | |
| 5 | +import polars as pl | |
| 6 | + | |
| 7 | +from countryatlas.models import NormalizedObservation | |
| 8 | +from countryatlas.pipeline.staging import rows_to_frame | |
| 9 | +from countryatlas.pipeline.validate import validate_frame | |
| 10 | +from countryatlas.registry import indicators_by_id | |
| 11 | + | |
| 12 | + | |
| 13 | +def _rows(ind: str, unit: str, series: dict[str, list[float]], start: int = 2000) -> pl.DataFrame: | |
| 14 | + now = datetime.now(UTC) | |
| 15 | + out = [] | |
| 16 | + for c, vals in series.items(): | |
| 17 | + for k, v in enumerate(vals): | |
| 18 | + out.append(NormalizedObservation(country_id=c, indicator_id=ind, period=date(start + k, 1, 1), year=start + k, | |
| 19 | + frequency="A", value=v, unit=unit, source_id="worldbank", source_dataset="WDI", | |
| 20 | + source_series_code="X", retrieved_at=now)) | |
| 21 | + return rows_to_frame(out) | |
| 22 | + | |
| 23 | + | |
| 24 | +def test_bounds_quarantine_and_jump_warning() -> None: | |
| 25 | + ind = indicators_by_id()["life-expectancy"] # bounds roughly [20, 100] | |
| 26 | + df = _rows("life-expectancy", ind.unit, {"CAN": [78, 78.5, 79, 79.4, 79.9, 80.2, 80.6, 81.0, 81.3, 250.0], | |
| 27 | + "FRA": [78, 78.4, 78.9, 79.3, 79.7, 80.1, 80.5, 60.0, 81.2, 81.5]}, 2010) | |
| 28 | + gv = validate_frame(df, ind, now=datetime(2021, 6, 1, tzinfo=UTC)) | |
| 29 | + st = dict(zip(gv.frame["country_id"].to_list(), gv.frame["status"].to_list(), strict=True)) # last per country wins | |
| 30 | + assert gv.frame.filter((pl.col("country_id") == "CAN") & (pl.col("year") == 2019))["status"][0] == "quarantined" | |
| 31 | + assert gv.frame.filter((pl.col("country_id") == "FRA") & (pl.col("year") == 2017))["status"][0] == "warning" | |
| 32 | + assert gv.n_quarantined == 1 and gv.n_warning >= 1 | |
| 33 | + assert not gv.quarantine_dataset | |
| 34 | + assert {i.code for i in gv.issues} >= {"out_of_bounds", "extreme_jump"} | |
| 35 | + assert st # sanity | |
| 36 | + | |
| 37 | + | |
| 38 | +def test_unit_mismatch_and_partial_download_quarantine_dataset() -> None: | |
| 39 | + ind = indicators_by_id()["gdp-per-capita"] | |
| 40 | + df = _rows("gdp-per-capita", "wrong unit", {"CAN": [1.0, 2.0, 3.0]}) | |
| 41 | + gv = validate_frame(df, ind) | |
| 42 | + assert gv.quarantine_dataset and any(i.code == "unit_mismatch" for i in gv.issues) | |
| 43 | + df2 = _rows("gdp-per-capita", ind.unit, {"CAN": [1.0, 2.0, 3.0]}) | |
| 44 | + gv2 = validate_frame(df2, ind, previous_rows=100) | |
| 45 | + assert gv2.quarantine_dataset and any(i.code == "partial_download" for i in gv2.issues) | |
| 46 | + gv3 = validate_frame(df2, ind, previous_rows=5) | |
| 47 | + assert not gv3.quarantine_dataset | |
| 48 | + | |
| 49 | + | |
| 50 | +def test_stale_flag_on_latest_row_only() -> None: | |
| 51 | + ind = indicators_by_id()["gdp-per-capita"] | |
| 52 | + df = _rows("gdp-per-capita", ind.unit, {"CAN": [100.0, 101.0, 102.0, 103.0, 104.0]}, 2010) # latest 2014 | |
| 53 | + gv = validate_frame(df, ind, now=datetime(2026, 9, 1, tzinfo=UTC)) | |
| 54 | + statuses = gv.frame.sort("period")["status"].to_list() | |
| 55 | + assert statuses[-1] == "stale" and statuses[:-1] == ["imported"] * 4 | |
added
tests/test_worldbank.py
+60 −0
@@ -0,0 +1,60 @@ | ||
| 1 | +from __future__ import annotations | |
| 2 | + | |
| 3 | +import json | |
| 4 | +from datetime import UTC, datetime | |
| 5 | +from pathlib import Path | |
| 6 | + | |
| 7 | +import pytest | |
| 8 | + | |
| 9 | +from countryatlas.connectors._util import ConnectorError | |
| 10 | +from countryatlas.connectors.worldbank import WorldBankConnector, _check_header | |
| 11 | +from countryatlas.models import IndicatorSourceSpec, RawPayload | |
| 12 | + | |
| 13 | + | |
| 14 | +def _raw(body: bytes, code: str = "NY.GDP.PCAP.CD") -> RawPayload: | |
| 15 | + return RawPayload(connector="worldbank", dataset="WDI", code=code, url="http://test", retrieved_at=datetime.now(UTC), | |
| 16 | + status_code=200, content_type="application/json", body=body, | |
| 17 | + source_updated_at=datetime(2026, 7, 13, tzinfo=UTC), meta={"lastupdated": "2026-07-13", "page": 1}) | |
| 18 | + | |
| 19 | + | |
| 20 | +def test_normalize_fixture_drops_aggregates_and_nulls(fixtures: Path) -> None: | |
| 21 | + body = (fixtures / "wb_NY.GDP.PCAP.CD.json").read_bytes() | |
| 22 | + doc = json.loads(body) | |
| 23 | + n_country_rows = sum(1 for r in doc[1] if r["countryiso3code"] in {"CAN", "USA", "FRA", "DEU", "JPN", "NGA", "BRA", "IND"} | |
| 24 | + and r["value"] is not None) | |
| 25 | + spec = IndicatorSourceSpec(indicator_id="gdp-per-capita", connector="worldbank", dataset="WDI", code="NY.GDP.PCAP.CD") | |
| 26 | + conn = WorldBankConnector.__new__(WorldBankConnector) # no HTTP client needed for normalize | |
| 27 | + rows = conn.normalize([_raw(body)], spec) | |
| 28 | + assert len(rows) == n_country_rows | |
| 29 | + assert {r.country_id for r in rows} <= {"CAN", "USA", "FRA", "DEU", "JPN", "NGA", "BRA", "IND"} | |
| 30 | + can = sorted((r for r in rows if r.country_id == "CAN"), key=lambda r: r.period) | |
| 31 | + assert can[-1].year >= 2023 and can[-1].period.month == 1 and can[-1].frequency == "A" | |
| 32 | + assert can[-1].unit == "current US$" | |
| 33 | + assert can[-1].source_updated_at == datetime(2026, 7, 13, tzinfo=UTC) | |
| 34 | + assert not any(r.is_forecast for r in rows) | |
| 35 | + report = WorldBankConnector.validate(conn, rows) | |
| 36 | + assert report.errors == 0 and not report.quarantine_dataset | |
| 37 | + | |
| 38 | + | |
| 39 | +def test_transform_applied() -> None: | |
| 40 | + body = json.dumps([{"page": 1, "pages": 1, "total": 1, "lastupdated": "2026-07-13"}, | |
| 41 | + [{"countryiso3code": "CAN", "country": {"id": "CA"}, "date": "2020", "value": 2.0, "obs_status": "E"}]]).encode() | |
| 42 | + spec = IndicatorSourceSpec(indicator_id="gdp", connector="worldbank", dataset="WDI", code="X", transform="x*1e9") | |
| 43 | + conn = WorldBankConnector.__new__(WorldBankConnector) | |
| 44 | + rows = conn.normalize(_raw(body, "X"), spec) | |
| 45 | + assert rows[0].value == 2e9 and rows[0].is_estimate is True | |
| 46 | + | |
| 47 | + | |
| 48 | +def test_retired_code_is_an_error(fixtures: Path) -> None: | |
| 49 | + doc = json.loads((fixtures / "wb_retired.json").read_text()) | |
| 50 | + with pytest.raises(ConnectorError, match="retired or invalid"): | |
| 51 | + _check_header(doc, "XX.OLD.CODE") | |
| 52 | + | |
| 53 | + | |
| 54 | +def test_period_parsing() -> None: | |
| 55 | + from countryatlas.connectors._util import parse_period | |
| 56 | + | |
| 57 | + assert parse_period("2023")[1:] == (2023, "A") | |
| 58 | + assert parse_period("2023Q3")[0].month == 7 | |
| 59 | + assert parse_period("2023M11")[0].month == 11 | |
| 60 | + assert parse_period("garbage") is None | |
| 61 | ||