SPB Git forge

spb/countryatlas

Public
20commits 1branches 0releases
268.3 MBsize
maindefault branch
12 days agolast push
TypeScript 57% Python 38.6% JavaScript 3.6% CSS 0.6%

API: weak ETag + public Cache-Control per snapshot (304 on If-None-Match); page cache headers for indicators/rankings/regions/compare/stories/sources; final integration sweep script

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 12 days ago (Sep 12, 2026) parent 9247240

4 changed files +201 −1

modified apps/web/next.config.ts +6 −0
@@ -41,6 +41,12 @@ const nextConfig: NextConfig = {
41 41 ],
42 42 },
43 43 { source: '/countries/:path*', headers: [PUBLIC_CACHE] },
44 + { source: '/indicators/:path*', headers: [PUBLIC_CACHE] },
45 + { source: '/rankings/:path*', headers: [PUBLIC_CACHE] },
46 + { source: '/regions/:path*', headers: [PUBLIC_CACHE] },
47 + { source: '/compare/:path*', headers: [PUBLIC_CACHE] },
48 + { source: '/stories/:path*', headers: [PUBLIC_CACHE] },
49 + { source: '/sources/:path*', headers: [PUBLIC_CACHE] },
44 50 { source: '/api/:path*', headers: [NO_STORE] },
45 51 { source: '/admin/:path*', headers: [NO_STORE] },
46 52 ];
added apps/web/qa/final-sweep.mjs +176 −0
@@ -0,0 +1,176 @@
1 +/**
2 + * Final integration sweep (production build + real API): every public route × 320/375/390/430/768/1440/1920,
3 + * light (plus dark at 390 and 1440). Asserts: HTTP 200, no horizontal overflow, no console errors, no failed
4 + * API requests, no "undefined/NaN/null" text, footer credits present, tap targets ≥ 44 px on phones (excluding
5 + * sr-only/inputs inside 44 px labels), and every internal link on the page resolves (HEAD/GET once per URL).
6 + *
7 + * node qa/final-sweep.mjs [BASE_URL] [--quick]
8 + */
9 +import { chromium } from '/Users/simon-pierreboucher/Desktop/uqo-eval/node_modules/playwright/index.mjs';
10 +import { mkdirSync, writeFileSync } from 'node:fs';
11 +
12 +const args = process.argv.slice(2);
13 +const quick = args.includes('--quick');
14 +const BASE = args.find((a) => a.startsWith('http')) ?? 'http://localhost:8290';
15 +const OUT = new URL('./screens/final/', import.meta.url).pathname;
16 +mkdirSync(OUT, { recursive: true });
17 +
18 +const ROUTES = [
19 + '/', '/explore', '/explore?indicator=life-expectancy&year=1990', '/trajectories', '/scatter', '/finder?f=gdp-per-capita:gt:40000&f=population:gt:10000000', '/extremes',
20 + '/peers', '/stories', '/stories/the-world-is-getting-older', '/download', '/updates', '/api', '/countries', '/countries/canada', '/countries/canada/economy',
21 + '/countries/nigeria', '/compare', '/compare/canada/australia', '/compare/canada/united-states/france?tab=economy', '/rankings', '/rankings/gdp-per-capita',
22 + '/rankings/life-expectancy?group=oecd&view=map', '/indicators', '/indicators/life-expectancy', '/indicators/inflation', '/regions', '/regions/g7', '/regions/compare',
23 + '/changes', '/sources', '/sources/worldbank', '/methodology',
24 +];
25 +const WIDTHS = quick ? [390, 1440] : [320, 375, 390, 430, 768, 1440, 1920];
26 +const slug = (p) => (p === '/' ? 'home' : p.slice(1).replace(/[/?=&:]+/g, '_'));
27 +const report = [];
28 +const linkStatus = new Map();
29 +const browser = await chromium.launch();
30 +
31 +async function checkLinks(page, hrefs) {
32 + const bad = [];
33 + for (const h of hrefs) {
34 + if (linkStatus.has(h)) {
35 + if (linkStatus.get(h) >= 400) bad.push(`${h} → ${linkStatus.get(h)}`);
36 + continue;
37 + }
38 + try {
39 + const r = await page.request.get(BASE + h, { maxRedirects: 3, timeout: 60_000 });
40 + linkStatus.set(h, r.status());
41 + if (r.status() >= 400) bad.push(`${h} → ${r.status()}`);
42 + } catch (e) {
43 + linkStatus.set(h, 599);
44 + bad.push(`${h} → ERR ${String(e).slice(0, 60)}`);
45 + }
46 + }
47 + return bad;
48 +}
49 +
50 +for (const width of WIDTHS) {
51 + const themes = width === 390 || width === 1440 ? ['light', 'dark'] : ['light'];
52 + for (const theme of themes) {
53 + const mobile = width < 768;
54 + const ctx = await browser.newContext({ viewport: { width, height: mobile ? 844 : 900 }, deviceScaleFactor: 1, isMobile: mobile, hasTouch: mobile, colorScheme: theme });
55 + const page = await ctx.newPage();
56 + const errors = [];
57 + const failed = [];
58 + page.on('pageerror', (e) => errors.push(`pageerror: ${String(e).slice(0, 200)}`));
59 + page.on('console', (m) => {
60 + if (m.type() === 'error') errors.push(m.text().slice(0, 200));
61 + });
62 + page.on('response', (r) => {
63 + if (r.url().includes('/api/v1/') && r.status() >= 400) failed.push(`${r.status()} ${r.url().slice(BASE.length, BASE.length + 100)}`);
64 + });
65 + for (const path of ROUTES) {
66 + let status = 0;
67 + try {
68 + const resp = await page.goto(BASE + path, { waitUntil: 'networkidle', timeout: 90_000 });
69 + status = resp?.status() ?? 0;
70 + } catch (e) {
71 + report.push({ path, width, theme, status: 'ERR', error: String(e).slice(0, 200) });
72 + continue;
73 + }
74 + await page.evaluate(() => document.fonts.ready);
75 + await page.waitForTimeout(500);
76 + const full = await page.evaluate(() => document.querySelector('main')?.getAttribute('data-layout') === 'full');
77 + if (!full) {
78 + await page.evaluate(async () => {
79 + const h = document.documentElement.scrollHeight;
80 + for (let y = 0; y < h; y += 800) {
81 + window.scrollTo(0, y);
82 + await new Promise((r) => setTimeout(r, 40));
83 + }
84 + window.scrollTo(0, 0);
85 + });
86 + await page.waitForLoadState('networkidle').catch(() => {});
87 + await page.waitForTimeout(400);
88 + }
89 + const m = await page.evaluate(
90 + ({ mobile }) => {
91 + const de = document.documentElement;
92 + const overflow = de.scrollWidth - de.clientWidth;
93 + const wide = [...document.querySelectorAll('body *')]
94 + .filter((el) => {
95 + const r = el.getBoundingClientRect();
96 + if (!(r.right > de.clientWidth + 1 && r.width > 0)) return false;
97 + // inside an intentional horizontal scroller?
98 + let p = el.parentElement;
99 + while (p) {
100 + const cs = getComputedStyle(p);
101 + if (/(auto|scroll)/.test(cs.overflowX)) return false;
102 + p = p.parentElement;
103 + }
104 + return true;
105 + })
106 + .slice(0, 5)
107 + .map((el) => `${el.tagName.toLowerCase()}.${String(el.className).split(' ').slice(0, 3).join('.')} right=${Math.round(el.getBoundingClientRect().right)}`);
108 + const vis = (el) => {
109 + const r = el.getBoundingClientRect();
110 + if (r.width === 0 || r.height === 0) return false;
111 + const cs = getComputedStyle(el);
112 + return cs.visibility !== 'hidden' && cs.display !== 'none';
113 + };
114 + const small = mobile
115 + ? [...document.querySelectorAll('a,button,[role=button],select,summary,[role=radio],[role=tab]')]
116 + .filter(vis)
117 + .filter((el) => {
118 + const r = el.getBoundingClientRect();
119 + if (!(r.height < 40 && r.width < 40)) return false;
120 + if (el.closest('svg')) return false;
121 + return true;
122 + })
123 + .map((el) => `${el.tagName.toLowerCase()} "${(el.getAttribute('aria-label') || el.textContent || '').trim().slice(0, 30)}" ${Math.round(el.getBoundingClientRect().width)}x${Math.round(el.getBoundingClientRect().height)}`)
124 + : [];
125 + const text = document.body.innerText || '';
126 + const bad = [];
127 + for (const re of [/\bundefined\b/g, /\bNaN\b/g, /(?<![\w"':/])null(?![\w"':])/g, /\[object Object\]/g]) {
128 + let mm;
129 + let n = 0;
130 + while ((mm = re.exec(text)) && n < 3) {
131 + bad.push(`${mm[0]} @ "${text.slice(Math.max(0, mm.index - 40), mm.index + 30).replace(/\s+/g, ' ')}"`);
132 + n++;
133 + }
134 + }
135 + const footer = document.querySelector('footer');
136 + const ft = footer ? footer.innerText : '';
137 + const credits = /Simon-Pierre Boucher/.test(ft) && /contact@spboucher\.ai/.test(ft) && /MacLustr/.test(ft) && !!footer?.querySelector('a[href="https://www.maclustr.io"]');
138 + const hrefs = [...new Set([...document.querySelectorAll('a[href^="/"]')].map((a) => a.getAttribute('href')).filter((h) => h && !h.startsWith('/api/v1/') && !h.startsWith('//')))];
139 + const keys = new Set();
140 + return { overflow, wide, small: small.slice(0, 8), nSmall: small.length, bad, credits, hrefs: hrefs.slice(0, 80), title: document.title, docH: de.scrollHeight, full: !!document.querySelector('main[data-layout="full"]') };
141 + },
142 + { mobile },
143 + );
144 + const badLinks = theme === 'light' && (width === 1440 || width === 390) ? await checkLinks(page, m.hrefs) : [];
145 + const file = `${slug(path)}-${width}-${theme}.png`;
146 + await page.screenshot({ path: OUT + file, fullPage: !m.full }).catch(() => {});
147 + report.push({ path, width, theme, status, ...m, hrefs: undefined, badLinks, errors: errors.splice(0), failed: failed.splice(0), file });
148 + writeFileSync(OUT + 'report.json', JSON.stringify(report, null, 2));
149 + }
150 + await ctx.close();
151 + }
152 +}
153 +await browser.close();
154 +
155 +let fails = 0;
156 +for (const r of report) {
157 + const flags = [];
158 + if (r.status !== 200) flags.push(`HTTP ${r.status}`);
159 + if (r.overflow > 0) flags.push(`OVERFLOW +${r.overflow}px`);
160 + if (r.wide?.length) flags.push(`${r.wide.length} wide`);
161 + if (r.nSmall) flags.push(`${r.nSmall} small`);
162 + if (r.bad?.length) flags.push(`BAD TEXT ${r.bad.length}`);
163 + if (r.errors?.length) flags.push(`${r.errors.length} console`);
164 + if (r.failed?.length) flags.push(`${r.failed.length} failed API`);
165 + if (r.credits === false) flags.push('NO CREDITS');
166 + if (r.badLinks?.length) flags.push(`${r.badLinks.length} broken links`);
167 + if (flags.length) fails++;
168 + console.log(`${String(r.width).padStart(4)} ${(r.theme || '').padEnd(5)} ${r.path.padEnd(60)} ${flags.join(' · ') || 'ok'}`);
169 + if (r.wide?.length) console.log(' wide:', r.wide.join(' | '));
170 + if (r.small?.length) console.log(' small:', r.small.slice(0, 4).join(' | '));
171 + if (r.bad?.length) console.log(' bad:', r.bad.join(' | '));
172 + if (r.errors?.length) console.log(' errors:', r.errors.slice(0, 3).join(' | '));
173 + if (r.failed?.length) console.log(' failed:', r.failed.slice(0, 3).join(' | '));
174 + if (r.badLinks?.length) console.log(' links:', r.badLinks.slice(0, 5).join(' | '));
175 +}
176 +console.log(`\n${report.length} renders, ${fails} with flags, ${linkStatus.size} internal links checked`);
modified src/countryatlas/api/main.py +18 −0
@@ -5,6 +5,7 @@ Run: `ca-api` (uvicorn, 1 worker, proxy headers) or
5 5 """
6 6 from __future__ import annotations
7 7
8 +import hashlib
8 9 import logging
9 10 import os
10 11 import time
@@ -48,6 +49,8 @@ log = logging.getLogger("countryatlas.api")
48 49
49 50 API_PREFIX = "/api/v1"
50 51 PROBLEM = "application/problem+json"
52 +# Public GET responses: 5 min fresh, an hour stale-while-revalidate; every ETag changes with the snapshot run id.
53 +PUBLIC_CACHE = "public, max-age=300, stale-while-revalidate=3600"
51 54
52 55
53 56 class ORJSONResponse(JSONResponse):
@@ -115,12 +118,24 @@ def create_app(db_path: str | Path | None = None, *, rate_limit_per_minute: int
115 118 run_id = snap.run_id if snap else None
116 119 cacheable = (public and request.method == "GET" and snap is not None and "/download." not in path)
117 120 key = cache.key(run_id or "", path, str(request.url.query)) if cacheable else None
121 + # Weak ETag = snapshot run id + request; a new snapshot changes every ETag, so clients can revalidate cheaply.
122 + etag = f'W/"{run_id}-{hashlib.sha1(f"{path}?{request.url.query}".encode()).hexdigest()[:12]}"' if cacheable else None
123 + if etag is not None and etag in (request.headers.get("if-none-match") or ""):
124 + resp = Response(status_code=304)
125 + resp.headers["ETag"] = etag
126 + resp.headers["Cache-Control"] = PUBLIC_CACHE
127 + if run_id:
128 + resp.headers["X-CountryAtlas-Run"] = run_id
129 + return resp
118 130 if key is not None:
119 131 hit = cache.get(key)
120 132 if hit is not None:
121 133 status, body, media = hit
122 134 resp = Response(content=body, status_code=status, media_type=media)
123 135 resp.headers["X-Cache"] = "HIT"
136 + if etag and status == 200:
137 + resp.headers["ETag"] = etag
138 + resp.headers["Cache-Control"] = PUBLIC_CACHE
124 139 if run_id:
125 140 resp.headers["X-CountryAtlas-Run"] = run_id
126 141 return resp
@@ -139,6 +154,9 @@ def create_app(db_path: str | Path | None = None, *, rate_limit_per_minute: int
139 154 headers.pop("content-length", None)
140 155 new = Response(content=body, status_code=response.status_code, headers=headers, media_type=response.media_type)
141 156 new.headers["X-Cache"] = "MISS"
157 + if etag:
158 + new.headers["ETag"] = etag
159 + new.headers["Cache-Control"] = PUBLIC_CACHE
142 160 return new
143 161 return response
144 162
modified src/countryatlas/stats.py +1 −1
@@ -97,7 +97,7 @@ def rank(values: Sequence[float | None], descending: bool = True) -> list[int |
97 97 order = -vals if descending else vals
98 98 # ties: rank = 1 + number of strictly better values
99 99 for idx, v in zip(np.flatnonzero(ok), order, strict=True):
100 − out[int(idx)] = int(1 + np.sum(order < v))
100 + out[int(idx)] = 1 + int(np.sum(order < v))
101 101 return out
102 102
103 103
104 104