registry scale-up prep: tier 4 (900 s), fragment loading, paginated targets API/page, 5-min corroboration polling

Simon-Pierre Boucher committed 11 days ago (Sep 13, 2026) parent b2ec043

9 changed files +106 −26

modified apps/api/src/internetpressure/api/admin.py +1 −1

		`@@ -103,7 +103,7 @@ class TargetIn(BaseModel):`
103	103	country: str \| None = None
104	104	region: str \| None = None
105	105	importance: int = Field(3, ge=1, le=5)
106		− tier: int = Field(2, ge=1, le=3)
	106	+ tier: int = Field(2, ge=1, le=4)
107	107	checks: list[str] = ["http", "dns", "ping"]
108	108	traceroute: bool = False
109	109	enabled: bool = True

modified apps/api/src/internetpressure/api/public.py +15 −4

		`@@ -412,10 +412,10 @@ def _z_of(q: tuple[float, float, float] \| None, current: Any) -> float \| None:`
412	412	async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any]]:
413	413	if not targets:
414	414	return []
415		− ids = ",".join(f"'{_esc(t['target_id'])}'" for t in targets)
	415	+ where_ids = f"AND target_id IN ({','.join(f"'{_esc(t['target_id'])}'" for t in targets)})" if len(targets) <= 400 else ""
416	416	rows = await ch.query(f"""
417	417	SELECT target_id, avg(ok) AS ok_ratio, quantileTDigest(0.5)(ttfb_ms) AS ttfb FROM measurements
418		− WHERE kind='http' AND target_id IN ({ids}) AND ts >= now() - INTERVAL 1 HOUR GROUP BY target_id
	418	+ WHERE kind='http' {where_ids} AND ts >= now() - INTERVAL 1 HOUR GROUP BY target_id
419	419	""")
420	420	stats = {r["target_id"]: r for r in rows}
421	421	tp = await _live("target_pressure") or {}
		`@@ -430,11 +430,22 @@ async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any]`
430	430
431	431
432	432	@router.get("/targets")
433		−async def targets(category: str \| None = None) -> dict[str, Any]:
	433	+async def targets(category: str \| None = None, q: str \| None = None, country: str \| None = None, tier: int \| None = None,
	434	+ limit: int = Query(300, ge=1, le=3000), offset: int = Query(0, ge=0)) -> dict[str, Any]:
434	435	ts = await list_targets(enabled_only=True)
435	436	if category:
436	437	ts = [t for t in ts if t["category"] == category]
437		− return {"targets": await _targets_summary(ts)}
	438	+ if country:
	439	+ ts = [t for t in ts if (t.get("country") or "").upper() == country.upper()]
	440	+ if tier:
	441	+ ts = [t for t in ts if int(t["tier"]) == tier]
	442	+ if q:
	443	+ ql = q.lower().strip()
	444	+ ts = [t for t in ts if ql in t["target_id"] or ql in t["name"].lower() or ql in t["hostname"].lower() or ql in (t.get("provider") or "").lower()]
	445	+ total = len(ts)
	446	+ cats = sorted({t["category"] for t in await list_targets(enabled_only=True)})
	447	+ page = ts[offset: offset + limit]
	448	+ return {"total": total, "limit": limit, "offset": offset, "categories": cats, "targets": await _targets_summary(page)}
438	449
439	450
440	451	@router.get("/target/{target_id}")

modified apps/api/src/internetpressure/corroboration/runner.py +2 −2

		`@@ -1,4 +1,4 @@`
1		−"""Polls every service with a `status` connector spec every 60 s and stores the normalised result in Postgres
	1	+"""Polls every service with a `status` connector spec every 5 min and stores the normalised result in Postgres
2	2	`vendor_status` (+ raw provenance). Failures are recorded, never raised into scoring."""
3	3
4	4	from __future__ import annotations
		`@@ -52,4 +52,4 @@ async def main(*, once: bool = False) -> None:`
52	52	log.exception("corroboration run failed: %s", exc)
53	53	if once:
54	54	break
55		− await asyncio.sleep(60)
	55	+ await asyncio.sleep(300) # hundreds of status pages: poll every 5 min, politely

added apps/api/src/internetpressure/db/migrations/pg/0002_tier4.sql +3 −0

		`@@ -0,0 +1,3 @@`
	1	+-- 2026-09-13 — tier 4 (every 900 s) for the long tail of the registry.
	2	+ALTER TABLE targets DROP CONSTRAINT IF EXISTS targets_tier_check;
	3	+ALTER TABLE targets ADD CONSTRAINT targets_tier_check CHECK (tier BETWEEN 1 AND 4);

modified apps/api/src/internetpressure/registry.py +20 −5

		`@@ -229,15 +229,30 @@ async def seed(*, targets: bool = True, services: bool = True, probes: bool = Tr`
229	229	counts = {"services": 0, "targets": 0, "probes": 0}
230	230	regions = load_regions()
231	231	if services:
232		− with open(s.services_path, encoding="utf-8") as fh:
233		− for svc in yaml.safe_load(fh)["services"]:
234		− await upsert_service(svc)
235		− counts["services"] += 1
	232	+ svc_files = [s.services_path] + sorted((s.services_path.parent / "services-fragments").glob("*.yaml"))
	233	+ seen_svc: set[str] = set()
	234	+ for f in svc_files:
	235	+ with open(f, encoding="utf-8") as fh:
	236	+ for svc in (yaml.safe_load(fh) or {}).get("services") or []:
	237	+ if svc["slug"] in seen_svc:
	238	+ continue
	239	+ seen_svc.add(svc["slug"])
	240	+ await upsert_service(svc)
	241	+ counts["services"] += 1
236	242	if targets:
237	243	with open(s.targets_path, encoding="utf-8") as fh:
238	244	doc = yaml.safe_load(fh)
239	245	defaults = doc.get("defaults") or {}
240		− for t in doc["targets"]:
	246	+ all_targets = list(doc["targets"])
	247	+ seen_ids = {str(t["id"]) for t in all_targets}
	248	+ for f in sorted((s.targets_path.parent / "fragments").glob("*.yaml")):
	249	+ with open(f, encoding="utf-8") as fh:
	250	+ for t in (yaml.safe_load(fh) or {}).get("targets") or []:
	251	+ if str(t["id"]) in seen_ids:
	252	+ continue
	253	+ seen_ids.add(str(t["id"]))
	254	+ all_targets.append(t)
	255	+ for t in all_targets:
241	256	row = {
242	257	"target_id": str(t["id"]), "name": str(t["name"]), "hostname": str(t["host"]), "url": t.get("url"), "ip": t.get("ip"),
243	258	"port": t.get("port", 443), "category": t["cat"], "provider": t.get("provider"), "service_id": t.get("svc"),

modified apps/web/src/app/(site)/routes/page.tsx +1 −1

		`@@ -8,7 +8,7 @@ export const metadata: Metadata = { title: 'Route Explorer', description: 'Probe`
8	8
9	9	export default async function RoutesPage({ searchParams }: { searchParams: Promise<{ probe?: string; target?: string }> }) {
10	10	const sp = await searchParams;
11		− const [pairs, probes, targets] = await Promise.all([apiTry<{ pairs: RoutePair[] }>('/api/v1/routes/pairs'), apiTry<{ probes: Probe[] }>('/api/v1/probes'), apiTry<{ targets: Target[] }>('/api/v1/targets')]);
	11	+ const [pairs, probes, targets] = await Promise.all([apiTry<{ pairs: RoutePair[] }>('/api/v1/routes/pairs'), apiTry<{ probes: Probe[] }>('/api/v1/probes'), apiTry<{ targets: Target[] }>('/api/v1/targets?limit=3000')]);
12	12	const list = pairs?.pairs ?? [];
13	13	// pre-select: URL → a changed pair → first pair
14	14	const initialPair = list.find((p) => p.probe_id === sp.probe && p.target_id === sp.target) ?? [...list].sort((a, b) => b.changed_24h - a.changed_24h)[0] ?? null;

modified apps/web/src/app/(site)/targets/page.tsx +15 −5

		`@@ -7,18 +7,28 @@ import type { Target } from '@/lib/types';`
7	7	export const dynamic = 'force-dynamic';
8	8	export const metadata: Metadata = { title: 'Target registry', description: 'The representative endpoints we measure: cloud, CDN, DNS, developer platforms, social, finance, government, streaming and more.' };
9	9
10		−export default async function TargetsPage({ searchParams }: { searchParams: Promise<{ q?: string; category?: string }> }) {
	10	+const PAGE = 250;
	11	+
	12	+type TargetsResponse = { total: number; limit: number; offset: number; categories: string[]; targets: Target[] };
	13	+
	14	+export default async function TargetsPage({ searchParams }: { searchParams: Promise<{ q?: string; category?: string; page?: string; tier?: string; country?: string }> }) {
11	15	const sp = await searchParams;
12		− const { targets } = await apiGet<{ targets: Target[] }>('/api/v1/targets');
13		− const categories = [...new Set(targets.map((t) => t.category))].sort();
	16	+ const page = Math.max(1, Number(sp.page ?? '1') \|\| 1);
	17	+ const params = new URLSearchParams({ limit: String(PAGE), offset: String((page - 1) * PAGE) });
	18	+ if (sp.q) params.set('q', sp.q);
	19	+ if (sp.category) params.set('category', sp.category);
	20	+ if (sp.tier) params.set('tier', sp.tier);
	21	+ if (sp.country) params.set('country', sp.country);
	22	+ const res = await apiGet<TargetsResponse>(`/api/v1/targets?${params.toString()}`);
	23	+ const pages = Math.max(1, Math.ceil(res.total / PAGE));
14	24	return (
15	25	<div className="pb-8">
16	26	<header className="pt-6 pb-4">
17	27	<p className="label">Registry</p>
18	28	<h1 className="mt-1 text-[26px] font-medium tracking-tight sm:text-[32px]">Targets</h1>
19		− <p className="mt-1 max-w-[760px] text-[13px] text-ink-2">{fmtInt(targets.length)} endpoints across {categories.length} categories. Tier 1 is checked every 20 s, tier 2 every 45 s, tier 3 every 3 min; traceroutes every 15 min. Importance (1–5) weights each target in the aggregates.</p>
	29	+ <p className="mt-1 max-w-[760px] text-[13px] text-ink-2">{fmtInt(res.total)} endpoints across {res.categories.length} categories. Tier 1 is checked every 20 s, tier 2 every 45 s, tier 3 every 5 min, tier 4 every 15 min; traceroutes every 15 min. Importance (1–5) weights each target in the aggregates.</p>
20	30	</header>
21		− <TargetsRegistry targets={targets} categories={categories} initialQuery={sp.q ?? ''} initialCategory={sp.category ?? ''} />
	31	+ <TargetsRegistry targets={res.targets} categories={res.categories} initialQuery={sp.q ?? ''} initialCategory={sp.category ?? ''} total={res.total} page={page} pages={pages} pageSize={PAGE} />
22	32	</div>
23	33	);
24	34	}

modified apps/web/src/components/targets/TargetsRegistry.tsx +48 −7

		`@@ -1,16 +1,44 @@`
1	1	'use client';
2	2
3	3	import Link from 'next/link';
4		−import { useMemo, useState } from 'react';
	4	+import { useRouter, useSearchParams } from 'next/navigation';
	5	+import { useEffect, useMemo, useRef, useState } from 'react';
5	6	import { PNum } from '@/components/ui/primitives';
6	7	import { fmtInt, fmtMs, fmtPct } from '@/lib/format';
7	8	import type { Target } from '@/lib/types';
8	9
9	10	type SortKey = 'pressure' \| 'name' \| 'ok_ratio_1h' \| 'ttfb_ms_median_1h' \| 'importance';
10	11
11		−export function TargetsRegistry({ targets, categories, initialQuery, initialCategory }: { targets: Target[]; categories: string[]; initialQuery: string; initialCategory: string }) {
	12	+export function TargetsRegistry({ targets, categories, initialQuery, initialCategory, total, page, pages, pageSize }: { targets: Target[]; categories: string[]; initialQuery: string; initialCategory: string; total: number; page: number; pages: number; pageSize: number }) {
12	13	const [q, setQ] = useState(initialQuery);
13	14	const [cat, setCat] = useState(initialCategory);
	15	+ const router = useRouter();
	16	+ const sp = useSearchParams();
	17	+ const first = useRef(true);
	18	+ // filters are applied server-side (the registry is paginated): push them to the URL, debounced
	19	+ useEffect(() => {
	20	+ if (first.current) {
	21	+ first.current = false;
	22	+ return;
	23	+ }
	24	+ const t = setTimeout(() => {
	25	+ const next = new URLSearchParams(sp.toString());
	26	+ if (q) next.set('q', q);
	27	+ else next.delete('q');
	28	+ if (cat) next.set('category', cat);
	29	+ else next.delete('category');
	30	+ next.delete('page');
	31	+ router.replace(`/targets?${next.toString()}`, { scroll: false });
	32	+ }, 350);
	33	+ return () => clearTimeout(t);
	34	+ // eslint-disable-next-line react-hooks/exhaustive-deps
	35	+ }, [q, cat]);
	36	+ const pageHref = (n: number) => {
	37	+ const next = new URLSearchParams(sp.toString());
	38	+ if (n > 1) next.set('page', String(n));
	39	+ else next.delete('page');
	40	+ return `/targets?${next.toString()}`;
	41	+ };
14	42	const [sort, setSort] = useState<SortKey>('pressure');
15	43	const [dir, setDir] = useState<1 \| -1>(-1);
16	44
		`@@ -59,7 +87,7 @@ export function TargetsRegistry({ targets, categories, initialQuery, initialCate`
59	87	</button>
60	88	))}
61	89	</div>
62		− <span className="num ml-auto text-[11px] text-ink-3">{fmtInt(rows.length)} shown</span>
	90	+ <span className="num ml-auto text-[11px] text-ink-3">{fmtInt(rows.length)} shown · {fmtInt(total)} match · page {page}/{pages}</span>
63	91	</div>
64	92	<div className="scroll-x -mx-3 mt-3 px-3">
65	93	<table className="tbl">
		`@@ -86,9 +114,13 @@ export function TargetsRegistry({ targets, categories, initialQuery, initialCate`
86	114	<td className="num hidden text-ink-2 md:table-cell">{t.hostname}</td>
87	115	<td className="hidden text-ink-2 sm:table-cell">{t.category}</td>
88	116	<td className="hidden lg:table-cell">
89		− <Link href={`/service/${t.service_id}`} className="text-ink-2 hover:text-accent">
90		− {t.service_id}
91		− </Link>
	117	+ {t.service_id ? (
	118	+ <Link href={`/service/${t.service_id}`} className="text-ink-2 hover:text-accent">
	119	+ {t.service_id}
	120	+ </Link>
	121	+ ) : (
	122	+ <span className="text-ink-3">—</span>
	123	+ )}
92	124	</td>
93	125	<td className="hidden sm:table-cell">
94	126	{t.country ? (
		`@@ -104,7 +136,7 @@ export function TargetsRegistry({ targets, categories, initialQuery, initialCate`
104	136	<td className="r">
105	137	<PNum value={t.pressure} />
106	138	</td>
107		− <td className="num r" style={{ color: t.ok_ratio_1h < 0.98 ? 'var(--p-high)' : undefined }}>
	139	+ <td className="num r" style={{ color: t.ok_ratio_1h != null && t.ok_ratio_1h < 0.98 ? 'var(--p-high)' : undefined }}>
108	140	{fmtPct(t.ok_ratio_1h, 1)}
109	141	</td>
110	142	<td className="num r text-ink-2">{fmtMs(t.ttfb_ms_median_1h)}</td>
		`@@ -113,6 +145,15 @@ export function TargetsRegistry({ targets, categories, initialQuery, initialCate`
113	145	</tbody>
114	146	</table>
115	147	</div>
	148	+ {pages > 1 && (
	149	+ <nav className="mt-3 flex items-center gap-2 text-[12px]" aria-label="Pagination">
	150	+ {page > 1 ? <Link href={pageHref(page - 1)} className="text-ink-2 hover:text-accent">← previous</Link> : <span className="text-ink-3">← previous</span>}
	151	+ <span className="num text-ink-3">
	152	+ {fmtInt((page - 1) * pageSize + 1)}–{fmtInt(Math.min(page * pageSize, total))} of {fmtInt(total)}
	153	+ </span>
	154	+ {page < pages ? <Link href={pageHref(page + 1)} className="text-ink-2 hover:text-accent">next →</Link> : <span className="text-ink-3">next →</span>}
	155	+ </nav>
	156	+ )}
116	157	</div>
117	158	);
118	159	}

modified packages/config/pressure.yaml +1 −1

		`@@ -100,7 +100,7 @@ fronts:`
100	100	min_intensity: 35
101	101
102	102	scheduler:
103		− tiers: { 1: 20, 2: 45, 3: 300 } # seconds between HTTP checks per tier (tier 3 = the long tail of ~1 700 targets)
	103	+ tiers: { 1: 20, 2: 45, 3: 300, 4: 900 } # seconds between HTTP checks per tier (4 = the long tail: government, media, retail…)
104	104	dns_every: 600 # 4 resolvers × 2 100 targets: one query per resolver every 10 min per probe
105	105	ping_every: 120
106	106	traceroute_every: 900
107	107