registry scale-up prep: tier 4 (900 s), fragment loading, paginated targets API/page, 5-min corroboration polling
9 changed files +106 −26
modified
apps/api/src/internetpressure/api/admin.py
+1 −1
@@ -103,7 +103,7 @@ class TargetIn(BaseModel): | |||||
| 103 | country: str | None = None | 103 | country: str | None = None | ||
| 104 | region: str | None = None | 104 | region: str | None = None | ||
| 105 | importance: int = Field(3, ge=1, le=5) | 105 | importance: int = Field(3, ge=1, le=5) | ||
| 106 | − tier: int = Field(2, ge=1, le=3) | 106 | + tier: int = Field(2, ge=1, le=4) | ||
| 107 | checks: list[str] = ["http", "dns", "ping"] | 107 | checks: list[str] = ["http", "dns", "ping"] | ||
| 108 | traceroute: bool = False | 108 | traceroute: bool = False | ||
| 109 | enabled: bool = True | 109 | enabled: bool = True | ||
modified
apps/api/src/internetpressure/api/public.py
+15 −4
@@ -412,10 +412,10 @@ def _z_of(q: tuple[float, float, float] | None, current: Any) -> float | None: | |||||
| 412 | async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any]]: | 412 | async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any]]: | ||
| 413 | if not targets: | 413 | if not targets: | ||
| 414 | return [] | 414 | return [] | ||
| 415 | − ids = ",".join(f"'{_esc(t['target_id'])}'" for t in targets) | 415 | + where_ids = f"AND target_id IN ({','.join(f"'{_esc(t['target_id'])}'" for t in targets)})" if len(targets) <= 400 else "" | ||
| 416 | rows = await ch.query(f""" | 416 | rows = await ch.query(f""" | ||
| 417 | SELECT target_id, avg(ok) AS ok_ratio, quantileTDigest(0.5)(ttfb_ms) AS ttfb FROM measurements | 417 | SELECT target_id, avg(ok) AS ok_ratio, quantileTDigest(0.5)(ttfb_ms) AS ttfb FROM measurements | ||
| 418 | − WHERE kind='http' AND target_id IN ({ids}) AND ts >= now() - INTERVAL 1 HOUR GROUP BY target_id | 418 | + WHERE kind='http' {where_ids} AND ts >= now() - INTERVAL 1 HOUR GROUP BY target_id | ||
| 419 | """) | 419 | """) | ||
| 420 | stats = {r["target_id"]: r for r in rows} | 420 | stats = {r["target_id"]: r for r in rows} | ||
| 421 | tp = await _live("target_pressure") or {} | 421 | tp = await _live("target_pressure") or {} | ||
@@ -430,11 +430,22 @@ async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any] | |||||
| 430 | 430 | ||||
| 431 | 431 | ||||
| 432 | @router.get("/targets") | 432 | @router.get("/targets") | ||
| 433 | −async def targets(category: str | None = None) -> dict[str, Any]: | 433 | +async def targets(category: str | None = None, q: str | None = None, country: str | None = None, tier: int | None = None, | ||
| 434 | + limit: int = Query(300, ge=1, le=3000), offset: int = Query(0, ge=0)) -> dict[str, Any]: | ||||
| 434 | ts = await list_targets(enabled_only=True) | 435 | ts = await list_targets(enabled_only=True) | ||
| 435 | if category: | 436 | if category: | ||
| 436 | ts = [t for t in ts if t["category"] == category] | 437 | ts = [t for t in ts if t["category"] == category] | ||
| 437 | − return {"targets": await _targets_summary(ts)} | 438 | + if country: | ||
| 439 | + ts = [t for t in ts if (t.get("country") or "").upper() == country.upper()] | ||||
| 440 | + if tier: | ||||
| 441 | + ts = [t for t in ts if int(t["tier"]) == tier] | ||||
| 442 | + if q: | ||||
| 443 | + ql = q.lower().strip() | ||||
| 444 | + ts = [t for t in ts if ql in t["target_id"] or ql in t["name"].lower() or ql in t["hostname"].lower() or ql in (t.get("provider") or "").lower()] | ||||
| 445 | + total = len(ts) | ||||
| 446 | + cats = sorted({t["category"] for t in await list_targets(enabled_only=True)}) | ||||
| 447 | + page = ts[offset: offset + limit] | ||||
| 448 | + return {"total": total, "limit": limit, "offset": offset, "categories": cats, "targets": await _targets_summary(page)} | ||||
| 438 | 449 | ||||
| 439 | 450 | ||||
| 440 | @router.get("/target/{target_id}") | 451 | @router.get("/target/{target_id}") | ||
modified
apps/api/src/internetpressure/corroboration/runner.py
+2 −2
@@ -1,4 +1,4 @@ | |||||
| 1 | −"""Polls every service with a `status` connector spec every 60 s and stores the normalised result in Postgres | 1 | +"""Polls every service with a `status` connector spec every 5 min and stores the normalised result in Postgres | ||
| 2 | `vendor_status` (+ raw provenance). Failures are recorded, never raised into scoring.""" | 2 | `vendor_status` (+ raw provenance). Failures are recorded, never raised into scoring.""" | ||
| 3 | 3 | ||||
| 4 | from __future__ import annotations | 4 | from __future__ import annotations | ||
@@ -52,4 +52,4 @@ async def main(*, once: bool = False) -> None: | |||||
| 52 | log.exception("corroboration run failed: %s", exc) | 52 | log.exception("corroboration run failed: %s", exc) | ||
| 53 | if once: | 53 | if once: | ||
| 54 | break | 54 | break | ||
| 55 | − await asyncio.sleep(60) | 55 | + await asyncio.sleep(300) # hundreds of status pages: poll every 5 min, politely | ||
added
apps/api/src/internetpressure/db/migrations/pg/0002_tier4.sql
+3 −0
@@ -0,0 +1,3 @@ | |||||
| 1 | +-- 2026-09-13 — tier 4 (every 900 s) for the long tail of the registry. | ||||
| 2 | +ALTER TABLE targets DROP CONSTRAINT IF EXISTS targets_tier_check; | ||||
| 3 | +ALTER TABLE targets ADD CONSTRAINT targets_tier_check CHECK (tier BETWEEN 1 AND 4); | ||||
modified
apps/api/src/internetpressure/registry.py
+20 −5
@@ -229,15 +229,30 @@ async def seed(*, targets: bool = True, services: bool = True, probes: bool = Tr | |||||
| 229 | counts = {"services": 0, "targets": 0, "probes": 0} | 229 | counts = {"services": 0, "targets": 0, "probes": 0} | ||
| 230 | regions = load_regions() | 230 | regions = load_regions() | ||
| 231 | if services: | 231 | if services: | ||
| 232 | − with open(s.services_path, encoding="utf-8") as fh: | 232 | + svc_files = [s.services_path] + sorted((s.services_path.parent / "services-fragments").glob("*.yaml")) | ||
| 233 | − for svc in yaml.safe_load(fh)["services"]: | 233 | + seen_svc: set[str] = set() | ||
| 234 | − await upsert_service(svc) | 234 | + for f in svc_files: | ||
| 235 | − counts["services"] += 1 | 235 | + with open(f, encoding="utf-8") as fh: | ||
| 236 | + for svc in (yaml.safe_load(fh) or {}).get("services") or []: | ||||
| 237 | + if svc["slug"] in seen_svc: | ||||
| 238 | + continue | ||||
| 239 | + seen_svc.add(svc["slug"]) | ||||
| 240 | + await upsert_service(svc) | ||||
| 241 | + counts["services"] += 1 | ||||
| 236 | if targets: | 242 | if targets: | ||
| 237 | with open(s.targets_path, encoding="utf-8") as fh: | 243 | with open(s.targets_path, encoding="utf-8") as fh: | ||
| 238 | doc = yaml.safe_load(fh) | 244 | doc = yaml.safe_load(fh) | ||
| 239 | defaults = doc.get("defaults") or {} | 245 | defaults = doc.get("defaults") or {} | ||
| 240 | − for t in doc["targets"]: | 246 | + all_targets = list(doc["targets"]) | ||
| 247 | + seen_ids = {str(t["id"]) for t in all_targets} | ||||
| 248 | + for f in sorted((s.targets_path.parent / "fragments").glob("*.yaml")): | ||||
| 249 | + with open(f, encoding="utf-8") as fh: | ||||
| 250 | + for t in (yaml.safe_load(fh) or {}).get("targets") or []: | ||||
| 251 | + if str(t["id"]) in seen_ids: | ||||
| 252 | + continue | ||||
| 253 | + seen_ids.add(str(t["id"])) | ||||
| 254 | + all_targets.append(t) | ||||
| 255 | + for t in all_targets: | ||||
| 241 | row = { | 256 | row = { | ||
| 242 | "target_id": str(t["id"]), "name": str(t["name"]), "hostname": str(t["host"]), "url": t.get("url"), "ip": t.get("ip"), | 257 | "target_id": str(t["id"]), "name": str(t["name"]), "hostname": str(t["host"]), "url": t.get("url"), "ip": t.get("ip"), | ||
| 243 | "port": t.get("port", 443), "category": t["cat"], "provider": t.get("provider"), "service_id": t.get("svc"), | 258 | "port": t.get("port", 443), "category": t["cat"], "provider": t.get("provider"), "service_id": t.get("svc"), | ||
modified
apps/web/src/app/(site)/routes/page.tsx
+1 −1
@@ -8,7 +8,7 @@ export const metadata: Metadata = { title: 'Route Explorer', description: 'Probe | |||||
| 8 | 8 | ||||
| 9 | export default async function RoutesPage({ searchParams }: { searchParams: Promise<{ probe?: string; target?: string }> }) { | 9 | export default async function RoutesPage({ searchParams }: { searchParams: Promise<{ probe?: string; target?: string }> }) { | ||
| 10 | const sp = await searchParams; | 10 | const sp = await searchParams; | ||
| 11 | − const [pairs, probes, targets] = await Promise.all([apiTry<{ pairs: RoutePair[] }>('/api/v1/routes/pairs'), apiTry<{ probes: Probe[] }>('/api/v1/probes'), apiTry<{ targets: Target[] }>('/api/v1/targets')]); | 11 | + const [pairs, probes, targets] = await Promise.all([apiTry<{ pairs: RoutePair[] }>('/api/v1/routes/pairs'), apiTry<{ probes: Probe[] }>('/api/v1/probes'), apiTry<{ targets: Target[] }>('/api/v1/targets?limit=3000')]); | ||
| 12 | const list = pairs?.pairs ?? []; | 12 | const list = pairs?.pairs ?? []; | ||
| 13 | // pre-select: URL → a changed pair → first pair | 13 | // pre-select: URL → a changed pair → first pair | ||
| 14 | const initialPair = list.find((p) => p.probe_id === sp.probe && p.target_id === sp.target) ?? [...list].sort((a, b) => b.changed_24h - a.changed_24h)[0] ?? null; | 14 | const initialPair = list.find((p) => p.probe_id === sp.probe && p.target_id === sp.target) ?? [...list].sort((a, b) => b.changed_24h - a.changed_24h)[0] ?? null; | ||
modified
apps/web/src/app/(site)/targets/page.tsx
+15 −5
@@ -7,18 +7,28 @@ import type { Target } from '@/lib/types'; | |||||
| 7 | export const dynamic = 'force-dynamic'; | 7 | export const dynamic = 'force-dynamic'; | ||
| 8 | export const metadata: Metadata = { title: 'Target registry', description: 'The representative endpoints we measure: cloud, CDN, DNS, developer platforms, social, finance, government, streaming and more.' }; | 8 | export const metadata: Metadata = { title: 'Target registry', description: 'The representative endpoints we measure: cloud, CDN, DNS, developer platforms, social, finance, government, streaming and more.' }; | ||
| 9 | 9 | ||||
| 10 | −export default async function TargetsPage({ searchParams }: { searchParams: Promise<{ q?: string; category?: string }> }) { | 10 | +const PAGE = 250; | ||
| 11 | + | ||||
| 12 | +type TargetsResponse = { total: number; limit: number; offset: number; categories: string[]; targets: Target[] }; | ||||
| 13 | + | ||||
| 14 | +export default async function TargetsPage({ searchParams }: { searchParams: Promise<{ q?: string; category?: string; page?: string; tier?: string; country?: string }> }) { | ||||
| 11 | const sp = await searchParams; | 15 | const sp = await searchParams; | ||
| 12 | − const { targets } = await apiGet<{ targets: Target[] }>('/api/v1/targets'); | 16 | + const page = Math.max(1, Number(sp.page ?? '1') || 1); | ||
| 13 | − const categories = [...new Set(targets.map((t) => t.category))].sort(); | 17 | + const params = new URLSearchParams({ limit: String(PAGE), offset: String((page - 1) * PAGE) }); | ||
| 18 | + if (sp.q) params.set('q', sp.q); | ||||
| 19 | + if (sp.category) params.set('category', sp.category); | ||||
| 20 | + if (sp.tier) params.set('tier', sp.tier); | ||||
| 21 | + if (sp.country) params.set('country', sp.country); | ||||
| 22 | + const res = await apiGet<TargetsResponse>(`/api/v1/targets?${params.toString()}`); | ||||
| 23 | + const pages = Math.max(1, Math.ceil(res.total / PAGE)); | ||||
| 14 | return ( | 24 | return ( | ||
| 15 | <div className="pb-8"> | 25 | <div className="pb-8"> | ||
| 16 | <header className="pt-6 pb-4"> | 26 | <header className="pt-6 pb-4"> | ||
| 17 | <p className="label">Registry</p> | 27 | <p className="label">Registry</p> | ||
| 18 | <h1 className="mt-1 text-[26px] font-medium tracking-tight sm:text-[32px]">Targets</h1> | 28 | <h1 className="mt-1 text-[26px] font-medium tracking-tight sm:text-[32px]">Targets</h1> | ||
| 19 | − <p className="mt-1 max-w-[760px] text-[13px] text-ink-2">{fmtInt(targets.length)} endpoints across {categories.length} categories. Tier 1 is checked every 20 s, tier 2 every 45 s, tier 3 every 3 min; traceroutes every 15 min. Importance (1–5) weights each target in the aggregates.</p> | 29 | + <p className="mt-1 max-w-[760px] text-[13px] text-ink-2">{fmtInt(res.total)} endpoints across {res.categories.length} categories. Tier 1 is checked every 20 s, tier 2 every 45 s, tier 3 every 5 min, tier 4 every 15 min; traceroutes every 15 min. Importance (1–5) weights each target in the aggregates.</p> | ||
| 20 | </header> | 30 | </header> | ||
| 21 | − <TargetsRegistry targets={targets} categories={categories} initialQuery={sp.q ?? ''} initialCategory={sp.category ?? ''} /> | 31 | + <TargetsRegistry targets={res.targets} categories={res.categories} initialQuery={sp.q ?? ''} initialCategory={sp.category ?? ''} total={res.total} page={page} pages={pages} pageSize={PAGE} /> | ||
| 22 | </div> | 32 | </div> | ||
| 23 | ); | 33 | ); | ||
| 24 | } | 34 | } | ||
modified
apps/web/src/components/targets/TargetsRegistry.tsx
+48 −7
@@ -1,16 +1,44 @@ | |||||
| 1 | 'use client'; | 1 | 'use client'; | ||
| 2 | 2 | ||||
| 3 | import Link from 'next/link'; | 3 | import Link from 'next/link'; | ||
| 4 | −import { useMemo, useState } from 'react'; | 4 | +import { useRouter, useSearchParams } from 'next/navigation'; | ||
| 5 | +import { useEffect, useMemo, useRef, useState } from 'react'; | ||||
| 5 | import { PNum } from '@/components/ui/primitives'; | 6 | import { PNum } from '@/components/ui/primitives'; | ||
| 6 | import { fmtInt, fmtMs, fmtPct } from '@/lib/format'; | 7 | import { fmtInt, fmtMs, fmtPct } from '@/lib/format'; | ||
| 7 | import type { Target } from '@/lib/types'; | 8 | import type { Target } from '@/lib/types'; | ||
| 8 | 9 | ||||
| 9 | type SortKey = 'pressure' | 'name' | 'ok_ratio_1h' | 'ttfb_ms_median_1h' | 'importance'; | 10 | type SortKey = 'pressure' | 'name' | 'ok_ratio_1h' | 'ttfb_ms_median_1h' | 'importance'; | ||
| 10 | 11 | ||||
| 11 | −export function TargetsRegistry({ targets, categories, initialQuery, initialCategory }: { targets: Target[]; categories: string[]; initialQuery: string; initialCategory: string }) { | 12 | +export function TargetsRegistry({ targets, categories, initialQuery, initialCategory, total, page, pages, pageSize }: { targets: Target[]; categories: string[]; initialQuery: string; initialCategory: string; total: number; page: number; pages: number; pageSize: number }) { | ||
| 12 | const [q, setQ] = useState(initialQuery); | 13 | const [q, setQ] = useState(initialQuery); | ||
| 13 | const [cat, setCat] = useState(initialCategory); | 14 | const [cat, setCat] = useState(initialCategory); | ||
| 15 | + const router = useRouter(); | ||||
| 16 | + const sp = useSearchParams(); | ||||
| 17 | + const first = useRef(true); | ||||
| 18 | + // filters are applied server-side (the registry is paginated): push them to the URL, debounced | ||||
| 19 | + useEffect(() => { | ||||
| 20 | + if (first.current) { | ||||
| 21 | + first.current = false; | ||||
| 22 | + return; | ||||
| 23 | + } | ||||
| 24 | + const t = setTimeout(() => { | ||||
| 25 | + const next = new URLSearchParams(sp.toString()); | ||||
| 26 | + if (q) next.set('q', q); | ||||
| 27 | + else next.delete('q'); | ||||
| 28 | + if (cat) next.set('category', cat); | ||||
| 29 | + else next.delete('category'); | ||||
| 30 | + next.delete('page'); | ||||
| 31 | + router.replace(`/targets?${next.toString()}`, { scroll: false }); | ||||
| 32 | + }, 350); | ||||
| 33 | + return () => clearTimeout(t); | ||||
| 34 | + // eslint-disable-next-line react-hooks/exhaustive-deps | ||||
| 35 | + }, [q, cat]); | ||||
| 36 | + const pageHref = (n: number) => { | ||||
| 37 | + const next = new URLSearchParams(sp.toString()); | ||||
| 38 | + if (n > 1) next.set('page', String(n)); | ||||
| 39 | + else next.delete('page'); | ||||
| 40 | + return `/targets?${next.toString()}`; | ||||
| 41 | + }; | ||||
| 14 | const [sort, setSort] = useState<SortKey>('pressure'); | 42 | const [sort, setSort] = useState<SortKey>('pressure'); | ||
| 15 | const [dir, setDir] = useState<1 | -1>(-1); | 43 | const [dir, setDir] = useState<1 | -1>(-1); | ||
| 16 | 44 | ||||
@@ -59,7 +87,7 @@ export function TargetsRegistry({ targets, categories, initialQuery, initialCate | |||||
| 59 | </button> | 87 | </button> | ||
| 60 | ))} | 88 | ))} | ||
| 61 | </div> | 89 | </div> | ||
| 62 | − <span className="num ml-auto text-[11px] text-ink-3">{fmtInt(rows.length)} shown</span> | 90 | + <span className="num ml-auto text-[11px] text-ink-3">{fmtInt(rows.length)} shown · {fmtInt(total)} match · page {page}/{pages}</span> | ||
| 63 | </div> | 91 | </div> | ||
| 64 | <div className="scroll-x -mx-3 mt-3 px-3"> | 92 | <div className="scroll-x -mx-3 mt-3 px-3"> | ||
| 65 | <table className="tbl"> | 93 | <table className="tbl"> | ||
@@ -86,9 +114,13 @@ export function TargetsRegistry({ targets, categories, initialQuery, initialCate | |||||
| 86 | <td className="num hidden text-ink-2 md:table-cell">{t.hostname}</td> | 114 | <td className="num hidden text-ink-2 md:table-cell">{t.hostname}</td> | ||
| 87 | <td className="hidden text-ink-2 sm:table-cell">{t.category}</td> | 115 | <td className="hidden text-ink-2 sm:table-cell">{t.category}</td> | ||
| 88 | <td className="hidden lg:table-cell"> | 116 | <td className="hidden lg:table-cell"> | ||
| 89 | − <Link href={`/service/${t.service_id}`} className="text-ink-2 hover:text-accent"> | 117 | + {t.service_id ? ( | ||
| 90 | − {t.service_id} | 118 | + <Link href={`/service/${t.service_id}`} className="text-ink-2 hover:text-accent"> | ||
| 91 | − </Link> | 119 | + {t.service_id} | ||
| 120 | + </Link> | ||||
| 121 | + ) : ( | ||||
| 122 | + <span className="text-ink-3">—</span> | ||||
| 123 | + )} | ||||
| 92 | </td> | 124 | </td> | ||
| 93 | <td className="hidden sm:table-cell"> | 125 | <td className="hidden sm:table-cell"> | ||
| 94 | {t.country ? ( | 126 | {t.country ? ( | ||
@@ -104,7 +136,7 @@ export function TargetsRegistry({ targets, categories, initialQuery, initialCate | |||||
| 104 | <td className="r"> | 136 | <td className="r"> | ||
| 105 | <PNum value={t.pressure} /> | 137 | <PNum value={t.pressure} /> | ||
| 106 | </td> | 138 | </td> | ||
| 107 | − <td className="num r" style={{ color: t.ok_ratio_1h < 0.98 ? 'var(--p-high)' : undefined }}> | 139 | + <td className="num r" style={{ color: t.ok_ratio_1h != null && t.ok_ratio_1h < 0.98 ? 'var(--p-high)' : undefined }}> | ||
| 108 | {fmtPct(t.ok_ratio_1h, 1)} | 140 | {fmtPct(t.ok_ratio_1h, 1)} | ||
| 109 | </td> | 141 | </td> | ||
| 110 | <td className="num r text-ink-2">{fmtMs(t.ttfb_ms_median_1h)}</td> | 142 | <td className="num r text-ink-2">{fmtMs(t.ttfb_ms_median_1h)}</td> | ||
@@ -113,6 +145,15 @@ export function TargetsRegistry({ targets, categories, initialQuery, initialCate | |||||
| 113 | </tbody> | 145 | </tbody> | ||
| 114 | </table> | 146 | </table> | ||
| 115 | </div> | 147 | </div> | ||
| 148 | + {pages > 1 && ( | ||||
| 149 | + <nav className="mt-3 flex items-center gap-2 text-[12px]" aria-label="Pagination"> | ||||
| 150 | + {page > 1 ? <Link href={pageHref(page - 1)} className="text-ink-2 hover:text-accent">← previous</Link> : <span className="text-ink-3">← previous</span>} | ||||
| 151 | + <span className="num text-ink-3"> | ||||
| 152 | + {fmtInt((page - 1) * pageSize + 1)}–{fmtInt(Math.min(page * pageSize, total))} of {fmtInt(total)} | ||||
| 153 | + </span> | ||||
| 154 | + {page < pages ? <Link href={pageHref(page + 1)} className="text-ink-2 hover:text-accent">next →</Link> : <span className="text-ink-3">next →</span>} | ||||
| 155 | + </nav> | ||||
| 156 | + )} | ||||
| 116 | </div> | 157 | </div> | ||
| 117 | ); | 158 | ); | ||
| 118 | } | 159 | } | ||
modified
packages/config/pressure.yaml
+1 −1
@@ -100,7 +100,7 @@ fronts: | |||||
| 100 | min_intensity: 35 | 100 | min_intensity: 35 | ||
| 101 | 101 | ||||
| 102 | scheduler: | 102 | scheduler: | ||
| 103 | − tiers: { 1: 20, 2: 45, 3: 300 } # seconds between HTTP checks per tier (tier 3 = the long tail of ~1 700 targets) | 103 | + tiers: { 1: 20, 2: 45, 3: 300, 4: 900 } # seconds between HTTP checks per tier (4 = the long tail: government, media, retail…) | ||
| 104 | dns_every: 600 # 4 resolvers × 2 100 targets: one query per resolver every 10 min per probe | 104 | dns_every: 600 # 4 resolvers × 2 100 targets: one query per resolver every 10 min per probe | ||
| 105 | ping_every: 120 | 105 | ping_every: 120 | ||
| 106 | traceroute_every: 900 | 106 | traceroute_every: 900 | ||
| 107 | 107 | ||||