web/api: null-safe pages, per-target pressure, matrix z-scores, stricter affected regions
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +67 −5
modified
apps/api/src/internetpressure/api/public.py
+30 −2
@@ -345,6 +345,7 @@ async def service(slug: str) -> dict[str, Any]: | ||
| 345 | 345 | base = await ch.query_one(f"""SELECT quantileTDigest(0.5)(ttfb_ms) AS ttfb FROM measurements WHERE kind='http' AND target_id IN ({','.join(f"'{_esc(t)}'" for t in tids) or "''"}) |
| 346 | 346 | AND ts >= now() - INTERVAL 7 DAY AND ts < now() - INTERVAL 10 MINUTE""") if tids else {} |
| 347 | 347 | probes = {p["probe_id"]: p for p in await list_probes()} |
| 348 | + zbase = await _ttfb_baselines(tids) | |
| 348 | 349 | matrix = [] |
| 349 | 350 | latest_by_target: dict[str, dict[str, Any]] = {} |
| 350 | 351 | for tid in tids: |
@@ -355,7 +356,7 @@ async def service(slug: str) -> dict[str, Any]: | ||
| 355 | 356 | for tid in tids: |
| 356 | 357 | m = latest_by_target.get(tid, {}).get(f"{pid}|http|") |
| 357 | 358 | if m: |
| 358 | − row["targets"].append({"target_id": tid, "ok": bool(m["ok"]), "ttfb_ms": r1(m.get("ttfb_ms")), "z": None, "ts": m["ts"], | |
| 359 | + row["targets"].append({"target_id": tid, "ok": bool(m["ok"]), "ttfb_ms": r1(m.get("ttfb_ms")), "z": _z_of(zbase.get((pid, tid)), m.get("ttfb_ms")), "ts": m["ts"], | |
| 359 | 360 | "error": m.get("error") or None, "http_status": m.get("http_status")}) |
| 360 | 361 | if row["targets"]: |
| 361 | 362 | matrix.append(row) |
@@ -386,6 +387,28 @@ async def service(slug: str) -> dict[str, Any]: | ||
| 386 | 387 | |
| 387 | 388 | # ── targets & probes ──────────────────────────────────────────────────────────────────────────────────────────────── |
| 388 | 389 | |
| 390 | +async def _ttfb_baselines(target_ids: list[str]) -> dict[tuple[str, str], tuple[float, float, float]]: | |
| 391 | + """7-day TTFB quartiles per (probe, target) — used for the robust z shown in matrices.""" | |
| 392 | + if not target_ids: | |
| 393 | + return {} | |
| 394 | + ids = ",".join(f"'{_esc(t)}'" for t in target_ids) | |
| 395 | + rows = await ch.query(f""" | |
| 396 | + SELECT probe_id, target_id, quantilesTDigest(0.25, 0.5, 0.75)(ttfb_ms) AS q FROM measurements | |
| 397 | + WHERE kind='http' AND target_id IN ({ids}) AND ts >= now() - INTERVAL 7 DAY AND ts < now() - INTERVAL 10 MINUTE | |
| 398 | + GROUP BY probe_id, target_id HAVING count() >= 20 | |
| 399 | + """) | |
| 400 | + return {(r["probe_id"], r["target_id"]): tuple(float(x) for x in r["q"]) for r in rows if r.get("q") and None not in r["q"]} | |
| 401 | + | |
| 402 | + | |
| 403 | +def _z_of(q: tuple[float, float, float] | None, current: Any) -> float | None: | |
| 404 | + if not q or current is None: | |
| 405 | + return None | |
| 406 | + from ..engine import scoring as S | |
| 407 | + | |
| 408 | + z = S.robust_z(float(current), q[1], S.mad_from_iqr(q[0], q[2])) | |
| 409 | + return r2(z) | |
| 410 | + | |
| 411 | + | |
| 389 | 412 | async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any]]: |
| 390 | 413 | if not targets: |
| 391 | 414 | return [] |
@@ -395,12 +418,13 @@ async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any] | ||
| 395 | 418 | WHERE kind='http' AND target_id IN ({ids}) AND ts >= now() - INTERVAL 1 HOUR GROUP BY target_id |
| 396 | 419 | """) |
| 397 | 420 | stats = {r["target_id"]: r for r in rows} |
| 421 | + tp = await _live("target_pressure") or {} | |
| 398 | 422 | out = [] |
| 399 | 423 | for t in targets: |
| 400 | 424 | s = stats.get(t["target_id"], {}) |
| 401 | 425 | out.append({"target_id": t["target_id"], "name": t["name"], "hostname": t["hostname"], "category": t["category"], |
| 402 | 426 | "provider": t.get("provider"), "service_id": t.get("service_id"), "country": t.get("country"), "region": t["region"], |
| 403 | − "importance": t["importance"], "tier": t["tier"], "pressure": None, | |
| 427 | + "importance": t["importance"], "tier": t["tier"], "pressure": tp.get(t["target_id"]), | |
| 404 | 428 | "ok_ratio_1h": r3(s.get("ok_ratio")), "ttfb_ms_median_1h": r1(s.get("ttfb"))}) |
| 405 | 429 | return out |
| 406 | 430 | |
@@ -427,6 +451,10 @@ async def target(target_id: str) -> dict[str, Any]: | ||
| 427 | 451 | "dns_ms": r1(m.get("dns_ms")), "tcp_ms": r1(m.get("tcp_ms")), "tls_ms": r1(m.get("tls_ms")), "ttfb_ms": r1(m.get("ttfb_ms")), |
| 428 | 452 | "http_status": m.get("http_status"), "resolved_ip": m.get("resolved_ip") or None, "packet_loss": m.get("packet_loss"), |
| 429 | 453 | "rtt_avg_ms": r1(m.get("rtt_avg_ms")), "dns_rcode": m.get("dns_rcode") or None, "dns_answers": m.get("dns_answers"), "z": None}) |
| 454 | + zb = await _ttfb_baselines([target_id]) | |
| 455 | + for m in latest: | |
| 456 | + if m["kind"] == "http": | |
| 457 | + m["z"] = _z_of(zb.get((m["probe_id"], target_id)), m.get("ttfb_ms")) | |
| 430 | 458 | latest.sort(key=lambda x: (x["probe_id"], x["kind"], x["resolver"] or "")) |
| 431 | 459 | series = await ch.query(f""" |
| 432 | 460 | SELECT toStartOfFifteenMinutes(ts) AS b, quantileTDigest(0.5)(ttfb_ms) AS ttfb, avg(ok) AS ok_ratio FROM measurements |
modified
apps/api/src/internetpressure/engine/loop.py
+34 −1
@@ -204,7 +204,7 @@ class Engine: | ||
| 204 | 204 | vs = next((v for v in vendor_rows if v["service_slug"] == slug), None) |
| 205 | 205 | pr = sc.pressure if allowed else None |
| 206 | 206 | lv, ll = cfg.level_for(pr) |
| 207 | − affected = sorted({s.tags.src_region for s in buckets.get(("service", slug), []) if s.pair.stress >= 0.4 and s.tags.probe_id != "*"}) | |
| 207 | + affected = _affected_regions(buckets.get(("service", slug), []), sc.pressure if allowed else None) | |
| 208 | 208 | services_out.append({ |
| 209 | 209 | "slug": slug, "name": svc["name"], "category": svc.get("category"), "importance": svc.get("importance"), |
| 210 | 210 | "pressure": r1(pr), "level": lv, "level_label": ll, "confidence": sc.confidence, |
@@ -256,6 +256,8 @@ class Engine: | ||
| 256 | 256 | route_changes_1h = sum(1 for s in sigs if s.signal_id == "route_change_rate" and (s.pair.current or 0) >= 1.0) |
| 257 | 257 | extra = {"dns_failures_per_min": round(dns_fail_2m / max(1.0, window / 60.0), 1), "route_changes_per_min": round(route_changes_1h / 60.0, 2)} |
| 258 | 258 | targets_by_asn = {k[1]: sorted({s.tags.target_id for s in buckets[k]}) for k in asn_keys} |
| 259 | + target_pressure = _target_pressure(sigs, float(eng.get("saturation_k", 1.2))) if allowed else {} | |
| 260 | + await rds.set_json("ip:live:target_pressure", target_pressure) | |
| 259 | 261 | await rds.set_json("ip:live:latency", lat) |
| 260 | 262 | await rds.set_json("ip:live:extra", extra) |
| 261 | 263 | await rds.set_json("ip:live:targets_by_asn", targets_by_asn) |
@@ -414,6 +416,37 @@ class Engine: | ||
| 414 | 416 | log.warning("history insert failed: %s", exc) |
| 415 | 417 | |
| 416 | 418 | |
| 419 | +def _affected_regions(sigs, pressure: float | None) -> list[str]: # type: ignore[no-untyped-def] | |
| 420 | + """Probe regions whose importance-weighted mean stress toward this service is ≥ 0.3 — only when the service | |
| 421 | + itself is at least 'normal-high' (≥ 20), so a single noisy pair never flags a region.""" | |
| 422 | + if pressure is None or pressure < 20: | |
| 423 | + return [] | |
| 424 | + acc: dict[str, list[float]] = {} | |
| 425 | + for s in sigs: | |
| 426 | + if s.tags.probe_id == "*": | |
| 427 | + continue | |
| 428 | + acc.setdefault(s.tags.src_region, []).append((s.pair.stress, s.pair.weight)) # type: ignore[arg-type] | |
| 429 | + out = [] | |
| 430 | + for reg, pairs in acc.items(): | |
| 431 | + tw = sum(w for _, w in pairs) | |
| 432 | + if tw > 0 and sum(st * w for st, w in pairs) / tw >= 0.3: | |
| 433 | + out.append(reg) | |
| 434 | + return sorted(out) | |
| 435 | + | |
| 436 | + | |
| 437 | +def _target_pressure(sigs, k: float) -> dict[str, float]: # type: ignore[no-untyped-def] | |
| 438 | + """Per-target pressure = saturated importance/coverage-weighted mean stress of all its pair signals.""" | |
| 439 | + acc: dict[str, list[tuple[float, float]]] = {} | |
| 440 | + for s in sigs: | |
| 441 | + acc.setdefault(s.tags.target_id, []).append((s.pair.stress, s.pair.weight)) | |
| 442 | + out: dict[str, float] = {} | |
| 443 | + for tid, pairs in acc.items(): | |
| 444 | + tw = sum(w for _, w in pairs) | |
| 445 | + if tw > 0: | |
| 446 | + out[tid] = round(S.saturate(sum(st * w for st, w in pairs) / tw, k), 1) | |
| 447 | + return out | |
| 448 | + | |
| 449 | + | |
| 417 | 450 | def _f(v: Any) -> float | None: |
| 418 | 451 | try: |
| 419 | 452 | return None if v is None else float(v) |
modified
apps/web/src/app/(site)/methodology/page.tsx
+1 −1
@@ -17,7 +17,7 @@ export default async function MethodologyPage() { | ||
| 17 | 17 | return ( |
| 18 | 18 | <div className="pb-8"> |
| 19 | 19 | <header className="pt-6 pb-4"> |
| 20 | − <p className="label">Methodology {m ? <span className="num normal-case tracking-normal text-ink-3">· config v{m.version} · updated {m.updated_at.slice(0, 10)}</span> : <span className="text-warn">· live config unavailable, showing documented defaults</span>}</p> | |
| 20 | + <p className="label">Methodology {m ? <span className="num normal-case tracking-normal text-ink-3">· config v{m.version} · {m.updated_at ? `updated ${m.updated_at.slice(0, 10)}` : `source: ${m.source ?? "file"}`}</span> : <span className="text-warn">· live config unavailable, showing documented defaults</span>}</p> | |
| 21 | 21 | <h1 className="mt-1 text-[26px] font-medium tracking-tight sm:text-[32px]">How the pressure is computed</h1> |
| 22 | 22 | <p className="mt-2 max-w-[780px] text-[14px] leading-relaxed text-ink-2">The Global Internet Pressure Index is a composite observability index between 0 and 100 describing how stressed, unstable, congested, degraded or abnormal the public Internet currently is. It is not scientific truth and it is not an uptime percentage: it is a baseline-relative synthesis of independent telemetry, and every number on this site decomposes into the signals that produced it.</p> |
| 23 | 23 | </header> |
modified
apps/web/src/lib/types.ts
+2 −1
@@ -536,7 +536,8 @@ export interface Methodology { | ||
| 536 | 536 | events?: Record<string, number>; |
| 537 | 537 | fronts?: Record<string, number>; |
| 538 | 538 | version: number; |
| 539 | − updated_at: string; | |
| 539 | + updated_at: string | null; | |
| 540 | + source?: string; | |
| 540 | 541 | } |
| 541 | 542 | |
| 542 | 543 | export interface SearchResult { |
| 543 | 544 | |