web/api: null-safe pages, per-target pressure, matrix z-scores, stricter affected regions
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +67 −5
modified
apps/api/src/internetpressure/api/public.py
+30 −2
@@ -345,6 +345,7 @@ async def service(slug: str) -> dict[str, Any]: | |||||
| 345 | base = await ch.query_one(f"""SELECT quantileTDigest(0.5)(ttfb_ms) AS ttfb FROM measurements WHERE kind='http' AND target_id IN ({','.join(f"'{_esc(t)}'" for t in tids) or "''"}) | 345 | base = await ch.query_one(f"""SELECT quantileTDigest(0.5)(ttfb_ms) AS ttfb FROM measurements WHERE kind='http' AND target_id IN ({','.join(f"'{_esc(t)}'" for t in tids) or "''"}) | ||
| 346 | AND ts >= now() - INTERVAL 7 DAY AND ts < now() - INTERVAL 10 MINUTE""") if tids else {} | 346 | AND ts >= now() - INTERVAL 7 DAY AND ts < now() - INTERVAL 10 MINUTE""") if tids else {} | ||
| 347 | probes = {p["probe_id"]: p for p in await list_probes()} | 347 | probes = {p["probe_id"]: p for p in await list_probes()} | ||
| 348 | + zbase = await _ttfb_baselines(tids) | ||||
| 348 | matrix = [] | 349 | matrix = [] | ||
| 349 | latest_by_target: dict[str, dict[str, Any]] = {} | 350 | latest_by_target: dict[str, dict[str, Any]] = {} | ||
| 350 | for tid in tids: | 351 | for tid in tids: | ||
@@ -355,7 +356,7 @@ async def service(slug: str) -> dict[str, Any]: | |||||
| 355 | for tid in tids: | 356 | for tid in tids: | ||
| 356 | m = latest_by_target.get(tid, {}).get(f"{pid}|http|") | 357 | m = latest_by_target.get(tid, {}).get(f"{pid}|http|") | ||
| 357 | if m: | 358 | if m: | ||
| 358 | − row["targets"].append({"target_id": tid, "ok": bool(m["ok"]), "ttfb_ms": r1(m.get("ttfb_ms")), "z": None, "ts": m["ts"], | 359 | + row["targets"].append({"target_id": tid, "ok": bool(m["ok"]), "ttfb_ms": r1(m.get("ttfb_ms")), "z": _z_of(zbase.get((pid, tid)), m.get("ttfb_ms")), "ts": m["ts"], | ||
| 359 | "error": m.get("error") or None, "http_status": m.get("http_status")}) | 360 | "error": m.get("error") or None, "http_status": m.get("http_status")}) | ||
| 360 | if row["targets"]: | 361 | if row["targets"]: | ||
| 361 | matrix.append(row) | 362 | matrix.append(row) | ||
@@ -386,6 +387,28 @@ async def service(slug: str) -> dict[str, Any]: | |||||
| 386 | 387 | ||||
| 387 | # ── targets & probes ──────────────────────────────────────────────────────────────────────────────────────────────── | 388 | # ── targets & probes ──────────────────────────────────────────────────────────────────────────────────────────────── | ||
| 388 | 389 | ||||
| 390 | +async def _ttfb_baselines(target_ids: list[str]) -> dict[tuple[str, str], tuple[float, float, float]]: | ||||
| 391 | + """7-day TTFB quartiles per (probe, target) — used for the robust z shown in matrices.""" | ||||
| 392 | + if not target_ids: | ||||
| 393 | + return {} | ||||
| 394 | + ids = ",".join(f"'{_esc(t)}'" for t in target_ids) | ||||
| 395 | + rows = await ch.query(f""" | ||||
| 396 | + SELECT probe_id, target_id, quantilesTDigest(0.25, 0.5, 0.75)(ttfb_ms) AS q FROM measurements | ||||
| 397 | + WHERE kind='http' AND target_id IN ({ids}) AND ts >= now() - INTERVAL 7 DAY AND ts < now() - INTERVAL 10 MINUTE | ||||
| 398 | + GROUP BY probe_id, target_id HAVING count() >= 20 | ||||
| 399 | + """) | ||||
| 400 | + return {(r["probe_id"], r["target_id"]): tuple(float(x) for x in r["q"]) for r in rows if r.get("q") and None not in r["q"]} | ||||
| 401 | + | ||||
| 402 | + | ||||
| 403 | +def _z_of(q: tuple[float, float, float] | None, current: Any) -> float | None: | ||||
| 404 | + if not q or current is None: | ||||
| 405 | + return None | ||||
| 406 | + from ..engine import scoring as S | ||||
| 407 | + | ||||
| 408 | + z = S.robust_z(float(current), q[1], S.mad_from_iqr(q[0], q[2])) | ||||
| 409 | + return r2(z) | ||||
| 410 | + | ||||
| 411 | + | ||||
| 389 | async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any]]: | 412 | async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any]]: | ||
| 390 | if not targets: | 413 | if not targets: | ||
| 391 | return [] | 414 | return [] | ||
@@ -395,12 +418,13 @@ async def _targets_summary(targets: list[dict[str, Any]]) -> list[dict[str, Any] | |||||
| 395 | WHERE kind='http' AND target_id IN ({ids}) AND ts >= now() - INTERVAL 1 HOUR GROUP BY target_id | 418 | WHERE kind='http' AND target_id IN ({ids}) AND ts >= now() - INTERVAL 1 HOUR GROUP BY target_id | ||
| 396 | """) | 419 | """) | ||
| 397 | stats = {r["target_id"]: r for r in rows} | 420 | stats = {r["target_id"]: r for r in rows} | ||
| 421 | + tp = await _live("target_pressure") or {} | ||||
| 398 | out = [] | 422 | out = [] | ||
| 399 | for t in targets: | 423 | for t in targets: | ||
| 400 | s = stats.get(t["target_id"], {}) | 424 | s = stats.get(t["target_id"], {}) | ||
| 401 | out.append({"target_id": t["target_id"], "name": t["name"], "hostname": t["hostname"], "category": t["category"], | 425 | out.append({"target_id": t["target_id"], "name": t["name"], "hostname": t["hostname"], "category": t["category"], | ||
| 402 | "provider": t.get("provider"), "service_id": t.get("service_id"), "country": t.get("country"), "region": t["region"], | 426 | "provider": t.get("provider"), "service_id": t.get("service_id"), "country": t.get("country"), "region": t["region"], | ||
| 403 | − "importance": t["importance"], "tier": t["tier"], "pressure": None, | 427 | + "importance": t["importance"], "tier": t["tier"], "pressure": tp.get(t["target_id"]), | ||
| 404 | "ok_ratio_1h": r3(s.get("ok_ratio")), "ttfb_ms_median_1h": r1(s.get("ttfb"))}) | 428 | "ok_ratio_1h": r3(s.get("ok_ratio")), "ttfb_ms_median_1h": r1(s.get("ttfb"))}) | ||
| 405 | return out | 429 | return out | ||
| 406 | 430 | ||||
@@ -427,6 +451,10 @@ async def target(target_id: str) -> dict[str, Any]: | |||||
| 427 | "dns_ms": r1(m.get("dns_ms")), "tcp_ms": r1(m.get("tcp_ms")), "tls_ms": r1(m.get("tls_ms")), "ttfb_ms": r1(m.get("ttfb_ms")), | 451 | "dns_ms": r1(m.get("dns_ms")), "tcp_ms": r1(m.get("tcp_ms")), "tls_ms": r1(m.get("tls_ms")), "ttfb_ms": r1(m.get("ttfb_ms")), | ||
| 428 | "http_status": m.get("http_status"), "resolved_ip": m.get("resolved_ip") or None, "packet_loss": m.get("packet_loss"), | 452 | "http_status": m.get("http_status"), "resolved_ip": m.get("resolved_ip") or None, "packet_loss": m.get("packet_loss"), | ||
| 429 | "rtt_avg_ms": r1(m.get("rtt_avg_ms")), "dns_rcode": m.get("dns_rcode") or None, "dns_answers": m.get("dns_answers"), "z": None}) | 453 | "rtt_avg_ms": r1(m.get("rtt_avg_ms")), "dns_rcode": m.get("dns_rcode") or None, "dns_answers": m.get("dns_answers"), "z": None}) | ||
| 454 | + zb = await _ttfb_baselines([target_id]) | ||||
| 455 | + for m in latest: | ||||
| 456 | + if m["kind"] == "http": | ||||
| 457 | + m["z"] = _z_of(zb.get((m["probe_id"], target_id)), m.get("ttfb_ms")) | ||||
| 430 | latest.sort(key=lambda x: (x["probe_id"], x["kind"], x["resolver"] or "")) | 458 | latest.sort(key=lambda x: (x["probe_id"], x["kind"], x["resolver"] or "")) | ||
| 431 | series = await ch.query(f""" | 459 | series = await ch.query(f""" | ||
| 432 | SELECT toStartOfFifteenMinutes(ts) AS b, quantileTDigest(0.5)(ttfb_ms) AS ttfb, avg(ok) AS ok_ratio FROM measurements | 460 | SELECT toStartOfFifteenMinutes(ts) AS b, quantileTDigest(0.5)(ttfb_ms) AS ttfb, avg(ok) AS ok_ratio FROM measurements | ||
modified
apps/api/src/internetpressure/engine/loop.py
+34 −1
@@ -204,7 +204,7 @@ class Engine: | |||||
| 204 | vs = next((v for v in vendor_rows if v["service_slug"] == slug), None) | 204 | vs = next((v for v in vendor_rows if v["service_slug"] == slug), None) | ||
| 205 | pr = sc.pressure if allowed else None | 205 | pr = sc.pressure if allowed else None | ||
| 206 | lv, ll = cfg.level_for(pr) | 206 | lv, ll = cfg.level_for(pr) | ||
| 207 | − affected = sorted({s.tags.src_region for s in buckets.get(("service", slug), []) if s.pair.stress >= 0.4 and s.tags.probe_id != "*"}) | 207 | + affected = _affected_regions(buckets.get(("service", slug), []), sc.pressure if allowed else None) | ||
| 208 | services_out.append({ | 208 | services_out.append({ | ||
| 209 | "slug": slug, "name": svc["name"], "category": svc.get("category"), "importance": svc.get("importance"), | 209 | "slug": slug, "name": svc["name"], "category": svc.get("category"), "importance": svc.get("importance"), | ||
| 210 | "pressure": r1(pr), "level": lv, "level_label": ll, "confidence": sc.confidence, | 210 | "pressure": r1(pr), "level": lv, "level_label": ll, "confidence": sc.confidence, | ||
@@ -256,6 +256,8 @@ class Engine: | |||||
| 256 | route_changes_1h = sum(1 for s in sigs if s.signal_id == "route_change_rate" and (s.pair.current or 0) >= 1.0) | 256 | route_changes_1h = sum(1 for s in sigs if s.signal_id == "route_change_rate" and (s.pair.current or 0) >= 1.0) | ||
| 257 | extra = {"dns_failures_per_min": round(dns_fail_2m / max(1.0, window / 60.0), 1), "route_changes_per_min": round(route_changes_1h / 60.0, 2)} | 257 | extra = {"dns_failures_per_min": round(dns_fail_2m / max(1.0, window / 60.0), 1), "route_changes_per_min": round(route_changes_1h / 60.0, 2)} | ||
| 258 | targets_by_asn = {k[1]: sorted({s.tags.target_id for s in buckets[k]}) for k in asn_keys} | 258 | targets_by_asn = {k[1]: sorted({s.tags.target_id for s in buckets[k]}) for k in asn_keys} | ||
| 259 | + target_pressure = _target_pressure(sigs, float(eng.get("saturation_k", 1.2))) if allowed else {} | ||||
| 260 | + await rds.set_json("ip:live:target_pressure", target_pressure) | ||||
| 259 | await rds.set_json("ip:live:latency", lat) | 261 | await rds.set_json("ip:live:latency", lat) | ||
| 260 | await rds.set_json("ip:live:extra", extra) | 262 | await rds.set_json("ip:live:extra", extra) | ||
| 261 | await rds.set_json("ip:live:targets_by_asn", targets_by_asn) | 263 | await rds.set_json("ip:live:targets_by_asn", targets_by_asn) | ||
@@ -414,6 +416,37 @@ class Engine: | |||||
| 414 | log.warning("history insert failed: %s", exc) | 416 | log.warning("history insert failed: %s", exc) | ||
| 415 | 417 | ||||
| 416 | 418 | ||||
| 419 | +def _affected_regions(sigs, pressure: float | None) -> list[str]: # type: ignore[no-untyped-def] | ||||
| 420 | + """Probe regions whose importance-weighted mean stress toward this service is ≥ 0.3 — only when the service | ||||
| 421 | + itself is at least 'normal-high' (≥ 20), so a single noisy pair never flags a region.""" | ||||
| 422 | + if pressure is None or pressure < 20: | ||||
| 423 | + return [] | ||||
| 424 | + acc: dict[str, list[float]] = {} | ||||
| 425 | + for s in sigs: | ||||
| 426 | + if s.tags.probe_id == "*": | ||||
| 427 | + continue | ||||
| 428 | + acc.setdefault(s.tags.src_region, []).append((s.pair.stress, s.pair.weight)) # type: ignore[arg-type] | ||||
| 429 | + out = [] | ||||
| 430 | + for reg, pairs in acc.items(): | ||||
| 431 | + tw = sum(w for _, w in pairs) | ||||
| 432 | + if tw > 0 and sum(st * w for st, w in pairs) / tw >= 0.3: | ||||
| 433 | + out.append(reg) | ||||
| 434 | + return sorted(out) | ||||
| 435 | + | ||||
| 436 | + | ||||
| 437 | +def _target_pressure(sigs, k: float) -> dict[str, float]: # type: ignore[no-untyped-def] | ||||
| 438 | + """Per-target pressure = saturated importance/coverage-weighted mean stress of all its pair signals.""" | ||||
| 439 | + acc: dict[str, list[tuple[float, float]]] = {} | ||||
| 440 | + for s in sigs: | ||||
| 441 | + acc.setdefault(s.tags.target_id, []).append((s.pair.stress, s.pair.weight)) | ||||
| 442 | + out: dict[str, float] = {} | ||||
| 443 | + for tid, pairs in acc.items(): | ||||
| 444 | + tw = sum(w for _, w in pairs) | ||||
| 445 | + if tw > 0: | ||||
| 446 | + out[tid] = round(S.saturate(sum(st * w for st, w in pairs) / tw, k), 1) | ||||
| 447 | + return out | ||||
| 448 | + | ||||
| 449 | + | ||||
| 417 | def _f(v: Any) -> float | None: | 450 | def _f(v: Any) -> float | None: | ||
| 418 | try: | 451 | try: | ||
| 419 | return None if v is None else float(v) | 452 | return None if v is None else float(v) | ||
modified
apps/web/src/app/(site)/methodology/page.tsx
+1 −1
@@ -17,7 +17,7 @@ export default async function MethodologyPage() { | |||||
| 17 | return ( | 17 | return ( | ||
| 18 | <div className="pb-8"> | 18 | <div className="pb-8"> | ||
| 19 | <header className="pt-6 pb-4"> | 19 | <header className="pt-6 pb-4"> | ||
| 20 | − <p className="label">Methodology {m ? <span className="num normal-case tracking-normal text-ink-3">· config v{m.version} · updated {m.updated_at.slice(0, 10)}</span> : <span className="text-warn">· live config unavailable, showing documented defaults</span>}</p> | 20 | + <p className="label">Methodology {m ? <span className="num normal-case tracking-normal text-ink-3">· config v{m.version} · {m.updated_at ? `updated ${m.updated_at.slice(0, 10)}` : `source: ${m.source ?? "file"}`}</span> : <span className="text-warn">· live config unavailable, showing documented defaults</span>}</p> | ||
| 21 | <h1 className="mt-1 text-[26px] font-medium tracking-tight sm:text-[32px]">How the pressure is computed</h1> | 21 | <h1 className="mt-1 text-[26px] font-medium tracking-tight sm:text-[32px]">How the pressure is computed</h1> | ||
| 22 | <p className="mt-2 max-w-[780px] text-[14px] leading-relaxed text-ink-2">The Global Internet Pressure Index is a composite observability index between 0 and 100 describing how stressed, unstable, congested, degraded or abnormal the public Internet currently is. It is not scientific truth and it is not an uptime percentage: it is a baseline-relative synthesis of independent telemetry, and every number on this site decomposes into the signals that produced it.</p> | 22 | <p className="mt-2 max-w-[780px] text-[14px] leading-relaxed text-ink-2">The Global Internet Pressure Index is a composite observability index between 0 and 100 describing how stressed, unstable, congested, degraded or abnormal the public Internet currently is. It is not scientific truth and it is not an uptime percentage: it is a baseline-relative synthesis of independent telemetry, and every number on this site decomposes into the signals that produced it.</p> | ||
| 23 | </header> | 23 | </header> | ||
modified
apps/web/src/lib/types.ts
+2 −1
@@ -536,7 +536,8 @@ export interface Methodology { | |||||
| 536 | events?: Record<string, number>; | 536 | events?: Record<string, number>; | ||
| 537 | fronts?: Record<string, number>; | 537 | fronts?: Record<string, number>; | ||
| 538 | version: number; | 538 | version: number; | ||
| 539 | − updated_at: string; | 539 | + updated_at: string | null; | ||
| 540 | + source?: string; | ||||
| 540 | } | 541 | } | ||
| 541 | 542 | ||||
| 542 | export interface SearchResult { | 543 | export interface SearchResult { | ||
| 543 | 544 | ||||