spb/internetpressure

1#!/usr/bin/env python32"""Validate and summarise the target registry (targets.yaml + fragments/*.yaml, services.yaml + services-fragments/*.yaml).34    python3 scripts/registry-check.py            # report + exit 1 on errors5    python3 scripts/registry-check.py --fix      # also normalise: tier ≥ 3 non-infra targets → checks [http]; numeric ids → str6"""78from __future__ import annotations910import collections11import pathlib12import re13import sys1415import yaml1617ROOT = pathlib.Path(__file__).resolve().parents[1]18T_MAIN = ROOT / "data/targets/targets.yaml"19T_FRAG = sorted((ROOT / "data/targets/fragments").glob("*.yaml"))20S_MAIN = ROOT / "data/seed/services.yaml"21S_FRAG = sorted((ROOT / "data/seed/services-fragments").glob("*.yaml"))22CATS = {"dns", "cdn", "cloud", "developer", "infrastructure", "search", "messaging", "social", "streaming", "commerce",23        "finance", "government", "news", "ai"}24INFRA_CATS = {"dns", "cdn", "cloud", "infrastructure"}25CC_RE = re.compile(r"^[A-Z]{2}$")2627fix = "--fix" in sys.argv28errors: list[str] = []293031def load_targets(path: pathlib.Path) -> list[dict]:32    doc = yaml.safe_load(path.read_text()) or {}33    return doc.get("targets") or []343536targets_by_file = {T_MAIN: load_targets(T_MAIN), **{f: load_targets(f) for f in T_FRAG}}37services = []38for f in [S_MAIN, *S_FRAG]:39    services += (yaml.safe_load(f.read_text()) or {}).get("services") or []40slugs = collections.Counter(s["slug"] for s in services)41dup_slugs = [slug for slug, n in slugs.items() if n > 1]42if dup_slugs:43    print(f"note: {len(dup_slugs)} service slugs defined in several fragments (the seed keeps the first): {', '.join(dup_slugs[:10])}")4445ids: collections.Counter = collections.Counter()46hosts: collections.Counter = collections.Counter()47per_cat: collections.Counter = collections.Counter()48per_tier: collections.Counter = collections.Counter()49per_cc: collections.Counter = collections.Counter()50checks_count: collections.Counter = collections.Counter()51tr = 052total = 053for f, ts in targets_by_file.items():54    for t in ts:55        total += 156        tid = str(t.get("id"))57        ids[tid] += 158        hosts[(t.get("host"), t.get("url"), t.get("ip"))] += 159        cat = t.get("cat")60        if cat not in CATS:61            errors.append(f"{f.name}: {tid} bad cat {cat!r}")62        cc = t.get("cc")63        if cc is not None and not (isinstance(cc, str) and CC_RE.match(cc)):64            errors.append(f"{f.name}: {tid} bad cc {cc!r}")65        tier = int(t.get("tier", 2))66        if tier not in (1, 2, 3, 4):67            errors.append(f"{f.name}: {tid} bad tier {tier}")68        imp = int(t.get("imp", 3))69        if imp not in (1, 2, 3, 4, 5):70            errors.append(f"{f.name}: {tid} bad imp {imp}")71        svc = t.get("svc")72        if svc and svc not in slugs:73            errors.append(f"{f.name}: {tid} unknown svc {svc}")74        checks = t.get("checks", ["http", "dns", "ping"])75        if fix:76            if not isinstance(t.get("id"), str):77                t["id"] = tid78            if tier >= 3 and cat not in INFRA_CATS and checks != ["http"]:79                t["checks"] = ["http"]80                checks = ["http"]81        per_cat[cat] += 182        per_tier[tier] += 183        per_cc[cc or "global"] += 184        for c in checks:85            checks_count[c] += 186        if t.get("tr"):87            tr += 188for tid, n in ids.items():89    if n > 1:90        errors.append(f"duplicate target id {tid} ×{n}")91dup_hosts = sum(n - 1 for n in hosts.values() if n > 1)92if dup_hosts:93    print(f"note: {dup_hosts} cross-fragment duplicate host/url entries (the seed keeps the first occurrence)")9495# probe load estimate (per probe, per second) with the configured cadence96cfg = yaml.safe_load((ROOT / "packages/config/pressure.yaml").read_text())["scheduler"]97tiers = {int(k): int(v) for k, v in cfg["tiers"].items()}98http_ps = sum(1 / tiers.get(int(t.get("tier", 2)), 300) for ts in targets_by_file.values() for t in ts if "http" in t.get("checks", ["http", "dns", "ping"]))99dns_ps = sum(4 / cfg["dns_every"] for ts in targets_by_file.values() for t in ts if "dns" in t.get("checks", ["http", "dns", "ping"]))100ping_ps = sum(1 / cfg["ping_every"] for ts in targets_by_file.values() for t in ts if "ping" in t.get("checks", ["http", "dns", "ping"]))101102print(f"targets: {total}  (main {len(targets_by_file[T_MAIN])} + fragments {total - len(targets_by_file[T_MAIN])})")103print(f"services: {len(services)}  with status connector: {sum(1 for s in services if s.get('status'))}")104print("per category:", dict(per_cat.most_common()))105print("per tier:", dict(sorted(per_tier.items())))106print(f"countries covered: {len(per_cc) - 1}   traceroute targets: {tr}")107print("checks:", dict(checks_count))108print(f"estimated load per probe: http {http_ps:.1f}/s · dns {dns_ps:.1f}/s · ping {ping_ps:.1f}/s → {http_ps + dns_ps + ping_ps:.1f} measurements/s "109      f"(×8 probes ≈ {(http_ps + dns_ps + ping_ps) * 8 * 86400 / 1e6:.0f} M rows/day)")110if fix:111    for f, ts in targets_by_file.items():112        doc = yaml.safe_load(f.read_text()) or {}113        doc["targets"] = ts114        # keep the compact flow style: dump one entry per line115        lines = [f"# {f.name} — normalised by scripts/registry-check.py --fix", "targets:"]116        if f == T_MAIN and doc.get("defaults"):117            lines.insert(1, "defaults: " + yaml.safe_dump(doc["defaults"], default_flow_style=True, width=10_000).strip())118        for t in ts:119            lines.append("  - " + yaml.safe_dump(t, default_flow_style=True, width=10_000, allow_unicode=True, sort_keys=False).strip())120        f.write_text("\n".join(lines) + "\n")121    print("normalised files written")122if errors:123    print(f"\n{len(errors)} error(s):")124    for e in errors[:60]:125        print(" -", e)126    sys.exit(1)127print("OK")128