spb/websensor

1#!/usr/bin/env python32"""Generate config/sources.d/32-web-posture.yaml — `extend: true` entries adding the web-posture3sensor class (robots.txt crawler policy, security.txt, TLS certificate, DNS records, HTTP security4headers, RDAP registration) to the tier S/A organizations of the founding registry.56Usage: python3 scripts/gen-web-posture.py > config/sources.d/32-web-posture.yaml7Then:  node node_modules/tsx/dist/cli.mjs apps/engine/src/validate.ts config/sources.d/32-web-posture.yaml --json /tmp/wp.json8       python3 scripts/prune-fragment.py config/sources.d/32-web-posture.yaml /tmp/wp.json9"""10import sys11from urllib.parse import urlparse1213import yaml1415TIERS = {"S", "A"}16MULTI = {"co.uk", "gc.ca", "gov.uk", "com.au", "co.jp", "org.uk", "ac.uk", "qc.ca", "on.ca", "bc.ca", "ab.ca", "europa.eu"}171819def apex(host: str) -> str:20    parts = host.lower().strip(".").split(".")21    if len(parts) >= 3 and ".".join(parts[-2:]) in MULTI:22        return ".".join(parts[-3:])23    return ".".join(parts[-2:])242526def main() -> None:27    d = yaml.safe_load(open("config/sources.yaml"))28    print("# WebSensor — web-posture sensor class (generated 2026-09-08 by scripts/gen-web-posture.py, then pruned by")29    print("# the live validator). For every tier S/A organization of the founding registry: crawler policy (robots.txt),")30    print("# security.txt, TLS certificate, DNS records, HTTP security headers and RDAP registration — tier D (6–24 h).")31    print("# Changes here are the classic *silent* changes: AI-crawler blocks, CA switches, hosting/DNS migrations,")32    print("# CSP/HSTS regressions, registrar transfers, expiring domains.")33    print("sources:")34    seen_hosts = set()35    for s in d["sources"]:36        if s.get("tier", "B") not in TIERS:37            continue38        home = s.get("homepage") or f"https://{s['domain']}"39        host = urlparse(home).hostname or s["domain"]40        ap = apex(host)41        if host in seen_hosts:42            continue43        seen_hosts.add(host)44        print(f"  - id: {s['id']}")45        print("    extend: true")46        print("    categories: [web-policy]")47        print("    sensors:")48        print(f'      - {{ name: crawler policy (robots.txt), url: "https://{host}/robots.txt", type: FILE, connector: http, tier: D }}')49        print(f'      - {{ name: security.txt, url: "https://{host}/.well-known/security.txt", type: FILE, connector: http, tier: D }}')50        print(f'      - {{ name: tls certificate, url: "tls://{host}", type: TLS, connector: tls, tier: D }}')51        print(f'      - {{ name: dns records, url: "dns://{ap}", type: DNS, connector: dns, tier: D }}')52        print(f'      - {{ name: http security headers, url: "{home.rstrip("/")}/", type: HTTP_HEADERS, connector: headers, tier: D }}')53        print(f'      - {{ name: domain registration (rdap), url: "https://rdap.org/domain/{ap}", type: JSON, connector: rdap, tier: D }}')545556if __name__ == "__main__":57    main()58