TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";2import { join } from "node:path";3import YAML from "yaml";4import { coverageKey, coverageSectorSchema, type CoverageMember, type CoverageSector } from "@websensor/core";5import { db, sql } from "@websensor/db";6import { cached } from "./cache";78/**9 * Global Observation Coverage Score — "how much of the world's high-value public web do we observe?"10 *11 * For every sector the universes in `config/coverage/*.yaml` are the denominator. A member is *covered* when at12 * least one active (non-shadow) sensor lives on its registrable domain (or on a host declared in `hints.hosts`).13 * breadth = Σ w·covered / Σ w (w = member importance 1–3)14 * depth = Σ w·min(1, sensors/5) / Σ w (five sensors per organization = full depth)15 * score = 100 · (0.7·breadth + 0.3·depth)16 * The global score is the sector-weight-weighted mean. Everything is explainable: the per-member table shows17 * which sensors count, what is in shadow and what the Factory is still queuing.18 */19export interface MemberCoverage {20 name: string;21 domain: string;22 country: string | null;23 importance: number;24 universe: string;25 covered: boolean;26 sensors: number;27 shadow: number;28 source_id: string | null;29 seed_status: string | null;30 depth: number;31}32export interface UniverseCoverage {33 key: string;34 label: string;35 provenance: string | null;36 members: number;37 covered: number;38 sensors: number;39 breadth: number;40 depth: number;41 score: number;42}43export interface SectorCoverage {44 sector: string;45 label: string;46 description: string | null;47 weight: number;48 members: number;49 covered: number;50 shadow_members: number;51 queued: number;52 sensors: number;53 breadth: number;54 depth: number;55 score: number;56 universes: UniverseCoverage[];57}5859const COVERAGE_DIR = process.env.WS_COVERAGE_DIR ?? "./config/coverage";6061let fileCache: { at: number; sectors: CoverageSector[]; issues: string[] } | null = null;62export function loadSectors(): { sectors: CoverageSector[]; issues: string[] } {63 if (fileCache && Date.now() - fileCache.at < 60_000) return fileCache;64 const byKey = new Map<string, CoverageSector>();65 const issues: string[] = [];66 if (existsSync(COVERAGE_DIR) && statSync(COVERAGE_DIR).isDirectory()) {67 for (const f of readdirSync(COVERAGE_DIR).filter((x) => /\.ya?ml$/.test(x) && !x.startsWith("_")).sort()) {68 try {69 const parsed = coverageSectorSchema.safeParse(YAML.parse(readFileSync(join(COVERAGE_DIR, f), "utf8")));70 if (!parsed.success) {71 issues.push(`${f}: ${parsed.error.issues[0]?.path.join(".")}: ${parsed.error.issues[0]?.message}`);72 continue;73 }74 const s = parsed.data;75 const prev = byKey.get(s.sector);76 if (prev) prev.universes.push(...s.universes);77 else byKey.set(s.sector, { ...s, universes: [...s.universes] });78 } catch (e) {79 issues.push(`${f}: ${(e as Error).message}`);80 }81 }82 }83 fileCache = { at: Date.now(), sectors: [...byKey.values()], issues };84 return fileCache;85}8687interface Observed {88 /** registrable domain → { sensors (active), shadow, sourceId } */89 byKey: Map<string, { sensors: number; shadow: number; sourceId: string | null }>;90 seeds: Map<string, string>;91}9293async function observed(): Promise<Observed> {94 const [hosts, srcs, seeds] = await Promise.all([95 db.execute<{ host: string; status: string; n: string; source_id: string }>(sql`96 select lower(split_part(split_part(url, '/', 3), ':', 1)) as host, case when status = 'SHADOW' then 'SHADOW' else 'ACTIVE' end as status, count(*)::text as n, min(source_id) as source_id97 from sensors where enabled and url like 'http%' group by 1, 2`).then((r) => r.rows),98 db.execute<{ id: string; domain: string }>(sql`select id, domain from sources where enabled and kind = 'registry'`).then((r) => r.rows),99 db.execute<{ domain: string; status: string }>(sql`select domain, status from factory_seeds`).then((r) => r.rows),100 ]);101 const byKey = new Map<string, { sensors: number; shadow: number; sourceId: string | null }>();102 for (const h of hosts) {103 if (!h.host) continue;104 const k = coverageKey(h.host);105 const cur = byKey.get(k) ?? { sensors: 0, shadow: 0, sourceId: null };106 if (h.status === "SHADOW") cur.shadow += Number(h.n);107 else cur.sensors += Number(h.n);108 cur.sourceId = cur.sourceId ?? h.source_id;109 byKey.set(k, cur);110 }111 for (const s of srcs) {112 const k = coverageKey(s.domain);113 const cur = byKey.get(k) ?? { sensors: 0, shadow: 0, sourceId: null };114 cur.sourceId = cur.sourceId ?? s.id;115 byKey.set(k, cur);116 }117 const seedMap = new Map<string, string>();118 for (const s of seeds) seedMap.set(coverageKey(s.domain), s.status);119 return { byKey, seeds: seedMap };120}121122function memberCoverage(m: CoverageMember, universe: string, obs: Observed): MemberCoverage {123 const keys = [coverageKey(m.domain), ...((m.hints?.hosts ?? []).map((h) => coverageKey(h)))];124 let sensors = 0;125 let shadow = 0;126 let sourceId: string | null = null;127 for (const k of new Set(keys)) {128 const o = obs.byKey.get(k);129 if (!o) continue;130 sensors += o.sensors;131 shadow += o.shadow;132 sourceId = sourceId ?? o.sourceId;133 }134 return { name: m.name, domain: m.domain, country: m.country ?? null, importance: m.importance, universe, covered: sensors > 0, sensors, shadow, source_id: sourceId, seed_status: obs.seeds.get(coverageKey(m.domain)) ?? null, depth: Math.min(1, sensors / 5) };135}136137function aggregate(members: MemberCoverage[]): { breadth: number; depth: number; score: number; covered: number; sensors: number } {138 let w = 0;139 let cov = 0;140 let dep = 0;141 let covered = 0;142 let sensors = 0;143 for (const m of members) {144 w += m.importance;145 if (m.covered) {146 cov += m.importance;147 covered++;148 }149 dep += m.importance * m.depth;150 sensors += m.sensors;151 }152 const breadth = w ? cov / w : 0;153 const depth = w ? dep / w : 0;154 return { breadth: round(breadth), depth: round(depth), score: round(100 * (0.7 * breadth + 0.3 * depth), 1), covered, sensors };155}156const round = (x: number, d = 3): number => Math.round(x * 10 ** d) / 10 ** d;157158export async function coverageSummary(): Promise<Record<string, unknown>> {159 return cached("coverage:summary", 5 * 60_000, async () => {160 const { sectors, issues } = loadSectors();161 const obs = await observed();162 const out: SectorCoverage[] = [];163 let wsum = 0;164 let ssum = 0;165 let members = 0;166 let covered = 0;167 let sensorsTotal = 0;168 for (const s of sectors) {169 const all: MemberCoverage[] = [];170 const universes: UniverseCoverage[] = [];171 const seenDomains = new Set<string>();172 for (const u of s.universes) {173 const ms = u.members.map((m) => memberCoverage(m, u.key, obs));174 const a = aggregate(ms);175 universes.push({ key: u.key, label: u.label, provenance: u.provenance ?? null, members: ms.length, covered: a.covered, sensors: a.sensors, breadth: a.breadth, depth: a.depth, score: a.score });176 for (const m of ms) {177 const k = coverageKey(m.domain);178 if (seenDomains.has(k)) continue;179 seenDomains.add(k);180 all.push(m);181 }182 }183 const a = aggregate(all);184 const sec: SectorCoverage = { sector: s.sector, label: s.label, description: s.description ?? null, weight: s.weight, members: all.length, covered: a.covered, shadow_members: all.filter((m) => !m.covered && m.shadow > 0).length, queued: all.filter((m) => !m.covered && (m.seed_status === "queued" || m.seed_status === "discovering")).length, sensors: a.sensors, breadth: a.breadth, depth: a.depth, score: a.score, universes };185 out.push(sec);186 wsum += s.weight;187 ssum += s.weight * a.score;188 members += all.length;189 covered += a.covered;190 sensorsTotal += a.sensors;191 }192 out.sort((a, b) => b.score - a.score);193 const [monitored, factory] = await Promise.all([194 db.execute<{ sources: string; sensors: string; shadow: string; countries: string }>(sql`select (select count(*) from sources where enabled and kind = 'registry')::text as sources, (select count(*) from sensors where enabled and status <> 'SHADOW')::text as sensors, (select count(*) from sensors where enabled and status = 'SHADOW')::text as shadow, (select count(distinct country) from sources where country is not null and enabled)::text as countries`).then((r) => r.rows[0]),195 db.execute<{ status: string; n: string }>(sql`select status, count(*)::text as n from factory_seeds group by status`).then((r) => Object.fromEntries(r.rows.map((x) => [x.status, Number(x.n)]))),196 ]);197 return {198 global_score: wsum ? round(ssum / wsum, 1) : 0,199 members,200 covered,201 sensors_matched: sensorsTotal,202 sectors: out.map(({ universes: _u, ...rest }) => rest),203 monitored: { sources: Number(monitored?.sources ?? 0), sensors: Number(monitored?.sensors ?? 0), shadow: Number(monitored?.shadow ?? 0), countries: Number(monitored?.countries ?? 0) },204 factory: factory,205 method: "breadth = importance-weighted share of members with ≥ 1 active sensor · depth = importance-weighted min(1, sensors/5) · score = 100·(0.7·breadth + 0.3·depth) · global = sector-weight-weighted mean",206 issues,207 generated_at: new Date().toISOString(),208 };209 });210}211212export async function coverageSector(sector: string): Promise<Record<string, unknown> | null> {213 return cached(`coverage:sector:${sector}`, 5 * 60_000, async () => {214 const { sectors } = loadSectors();215 const s = sectors.find((x) => x.sector === sector);216 if (!s) return null;217 const obs = await observed();218 const universes = s.universes.map((u) => {219 const members = u.members.map((m) => memberCoverage(m, u.key, obs)).sort((a, b) => Number(b.covered) - Number(a.covered) || b.importance - a.importance || b.sensors - a.sensors || a.name.localeCompare(b.name));220 const a = aggregate(members);221 return { key: u.key, label: u.label, description: u.description ?? null, provenance: u.provenance ?? null, members: members.length, covered: a.covered, sensors: a.sensors, breadth: a.breadth, depth: a.depth, score: a.score, items: members };222 });223 const flat = new Map<string, MemberCoverage>();224 for (const u of universes) for (const m of u.items) if (!flat.has(coverageKey(m.domain))) flat.set(coverageKey(m.domain), m);225 const a = aggregate([...flat.values()]);226 const byCountry = new Map<string, { members: number; covered: number }>();227 for (const m of flat.values()) {228 const c = m.country ?? "—";229 const cur = byCountry.get(c) ?? { members: 0, covered: 0 };230 cur.members++;231 if (m.covered) cur.covered++;232 byCountry.set(c, cur);233 }234 return { sector: s.sector, label: s.label, description: s.description ?? null, weight: s.weight, categories: s.categories, members: flat.size, covered: a.covered, sensors: a.sensors, breadth: a.breadth, depth: a.depth, score: a.score, shadow_members: [...flat.values()].filter((m) => !m.covered && m.shadow > 0).length, queued: [...flat.values()].filter((m) => !m.covered && (m.seed_status === "queued" || m.seed_status === "discovering")).length, universes, by_country: [...byCountry.entries()].map(([country, v]) => ({ country, ...v })).sort((x, y) => y.members - x.members), generated_at: new Date().toISOString() };235 });236}237