TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { z } from "zod";2import { TIERS } from "./taxonomy";34/**5 * Coverage universes (`config/coverage/*.yaml`). A universe is the list of organizations WebSensor *should*6 * observe for a sector — the denominator of the Global Observation Coverage Score. The same files feed the7 * Source Factory: every member without a monitored domain becomes an expansion seed.8 *9 * One file per sector:10 *11 * sector: ai12 * label: Artificial intelligence13 * weight: 1.2 # weight of the sector in the global score (default 1)14 * categories: [ai, technology] # default categories applied to members (registry vocabulary)15 * tier: B # default tier for factory-created sources16 * universes:17 * - key: frontier-labs18 * label: Frontier labs & model providers19 * members:20 * - { name: OpenAI, domain: openai.com, country: US, importance: 3, hints: { github_org: openai, hf_author: openai } }21 */22export const memberHintUrlSchema = z.object({23 url: z.string().url(),24 /** what the page is: news | press | blog | ir | filings | changelog | releases | security | pricing | legal | careers | docs | api | status | models | data | other */25 kind: z.string().default("other"),26 connector: z.string().optional(),27 type: z.string().optional(),28 tier: z.enum(TIERS).optional(),29 config: z.record(z.string(), z.unknown()).optional(),30 name: z.string().optional(),31});3233export const memberHintsSchema = z.object({34 /** SEC CIK (digits, any length) → EDGAR submissions sensor */35 cik: z.union([z.string(), z.number()]).optional(),36 ticker: z.string().optional(),37 exchange: z.string().optional(),38 /** GitHub organization → most active repositories' releases */39 github_org: z.string().optional(),40 /** explicit repositories `owner/name` */41 github_repos: z.array(z.string()).optional(),42 /** Hugging Face author/organization → model list sensor */43 hf_author: z.string().optional(),44 /** status page root (Atlassian Statuspage, Instatus, incident.io, Status.io) */45 status_url: z.string().url().optional(),46 /** additional hosts that belong to the organization (blog.example.org, investors.example.org…) */47 hosts: z.array(z.string()).optional(),48 /** explicit candidate URLs (validated by the factory before becoming sensors) */49 urls: z.array(memberHintUrlSchema).optional(),50 /** probe robots/TLS/DNS/headers posture sensors (default: only for importance 3) */51 posture: z.boolean().optional(),52 /** Wikidata QID, for provenance */53 wikidata: z.string().optional(),54});5556export const coverageMemberSchema = z.object({57 name: z.string().min(1),58 /** bare registrable domain, lower-case, no scheme, no `www.` */59 domain: z60 .string()61 .min(3)62 .transform((d) => d.trim().toLowerCase().replace(/^https?:\/\//, "").replace(/^www\./, "").replace(/\/.*$/, "")),63 country: z.string().regex(/^[A-Z]{2,3}$/).optional(),64 language: z.string().regex(/^[a-z]{2}$/).optional(),65 /** 1 = long tail · 2 = notable (default) · 3 = systemically important */66 importance: z.union([z.literal(1), z.literal(2), z.literal(3)]).default(2),67 aliases: z.array(z.string()).default([]),68 categories: z.array(z.string()).optional(),69 tier: z.enum(TIERS).optional(),70 first_party: z.boolean().optional(),71 hints: memberHintsSchema.default({}),72 note: z.string().optional(),73});74export type CoverageMember = z.infer<typeof coverageMemberSchema>;7576export const coverageUniverseSchema = z.object({77 key: z.string().regex(/^[a-z0-9][a-z0-9-]*$/),78 label: z.string(),79 description: z.string().optional(),80 /** provenance of the list (URL of the index constituents page, Wikidata query…) */81 provenance: z.string().optional(),82 members: z.array(coverageMemberSchema),83});84export type CoverageUniverse = z.infer<typeof coverageUniverseSchema>;8586export const coverageSectorSchema = z.object({87 sector: z.string().regex(/^[a-z0-9][a-z0-9-]*$/),88 label: z.string(),89 description: z.string().optional(),90 weight: z.number().positive().default(1),91 categories: z.array(z.string()).default([]),92 tier: z.enum(TIERS).default("B"),93 universes: z.array(coverageUniverseSchema),94});95export type CoverageSector = z.infer<typeof coverageSectorSchema>;9697/**98 * Registrable domain (eTLD+1) with a compact public-suffix list covering the multi-label suffixes that99 * matter for official sources (governments, universities, companies). Good enough to match a sensor host100 * (`ir.example.com`, `www.gov.uk`) to a universe member; not a full PSL.101 */102const MULTI_SUFFIX = new Set([103 "co.uk", "org.uk", "gov.uk", "ac.uk", "nhs.uk", "police.uk", "ltd.uk", "plc.uk", "me.uk", "net.uk", "sch.uk",104 "com.au", "net.au", "org.au", "gov.au", "edu.au", "id.au", "asn.au",105 "gc.ca", "qc.ca", "on.ca", "bc.ca", "ab.ca", "mb.ca", "sk.ca", "ns.ca", "nb.ca", "nl.ca", "pe.ca", "nt.ca", "nu.ca", "yk.ca", "gouv.qc.ca",106 "co.jp", "ne.jp", "or.jp", "go.jp", "ac.jp", "ad.jp", "lg.jp", "ed.jp",107 "co.kr", "go.kr", "or.kr", "ac.kr", "re.kr", "ne.kr",108 "com.cn", "gov.cn", "org.cn", "edu.cn", "net.cn", "ac.cn",109 "com.hk", "gov.hk", "org.hk", "edu.hk",110 "com.tw", "gov.tw", "org.tw", "edu.tw",111 "com.sg", "gov.sg", "org.sg", "edu.sg",112 "co.in", "gov.in", "nic.in", "org.in", "ac.in", "net.in", "res.in", "edu.in",113 "com.br", "gov.br", "org.br", "edu.br", "net.br", "leg.br", "jus.br", "mil.br",114 "com.mx", "gob.mx", "org.mx", "edu.mx",115 "com.ar", "gob.ar", "gov.ar", "org.ar", "edu.ar",116 "gob.cl", "gov.cl", "cl", "gob.pe", "gob.es", "gov.co", "gob.ec", "gob.bo", "gub.uy", "gob.pa", "gob.do", "gob.gt", "gob.hn", "gob.sv", "gob.ni", "gob.cu", "gob.ve", "gob.py",117 "co.za", "gov.za", "org.za", "ac.za", "net.za",118 "co.nz", "govt.nz", "org.nz", "ac.nz", "net.nz",119 "com.tr", "gov.tr", "org.tr", "edu.tr",120 "co.il", "gov.il", "org.il", "ac.il", "muni.il",121 "gov.ae", "ac.ae", "co.ae",122 "gov.sa", "com.sa", "edu.sa", "org.sa",123 "gov.qa", "com.qa", "gov.kw", "gov.bh", "gov.om", "gov.jo", "gov.lb", "gov.eg", "com.eg", "gov.ma", "gov.dz", "gov.tn",124 "gov.ng", "com.ng", "org.ng", "go.ke", "co.ke", "or.ke", "ac.ke", "gov.gh", "com.gh", "gov.et", "gov.rw", "go.tz", "co.tz", "go.ug", "co.ug", "gov.zm", "gov.zw", "gov.bw", "gov.na", "gov.mz", "gov.sn", "gouv.sn", "gouv.ci", "gouv.bf", "gouv.ml", "gouv.ne", "gouv.tg", "gouv.bj", "gouv.cm", "gov.mu",125 "com.my", "gov.my", "org.my", "edu.my", "go.th", "co.th", "or.th", "ac.th", "gov.vn", "com.vn", "edu.vn", "gov.ph", "com.ph", "org.ph", "edu.ph", "go.id", "co.id", "or.id", "ac.id", "gov.bd", "com.bd", "gov.pk", "com.pk", "edu.pk", "gov.lk", "gov.np", "gov.kh", "gov.la", "gov.mm", "gov.bn", "gov.mn",126 "gov.kz", "gov.uz", "gov.kg", "gov.tj", "gov.tm", "gov.az", "gov.ge", "gov.am", "gov.by", "gov.ua", "com.ua", "org.ua", "edu.ua", "gov.md", "gov.ru", "com.ru", "org.ru", "edu.ru", "ac.ru",127 "gov.pl", "com.pl", "org.pl", "edu.pl", "gov.cz", "gov.sk", "gov.hu", "gov.ro", "gov.bg", "gov.hr", "gov.si", "gov.rs", "gov.ba", "gov.mk", "gov.al", "gov.me", "gov.lv", "gov.lt", "gov.ee", "gov.gr", "gov.cy", "gov.mt", "gov.ie", "gov.pt", "gov.it", "gouv.fr", "gov.lu", "gov.be", "gov.nl", "gov.is", "gov.li",128 "com.es", "org.es", "edu.es", "gob.es", "com.pt", "edu.pt", "co.it", "edu.it", "com.gr", "edu.gr", "com.pl", "com.ro", "com.tr",129 "europa.eu", "canada.ca", "admin.ch", "bund.de", "edu.mo", "gov.mo", "ac.ir", "gov.ir", "edu.qa", "edu.lb", "edu.bn", "edu.kw", "edu.jo", "edu.om", "edu.eg", "ac.za", "edu.gh", "edu.ng", "ac.ke", "ac.tz", "ac.ug", "edu.et", "com.co", "edu.co", "org.co", "net.co", "gov.co", "gov.ky", "gov.bm", "gov.je", "gov.gg",130 "gov.uk", "parliament.uk", "judiciary.uk", "mod.uk", "nhs.uk", "royal.uk",131 "state.gov", "mil", "fed.us", "nsn.us", "k12.us", "lib.us", "dni.us", "isa.us", "kids.us",132 "ac.at", "co.at", "gv.at", "or.at", "ac.be", "co.nl", "ac.nz", "ac.uk", "ac.jp", "edu.au", "edu.sg", "edu.hk",133 "org.au", "com.au", "co.ke",134]);135136/** Generic second-level labels under country-code TLDs (`gov.py`, `go.cr`, `gouv.ht`, `ac.ir`, `co.zw`…): institutions there are distinct organizations. */137const CC_SECOND_LEVEL = new Set(["gov", "gouv", "gob", "go", "gv", "govt", "gub", "edu", "ac", "co", "com", "org", "net", "or", "ne", "mil", "sch", "nic", "res", "ltd", "plc", "me", "id", "asn", "lg", "ed", "ad", "re", "muni", "jus", "leg", "int", "info", "biz", "web", "pro", "nom", "ind", "firm", "gen", "per", "mod", "nhs", "police", "parliament", "judiciary", "govmu"]);138139export function registrableDomain(host: string): string {140 const h = host.toLowerCase().replace(/\.$/, "").replace(/^www\./, "");141 const parts = h.split(".");142 if (parts.length <= 2) return h;143 const last2 = parts.slice(-2).join(".");144 const last3 = parts.slice(-3).join(".");145 if (MULTI_SUFFIX.has(last3) && parts.length >= 4) return parts.slice(-4).join(".");146 if (MULTI_SUFFIX.has(last2)) return last3;147 const tld = parts[parts.length - 1]!;148 if (tld.length === 2 && CC_SECOND_LEVEL.has(parts[parts.length - 2]!)) return last3;149 return last2;150}151152/** Normalize a member/source domain to its comparison key. */153export function coverageKey(domain: string): string {154 return registrableDomain(domain.trim().toLowerCase().replace(/^https?:\/\//, "").replace(/\/.*$/, ""));155}156