TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { canonicalJson, sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { NormalizeError, type WebSensorConnector } from "./types";45/**6 * HTTP response-headers connector (security + infrastructure posture). Performs a GET (HEAD is often7 * refused or answered differently by CDNs) and keeps only the headers that describe policy or stack:8 * HSTS, CSP, frame/content-type/referrer/permissions policies, CORS, cookies flags (names only),9 * server / powered-by / CDN markers, cache policy, alt-svc, reporting endpoints, robots tag.10 * Values that change on every request (dates, request ids, ray ids, ages, ETags) are excluded.11 *12 * Config: { method?: "GET" | "HEAD", include?: [extra header names], exclude?: [names] }13 */14const KEEP = [15 "strict-transport-security",16 "content-security-policy",17 "content-security-policy-report-only",18 "x-frame-options",19 "x-content-type-options",20 "referrer-policy",21 "permissions-policy",22 "cross-origin-opener-policy",23 "cross-origin-embedder-policy",24 "cross-origin-resource-policy",25 "access-control-allow-origin",26 "access-control-allow-methods",27 "access-control-allow-headers",28 "access-control-allow-credentials",29 "server",30 "x-powered-by",31 "via",32 "x-served-by",33 "x-cache",34 "cf-cache-status",35 "x-vercel-cache",36 "x-amz-cf-pop",37 "x-akamai-transformed",38 "x-azure-ref",39 "x-fastly-request-id",40 "cache-control",41 "content-type",42 "content-language",43 "alt-svc",44 "report-to",45 "reporting-endpoints",46 "nel",47 "x-robots-tag",48 "link",49 "vary",50 "content-encoding",51 "x-dns-prefetch-control",52 "expect-ct",53 "x-xss-protection",54 "origin-agent-cluster",55 "accept-ch",56 "critical-ch",57 "x-ua-compatible",58 "www-authenticate",59 "location",60];61const CDN_MARKERS: [RegExp, string][] = [62 [/cloudflare/i, "Cloudflare"],63 [/cloudfront|x-amz-cf/i, "CloudFront"],64 [/akamai/i, "Akamai"],65 [/fastly/i, "Fastly"],66 [/vercel/i, "Vercel"],67 [/netlify/i, "Netlify"],68 [/azure|x-azure/i, "Azure"],69 [/gws|google/i, "Google"],70 [/nginx/i, "nginx"],71 [/apache/i, "Apache"],72 [/microsoft-iis/i, "IIS"],73 [/envoy/i, "Envoy"],74 [/caddy/i, "Caddy"],75 [/openresty/i, "OpenResty"],76 [/litespeed/i, "LiteSpeed"],77 [/imperva|incapsula/i, "Imperva"],78 [/sucuri/i, "Sucuri"],79];8081/** Strip per-request noise inside kept headers (nonces, request ids, ray ids). */82function scrub(name: string, value: string): string {83 let v = value;84 if (name.startsWith("content-security-policy")) v = v.replace(/'nonce-[^']+'/g, "'nonce-…'").replace(/\s+/g, " ").trim();85 if (name === "x-fastly-request-id" || name === "x-served-by" || name === "x-amz-cf-pop" || name === "x-azure-ref") return "present";86 if (name === "link") v = v.replace(/<[^>]*>/g, (m) => m.replace(/[?&](v|ver|hash|_)=[^&>]+/g, ""));87 if (name === "set-cookie") return v;88 return v;89}9091export class HeadersConnector implements WebSensorConnector {92 mode = "json" as const;93 metadata(): ConnectorMetadata {94 return { key: "headers", name: "HTTP response headers", sensorTypes: ["HTTP_HEADERS"], description: "Security & infrastructure headers (HSTS, CSP, CORS, server/CDN markers, cache policy, cookies flags)", version: "1.0.0" };95 }96 async fetch(endpoint: SensorEndpoint): Promise<Observation> {97 const cfg = endpoint.config as { method?: "GET" | "HEAD"; headers?: Record<string, string> };98 const obs = await httpFetchWithRetry(endpoint.id, endpoint.url, { method: cfg.method ?? "GET", headers: cfg.headers, accept: "text/html,application/xhtml+xml,*/*;q=0.8", keepAllHeaders: true, maxBytes: 24 * 1024 * 1024, timeoutMs: 25_000 });99 // The body is irrelevant; keep the observation small (headers live in meta).100 if (obs.body) obs.body = Buffer.from(JSON.stringify({ status: obs.meta.status, finalUrl: obs.meta.finalUrl, headers: obs.meta.headers }));101 else if (!obs.error) obs.body = Buffer.from(JSON.stringify({ status: obs.meta.status, finalUrl: obs.meta.finalUrl, headers: obs.meta.headers }));102 return obs;103 }104 async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {105 if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");106 const j = JSON.parse(obs.body.toString("utf8")) as { status: number; finalUrl: string; headers: Record<string, string> };107 const cfg = endpoint.config as { include?: string[]; exclude?: string[] };108 const keep = new Set([...KEEP, ...(cfg.include ?? []).map((s) => s.toLowerCase())]);109 for (const x of cfg.exclude ?? []) keep.delete(x.toLowerCase());110 const kept: Record<string, string> = {};111 for (const [k, v] of Object.entries(j.headers)) {112 const name = k.toLowerCase();113 if (keep.has(name)) kept[name] = scrub(name, v);114 }115 // Cookies: names + flags only (values are secrets / per-session).116 const cookies = (j.headers["set-cookie"] ?? "")117 .split(/,(?=\s*[A-Za-z0-9_.-]+=)/)118 .map((c) => c.trim())119 .filter(Boolean)120 .map((c) => {121 const [nv, ...attrs] = c.split(";");122 return `${nv!.split("=")[0]!.trim()} [${attrs.map((a) => a.trim().split("=")[0]!.toLowerCase()).filter((a) => /secure|httponly|samesite|partitioned/.test(a)).sort().join(",")}]`;123 })124 .sort();125 const posture = {126 status: j.status,127 finalUrl: j.finalUrl.replace(/\/$/, ""),128 https: j.finalUrl.startsWith("https://"),129 hsts: Boolean(kept["strict-transport-security"]),130 hstsPreload: /preload/i.test(kept["strict-transport-security"] ?? ""),131 csp: Boolean(kept["content-security-policy"]),132 frameProtection: Boolean(kept["x-frame-options"]) || /frame-ancestors/i.test(kept["content-security-policy"] ?? ""),133 nosniff: /nosniff/i.test(kept["x-content-type-options"] ?? ""),134 referrerPolicy: kept["referrer-policy"] ?? null,135 permissionsPolicy: Boolean(kept["permissions-policy"]),136 cors: kept["access-control-allow-origin"] ?? null,137 server: kept.server ?? null,138 poweredBy: kept["x-powered-by"] ?? null,139 stack: CDN_MARKERS.filter(([re]) => re.test(`${kept.server ?? ""} ${kept.via ?? ""} ${kept["x-powered-by"] ?? ""} ${Object.keys(kept).join(" ")}`)).map(([, n]) => n),140 cookies,141 };142 const json = { posture, headers: kept };143 const c = canonicalJson(json);144 const score = [posture.https, posture.hsts, posture.csp, posture.frameProtection, posture.nosniff, Boolean(posture.referrerPolicy), posture.permissionsPolicy].filter(Boolean).length;145 return { mode: "json", json, title: `Headers ${new URL(endpoint.url).hostname}`, rawHash: sha256(obs.body), canonicalHash: sha256(c), semanticHash: simhash(c), extractionConfidence: 1, extra: { securityScore: `${score}/7`, stack: posture.stack, server: posture.server, status: j.status } };146 }147}148