SPB Git

Repository Public

net.ts 6.5 KB 103 lines typescript

1/**2 * Outbound HTTP for source checks and "import from a link". The URL is untrusted input, so:3 * https only, the host must resolve to public addresses only (no loopback, private, link-local, CGNAT, multicast,4 * metadata ranges — the cluster's LAN, WireGuard and Tailscale networks are all excluded), every redirect hop is5 * re-checked, the body is capped, and no cookie or credential is ever sent.6 */7import nodeDns from "node:dns";8import dns from "node:dns/promises";9import https from "node:https";10import net from "node:net";1112export class NetError extends Error { constructor(public code: string, message: string) { super(message); } }1314function isPublicV4(ip: string): boolean {15  const [a, b] = ip.split(".").map(Number);16  if (a === 0 || a === 10 || a === 127 || a >= 224) return false;17  if (a === 100 && b >= 64 && b <= 127) return false; // CGNAT / Tailscale18  if (a === 169 && b === 254) return false;19  if (a === 172 && b >= 16 && b <= 31) return false;20  if (a === 192 && (b === 168 || b === 0)) return false;21  if (a === 198 && (b === 18 || b === 19)) return false;22  return true;23}24export function isPublicAddress(ip: string): boolean {25  if (net.isIPv4(ip)) return isPublicV4(ip);26  if (!net.isIPv6(ip)) return false;27  const v = ip.toLowerCase();28  const mapped = v.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);29  if (mapped) return isPublicV4(mapped[1]);30  if (v === "::" || v === "::1") return false;31  return !/^(fc|fd|fe8|fe9|fea|feb|ff)/.test(v) && !v.startsWith("::ffff:") && !v.startsWith("64:ff9b:");32}3334export async function assertPublicUrl(raw: string): Promise<URL> {35  let u: URL;36  try { u = new URL(raw); } catch { throw new NetError("bad_url", "Not a URL"); }37  if (u.protocol !== "https:") throw new NetError("bad_url", "Only https:// links are fetched");38  if (u.username || u.password) throw new NetError("bad_url", "Links with credentials are refused");39  if (u.port && u.port !== "443") throw new NetError("bad_url", "Only the default https port is allowed");40  const host = u.hostname.replace(/^\[|\]$/g, "");41  const addrs = net.isIP(host) ? [host] : (await dns.lookup(host, { all: true }).catch(() => { throw new NetError("dns", "Host not found"); })).map((a) => a.address);42  if (!addrs.length || !addrs.every(isPublicAddress)) throw new NetError("private_address", "This host does not resolve to a public address");43  return u;44}4546export interface Fetched { status: number; url: string; contentType: string; body: Buffer }4748/**49 * Name resolution used by the connection itself: only public addresses are ever handed to the socket. Because the50 * check happens at connect time, a host cannot pass the first check and then answer with a private address51 * (DNS rebinding).52 */53const publicLookup: net.LookupFunction = (hostname, options, callback) => {54  nodeDns.lookup(hostname, { all: true }, (err, addresses) => {55    if (err) return callback(err, "", 4);56    const ok = addresses.filter((a) => isPublicAddress(a.address));57    if (!ok.length || ok.length !== addresses.length) return callback(Object.assign(new Error("private address"), { code: "EPRIVATE" }), "", 4);58    if ((options as { all?: boolean }).all) (callback as unknown as (e: null, a: typeof ok) => void)(null, ok);59    else callback(null, ok[0].address, ok[0].family);60  });61};6263function requestOnce(url: URL, opts: { method: "GET" | "HEAD"; accept: string; timeoutMs: number; maxBytes: number }): Promise<{ status: number; location: string | null; contentType: string; body: Buffer }> {64  return new Promise((resolve, reject) => {65    const req = https.request(url, { method: opts.method, lookup: publicLookup, timeout: opts.timeoutMs, headers: { "User-Agent": "SheetBot/1.0 (reference checker)", Accept: opts.accept, "Accept-Encoding": "identity" } }, (res) => {66      const status = res.statusCode ?? 0, location = res.headers.location ?? null, contentType = String(res.headers["content-type"] ?? "");67      if ((status >= 300 && status < 400) || opts.method === "HEAD") { res.resume(); return resolve({ status, location, contentType, body: Buffer.alloc(0) }); }68      const chunks: Buffer[] = []; let size = 0;69      res.on("data", (c: Buffer) => { size += c.length; if (size > opts.maxBytes) { req.destroy(new NetError("too_large", "The page is too large")); return; } chunks.push(c); });70      res.on("end", () => resolve({ status, location, contentType, body: Buffer.concat(chunks) }));71      res.on("error", () => reject(new NetError("network", "Network error")));72    });73    req.on("timeout", () => req.destroy(new NetError("network", "Timed out")));74    req.on("error", (e: NodeJS.ErrnoException) => reject(e instanceof NetError ? e : e.code === "EPRIVATE" ? new NetError("private_address", "This host does not resolve to a public address") : e.code === "ENOTFOUND" ? new NetError("dns", "Host not found") : new NetError("network", "Network error")));75    req.end();76  });77}7879/** GET with manual, re-validated redirects. Never throws on HTTP status; throws NetError on policy or network failure. */80export async function safeFetch(raw: string, opts: { maxBytes?: number; timeoutMs?: number; accept?: string; method?: "GET" | "HEAD" } = {}): Promise<Fetched> {81  let url = await assertPublicUrl(raw);82  for (let hop = 0; hop < 5; hop++) {83    const r = await requestOnce(url, { method: opts.method ?? "GET", accept: opts.accept ?? "*/*", timeoutMs: opts.timeoutMs ?? 15_000, maxBytes: opts.maxBytes ?? 2_000_000 });84    if (r.status >= 300 && r.status < 400) {85      if (!r.location) throw new NetError("redirect", "Redirect without a location");86      url = await assertPublicUrl(new URL(r.location, url).toString());87      continue;88    }89    return { status: r.status, url: url.toString(), contentType: r.contentType, body: r.body };90  }91  throw new NetError("redirect", "Too many redirects");92}9394/** Readable text of an HTML page: scripts, styles and tags removed. Good enough to hand to the model as source material. */95export function htmlToText(html: string): { title: string; text: string } {96  const title = (html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? "").replace(/\s+/g, " ").trim().slice(0, 300);97  const text = html.replace(/<(script|style|noscript|svg|template|head)\b[\s\S]*?<\/\1>/gi, " ").replace(/<!--[\s\S]*?-->/g, " ")98    .replace(/<\/(p|div|section|article|li|h[1-6]|tr|br)\s*>/gi, "\n").replace(/<br\s*\/?>/gi, "\n").replace(/<[^>]+>/g, " ")99    .replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'")100    .replace(/[ \t]+/g, " ").replace(/ ?\n ?/g, "\n").replace(/\n{3,}/g, "\n\n").trim();101  return { title, text };102}103
Files main
Menu

Theme