SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
2.7 KB · 83 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2import { lookup } from "node:dns/promises";3import { isIP } from "node:net";4import { err, ok, tendrilError, type Result } from "@tendril/shared";5import { isBlockedIp } from "./ip.js";67const ALLOWED_PORTS = new Set([80, 443, 8080, 8443]);8const DEFAULT_PORT: Readonly<Record<string, number>> = { "http:": 80, "https:": 443 };910export interface SafeTarget {11  readonly url: URL;12  readonly host: string;13  readonly port: number;14  /** All resolved addresses, all verified public. Pin these to defeat DNS rebinding. */15  readonly addresses: readonly string[];16}1718export type Resolver = (host: string) => Promise<readonly string[]>;1920const defaultResolver: Resolver = async (host) => {21  const results = await lookup(host, { all: true });22  return results.map((r) => r.address);23};2425/**26 * Validate a URL for egress (§16.5). Rejects non-http(s) schemes and disallowed27 * ports up front, then resolves DNS and rejects if *any* resolved address is28 * private/loopback/link-local. The returned `addresses` should be pinned by the29 * caller when connecting, so a rebind between validation and fetch cannot slip30 * a private IP through.31 */32export async function validateEgress(33  raw: string,34  resolver: Resolver = defaultResolver,35): Promise<Result<SafeTarget>> {36  let url: URL;37  try {38    url = new URL(raw);39  } catch {40    return err(tendrilError("ERR_INVALID_URL", { details: { url: raw } }));41  }4243  if (url.protocol !== "http:" && url.protocol !== "https:") {44    return err(tendrilError("ERR_INVALID_URL", { details: { scheme: url.protocol } }));45  }4647  const port = url.port !== "" ? Number(url.port) : DEFAULT_PORT[url.protocol] ?? 0;48  if (!ALLOWED_PORTS.has(port)) {49    return err(tendrilError("ERR_SSRF_BLOCKED", { message: "Port not allowed", details: { port } }));50  }5152  const host = url.hostname.toLowerCase();5354  const literal = host.startsWith("[") ? host.slice(1, -1) : host;55  const hostIsIp = isIP(literal) !== 0;56  if (hostIsIp && isBlockedIp(literal)) {57    return err(tendrilError("ERR_SSRF_BLOCKED", { details: { host } }));58  }5960  let addresses: readonly string[];61  if (hostIsIp) {62    addresses = [literal];63  } else {64    try {65      addresses = await resolver(literal);66    } catch (cause) {67      return err(tendrilError("ERR_INVALID_URL", { message: "DNS resolution failed", details: { host }, cause }));68    }69  }7071  if (addresses.length === 0) {72    return err(tendrilError("ERR_INVALID_URL", { message: "No addresses resolved", details: { host } }));73  }7475  for (const addr of addresses) {76    if (isBlockedIp(addr)) {77      return err(tendrilError("ERR_SSRF_BLOCKED", { details: { host, resolved: addr } }));78    }79  }8081  return ok({ url, host, port, addresses });82}83