spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { lookup } from "node:dns/promises";3import { isIP } from "node:net";4import { err, ok, tendrilError, type Result } from "@tendril/shared";5import { isBlockedIp } from "./ip.js";67const ALLOWED_PORTS = new Set([80, 443, 8080, 8443]);8const DEFAULT_PORT: Readonly<Record<string, number>> = { "http:": 80, "https:": 443 };910export interface SafeTarget {11 readonly url: URL;12 readonly host: string;13 readonly port: number;14 /** All resolved addresses, all verified public. Pin these to defeat DNS rebinding. */15 readonly addresses: readonly string[];16}1718export type Resolver = (host: string) => Promise<readonly string[]>;1920const defaultResolver: Resolver = async (host) => {21 const results = await lookup(host, { all: true });22 return results.map((r) => r.address);23};2425/**26 * Validate a URL for egress (§16.5). Rejects non-http(s) schemes and disallowed27 * ports up front, then resolves DNS and rejects if *any* resolved address is28 * private/loopback/link-local. The returned `addresses` should be pinned by the29 * caller when connecting, so a rebind between validation and fetch cannot slip30 * a private IP through.31 */32export async function validateEgress(33 raw: string,34 resolver: Resolver = defaultResolver,35): Promise<Result<SafeTarget>> {36 let url: URL;37 try {38 url = new URL(raw);39 } catch {40 return err(tendrilError("ERR_INVALID_URL", { details: { url: raw } }));41 }4243 if (url.protocol !== "http:" && url.protocol !== "https:") {44 return err(tendrilError("ERR_INVALID_URL", { details: { scheme: url.protocol } }));45 }4647 const port = url.port !== "" ? Number(url.port) : DEFAULT_PORT[url.protocol] ?? 0;48 if (!ALLOWED_PORTS.has(port)) {49 return err(tendrilError("ERR_SSRF_BLOCKED", { message: "Port not allowed", details: { port } }));50 }5152 const host = url.hostname.toLowerCase();5354 const literal = host.startsWith("[") ? host.slice(1, -1) : host;55 const hostIsIp = isIP(literal) !== 0;56 if (hostIsIp && isBlockedIp(literal)) {57 return err(tendrilError("ERR_SSRF_BLOCKED", { details: { host } }));58 }5960 let addresses: readonly string[];61 if (hostIsIp) {62 addresses = [literal];63 } else {64 try {65 addresses = await resolver(literal);66 } catch (cause) {67 return err(tendrilError("ERR_INVALID_URL", { message: "DNS resolution failed", details: { host }, cause }));68 }69 }7071 if (addresses.length === 0) {72 return err(tendrilError("ERR_INVALID_URL", { message: "No addresses resolved", details: { host } }));73 }7475 for (const addr of addresses) {76 if (isBlockedIp(addr)) {77 return err(tendrilError("ERR_SSRF_BLOCKED", { details: { host, resolved: addr } }));78 }79 }8081 return ok({ url, host, port, addresses });82}83