spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import { Agent, request } from "undici";3import type { LookupFunction } from "node:net";4import { brotliDecompressSync, gunzipSync, inflateSync } from "node:zlib";5import {6  err,7  normalizeUrl,8  ok,9  tendrilError,10  type FetchResult,11  type RedirectHop,12  type Result,13} from "@tendril/shared";14import { validateEgress, type Resolver } from "@tendril/egress";15import { buildHeaders } from "./headers.js";1617const MAX_REDIRECTS = 5;18const DEFAULT_MAX_BYTES = 20 * 1024 * 1024;1920const pinnedAddresses = new Map<string, string>();2122const pinnedLookup: LookupFunction = (hostname, options, callback) => {23  const pinned = pinnedAddresses.get(hostname);24  if (pinned === undefined) {25    callback(new Error(`no pinned address for ${hostname}`) as NodeJS.ErrnoException, "");26    return;27  }28  const family = pinned.includes(":") ? 6 : 4;29  if (options.all === true) {30    callback(null, [{ address: pinned, family }]);31  } else {32    callback(null, pinned, family);33  }34};3536const agent = new Agent({37  connections: 64,38  pipelining: 1,39  keepAliveTimeout: 30_000,40  keepAliveMaxTimeout: 120_000,41  bodyTimeout: 20_000,42  headersTimeout: 10_000,43  connect: {44    timeout: 8_000,45    rejectUnauthorized: true,46    lookup: pinnedLookup,47  },48});4950export interface HttpFetchOptions {51  readonly timeout?: number;52  readonly maxBytes?: number;53  readonly headers?: Readonly<Record<string, string>>;54  readonly userAgent?: string;55  readonly resolver?: Resolver;56}5758function decompress(buf: Buffer, encoding: string | undefined): Buffer {59  try {60    switch ((encoding ?? "").toLowerCase()) {61      case "gzip":62        return gunzipSync(buf);63      case "deflate":64        return inflateSync(buf);65      case "br":66        return brotliDecompressSync(buf);67      default:68        return buf;69    }70  } catch {71    return buf;72  }73}7475async function readCapped(body: AsyncIterable<Buffer>, max: number): Promise<Buffer | null> {76  const chunks: Buffer[] = [];77  let total = 0;78  for await (const chunk of body) {79    total += chunk.length;80    if (total > max) return null;81    chunks.push(chunk);82  }83  return Buffer.concat(chunks);84}8586const META_REFRESH_RE = /<meta[^>]+http-equiv=["']?refresh["']?[^>]*content=["'][^"']*url=([^"'>\s]+)/i;8788function firstHeader(value: string | string[] | undefined): string | undefined {89  if (Array.isArray(value)) return value[0];90  return value;91}9293/**94 * Tier 0 HTTP fetch (§3). Handles redirects manually so it can re-validate SSRF95 * on every hop (§16.5), record the full chain, detect meta-refresh redirects the96 * transport cannot see, and stop on a redirect loop. Connections are pinned to97 * the SSRF-validated IP to defeat DNS rebinding between validation and connect.98 */99export async function httpFetch(rawUrl: string, options: HttpFetchOptions = {}): Promise<Result<FetchResult>> {100  const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;101  const redirects: RedirectHop[] = [];102  const seen = new Set<string>();103104  let current = rawUrl;105  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {106    const norm = normalizeUrl(current);107    if (!norm.ok) return norm;108    if (seen.has(norm.value)) return err(tendrilError("ERR_REDIRECT_LOOP", { details: { url: current } }));109    seen.add(norm.value);110111    const safe = await validateEgress(current, options.resolver);112    if (!safe.ok) return safe;113    const address = safe.value.addresses[0];114    if (address === undefined) return err(tendrilError("ERR_SSRF_BLOCKED", { details: { host: safe.value.host } }));115    pinnedAddresses.set(safe.value.host, address);116117    let res: Awaited<ReturnType<typeof request>>;118    try {119      res = await request(current, {120        dispatcher: agent,121        method: "GET",122        headersTimeout: options.timeout ?? 10_000,123        bodyTimeout: options.timeout ?? 20_000,124        headers: buildHeaders(safe.value.host, options.headers, options.userAgent),125      });126    } catch (cause) {127      return err(tendrilError("ERR_TARGET_5XX", { message: "Transport error", details: { url: current }, cause }));128    }129130    const status = res.statusCode;131    const location = firstHeader(res.headers["location"]);132133    if (status >= 300 && status < 400 && location !== undefined && location !== "") {134      res.body.dump().catch(() => undefined);135      if (hop === MAX_REDIRECTS) return err(tendrilError("ERR_REDIRECT_LOOP", { details: { url: current } }));136      let next: string;137      try {138        next = new URL(location, current).toString();139      } catch {140        return err(tendrilError("ERR_INVALID_URL", { details: { location } }));141      }142      redirects.push({ from: current, to: next, status });143      current = next;144      continue;145    }146147    const rawBody = await readCapped(res.body, maxBytes);148    if (rawBody === null) return err(tendrilError("ERR_TOO_LARGE", { details: { max: maxBytes } }));149150    const contentType = (firstHeader(res.headers["content-type"]) ?? "").toLowerCase();151    const encoding = firstHeader(res.headers["content-encoding"]);152    const decoded = decompress(rawBody, encoding);153    const bodyText = decoded.toString("utf8");154155    const isHtml = contentType.includes("text/html") || contentType.includes("xhtml") || contentType === "";156    if (isHtml && hop < MAX_REDIRECTS) {157      const meta = META_REFRESH_RE.exec(bodyText);158      if (meta?.[1] !== undefined) {159        let next: string;160        try {161          next = new URL(meta[1], current).toString();162        } catch {163          next = "";164        }165        if (next !== "" && !seen.has(normalizeUrl(next).ok ? (normalizeUrl(next) as { value: string }).value : next)) {166          redirects.push({ from: current, to: next, status: 200 });167          current = next;168          continue;169        }170      }171    }172173    const headers: Record<string, string> = {};174    for (const [key, value] of Object.entries(res.headers)) {175      headers[key] = Array.isArray(value) ? value.join(", ") : (value ?? "");176    }177178    const result: FetchResult = {179      tier: "http",180      status,181      finalUrl: current,182      contentType,183      body: bodyText,184      bodyBytes: decoded.length,185      redirects,186      headers,187    };188    return ok(result);189  }190191  return err(tendrilError("ERR_REDIRECT_LOOP", { details: { url: rawUrl } }));192}193