SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
3.7 KB · 138 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2import { isIP } from "node:net";34interface Cidr {5  readonly base: bigint;6  readonly bits: number;7  readonly family: 4 | 6;8}910function ipv4ToBigInt(ip: string): bigint | null {11  const parts = ip.split(".");12  if (parts.length !== 4) return null;13  let acc = 0n;14  for (const part of parts) {15    if (!/^\d{1,3}$/.test(part)) return null;16    const n = Number(part);17    if (n > 255) return null;18    acc = (acc << 8n) | BigInt(n);19  }20  return acc;21}2223function ipv6ToBigInt(ip: string): bigint | null {24  let addr = ip;25  const zone = addr.indexOf("%");26  if (zone !== -1) addr = addr.slice(0, zone);2728  let mappedSuffix = 0n;29  let embeddedV4 = false;30  const lastColon = addr.lastIndexOf(":");31  const tail = addr.slice(lastColon + 1);32  if (tail.includes(".")) {33    const v4 = ipv4ToBigInt(tail);34    if (v4 === null) return null;35    mappedSuffix = v4;36    embeddedV4 = true;37    addr = addr.slice(0, lastColon + 1) + "0:0";38  }3940  const halves = addr.split("::");41  if (halves.length > 2) return null;42  const head = halves[0] === "" || halves[0] === undefined ? [] : halves[0].split(":");43  const tailGroups = halves.length === 2 ? (halves[1] === "" ? [] : (halves[1] ?? "").split(":")) : [];4445  const groups: string[] = [];46  if (halves.length === 2) {47    const missing = 8 - head.length - tailGroups.length;48    if (missing < 0) return null;49    groups.push(...head, ...Array<string>(missing).fill("0"), ...tailGroups);50  } else {51    groups.push(...head);52  }53  if (groups.length !== 8) return null;5455  let acc = 0n;56  for (const g of groups) {57    if (!/^[0-9a-fA-F]{1,4}$/.test(g)) return null;58    acc = (acc << 16n) | BigInt(parseInt(g, 16));59  }60  if (embeddedV4) {61    acc = (acc & ~0xffffffffn) | mappedSuffix;62  }63  return acc;64}6566export function ipToBigInt(ip: string, family: 4 | 6): bigint | null {67  return family === 4 ? ipv4ToBigInt(ip) : ipv6ToBigInt(ip);68}6970function cidr(spec: string, family: 4 | 6): Cidr {71  const [addr, bitsRaw] = spec.split("/");72  const bits = Number(bitsRaw);73  const base = ipToBigInt(addr ?? "", family);74  if (base === null) throw new Error(`bad cidr ${spec}`);75  return { base, bits, family };76}7778const V4_BLOCKED: readonly Cidr[] = [79  "0.0.0.0/8",80  "10.0.0.0/8",81  "100.64.0.0/10",82  "127.0.0.0/8",83  "169.254.0.0/16",84  "172.16.0.0/12",85  "192.0.0.0/24",86  "192.0.2.0/24",87  "192.168.0.0/16",88  "198.18.0.0/15",89  "198.51.100.0/24",90  "203.0.113.0/24",91  "224.0.0.0/4",92  "240.0.0.0/4",93].map((s) => cidr(s, 4));9495const V6_BLOCKED: readonly Cidr[] = [96  "::1/128",97  "::/128",98  "fc00::/7",99  "fe80::/10",100  "ff00::/8",101  "2001:db8::/32",102].map((s) => cidr(s, 6));103104const V4_FULL_BITS = 32;105const V6_FULL_BITS = 128;106const V4_MAPPED_HIGH = 0xffffn; // high 96 bits of ::ffff:a.b.c.d107108function inCidr(value: bigint, c: Cidr): boolean {109  const full = c.family === 4 ? V4_FULL_BITS : V6_FULL_BITS;110  if (c.bits === 0) return true;111  const shift = BigInt(full - c.bits);112  return value >> shift === c.base >> shift;113}114115/**116 * True when an IP literal points at a private, loopback, link-local, multicast,117 * or otherwise forbidden destination (§16.5). Pure — safe to unit test. IPv4-mapped118 * IPv6 (`::ffff:a.b.c.d`) is unwrapped and evaluated as IPv4.119 */120export function isBlockedIp(ip: string): boolean {121  const family = isIP(ip);122  if (family === 0) return true;123124  if (family === 4) {125    const v = ipv4ToBigInt(ip);126    if (v === null) return true;127    return V4_BLOCKED.some((c) => inCidr(v, c));128  }129130  const v = ipv6ToBigInt(ip);131  if (v === null) return true;132  if (v >> 32n === V4_MAPPED_HIGH) {133    const embedded = v & 0xffffffffn;134    return V4_BLOCKED.some((c) => inCidr(embedded, c));135  }136  return V6_BLOCKED.some((c) => inCidr(v, c));137}138