spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { isIP } from "node:net";34interface Cidr {5 readonly base: bigint;6 readonly bits: number;7 readonly family: 4 | 6;8}910function ipv4ToBigInt(ip: string): bigint | null {11 const parts = ip.split(".");12 if (parts.length !== 4) return null;13 let acc = 0n;14 for (const part of parts) {15 if (!/^\d{1,3}$/.test(part)) return null;16 const n = Number(part);17 if (n > 255) return null;18 acc = (acc << 8n) | BigInt(n);19 }20 return acc;21}2223function ipv6ToBigInt(ip: string): bigint | null {24 let addr = ip;25 const zone = addr.indexOf("%");26 if (zone !== -1) addr = addr.slice(0, zone);2728 let mappedSuffix = 0n;29 let embeddedV4 = false;30 const lastColon = addr.lastIndexOf(":");31 const tail = addr.slice(lastColon + 1);32 if (tail.includes(".")) {33 const v4 = ipv4ToBigInt(tail);34 if (v4 === null) return null;35 mappedSuffix = v4;36 embeddedV4 = true;37 addr = addr.slice(0, lastColon + 1) + "0:0";38 }3940 const halves = addr.split("::");41 if (halves.length > 2) return null;42 const head = halves[0] === "" || halves[0] === undefined ? [] : halves[0].split(":");43 const tailGroups = halves.length === 2 ? (halves[1] === "" ? [] : (halves[1] ?? "").split(":")) : [];4445 const groups: string[] = [];46 if (halves.length === 2) {47 const missing = 8 - head.length - tailGroups.length;48 if (missing < 0) return null;49 groups.push(...head, ...Array<string>(missing).fill("0"), ...tailGroups);50 } else {51 groups.push(...head);52 }53 if (groups.length !== 8) return null;5455 let acc = 0n;56 for (const g of groups) {57 if (!/^[0-9a-fA-F]{1,4}$/.test(g)) return null;58 acc = (acc << 16n) | BigInt(parseInt(g, 16));59 }60 if (embeddedV4) {61 acc = (acc & ~0xffffffffn) | mappedSuffix;62 }63 return acc;64}6566export function ipToBigInt(ip: string, family: 4 | 6): bigint | null {67 return family === 4 ? ipv4ToBigInt(ip) : ipv6ToBigInt(ip);68}6970function cidr(spec: string, family: 4 | 6): Cidr {71 const [addr, bitsRaw] = spec.split("/");72 const bits = Number(bitsRaw);73 const base = ipToBigInt(addr ?? "", family);74 if (base === null) throw new Error(`bad cidr ${spec}`);75 return { base, bits, family };76}7778const V4_BLOCKED: readonly Cidr[] = [79 "0.0.0.0/8",80 "10.0.0.0/8",81 "100.64.0.0/10",82 "127.0.0.0/8",83 "169.254.0.0/16",84 "172.16.0.0/12",85 "192.0.0.0/24",86 "192.0.2.0/24",87 "192.168.0.0/16",88 "198.18.0.0/15",89 "198.51.100.0/24",90 "203.0.113.0/24",91 "224.0.0.0/4",92 "240.0.0.0/4",93].map((s) => cidr(s, 4));9495const V6_BLOCKED: readonly Cidr[] = [96 "::1/128",97 "::/128",98 "fc00::/7",99 "fe80::/10",100 "ff00::/8",101 "2001:db8::/32",102].map((s) => cidr(s, 6));103104const V4_FULL_BITS = 32;105const V6_FULL_BITS = 128;106const V4_MAPPED_HIGH = 0xffffn; // high 96 bits of ::ffff:a.b.c.d107108function inCidr(value: bigint, c: Cidr): boolean {109 const full = c.family === 4 ? V4_FULL_BITS : V6_FULL_BITS;110 if (c.bits === 0) return true;111 const shift = BigInt(full - c.bits);112 return value >> shift === c.base >> shift;113}114115/**116 * True when an IP literal points at a private, loopback, link-local, multicast,117 * or otherwise forbidden destination (§16.5). Pure — safe to unit test. IPv4-mapped118 * IPv6 (`::ffff:a.b.c.d`) is unwrapped and evaluated as IPv4.119 */120export function isBlockedIp(ip: string): boolean {121 const family = isIP(ip);122 if (family === 0) return true;123124 if (family === 4) {125 const v = ipv4ToBigInt(ip);126 if (v === null) return true;127 return V4_BLOCKED.some((c) => inCidr(v, c));128 }129130 const v = ipv6ToBigInt(ip);131 if (v === null) return true;132 if (v >> 32n === V4_MAPPED_HIGH) {133 const embedded = v & 0xffffffffn;134 return V4_BLOCKED.some((c) => inCidr(embedded, c));135 }136 return V6_BLOCKED.some((c) => inCidr(v, c));137}138