spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2export interface EscalationInput {3 readonly status: number;4 readonly contentType: string;5 readonly body: string;6}78export type TierDecision =9 | { readonly action: "proceed" }10 | { readonly action: "escalate"; readonly reason: string }11 | { readonly action: "non-html"; readonly reason: string };1213type Severity = "decisive" | "strong" | "moderate";14interface Signal {15 readonly severity: Severity;16 readonly reason: string;17}1819const CHALLENGE_MARKERS = [20 "challenge-platform",21 "cf-browser-verification",22 "_Incapsula_",23 "px-captcha",24 "datadome",25];2627const EMPTY_ROOT_RE = /<[^>]+\bid=["']?(root|__next|app)["']?[^>]*>\s*<\/[a-z]+>/i;28const NOSCRIPT_JS_RE = /<noscript[^>]*>[\s\S]*?(enable\s+javascript|javascript\s+is\s+required)[\s\S]*?<\/noscript>/i;29const TITLE_RE = /<title[^>]*>\s*(just a moment|attention required|access denied)/i;30const HREF_RE = /<a\s[^>]*href\s*=/i;3132function isHtml(contentType: string): boolean {33 const ct = contentType.toLowerCase();34 return ct.includes("text/html") || ct.includes("application/xhtml+xml") || ct === "";35}3637function textHtmlRatio(html: string): number {38 if (html.length === 0) return 0;39 const text = html40 .replace(/<script[\s\S]*?<\/script>/gi, "")41 .replace(/<style[\s\S]*?<\/style>/gi, "")42 .replace(/<[^>]+>/g, "")43 .replace(/\s+/g, " ")44 .trim();45 return text.length / html.length;46}4748/**49 * Decide whether a Tier 0 response should escalate to Tier 1 (§3.4). Pure, no I/O.50 * Decisive signals escalate on their own; two or more strong/moderate signals51 * escalate together. Non-HTML content types route to §9 instead of escalating.52 */53export function shouldEscalate(input: EscalationInput): TierDecision {54 if (!isHtml(input.contentType)) {55 return { action: "non-html", reason: `content-type:${input.contentType || "unknown"}` };56 }5758 const signals: Signal[] = [];59 const body = input.body;60 const bytes = Buffer.byteLength(body, "utf8");6162 if (input.status === 403 || input.status === 429 || input.status === 503) {63 signals.push({ severity: "decisive", reason: `status:${input.status}` });64 }6566 if (bytes < 2048 && EMPTY_ROOT_RE.test(body)) {67 signals.push({ severity: "decisive", reason: "empty-spa-root" });68 }6970 for (const marker of CHALLENGE_MARKERS) {71 if (body.includes(marker)) {72 signals.push({ severity: "decisive", reason: `challenge:${marker}` });73 break;74 }75 }7677 if (TITLE_RE.test(body)) {78 signals.push({ severity: "decisive", reason: "challenge-title" });79 }8081 if (textHtmlRatio(body) < 0.05) {82 signals.push({ severity: "strong", reason: "low-text-ratio" });83 }8485 if (NOSCRIPT_JS_RE.test(body)) {86 signals.push({ severity: "strong", reason: "noscript-js-required" });87 }8889 if (bytes > 10_240 && !HREF_RE.test(body)) {90 signals.push({ severity: "moderate", reason: "no-links" });91 }9293 const decisive = signals.find((s) => s.severity === "decisive");94 if (decisive) {95 return { action: "escalate", reason: signals.map((s) => s.reason).join(",") };96 }9798 const weak = signals.filter((s) => s.severity === "strong" || s.severity === "moderate");99 if (weak.length >= 2) {100 return { action: "escalate", reason: weak.map((s) => s.reason).join(",") };101 }102103 return { action: "proceed" };104}105