spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { shouldEscalate } from "./escalate.js";45const goodHtml =6 "<html><head><title>Article</title></head><body>" +7 "<h1>Heading</h1>" +8 "<p>" +9 "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod. ".repeat(20) +10 "</p>" +11 '<a href="/next">next</a></body></html>';1213describe("shouldEscalate", () => {14 it("proceeds on a healthy article page", () => {15 expect(shouldEscalate({ status: 200, contentType: "text/html", body: goodHtml })).toEqual({16 action: "proceed",17 });18 });1920 it("routes non-HTML content to §9 without escalating", () => {21 const d = shouldEscalate({ status: 200, contentType: "application/pdf", body: "%PDF-1.7" });22 expect(d.action).toBe("non-html");23 });2425 it("escalates on 403/429/503 (decisive)", () => {26 for (const status of [403, 429, 503]) {27 const d = shouldEscalate({ status, contentType: "text/html", body: goodHtml });28 expect(d.action).toBe("escalate");29 if (d.action === "escalate") expect(d.reason).toContain(`status:${status}`);30 }31 });3233 it("escalates on a Cloudflare challenge title (decisive)", () => {34 const d = shouldEscalate({35 status: 200,36 contentType: "text/html",37 body: "<html><head><title>Just a moment...</title></head><body></body></html>",38 });39 expect(d.action).toBe("escalate");40 if (d.action === "escalate") expect(d.reason).toContain("challenge-title");41 });4243 it("escalates on known challenge markers (decisive)", () => {44 const d = shouldEscalate({45 status: 200,46 contentType: "text/html",47 body: `<html><body>${goodHtml}<script>datadome</script></body></html>`,48 });49 expect(d.action).toBe("escalate");50 if (d.action === "escalate") expect(d.reason).toContain("challenge:datadome");51 });5253 it("escalates on an empty SPA root under 2KB (decisive)", () => {54 const d = shouldEscalate({55 status: 200,56 contentType: "text/html",57 body: '<html><body><div id="root"></div></body></html>',58 });59 expect(d.action).toBe("escalate");60 if (d.action === "escalate") expect(d.reason).toContain("empty-spa-root");61 });6263 it("does not escalate on a single strong signal alone", () => {64 const body = `<html><body><noscript>Please enable JavaScript to continue</noscript>${goodHtml}</body></html>`;65 const d = shouldEscalate({ status: 200, contentType: "text/html", body });66 expect(d.action).toBe("proceed");67 });6869 it("escalates on two weak signals together", () => {70 const filler = "x".repeat(11_000);71 const body = `<html><body><noscript>JavaScript is required</noscript><!--${filler}--></body></html>`;72 const d = shouldEscalate({ status: 200, contentType: "text/html", body });73 expect(d.action).toBe("escalate");74 });75});76