SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
3.2 KB · 105 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2export interface EscalationInput {3  readonly status: number;4  readonly contentType: string;5  readonly body: string;6}78export type TierDecision =9  | { readonly action: "proceed" }10  | { readonly action: "escalate"; readonly reason: string }11  | { readonly action: "non-html"; readonly reason: string };1213type Severity = "decisive" | "strong" | "moderate";14interface Signal {15  readonly severity: Severity;16  readonly reason: string;17}1819const CHALLENGE_MARKERS = [20  "challenge-platform",21  "cf-browser-verification",22  "_Incapsula_",23  "px-captcha",24  "datadome",25];2627const EMPTY_ROOT_RE = /<[^>]+\bid=["']?(root|__next|app)["']?[^>]*>\s*<\/[a-z]+>/i;28const NOSCRIPT_JS_RE = /<noscript[^>]*>[\s\S]*?(enable\s+javascript|javascript\s+is\s+required)[\s\S]*?<\/noscript>/i;29const TITLE_RE = /<title[^>]*>\s*(just a moment|attention required|access denied)/i;30const HREF_RE = /<a\s[^>]*href\s*=/i;3132function isHtml(contentType: string): boolean {33  const ct = contentType.toLowerCase();34  return ct.includes("text/html") || ct.includes("application/xhtml+xml") || ct === "";35}3637function textHtmlRatio(html: string): number {38  if (html.length === 0) return 0;39  const text = html40    .replace(/<script[\s\S]*?<\/script>/gi, "")41    .replace(/<style[\s\S]*?<\/style>/gi, "")42    .replace(/<[^>]+>/g, "")43    .replace(/\s+/g, " ")44    .trim();45  return text.length / html.length;46}4748/**49 * Decide whether a Tier 0 response should escalate to Tier 1 (§3.4). Pure, no I/O.50 * Decisive signals escalate on their own; two or more strong/moderate signals51 * escalate together. Non-HTML content types route to §9 instead of escalating.52 */53export function shouldEscalate(input: EscalationInput): TierDecision {54  if (!isHtml(input.contentType)) {55    return { action: "non-html", reason: `content-type:${input.contentType || "unknown"}` };56  }5758  const signals: Signal[] = [];59  const body = input.body;60  const bytes = Buffer.byteLength(body, "utf8");6162  if (input.status === 403 || input.status === 429 || input.status === 503) {63    signals.push({ severity: "decisive", reason: `status:${input.status}` });64  }6566  if (bytes < 2048 && EMPTY_ROOT_RE.test(body)) {67    signals.push({ severity: "decisive", reason: "empty-spa-root" });68  }6970  for (const marker of CHALLENGE_MARKERS) {71    if (body.includes(marker)) {72      signals.push({ severity: "decisive", reason: `challenge:${marker}` });73      break;74    }75  }7677  if (TITLE_RE.test(body)) {78    signals.push({ severity: "decisive", reason: "challenge-title" });79  }8081  if (textHtmlRatio(body) < 0.05) {82    signals.push({ severity: "strong", reason: "low-text-ratio" });83  }8485  if (NOSCRIPT_JS_RE.test(body)) {86    signals.push({ severity: "strong", reason: "noscript-js-required" });87  }8889  if (bytes > 10_240 && !HREF_RE.test(body)) {90    signals.push({ severity: "moderate", reason: "no-links" });91  }9293  const decisive = signals.find((s) => s.severity === "decisive");94  if (decisive) {95    return { action: "escalate", reason: signals.map((s) => s.reason).join(",") };96  }9798  const weak = signals.filter((s) => s.severity === "strong" || s.severity === "moderate");99  if (weak.length >= 2) {100    return { action: "escalate", reason: weak.map((s) => s.reason).join(",") };101  }102103  return { action: "proceed" };104}105