SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
3.4 KB · 87 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2export type ErrorCode =3  | "ERR_INVALID_URL"4  | "ERR_SSRF_BLOCKED"5  | "ERR_UNSUPPORTED_TYPE"6  | "ERR_TOO_LARGE"7  | "ERR_UNAUTHORIZED"8  | "ERR_QUOTA_EXCEEDED"9  | "ERR_RATE_LIMITED"10  | "ERR_ROBOTS_DENIED"11  | "ERR_TARGET_BLOCKED"12  | "ERR_TARGET_4XX"13  | "ERR_TARGET_5XX"14  | "ERR_TIER_TIMEOUT"15  | "ERR_POOL_EXHAUSTED"16  | "ERR_DAEMON_DOWN"17  | "ERR_SAFARI_BUSY"18  | "ERR_PROFILE_EXPIRED"19  | "ERR_EXTRACT_FAILED"20  | "ERR_REDIRECT_LOOP"21  | "ERR_INTERNAL";2223export interface ErrorSpec {24  readonly http: number;25  readonly retryable: boolean;26  readonly message: string;27}2829/**30 * The single source of truth for the §15 error taxonomy: code → HTTP status,31 * retry-ability, and a default human message. The API layer reads `http`; the32 * queue layer reads `retryable`. Do not duplicate this mapping anywhere else.33 */34export const ERROR_TAXONOMY: Readonly<Record<ErrorCode, ErrorSpec>> = {35  ERR_INVALID_URL: { http: 400, retryable: false, message: "Malformed or unsupported scheme" },36  ERR_SSRF_BLOCKED: { http: 400, retryable: false, message: "Resolved to a private or forbidden address" },37  ERR_UNSUPPORTED_TYPE: { http: 415, retryable: false, message: "Content type not handled" },38  ERR_TOO_LARGE: { http: 413, retryable: false, message: "Exceeded maxSizeBytes" },39  ERR_UNAUTHORIZED: { http: 401, retryable: false, message: "Bad or revoked key" },40  ERR_QUOTA_EXCEEDED: { http: 402, retryable: false, message: "Plan limit hit" },41  ERR_RATE_LIMITED: { http: 429, retryable: true, message: "Rate limited" },42  ERR_ROBOTS_DENIED: { http: 403, retryable: false, message: "robots.txt disallows" },43  ERR_TARGET_BLOCKED: { http: 502, retryable: true, message: "Target returned a challenge at every tier" },44  ERR_TARGET_4XX: { http: 502, retryable: false, message: "Upstream 4xx status" },45  ERR_TARGET_5XX: { http: 502, retryable: true, message: "Upstream 5xx status" },46  ERR_TIER_TIMEOUT: { http: 504, retryable: true, message: "Render exceeded timeout" },47  ERR_POOL_EXHAUSTED: { http: 503, retryable: true, message: "No WebView available in 30s" },48  ERR_DAEMON_DOWN: { http: 503, retryable: true, message: "Swift daemon unreachable, circuit open" },49  ERR_SAFARI_BUSY: { http: 503, retryable: true, message: "Tier 2 session conflict" },50  ERR_PROFILE_EXPIRED: { http: 409, retryable: false, message: "Session profile needs re-auth" },51  ERR_EXTRACT_FAILED: { http: 422, retryable: false, message: "Schema produced no fields" },52  ERR_REDIRECT_LOOP: { http: 502, retryable: false, message: "Same URL twice in a redirect chain" },53  ERR_INTERNAL: { http: 500, retryable: true, message: "Internal error" },54};5556export interface TendrilError {57  readonly code: ErrorCode;58  readonly message: string;59  readonly details?: Readonly<Record<string, unknown>>;60  readonly cause?: unknown;61}6263export function tendrilError(64  code: ErrorCode,65  overrides?: { message?: string; details?: Record<string, unknown>; cause?: unknown },66): TendrilError {67  const base: TendrilError = {68    code,69    message: overrides?.message ?? ERROR_TAXONOMY[code].message,70  };71  const details = overrides?.details;72  const cause = overrides?.cause;73  return {74    ...base,75    ...(details !== undefined ? { details } : {}),76    ...(cause !== undefined ? { cause } : {}),77  };78}7980export function httpStatusFor(code: ErrorCode): number {81  return ERROR_TAXONOMY[code].http;82}8384export function isRetryable(code: ErrorCode): boolean {85  return ERROR_TAXONOMY[code].retryable;86}87