TypeScript 97.5%
SQL 1.4%
Python 0.8%
1/** Minimal, dependency-free HTML → readable text extraction. */2export function htmlToText(html: string): string {3 let s = html;4 s = s.replace(/<script[\s\S]*?<\/script>/gi, " ");5 s = s.replace(/<style[\s\S]*?<\/style>/gi, " ");6 s = s.replace(/<noscript[\s\S]*?<\/noscript>/gi, " ");7 s = s.replace(/<template[\s\S]*?<\/template>/gi, " ");8 s = s.replace(/<svg[\s\S]*?<\/svg>/gi, " ");9 s = s.replace(/<!--[\s\S]*?-->/g, " ");10 s = s.replace(/<(br|\/p|\/div|\/li|\/h[1-6]|\/tr|\/section|\/article|\/header|\/footer|\/blockquote|\/pre|\/table|\/ul|\/ol|\/dd|\/dt)[^>]*>/gi, "\n");11 s = s.replace(/<\/t[dh]>/gi, "\t");12 s = s.replace(/<[^>]+>/g, " ");13 s = decodeEntities(s);14 s = s.replace(/[ \t\f\v]+/g, " ");15 s = s.replace(/\s*\n\s*/g, "\n");16 s = s.replace(/\n{3,}/g, "\n\n");17 return s.trim();18}1920const ENTITIES: Record<string, string> = {21 amp: "&",22 lt: "<",23 gt: ">",24 quot: '"',25 apos: "'",26 nbsp: " ",27 copy: "©",28 reg: "®",29 trade: "™",30 hellip: "…",31 mdash: "—",32 ndash: "–",33 laquo: "«",34 raquo: "»",35 lsquo: "‘",36 rsquo: "’",37 ldquo: "“",38 rdquo: "”",39 bull: "•",40 middot: "·",41 euro: "€",42 pound: "£",43 yen: "¥",44 cent: "¢",45 deg: "°",46 times: "×",47 divide: "÷",48 plusmn: "±",49 frac12: "½",50 frac14: "¼",51 frac34: "¾",52 eacute: "é",53 egrave: "è",54 ecirc: "ê",55 euml: "ë",56 agrave: "à",57 aacute: "á",58 acirc: "â",59 auml: "ä",60 ccedil: "ç",61 iuml: "ï",62 icirc: "î",63 ocirc: "ô",64 ouml: "ö",65 ugrave: "ù",66 ucirc: "û",67 uuml: "ü",68 ntilde: "ñ",69 szlig: "ß",70 oelig: "œ",71 aelig: "æ",72 Eacute: "É",73 Agrave: "À",74 Ccedil: "Ç",75};7677export function decodeEntities(s: string): string {78 return s79 .replace(/&#x([0-9a-f]+);/gi, (_, h) => safeChar(parseInt(h, 16)))80 .replace(/&#(\d+);/g, (_, d) => safeChar(parseInt(d, 10)))81 .replace(/&([a-z0-9]+);/gi, (m, name) => ENTITIES[name] ?? ENTITIES[name.toLowerCase()] ?? m);82}8384function safeChar(code: number): string {85 try {86 return String.fromCodePoint(code);87 } catch {88 return "";89 }90}9192export function extractTitle(html: string): string | null {93 const m = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);94 return m ? decodeEntities(m[1]!).replace(/\s+/g, " ").trim().slice(0, 300) || null : null;95}9697// ---------------------------------------------------------------------------98// Block / anti-bot detection99// ---------------------------------------------------------------------------100101export type BlockVendor =102 | "cloudflare"103 | "datadome"104 | "perimeterx"105 | "akamai"106 | "kasada"107 | "imperva"108 | "aws_waf"109 | "vercel"110 | "shape"111 | "distil"112 | "fastly"113 | "google"114 | "sucuri"115 | "generic";116117export interface BlockVerdict {118 blocked: boolean;119 /** Stable reason code: http_403, http_429, cloudflare_challenge, captcha, anti_bot, waf, soft_block, empty_html, rate_limited… */120 reason?: string;121 vendor?: BlockVendor;122 /** True when the block is a JavaScript challenge that a real browser can typically pass. */123 challenge?: boolean;124 /** Retry-After hint in milliseconds, when the origin sent one. */125 retryAfterMs?: number;126}127128const HTML_CT = /text\/html|application\/xhtml/i;129130function parseRetryAfter(v: string | undefined): number | undefined {131 if (!v) return undefined;132 const n = Number(v);133 if (Number.isFinite(n)) return Math.max(0, Math.round(n * 1000));134 const d = Date.parse(v);135 if (!Number.isNaN(d)) return Math.max(0, d - Date.now());136 return undefined;137}138139/**140 * Heuristic block-page detection used by the retry engine. Looks at the status, the response141 * headers (WAF signatures) and the first 40 KB of the body (challenge markup, captcha vendors,142 * "access denied" pages served with a 200).143 */144export function looksBlocked(status: number, body: string, headers: Record<string, string>): BlockVerdict {145 const h = lowerKeys(headers);146 const server = (h["server"] ?? "").toLowerCase();147 const ct = h["content-type"] ?? "";148 const isHtml = !ct || HTML_CT.test(ct);149 const head = body.slice(0, 40_000);150 const retryAfterMs = parseRetryAfter(h["retry-after"]);151152 // --- header-level signals (strongest) -------------------------------------153 if (h["cf-mitigated"] === "challenge") return { blocked: true, reason: "cloudflare_challenge", vendor: "cloudflare", challenge: true, retryAfterMs };154 if (h["x-datadome"] || h["x-dd-b"] || /datadome/i.test(h["set-cookie"] ?? "")) {155 if (status === 403 || status === 401 || status === 429 || /captcha-delivery|geo\.captcha|dd\.js|DataDome/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "datadome", challenge: true, retryAfterMs };156 }157 if (h["x-kpsdk-ct"] || h["x-kpsdk-c"] || (/x-kpsdk|kpsdk-/i.test(head) && status >= 400)) return { blocked: true, reason: "anti_bot", vendor: "kasada", challenge: true, retryAfterMs };158 if (h["x-amzn-waf-action"] === "challenge" || h["x-amzn-waf-action"] === "captcha") return { blocked: true, reason: "waf", vendor: "aws_waf", challenge: true, retryAfterMs };159 if (h["x-vercel-mitigated"] === "challenge" || h["x-vercel-protection-bypass"] !== undefined && status === 403) return { blocked: true, reason: "anti_bot", vendor: "vercel", challenge: true, retryAfterMs };160 if (status === 403 && (h["x-iinfo"] || /incap_ses|visid_incap/i.test(h["set-cookie"] ?? ""))) return { blocked: true, reason: "waf", vendor: "imperva", challenge: /_Incapsula_Resource/i.test(head), retryAfterMs };161 if (status === 403 && /_abck|ak_bmsc|bm_sz/i.test(h["set-cookie"] ?? "")) return { blocked: true, reason: "anti_bot", vendor: "akamai", challenge: false, retryAfterMs };162163 // --- status-level signals --------------------------------------------------164 if (status === 407) return { blocked: true, reason: "http_407", vendor: "generic" };165 if (status === 429) return { blocked: true, reason: "rate_limited", vendor: vendorFrom(server, head), retryAfterMs };166 if (status === 999) return { blocked: true, reason: "http_999", vendor: "generic" };167 if (status === 403) return { blocked: true, reason: "http_403", vendor: vendorFrom(server, head), challenge: /cf-chl|challenge-platform|__cf_chl|Just a moment/i.test(head), retryAfterMs };168 if (status === 401 && /captcha|challenge|bot|automated/i.test(head)) return { blocked: true, reason: "http_401", vendor: vendorFrom(server, head) };169 if (status === 503 && /cloudflare|just a moment|attention required|checking your browser|ddos-guard|checking if the site connection is secure/i.test(head)) {170 return { blocked: true, reason: "cloudflare_challenge", vendor: "cloudflare", challenge: true, retryAfterMs };171 }172 if ((status === 503 || status === 520 || status === 521 || status === 522 || status === 523 || status === 524 || status === 525 || status === 526 || status === 530) && /cloudflare/i.test(server + " " + head)) {173 return { blocked: true, reason: `origin_${status}`, vendor: "cloudflare", challenge: false, retryAfterMs };174 }175 if (status === 405 && /akamai/i.test(server)) return { blocked: true, reason: "waf", vendor: "akamai" };176 if (status === 202 && /datadome|captcha-delivery/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "datadome", challenge: true };177178 // --- body-level signals (only meaningful for HTML) -------------------------179 // Vendor beacons (Cloudflare JSD, DataDome tags.js, PerimeterX, Imperva, Kasada ips.js, AWS WAF180 // challenge.js) are present on EVERY page of a protected site, so on a 2xx only interstitial-specific181 // markers count; on 4xx/5xx the mere presence of a vendor script is enough.182 if (!isHtml) return { blocked: false };183 const denied = status >= 400;184 if (/cf-chl-bypass|__cf_chl_f_tk|__cf_chl_rt_tk|__cf_chl_tk|cf_chl_opt|<title>\s*Just a moment|cf-chl-widget|challenge-error-text|cf-challenge-running|id="challenge-(running|stage|form)"|Checking your browser before accessing|Verify you are human by completing the action/i.test(head) || (denied && /cf-chl|challenge-platform|cf-turnstile|challenges\.cloudflare\.com/i.test(head))) {185 return { blocked: true, reason: "cloudflare_challenge", vendor: "cloudflare", challenge: true, retryAfterMs };186 }187 if (/geo\.captcha-delivery\.com|<title>\s*DataDome|dd\.js\?|ddCaptcha/i.test(head) || (denied && /datadome/i.test(head))) return { blocked: true, reason: "anti_bot", vendor: "datadome", challenge: true };188 if (/px-captcha|human-challenge|<title>\s*Access to this page has been denied/i.test(head) || (denied && /perimeterx|_pxhd|_pxvid/i.test(head))) return { blocked: true, reason: "anti_bot", vendor: "perimeterx", challenge: true };189 if (/Reference #\d|Reference #\d+\.[0-9a-f]+\.\d+\.[0-9a-f]+|errors\.edgesuite\.net|akamai\.com\/us\/en\/policies/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "akamai", challenge: false };190 if (/Incapsula incident|Request unsuccessful\. Incapsula/i.test(head) || (denied && /_Incapsula_Resource/i.test(head))) return { blocked: true, reason: "waf", vendor: "imperva", challenge: true };191 if (/<title>\s*Human Verification|aws-waf-token.*captcha|awswaf.*captcha/i.test(head) || (denied && /awswaf|challenge\.js\?/i.test(head))) return { blocked: true, reason: "waf", vendor: "aws_waf", challenge: true };192 if (/<title>\s*Kasada/i.test(head) || (denied && /kpsdk|ips\.js/i.test(head))) return { blocked: true, reason: "anti_bot", vendor: "kasada", challenge: true };193 if (/Pardon Our Interruption|distil_r_captcha|distil_referrer/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "distil", challenge: true };194 if (/Vercel Security Checkpoint|_vercel_challenge/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "vercel", challenge: true };195 if (/<title>\s*Blocked by Shape|_imp_apg_r_/i.test(head) || (denied && /shape-security/i.test(head))) return { blocked: true, reason: "anti_bot", vendor: "shape" };196 if (/Sucuri WebSite Firewall/i.test(head) || (denied && /sucuri\.net/i.test(head))) return { blocked: true, reason: "waf", vendor: "sucuri" };197 if (/www\.google\.com\/sorry\/|Our systems have detected unusual traffic/i.test(head)) return { blocked: true, reason: "captcha", vendor: "google", challenge: false };198 if (/recaptcha\/api\.js|g-recaptcha|hcaptcha\.com|h-captcha|arkoselabs|funcaptcha|<title>\s*[^<]*captcha/i.test(head) && /verify|robot|human|unusual traffic|security check|confirm you are/i.test(head)) {199 return { blocked: true, reason: "captcha", vendor: vendorFrom(server, head), challenge: false };200 }201 if (/<title>\s*(Access Denied|Request Blocked|Forbidden|Blocked|Bot Detected|Security Check|Attention Required|Verification Required|Are you a robot)/i.test(head)) {202 return { blocked: true, reason: "soft_block", vendor: vendorFrom(server, head) };203 }204 if (/enable javascript and cookies to continue|please enable javascript|checking your browser before accessing|verify you are human|verifying you are human|are you a robot|unusual traffic from your (computer|network)|automated access to this (site|page)|suspected bot|request could not be satisfied.*(bot|blocked)/i.test(head) && body.length < 60_000) {205 return { blocked: true, reason: "soft_block", vendor: vendorFrom(server, head), challenge: /enable javascript|checking your browser|verifying/i.test(head) };206 }207 if (status >= 400 && /akamai|imperva|sucuri|cloudflare|awselb|varnish.*block/i.test(server)) return { blocked: true, reason: "waf", vendor: vendorFrom(server, head) };208 // A 200 whose document is a script-only shell with no visible text is typically a JS gate.209 if (status === 200 && /<html/i.test(head) && /<script/i.test(head) && body.length < 20_000) {210 const visible = body.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, "").replace(/\s+/g, "");211 if (visible.length < 20 && !/<(img|a|form|input|p|h1|h2|main|article)\b/i.test(head)) return { blocked: true, reason: "empty_html", vendor: "generic", challenge: true };212 }213 return { blocked: false };214}215216function vendorFrom(server: string, head: string): BlockVendor {217 if (/cloudflare/i.test(server) || /cloudflare/i.test(head)) return "cloudflare";218 if (/akamai/i.test(server) || /akamai/i.test(head)) return "akamai";219 if (/imperva|incapsula/i.test(server + head)) return "imperva";220 if (/datadome/i.test(head)) return "datadome";221 if (/perimeterx|_px/i.test(head)) return "perimeterx";222 if (/kasada|kpsdk/i.test(head)) return "kasada";223 if (/awselb|aws/i.test(server) && /waf/i.test(head)) return "aws_waf";224 if (/vercel/i.test(server)) return "vercel";225 if (/sucuri/i.test(server + head)) return "sucuri";226 if (/fastly|varnish/i.test(server)) return "fastly";227 return "generic";228}229230function lowerKeys(h: Record<string, string>): Record<string, string> {231 const out: Record<string, string> = {};232 for (const [k, v] of Object.entries(h)) out[k.toLowerCase()] = v;233 return out;234}235236/** Whether an origin status is worth retrying on another route (transient upstream/edge errors). */237export function isTransientStatus(status: number): boolean {238 return status === 502 || status === 503 || status === 504 || status === 408 || (status >= 520 && status <= 530);239}240