spb/rareindex

1import type { ExtractionResult } from '@rareindex/shared';2import type { FetchOptions } from '../types.js';3import { policyFor } from '../domains.js';45export const DEFAULT_UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; market-data research; contact: data@rareindex.io)';6/** Used only where connectors/domains.json sets userAgent:"browser" (site serves bots a degraded page; terms allow). */7export const BROWSER_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15';89export interface HttpEngineOptions {10  userAgent?: string;11  defaultTimeoutMs?: number;12  maxRetries?: number;13}1415/**16 * Direct HTTP engine for official APIs, feeds and plain public pages. Honest user agent, no17 * evasion, exponential backoff on 429/5xx, and conditional requests when the caller passes18 * If-None-Match / If-Modified-Since headers (budget engine).19 */20export function createHttpEngine(opts: HttpEngineOptions = {}) {21  const ua = opts.userAgent ?? DEFAULT_UA;2223  async function fetchUrl(url: string, o: FetchOptions = {}): Promise<ExtractionResult> {24    const started = Date.now();25    const policy = policyFor(url);26    const maxRetries = opts.maxRetries ?? policy.maxRetries;27    const timeoutMs = o.timeoutMs ?? opts.defaultTimeoutMs ?? policy.timeoutMs;28    let attempt = 0;29    let lastErr: string | null = null;30    while (attempt <= maxRetries) {31      attempt++;32      const ctrl = new AbortController();33      const timer = setTimeout(() => ctrl.abort(), timeoutMs);34      try {35        const res = await fetch(url, {36          method: o.method ?? 'GET',37          headers: {38            'user-agent': o.headers?.['user-agent'] ?? (policyFor(url).userAgent === 'browser' ? BROWSER_UA : ua),39            accept: o.responseType === 'text' ? 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' : 'application/json, text/plain, */*',40            'accept-language': 'en-US,en;q=0.9',41            ...(o.body !== undefined ? { 'content-type': 'application/json' } : {}),42            ...(o.headers ?? {}),43          },44          body: o.body !== undefined ? JSON.stringify(o.body) : undefined,45          signal: ctrl.signal,46          redirect: 'follow',47        });48        clearTimeout(timer);49        const status = res.status;50        if (status === 304) {51          return { success: true, engine: 'api', url, finalUrl: res.url, httpStatus: 304, html: null, markdown: null, json: null, qualityScore: 1, requiresReview: false, error: null, costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };52        }53        if ((status === 429 || status >= 500) && attempt <= maxRetries) {54          const retryAfter = Number(res.headers.get('retry-after'));55          // Never sleep longer than a minute inside a fetch: a long Retry-After (comics.org sends ~30 min) is a56          // hard rate limit → return the 429 and let the scheduler/circuit breaker back off instead.57          if (Number.isFinite(retryAfter) && retryAfter > 60) {58            return { success: false, engine: 'api', url, finalUrl: res.url, httpStatus: status, html: null, markdown: null, json: null, qualityScore: 0, requiresReview: false, error: `HTTP ${status} (retry-after ${retryAfter}s)`, costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };59          }60          const wait = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : Math.min(30_000, 1000 * 2 ** attempt);61          lastErr = `HTTP ${status}`;62          await new Promise((r) => setTimeout(r, wait));63          continue;64        }65        const ctype = res.headers.get('content-type') ?? '';66        if (o.responseType === 'binary') {67          const buf = new Uint8Array(await res.arrayBuffer());68          const okB = res.ok && (o.failOnHttpError === false || status < 400);69          return { success: okB, engine: 'api', url, finalUrl: res.url, httpStatus: status, html: null, markdown: null, json: null, buffer: buf, qualityScore: okB ? 1 : 0, requiresReview: false, error: okB ? null : `HTTP ${status}`, costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };70        }71        const isJson = o.responseType !== 'text' && (ctype.includes('json') || o.responseType === 'json');72        let json: unknown = null;73        let text: string | null = null;74        if (isJson) {75          const raw = await res.text();76          try {77            json = raw ? JSON.parse(raw) : null;78          } catch {79            text = raw;80          }81        } else {82          text = await res.text();83        }84        const ok = res.ok && (o.failOnHttpError === false || status < 400);85        return {86          success: ok,87          engine: 'api',88          url,89          finalUrl: res.url,90          httpStatus: status,91          html: text,92          markdown: null,93          json,94          qualityScore: ok ? 1 : 0,95          requiresReview: false,96          error: ok ? null : `HTTP ${status}`,97          costCredits: 0,98          durationMs: Date.now() - started,99          fetchedAt: new Date(),100        };101      } catch (err) {102        clearTimeout(timer);103        lastErr = err instanceof Error ? err.message : String(err);104        if (attempt > maxRetries) break;105        await new Promise((r) => setTimeout(r, Math.min(15_000, 500 * 2 ** attempt)));106      }107    }108    return { success: false, engine: 'api', url, finalUrl: null, httpStatus: null, html: null, markdown: null, json: null, qualityScore: 0, requiresReview: false, error: lastErr ?? 'fetch failed', costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };109  }110111  return { fetch: fetchUrl };112}113114export type HttpEngine = ReturnType<typeof createHttpEngine>;115