1import type { ExtractionResult } from '@rareindex/shared';2import type { FetchOptions } from '../types.js';34const DEFAULT_UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; market-data research; contact: data@rareindex.io)';56export interface HttpEngineOptions {7 userAgent?: string;8 defaultTimeoutMs?: number;9 maxRetries?: number;10}1112/**13 * Direct HTTP engine for official APIs, feeds and plain public pages. Honest user agent, no14 * evasion, exponential backoff on 429/5xx, and conditional requests when the caller passes15 * If-None-Match / If-Modified-Since headers (budget engine).16 */17export function createHttpEngine(opts: HttpEngineOptions = {}) {18 const ua = opts.userAgent ?? DEFAULT_UA;19 const maxRetries = opts.maxRetries ?? 3;2021 async function fetchUrl(url: string, o: FetchOptions = {}): Promise<ExtractionResult> {22 const started = Date.now();23 const timeoutMs = o.timeoutMs ?? opts.defaultTimeoutMs ?? 30_000;24 let attempt = 0;25 let lastErr: string | null = null;26 while (attempt <= maxRetries) {27 attempt++;28 const ctrl = new AbortController();29 const timer = setTimeout(() => ctrl.abort(), timeoutMs);30 try {31 const res = await fetch(url, {32 method: o.method ?? 'GET',33 headers: {34 'user-agent': ua,35 accept: o.responseType === 'text' ? 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' : 'application/json, text/plain, */*',36 'accept-language': 'en-US,en;q=0.9',37 ...(o.body !== undefined ? { 'content-type': 'application/json' } : {}),38 ...(o.headers ?? {}),39 },40 body: o.body !== undefined ? JSON.stringify(o.body) : undefined,41 signal: ctrl.signal,42 redirect: 'follow',43 });44 clearTimeout(timer);45 const status = res.status;46 if (status === 304) {47 return { success: true, engine: 'api', url, finalUrl: res.url, httpStatus: 304, html: null, markdown: null, json: null, qualityScore: 1, requiresReview: false, error: null, costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };48 }49 if ((status === 429 || status >= 500) && attempt <= maxRetries) {50 const retryAfter = Number(res.headers.get('retry-after'));51 const wait = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : Math.min(30_000, 1000 * 2 ** attempt);52 lastErr = `HTTP ${status}`;53 await new Promise((r) => setTimeout(r, wait));54 continue;55 }56 const ctype = res.headers.get('content-type') ?? '';57 const isJson = o.responseType !== 'text' && (ctype.includes('json') || o.responseType === 'json');58 let json: unknown = null;59 let text: string | null = null;60 if (isJson) {61 const raw = await res.text();62 try {63 json = raw ? JSON.parse(raw) : null;64 } catch {65 text = raw;66 }67 } else {68 text = await res.text();69 }70 const ok = res.ok && (o.failOnHttpError === false || status < 400);71 return {72 success: ok,73 engine: 'api',74 url,75 finalUrl: res.url,76 httpStatus: status,77 html: text,78 markdown: null,79 json,80 qualityScore: ok ? 1 : 0,81 requiresReview: false,82 error: ok ? null : `HTTP ${status}`,83 costCredits: 0,84 durationMs: Date.now() - started,85 fetchedAt: new Date(),86 };87 } catch (err) {88 clearTimeout(timer);89 lastErr = err instanceof Error ? err.message : String(err);90 if (attempt > maxRetries) break;91 await new Promise((r) => setTimeout(r, Math.min(15_000, 500 * 2 ** attempt)));92 }93 }94 return { success: false, engine: 'api', url, finalUrl: null, httpStatus: null, html: null, markdown: null, json: null, qualityScore: 0, requiresReview: false, error: lastErr ?? 'fetch failed', costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };95 }9697 return { fetch: fetchUrl };98}99100export type HttpEngine = ReturnType<typeof createHttpEngine>;101