import { setTimeout as sleep } from 'node:timers/promises'; import type { ConnectorManifest } from './manifest.js'; export interface HttpStats { requests: number; failures: number; rateLimitEvents: number; retries: number; } export class BodyTimeoutError extends Error { constructor( public readonly url: string, public readonly timeoutMs: number, ) { super(`body read timed out after ${timeoutMs} ms for ${url}`); this.name = 'BodyTimeoutError'; } } export class HttpError extends Error { constructor( public readonly status: number, public readonly url: string, public readonly bodySnippet: string, ) { super(`HTTP ${status} for ${url}: ${bodySnippet.slice(0, 200)}`); } } /** Token bucket rate limiter (CLAUDE.md §229). */ class TokenBucket { private tokens: number; private last = Date.now(); constructor( private readonly ratePerSec: number, private readonly burst: number, ) { this.tokens = burst; } async take(): Promise { for (;;) { const now = Date.now(); this.tokens = Math.min(this.burst, this.tokens + ((now - this.last) / 1000) * this.ratePerSec); this.last = now; if (this.tokens >= 1) { this.tokens -= 1; return; } await sleep(Math.ceil(((1 - this.tokens) / this.ratePerSec) * 1000)); } } } class Semaphore { private queue: Array<() => void> = []; private active = 0; constructor(private readonly max: number) {} async acquire(): Promise<() => void> { if (this.active >= this.max) await new Promise((resolve) => this.queue.push(resolve)); this.active++; let released = false; return () => { if (released) return; released = true; this.active--; const next = this.queue.shift(); if (next) next(); }; } } export interface HttpClientOptions { userAgent?: string; timeoutMs?: number; headers?: Record; } /** * Source-aware HTTP client: token bucket, bounded concurrency, exponential backoff with jitter, * Retry-After respect, request accounting for the ingest run. */ export class HttpClient { readonly stats: HttpStats = { requests: 0, failures: 0, rateLimitEvents: 0, retries: 0 }; private readonly bucket: TokenBucket | null; private readonly sem: Semaphore; private readonly retry: NonNullable; private readonly ua: string; private readonly timeoutMs: number; private readonly headers: Record; constructor(manifest: ConnectorManifest, opts: HttpClientOptions = {}) { const rps = manifest.rateLimits.requestsPerSecond ?? (manifest.rateLimits.requestsPerMinute ? manifest.rateLimits.requestsPerMinute / 60 : undefined); this.bucket = rps ? new TokenBucket(rps, Math.max(1, Math.ceil(rps))) : null; this.sem = new Semaphore(manifest.rateLimits.maxConcurrency); this.retry = manifest.retryPolicy; // Plain product token only: some WAFs (e.g. oncotree.mskcc.org, Akamai) reject UAs containing URLs/parentheses. this.ua = opts.userAgent ?? `CancerIndex/0.1`; this.timeoutMs = opts.timeoutMs ?? 60_000; this.headers = opts.headers ?? {}; } /** One attempt: returns the Response (any status) or throws on network error. */ private async attempt(url: string, init: RequestInit): Promise { const release = await this.sem.acquire(); try { if (this.bucket) await this.bucket.take(); this.stats.requests++; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), this.timeoutMs); try { return await fetch(url, { ...init, headers: { 'user-agent': this.ua, accept: 'application/json, text/plain, */*', ...this.headers, ...(init.headers as Record | undefined) }, signal: controller.signal, }); } finally { clearTimeout(timer); } } finally { release(); } } async request(url: string, init: RequestInit = {}): Promise { for (let attempt = 0; ; attempt++) { let res: Response; try { res = await this.attempt(url, init); } catch (err) { if (attempt >= this.retry.maxRetries) { this.stats.failures++; throw err; } this.stats.retries++; await sleep(this.backoff(attempt)); continue; } if (res.ok) return res; const body = await res.text().catch(() => ''); if (res.status === 429) this.stats.rateLimitEvents++; const retryable = res.status === 429 || res.status >= 500 || res.status === 408; if (!retryable || attempt >= this.retry.maxRetries) { this.stats.failures++; throw new HttpError(res.status, url, body); } const retryAfter = Number(res.headers.get('retry-after')); const delay = Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter * 1000, 120_000) : this.backoff(attempt); this.stats.retries++; await sleep(delay); } } /** * Read a body under the same time budget as the headers. Without this a stalled body (server * accepted the request, sent headers, then went silent — observed with ChEMBL) hangs forever * because the AbortController is cleared once headers arrive. */ private async readBody(res: Response, url: string, read: (r: Response) => Promise): Promise { let timer: ReturnType | undefined; const timeout = new Promise((_, reject) => { timer = setTimeout(() => { res.body?.cancel().catch(() => {}); reject(new BodyTimeoutError(url, this.timeoutMs)); }, this.timeoutMs); }); try { return await Promise.race([read(res), timeout]); } finally { clearTimeout(timer); } } /** Request + body read with retries on body stalls (same backoff as request()). */ private async withBody(url: string, init: RequestInit | undefined, read: (r: Response) => Promise): Promise { for (let attempt = 0; ; attempt++) { const res = await this.request(url, init); try { return await this.readBody(res, url, read); } catch (err) { if (!(err instanceof BodyTimeoutError) || attempt >= this.retry.maxRetries) { this.stats.failures++; throw err; } this.stats.retries++; await sleep(this.backoff(attempt)); } } } async json(url: string, init?: RequestInit): Promise { return this.withBody(url, init, (r) => r.json() as Promise); } async text(url: string, init?: RequestInit): Promise { return this.withBody(url, init, (r) => r.text()); } async postJson(url: string, body: unknown, init: RequestInit = {}): Promise { return this.json(url, { ...init, method: 'POST', headers: { 'content-type': 'application/json', ...(init.headers as Record | undefined) }, body: JSON.stringify(body) }); } private backoff(attempt: number): number { const base = Math.min(this.retry.maxDelayMs, this.retry.baseDelayMs * 2 ** attempt); return Math.round(base / 2 + Math.random() * (base / 2)); } }