spb/market-atlas

1import { isIP } from "node:net";2import { lookup } from "node:dns/promises";3import { RateLimiter, backoffMs } from "./ratelimit.js";4import { redactUrl } from "./redact.js";56export interface HttpRequestOptions {7  method?: "GET" | "POST" | "HEAD";8  headers?: Record<string, string>;9  body?: string;10  timeoutMs?: number;11  /** Use conditional requests (ETag / If-Modified-Since) keyed by URL. Default true for GET. */12  conditional?: boolean;13  retries?: number;14  /** Accept 304 as success and return cached body. */15  acceptNotModified?: boolean;16}1718export interface HttpResponse {19  status: number;20  ok: boolean;21  notModified: boolean;22  headers: Record<string, string>;23  text: string;24  url: string;25  durationMs: number;26  fromCache: boolean;27}2829export class HttpError extends Error {30  constructor(31    message: string,32    public status: number,33    public url: string,34    public retryAfterMs: number | null = null,35  ) {36    super(message);37    this.name = "HttpError";38  }39}4041interface CacheEntry {42  etag: string | null;43  lastModified: string | null;44  body: string;45  headers: Record<string, string>;46}4748export interface HttpClientOptions {49  userAgent: string;50  limiter?: RateLimiter;51  defaultTimeoutMs?: number;52  /** Called for every request (metrics). */53  onRequest?: (info: { host: string; status: number; durationMs: number; ok: boolean }) => void;54  fetchImpl?: typeof fetch;55}5657/**58 * Polite HTTP client: central per-host rate limiter, ETag/If-Modified-Since cache,59 * bounded retries with jittered backoff, secret-free URLs in errors, identified User-Agent.60 */61export class HttpClient {62  private cache = new Map<string, CacheEntry>();63  readonly limiter: RateLimiter;64  private fetchImpl: typeof fetch;6566  constructor(private opts: HttpClientOptions) {67    this.limiter = opts.limiter ?? new RateLimiter();68    this.fetchImpl = opts.fetchImpl ?? fetch;69  }7071  async request(url: string, options: HttpRequestOptions = {}): Promise<HttpResponse> {72    const method = options.method ?? "GET";73    const u = new URL(url);74    const retries = options.retries ?? 2;75    const conditional = options.conditional ?? method === "GET";76    let attempt = 0;77    for (;;) {78      await this.limiter.acquire(u.host);79      const started = Date.now();80      const headers: Record<string, string> = {81        "user-agent": this.opts.userAgent,82        accept: "application/json, text/xml, application/xml, text/html, text/csv, text/plain;q=0.9, */*;q=0.8",83        "accept-encoding": "gzip, br",84        ...(options.headers ?? {}),85      };86      const cached = conditional ? this.cache.get(url) : undefined;87      if (cached?.etag) headers["if-none-match"] = cached.etag;88      if (cached?.lastModified) headers["if-modified-since"] = cached.lastModified;8990      const ctrl = new AbortController();91      const timer = setTimeout(() => ctrl.abort(), options.timeoutMs ?? this.opts.defaultTimeoutMs ?? 20_000);92      try {93        const res = await this.fetchImpl(url, { method, headers, body: options.body, signal: ctrl.signal, redirect: "follow" });94        const durationMs = Date.now() - started;95        const resHeaders: Record<string, string> = {};96        res.headers.forEach((v, k) => (resHeaders[k] = v));97        this.opts.onRequest?.({ host: u.host, status: res.status, durationMs, ok: res.ok || res.status === 304 });9899        if (res.status === 304 && cached) {100          return { status: 304, ok: true, notModified: true, headers: resHeaders, text: cached.body, url, durationMs, fromCache: true };101        }102        if (res.status === 429 || res.status >= 500) {103          const ra = res.headers.get("retry-after");104          const retryAfterMs = ra ? (Number.isFinite(Number(ra)) ? Number(ra) * 1000 : Math.max(0, Date.parse(ra) - Date.now())) : null;105          if (attempt < retries) {106            attempt++;107            await sleep(retryAfterMs ?? backoffMs(attempt, 750, 30_000));108            continue;109          }110          throw new HttpError(`HTTP ${res.status} from ${redactUrl(url)}`, res.status, redactUrl(url), retryAfterMs);111        }112        const text = await res.text();113        if (!res.ok) throw new HttpError(`HTTP ${res.status} from ${redactUrl(url)}`, res.status, redactUrl(url));114        if (conditional) {115          const etag = res.headers.get("etag");116          const lastModified = res.headers.get("last-modified");117          if (etag || lastModified) this.cache.set(url, { etag, lastModified, body: text, headers: resHeaders });118        }119        return { status: res.status, ok: true, notModified: false, headers: resHeaders, text, url, durationMs, fromCache: false };120      } catch (err) {121        if (err instanceof HttpError) throw err;122        if (attempt < Math.max(retries, 3)) {123          attempt++;124          await sleep(1000 + backoffMs(attempt, 1500, 20_000)); // DNS/TLS hiccups at boot need more than a few hundred ms125          continue;126        }127        const msg = err instanceof Error ? `${err.message}${(err as { cause?: { code?: string } }).cause?.code ? ` [${(err as { cause?: { code?: string } }).cause?.code}]` : ""}` : String(err);128        throw new HttpError(`request failed: ${msg} (${redactUrl(url)})`, 0, redactUrl(url));129      } finally {130        clearTimeout(timer);131      }132    }133  }134135  async getText(url: string, options?: HttpRequestOptions): Promise<HttpResponse> {136    return this.request(url, { ...options, method: "GET" });137  }138139  async getJson<T = unknown>(url: string, options?: HttpRequestOptions): Promise<{ data: T; response: HttpResponse }> {140    const response = await this.request(url, { ...options, method: "GET", headers: { accept: "application/json", ...(options?.headers ?? {}) } });141    try {142      return { data: JSON.parse(response.text) as T, response };143    } catch {144      throw new HttpError(`invalid JSON from ${redactUrl(url)}`, response.status, redactUrl(url));145    }146  }147148  cacheSize(): number {149    return this.cache.size;150  }151}152153export function sleep(ms: number): Promise<void> {154  return new Promise((r) => setTimeout(r, ms));155}156157const PRIVATE_V4 = [158  /^10\./,159  /^127\./,160  /^0\./,161  /^169\.254\./,162  /^172\.(1[6-9]|2\d|3[01])\./,163  /^192\.168\./,164  /^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./,165  /^22[4-9]\./,166  /^2[3-5]\d\./,167];168169export function isPrivateAddress(ip: string): boolean {170  const v = isIP(ip);171  if (v === 4) return PRIVATE_V4.some((re) => re.test(ip));172  if (v === 6) {173    const low = ip.toLowerCase();174    return low === "::1" || low === "::" || low.startsWith("fc") || low.startsWith("fd") || low.startsWith("fe80") || low.startsWith("::ffff:");175  }176  return true;177}178179/**180 * SSRF guard for user-supplied URLs (discovery): public http(s) only, no private/link-local/metadata181 * targets, resolved addresses checked too. Throws on violation.182 */183export async function assertPublicUrl(input: string): Promise<URL> {184  const u = new URL(input);185  if (u.protocol !== "http:" && u.protocol !== "https:") throw new Error("only http(s) URLs are allowed");186  if (u.username || u.password) throw new Error("credentials in URL are not allowed");187  const host = u.hostname.toLowerCase();188  if (host === "localhost" || host.endsWith(".local") || host.endsWith(".internal") || host.endsWith(".maclustr.io") || host === "metadata.google.internal")189    throw new Error("internal hostnames are not allowed");190  if (isIP(host)) {191    if (isPrivateAddress(host)) throw new Error("private addresses are not allowed");192    return u;193  }194  const addrs = await lookup(host, { all: true });195  if (!addrs.length) throw new Error("hostname does not resolve");196  for (const a of addrs) if (isPrivateAddress(a.address)) throw new Error("hostname resolves to a private address");197  return u;198}199