SPB Git

Repository Public Research preview

http.mjs 6.9 KB 134 lines javascript

1/**2 * =============================================================================3 *                  SP BOUCHER | SOFTWARE & RESEARCH4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : LLM API toolkits8 * File             : scholarly/skill/citation-verification/scripts/lib/http.mjs9 * Module           : scholarly10 * Repository       : https://git.spboucher.ai/llm-api-toolkits11 * Created          : 2026-11-1012 * Last Modified    : 2026-11-1013 *14 * Author           : Simon-Pierre Boucher15 * Position         : Professor of Finance16 * Institution      : Université du Québec en Outaouais (UQO)17 * Organization     : SP Boucher18 * Website          : https://spboucher.ai19 * Git              : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development      : dev@spboucher.ai23 * Support          : support@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142/**43 * HTTP layer shared by every source: generic User-Agent, timeouts, retries with backoff on 429/5xx,44 * per-host concurrency and spacing, and a small TTL cache. No dependency, no credential.45 */46export class ScholarlyError extends Error {47  constructor(code, message, extra) { super(message); this.name = "ScholarlyError"; this.code = code; Object.assign(this, extra ?? {}); }48}4950export const USER_AGENT = "scholarly-mcp/1.0";51const HOST_LIMITS = {52  "api.crossref.org": { concurrency: 3, gapMs: 60 },53  "api.openalex.org": { concurrency: 2, gapMs: 120 },54  "export.arxiv.org": { concurrency: 1, gapMs: 3000 }, // arXiv asks for one request every three seconds55  "doi.org": { concurrency: 3, gapMs: 60 },56};57const sleep = (ms) => new Promise((r) => setTimeout(r, ms));5859export function createHttp({ fetchImpl = globalThis.fetch, timeoutMs = 20000, retries = 3, cacheTtlMs = 15 * 60_000, cacheMax = 500, now = () => Date.now(), wait = sleep } = {}) {60  const cache = new Map();61  const hosts = new Map();62  const stats = { requests: 0, cache_hits: 0, retries: 0, by_host: {}, openalex_budget: null };6364  function slot(host) {65    if (!hosts.has(host)) hosts.set(host, { active: 0, queue: [], last: 0 });66    return hosts.get(host);67  }68  async function acquire(host) {69    const lim = HOST_LIMITS[host] ?? { concurrency: 2, gapMs: 100 }, s = slot(host);70    if (s.active >= lim.concurrency) await new Promise((res) => s.queue.push(res));71    s.active++;72    const gap = s.last + lim.gapMs - now();73    s.last = Math.max(now(), s.last + lim.gapMs);74    if (gap > 0) await wait(gap);75  }76  function release(host) { const s = slot(host); s.active--; const next = s.queue.shift(); if (next) next(); }7778  /** GET a URL. `as` is "json" or "text". 404 raises not_found; 429 after retries raises rate_limited. */79  async function get(url, { as = "json", headers = {}, cache: useCache = true, notFoundMessage } = {}) {80    const key = `${as} ${url} ${headers.Accept ?? ""}`;81    if (useCache) {82      const hit = cache.get(key);83      if (hit && hit.expires > now()) { stats.cache_hits++; return hit.value; }84    }85    const host = new URL(url).host;86    let lastErr;87    for (let attempt = 0; attempt <= retries; attempt++) {88      await acquire(host);89      let res;90      try {91        stats.requests++; stats.by_host[host] = (stats.by_host[host] ?? 0) + 1;92        const ctl = new AbortController(), timer = setTimeout(() => ctl.abort(), timeoutMs);93        try { res = await fetchImpl(url, { headers: { "User-Agent": USER_AGENT, Accept: as === "json" ? "application/json" : "*/*", ...headers }, redirect: "follow", signal: ctl.signal }); }94        finally { clearTimeout(timer); }95      } catch (e) {96        release(host);97        lastErr = new ScholarlyError(e?.name === "AbortError" ? "timeout" : "upstream_error", e?.name === "AbortError" ? `${host} did not answer within ${Math.round(timeoutMs / 1000)} s.` : `Could not reach ${host}: ${String(e?.message ?? e).slice(0, 160)}`, { host });98        if (attempt < retries) { stats.retries++; await wait(400 * 2 ** attempt); continue; }99        throw lastErr;100      }101      release(host);102      if (host === "api.openalex.org") { const rem = res.headers.get("x-ratelimit-remaining-usd"), lim = res.headers.get("x-ratelimit-limit-usd"); if (rem != null) stats.openalex_budget = { remaining_usd: Number(rem), limit_usd: lim == null ? null : Number(lim) }; }103      if (res.status === 404 || res.status === 410) throw new ScholarlyError("not_found", notFoundMessage ?? `Not found at ${host}.`, { host, status: res.status });104      if (res.status === 429 || res.status >= 500) {105        const ra = Number(res.headers.get("retry-after"));106        lastErr = new ScholarlyError(res.status === 429 ? "rate_limited" : "upstream_error", res.status === 429 ? `${host} is rate limiting this client${host === "api.openalex.org" ? " (the key-less OpenAlex budget is small and shared per IP address; set OPENALEX_API_KEY)" : ""}.` : `${host} answered HTTP ${res.status}.`, { host, status: res.status });107        if (attempt < retries && !(res.status === 429 && ra > 30)) { stats.retries++; await wait(Number.isFinite(ra) && ra > 0 ? Math.min(ra * 1000, 15000) : 600 * 2 ** attempt); continue; }108        throw lastErr;109      }110      if (!res.ok) throw new ScholarlyError(res.status === 400 || res.status === 422 ? "invalid_input" : "upstream_error", `${host} answered HTTP ${res.status}.`, { host, status: res.status });111      let value;112      try { value = as === "json" ? await res.json() : await res.text(); }113      catch { throw new ScholarlyError("upstream_error", `${host} returned a body that could not be read as ${as}.`, { host }); }114      if (useCache) {115        if (cache.size >= cacheMax) cache.delete(cache.keys().next().value);116        cache.set(key, { value, expires: now() + cacheTtlMs });117      }118      return value;119    }120    throw lastErr;121  }122  return { get, stats, clearCache: () => cache.clear(), cacheSize: () => cache.size };123}124125/** Run async jobs with a fixed number in flight; results keep input order. */126export async function mapLimit(items, limit, fn) {127  const out = new Array(items.length);128  let next = 0;129  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {130    while (next < items.length) { const i = next++; out[i] = await fn(items[i], i); }131  }));132  return out;133}134
Files main
Menu

Theme