Repository Public Research preview
1/**2 * =============================================================================3 * SP BOUCHER | SOFTWARE & RESEARCH4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : LLM API toolkits8 * File : scholarly/mcp/lib/http.mjs9 * Module : scholarly10 * Repository : https://git.spboucher.ai/llm-api-toolkits11 * Created : 2026-11-1012 * Last Modified : 2026-11-1013 *14 * Author : Simon-Pierre Boucher15 * Position : Professor of Finance16 * Institution : Université du Québec en Outaouais (UQO)17 * Organization : SP Boucher18 * Website : https://spboucher.ai19 * Git : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development : dev@spboucher.ai23 * Support : support@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142/**43 * HTTP layer shared by every source: generic User-Agent, timeouts, retries with backoff on 429/5xx,44 * per-host concurrency and spacing, and a small TTL cache. No dependency, no credential.45 */46export class ScholarlyError extends Error {47 constructor(code, message, extra) { super(message); this.name = "ScholarlyError"; this.code = code; Object.assign(this, extra ?? {}); }48}4950export const USER_AGENT = "scholarly-mcp/1.0";51const HOST_LIMITS = {52 "api.crossref.org": { concurrency: 3, gapMs: 60 },53 "api.openalex.org": { concurrency: 2, gapMs: 120 },54 "export.arxiv.org": { concurrency: 1, gapMs: 3000 }, // arXiv asks for one request every three seconds55 "doi.org": { concurrency: 3, gapMs: 60 },56};57const sleep = (ms) => new Promise((r) => setTimeout(r, ms));5859export function createHttp({ fetchImpl = globalThis.fetch, timeoutMs = 20000, retries = 3, cacheTtlMs = 15 * 60_000, cacheMax = 500, now = () => Date.now(), wait = sleep } = {}) {60 const cache = new Map();61 const hosts = new Map();62 const stats = { requests: 0, cache_hits: 0, retries: 0, by_host: {}, openalex_budget: null };6364 function slot(host) {65 if (!hosts.has(host)) hosts.set(host, { active: 0, queue: [], last: 0 });66 return hosts.get(host);67 }68 async function acquire(host) {69 const lim = HOST_LIMITS[host] ?? { concurrency: 2, gapMs: 100 }, s = slot(host);70 if (s.active >= lim.concurrency) await new Promise((res) => s.queue.push(res));71 s.active++;72 const gap = s.last + lim.gapMs - now();73 s.last = Math.max(now(), s.last + lim.gapMs);74 if (gap > 0) await wait(gap);75 }76 function release(host) { const s = slot(host); s.active--; const next = s.queue.shift(); if (next) next(); }7778 /** GET a URL. `as` is "json" or "text". 404 raises not_found; 429 after retries raises rate_limited. */79 async function get(url, { as = "json", headers = {}, cache: useCache = true, notFoundMessage } = {}) {80 const key = `${as} ${url} ${headers.Accept ?? ""}`;81 if (useCache) {82 const hit = cache.get(key);83 if (hit && hit.expires > now()) { stats.cache_hits++; return hit.value; }84 }85 const host = new URL(url).host;86 let lastErr;87 for (let attempt = 0; attempt <= retries; attempt++) {88 await acquire(host);89 let res;90 try {91 stats.requests++; stats.by_host[host] = (stats.by_host[host] ?? 0) + 1;92 const ctl = new AbortController(), timer = setTimeout(() => ctl.abort(), timeoutMs);93 try { res = await fetchImpl(url, { headers: { "User-Agent": USER_AGENT, Accept: as === "json" ? "application/json" : "*/*", ...headers }, redirect: "follow", signal: ctl.signal }); }94 finally { clearTimeout(timer); }95 } catch (e) {96 release(host);97 lastErr = new ScholarlyError(e?.name === "AbortError" ? "timeout" : "upstream_error", e?.name === "AbortError" ? `${host} did not answer within ${Math.round(timeoutMs / 1000)} s.` : `Could not reach ${host}: ${String(e?.message ?? e).slice(0, 160)}`, { host });98 if (attempt < retries) { stats.retries++; await wait(400 * 2 ** attempt); continue; }99 throw lastErr;100 }101 release(host);102 if (host === "api.openalex.org") { const rem = res.headers.get("x-ratelimit-remaining-usd"), lim = res.headers.get("x-ratelimit-limit-usd"); if (rem != null) stats.openalex_budget = { remaining_usd: Number(rem), limit_usd: lim == null ? null : Number(lim) }; }103 if (res.status === 404 || res.status === 410) throw new ScholarlyError("not_found", notFoundMessage ?? `Not found at ${host}.`, { host, status: res.status });104 if (res.status === 429 || res.status >= 500) {105 const ra = Number(res.headers.get("retry-after"));106 lastErr = new ScholarlyError(res.status === 429 ? "rate_limited" : "upstream_error", res.status === 429 ? `${host} is rate limiting this client${host === "api.openalex.org" ? " (the key-less OpenAlex budget is small and shared per IP address; set OPENALEX_API_KEY)" : ""}.` : `${host} answered HTTP ${res.status}.`, { host, status: res.status });107 if (attempt < retries && !(res.status === 429 && ra > 30)) { stats.retries++; await wait(Number.isFinite(ra) && ra > 0 ? Math.min(ra * 1000, 15000) : 600 * 2 ** attempt); continue; }108 throw lastErr;109 }110 if (!res.ok) throw new ScholarlyError(res.status === 400 || res.status === 422 ? "invalid_input" : "upstream_error", `${host} answered HTTP ${res.status}.`, { host, status: res.status });111 let value;112 try { value = as === "json" ? await res.json() : await res.text(); }113 catch { throw new ScholarlyError("upstream_error", `${host} returned a body that could not be read as ${as}.`, { host }); }114 if (useCache) {115 if (cache.size >= cacheMax) cache.delete(cache.keys().next().value);116 cache.set(key, { value, expires: now() + cacheTtlMs });117 }118 return value;119 }120 throw lastErr;121 }122 return { get, stats, clearCache: () => cache.clear(), cacheSize: () => cache.size };123}124125/** Run async jobs with a fixed number in flight; results keep input order. */126export async function mapLimit(items, limit, fn) {127 const out = new Array(items.length);128 let next = 0;129 await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {130 while (next < items.length) { const i = next++; out[i] = await fn(items[i], i); }131 }));132 return out;133}134