SPB Git

Repository Public Research preview

sources.mjs 13.9 KB 183 lines javascript

1/**2 * =============================================================================3 *                  SP BOUCHER | SOFTWARE & RESEARCH4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : LLM API toolkits8 * File             : scholarly/mcp/lib/sources.mjs9 * Module           : scholarly10 * Repository       : https://git.spboucher.ai/llm-api-toolkits11 * Created          : 2026-11-1012 * Last Modified    : 2026-11-1013 *14 * Author           : Simon-Pierre Boucher15 * Position         : Professor of Finance16 * Institution      : Université du Québec en Outaouais (UQO)17 * Organization     : SP Boucher18 * Website          : https://spboucher.ai19 * Git              : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development      : dev@spboucher.ai23 * Support          : support@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142import { ScholarlyError } from "./http.mjs";43import { normalizeArxivId, normalizeDoi, parseYear } from "./normalize.mjs";4445/**46 * One function per upstream service, each mapping its answer to the same Work shape.47 * Crossref is the backbone (free, generous); OpenAlex adds the citation graph, open-access48 * locations and a retraction flag but has a small key-less budget; arXiv covers preprints;49 * doi.org content negotiation resolves DOIs from any registration agency (DataCite included).50 */51const clean = (s) => (s == null ? null : String(s).replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim() || null);52const first = (a) => (Array.isArray(a) ? a[0] ?? null : a ?? null);5354export function workFromCrossref(m) {55  const date = m.issued?.["date-parts"]?.[0] ?? m.published?.["date-parts"]?.[0] ?? [];56  const updates = (m["updated-by"] ?? []).map((u) => ({ type: u.type, label: u.label ?? null, doi: normalizeDoi(u.DOI ?? ""), date: u.updated?.["date-time"]?.slice(0, 10) ?? null, source: u.source ?? null }));57  const retraction = updates.find((u) => /retraction|withdrawal|removal/i.test(u.type ?? "")) ?? null;58  return {59    source: "crossref", doi: normalizeDoi(m.DOI ?? ""), title: clean(first(m.title)), subtitle: clean(first(m.subtitle)),60    authors: (m.author ?? []).map((a) => ({ family: a.family ?? null, given: a.given ?? null, name: a.name ?? ([a.given, a.family].filter(Boolean).join(" ") || null), orcid: a.ORCID ? a.ORCID.replace(/^https?:\/\/orcid\.org\//, "") : null })),61    year: date[0] ?? null, date: date.length ? date.map((x, i) => (i ? String(x).padStart(2, "0") : x)).join("-") : null,62    venue: clean(first(m["container-title"])), volume: m.volume ?? null, issue: m.issue ?? null, pages: m.page ?? null, publisher: m.publisher ?? null,63    type: m.type ?? null, url: m.URL ?? (m.DOI ? `https://doi.org/${m.DOI}` : null), abstract: clean(m.abstract), citation_count: m["is-referenced-by-count"] ?? null,64    reference_count: m["reference-count"] ?? m["references-count"] ?? null, issn: m.ISSN ?? [], isbn: m.ISBN ?? [],65    is_retracted: !!retraction || /^\s*(retracted|withdrawn)\s*[:\-–]/i.test(first(m.title) ?? ""), retraction, updates,66    updates_other: (m["update-to"] ?? []).map((u) => ({ type: u.type, doi: normalizeDoi(u.DOI ?? "") })), // this record IS a notice about another work67    open_access: null, ids: { doi: normalizeDoi(m.DOI ?? "") },68  };69}7071export function abstractFromInvertedIndex(idx) {72  if (!idx || typeof idx !== "object") return null;73  const words = [];74  for (const [w, pos] of Object.entries(idx)) for (const p of pos) words[p] = w;75  return words.filter((w) => w !== undefined).join(" ") || null;76}7778export function workFromOpenAlex(w) {79  const loc = w.primary_location ?? {}, b = w.biblio ?? {}, oa = w.open_access ?? null, best = w.best_oa_location ?? null;80  return {81    source: "openalex", doi: normalizeDoi(w.doi ?? ""), title: clean(w.title ?? w.display_name), subtitle: null,82    authors: (w.authorships ?? []).map((a) => ({ family: null, given: null, name: a.author?.display_name ?? a.raw_author_name ?? null, orcid: a.author?.orcid ? a.author.orcid.replace(/^https?:\/\/orcid\.org\//, "") : null, openalex_id: a.author?.id?.split("/").pop() ?? null })),83    year: w.publication_year ?? null, date: w.publication_date ?? null, venue: clean(loc.source?.display_name), volume: b.volume ?? null, issue: b.issue ?? null,84    pages: b.first_page ? (b.last_page && b.last_page !== b.first_page ? `${b.first_page}-${b.last_page}` : b.first_page) : null, publisher: loc.source?.host_organization_name ?? null,85    type: w.type ?? null, url: loc.landing_page_url ?? w.doi ?? w.id ?? null, abstract: abstractFromInvertedIndex(w.abstract_inverted_index), citation_count: w.cited_by_count ?? null,86    reference_count: w.referenced_works_count ?? (Array.isArray(w.referenced_works) ? w.referenced_works.length : null), issn: loc.source?.issn ?? [], isbn: [],87    is_retracted: w.is_retracted === true, retraction: null, updates: [], updates_other: [],88    open_access: oa ? { is_oa: !!oa.is_oa, status: oa.oa_status ?? null, url: oa.oa_url ?? best?.pdf_url ?? best?.landing_page_url ?? null, pdf_url: best?.pdf_url ?? null, host: best?.source?.display_name ?? null, version: best?.version ?? null, license: best?.license ?? null } : null,89    ids: { doi: normalizeDoi(w.doi ?? ""), openalex: w.id?.split("/").pop() ?? null, pmid: w.ids?.pmid?.split("/").pop() ?? null },90  };91}9293export function workFromCsl(c) {94  const date = c.issued?.["date-parts"]?.[0] ?? [];95  return {96    source: "doi.org", doi: normalizeDoi(c.DOI ?? ""), title: clean(c.title), subtitle: null,97    authors: (c.author ?? []).map((a) => ({ family: a.family ?? null, given: a.given ?? null, name: a.literal ?? ([a.given, a.family].filter(Boolean).join(" ") || null), orcid: null })),98    year: date[0] ?? parseYear(c.issued?.raw) ?? null, date: date.length ? date.map((x, i) => (i ? String(x).padStart(2, "0") : x)).join("-") : null,99    venue: clean(c["container-title"]) ?? (c.publisher === "arXiv" ? "arXiv" : null), volume: c.volume ?? null, issue: c.issue ?? null, pages: c.page ?? null, publisher: c.publisher ?? null,100    type: c.type ?? null, url: c.URL ?? null, abstract: clean(c.abstract), citation_count: null, reference_count: null, issn: [], isbn: [],101    is_retracted: false, retraction: null, updates: [], updates_other: [], open_access: null, ids: { doi: normalizeDoi(c.DOI ?? "") },102  };103}104105const xml = (s) => String(s ?? "").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, "&").replace(/\s+/g, " ").trim();106const tag = (block, name) => { const m = new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`).exec(block); return m ? xml(m[1]) : null; };107108/** Parse an arXiv Atom feed. The feed is small and regular, so no XML library is needed. */109export function worksFromArxivFeed(feed) {110  const total = Number(/<opensearch:totalResults[^>]*>(\d+)</.exec(feed)?.[1] ?? 0);111  const works = [...String(feed).matchAll(/<entry>([\s\S]*?)<\/entry>/g)].map(([, e]) => {112    const idUrl = tag(e, "id") ?? "", id = normalizeArxivId(idUrl), version = /v(\d+)$/.exec(idUrl)?.[1] ?? null;113    const published = tag(e, "published"), doi = tag(e, "arxiv:doi");114    return {115      source: "arxiv", doi: normalizeDoi(doi ?? "") ?? (id ? `10.48550/arxiv.${id.toLowerCase()}` : null), published_doi: normalizeDoi(doi ?? ""), title: tag(e, "title"), subtitle: null,116      authors: [...e.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => ({ family: null, given: null, name: xml(m[1]), orcid: null })),117      year: published ? Number(published.slice(0, 4)) : null, date: published ? published.slice(0, 10) : null, updated: tag(e, "updated")?.slice(0, 10) ?? null,118      venue: "arXiv", journal_ref: tag(e, "arxiv:journal_ref"), volume: null, issue: null, pages: null, publisher: "arXiv", type: "preprint",119      url: id ? `https://arxiv.org/abs/${id}` : idUrl, abstract: tag(e, "summary"), citation_count: null, reference_count: null, issn: [], isbn: [],120      is_retracted: /withdrawn/i.test(tag(e, "arxiv:comment") ?? ""), retraction: null, updates: [], updates_other: [], comment: tag(e, "arxiv:comment"),121      categories: [...e.matchAll(/<category term="([^"]+)"/g)].map((m) => m[1]), version: version ? Number(version) : null,122      open_access: { is_oa: true, status: "green", url: id ? `https://arxiv.org/abs/${id}` : null, pdf_url: id ? `https://arxiv.org/pdf/${id}` : null, host: "arXiv", version: "submittedVersion", license: null },123      ids: { arxiv: id, doi: normalizeDoi(doi ?? "") },124    };125  }).filter((w) => w.title && !/^Error$/i.test(w.title));126  return { total, works };127}128129export function createSources(http, env = {}) {130  const mailto = env.SCHOLARLY_MAILTO && /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(env.SCHOLARLY_MAILTO) ? env.SCHOLARLY_MAILTO : null;131  const oaKey = env.OPENALEX_API_KEY || null;132  const cr = (path, params = {}) => { const u = new URL(`https://api.crossref.org${path}`); for (const [k, v] of Object.entries(params)) if (v != null && v !== "") u.searchParams.set(k, String(v)); if (mailto) u.searchParams.set("mailto", mailto); return u.href; };133  const oa = (path, params = {}) => { const u = new URL(`https://api.openalex.org${path}`); for (const [k, v] of Object.entries(params)) if (v != null && v !== "") u.searchParams.set(k, String(v)); if (mailto) u.searchParams.set("mailto", mailto); if (oaKey) u.searchParams.set("api_key", oaKey); return u.href; };134  const LIST = "id,doi,title,display_name,publication_year,publication_date,type,cited_by_count,is_retracted,authorships,primary_location,open_access,best_oa_location,biblio,ids";135  const encDoi = (doi) => encodeURIComponent(doi).replace(/%2F/gi, "/");136137  return {138    config: { mailto_set: !!mailto, openalex_key_set: !!oaKey },139    async crossrefWork(doi) {140      const d = await http.get(cr(`/works/${encDoi(doi)}`), { notFoundMessage: `Crossref has no record for DOI ${doi}.` });141      return { work: workFromCrossref(d.message), references: (d.message.reference ?? []).map((r) => ({ doi: normalizeDoi(r.DOI ?? ""), title: clean(r["article-title"] ?? r["volume-title"]), first_author: r.author ?? null, year: parseYear(r.year), venue: clean(r["journal-title"]), unstructured: clean(r.unstructured) })) };142    },143    async crossrefSearch({ query, bibliographic, author, title, from_year, to_year, type, rows = 10, offset = 0, sort }) {144      const filters = [];145      if (from_year) filters.push(`from-pub-date:${from_year}`);146      if (to_year) filters.push(`until-pub-date:${to_year}`);147      if (type) filters.push(`type:${type}`);148      const d = await http.get(cr("/works", { query, "query.bibliographic": bibliographic, "query.author": author, "query.title": title, filter: filters.join(",") || null, rows, offset: offset || null, sort: sort === "citations" ? "is-referenced-by-count" : sort === "date" ? "published" : null, order: sort && sort !== "relevance" ? "desc" : null }));149      return { total: d.message["total-results"], works: d.message.items.map((m) => ({ ...workFromCrossref(m), relevance: m.score ?? null })) };150    },151    async openalexWork(id) { // id: "doi:…" (full https DOI url), "W…", "pmid:…"152      return workFromOpenAlex(await http.get(oa(`/works/${id}`), { notFoundMessage: `OpenAlex has no record for ${id}.` }));153    },154    async openalexRaw(id, select) { return http.get(oa(`/works/${id}`, { select })); },155    async openalexList(params) {156      const d = await http.get(oa("/works", { select: LIST, ...params }));157      return { total: d.meta?.count ?? null, works: (d.results ?? []).map(workFromOpenAlex) };158    },159    async openalexAuthors(name, per = 5) {160      const d = await http.get(oa("/authors", { search: name, "per-page": per, select: "id,display_name,orcid,works_count,cited_by_count,last_known_institutions" }));161      return (d.results ?? []).map((a) => ({ openalex_id: a.id.split("/").pop(), name: a.display_name, orcid: a.orcid ? a.orcid.replace(/^https?:\/\/orcid\.org\//, "") : null, works_count: a.works_count, cited_by_count: a.cited_by_count, institutions: (a.last_known_institutions ?? []).map((i) => i.display_name) }));162    },163    async arxiv({ query, ids, start = 0, max = 10, sort }) {164      const u = new URL("https://export.arxiv.org/api/query");165      if (ids?.length) u.searchParams.set("id_list", ids.join(","));166      if (query) u.searchParams.set("search_query", query);167      u.searchParams.set("start", String(start)); u.searchParams.set("max_results", String(max));168      if (sort === "date") { u.searchParams.set("sortBy", "submittedDate"); u.searchParams.set("sortOrder", "descending"); }169      return worksFromArxivFeed(await http.get(u.href, { as: "text" }));170    },171    async doiCsl(doi) {172      const d = await http.get(`https://doi.org/${encDoi(doi)}`, { headers: { Accept: "application/vnd.citationstyles.csl+json" }, notFoundMessage: `DOI ${doi} does not resolve: it is not registered with any agency.` });173      if (!d || typeof d !== "object" || (!d.title && !d.DOI)) throw new ScholarlyError("not_found", `DOI ${doi} resolves, but its registration agency returned no bibliographic metadata.`);174      return workFromCsl(d);175    },176    async doiBibtex(doi) {177      const t = await http.get(`https://doi.org/${encDoi(doi)}`, { as: "text", headers: { Accept: "application/x-bibtex" }, notFoundMessage: `DOI ${doi} does not resolve: it is not registered with any agency.` });178      if (!/^\s*@\w+\s*\{/.test(t)) throw new ScholarlyError("upstream_error", `The registration agency of ${doi} did not return BibTeX.`);179      return t.trim();180    },181  };182}183
Files main
Menu

Theme