spb/autoscholar
Public
1import type { SourceRecord } from "@autoscholar/schemas";2import { assertPublicUrl } from "./policy.js";34const UA = "AutoScholar.ai research-runner (mailto:research@autoscholar.ai)";56async function getJson(url: string, timeoutMs = 15_000): Promise<unknown> {7 await assertPublicUrl(url);8 const ctrl = new AbortController();9 const t = setTimeout(() => ctrl.abort(), timeoutMs);10 try {11 const res = await fetch(url, {12 headers: { "user-agent": UA, accept: "application/json" },13 signal: ctrl.signal,14 });15 if (!res.ok) throw new Error(`HTTP ${res.status}`);16 return await res.json();17 } finally {18 clearTimeout(t);19 }20}2122async function getText(23 url: string,24 timeoutMs = 15_000,25): Promise<{ status: number; text: string; contentType: string }> {26 await assertPublicUrl(url);27 const ctrl = new AbortController();28 const t = setTimeout(() => ctrl.abort(), timeoutMs);29 try {30 const res = await fetch(url, { headers: { "user-agent": UA }, signal: ctrl.signal, redirect: "follow" });31 const ct = res.headers.get("content-type") ?? "";32 const text =33 ct.includes("text") || ct.includes("xml") || ct.includes("json")34 ? (await res.text()).slice(0, 200_000)35 : "";36 return { status: res.status, text, contentType: ct };37 } finally {38 clearTimeout(t);39 }40}4142export function normalizeTitle(s: string): string {43 return s44 .toLowerCase()45 .replace(/<[^>]+>/g, "")46 .replace(/[^a-z0-9 ]+/g, " ")47 .replace(/\s+/g, " ")48 .trim();49}5051export function titleSimilarity(a: string, b: string): number {52 const A = new Set(53 normalizeTitle(a)54 .split(" ")55 .filter((w) => w.length > 2),56 );57 const B = new Set(58 normalizeTitle(b)59 .split(" ")60 .filter((w) => w.length > 2),61 );62 if (A.size === 0 || B.size === 0) return 0;63 let inter = 0;64 for (const w of A) if (B.has(w)) inter++;65 return inter / Math.min(A.size, B.size);66}6768export interface VerifiedMeta {69 title: string;70 authors: string[];71 year?: number;72 venue?: string;73 url?: string;74 doi?: string;75 bibtex: string;76 method: string;77 raw?: unknown;78}7980function bibKeySafe(k: string): string {81 return k.replace(/[^A-Za-z0-9:_-]/g, "");82}83function esc(s: string): string {84 return s85 .replace(/[\\{}]/g, "")86 .replace(/&/g, "\\&")87 .replace(/%/g, "\\%")88 .replace(/#/g, "\\#")89 .replace(/_/g, "\\_");90}91function bibField(k: string, v: string | number | undefined): string {92 return v === undefined || v === ""93 ? ""94 : ` ${k.padEnd(12)} = {${typeof v === "number" ? v : esc(String(v))}},\n`;95}9697function crossrefToBib(key: string, w: Record<string, unknown>): VerifiedMeta {98 const title = Array.isArray(w.title) ? String(w.title[0] ?? "") : String(w.title ?? "");99 const authors = ((w.author as Array<{ family?: string; given?: string; name?: string }>) ?? [])100 .map((a) => (a.family ? `${a.family}, ${a.given ?? ""}`.trim().replace(/,$/, "") : (a.name ?? "")))101 .filter(Boolean);102 const dateParts =103 (w.issued as { "date-parts"?: number[][] })?.["date-parts"] ??104 (w.published as { "date-parts"?: number[][] })?.["date-parts"] ??105 [];106 const year = dateParts[0]?.[0];107 const venue = Array.isArray(w["container-title"]) ? String(w["container-title"][0] ?? "") : undefined;108 const doi = String(w.DOI ?? "");109 const type = String(w.type ?? "");110 const bibType = type.includes("journal")111 ? "article"112 : type.includes("book-chapter")113 ? "incollection"114 : type.includes("book")115 ? "book"116 : type.includes("proceedings")117 ? "inproceedings"118 : type.includes("posted-content")119 ? "misc"120 : "article";121 const bibtex = `@${bibType}{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField(bibType === "article" ? "journal" : "booktitle", venue)}${bibField("year", year)}${bibField("volume", w.volume as string)}${bibField("number", w.issue as string)}${bibField("pages", w.page as string)}${bibField("publisher", w.publisher as string)}${bibField("doi", doi)}${bibField("url", `https://doi.org/${doi}`)}}\n`;122 return {123 title,124 authors,125 year,126 venue,127 doi,128 url: `https://doi.org/${doi}`,129 bibtex,130 method: "crossref",131 raw: { type },132 };133}134135export async function verifyDoi(key: string, doi: string): Promise<VerifiedMeta> {136 const clean = doi.replace(/^https?:\/\/(dx\.)?doi\.org\//i, "").trim();137 try {138 const j = (await getJson(`https://api.crossref.org/works/${encodeURIComponent(clean)}`)) as {139 message: Record<string, unknown>;140 };141 return crossrefToBib(key, j.message);142 } catch (e) {143 const j = (await getJson(144 `https://api.openalex.org/works/https://doi.org/${encodeURIComponent(clean)}`,145 )) as Record<string, unknown>;146 const title = String(j.title ?? j.display_name ?? "");147 const authors = ((j.authorships as Array<{ author?: { display_name?: string } }>) ?? [])148 .map((a) => a.author?.display_name ?? "")149 .filter(Boolean);150 const year = j.publication_year as number | undefined;151 const venue = (j.primary_location as { source?: { display_name?: string } })?.source?.display_name;152 const bibtex = `@article{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField("journal", venue)}${bibField("year", year)}${bibField("doi", clean)}${bibField("url", `https://doi.org/${clean}`)}}\n`;153 return {154 title,155 authors,156 year,157 venue,158 doi: clean,159 url: `https://doi.org/${clean}`,160 bibtex,161 method: `openalex (crossref failed: ${(e as Error).message})`,162 };163 }164}165166export async function verifyArxiv(key: string, id: string): Promise<VerifiedMeta> {167 const clean = id168 .replace(/^(arxiv:|https?:\/\/arxiv\.org\/(abs|pdf)\/)/i, "")169 .replace(/(v\d+)?(\.pdf)?$/, "");170 const { status, text } = await getText(171 `https://export.arxiv.org/api/query?id_list=${encodeURIComponent(clean)}`,172 );173 if (status !== 200) throw new Error(`arXiv HTTP ${status}`);174 const entry = text.split("<entry>")[1];175 if (!entry || /<title>Error<\/title>/.test(entry)) throw new Error("arXiv id not found");176 const title = (entry.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? "").replace(/\s+/g, " ").trim();177 const authors = [...entry.matchAll(/<name>([\s\S]*?)<\/name>/g)].map((m) => (m[1] ?? "").trim());178 const year = Number(entry.match(/<published>(\d{4})/)?.[1] ?? "0") || undefined;179 const doi = entry.match(/<arxiv:doi[^>]*>([\s\S]*?)<\/arxiv:doi>/)?.[1]?.trim();180 const cat = entry.match(/<arxiv:primary_category[^>]*term="([^"]+)"/)?.[1];181 const bibtex = `@article{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField("journal", `arXiv preprint arXiv:${clean}`)}${bibField("year", year)}${bibField("eprint", clean)} archivePrefix = {arXiv},\n${bibField("primaryClass", cat)}${bibField("doi", doi)}${bibField("url", `https://arxiv.org/abs/${clean}`)}}\n`;182 return {183 title,184 authors,185 year,186 venue: "arXiv",187 doi,188 url: `https://arxiv.org/abs/${clean}`,189 bibtex,190 method: "arxiv-api",191 };192}193194export async function verifyPubmed(key: string, pmid: string): Promise<VerifiedMeta> {195 const clean = pmid.replace(/^pmid:/i, "").trim();196 const j = (await getJson(197 `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${encodeURIComponent(clean)}&retmode=json`,198 )) as { result?: Record<string, Record<string, unknown>> };199 const r = j.result?.[clean];200 if (!r || r.error) throw new Error("PMID not found");201 const title = String(r.title ?? "");202 const authors = ((r.authors as Array<{ name?: string }>) ?? []).map((a) => a.name ?? "").filter(Boolean);203 const year = Number(String(r.pubdate ?? "").slice(0, 4)) || undefined;204 const venue = String(r.fulljournalname ?? r.source ?? "");205 const doi = ((r.articleids as Array<{ idtype?: string; value?: string }>) ?? []).find(206 (x) => x.idtype === "doi",207 )?.value;208 const bibtex = `@article{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField("journal", venue)}${bibField("year", year)}${bibField("volume", r.volume as string)}${bibField("pages", r.pages as string)}${bibField("doi", doi)} note = {PMID: ${clean}},\n${bibField("url", `https://pubmed.ncbi.nlm.nih.gov/${clean}/`)}}\n`;209 return {210 title,211 authors,212 year,213 venue,214 doi,215 url: `https://pubmed.ncbi.nlm.nih.gov/${clean}/`,216 bibtex,217 method: "pubmed-esummary",218 };219}220221export async function verifyOpenAlex(key: string, id: string): Promise<VerifiedMeta> {222 const clean = id.replace(/^https?:\/\/openalex\.org\//i, "");223 const j = (await getJson(`https://api.openalex.org/works/${encodeURIComponent(clean)}`)) as Record<224 string,225 unknown226 >;227 const doi = String(j.doi ?? "").replace(/^https?:\/\/doi\.org\//, "");228 if (doi) return { ...(await verifyDoi(key, doi)), method: "openalex→crossref" };229 const title = String(j.title ?? j.display_name ?? "");230 const authors = ((j.authorships as Array<{ author?: { display_name?: string } }>) ?? [])231 .map((a) => a.author?.display_name ?? "")232 .filter(Boolean);233 const year = j.publication_year as number | undefined;234 const bibtex = `@misc{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField("year", year)}${bibField("url", `https://openalex.org/${clean}`)}}\n`;235 return { title, authors, year, url: `https://openalex.org/${clean}`, bibtex, method: "openalex" };236}237238export async function verifyUrl(key: string, url: string, claimedTitle: string): Promise<VerifiedMeta> {239 const { status, text } = await getText(url);240 if (status < 200 || status >= 400) throw new Error(`HTTP ${status}`);241 const pageTitle = (text.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? "").replace(/\s+/g, " ").trim();242 const year = new Date().getUTCFullYear();243 const bibtex = `@misc{${bibKeySafe(key)},\n${bibField("title", claimedTitle || pageTitle)}${bibField("howpublished", `\\url{${url}}`)}${bibField("year", year)} note = {Accessed ${new Date().toISOString().slice(0, 10)}},\n}\n`;244 return {245 title: claimedTitle || pageTitle,246 authors: [],247 year,248 url,249 bibtex,250 method: `url-reachable (${status})`,251 raw: { pageTitle },252 };253}254255/** Verifies a claimed source by identifier and compares the claimed title. */256export async function verifySource(257 key: string,258 kind: SourceRecord["kind"],259 identifier: string,260 claimedTitle: string,261): Promise<{ meta: VerifiedMeta; similarity: number; verified: boolean; reason?: string }> {262 let meta: VerifiedMeta;263 switch (kind) {264 case "doi":265 meta = await verifyDoi(key, identifier);266 break;267 case "arxiv":268 meta = await verifyArxiv(key, identifier);269 break;270 case "pmid":271 meta = await verifyPubmed(key, identifier);272 break;273 case "openalex":274 meta = await verifyOpenAlex(key, identifier);275 break;276 case "url":277 meta = await verifyUrl(key, identifier, claimedTitle);278 break;279 default:280 throw new Error(`unsupported identifier kind ${kind}`);281 }282 const similarity = claimedTitle ? titleSimilarity(claimedTitle, meta.title) : 1;283 const threshold = kind === "url" ? 0.2 : 0.5;284 const verified = similarity >= threshold && meta.title.length > 3;285 return {286 meta,287 similarity,288 verified,289 reason: verified290 ? undefined291 : `claimed title does not match the resolved record (similarity ${similarity.toFixed(2)}): "${meta.title.slice(0, 120)}"`,292 };293}294295export interface SearchHit {296 kind: SourceRecord["kind"];297 identifier: string;298 title: string;299 authors: string[];300 year?: number;301 venue?: string;302 abstract?: string;303 citedBy?: number;304 url?: string;305}306307export async function scholarSearch(308 queryText: string,309 source: "openalex" | "crossref" | "arxiv" | "semanticscholar" | "pubmed",310 limit = 10,311 fromYear?: number,312): Promise<SearchHit[]> {313 const n = Math.min(Math.max(limit, 1), 25);314 if (source === "openalex") {315 const j = (await getJson(316 `https://api.openalex.org/works?search=${encodeURIComponent(queryText)}&per-page=${n}${fromYear ? `&filter=from_publication_date:${fromYear}-01-01` : ""}&select=id,doi,title,display_name,publication_year,authorships,primary_location,cited_by_count,abstract_inverted_index`,317 )) as { results: Array<Record<string, unknown>> };318 return j.results.map((w) => {319 const doi = String(w.doi ?? "").replace(/^https?:\/\/doi\.org\//, "");320 const inv = w.abstract_inverted_index as Record<string, number[]> | undefined;321 let abstract: string | undefined;322 if (inv) {323 const words: string[] = [];324 for (const [word, idxs] of Object.entries(inv)) for (const i of idxs) words[i] = word;325 abstract = words.join(" ").slice(0, 1200);326 }327 return {328 kind: doi ? "doi" : "openalex",329 identifier: doi || String(w.id ?? "").replace("https://openalex.org/", ""),330 title: String(w.title ?? w.display_name ?? ""),331 authors: ((w.authorships as Array<{ author?: { display_name?: string } }>) ?? [])332 .slice(0, 6)333 .map((a) => a.author?.display_name ?? ""),334 year: w.publication_year as number,335 venue: (w.primary_location as { source?: { display_name?: string } })?.source?.display_name,336 abstract,337 citedBy: w.cited_by_count as number,338 url: doi ? `https://doi.org/${doi}` : String(w.id ?? ""),339 } as SearchHit;340 });341 }342 if (source === "crossref") {343 const j = (await getJson(344 `https://api.crossref.org/works?query=${encodeURIComponent(queryText)}&rows=${n}${fromYear ? `&filter=from-pub-date:${fromYear}` : ""}&select=DOI,title,author,issued,container-title,is-referenced-by-count,abstract`,345 )) as { message: { items: Array<Record<string, unknown>> } };346 return j.message.items.map((w) => {347 const m = crossrefToBib("x", w);348 return {349 kind: "doi",350 identifier: m.doi ?? "",351 title: m.title,352 authors: m.authors.slice(0, 6),353 year: m.year,354 venue: m.venue,355 abstract:356 typeof w.abstract === "string" ? w.abstract.replace(/<[^>]+>/g, "").slice(0, 1200) : undefined,357 citedBy: w["is-referenced-by-count"] as number,358 url: m.url,359 } as SearchHit;360 });361 }362 if (source === "arxiv") {363 const { text } = await getText(364 `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(queryText)}&start=0&max_results=${n}&sortBy=relevance`,365 );366 return text367 .split("<entry>")368 .slice(1)369 .map((e) => {370 const id = (e.match(/<id>https?:\/\/arxiv\.org\/abs\/([^<]+)<\/id>/)?.[1] ?? "").replace(/v\d+$/, "");371 return {372 kind: "arxiv",373 identifier: id,374 title: (e.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? "").replace(/\s+/g, " ").trim(),375 authors: [...e.matchAll(/<name>([\s\S]*?)<\/name>/g)].slice(0, 6).map((m) => (m[1] ?? "").trim()),376 year: Number(e.match(/<published>(\d{4})/)?.[1]) || undefined,377 venue: "arXiv",378 abstract: (e.match(/<summary>([\s\S]*?)<\/summary>/)?.[1] ?? "")379 .replace(/\s+/g, " ")380 .trim()381 .slice(0, 1200),382 url: `https://arxiv.org/abs/${id}`,383 } as SearchHit;384 })385 .filter((h) => h.identifier);386 }387 if (source === "semanticscholar") {388 const j = (await getJson(389 `https://api.semanticscholar.org/graph/v1/paper/search?query=${encodeURIComponent(queryText)}&limit=${n}&fields=title,authors,year,venue,externalIds,abstract,citationCount${fromYear ? `&year=${fromYear}-` : ""}`,390 )) as { data?: Array<Record<string, unknown>> };391 return (j.data ?? []).map((p) => {392 const ext = (p.externalIds as Record<string, string>) ?? {};393 const kind: SearchHit["kind"] = ext.DOI ? "doi" : ext.ArXiv ? "arxiv" : ext.PubMed ? "pmid" : "url";394 const identifier =395 ext.DOI ?? ext.ArXiv ?? ext.PubMed ?? `https://www.semanticscholar.org/paper/${p.paperId}`;396 return {397 kind,398 identifier,399 title: String(p.title ?? ""),400 authors: ((p.authors as Array<{ name?: string }>) ?? []).slice(0, 6).map((a) => a.name ?? ""),401 year: p.year as number,402 venue: p.venue as string,403 abstract: typeof p.abstract === "string" ? p.abstract.slice(0, 1200) : undefined,404 citedBy: p.citationCount as number,405 } as SearchHit;406 });407 }408 // pubmed409 const s = (await getJson(410 `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=${encodeURIComponent(queryText)}&retmax=${n}&retmode=json${fromYear ? `&mindate=${fromYear}&datetype=pdat` : ""}`,411 )) as { esearchresult?: { idlist?: string[] } };412 const ids = s.esearchresult?.idlist ?? [];413 if (ids.length === 0) return [];414 const j = (await getJson(415 `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${ids.join(",")}&retmode=json`,416 )) as { result?: Record<string, Record<string, unknown>> };417 return ids.map((id) => {418 const r = j.result?.[id] ?? {};419 return {420 kind: "pmid",421 identifier: id,422 title: String(r.title ?? ""),423 authors: ((r.authors as Array<{ name?: string }>) ?? []).slice(0, 6).map((a) => a.name ?? ""),424 year: Number(String(r.pubdate ?? "").slice(0, 4)) || undefined,425 venue: String(r.fulljournalname ?? r.source ?? ""),426 url: `https://pubmed.ncbi.nlm.nih.gov/${id}/`,427 } as SearchHit;428 });429}430