import type { SourceRecord } from "@autoscholar/schemas"; import { assertPublicUrl } from "./policy.js"; const UA = "AutoScholar.ai research-runner (mailto:research@autoscholar.ai)"; async function getJson(url: string, timeoutMs = 15_000): Promise { await assertPublicUrl(url); const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); try { const res = await fetch(url, { headers: { "user-agent": UA, accept: "application/json" }, signal: ctrl.signal, }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return await res.json(); } finally { clearTimeout(t); } } async function getText( url: string, timeoutMs = 15_000, ): Promise<{ status: number; text: string; contentType: string }> { await assertPublicUrl(url); const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); try { const res = await fetch(url, { headers: { "user-agent": UA }, signal: ctrl.signal, redirect: "follow" }); const ct = res.headers.get("content-type") ?? ""; const text = ct.includes("text") || ct.includes("xml") || ct.includes("json") ? (await res.text()).slice(0, 200_000) : ""; return { status: res.status, text, contentType: ct }; } finally { clearTimeout(t); } } export function normalizeTitle(s: string): string { return s .toLowerCase() .replace(/<[^>]+>/g, "") .replace(/[^a-z0-9 ]+/g, " ") .replace(/\s+/g, " ") .trim(); } export function titleSimilarity(a: string, b: string): number { const A = new Set( normalizeTitle(a) .split(" ") .filter((w) => w.length > 2), ); const B = new Set( normalizeTitle(b) .split(" ") .filter((w) => w.length > 2), ); if (A.size === 0 || B.size === 0) return 0; let inter = 0; for (const w of A) if (B.has(w)) inter++; return inter / Math.min(A.size, B.size); } export interface VerifiedMeta { title: string; authors: string[]; year?: number; venue?: string; url?: string; doi?: string; bibtex: string; method: string; raw?: unknown; } function bibKeySafe(k: string): string { return k.replace(/[^A-Za-z0-9:_-]/g, ""); } function esc(s: string): string { return s .replace(/[\\{}]/g, "") .replace(/&/g, "\\&") .replace(/%/g, "\\%") .replace(/#/g, "\\#") .replace(/_/g, "\\_"); } function bibField(k: string, v: string | number | undefined): string { return v === undefined || v === "" ? "" : ` ${k.padEnd(12)} = {${typeof v === "number" ? v : esc(String(v))}},\n`; } function crossrefToBib(key: string, w: Record): VerifiedMeta { const title = Array.isArray(w.title) ? String(w.title[0] ?? "") : String(w.title ?? ""); const authors = ((w.author as Array<{ family?: string; given?: string; name?: string }>) ?? []) .map((a) => (a.family ? `${a.family}, ${a.given ?? ""}`.trim().replace(/,$/, "") : (a.name ?? ""))) .filter(Boolean); const dateParts = (w.issued as { "date-parts"?: number[][] })?.["date-parts"] ?? (w.published as { "date-parts"?: number[][] })?.["date-parts"] ?? []; const year = dateParts[0]?.[0]; const venue = Array.isArray(w["container-title"]) ? String(w["container-title"][0] ?? "") : undefined; const doi = String(w.DOI ?? ""); const type = String(w.type ?? ""); const bibType = type.includes("journal") ? "article" : type.includes("book-chapter") ? "incollection" : type.includes("book") ? "book" : type.includes("proceedings") ? "inproceedings" : type.includes("posted-content") ? "misc" : "article"; const bibtex = `@${bibType}{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField(bibType === "article" ? "journal" : "booktitle", venue)}${bibField("year", year)}${bibField("volume", w.volume as string)}${bibField("number", w.issue as string)}${bibField("pages", w.page as string)}${bibField("publisher", w.publisher as string)}${bibField("doi", doi)}${bibField("url", `https://doi.org/${doi}`)}}\n`; return { title, authors, year, venue, doi, url: `https://doi.org/${doi}`, bibtex, method: "crossref", raw: { type }, }; } export async function verifyDoi(key: string, doi: string): Promise { const clean = doi.replace(/^https?:\/\/(dx\.)?doi\.org\//i, "").trim(); try { const j = (await getJson(`https://api.crossref.org/works/${encodeURIComponent(clean)}`)) as { message: Record; }; return crossrefToBib(key, j.message); } catch (e) { const j = (await getJson( `https://api.openalex.org/works/https://doi.org/${encodeURIComponent(clean)}`, )) as Record; const title = String(j.title ?? j.display_name ?? ""); const authors = ((j.authorships as Array<{ author?: { display_name?: string } }>) ?? []) .map((a) => a.author?.display_name ?? "") .filter(Boolean); const year = j.publication_year as number | undefined; const venue = (j.primary_location as { source?: { display_name?: string } })?.source?.display_name; const bibtex = `@article{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField("journal", venue)}${bibField("year", year)}${bibField("doi", clean)}${bibField("url", `https://doi.org/${clean}`)}}\n`; return { title, authors, year, venue, doi: clean, url: `https://doi.org/${clean}`, bibtex, method: `openalex (crossref failed: ${(e as Error).message})`, }; } } export async function verifyArxiv(key: string, id: string): Promise { const clean = id .replace(/^(arxiv:|https?:\/\/arxiv\.org\/(abs|pdf)\/)/i, "") .replace(/(v\d+)?(\.pdf)?$/, ""); const { status, text } = await getText( `https://export.arxiv.org/api/query?id_list=${encodeURIComponent(clean)}`, ); if (status !== 200) throw new Error(`arXiv HTTP ${status}`); const entry = text.split("")[1]; if (!entry || /Error<\/title>/.test(entry)) throw new Error("arXiv id not found"); const title = (entry.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? "").replace(/\s+/g, " ").trim(); const authors = [...entry.matchAll(/<name>([\s\S]*?)<\/name>/g)].map((m) => (m[1] ?? "").trim()); const year = Number(entry.match(/<published>(\d{4})/)?.[1] ?? "0") || undefined; const doi = entry.match(/<arxiv:doi[^>]*>([\s\S]*?)<\/arxiv:doi>/)?.[1]?.trim(); const cat = entry.match(/<arxiv:primary_category[^>]*term="([^"]+)"/)?.[1]; const bibtex = `@article{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField("journal", `arXiv preprint arXiv:${clean}`)}${bibField("year", year)}${bibField("eprint", clean)} archivePrefix = {arXiv},\n${bibField("primaryClass", cat)}${bibField("doi", doi)}${bibField("url", `https://arxiv.org/abs/${clean}`)}}\n`; return { title, authors, year, venue: "arXiv", doi, url: `https://arxiv.org/abs/${clean}`, bibtex, method: "arxiv-api", }; } export async function verifyPubmed(key: string, pmid: string): Promise<VerifiedMeta> { const clean = pmid.replace(/^pmid:/i, "").trim(); const j = (await getJson( `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${encodeURIComponent(clean)}&retmode=json`, )) as { result?: Record<string, Record<string, unknown>> }; const r = j.result?.[clean]; if (!r || r.error) throw new Error("PMID not found"); const title = String(r.title ?? ""); const authors = ((r.authors as Array<{ name?: string }>) ?? []).map((a) => a.name ?? "").filter(Boolean); const year = Number(String(r.pubdate ?? "").slice(0, 4)) || undefined; const venue = String(r.fulljournalname ?? r.source ?? ""); const doi = ((r.articleids as Array<{ idtype?: string; value?: string }>) ?? []).find( (x) => x.idtype === "doi", )?.value; const bibtex = `@article{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField("journal", venue)}${bibField("year", year)}${bibField("volume", r.volume as string)}${bibField("pages", r.pages as string)}${bibField("doi", doi)} note = {PMID: ${clean}},\n${bibField("url", `https://pubmed.ncbi.nlm.nih.gov/${clean}/`)}}\n`; return { title, authors, year, venue, doi, url: `https://pubmed.ncbi.nlm.nih.gov/${clean}/`, bibtex, method: "pubmed-esummary", }; } export async function verifyOpenAlex(key: string, id: string): Promise<VerifiedMeta> { const clean = id.replace(/^https?:\/\/openalex\.org\//i, ""); const j = (await getJson(`https://api.openalex.org/works/${encodeURIComponent(clean)}`)) as Record< string, unknown >; const doi = String(j.doi ?? "").replace(/^https?:\/\/doi\.org\//, ""); if (doi) return { ...(await verifyDoi(key, doi)), method: "openalex→crossref" }; const title = String(j.title ?? j.display_name ?? ""); const authors = ((j.authorships as Array<{ author?: { display_name?: string } }>) ?? []) .map((a) => a.author?.display_name ?? "") .filter(Boolean); const year = j.publication_year as number | undefined; const bibtex = `@misc{${bibKeySafe(key)},\n${bibField("title", title)}${bibField("author", authors.join(" and "))}${bibField("year", year)}${bibField("url", `https://openalex.org/${clean}`)}}\n`; return { title, authors, year, url: `https://openalex.org/${clean}`, bibtex, method: "openalex" }; } export async function verifyUrl(key: string, url: string, claimedTitle: string): Promise<VerifiedMeta> { const { status, text } = await getText(url); if (status < 200 || status >= 400) throw new Error(`HTTP ${status}`); const pageTitle = (text.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? "").replace(/\s+/g, " ").trim(); const year = new Date().getUTCFullYear(); const bibtex = `@misc{${bibKeySafe(key)},\n${bibField("title", claimedTitle || pageTitle)}${bibField("howpublished", `\\url{${url}}`)}${bibField("year", year)} note = {Accessed ${new Date().toISOString().slice(0, 10)}},\n}\n`; return { title: claimedTitle || pageTitle, authors: [], year, url, bibtex, method: `url-reachable (${status})`, raw: { pageTitle }, }; } /** Verifies a claimed source by identifier and compares the claimed title. */ export async function verifySource( key: string, kind: SourceRecord["kind"], identifier: string, claimedTitle: string, ): Promise<{ meta: VerifiedMeta; similarity: number; verified: boolean; reason?: string }> { let meta: VerifiedMeta; switch (kind) { case "doi": meta = await verifyDoi(key, identifier); break; case "arxiv": meta = await verifyArxiv(key, identifier); break; case "pmid": meta = await verifyPubmed(key, identifier); break; case "openalex": meta = await verifyOpenAlex(key, identifier); break; case "url": meta = await verifyUrl(key, identifier, claimedTitle); break; default: throw new Error(`unsupported identifier kind ${kind}`); } const similarity = claimedTitle ? titleSimilarity(claimedTitle, meta.title) : 1; const threshold = kind === "url" ? 0.2 : 0.5; const verified = similarity >= threshold && meta.title.length > 3; return { meta, similarity, verified, reason: verified ? undefined : `claimed title does not match the resolved record (similarity ${similarity.toFixed(2)}): "${meta.title.slice(0, 120)}"`, }; } export interface SearchHit { kind: SourceRecord["kind"]; identifier: string; title: string; authors: string[]; year?: number; venue?: string; abstract?: string; citedBy?: number; url?: string; } export async function scholarSearch( queryText: string, source: "openalex" | "crossref" | "arxiv" | "semanticscholar" | "pubmed", limit = 10, fromYear?: number, ): Promise<SearchHit[]> { const n = Math.min(Math.max(limit, 1), 25); if (source === "openalex") { const j = (await getJson( `https://api.openalex.org/works?search=${encodeURIComponent(queryText)}&per-page=${n}${fromYear ? `&filter=from_publication_date:${fromYear}-01-01` : ""}&select=id,doi,title,display_name,publication_year,authorships,primary_location,cited_by_count,abstract_inverted_index`, )) as { results: Array<Record<string, unknown>> }; return j.results.map((w) => { const doi = String(w.doi ?? "").replace(/^https?:\/\/doi\.org\//, ""); const inv = w.abstract_inverted_index as Record<string, number[]> | undefined; let abstract: string | undefined; if (inv) { const words: string[] = []; for (const [word, idxs] of Object.entries(inv)) for (const i of idxs) words[i] = word; abstract = words.join(" ").slice(0, 1200); } return { kind: doi ? "doi" : "openalex", identifier: doi || String(w.id ?? "").replace("https://openalex.org/", ""), title: String(w.title ?? w.display_name ?? ""), authors: ((w.authorships as Array<{ author?: { display_name?: string } }>) ?? []) .slice(0, 6) .map((a) => a.author?.display_name ?? ""), year: w.publication_year as number, venue: (w.primary_location as { source?: { display_name?: string } })?.source?.display_name, abstract, citedBy: w.cited_by_count as number, url: doi ? `https://doi.org/${doi}` : String(w.id ?? ""), } as SearchHit; }); } if (source === "crossref") { const j = (await getJson( `https://api.crossref.org/works?query=${encodeURIComponent(queryText)}&rows=${n}${fromYear ? `&filter=from-pub-date:${fromYear}` : ""}&select=DOI,title,author,issued,container-title,is-referenced-by-count,abstract`, )) as { message: { items: Array<Record<string, unknown>> } }; return j.message.items.map((w) => { const m = crossrefToBib("x", w); return { kind: "doi", identifier: m.doi ?? "", title: m.title, authors: m.authors.slice(0, 6), year: m.year, venue: m.venue, abstract: typeof w.abstract === "string" ? w.abstract.replace(/<[^>]+>/g, "").slice(0, 1200) : undefined, citedBy: w["is-referenced-by-count"] as number, url: m.url, } as SearchHit; }); } if (source === "arxiv") { const { text } = await getText( `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(queryText)}&start=0&max_results=${n}&sortBy=relevance`, ); return text .split("<entry>") .slice(1) .map((e) => { const id = (e.match(/<id>https?:\/\/arxiv\.org\/abs\/([^<]+)<\/id>/)?.[1] ?? "").replace(/v\d+$/, ""); return { kind: "arxiv", identifier: id, title: (e.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? "").replace(/\s+/g, " ").trim(), authors: [...e.matchAll(/<name>([\s\S]*?)<\/name>/g)].slice(0, 6).map((m) => (m[1] ?? "").trim()), year: Number(e.match(/<published>(\d{4})/)?.[1]) || undefined, venue: "arXiv", abstract: (e.match(/<summary>([\s\S]*?)<\/summary>/)?.[1] ?? "") .replace(/\s+/g, " ") .trim() .slice(0, 1200), url: `https://arxiv.org/abs/${id}`, } as SearchHit; }) .filter((h) => h.identifier); } if (source === "semanticscholar") { const j = (await getJson( `https://api.semanticscholar.org/graph/v1/paper/search?query=${encodeURIComponent(queryText)}&limit=${n}&fields=title,authors,year,venue,externalIds,abstract,citationCount${fromYear ? `&year=${fromYear}-` : ""}`, )) as { data?: Array<Record<string, unknown>> }; return (j.data ?? []).map((p) => { const ext = (p.externalIds as Record<string, string>) ?? {}; const kind: SearchHit["kind"] = ext.DOI ? "doi" : ext.ArXiv ? "arxiv" : ext.PubMed ? "pmid" : "url"; const identifier = ext.DOI ?? ext.ArXiv ?? ext.PubMed ?? `https://www.semanticscholar.org/paper/${p.paperId}`; return { kind, identifier, title: String(p.title ?? ""), authors: ((p.authors as Array<{ name?: string }>) ?? []).slice(0, 6).map((a) => a.name ?? ""), year: p.year as number, venue: p.venue as string, abstract: typeof p.abstract === "string" ? p.abstract.slice(0, 1200) : undefined, citedBy: p.citationCount as number, } as SearchHit; }); } // pubmed const s = (await getJson( `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=${encodeURIComponent(queryText)}&retmax=${n}&retmode=json${fromYear ? `&mindate=${fromYear}&datetype=pdat` : ""}`, )) as { esearchresult?: { idlist?: string[] } }; const ids = s.esearchresult?.idlist ?? []; if (ids.length === 0) return []; const j = (await getJson( `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${ids.join(",")}&retmode=json`, )) as { result?: Record<string, Record<string, unknown>> }; return ids.map((id) => { const r = j.result?.[id] ?? {}; return { kind: "pmid", identifier: id, title: String(r.title ?? ""), authors: ((r.authors as Array<{ name?: string }>) ?? []).slice(0, 6).map((a) => a.name ?? ""), year: Number(String(r.pubdate ?? "").slice(0, 4)) || undefined, venue: String(r.fulljournalname ?? r.source ?? ""), url: `https://pubmed.ncbi.nlm.nih.gov/${id}/`, } as SearchHit; }); }