SPB Git

Repository Public

sources.ts 5.4 KB 77 lines typescript

1/**2 * Reference checking. For each source of a sheet the server — not the model — looks the reference up:3 *   - DOI  → Crossref (api.crossref.org/works/<doi>), falling back to doi.org; the registered title must match4 *            the title written in the sheet (token overlap), otherwise the source stays unverified with a note.5 *   - URL  → an https GET through the outbound guard; 2xx means the page exists.6 * "resolved" therefore means: this record exists and carries this title. It does not mean the sheet represents7 * the work faithfully — that is the author's review ("reviewed").8 */9import type { Sheet, Source } from "@/schema";10import { htmlToText, NetError, safeFetch } from "./net";1112const tokens = (s: string) => new Set(s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/<[^>]+>/g, " ").split(/[^a-z0-9]+/).filter((w) => w.length > 2));13export function titleSimilarity(a: string, b: string): number {14  const A = tokens(a), B = tokens(b);15  if (!A.size || !B.size) return 0;16  let common = 0; for (const w of A) if (B.has(w)) common++;17  return common / Math.min(A.size, B.size);18}1920export interface SourceCheck { id: string; status: Source["status"]; detail: string; registeredTitle?: string; registeredYear?: number }2122export async function checkSource(s: Source): Promise<SourceCheck> {23  if (s.status === "reviewed") return { id: s.id, status: "reviewed", detail: "kept: reviewed by the author" };24  if (s.doi) {25    try {26      const r = await safeFetch(`https://api.crossref.org/works/${encodeURIComponent(s.doi)}`, { accept: "application/json", maxBytes: 1_500_000 });27      if (r.status === 200) {28        const m = (JSON.parse(r.body.toString("utf8")) as { message?: { title?: string[]; issued?: { "date-parts"?: number[][] } } }).message;29        const title = m?.title?.[0] ?? "";30        const year = m?.issued?.["date-parts"]?.[0]?.[0];31        const sim = titleSimilarity(s.title, title);32        if (sim >= 0.6) return { id: s.id, status: "resolved", detail: `Crossref record found; title match ${(sim * 100).toFixed(0)} %${year && s.year && year !== s.year ? `; year on record ${year}, in the sheet ${s.year}` : ""}`, registeredTitle: title, registeredYear: year };33        return { id: s.id, status: "unverified", detail: `The DOI exists but is registered for a different title: “${title.slice(0, 160)}”`, registeredTitle: title, registeredYear: year };34      }35      if (r.status === 404) {36        // Not every DOI is registered with Crossref (DataCite, arXiv…): ask the resolver itself.37        const d = await safeFetch(`https://doi.org/${encodeURI(s.doi)}`, { accept: "text/html", maxBytes: 3_000_000 });38        if (d.status >= 200 && d.status < 300) {39          const { title } = htmlToText(d.body.toString("utf8"));40          const sim = titleSimilarity(s.title, title);41          return sim >= 0.5 ? { id: s.id, status: "resolved", detail: `The DOI resolves to a page whose title matches (${(sim * 100).toFixed(0)} %)`, registeredTitle: title }42            : { id: s.id, status: "unverified", detail: `The DOI resolves, but the page title could not be matched: “${title.slice(0, 160)}”` };43        }44        return { id: s.id, status: "unverified", detail: `No such DOI (resolver answered HTTP ${d.status})` };45      }46      return { id: s.id, status: "unreachable", detail: `Crossref answered HTTP ${r.status}` };47    } catch (e) { return { id: s.id, status: "unreachable", detail: e instanceof NetError ? e.message : "network error" }; }48  }49  if (s.url) {50    try {51      const r = await safeFetch(s.url, { accept: "text/html,*/*;q=0.5", maxBytes: 3_000_000 });52      if (r.status >= 200 && r.status < 300) {53        const { title } = /html/i.test(r.contentType) ? htmlToText(r.body.toString("utf8")) : { title: "" };54        const sim = title ? titleSimilarity(s.title, title) : 0;55        return { id: s.id, status: "resolved", detail: title ? `The page exists (“${title.slice(0, 120)}”, title match ${(sim * 100).toFixed(0)} %)` : "The address answers", registeredTitle: title || undefined };56      }57      // Publishers often refuse automated readers: that says nothing about the reference.58      if (r.status === 401 || r.status === 403 || r.status === 429) return { id: s.id, status: "unreachable", detail: `The site refuses automated checks (HTTP ${r.status}); check it by hand` };59      return { id: s.id, status: "unverified", detail: `The address answered HTTP ${r.status}` };60    } catch (e) { return { id: s.id, status: "unreachable", detail: e instanceof NetError ? e.message : "network error" }; }61  }62  return { id: s.id, status: "unverified", detail: "No DOI or URL to check; confirm it by hand" };63}6465/** Checks every source (a few at a time) and returns the document with updated statuses plus a report. */66export async function checkSources(doc: Sheet, onProgress?: (done: number, total: number) => void): Promise<{ doc: Sheet; checks: SourceCheck[] }> {67  const checks: SourceCheck[] = [];68  const queue = [...doc.sources];69  let done = 0;70  await Promise.all(Array.from({ length: Math.min(3, queue.length) }, async () => {71    for (let s = queue.shift(); s; s = queue.shift()) { checks.push(await checkSource(s)); onProgress?.(++done, doc.sources.length); }72  }));73  const at = new Date().toISOString();74  const byId = new Map(checks.map((c) => [c.id, c]));75  return { checks: doc.sources.map((s) => byId.get(s.id)!), doc: { ...doc, sources: doc.sources.map((s) => ({ ...s, status: byId.get(s.id)!.status, checkedAt: at })) } };76}77
Files main
Menu

Theme