Repository Public Research preview
1/**2 * =============================================================================3 * SP BOUCHER | SOFTWARE & RESEARCH4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : LLM API toolkits8 * File : scholarly/skill/citation-verification/scripts/lib/normalize.mjs9 * Module : scholarly10 * Repository : https://git.spboucher.ai/llm-api-toolkits11 * Created : 2026-11-1012 * Last Modified : 2026-11-1013 *14 * Author : Simon-Pierre Boucher15 * Position : Professor of Finance16 * Institution : Université du Québec en Outaouais (UQO)17 * Organization : SP Boucher18 * Website : https://spboucher.ai19 * Git : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development : dev@spboucher.ai23 * Support : support@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142/** Pure text helpers: identifiers, title and name normalisation, similarity. No network. */4344export function normalizeDoi(input) {45 if (typeof input !== "string") return null;46 let s = input.trim().replace(/^(?:https?:\/\/)?(?:dx\.)?doi\.org\//i, "").replace(/^doi:\s*/i, "").trim();47 s = s.replace(/[\s.,;]+$/, "");48 // A trailing bracket belongs to the DOI only when it closes one opened inside it.49 const pairs = { ")": "(", "]": "[", "}": "{" };50 while (/[)\]}]$/.test(s)) {51 const close = s.at(-1), open = pairs[close];52 if (s.split(close).length <= s.split(open).length) break;53 s = s.slice(0, -1).replace(/[\s.,;]+$/, "");54 }55 return /^10\.\d{4,9}\/\S+$/.test(s) ? s.toLowerCase() : null;56}57export const DOI_IN_TEXT = /\b10\.\d{4,9}\/[^\s"]+/i;5859export function extractDoi(text) {60 const m = DOI_IN_TEXT.exec(String(text ?? ""));61 return m ? normalizeDoi(m[0]) : null;62}6364/** arXiv identifiers: new style 1706.03762 (optional vN) and old style hep-th/9901001. */65export function normalizeArxivId(input) {66 if (typeof input !== "string") return null;67 const s = input.trim().replace(/^(?:https?:\/\/)?arxiv\.org\/(?:abs|pdf)\//i, "").replace(/^arxiv:\s*/i, "").replace(/\.pdf$/i, "");68 const m = /^(\d{4}\.\d{4,5}|[a-z-]+(?:\.[A-Z]{2})?\/\d{7})(v\d+)?$/i.exec(s);69 return m ? m[1] : null;70}71export function extractArxivId(text) {72 const m = /arxiv[:\s/]+(?:abs\/)?(\d{4}\.\d{4,5}|[a-z-]+(?:\.[A-Z]{2})?\/\d{7})/i.exec(String(text ?? ""));73 return m ? m[1] : null;74}7576const STOP = new Set(["a", "an", "the", "of", "and", "in", "on", "for", "to", "with", "by", "at", "from", "de", "la", "le", "les", "des", "du", "et", "en", "un", "une"]);7778export function foldText(s) {79 return String(s ?? "").normalize("NFKD").replace(/[̀-ͯ]/g, "").replace(/ß/g, "ss").replace(/[øØ]/g, "o").replace(/[æÆ]/g, "ae").replace(/[łŁ]/g, "l").replace(/[đĐ]/g, "d").toLowerCase();80}81export function normalizeTitle(s) {82 return foldText(String(s ?? "").replace(/<[^>]+>/g, " ").replace(/^\s*(retracted|withdrawn|retraction notice to|erratum to|corrigendum to)\s*[:\-–]\s*/i, ""))83 .replace(/&/g, "and").replace(/[^a-z0-9]+/g, " ").trim();84}85export function titleTokens(s) {86 return normalizeTitle(s).split(" ").filter((t) => t && !STOP.has(t));87}8889/** Sørensen–Dice on token multisets, 0..1. A title that is the other plus a subtitle scores at least 0.9. */90export function titleSimilarity(a, b) {91 const ta = titleTokens(a), tb = titleTokens(b);92 if (!ta.length || !tb.length) return 0;93 const count = new Map();94 for (const t of ta) count.set(t, (count.get(t) ?? 0) + 1);95 let inter = 0;96 for (const t of tb) { const c = count.get(t) ?? 0; if (c > 0) { inter++; count.set(t, c - 1); } }97 const dice = (2 * inter) / (ta.length + tb.length);98 const [short, long] = ta.length <= tb.length ? [ta, tb] : [tb, ta];99 const prefix = short.length >= 4 && short.every((t, i) => long[i] === t);100 return Math.round(Math.max(dice, prefix ? 0.9 : 0) * 1000) / 1000;101}102103/** Family name of an author given as {family}, "Family, Given" or "Given Family". */104export function familyName(author) {105 if (!author) return "";106 if (typeof author === "object") return foldText(author.family ?? familyName(author.name ?? "")).replace(/[^a-z ]/g, " ").replace(/\s+/g, " ").trim();107 const s = String(author).trim();108 if (!s) return "";109 if (s.includes(",")) return foldText(s.split(",")[0]).replace(/[^a-z ]/g, " ").replace(/\s+/g, " ").trim();110 const parts = s.replace(/\s+(jr|sr|ii|iii|iv)\.?$/i, "").split(/\s+/);111 // Keep lower-case particles with the family name: "van der Waals", "de la Cruz".112 let i = parts.length - 1;113 while (i > 0 && /^(van|von|der|den|de|del|della|di|da|dos|du|la|le|ter|ten|bin|ibn|al)$/i.test(parts[i - 1])) i--;114 return foldText(parts.slice(i).join(" ")).replace(/[^a-z ]/g, " ").replace(/\s+/g, " ").trim();115}116const nameTokens = (author) => foldText(typeof author === "object" && author ? `${author.given ?? ""} ${author.family ?? author.name ?? ""}` : String(author ?? "")).replace(/[^a-z ]/g, " ").split(/\s+/).filter((t) => t.length >= 3);117118/** True when two author strings plausibly name the same family. Tolerates initials and noisy index data ("Tim Bollerslev B"). */119export function sameFamily(a, b) {120 const x = familyName(a), y = familyName(b);121 if (!x && !y) return false;122 if (x && x === y) return true;123 const lx = x.split(" ").at(-1) ?? "", ly = y.split(" ").at(-1) ?? "";124 if (lx.length >= 3 && lx === ly) return true;125 return (lx.length >= 3 && nameTokens(b).includes(lx)) || (ly.length >= 3 && nameTokens(a).includes(ly));126}127128export function parseYear(v) {129 const m = /\b(1[5-9]\d\d|20\d\d)\b/.exec(String(v ?? ""));130 return m ? Number(m[1]) : null;131}132