SPB Git

Repository Public Research preview

normalize.mjs 6.2 KB 132 lines javascript

1/**2 * =============================================================================3 *                  SP BOUCHER | SOFTWARE & RESEARCH4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : LLM API toolkits8 * File             : scholarly/skill/citation-verification/scripts/lib/normalize.mjs9 * Module           : scholarly10 * Repository       : https://git.spboucher.ai/llm-api-toolkits11 * Created          : 2026-11-1012 * Last Modified    : 2026-11-1013 *14 * Author           : Simon-Pierre Boucher15 * Position         : Professor of Finance16 * Institution      : Université du Québec en Outaouais (UQO)17 * Organization     : SP Boucher18 * Website          : https://spboucher.ai19 * Git              : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development      : dev@spboucher.ai23 * Support          : support@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142/** Pure text helpers: identifiers, title and name normalisation, similarity. No network. */4344export function normalizeDoi(input) {45  if (typeof input !== "string") return null;46  let s = input.trim().replace(/^(?:https?:\/\/)?(?:dx\.)?doi\.org\//i, "").replace(/^doi:\s*/i, "").trim();47  s = s.replace(/[\s.,;]+$/, "");48  // A trailing bracket belongs to the DOI only when it closes one opened inside it.49  const pairs = { ")": "(", "]": "[", "}": "{" };50  while (/[)\]}]$/.test(s)) {51    const close = s.at(-1), open = pairs[close];52    if (s.split(close).length <= s.split(open).length) break;53    s = s.slice(0, -1).replace(/[\s.,;]+$/, "");54  }55  return /^10\.\d{4,9}\/\S+$/.test(s) ? s.toLowerCase() : null;56}57export const DOI_IN_TEXT = /\b10\.\d{4,9}\/[^\s"]+/i;5859export function extractDoi(text) {60  const m = DOI_IN_TEXT.exec(String(text ?? ""));61  return m ? normalizeDoi(m[0]) : null;62}6364/** arXiv identifiers: new style 1706.03762 (optional vN) and old style hep-th/9901001. */65export function normalizeArxivId(input) {66  if (typeof input !== "string") return null;67  const s = input.trim().replace(/^(?:https?:\/\/)?arxiv\.org\/(?:abs|pdf)\//i, "").replace(/^arxiv:\s*/i, "").replace(/\.pdf$/i, "");68  const m = /^(\d{4}\.\d{4,5}|[a-z-]+(?:\.[A-Z]{2})?\/\d{7})(v\d+)?$/i.exec(s);69  return m ? m[1] : null;70}71export function extractArxivId(text) {72  const m = /arxiv[:\s/]+(?:abs\/)?(\d{4}\.\d{4,5}|[a-z-]+(?:\.[A-Z]{2})?\/\d{7})/i.exec(String(text ?? ""));73  return m ? m[1] : null;74}7576const STOP = new Set(["a", "an", "the", "of", "and", "in", "on", "for", "to", "with", "by", "at", "from", "de", "la", "le", "les", "des", "du", "et", "en", "un", "une"]);7778export function foldText(s) {79  return String(s ?? "").normalize("NFKD").replace(/[̀-ͯ]/g, "").replace(/ß/g, "ss").replace(/[øØ]/g, "o").replace(/[æÆ]/g, "ae").replace(/[łŁ]/g, "l").replace(/[đĐ]/g, "d").toLowerCase();80}81export function normalizeTitle(s) {82  return foldText(String(s ?? "").replace(/<[^>]+>/g, " ").replace(/^\s*(retracted|withdrawn|retraction notice to|erratum to|corrigendum to)\s*[:\-–]\s*/i, ""))83    .replace(/&amp;/g, "and").replace(/[^a-z0-9]+/g, " ").trim();84}85export function titleTokens(s) {86  return normalizeTitle(s).split(" ").filter((t) => t && !STOP.has(t));87}8889/** Sørensen–Dice on token multisets, 0..1. A title that is the other plus a subtitle scores at least 0.9. */90export function titleSimilarity(a, b) {91  const ta = titleTokens(a), tb = titleTokens(b);92  if (!ta.length || !tb.length) return 0;93  const count = new Map();94  for (const t of ta) count.set(t, (count.get(t) ?? 0) + 1);95  let inter = 0;96  for (const t of tb) { const c = count.get(t) ?? 0; if (c > 0) { inter++; count.set(t, c - 1); } }97  const dice = (2 * inter) / (ta.length + tb.length);98  const [short, long] = ta.length <= tb.length ? [ta, tb] : [tb, ta];99  const prefix = short.length >= 4 && short.every((t, i) => long[i] === t);100  return Math.round(Math.max(dice, prefix ? 0.9 : 0) * 1000) / 1000;101}102103/** Family name of an author given as {family}, "Family, Given" or "Given Family". */104export function familyName(author) {105  if (!author) return "";106  if (typeof author === "object") return foldText(author.family ?? familyName(author.name ?? "")).replace(/[^a-z ]/g, " ").replace(/\s+/g, " ").trim();107  const s = String(author).trim();108  if (!s) return "";109  if (s.includes(",")) return foldText(s.split(",")[0]).replace(/[^a-z ]/g, " ").replace(/\s+/g, " ").trim();110  const parts = s.replace(/\s+(jr|sr|ii|iii|iv)\.?$/i, "").split(/\s+/);111  // Keep lower-case particles with the family name: "van der Waals", "de la Cruz".112  let i = parts.length - 1;113  while (i > 0 && /^(van|von|der|den|de|del|della|di|da|dos|du|la|le|ter|ten|bin|ibn|al)$/i.test(parts[i - 1])) i--;114  return foldText(parts.slice(i).join(" ")).replace(/[^a-z ]/g, " ").replace(/\s+/g, " ").trim();115}116const nameTokens = (author) => foldText(typeof author === "object" && author ? `${author.given ?? ""} ${author.family ?? author.name ?? ""}` : String(author ?? "")).replace(/[^a-z ]/g, " ").split(/\s+/).filter((t) => t.length >= 3);117118/** True when two author strings plausibly name the same family. Tolerates initials and noisy index data ("Tim Bollerslev B"). */119export function sameFamily(a, b) {120  const x = familyName(a), y = familyName(b);121  if (!x && !y) return false;122  if (x && x === y) return true;123  const lx = x.split(" ").at(-1) ?? "", ly = y.split(" ").at(-1) ?? "";124  if (lx.length >= 3 && lx === ly) return true;125  return (lx.length >= 3 && nameTokens(b).includes(lx)) || (ly.length >= 3 && nameTokens(a).includes(ly));126}127128export function parseYear(v) {129  const m = /\b(1[5-9]\d\d|20\d\d)\b/.exec(String(v ?? ""));130  return m ? Number(m[1]) : null;131}132
Files main
Menu

Theme