SPB Git

Repository Public

BibliographyImporter.swift 9.5 KB 176 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocReferences/BibliographyImporter.swift3// Description: Format detection for bibliography files (.bib, .ris, CSL-JSON), merging of imported entries into a library with4//              de-duplication and deterministic key renaming, and the completeness checks used by the document auditor.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import DocModel1011public enum BibliographyFormat: String, Sendable, CaseIterable { case bibtex, ris, cslJSON }1213public enum BibliographyImporter {14    /// Detects the format from the content first (it is what will be parsed), then from the file extension.15    public static func detectFormat(_ text: String, fileExtension: String?) -> BibliographyFormat? {16        let head = text.prefix(4000).drop(while: { $0.isWhitespace || $0 == "\u{FEFF}" })17        if let c = head.first, c == "[" || c == "{" { return .cslJSON }18        let lines = text.prefix(20000).split(whereSeparator: \.isNewline)19        if lines.contains(where: { $0.hasPrefix("TY  - ") || $0.hasPrefix("TY - ") }) { return .ris }20        if text.range(of: #"@[A-Za-z]+\s*[{(]"#, options: .regularExpression) != nil { return .bibtex }21        switch (fileExtension ?? "").lowercased().trimmingCharacters(in: CharacterSet(charactersIn: ".")) {22        case "bib", "bibtex", "biblatex": return .bibtex23        case "ris": return .ris24        case "json", "csl", "csljson": return .cslJSON25        default: return nil26        }27    }2829    public static func importEntries(_ data: Data, fileExtension: String?) -> BibliographyParseResult {30        var note: String?31        let text: String32        if data.starts(with: [0xFF, 0xFE]) || data.starts(with: [0xFE, 0xFF]), let s = String(data: data, encoding: .utf16) { text = s }33        else if let s = String(data: data, encoding: .utf8) { text = s }34        else {35            text = String(decoding: data, as: Unicode.ASCII.self).isEmpty ? "" : (String(data: data, encoding: .isoLatin1) ?? "")36            note = "The file is not valid UTF-8; it was read as ISO Latin-1. Check the accented characters."37        }38        guard let format = detectFormat(text, fileExtension: fileExtension) else {39            return BibliographyParseResult(entries: [], issues: ["The format was not recognised: expected BibTeX/BibLaTeX (.bib), RIS (.ris) or CSL-JSON (.json)."])40        }41        var result: BibliographyParseResult42        switch format {43        case .bibtex: result = BibTeXParser.parse(text)44        case .ris: result = RISParser.parse(text)45        case .cslJSON: result = CSLJSON.parse(Data(text.utf8))46        }47        if let note { result.issues.insert(note, at: 0) }48        return result49    }5051    /// "key-2", "key-3"…: the first one that is free.52    public static func uniqueKey(_ base: String, existing: Set<String>) -> String {53        guard existing.contains(base) else { return base }54        var n = 255        while existing.contains("\(base)-\(n)") { n += 1 }56        return "\(base)-\(n)"57    }5859    private static func doiKey(_ e: BibEntry) -> String? {60        guard let d = e.fields["DOI"].map({ TextFold.bareDOI($0).lowercased() }), !d.isEmpty else { return nil }61        return d62    }63    private static func titleKey(_ e: BibEntry) -> String? {64        let t = TextFold.alphanumeric(e.title)65        guard !t.isEmpty, let y = e.year else { return nil }66        return "\(t)|\(y)"67    }68    private static func sameWork(_ a: BibEntry, _ b: BibEntry) -> Bool {69        if a == b { return true }70        if let da = doiKey(a), let db = doiKey(b) { return da == db }71        if let ta = titleKey(a), let tb = titleKey(b) { return ta == tb }72        return false73    }7475    /// Merges imported entries into an existing library: de-duplicates by key, then by DOI, then by normalised title + year;76    /// an incoming entry whose key is taken by a different work is renamed "key-2", "key-3"… Existing entries are never modified.77    /// The log has one line per skipped or renamed entry.78    public static func merge(_ incoming: [BibEntry], into existing: [BibEntry]) -> (entries: [BibEntry], log: [String]) {79        var entries = existing80        var log: [String] = []81        var byKey: [String: Int] = [:], byDOI: [String: Int] = [:], byTitle: [String: Int] = [:]82        func index(_ e: BibEntry, _ i: Int) {83            if byKey[e.key] == nil { byKey[e.key] = i }84            if let d = doiKey(e), byDOI[d] == nil { byDOI[d] = i }85            if let t = titleKey(e), byTitle[t] == nil { byTitle[t] = i }86        }87        for (i, e) in existing.enumerated() { index(e, i) }88        var added = 089        for var e in incoming {90            if let i = byKey[e.key], sameWork(entries[i], e) {91                log.append("\"\(e.key)\": already in the library (same key); skipped.")92                continue93            }94            if let d = doiKey(e), let i = byDOI[d] {95                log.append("\"\(e.key)\": duplicate of \"\(entries[i].key)\" (same DOI \(d)); skipped.")96                continue97            }98            if let t = titleKey(e), let i = byTitle[t], doiKey(e) == nil || doiKey(entries[i]) == nil {99                log.append("\"\(e.key)\": duplicate of \"\(entries[i].key)\" (same title and year); skipped.")100                continue101            }102            if byKey[e.key] != nil {103                let renamed = uniqueKey(e.key, existing: Set(byKey.keys))104                log.append("\"\(e.key)\": the key is already used by a different work; imported as \"\(renamed)\".")105                e.key = renamed106            }107            entries.append(e)108            index(e, entries.count - 1)109            added += 1110        }111        log.append("\(added) added, \(incoming.count - added) skipped; the library now has \(entries.count) entries.")112        return (entries, log)113    }114}115116public enum ReferenceChecks {117    /// Fields a complete record of this type needs, whatever the citation style (journal article: author, date, title, journal, volume,118    /// pages or DOI…). Returns the missing ones as CSL variable names ("issued" is the publication date).119    public static func missingFields(_ entry: BibEntry) -> [String] {120        var missing: [String] = []121        func has(_ k: String) -> Bool { !(entry.fields[k] ?? "").trimmingCharacters(in: .whitespaces).isEmpty }122        func need(_ k: String) { if !has(k) { missing.append(k) } }123        let bookLike = ["book", "report", "periodical", "dataset", "software"].contains(entry.type)124        if entry.authors.isEmpty && !(bookLike && !entry.editors.isEmpty) { missing.append("author") }125        if entry.year == nil && !has(ReferenceField.issuedLiteral) && !has(ReferenceField.rawDate) { missing.append("issued") }126        need("title")127        switch entry.type {128        case "article-journal", "article":129            need("container-title"); need("volume")130            if !has("page") && !has(ReferenceField.articleNumber) && !has("DOI") { missing.append("page or DOI") }131        case "article-magazine", "article-newspaper", "review": need("container-title")132        case "book": need("publisher")133        case "chapter", "entry-encyclopedia", "entry-dictionary", "entry":134            need("container-title"); need("publisher"); need("page")135        case "paper-conference":136            if !has("container-title") && !has("event-title") { missing.append("container-title") }137        case "thesis": need("genre"); need("publisher")138        case "report": need("publisher")139        case "webpage", "post", "post-weblog": need("URL")140        default: break141        }142        return missing143    }144145    private static let stopWords: Set<String> = ["a", "an", "the", "of", "on", "in", "and", "for", "to", "from", "with", "at", "by", "is", "are", "as",146                                                 "le", "la", "les", "l", "un", "une", "des", "du", "de", "d", "et", "en", "sur", "pour", "dans", "au", "aux"]147148    /// "boucher2024volatility": first author's family name, year, first significant title word; "a", "b"… is appended when the key is taken.149    public static func suggestedKey(for entry: BibEntry, existing: Set<String>) -> String {150        func ascii(_ s: String) -> String { String(TextFold.fold(s).filter { ($0 >= "a" && $0 <= "z") || ($0 >= "0" && $0 <= "9") }) }151        let person = (entry.authors.first { !$0.isOthers } ?? entry.editors.first { !$0.isOthers })152        var base = ""153        if let p = person {154            if p.isLiteral {155                // Institutions: the acronym of the significant words ("World Health Organization" → "who"), or the single word.156                let words = p.literal!.split(separator: " ").filter { !stopWords.contains($0.lowercased()) }157                base = words.count >= 2 ? ascii(words.map { String($0.prefix(1)) }.joined()) : ascii(p.literal!)158            } else {159                base = ascii(p.family.split(separator: " ").last.map(String.init) ?? p.family)160            }161        }162        if let y = entry.year { base += String(y) }163        let words = TextFold.stripMarkup(entry.title).split(whereSeparator: { !($0.isLetter || $0.isNumber) }).map { ascii(String($0)) }164        if let w = words.first(where: { !$0.isEmpty && !stopWords.contains($0) }) { base += w }165        if base.isEmpty { base = "ref" }166        guard existing.contains(base) else { return base }167        for u in UInt8(ascii: "a")...UInt8(ascii: "z") {168            let candidate = base + String(UnicodeScalar(u))169            if !existing.contains(candidate) { return candidate }170        }171        var n = 2172        while existing.contains("\(base)\(n)") { n += 1 }173        return "\(base)\(n)"174    }175}176
Files main
Menu

Theme