SPB Git

Repository Public

Tokenizer.swift 4.2 KB 103 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocReview/Tokenizer.swift3// Description: Paragraph tokenisation for the word-level diff (words, punctuation, white space, inline objects) and content fingerprints.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import DocModel910/// Maps strings to dense integers so sequences can be compared by integer equality (no hash collisions).11final class Interner {12    private var map: [String: Int] = [:]13    func id(_ s: String) -> Int {14        if let v = map[s] { return v }15        let v = map.count16        map[s] = v17        return v18    }19}2021/// A token of a paragraph. `loc` and `len` are UTF-16 offsets into `Paragraph.plainText`, always on composed-character boundaries.22struct Token {23    enum Kind: UInt8 { case word, space, punctuation, object }24    var loc: Int25    var len: Int26    var key: Int27    var kind: Kind28    var end: Int { loc + len }29    /// Words and inline objects carry content; white space and punctuation do not count as words.30    var isContent: Bool { kind == .word || kind == .object }31}3233enum Tokenizer {34    static let whitespace = CharacterSet.whitespacesAndNewlines.union(CharacterSet(charactersIn: "\u{2028}\u{2029}\u{00A0}\u{202F}\u{FEFF}"))3536    static func isSpace(_ unit: unichar) -> Bool {37        guard let s = UnicodeScalar(unit) else { return false }38        return whitespace.contains(s)39    }4041    /// Splits `text` into (location, length, kind) pieces.42    ///43    /// Words come from `NSString.enumerateSubstrings(.byWords)`, which follows the Unicode word-break rules and uses the system44    /// dictionaries for scripts written without spaces (Chinese, Japanese, Thai), so CJK text is compared word by word instead of45    /// as one giant token. Everything between two words is cut into runs of white space and single composed characters46    /// (punctuation, symbols, emoji sequences), so no token boundary ever falls inside a surrogate pair, a ZWJ sequence or47    /// between a base letter and its combining marks.48    static func pieces(of text: NSString, in range: NSRange, _ emit: (Int, Int, Token.Kind) -> Void) {49        guard range.length > 0 else { return }50        var words: [NSRange] = []51        text.enumerateSubstrings(in: range, options: [.byWords, .substringNotRequired]) { _, r, _, _ in52            if r.length > 0 { words.append(r) }53        }54        var pos = range.location55        let end = range.location + range.length56        func gap(_ upTo: Int) {57            while pos < upTo {58                let unit = text.character(at: pos)59                if isSpace(unit) {60                    var e = pos + 161                    while e < upTo && isSpace(text.character(at: e)) { e += 1 }62                    emit(pos, e - pos, .space)63                    pos = e64                } else {65                    let r = text.rangeOfComposedCharacterSequence(at: pos)66                    let e = min(upTo, max(pos + 1, r.location + r.length))67                    emit(pos, e - pos, .punctuation)68                    pos = e69                }70            }71        }72        for w in words where w.location >= pos {73            gap(w.location)74            let e = min(end, w.location + w.length)75            emit(w.location, e - w.location, .word)76            pos = e77        }78        gap(end)79    }8081    /// A fast count of the words of a string (same definition as the status bar: runs of letters and digits).82    static func quickWordCount(_ s: String) -> Int {83        var n = 084        var inWord = false85        for u in s.unicodeScalars {86            let w = u.properties.isAlphabetic || (u.value >= 48 && u.value <= 57) || u.properties.numericType != nil87            if w && !inWord { n += 1 }88            inWord = w || (inWord && (u == "'" || u == "’" || u.properties.generalCategory == .nonspacingMark))89        }90        return n91    }92}9394/// Multiset similarity of two sorted integer bags: 2·|A ∩ B| ÷ (|A| + |B|) (Sørensen–Dice).95func diceSimilarity(_ a: [Int], _ b: [Int]) -> Double {96    if a.isEmpty || b.isEmpty { return 0 }97    var i = 0, j = 0, common = 098    while i < a.count && j < b.count {99        if a[i] == b[j] { common += 1; i += 1; j += 1 } else if a[i] < b[j] { i += 1 } else { j += 1 }100    }101    return 2 * Double(common) / Double(a.count + b.count)102}103
Files main
Menu

Theme