Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocReview/Tokenizer.swift3// Description: Paragraph tokenisation for the word-level diff (words, punctuation, white space, inline objects) and content fingerprints.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import DocModel910/// Maps strings to dense integers so sequences can be compared by integer equality (no hash collisions).11final class Interner {12 private var map: [String: Int] = [:]13 func id(_ s: String) -> Int {14 if let v = map[s] { return v }15 let v = map.count16 map[s] = v17 return v18 }19}2021/// A token of a paragraph. `loc` and `len` are UTF-16 offsets into `Paragraph.plainText`, always on composed-character boundaries.22struct Token {23 enum Kind: UInt8 { case word, space, punctuation, object }24 var loc: Int25 var len: Int26 var key: Int27 var kind: Kind28 var end: Int { loc + len }29 /// Words and inline objects carry content; white space and punctuation do not count as words.30 var isContent: Bool { kind == .word || kind == .object }31}3233enum Tokenizer {34 static let whitespace = CharacterSet.whitespacesAndNewlines.union(CharacterSet(charactersIn: "\u{2028}\u{2029}\u{00A0}\u{202F}\u{FEFF}"))3536 static func isSpace(_ unit: unichar) -> Bool {37 guard let s = UnicodeScalar(unit) else { return false }38 return whitespace.contains(s)39 }4041 /// Splits `text` into (location, length, kind) pieces.42 ///43 /// Words come from `NSString.enumerateSubstrings(.byWords)`, which follows the Unicode word-break rules and uses the system44 /// dictionaries for scripts written without spaces (Chinese, Japanese, Thai), so CJK text is compared word by word instead of45 /// as one giant token. Everything between two words is cut into runs of white space and single composed characters46 /// (punctuation, symbols, emoji sequences), so no token boundary ever falls inside a surrogate pair, a ZWJ sequence or47 /// between a base letter and its combining marks.48 static func pieces(of text: NSString, in range: NSRange, _ emit: (Int, Int, Token.Kind) -> Void) {49 guard range.length > 0 else { return }50 var words: [NSRange] = []51 text.enumerateSubstrings(in: range, options: [.byWords, .substringNotRequired]) { _, r, _, _ in52 if r.length > 0 { words.append(r) }53 }54 var pos = range.location55 let end = range.location + range.length56 func gap(_ upTo: Int) {57 while pos < upTo {58 let unit = text.character(at: pos)59 if isSpace(unit) {60 var e = pos + 161 while e < upTo && isSpace(text.character(at: e)) { e += 1 }62 emit(pos, e - pos, .space)63 pos = e64 } else {65 let r = text.rangeOfComposedCharacterSequence(at: pos)66 let e = min(upTo, max(pos + 1, r.location + r.length))67 emit(pos, e - pos, .punctuation)68 pos = e69 }70 }71 }72 for w in words where w.location >= pos {73 gap(w.location)74 let e = min(end, w.location + w.length)75 emit(w.location, e - w.location, .word)76 pos = e77 }78 gap(end)79 }8081 /// A fast count of the words of a string (same definition as the status bar: runs of letters and digits).82 static func quickWordCount(_ s: String) -> Int {83 var n = 084 var inWord = false85 for u in s.unicodeScalars {86 let w = u.properties.isAlphabetic || (u.value >= 48 && u.value <= 57) || u.properties.numericType != nil87 if w && !inWord { n += 1 }88 inWord = w || (inWord && (u == "'" || u == "’" || u.properties.generalCategory == .nonspacingMark))89 }90 return n91 }92}9394/// Multiset similarity of two sorted integer bags: 2·|A ∩ B| ÷ (|A| + |B|) (Sørensen–Dice).95func diceSimilarity(_ a: [Int], _ b: [Int]) -> Double {96 if a.isEmpty || b.isEmpty { return 0 }97 var i = 0, j = 0, common = 098 while i < a.count && j < b.count {99 if a[i] == b[j] { common += 1; i += 1; j += 1 } else if a[i] < b[j] { i += 1 } else { j += 1 }100 }101 return 2 * Double(common) / Double(a.count + b.count)102}103