Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocAutomation/Intelligence/TextSupport.swift3// Description: Deterministic helpers around generation: token estimation, paragraph-boundary chunking, the post-check that4// finds citation-like passages absent from the source, and a word-level diff.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation910// MARK: - Tokens and chunks1112public enum TextChunker {13 /// A deliberately high estimate of the number of tokens: about one token per three characters of alphabetic text14 /// (the measured ratio is closer to four), one per character for CJK scripts.15 public static func estimatedTokens(_ text: String) -> Int {16 var wide = 0, narrow = 017 for s in text.unicodeScalars {18 if (0x2E80...0x9FFF).contains(s.value) || (0xAC00...0xD7AF).contains(s.value) || (0x3040...0x30FF).contains(s.value) || s.value >= 0x20000 { wide += 1 }19 else { narrow += 1 }20 }21 return wide + (narrow + 2) / 3 + 122 }2324 /// Splits `text` into chunks of at most `budget` (estimated) tokens, cutting between paragraphs. A paragraph that is too25 /// long by itself is cut between sentences, and a sentence that is still too long is cut between words.26 public static func chunks(_ text: String, budget: Int, tokens: (String) -> Int = estimatedTokens) -> [String] {27 let budget = max(32, budget)28 let paragraphs = text.components(separatedBy: "\n").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }29 var units: [String] = []30 for p in paragraphs {31 if tokens(p) <= budget { units.append(p); continue }32 for s in sentences(p) {33 if tokens(s) <= budget { units.append(s) } else { units += hardSplit(s, budget: budget, tokens: tokens) }34 }35 }36 var chunks: [String] = []37 var current: [String] = []38 var used = 039 for u in units {40 let t = tokens(u) + 141 if !current.isEmpty, used + t > budget { chunks.append(current.joined(separator: "\n\n")); current = []; used = 0 }42 current.append(u); used += t43 }44 if !current.isEmpty { chunks.append(current.joined(separator: "\n\n")) }45 return chunks46 }4748 public static func sentences(_ text: String) -> [String] {49 var out: [String] = []50 text.enumerateSubstrings(in: text.startIndex..<text.endIndex, options: [.bySentences, .localized]) { s, _, _, _ in51 if let s = s?.trimmingCharacters(in: .whitespacesAndNewlines), !s.isEmpty { out.append(s) }52 }53 return out.isEmpty ? [text] : out54 }5556 private static func hardSplit(_ text: String, budget: Int, tokens: (String) -> Int) -> [String] {57 var out: [String] = []58 var current = ""59 for word in text.split(separator: " ", omittingEmptySubsequences: true) {60 let candidate = current.isEmpty ? String(word) : current + " " + word61 if tokens(candidate) > budget, !current.isEmpty { out.append(current); current = String(word) } else { current = candidate }62 }63 if !current.isEmpty { out.append(current) }64 return out65 }66}6768// MARK: - Citation guard6970/// A passage of a generated text that looks like a citation, a reference, a quotation or a link and is absent from the source.71public struct UnsupportedPassage: Sendable, Hashable, Identifiable {72 public enum Kind: String, Sendable { case authorYear, numericReference, quotation, link }73 public var kind: Kind74 public var text: String75 public var range: Range<String.Index>76 /// UTF-16 offset of the passage in the generated text.77 public var offset: Int78 public var id: String { "\(kind.rawValue)|\(offset)|\(text)" }79}8081public enum CitationGuard {82 private static func regex(_ pattern: String) -> NSRegularExpression {83 // The patterns are literals of this file.84 try! NSRegularExpression(pattern: pattern, options: [])85 }86 private static let year = #"(?:1[5-9]|20)\d{2}[a-z]?"#87 /// "(Smith, 2020)", "(Smith et al. 2019, p. 4; Jones 2021)".88 private static let parenthetical = regex(#"\([^()\n]{0,160}?\b"# + year + #"\b[^()\n]{0,80}?\)"#)89 /// "Smith (2020)", "Smith and Jones (2020)", "Smith et al. (2020)".90 private static let narrative = regex(#"\b\p{Lu}[\p{L}'’-]+(?:\s+(?:et al\.|et coll\.|and|et|&)\s*(?:\p{Lu}[\p{L}'’-]+)?)?\s+\("# + year + #"(?:,[^()\n]{0,40})?\)"#)91 /// "[12]", "[1, 3–5]".92 private static let numeric = regex(#"\[\d{1,4}(?:\s*[,;–-]\s*\d{1,4})*\]"#)93 private static let quotation = regex(#"“([^”\n]{12,400})”|«\s*([^»\n]{12,400}?)\s*»|"([^"\n]{12,400})""#)94 private static let link = regex(#"\bhttps?://[^\s)\]>]+|\b10\.\d{4,9}/[^\s)\]>]+"#)9596 /// Letters and digits only, lower-cased: comparison ignores spacing, punctuation and case.97 static func key(_ s: String) -> String {98 String(String.UnicodeScalarView(s.lowercased().folding(options: .diacriticInsensitive, locale: nil).unicodeScalars.filter { CharacterSet.alphanumerics.contains($0) }))99 }100101 /// Citation-like passages of `result` that do not occur in `source`. They are reported, never removed.102 public static func unsupportedPassages(in result: String, source: String) -> [UnsupportedPassage] {103 let sourceKey = key(source)104 let ns = result as NSString105 let whole = NSRange(location: 0, length: ns.length)106 var found: [UnsupportedPassage] = []107 var covered: [NSRange] = []108109 func add(_ range: NSRange, _ kind: UnsupportedPassage.Kind) {110 guard !covered.contains(where: { NSIntersectionRange($0, range).length > 0 }), let r = Range(range, in: result) else { return }111 covered.append(range)112 found.append(UnsupportedPassage(kind: kind, text: String(result[r]), range: r, offset: range.location))113 }114115 for m in parenthetical.matches(in: result, range: whole) {116 let text = ns.substring(with: m.range)117 // A name is needed for a citation: "(in 2020)" or "(2020)" alone is a date.118 guard text.range(of: #"\p{Lu}\p{Ll}"#, options: .regularExpression) != nil else { continue }119 // Each "; "-separated item is checked: one invented source among real ones is enough.120 let items = text.dropFirst().dropLast().split(separator: ";").map(String.init)121 if items.contains(where: { $0.range(of: year, options: .regularExpression) != nil && !sourceKey.contains(key($0)) }) { add(m.range, .authorYear) }122 }123 for m in narrative.matches(in: result, range: whole) where !sourceKey.contains(key(ns.substring(with: m.range))) { add(m.range, .authorYear) }124 let sourceCompact = source.filter { !$0.isWhitespace }125 for m in numeric.matches(in: result, range: whole) {126 let text = ns.substring(with: m.range).filter { !$0.isWhitespace }127 if !sourceCompact.contains(text) { add(m.range, .numericReference) }128 }129 for m in quotation.matches(in: result, range: whole) {130 let inner = (1..<m.numberOfRanges).map { m.range(at: $0) }.first { $0.location != NSNotFound }.map { ns.substring(with: $0) } ?? ""131 // Three words or more: shorter quoted terms are usually emphasis, not quotations.132 guard inner.split(whereSeparator: { $0.isWhitespace }).count >= 3 else { continue }133 if !sourceKey.contains(key(inner)) { add(m.range, .quotation) }134 }135 for m in link.matches(in: result, range: whole) {136 let text = ns.substring(with: m.range).trimmingCharacters(in: CharacterSet(charactersIn: ".,;"))137 if !source.lowercased().contains(text.lowercased()) { add(m.range, .link) }138 }139 return found.sorted { $0.range.lowerBound < $1.range.lowerBound }140 }141}142143// MARK: - Word diff144145public struct WordDiffSegment: Sendable, Hashable {146 public enum Kind: Sendable { case equal, inserted, deleted }147 public var kind: Kind148 public var text: String149 public init(kind: Kind, text: String) { self.kind = kind; self.text = text }150}151152public enum WordDiff {153 /// Words and the whitespace or punctuation between them, so that joining the tokens gives the text back.154 static func tokens(_ s: String) -> [String] {155 var out: [String] = []156 var current = ""157 var last = -1158 for ch in s {159 let kind = ch.isLetter || ch.isNumber || ch == "'" || ch == "’" ? 0 : (ch.isWhitespace ? 1 : 2)160 if kind != last, !current.isEmpty { out.append(current); current = "" }161 last = kind162 current.append(ch)163 }164 if !current.isEmpty { out.append(current) }165 return out166 }167168 /// A word-level diff (longest common subsequence). Texts too long for it are reported as one deletion and one insertion.169 public static func diff(_ old: String, _ new: String) -> [WordDiffSegment] {170 let a = tokens(old), b = tokens(new)171 guard !a.isEmpty || !b.isEmpty else { return [] }172 guard a.count * b.count <= 9_000_000 else {173 return [WordDiffSegment(kind: .deleted, text: old), WordDiffSegment(kind: .inserted, text: new)].filter { !$0.text.isEmpty }174 }175 let n = a.count, m = b.count176 var lcs = [Int32](repeating: 0, count: (n + 1) * (m + 1))177 if n > 0, m > 0 {178 for i in stride(from: n - 1, through: 0, by: -1) {179 for j in stride(from: m - 1, through: 0, by: -1) {180 lcs[i * (m + 1) + j] = a[i] == b[j] ? lcs[(i + 1) * (m + 1) + j + 1] + 1 : max(lcs[(i + 1) * (m + 1) + j], lcs[i * (m + 1) + j + 1])181 }182 }183 }184 var out: [WordDiffSegment] = []185 func push(_ kind: WordDiffSegment.Kind, _ t: String) {186 if let last = out.last, last.kind == kind { out[out.count - 1].text += t } else { out.append(WordDiffSegment(kind: kind, text: t)) }187 }188 var i = 0, j = 0189 while i < n, j < m {190 if a[i] == b[j] { push(.equal, a[i]); i += 1; j += 1 }191 else if lcs[(i + 1) * (m + 1) + j] >= lcs[i * (m + 1) + j + 1] { push(.deleted, a[i]); i += 1 }192 else { push(.inserted, b[j]); j += 1 }193 }194 while i < n { push(.deleted, a[i]); i += 1 }195 while j < m { push(.inserted, b[j]); j += 1 }196 return out197 }198}199