SPB Git

Repository Public

TextSupport.swift 10.2 KB 199 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocAutomation/Intelligence/TextSupport.swift3// Description: Deterministic helpers around generation: token estimation, paragraph-boundary chunking, the post-check that4//              finds citation-like passages absent from the source, and a word-level diff.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation910// MARK: - Tokens and chunks1112public enum TextChunker {13    /// A deliberately high estimate of the number of tokens: about one token per three characters of alphabetic text14    /// (the measured ratio is closer to four), one per character for CJK scripts.15    public static func estimatedTokens(_ text: String) -> Int {16        var wide = 0, narrow = 017        for s in text.unicodeScalars {18            if (0x2E80...0x9FFF).contains(s.value) || (0xAC00...0xD7AF).contains(s.value) || (0x3040...0x30FF).contains(s.value) || s.value >= 0x20000 { wide += 1 }19            else { narrow += 1 }20        }21        return wide + (narrow + 2) / 3 + 122    }2324    /// Splits `text` into chunks of at most `budget` (estimated) tokens, cutting between paragraphs. A paragraph that is too25    /// long by itself is cut between sentences, and a sentence that is still too long is cut between words.26    public static func chunks(_ text: String, budget: Int, tokens: (String) -> Int = estimatedTokens) -> [String] {27        let budget = max(32, budget)28        let paragraphs = text.components(separatedBy: "\n").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }29        var units: [String] = []30        for p in paragraphs {31            if tokens(p) <= budget { units.append(p); continue }32            for s in sentences(p) {33                if tokens(s) <= budget { units.append(s) } else { units += hardSplit(s, budget: budget, tokens: tokens) }34            }35        }36        var chunks: [String] = []37        var current: [String] = []38        var used = 039        for u in units {40            let t = tokens(u) + 141            if !current.isEmpty, used + t > budget { chunks.append(current.joined(separator: "\n\n")); current = []; used = 0 }42            current.append(u); used += t43        }44        if !current.isEmpty { chunks.append(current.joined(separator: "\n\n")) }45        return chunks46    }4748    public static func sentences(_ text: String) -> [String] {49        var out: [String] = []50        text.enumerateSubstrings(in: text.startIndex..<text.endIndex, options: [.bySentences, .localized]) { s, _, _, _ in51            if let s = s?.trimmingCharacters(in: .whitespacesAndNewlines), !s.isEmpty { out.append(s) }52        }53        return out.isEmpty ? [text] : out54    }5556    private static func hardSplit(_ text: String, budget: Int, tokens: (String) -> Int) -> [String] {57        var out: [String] = []58        var current = ""59        for word in text.split(separator: " ", omittingEmptySubsequences: true) {60            let candidate = current.isEmpty ? String(word) : current + " " + word61            if tokens(candidate) > budget, !current.isEmpty { out.append(current); current = String(word) } else { current = candidate }62        }63        if !current.isEmpty { out.append(current) }64        return out65    }66}6768// MARK: - Citation guard6970/// A passage of a generated text that looks like a citation, a reference, a quotation or a link and is absent from the source.71public struct UnsupportedPassage: Sendable, Hashable, Identifiable {72    public enum Kind: String, Sendable { case authorYear, numericReference, quotation, link }73    public var kind: Kind74    public var text: String75    public var range: Range<String.Index>76    /// UTF-16 offset of the passage in the generated text.77    public var offset: Int78    public var id: String { "\(kind.rawValue)|\(offset)|\(text)" }79}8081public enum CitationGuard {82    private static func regex(_ pattern: String) -> NSRegularExpression {83        // The patterns are literals of this file.84        try! NSRegularExpression(pattern: pattern, options: [])85    }86    private static let year = #"(?:1[5-9]|20)\d{2}[a-z]?"#87    /// "(Smith, 2020)", "(Smith et al. 2019, p. 4; Jones 2021)".88    private static let parenthetical = regex(#"\([^()\n]{0,160}?\b"# + year + #"\b[^()\n]{0,80}?\)"#)89    /// "Smith (2020)", "Smith and Jones (2020)", "Smith et al. (2020)".90    private static let narrative = regex(#"\b\p{Lu}[\p{L}'’-]+(?:\s+(?:et al\.|et coll\.|and|et|&)\s*(?:\p{Lu}[\p{L}'’-]+)?)?\s+\("# + year + #"(?:,[^()\n]{0,40})?\)"#)91    /// "[12]", "[1, 3–5]".92    private static let numeric = regex(#"\[\d{1,4}(?:\s*[,;–-]\s*\d{1,4})*\]"#)93    private static let quotation = regex(#"“([^”\n]{12,400})”|«\s*([^»\n]{12,400}?)\s*»|"([^"\n]{12,400})""#)94    private static let link = regex(#"\bhttps?://[^\s)\]>]+|\b10\.\d{4,9}/[^\s)\]>]+"#)9596    /// Letters and digits only, lower-cased: comparison ignores spacing, punctuation and case.97    static func key(_ s: String) -> String {98        String(String.UnicodeScalarView(s.lowercased().folding(options: .diacriticInsensitive, locale: nil).unicodeScalars.filter { CharacterSet.alphanumerics.contains($0) }))99    }100101    /// Citation-like passages of `result` that do not occur in `source`. They are reported, never removed.102    public static func unsupportedPassages(in result: String, source: String) -> [UnsupportedPassage] {103        let sourceKey = key(source)104        let ns = result as NSString105        let whole = NSRange(location: 0, length: ns.length)106        var found: [UnsupportedPassage] = []107        var covered: [NSRange] = []108109        func add(_ range: NSRange, _ kind: UnsupportedPassage.Kind) {110            guard !covered.contains(where: { NSIntersectionRange($0, range).length > 0 }), let r = Range(range, in: result) else { return }111            covered.append(range)112            found.append(UnsupportedPassage(kind: kind, text: String(result[r]), range: r, offset: range.location))113        }114115        for m in parenthetical.matches(in: result, range: whole) {116            let text = ns.substring(with: m.range)117            // A name is needed for a citation: "(in 2020)" or "(2020)" alone is a date.118            guard text.range(of: #"\p{Lu}\p{Ll}"#, options: .regularExpression) != nil else { continue }119            // Each "; "-separated item is checked: one invented source among real ones is enough.120            let items = text.dropFirst().dropLast().split(separator: ";").map(String.init)121            if items.contains(where: { $0.range(of: year, options: .regularExpression) != nil && !sourceKey.contains(key($0)) }) { add(m.range, .authorYear) }122        }123        for m in narrative.matches(in: result, range: whole) where !sourceKey.contains(key(ns.substring(with: m.range))) { add(m.range, .authorYear) }124        let sourceCompact = source.filter { !$0.isWhitespace }125        for m in numeric.matches(in: result, range: whole) {126            let text = ns.substring(with: m.range).filter { !$0.isWhitespace }127            if !sourceCompact.contains(text) { add(m.range, .numericReference) }128        }129        for m in quotation.matches(in: result, range: whole) {130            let inner = (1..<m.numberOfRanges).map { m.range(at: $0) }.first { $0.location != NSNotFound }.map { ns.substring(with: $0) } ?? ""131            // Three words or more: shorter quoted terms are usually emphasis, not quotations.132            guard inner.split(whereSeparator: { $0.isWhitespace }).count >= 3 else { continue }133            if !sourceKey.contains(key(inner)) { add(m.range, .quotation) }134        }135        for m in link.matches(in: result, range: whole) {136            let text = ns.substring(with: m.range).trimmingCharacters(in: CharacterSet(charactersIn: ".,;"))137            if !source.lowercased().contains(text.lowercased()) { add(m.range, .link) }138        }139        return found.sorted { $0.range.lowerBound < $1.range.lowerBound }140    }141}142143// MARK: - Word diff144145public struct WordDiffSegment: Sendable, Hashable {146    public enum Kind: Sendable { case equal, inserted, deleted }147    public var kind: Kind148    public var text: String149    public init(kind: Kind, text: String) { self.kind = kind; self.text = text }150}151152public enum WordDiff {153    /// Words and the whitespace or punctuation between them, so that joining the tokens gives the text back.154    static func tokens(_ s: String) -> [String] {155        var out: [String] = []156        var current = ""157        var last = -1158        for ch in s {159            let kind = ch.isLetter || ch.isNumber || ch == "'" || ch == "’" ? 0 : (ch.isWhitespace ? 1 : 2)160            if kind != last, !current.isEmpty { out.append(current); current = "" }161            last = kind162            current.append(ch)163        }164        if !current.isEmpty { out.append(current) }165        return out166    }167168    /// A word-level diff (longest common subsequence). Texts too long for it are reported as one deletion and one insertion.169    public static func diff(_ old: String, _ new: String) -> [WordDiffSegment] {170        let a = tokens(old), b = tokens(new)171        guard !a.isEmpty || !b.isEmpty else { return [] }172        guard a.count * b.count <= 9_000_000 else {173            return [WordDiffSegment(kind: .deleted, text: old), WordDiffSegment(kind: .inserted, text: new)].filter { !$0.text.isEmpty }174        }175        let n = a.count, m = b.count176        var lcs = [Int32](repeating: 0, count: (n + 1) * (m + 1))177        if n > 0, m > 0 {178            for i in stride(from: n - 1, through: 0, by: -1) {179                for j in stride(from: m - 1, through: 0, by: -1) {180                    lcs[i * (m + 1) + j] = a[i] == b[j] ? lcs[(i + 1) * (m + 1) + j + 1] + 1 : max(lcs[(i + 1) * (m + 1) + j], lcs[i * (m + 1) + j + 1])181                }182            }183        }184        var out: [WordDiffSegment] = []185        func push(_ kind: WordDiffSegment.Kind, _ t: String) {186            if let last = out.last, last.kind == kind { out[out.count - 1].text += t } else { out.append(WordDiffSegment(kind: kind, text: t)) }187        }188        var i = 0, j = 0189        while i < n, j < m {190            if a[i] == b[j] { push(.equal, a[i]); i += 1; j += 1 }191            else if lcs[(i + 1) * (m + 1) + j] >= lcs[i * (m + 1) + j + 1] { push(.deleted, a[i]); i += 1 }192            else { push(.inserted, b[j]); j += 1 }193        }194        while i < n { push(.deleted, a[i]); i += 1 }195        while j < m { push(.inserted, b[j]); j += 1 }196        return out197    }198}199
Files main
Menu

Theme