SPB Git

Repository Public

TextTools.swift 10.1 KB 200 lines swift

1// ORIX Docs2// File: App/Sources/Intelligence/TextTools.swift3// Description: Rule-based and statistical text analyses that need no language model: readability, language identification4//              per paragraph and related-paragraph search, on the NaturalLanguage framework. Deterministic and always available.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import NaturalLanguage10import DocModel1112/// A place in the document a result points at.13nonisolated struct TextLocation: Sendable, Hashable, Identifiable {14    var story: StoryID = .body15    var path: ParaPath16    var range: Range<Int>?17    var excerpt: String18    var id: String { "\(path.block)-\(path.cell?.row ?? -1)-\(path.cell?.cell ?? -1)-\(path.cell?.paragraph ?? -1)-\(range?.lowerBound ?? -1)" }19}2021nonisolated struct ReadabilityReport: Sendable {22    struct Bucket: Sendable, Identifiable { var label: String; var count: Int; var id: String { label } }23    struct LongSentence: Sendable, Identifiable { var location: TextLocation; var words: Int; var id: String { location.id } }24    var words = 025    var sentences = 026    var averageWords = 0.027    var medianWords = 028    var longestWords = 029    var buckets: [Bucket] = []30    /// Sentences longer than `threshold` words, longest first.31    var long: [LongSentence] = []32    var threshold = 3533    /// At 230 words per minute.34    var readingMinutes = 035}3637nonisolated struct LanguageReport: Sendable {38    struct Share: Sendable, Identifiable { var code: String; var paragraphs: Int; var id: String { code } }39    struct Outlier: Sendable, Identifiable { var location: TextLocation; var code: String; var confidence: Double; var id: String { location.id } }40    var shares: [Share] = []41    /// Paragraphs in a language other than the dominant one.42    var outliers: [Outlier] = []43    /// Paragraphs too short or too ambiguous to identify.44    var undetermined = 045    var dominant: String? { shares.first?.code }46}4748nonisolated struct RelatedReport: Sendable {49    enum Method: Sendable, Equatable {50        /// Sentence embeddings of the NaturalLanguage framework for this language.51        case sentenceEmbedding(language: String)52        /// Shared vocabulary (cosine similarity of word counts); `reason` says why embeddings were not used.53        case lexical(reason: String)54    }55    struct Item: Sendable, Identifiable { var location: TextLocation; var score: Double; var id: String { location.id } }56    var method: Method57    var items: [Item] = []58}5960nonisolated enum TextTools {61    private static func excerpt(_ s: String, _ limit: Int = 110) -> String {62        let t = s.replacingOccurrences(of: objectReplacementCharacter, with: "").replacingOccurrences(of: lineSeparatorCharacter, with: " ").trimmingCharacters(in: .whitespacesAndNewlines)63        return t.count > limit ? String(t.prefix(limit)).trimmingCharacters(in: .whitespaces) + "…" : t64    }6566    private static func wordCount(_ text: String, tokenizer: NLTokenizer) -> Int {67        tokenizer.string = text68        var n = 069        tokenizer.enumerateTokens(in: text.startIndex..<text.endIndex) { _, _ in n += 1; return true }70        return n71    }7273    // MARK: Readability7475    static func readability(_ doc: DocumentModel, threshold: Int = 35) -> ReadabilityReport {76        var report = ReadabilityReport(threshold: threshold)77        let sentenceTokenizer = NLTokenizer(unit: .sentence)78        let wordTokenizer = NLTokenizer(unit: .word)79        var lengths: [Int] = []80        doc.forEachBodyParagraph { p, path in81            // Headings, captions and code are not prose.82            if p.style.hasPrefix("Heading") || [StyleID.title, StyleID.subtitle, StyleID.caption, StyleID.code].contains(p.style) { return }83            let text = p.plainText84            guard text.count > 1 else { return }85            sentenceTokenizer.string = text86            var ranges: [Range<String.Index>] = []87            sentenceTokenizer.enumerateTokens(in: text.startIndex..<text.endIndex) { r, _ in ranges.append(r); return true }88            for r in ranges {89                let sentence = String(text[r])90                let n = wordCount(sentence, tokenizer: wordTokenizer)91                guard n > 0 else { continue }92                lengths.append(n)93                if n > threshold {94                    let ns = NSRange(r, in: text)95                    let trimmed = sentence.trimmingCharacters(in: .whitespacesAndNewlines)96                    let length = (trimmed as NSString).length97                    report.long.append(.init(location: TextLocation(path: path, range: ns.location..<(ns.location + length), excerpt: excerpt(sentence)), words: n))98                }99            }100        }101        report.sentences = lengths.count102        report.words = lengths.reduce(0, +)103        if !lengths.isEmpty {104            let sorted = lengths.sorted()105            report.averageWords = Double(report.words) / Double(lengths.count)106            report.medianWords = sorted[sorted.count / 2]107            report.longestWords = sorted[sorted.count - 1]108        }109        let edges: [(String, ClosedRange<Int>)] = [("1–10", 1...10), ("11–20", 11...20), ("21–30", 21...30), ("31–40", 31...40), ("41+", 41...Int.max)]110        report.buckets = edges.map { label, range in .init(label: label, count: lengths.filter { range.contains($0) }.count) }111        report.long.sort { $0.words > $1.words }112        report.readingMinutes = report.words == 0 ? 0 : max(1, Int((Double(report.words) / 230).rounded(.up)))113        return report114    }115116    // MARK: Languages117118    static func dominantLanguage(of text: String) -> (code: String, confidence: Double)? {119        let recognizer = NLLanguageRecognizer()120        recognizer.processString(text)121        guard let best = recognizer.languageHypotheses(withMaximum: 1).first else { return nil }122        return (best.key.rawValue, best.value)123    }124125    static func languages(_ doc: DocumentModel) -> LanguageReport {126        var report = LanguageReport()127        var counts: [String: Int] = [:]128        var identified: [(TextLocation, String, Double)] = []129        doc.forEachBodyParagraph { p, path in130            let text = p.visibleText.trimmingCharacters(in: .whitespacesAndNewlines)131            guard !text.isEmpty, p.style != StyleID.code else { return }132            // Short fragments are guessed badly: they are counted as undetermined rather than mislabelled.133            guard text.count >= 40, let (code, confidence) = dominantLanguage(of: text), confidence >= 0.6 else { report.undetermined += 1; return }134            counts[code, default: 0] += 1135            identified.append((TextLocation(path: path, range: nil, excerpt: excerpt(text)), code, confidence))136        }137        report.shares = counts.map { .init(code: $0.key, paragraphs: $0.value) }.sorted { $0.paragraphs != $1.paragraphs ? $0.paragraphs > $1.paragraphs : $0.code < $1.code }138        if let dominant = report.dominant {139            report.outliers = identified.filter { $0.1 != dominant }.map { .init(location: $0.0, code: $0.1, confidence: $0.2) }140        }141        return report142    }143144    // MARK: Related paragraphs145146    private static func wordVector(_ text: String, tokenizer: NLTokenizer) -> [String: Double] {147        let lower = text.lowercased()148        tokenizer.string = lower149        var v: [String: Double] = [:]150        tokenizer.enumerateTokens(in: lower.startIndex..<lower.endIndex) { r, _ in151            let w = String(lower[r])152            if w.count >= 4 { v[w, default: 0] += 1 }153            return true154        }155        return v156    }157158    private static func cosine(_ a: [String: Double], _ b: [String: Double]) -> Double {159        guard !a.isEmpty, !b.isEmpty else { return 0 }160        var dot = 0.0161        for (k, x) in a { if let y = b[k] { dot += x * y } }162        let na = a.values.reduce(0) { $0 + $1 * $1 }.squareRoot(), nb = b.values.reduce(0) { $0 + $1 * $1 }.squareRoot()163        return na > 0 && nb > 0 ? dot / (na * nb) : 0164    }165166    /// Paragraphs of the body closest in meaning (or, failing that, in vocabulary) to `query`. `excluding` are the paragraphs the query came from.167    static func related(to query: String, in doc: DocumentModel, excluding: Set<ParaPath>, limit: Int = 8) -> RelatedReport {168        let query = String(query.trimmingCharacters(in: .whitespacesAndNewlines).prefix(1500))169        var candidates: [(TextLocation, String)] = []170        doc.forEachBodyParagraph { p, path in171            guard !excluding.contains(path), candidates.count < 1500 else { return }172            let text = p.visibleText.trimmingCharacters(in: .whitespacesAndNewlines)173            if text.count >= 30 { candidates.append((TextLocation(path: path, range: nil, excerpt: excerpt(text)), String(text.prefix(1500)))) }174        }175        guard !query.isEmpty else { return RelatedReport(method: .lexical(reason: "empty")) }176177        let language = dominantLanguage(of: query).flatMap { $0.confidence >= 0.5 ? $0.code : nil }178        if let language, let embedding = NLEmbedding.sentenceEmbedding(for: NLLanguage(rawValue: language)), embedding.vector(for: query) != nil {179            var items: [RelatedReport.Item] = []180            for (location, text) in candidates {181                let d = embedding.distance(between: query, and: text, distanceType: .cosine)182                guard d.isFinite, d < 2 else { continue }183                items.append(.init(location: location, score: max(0, 1 - d)))184            }185            items.sort { $0.score > $1.score }186            return RelatedReport(method: .sentenceEmbedding(language: language), items: Array(items.prefix(limit)))187        }188189        let tokenizer = NLTokenizer(unit: .word)190        let q = wordVector(query, tokenizer: tokenizer)191        var items = candidates.map { RelatedReport.Item(location: $0.0, score: cosine(q, wordVector($0.1, tokenizer: tokenizer))) }.filter { $0.score > 0.05 }192        items.sort { $0.score > $1.score }193        return RelatedReport(method: .lexical(reason: language ?? "unknown"), items: Array(items.prefix(limit)))194    }195196    static func languageName(_ code: String) -> String {197        Locale.current.localizedString(forIdentifier: code) ?? Locale.current.localizedString(forLanguageCode: code) ?? code198    }199}200
Files main
Menu

Theme