Repository Public
1// ORIX Docs2// File: App/Sources/Intelligence/TextTools.swift3// Description: Rule-based and statistical text analyses that need no language model: readability, language identification4// per paragraph and related-paragraph search, on the NaturalLanguage framework. Deterministic and always available.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import NaturalLanguage10import DocModel1112/// A place in the document a result points at.13nonisolated struct TextLocation: Sendable, Hashable, Identifiable {14 var story: StoryID = .body15 var path: ParaPath16 var range: Range<Int>?17 var excerpt: String18 var id: String { "\(path.block)-\(path.cell?.row ?? -1)-\(path.cell?.cell ?? -1)-\(path.cell?.paragraph ?? -1)-\(range?.lowerBound ?? -1)" }19}2021nonisolated struct ReadabilityReport: Sendable {22 struct Bucket: Sendable, Identifiable { var label: String; var count: Int; var id: String { label } }23 struct LongSentence: Sendable, Identifiable { var location: TextLocation; var words: Int; var id: String { location.id } }24 var words = 025 var sentences = 026 var averageWords = 0.027 var medianWords = 028 var longestWords = 029 var buckets: [Bucket] = []30 /// Sentences longer than `threshold` words, longest first.31 var long: [LongSentence] = []32 var threshold = 3533 /// At 230 words per minute.34 var readingMinutes = 035}3637nonisolated struct LanguageReport: Sendable {38 struct Share: Sendable, Identifiable { var code: String; var paragraphs: Int; var id: String { code } }39 struct Outlier: Sendable, Identifiable { var location: TextLocation; var code: String; var confidence: Double; var id: String { location.id } }40 var shares: [Share] = []41 /// Paragraphs in a language other than the dominant one.42 var outliers: [Outlier] = []43 /// Paragraphs too short or too ambiguous to identify.44 var undetermined = 045 var dominant: String? { shares.first?.code }46}4748nonisolated struct RelatedReport: Sendable {49 enum Method: Sendable, Equatable {50 /// Sentence embeddings of the NaturalLanguage framework for this language.51 case sentenceEmbedding(language: String)52 /// Shared vocabulary (cosine similarity of word counts); `reason` says why embeddings were not used.53 case lexical(reason: String)54 }55 struct Item: Sendable, Identifiable { var location: TextLocation; var score: Double; var id: String { location.id } }56 var method: Method57 var items: [Item] = []58}5960nonisolated enum TextTools {61 private static func excerpt(_ s: String, _ limit: Int = 110) -> String {62 let t = s.replacingOccurrences(of: objectReplacementCharacter, with: "").replacingOccurrences(of: lineSeparatorCharacter, with: " ").trimmingCharacters(in: .whitespacesAndNewlines)63 return t.count > limit ? String(t.prefix(limit)).trimmingCharacters(in: .whitespaces) + "…" : t64 }6566 private static func wordCount(_ text: String, tokenizer: NLTokenizer) -> Int {67 tokenizer.string = text68 var n = 069 tokenizer.enumerateTokens(in: text.startIndex..<text.endIndex) { _, _ in n += 1; return true }70 return n71 }7273 // MARK: Readability7475 static func readability(_ doc: DocumentModel, threshold: Int = 35) -> ReadabilityReport {76 var report = ReadabilityReport(threshold: threshold)77 let sentenceTokenizer = NLTokenizer(unit: .sentence)78 let wordTokenizer = NLTokenizer(unit: .word)79 var lengths: [Int] = []80 doc.forEachBodyParagraph { p, path in81 // Headings, captions and code are not prose.82 if p.style.hasPrefix("Heading") || [StyleID.title, StyleID.subtitle, StyleID.caption, StyleID.code].contains(p.style) { return }83 let text = p.plainText84 guard text.count > 1 else { return }85 sentenceTokenizer.string = text86 var ranges: [Range<String.Index>] = []87 sentenceTokenizer.enumerateTokens(in: text.startIndex..<text.endIndex) { r, _ in ranges.append(r); return true }88 for r in ranges {89 let sentence = String(text[r])90 let n = wordCount(sentence, tokenizer: wordTokenizer)91 guard n > 0 else { continue }92 lengths.append(n)93 if n > threshold {94 let ns = NSRange(r, in: text)95 let trimmed = sentence.trimmingCharacters(in: .whitespacesAndNewlines)96 let length = (trimmed as NSString).length97 report.long.append(.init(location: TextLocation(path: path, range: ns.location..<(ns.location + length), excerpt: excerpt(sentence)), words: n))98 }99 }100 }101 report.sentences = lengths.count102 report.words = lengths.reduce(0, +)103 if !lengths.isEmpty {104 let sorted = lengths.sorted()105 report.averageWords = Double(report.words) / Double(lengths.count)106 report.medianWords = sorted[sorted.count / 2]107 report.longestWords = sorted[sorted.count - 1]108 }109 let edges: [(String, ClosedRange<Int>)] = [("1–10", 1...10), ("11–20", 11...20), ("21–30", 21...30), ("31–40", 31...40), ("41+", 41...Int.max)]110 report.buckets = edges.map { label, range in .init(label: label, count: lengths.filter { range.contains($0) }.count) }111 report.long.sort { $0.words > $1.words }112 report.readingMinutes = report.words == 0 ? 0 : max(1, Int((Double(report.words) / 230).rounded(.up)))113 return report114 }115116 // MARK: Languages117118 static func dominantLanguage(of text: String) -> (code: String, confidence: Double)? {119 let recognizer = NLLanguageRecognizer()120 recognizer.processString(text)121 guard let best = recognizer.languageHypotheses(withMaximum: 1).first else { return nil }122 return (best.key.rawValue, best.value)123 }124125 static func languages(_ doc: DocumentModel) -> LanguageReport {126 var report = LanguageReport()127 var counts: [String: Int] = [:]128 var identified: [(TextLocation, String, Double)] = []129 doc.forEachBodyParagraph { p, path in130 let text = p.visibleText.trimmingCharacters(in: .whitespacesAndNewlines)131 guard !text.isEmpty, p.style != StyleID.code else { return }132 // Short fragments are guessed badly: they are counted as undetermined rather than mislabelled.133 guard text.count >= 40, let (code, confidence) = dominantLanguage(of: text), confidence >= 0.6 else { report.undetermined += 1; return }134 counts[code, default: 0] += 1135 identified.append((TextLocation(path: path, range: nil, excerpt: excerpt(text)), code, confidence))136 }137 report.shares = counts.map { .init(code: $0.key, paragraphs: $0.value) }.sorted { $0.paragraphs != $1.paragraphs ? $0.paragraphs > $1.paragraphs : $0.code < $1.code }138 if let dominant = report.dominant {139 report.outliers = identified.filter { $0.1 != dominant }.map { .init(location: $0.0, code: $0.1, confidence: $0.2) }140 }141 return report142 }143144 // MARK: Related paragraphs145146 private static func wordVector(_ text: String, tokenizer: NLTokenizer) -> [String: Double] {147 let lower = text.lowercased()148 tokenizer.string = lower149 var v: [String: Double] = [:]150 tokenizer.enumerateTokens(in: lower.startIndex..<lower.endIndex) { r, _ in151 let w = String(lower[r])152 if w.count >= 4 { v[w, default: 0] += 1 }153 return true154 }155 return v156 }157158 private static func cosine(_ a: [String: Double], _ b: [String: Double]) -> Double {159 guard !a.isEmpty, !b.isEmpty else { return 0 }160 var dot = 0.0161 for (k, x) in a { if let y = b[k] { dot += x * y } }162 let na = a.values.reduce(0) { $0 + $1 * $1 }.squareRoot(), nb = b.values.reduce(0) { $0 + $1 * $1 }.squareRoot()163 return na > 0 && nb > 0 ? dot / (na * nb) : 0164 }165166 /// Paragraphs of the body closest in meaning (or, failing that, in vocabulary) to `query`. `excluding` are the paragraphs the query came from.167 static func related(to query: String, in doc: DocumentModel, excluding: Set<ParaPath>, limit: Int = 8) -> RelatedReport {168 let query = String(query.trimmingCharacters(in: .whitespacesAndNewlines).prefix(1500))169 var candidates: [(TextLocation, String)] = []170 doc.forEachBodyParagraph { p, path in171 guard !excluding.contains(path), candidates.count < 1500 else { return }172 let text = p.visibleText.trimmingCharacters(in: .whitespacesAndNewlines)173 if text.count >= 30 { candidates.append((TextLocation(path: path, range: nil, excerpt: excerpt(text)), String(text.prefix(1500)))) }174 }175 guard !query.isEmpty else { return RelatedReport(method: .lexical(reason: "empty")) }176177 let language = dominantLanguage(of: query).flatMap { $0.confidence >= 0.5 ? $0.code : nil }178 if let language, let embedding = NLEmbedding.sentenceEmbedding(for: NLLanguage(rawValue: language)), embedding.vector(for: query) != nil {179 var items: [RelatedReport.Item] = []180 for (location, text) in candidates {181 let d = embedding.distance(between: query, and: text, distanceType: .cosine)182 guard d.isFinite, d < 2 else { continue }183 items.append(.init(location: location, score: max(0, 1 - d)))184 }185 items.sort { $0.score > $1.score }186 return RelatedReport(method: .sentenceEmbedding(language: language), items: Array(items.prefix(limit)))187 }188189 let tokenizer = NLTokenizer(unit: .word)190 let q = wordVector(query, tokenizer: tokenizer)191 var items = candidates.map { RelatedReport.Item(location: $0.0, score: cosine(q, wordVector($0.1, tokenizer: tokenizer))) }.filter { $0.score > 0.05 }192 items.sort { $0.score > $1.score }193 return RelatedReport(method: .lexical(reason: language ?? "unknown"), items: Array(items.prefix(limit)))194 }195196 static func languageName(_ code: String) -> String {197 Locale.current.localizedString(forIdentifier: code) ?? Locale.current.localizedString(forLanguageCode: code) ?? code198 }199}200