Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocAudit/ResearchAssistant.swift3// Description: Research assistant: search in one or several documents, cited passages, unreferenced captions, outline, action items.4// Everything is rule-based and grounded in the document text; nothing is generated.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import DocModel1011public enum ResearchAssistant {12 public struct Hit: Sendable, Identifiable, Hashable {13 public var id: String14 public var documentName: String15 public var story: StoryID16 public var path: ParaPath17 /// UTF-16 range in the paragraph text (`Paragraph.plainText`).18 public var range: Range<Int>19 /// The match with some text around it.20 public var context: String21 /// Headings above the paragraph, outermost first (body paragraphs only).22 public var headingTrail: [String]2324 public init(id: String, documentName: String, story: StoryID, path: ParaPath, range: Range<Int>, context: String, headingTrail: [String]) {25 self.id = id; self.documentName = documentName; self.story = story; self.path = path; self.range = range26 self.context = context; self.headingTrail = headingTrail27 }28 }2930 public struct SearchOptions: Sendable {31 public var caseSensitive: Bool32 /// When false "resume" finds "résumé".33 public var diacriticSensitive: Bool34 public var wholeWord: Bool35 /// The query is an ICU regular expression.36 public var regex: Bool37 public var includeTables: Bool38 public var includeNotes: Bool39 public var includeHeadersAndFooters: Bool40 /// UTF-16 units of context kept on each side of a match.41 public var contextRadius: Int42 public var maxHits: Int?4344 public init(caseSensitive: Bool = false, diacriticSensitive: Bool = false, wholeWord: Bool = false, regex: Bool = false,45 includeTables: Bool = true, includeNotes: Bool = true, includeHeadersAndFooters: Bool = true, contextRadius: Int = 40, maxHits: Int? = nil) {46 self.caseSensitive = caseSensitive; self.diacriticSensitive = diacriticSensitive; self.wholeWord = wholeWord; self.regex = regex47 self.includeTables = includeTables; self.includeNotes = includeNotes; self.includeHeadersAndFooters = includeHeadersAndFooters48 self.contextRadius = contextRadius; self.maxHits = maxHits49 }50 }5152 public struct AcronymFinding: Sendable, Identifiable, Hashable {53 public var id: String { acronym }54 public var acronym: String55 /// Number of uses in the document.56 public var count: Int57 /// Evidence: the first use.58 public var firstUse: Hit59 }6061 public struct CitedPassage: Sendable, Identifiable, Hashable {62 public var id: String63 /// The sentence as written, citations shown as "[key; key]".64 public var sentence: String65 /// Citation keys of the sentence, in order of appearance.66 public var keys: [String]67 public var story: StoryID68 public var path: ParaPath69 /// UTF-16 range of the sentence in the paragraph.70 public var range: Range<Int>71 public var headingTrail: [String]72 }7374 public struct CaptionFinding: Sendable, Identifiable, Hashable {75 public var id: String { paragraphID }76 /// Sequence name: "Figure", "Table" …77 public var label: String78 public var number: Int79 /// Caption text as displayed ("Figure 2: Yield curve").80 public var text: String81 public var paragraphID: String82 public var blockIndex: Int83 }8485 public struct RepetitionFinding: Sendable, Identifiable, Hashable {86 public enum Kind: String, Sendable { case word, phrase }87 public var id: String88 public var kind: Kind89 /// The lemma (word) or the repeated phrase.90 public var text: String91 /// Occurrences inside the window.92 public var count: Int93 /// Number of consecutive words the occurrences fall in.94 public var span: Int95 public var occurrences: [Hit]96 }9798 // MARK: Search99100 /// Finds `query` in the body, tables, notes, headers and footers of a document. Case- and diacritic-insensitive unless the options101 /// say otherwise. Text marked as deleted by a tracked change is not searched. An invalid regular expression or an empty query gives no hit.102 public static func search(_ query: String, in doc: DocumentModel, documentName: String, options: SearchOptions = SearchOptions()) -> [Hit] {103 guard !query.isEmpty else { return [] }104 var regex: NSRegularExpression?105 if options.regex {106 var pattern = options.diacriticSensitive ? query : query.folding(options: .diacriticInsensitive, locale: nil)107 if options.wholeWord { pattern = "\\b(?:\(pattern))\\b" }108 guard let r = try? NSRegularExpression(pattern: pattern, options: options.caseSensitive ? [] : [.caseInsensitive]) else { return [] }109 regex = r110 }111 var compare: NSString.CompareOptions = []112 if !options.caseSensitive { compare.insert(.caseInsensitive) }113 if !options.diacriticSensitive { compare.insert(.diacriticInsensitive) }114 let trails = HeadingTrails(doc)115 var hits: [Hit] = []116 let limit = options.maxHits ?? Int.max117 for ref in Scan.paragraphs(of: doc) {118 if hits.count >= limit { break }119 switch ref.story {120 case .body: if ref.path.cell != nil && !options.includeTables { continue }121 case .note: if !options.includeNotes { continue }122 case .header, .footer: if !options.includeHeadersAndFooters { continue }123 }124 let text = Scan.maskedText(ref.paragraph)125 let ns = text as NSString126 guard ns.length > 0 else { continue }127 var ranges: [NSRange] = []128 if let regex {129 // Diacritic-insensitive matching needs a folded haystack; it is used only when folding keeps every offset in place.130 var haystack = text131 if !options.diacriticSensitive {132 let foldedText = text.folding(options: .diacriticInsensitive, locale: nil)133 if (foldedText as NSString).length == ns.length { haystack = foldedText }134 }135 for m in regex.matches(in: haystack, range: NSRange(location: 0, length: ns.length)) where m.range.length > 0 { ranges.append(m.range) }136 } else {137 var from = 0138 while from < ns.length {139 let r = ns.range(of: query, options: compare, range: NSRange(location: from, length: ns.length - from))140 if r.location == NSNotFound || r.length == 0 { break }141 if !options.wholeWord || isWholeWord(ns, r) { ranges.append(r) }142 from = r.location + max(1, r.length)143 }144 }145 for r in ranges {146 if hits.count >= limit { break }147 let range = r.location..<(r.location + r.length)148 hits.append(Hit(id: "\(documentName)#\(ref.story.sortKey)#\(ref.path.sortKey)#\(r.location)", documentName: documentName, story: ref.story, path: ref.path,149 range: range, context: Scan.context(text, around: range, radius: options.contextRadius),150 headingTrail: ref.story == .body ? trails.trail(ofBlock: ref.path.block) : []))151 }152 }153 return hits154 }155156 /// Searches several documents; hits are grouped by document, in the order given.157 public static func search(_ query: String, across docs: [(name: String, document: DocumentModel)], options: SearchOptions = SearchOptions()) -> [Hit] {158 var out: [Hit] = []159 for d in docs {160 var o = options161 if let m = options.maxHits { o.maxHits = m - out.count; if m - out.count <= 0 { break } }162 out.append(contentsOf: search(query, in: d.document, documentName: d.name, options: o))163 }164 return out165 }166167 private static func isWordUnit(_ ns: NSString, _ index: Int) -> Bool {168 let r = ns.rangeOfComposedCharacterSequence(at: index)169 let s = ns.substring(with: r)170 guard let u = s.unicodeScalars.first else { return false }171 return u.properties.isAlphabetic || u.properties.numericType != nil || u == "_"172 }173174 private static func isWholeWord(_ ns: NSString, _ r: NSRange) -> Bool {175 if r.location > 0 && isWordUnit(ns, r.location - 1) { return false }176 let end = r.location + r.length177 if end < ns.length && isWordUnit(ns, end) { return false }178 return true179 }180181 // MARK: Outline182183 /// The headings of the body in document order (same data as the navigation pane).184 public static func outline(of doc: DocumentModel) -> [OutlineEntry] { Numbering(doc).outline }185186 // MARK: Cited passages187188 /// Every sentence that contains a citation, with the keys it cites. The sentence is quoted from the text; nothing is inferred.189 /// A citation placed after the final period of a sentence belongs to that sentence.190 public static func citedPassages(in doc: DocumentModel) -> [CitedPassage] {191 let trails = HeadingTrails(doc)192 var out: [CitedPassage] = []193 for ref in Scan.paragraphs(of: doc) {194 let p = ref.paragraph195 var citations: [(offset: Int, keys: [String])] = []196 var pos = 0197 for r in p.runs {198 if !r.isDeleted, case .citation(let c)? = r.inline { citations.append((pos, c.items.map(\.key))) }199 pos += r.length200 }201 guard !citations.isEmpty else { continue }202 let text = Scan.maskedText(p)203 let ns = text as NSString204 let sentences = Scan.sentenceRanges(ns)205 guard !sentences.isEmpty else { continue }206 var keysOf: [Int: [String]] = [:]207 for c in citations {208 var index = sentences.firstIndex { c.offset >= $0.location && c.offset < $0.location + $0.length } ?? (sentences.count - 1)209 // A citation that opens a "sentence" holding no word of its own trails the previous sentence.210 if index > 0 {211 let s = sentences[index]212 let rest = ns.substring(with: s).replacingOccurrences(of: objectReplacementCharacter, with: "")213 let before = ns.substring(with: NSRange(location: s.location, length: c.offset - s.location)).replacingOccurrences(of: objectReplacementCharacter, with: "")214 if Scan.wordCount(rest) == 0 && Scan.wordCount(before) == 0 { index -= 1 }215 }216 for k in c.keys where !(keysOf[index] ?? []).contains(k) { keysOf[index, default: []].append(k) }217 }218 for index in keysOf.keys.sorted() {219 let s = sentences[index]220 // Rebuild the sentence with citations shown by key.221 var shown = ""222 var cursor = s.location223 for c in citations where c.offset >= s.location && c.offset < s.location + s.length {224 shown += ns.substring(with: NSRange(location: cursor, length: c.offset - cursor)) + "[" + c.keys.joined(separator: "; ") + "]"225 cursor = c.offset + 1226 }227 shown += ns.substring(with: NSRange(location: cursor, length: s.location + s.length - cursor))228 shown = shown.replacingOccurrences(of: objectReplacementCharacter, with: "").replacingOccurrences(of: lineSeparatorCharacter, with: " ")229 .trimmingCharacters(in: .whitespacesAndNewlines)230 out.append(CitedPassage(id: "\(ref.story.sortKey)#\(ref.path.sortKey)#\(s.location)", sentence: shown, keys: keysOf[index]!, story: ref.story, path: ref.path,231 range: s.location..<(s.location + s.length), headingTrail: ref.story == .body ? trails.trail(ofBlock: ref.path.block) : []))232 }233 }234 return out235 }236237 // MARK: Captions238239 /// Figures, tables and other numbered captions that no cross-reference targets and that the text never mentions by label and240 /// number ("Figure 3", "Fig. 3", "Tableau 2").241 public static func unreferencedCaptions(in doc: DocumentModel) -> [CaptionFinding] {242 let numbering = Numbering(doc)243 guard !numbering.captions.isEmpty else { return [] }244 var targeted: Set<String> = []245 var mentions: Set<String> = []246 let pattern = try? NSRegularExpression(pattern: "(?<![\\p{L}])(figures?|fig\\.?|tables?|tableaux?|tab\\.?|équations?|equations?|eq\\.?)[\\s\\u00A0\\u202F]*\\(?(\\d+)", options: [.caseInsensitive])247 for ref in Scan.paragraphs(of: doc) {248 let p = ref.paragraph249 for r in p.runs where !r.isDeleted {250 if case .crossRef(let x)? = r.inline {251 targeted.insert(x.target)252 if let para = numbering.bookmarks[x.target] { targeted.insert(para) }253 }254 if let link = r.props.link, link.hasPrefix("#") {255 let name = String(link.dropFirst())256 targeted.insert(name)257 if let para = numbering.bookmarks[name] { targeted.insert(para) }258 }259 }260 // A caption does not reference itself.261 if numbering.captions[p.id] != nil { continue }262 let text = p.visibleText263 guard let pattern, text.utf16.count >= 5 else { continue }264 let ns = text as NSString265 for m in pattern.matches(in: text, range: NSRange(location: 0, length: ns.length)) {266 let word = ns.substring(with: m.range(at: 1)).lowercased()267 let number = ns.substring(with: m.range(at: 2))268 let label = word.hasPrefix("fig") ? "figure" : (word.hasPrefix("tab") ? "table" : "equation")269 mentions.insert("\(label) \(number)")270 }271 }272 func family(_ label: String) -> String {273 let l = Scan.folded(label)274 if l.hasPrefix("fig") { return "figure" }275 if l.hasPrefix("tab") { return "table" }276 if l.hasPrefix("eq") { return "equation" }277 return l278 }279 return numbering.captions280 .filter { !targeted.contains($0.key) && !mentions.contains("\(family($0.value.label)) \($0.value.number)") }281 .map { CaptionFinding(label: $0.value.label, number: $0.value.number, text: $0.value.text, paragraphID: $0.key, blockIndex: $0.value.blockIndex) }282 .sorted { ($0.blockIndex, $0.label, $0.number) < ($1.blockIndex, $1.label, $1.number) }283 }284285 // MARK: Action items286287 /// Paragraphs that look like something left to do. **Rule-based, not AI**: a fixed list of markers ("TODO", "À faire", "must",288 /// "action:", "next steps:" …) and unchecked check-box list items ("[ ]", "☐"). One hit per paragraph, at the first marker.289 public static func actionItems(in doc: DocumentModel) -> [Hit] {290 let patterns: [(String, NSRegularExpression.Options)] = [291 ("^\\s*(?:[-*•]\\s*)?(?:\\[\\s?\\]|☐|❏|□)", []),292 ("\\b(?:TODO|TO DO|FIXME|TBD|TBC|XXX)\\b", []),293 ("(?<![\\p{L}])(?:à|a) (?:faire|compléter|completer|vérifier|verifier|valider|confirmer|revoir|corriger)(?![\\p{L}])", [.caseInsensitive]),294 ("(?<![\\p{L}])(?:action(?: item)?s?|next steps?|follow[- ]up|to be done|prochaines? étapes?|suivi|à suivre)\\s*:", [.caseInsensitive]),295 ("\\b(?:must|shall|needs? to|has to|have to)\\b", [.caseInsensitive]),296 ("(?<![\\p{L}])(?:doit|doivent|devra|devront|il faut|il faudra)(?![\\p{L}])", [.caseInsensitive]),297 ]298 let regexes = patterns.compactMap { try? NSRegularExpression(pattern: $0.0, options: $0.1) }299 let trails = HeadingTrails(doc)300 let name = doc.meta.title301 var hits: [Hit] = []302 for ref in Scan.paragraphs(of: doc) {303 let text = Scan.maskedText(ref.paragraph)304 let ns = text as NSString305 guard ns.length > 1 else { continue }306 var best: NSRange?307 for regex in regexes {308 if let m = regex.firstMatch(in: text, range: NSRange(location: 0, length: ns.length)), m.range.length > 0 {309 if best == nil || m.range.location < best!.location { best = m.range }310 }311 }312 guard let r = best else { continue }313 let range = r.location..<(r.location + r.length)314 hits.append(Hit(id: "action#\(ref.story.sortKey)#\(ref.path.sortKey)#\(r.location)", documentName: name, story: ref.story, path: ref.path, range: range,315 context: Scan.context(text, around: range, radius: 80), headingTrail: ref.story == .body ? trails.trail(ofBlock: ref.path.block) : []))316 }317 return hits318 }319}320321/// Heading trail of every body block, computed once.322struct HeadingTrails {323 private var trails: [[String]] = [[]]324 private var indexOfBlock: [Int] = []325326 init(_ doc: DocumentModel) {327 var levels = HeadingLevels(doc)328 var stack: [(level: Int, text: String)] = []329 var current = 0330 indexOfBlock.reserveCapacity(doc.body.count)331 for block in doc.body {332 if case .paragraph(let p) = block {333 let level = levels.level(p)334 if level > 0 {335 let text = p.visibleText.trimmingCharacters(in: .whitespacesAndNewlines)336 if !text.isEmpty {337 // The heading itself is filed under its parents.338 while let last = stack.last, last.level >= level { stack.removeLast() }339 indexOfBlock.append(trails.count)340 trails.append(stack.map(\.text))341 stack.append((level, text))342 current = trails.count343 trails.append(stack.map(\.text))344 continue345 }346 }347 }348 indexOfBlock.append(current)349 }350 }351352 func trail(ofBlock index: Int) -> [String] {353 indexOfBlock.indices.contains(index) ? trails[indexOfBlock[index]] : []354 }355}356