SPB Git

Repository Public

ResearchAssistant.swift 18.2 KB 356 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocAudit/ResearchAssistant.swift3// Description: Research assistant: search in one or several documents, cited passages, unreferenced captions, outline, action items.4//              Everything is rule-based and grounded in the document text; nothing is generated.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import DocModel1011public enum ResearchAssistant {12    public struct Hit: Sendable, Identifiable, Hashable {13        public var id: String14        public var documentName: String15        public var story: StoryID16        public var path: ParaPath17        /// UTF-16 range in the paragraph text (`Paragraph.plainText`).18        public var range: Range<Int>19        /// The match with some text around it.20        public var context: String21        /// Headings above the paragraph, outermost first (body paragraphs only).22        public var headingTrail: [String]2324        public init(id: String, documentName: String, story: StoryID, path: ParaPath, range: Range<Int>, context: String, headingTrail: [String]) {25            self.id = id; self.documentName = documentName; self.story = story; self.path = path; self.range = range26            self.context = context; self.headingTrail = headingTrail27        }28    }2930    public struct SearchOptions: Sendable {31        public var caseSensitive: Bool32        /// When false "resume" finds "résumé".33        public var diacriticSensitive: Bool34        public var wholeWord: Bool35        /// The query is an ICU regular expression.36        public var regex: Bool37        public var includeTables: Bool38        public var includeNotes: Bool39        public var includeHeadersAndFooters: Bool40        /// UTF-16 units of context kept on each side of a match.41        public var contextRadius: Int42        public var maxHits: Int?4344        public init(caseSensitive: Bool = false, diacriticSensitive: Bool = false, wholeWord: Bool = false, regex: Bool = false,45                    includeTables: Bool = true, includeNotes: Bool = true, includeHeadersAndFooters: Bool = true, contextRadius: Int = 40, maxHits: Int? = nil) {46            self.caseSensitive = caseSensitive; self.diacriticSensitive = diacriticSensitive; self.wholeWord = wholeWord; self.regex = regex47            self.includeTables = includeTables; self.includeNotes = includeNotes; self.includeHeadersAndFooters = includeHeadersAndFooters48            self.contextRadius = contextRadius; self.maxHits = maxHits49        }50    }5152    public struct AcronymFinding: Sendable, Identifiable, Hashable {53        public var id: String { acronym }54        public var acronym: String55        /// Number of uses in the document.56        public var count: Int57        /// Evidence: the first use.58        public var firstUse: Hit59    }6061    public struct CitedPassage: Sendable, Identifiable, Hashable {62        public var id: String63        /// The sentence as written, citations shown as "[key; key]".64        public var sentence: String65        /// Citation keys of the sentence, in order of appearance.66        public var keys: [String]67        public var story: StoryID68        public var path: ParaPath69        /// UTF-16 range of the sentence in the paragraph.70        public var range: Range<Int>71        public var headingTrail: [String]72    }7374    public struct CaptionFinding: Sendable, Identifiable, Hashable {75        public var id: String { paragraphID }76        /// Sequence name: "Figure", "Table" …77        public var label: String78        public var number: Int79        /// Caption text as displayed ("Figure 2: Yield curve").80        public var text: String81        public var paragraphID: String82        public var blockIndex: Int83    }8485    public struct RepetitionFinding: Sendable, Identifiable, Hashable {86        public enum Kind: String, Sendable { case word, phrase }87        public var id: String88        public var kind: Kind89        /// The lemma (word) or the repeated phrase.90        public var text: String91        /// Occurrences inside the window.92        public var count: Int93        /// Number of consecutive words the occurrences fall in.94        public var span: Int95        public var occurrences: [Hit]96    }9798    // MARK: Search99100    /// Finds `query` in the body, tables, notes, headers and footers of a document. Case- and diacritic-insensitive unless the options101    /// say otherwise. Text marked as deleted by a tracked change is not searched. An invalid regular expression or an empty query gives no hit.102    public static func search(_ query: String, in doc: DocumentModel, documentName: String, options: SearchOptions = SearchOptions()) -> [Hit] {103        guard !query.isEmpty else { return [] }104        var regex: NSRegularExpression?105        if options.regex {106            var pattern = options.diacriticSensitive ? query : query.folding(options: .diacriticInsensitive, locale: nil)107            if options.wholeWord { pattern = "\\b(?:\(pattern))\\b" }108            guard let r = try? NSRegularExpression(pattern: pattern, options: options.caseSensitive ? [] : [.caseInsensitive]) else { return [] }109            regex = r110        }111        var compare: NSString.CompareOptions = []112        if !options.caseSensitive { compare.insert(.caseInsensitive) }113        if !options.diacriticSensitive { compare.insert(.diacriticInsensitive) }114        let trails = HeadingTrails(doc)115        var hits: [Hit] = []116        let limit = options.maxHits ?? Int.max117        for ref in Scan.paragraphs(of: doc) {118            if hits.count >= limit { break }119            switch ref.story {120            case .body: if ref.path.cell != nil && !options.includeTables { continue }121            case .note: if !options.includeNotes { continue }122            case .header, .footer: if !options.includeHeadersAndFooters { continue }123            }124            let text = Scan.maskedText(ref.paragraph)125            let ns = text as NSString126            guard ns.length > 0 else { continue }127            var ranges: [NSRange] = []128            if let regex {129                // Diacritic-insensitive matching needs a folded haystack; it is used only when folding keeps every offset in place.130                var haystack = text131                if !options.diacriticSensitive {132                    let foldedText = text.folding(options: .diacriticInsensitive, locale: nil)133                    if (foldedText as NSString).length == ns.length { haystack = foldedText }134                }135                for m in regex.matches(in: haystack, range: NSRange(location: 0, length: ns.length)) where m.range.length > 0 { ranges.append(m.range) }136            } else {137                var from = 0138                while from < ns.length {139                    let r = ns.range(of: query, options: compare, range: NSRange(location: from, length: ns.length - from))140                    if r.location == NSNotFound || r.length == 0 { break }141                    if !options.wholeWord || isWholeWord(ns, r) { ranges.append(r) }142                    from = r.location + max(1, r.length)143                }144            }145            for r in ranges {146                if hits.count >= limit { break }147                let range = r.location..<(r.location + r.length)148                hits.append(Hit(id: "\(documentName)#\(ref.story.sortKey)#\(ref.path.sortKey)#\(r.location)", documentName: documentName, story: ref.story, path: ref.path,149                                range: range, context: Scan.context(text, around: range, radius: options.contextRadius),150                                headingTrail: ref.story == .body ? trails.trail(ofBlock: ref.path.block) : []))151            }152        }153        return hits154    }155156    /// Searches several documents; hits are grouped by document, in the order given.157    public static func search(_ query: String, across docs: [(name: String, document: DocumentModel)], options: SearchOptions = SearchOptions()) -> [Hit] {158        var out: [Hit] = []159        for d in docs {160            var o = options161            if let m = options.maxHits { o.maxHits = m - out.count; if m - out.count <= 0 { break } }162            out.append(contentsOf: search(query, in: d.document, documentName: d.name, options: o))163        }164        return out165    }166167    private static func isWordUnit(_ ns: NSString, _ index: Int) -> Bool {168        let r = ns.rangeOfComposedCharacterSequence(at: index)169        let s = ns.substring(with: r)170        guard let u = s.unicodeScalars.first else { return false }171        return u.properties.isAlphabetic || u.properties.numericType != nil || u == "_"172    }173174    private static func isWholeWord(_ ns: NSString, _ r: NSRange) -> Bool {175        if r.location > 0 && isWordUnit(ns, r.location - 1) { return false }176        let end = r.location + r.length177        if end < ns.length && isWordUnit(ns, end) { return false }178        return true179    }180181    // MARK: Outline182183    /// The headings of the body in document order (same data as the navigation pane).184    public static func outline(of doc: DocumentModel) -> [OutlineEntry] { Numbering(doc).outline }185186    // MARK: Cited passages187188    /// Every sentence that contains a citation, with the keys it cites. The sentence is quoted from the text; nothing is inferred.189    /// A citation placed after the final period of a sentence belongs to that sentence.190    public static func citedPassages(in doc: DocumentModel) -> [CitedPassage] {191        let trails = HeadingTrails(doc)192        var out: [CitedPassage] = []193        for ref in Scan.paragraphs(of: doc) {194            let p = ref.paragraph195            var citations: [(offset: Int, keys: [String])] = []196            var pos = 0197            for r in p.runs {198                if !r.isDeleted, case .citation(let c)? = r.inline { citations.append((pos, c.items.map(\.key))) }199                pos += r.length200            }201            guard !citations.isEmpty else { continue }202            let text = Scan.maskedText(p)203            let ns = text as NSString204            let sentences = Scan.sentenceRanges(ns)205            guard !sentences.isEmpty else { continue }206            var keysOf: [Int: [String]] = [:]207            for c in citations {208                var index = sentences.firstIndex { c.offset >= $0.location && c.offset < $0.location + $0.length } ?? (sentences.count - 1)209                // A citation that opens a "sentence" holding no word of its own trails the previous sentence.210                if index > 0 {211                    let s = sentences[index]212                    let rest = ns.substring(with: s).replacingOccurrences(of: objectReplacementCharacter, with: "")213                    let before = ns.substring(with: NSRange(location: s.location, length: c.offset - s.location)).replacingOccurrences(of: objectReplacementCharacter, with: "")214                    if Scan.wordCount(rest) == 0 && Scan.wordCount(before) == 0 { index -= 1 }215                }216                for k in c.keys where !(keysOf[index] ?? []).contains(k) { keysOf[index, default: []].append(k) }217            }218            for index in keysOf.keys.sorted() {219                let s = sentences[index]220                // Rebuild the sentence with citations shown by key.221                var shown = ""222                var cursor = s.location223                for c in citations where c.offset >= s.location && c.offset < s.location + s.length {224                    shown += ns.substring(with: NSRange(location: cursor, length: c.offset - cursor)) + "[" + c.keys.joined(separator: "; ") + "]"225                    cursor = c.offset + 1226                }227                shown += ns.substring(with: NSRange(location: cursor, length: s.location + s.length - cursor))228                shown = shown.replacingOccurrences(of: objectReplacementCharacter, with: "").replacingOccurrences(of: lineSeparatorCharacter, with: " ")229                    .trimmingCharacters(in: .whitespacesAndNewlines)230                out.append(CitedPassage(id: "\(ref.story.sortKey)#\(ref.path.sortKey)#\(s.location)", sentence: shown, keys: keysOf[index]!, story: ref.story, path: ref.path,231                                        range: s.location..<(s.location + s.length), headingTrail: ref.story == .body ? trails.trail(ofBlock: ref.path.block) : []))232            }233        }234        return out235    }236237    // MARK: Captions238239    /// Figures, tables and other numbered captions that no cross-reference targets and that the text never mentions by label and240    /// number ("Figure 3", "Fig. 3", "Tableau 2").241    public static func unreferencedCaptions(in doc: DocumentModel) -> [CaptionFinding] {242        let numbering = Numbering(doc)243        guard !numbering.captions.isEmpty else { return [] }244        var targeted: Set<String> = []245        var mentions: Set<String> = []246        let pattern = try? NSRegularExpression(pattern: "(?<![\\p{L}])(figures?|fig\\.?|tables?|tableaux?|tab\\.?|équations?|equations?|eq\\.?)[\\s\\u00A0\\u202F]*\\(?(\\d+)", options: [.caseInsensitive])247        for ref in Scan.paragraphs(of: doc) {248            let p = ref.paragraph249            for r in p.runs where !r.isDeleted {250                if case .crossRef(let x)? = r.inline {251                    targeted.insert(x.target)252                    if let para = numbering.bookmarks[x.target] { targeted.insert(para) }253                }254                if let link = r.props.link, link.hasPrefix("#") {255                    let name = String(link.dropFirst())256                    targeted.insert(name)257                    if let para = numbering.bookmarks[name] { targeted.insert(para) }258                }259            }260            // A caption does not reference itself.261            if numbering.captions[p.id] != nil { continue }262            let text = p.visibleText263            guard let pattern, text.utf16.count >= 5 else { continue }264            let ns = text as NSString265            for m in pattern.matches(in: text, range: NSRange(location: 0, length: ns.length)) {266                let word = ns.substring(with: m.range(at: 1)).lowercased()267                let number = ns.substring(with: m.range(at: 2))268                let label = word.hasPrefix("fig") ? "figure" : (word.hasPrefix("tab") ? "table" : "equation")269                mentions.insert("\(label) \(number)")270            }271        }272        func family(_ label: String) -> String {273            let l = Scan.folded(label)274            if l.hasPrefix("fig") { return "figure" }275            if l.hasPrefix("tab") { return "table" }276            if l.hasPrefix("eq") { return "equation" }277            return l278        }279        return numbering.captions280            .filter { !targeted.contains($0.key) && !mentions.contains("\(family($0.value.label)) \($0.value.number)") }281            .map { CaptionFinding(label: $0.value.label, number: $0.value.number, text: $0.value.text, paragraphID: $0.key, blockIndex: $0.value.blockIndex) }282            .sorted { ($0.blockIndex, $0.label, $0.number) < ($1.blockIndex, $1.label, $1.number) }283    }284285    // MARK: Action items286287    /// Paragraphs that look like something left to do. **Rule-based, not AI**: a fixed list of markers ("TODO", "À faire", "must",288    /// "action:", "next steps:" …) and unchecked check-box list items ("[ ]", "☐"). One hit per paragraph, at the first marker.289    public static func actionItems(in doc: DocumentModel) -> [Hit] {290        let patterns: [(String, NSRegularExpression.Options)] = [291            ("^\\s*(?:[-*•]\\s*)?(?:\\[\\s?\\]|☐|❏|□)", []),292            ("\\b(?:TODO|TO DO|FIXME|TBD|TBC|XXX)\\b", []),293            ("(?<![\\p{L}])(?:à|a) (?:faire|compléter|completer|vérifier|verifier|valider|confirmer|revoir|corriger)(?![\\p{L}])", [.caseInsensitive]),294            ("(?<![\\p{L}])(?:action(?: item)?s?|next steps?|follow[- ]up|to be done|prochaines? étapes?|suivi|à suivre)\\s*:", [.caseInsensitive]),295            ("\\b(?:must|shall|needs? to|has to|have to)\\b", [.caseInsensitive]),296            ("(?<![\\p{L}])(?:doit|doivent|devra|devront|il faut|il faudra)(?![\\p{L}])", [.caseInsensitive]),297        ]298        let regexes = patterns.compactMap { try? NSRegularExpression(pattern: $0.0, options: $0.1) }299        let trails = HeadingTrails(doc)300        let name = doc.meta.title301        var hits: [Hit] = []302        for ref in Scan.paragraphs(of: doc) {303            let text = Scan.maskedText(ref.paragraph)304            let ns = text as NSString305            guard ns.length > 1 else { continue }306            var best: NSRange?307            for regex in regexes {308                if let m = regex.firstMatch(in: text, range: NSRange(location: 0, length: ns.length)), m.range.length > 0 {309                    if best == nil || m.range.location < best!.location { best = m.range }310                }311            }312            guard let r = best else { continue }313            let range = r.location..<(r.location + r.length)314            hits.append(Hit(id: "action#\(ref.story.sortKey)#\(ref.path.sortKey)#\(r.location)", documentName: name, story: ref.story, path: ref.path, range: range,315                            context: Scan.context(text, around: range, radius: 80), headingTrail: ref.story == .body ? trails.trail(ofBlock: ref.path.block) : []))316        }317        return hits318    }319}320321/// Heading trail of every body block, computed once.322struct HeadingTrails {323    private var trails: [[String]] = [[]]324    private var indexOfBlock: [Int] = []325326    init(_ doc: DocumentModel) {327        var levels = HeadingLevels(doc)328        var stack: [(level: Int, text: String)] = []329        var current = 0330        indexOfBlock.reserveCapacity(doc.body.count)331        for block in doc.body {332            if case .paragraph(let p) = block {333                let level = levels.level(p)334                if level > 0 {335                    let text = p.visibleText.trimmingCharacters(in: .whitespacesAndNewlines)336                    if !text.isEmpty {337                        // The heading itself is filed under its parents.338                        while let last = stack.last, last.level >= level { stack.removeLast() }339                        indexOfBlock.append(trails.count)340                        trails.append(stack.map(\.text))341                        stack.append((level, text))342                        current = trails.count343                        trails.append(stack.map(\.text))344                        continue345                    }346                }347            }348            indexOfBlock.append(current)349        }350    }351352    func trail(ofBlock index: Int) -> [String] {353        indexOfBlock.indices.contains(index) ? trails[indexOfBlock[index]] : []354    }355}356
Files main
Menu

Theme