SPB Git

Repository Public

ResearchMode.swift 8.9 KB 152 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatAgents/Sources/ResearchMode.swift3// Description: Research Mode: plan sub-questions, retrieve passages for each, draft a cited answer, then verify every citation against the provided sources.4import Foundation5import LocalChatCore67public struct ResearchResult: Sendable {8    public var answer: String9    public var plan: [String]10    public var sources: [SourceReference]          // index i is cited as [i + 1]11    public var invalidCitations: [Int]             // numbers cited that match no source12    public var uncitedSentences: [String]          // factual-looking sentences without any citation13    public var unsupportedClaims: [String]         // flagged by the model-based verification pass14    public var isGrounded: Bool { invalidCitations.isEmpty && unsupportedClaims.isEmpty && !sources.isEmpty }15}1617public enum ResearchEvent: Sendable {18    case planning19    case planned([String])20    case retrieving(question: String)21    case retrieved(question: String, count: Int)22    case drafting23    case text(String)24    case verifying25    case finished(ResearchResult)26    case failed(String)27}2829public struct ResearchMode: Sendable {30    let generator: any TextGenerating31    let knowledge: any KnowledgeSearching32    public var maxQuestions = 433    public var passagesPerQuestion = 434    public var maxSources = 103536    public init(generator: any TextGenerating, knowledge: any KnowledgeSearching) { self.generator = generator; self.knowledge = knowledge }3738    public func run(question: String, modelID: String, collections: [UUID], settings: GenerationSettings = .default, verifyWithModel: Bool = true) -> AsyncStream<ResearchEvent> {39        AsyncStream { continuation in40            let task = Task {41                do {42                    let result = try await self.perform(question, modelID, collections, settings, verifyWithModel) { continuation.yield($0) }43                    continuation.yield(.finished(result))44                } catch is CancellationError {45                } catch { continuation.yield(.failed((error as? LocalizedError)?.errorDescription ?? String(describing: error))) }46                continuation.finish()47            }48            continuation.onTermination = { _ in task.cancel() }49        }50    }5152    /// Accepts a JSON array, an object with a list inside, or a numbered / bulleted list.53    public static func parsePlan(_ text: String, limit: Int) -> [String] {54        var body = text55        if let open = body.range(of: "</think>") { body = String(body[open.upperBound...]) }56        var items: [String] = []57        if let start = body.firstIndex(of: "["), let end = body.lastIndex(of: "]"), start < end, let value = JSONValue.parse(String(body[start...end])) {58            items = value.arrayValue?.compactMap { $0.stringValue ?? $0["question"]?.stringValue } ?? []59        }60        if items.isEmpty {61            items = body.components(separatedBy: .newlines).compactMap { line in62                let t = line.trimmingCharacters(in: .whitespaces)63                guard let r = t.range(of: #"^(\d+[.)]|[-*•])\s+"#, options: .regularExpression) else { return nil }64                return String(t[r.upperBound...])65            }66        }67        var seen = Set<String>()68        return items.map { $0.trimmingCharacters(in: CharacterSet(charactersIn: " \"'`")) }.filter { $0.count > 3 && seen.insert($0.lowercased()).inserted }.prefix(limit).map { $0 }69    }7071    public static func citedNumbers(_ text: String) -> [Int] {72        guard let regex = try? NSRegularExpression(pattern: #"\[(\d+(?:\s*[,–-]\s*\d+)*)\]"#) else { return [] }73        var out: [Int] = []74        for match in regex.matches(in: text, range: NSRange(text.startIndex..., in: text)) {75            guard let r = Range(match.range(at: 1), in: text) else { continue }76            for part in text[r].components(separatedBy: ",") {77                let bounds = part.components(separatedBy: CharacterSet(charactersIn: "–-")).compactMap { Int($0.trimmingCharacters(in: .whitespaces)) }78                if bounds.count == 2, bounds[0] <= bounds[1], bounds[1] - bounds[0] < 50 { out += Array(bounds[0]...bounds[1]) } else { out += bounds }79            }80        }81        return out82    }8384    /// Deterministic check: citations that point to no source, and declarative sentences that cite nothing.85    public static func checkCitations(answer: String, sourceCount: Int) -> (invalid: [Int], uncited: [String]) {86        let invalid = Array(Set(citedNumbers(answer).filter { $0 < 1 || $0 > sourceCount })).sorted()87        let paragraphs = answer.components(separatedBy: "\n").map { $0.trimmingCharacters(in: .whitespaces) }88            .filter { !$0.isEmpty && !$0.hasPrefix("#") && !$0.hasPrefix("```") }89        var uncited: [String] = []90        for p in paragraphs {91            for sentence in p.components(separatedBy: ". ").map({ $0.trimmingCharacters(in: .whitespaces) }) where sentence.count > 60 {92                // a citation at the end of the paragraph covers its sentences93                if citedNumbers(sentence).isEmpty && citedNumbers(String(p.suffix(12))).isEmpty && !sentence.hasSuffix("?") && !sentence.hasSuffix(":") { uncited.append(sentence) }94            }95        }96        return (invalid, uncited)97    }9899    private func perform(_ question: String, _ modelID: String, _ collections: [UUID], _ settings: GenerationSettings, _ verifyWithModel: Bool,100                         _ emit: @Sendable (ResearchEvent) -> Void) async throws -> ResearchResult {101        var quiet = settings; quiet.temperature = min(settings.temperature, 0.3); quiet.maxTokens = min(settings.maxTokens, 400); quiet.reasoningEnabled = false102103        emit(.planning)104        let planPrompt = "Break the research question below into at most \(maxQuestions) short, self-contained search queries that together cover it. " +105            "Answer with a JSON array of strings only.\n\nQuestion: \(question)"106        let planText = try await generator.complete(modelID: modelID, request: GenerationRequest(messages: [.user(planPrompt)], settings: quiet)).text107        var plan = Self.parsePlan(planText, limit: maxQuestions)108        if !plan.contains(where: { $0.caseInsensitiveCompare(question) == .orderedSame }) { plan.insert(question, at: 0) }   // always search the question itself109        emit(.planned(plan))110111        var hits: [KnowledgeHit] = []; var seen = Set<String>()112        for query in plan {113            try Task.checkCancellation()114            emit(.retrieving(question: query))115            let found = try await knowledge.search(query, collections: collections, limit: passagesPerQuestion)116            var added = 0117            for hit in found where seen.insert("\(hit.document.id)#\(hit.chunkIndex)").inserted { hits.append(hit); added += 1 }118            emit(.retrieved(question: query, count: added))119        }120        hits = Array(hits.sorted { $0.score > $1.score }.prefix(maxSources))121        let sources = hits.map(\.reference)122        guard !hits.isEmpty else {123            let text = "No passage in the selected collections is relevant to this question, so no grounded answer can be given."124            emit(.text(text))125            return ResearchResult(answer: text, plan: plan, sources: [], invalidCitations: [], uncitedSentences: [], unsupportedClaims: [])126        }127128        emit(.drafting)129        let block = hits.enumerated().map { "[\($0.offset + 1)] \($0.element.document.name)\($0.element.page.map { ", p. \($0)" } ?? "")\n\($0.element.text)" }.joined(separator: "\n\n")130        let system = "You are a careful research analyst. Answer only from the numbered sources. Put the supporting source numbers in brackets, like [2], " +131            "after every claim. If the sources do not answer part of the question, say so explicitly. Do not use outside knowledge."132        var answer = ""133        let draft = GenerationRequest(messages: [.system(system), .user("Sources:\n\n\(block)\n\nQuestion: \(question)")], settings: settings)134        for try await event in generator.generate(modelID: modelID, request: draft) {135            try Task.checkCancellation()136            if case .text(let t) = event { answer += t; emit(.text(t)) }137        }138139        emit(.verifying)140        let check = Self.checkCitations(answer: answer, sourceCount: hits.count)141        var unsupported: [String] = []142        if verifyWithModel {143            let prompt = "Sources:\n\n\(block)\n\nAnswer to check:\n\(answer)\n\nList every claim of the answer that the cited sources do not support. " +144                "Answer with a JSON array of short strings; answer [] when every claim is supported."145            if let text = try? await generator.complete(modelID: modelID, request: GenerationRequest(messages: [.user(prompt)], settings: quiet)).text {146                unsupported = Self.parsePlan(text, limit: 20)147            }148        }149        return ResearchResult(answer: answer, plan: plan, sources: sources, invalidCitations: check.invalid, uncitedSentences: check.uncited, unsupportedClaims: unsupported)150    }151}152
Files main
Menu

Theme