SPB Git

Repository Public

IntelligenceRunner.swift 14.0 KB 256 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocAutomation/Intelligence/IntelligenceRunner.swift3// Description: Runs an intelligence action on a text within the model's context window: directly when it fits, part by part4//              or by map-reduce when it does not, then checks the result for citation-like passages absent from the source.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation910public struct IntelligenceOutcome: Sendable, Equatable {11    public enum Strategy: Sendable, Equatable {12        /// The whole text went to the model in one request.13        case direct14        /// A long text was rewritten part by part (parts cut between paragraphs) and the parts joined.15        case partByPart(parts: Int)16        /// A long text was condensed part by part, then the action ran on the notes. Detail is lost on the way.17        case mapReduce(parts: Int, passes: Int)18    }19    public var text: String20    public var strategy: Strategy21    /// Tokens of the source text: measured by the model when `tokensMeasured`, estimated otherwise.22    public var inputTokens: Int23    public var tokensMeasured: Bool24    public var contextSize: Int25    /// Citation-like passages of the result that are not in the source. Shown to the user; never removed silently.26    public var unsupported: [UnsupportedPassage]27    /// Seconds, and seconds until the first text arrived.28    public var duration: Double29    /// A rewrite came back identical to the source (ignoring spacing): there is nothing to accept.30    public var isUnchanged: Bool31    public var timeToFirstText: Double?32    public var requests: Int33}3435public enum IntelligenceEvent: Sendable {36    public enum Stage: Sendable, Equatable {37        case preparing38        case generating39        case readingPart(Int, of: Int)40        case rewritingPart(Int, of: Int)41        case combining42    }43    case stage(Stage)44    /// The proposal so far (the whole text, not a delta).45    case partial(String)46    case finished(IntelligenceOutcome)47}4849public struct IntelligenceRunner: Sendable {50    public let engine: any TextIntelligence51    public init(engine: any TextIntelligence) { self.engine = engine }5253    /// Runs `action` on `text`. The stream ends with `.finished`, or throws an `IntelligenceError`.54    /// Cancelling the task that consumes the stream stops the generation.55    public func run(_ action: IntelligenceAction, on text: String) -> AsyncThrowingStream<IntelligenceEvent, Error> {56        AsyncThrowingStream { continuation in57            let task = Task {58                do {59                    let outcome = try await self.execute(action, text: text) { continuation.yield($0) }60                    continuation.yield(.finished(outcome))61                    continuation.finish()62                } catch is CancellationError {63                    continuation.finish(throwing: CancellationError())64                } catch {65                    continuation.finish(throwing: error)66                }67            }68            continuation.onTermination = { _ in task.cancel() }69        }70    }7172    /// The same, without streaming.73    public func result(_ action: IntelligenceAction, on text: String) async throws -> IntelligenceOutcome {74        try await execute(action, text: text) { _ in }75    }7677    // MARK: Implementation7879    private struct Counter { var requests = 0; var firstText: Date? }8081    private func execute(_ action: IntelligenceAction, text raw: String, emit: @Sendable (IntelligenceEvent) -> Void) async throws -> IntelligenceOutcome {82        let source = raw.trimmingCharacters(in: .whitespacesAndNewlines)83        guard !source.isEmpty else { throw IntelligenceError.emptyInput }84        let availability = engine.availability85        guard availability.isAvailable else { throw IntelligenceError.unavailable(availability) }86        let started = Date()87        emit(.stage(.preparing))8889        let context = max(1024, engine.contextSize)90        let instructions = IntelligencePrompts.instructions(for: action)91        let overhead = await engine.tokenCount(instructions) + 9692        let inputTokens = await engine.tokenCount(source)93        var counter = Counter()9495        // A rewrite returns about as much text as it receives: the window is shared in two. Other actions keep a quarter for the answer.96        func inputBudget(scale: Double) -> Int {97            let free = Double(context - overhead)98            return max(128, Int((action.replacesSource ? free / 2.2 : free * 0.72) * scale))99        }100101        // A bound on every answer: without one, a model that starts repeating itself writes until the window is full.102        // A rewrite may be somewhat longer than its source; other results are much shorter than theirs.103        func responseLimit(input: Int) -> Int {104            action.replacesSource ? min(context / 2, input + input / 2 + 160) : min(context / 4, max(400, input / 2 + 200))105        }106107        var scale = 1.0108        var forceParts = false109        while true {110            do {111                try Task.checkCancellation()112                let budget = inputBudget(scale: scale)113                let text: String114                let strategy: IntelligenceOutcome.Strategy115                if inputTokens <= budget, !forceParts {116                    emit(.stage(.generating))117                    let prompt = IntelligencePrompts.prompt(for: action, text: source)118                    var answer = try await complete(IntelligenceRequest(instructions: instructions, prompt: prompt, maximumResponseTokens: responseLimit(input: inputTokens),119                                                                       temperature: action.replacesSource ? 0.6 : 0.2), counter: &counter) { emit(.partial($0)) }120                    if action.replacesSource, IntelligenceRunner.sameText(answer, source) {121                        // The model sometimes hands a rewrite back untouched. One more attempt, said plainly and sampled more freely.122                        let insist = prompt + "\n\nDo not return the text as it is. Apply the task to it now and give only the new version."123                        answer = try await complete(IntelligenceRequest(instructions: instructions, prompt: insist, maximumResponseTokens: responseLimit(input: inputTokens),124                                                                       temperature: 0.9), counter: &counter) { emit(.partial($0)) }125                    }126                    text = answer127                    strategy = .direct128                } else if action.replacesSource {129                    (text, strategy) = try await partByPart(action, source: source, instructions: instructions, budget: budget, limit: responseLimit(input: budget), counter: &counter, emit: emit)130                } else {131                    (text, strategy) = try await mapReduce(action, source: source, instructions: instructions, context: context, scale: scale, counter: &counter, emit: emit)132                }133                let cleaned = IntelligenceRunner.clean(text)134                return IntelligenceOutcome(text: cleaned, strategy: strategy, inputTokens: inputTokens, tokensMeasured: engine.measuresTokens, contextSize: context,135                                           unsupported: CitationGuard.unsupportedPassages(in: cleaned, source: source), duration: Date().timeIntervalSince(started),136                                           isUnchanged: action.replacesSource && IntelligenceRunner.sameText(cleaned, source),137                                           timeToFirstText: counter.firstText.map { $0.timeIntervalSince(started) }, requests: counter.requests)138            } catch IntelligenceError.contextExceeded where scale > 0.3 {139                // The estimate was too optimistic: start again with smaller parts.140                scale *= 0.5141                forceParts = true142            }143        }144    }145146    private func complete(_ request: IntelligenceRequest, counter: inout Counter, partial: (String) -> Void) async throws -> String {147        counter.requests += 1148        var last = ""149        for try await text in engine.generate(request) {150            try Task.checkCancellation()151            if counter.firstText == nil, !text.isEmpty { counter.firstText = Date() }152            last = text153            partial(text)154        }155        try Task.checkCancellation()156        return last157    }158159    private func partByPart(_ action: IntelligenceAction, source: String, instructions: String, budget: Int, limit: Int, counter: inout Counter,160                            emit: @Sendable (IntelligenceEvent) -> Void) async throws -> (String, IntelligenceOutcome.Strategy) {161        let parts = TextChunker.chunks(source, budget: budget)162        var done: [String] = []163        for (i, part) in parts.enumerated() {164            emit(.stage(.rewritingPart(i + 1, of: parts.count)))165            let prefix = done.joined(separator: "\n\n")166            let text = try await complete(IntelligenceRequest(instructions: instructions, prompt: IntelligencePrompts.prompt(for: action, text: part), maximumResponseTokens: limit, temperature: 0.6),167                                          counter: &counter) { emit(.partial(prefix.isEmpty ? $0 : prefix + "\n\n" + $0)) }168            done.append(IntelligenceRunner.clean(text))169        }170        return (done.joined(separator: "\n\n"), .partByPart(parts: parts.count))171    }172173    private func mapReduce(_ action: IntelligenceAction, source: String, instructions: String, context: Int, scale: Double, counter: inout Counter,174                           emit: @Sendable (IntelligenceEvent) -> Void) async throws -> (String, IntelligenceOutcome.Strategy) {175        let mapInstructions = IntelligencePrompts.mapInstructions(for: action)176        let mapOverhead = await engine.tokenCount(mapInstructions) + 96177        // Notes are about a quarter of a part: the part may take most of the window.178        let mapBudget = max(128, Int(Double(context - mapOverhead) * 0.68 * scale))179        let reduceBudget = max(128, Int(Double(context - (await engine.tokenCount(instructions)) - 96) * 0.72 * scale))180181        var material = source182        var firstParts = 0183        var passes = 0184        var notes: [String] = []185        while true {186            passes += 1187            let parts = TextChunker.chunks(material, budget: mapBudget)188            if firstParts == 0 { firstParts = parts.count }189            notes = []190            for (i, part) in parts.enumerated() {191                emit(.stage(.readingPart(i + 1, of: parts.count)))192                let text = try await complete(IntelligenceRequest(instructions: mapInstructions, prompt: IntelligencePrompts.prompt(for: action, text: part),193                                                                 maximumResponseTokens: max(256, mapBudget / 3), temperature: 0.2),194                                              counter: &counter) { _ in }195                let note = IntelligenceRunner.clean(text)196                if !note.isEmpty, note.lowercased() != "(none)" { notes.append(note) }197            }198            let joined = notes.joined(separator: "\n\n")199            if notes.isEmpty || TextChunker.estimatedTokens(IntelligencePrompts.reducePrompt(for: action, notes: notes)) <= reduceBudget || passes >= 4 { break }200            // Still too long for one request: condense the notes once more.201            material = joined202        }203        emit(.stage(.combining))204        if notes.isEmpty, action == .actionItems {205            // Nothing found in any part: let the model phrase that in the language of the text, from a short sample.206            notes = [String(source.prefix(600))]207        }208        let text = try await complete(IntelligenceRequest(instructions: instructions, prompt: IntelligencePrompts.reducePrompt(for: action, notes: notes), maximumResponseTokens: context / 4, temperature: 0.2),209                                      counter: &counter) { emit(.partial($0)) }210        return (text, .mapReduce(parts: firstParts, passes: passes))211    }212213    static func sameText(_ a: String, _ b: String) -> Bool { a.filter { !$0.isWhitespace } == b.filter { !$0.isWhitespace } }214215    /// Trims the answer and removes a Markdown code fence wrapped around it.216    static func clean(_ text: String) -> String {217        var t = text.trimmingCharacters(in: .whitespacesAndNewlines)218        if t.hasPrefix("```"), t.hasSuffix("```"), t.count > 6 {219            var lines = t.components(separatedBy: "\n")220            lines.removeFirst()221            if lines.last?.trimmingCharacters(in: .whitespaces) == "```" { lines.removeLast() }222            t = lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)223        }224        return t225    }226}227228/// A scripted engine for tests and previews: answers from a closure, on this Mac, instantly.229public struct ScriptedIntelligence: TextIntelligence {230    public var availability: IntelligenceAvailability231    public var contextSize: Int232    public var measuresTokens: Bool { false }233    public var respond: @Sendable (IntelligenceRequest) throws -> String234235    public init(availability: IntelligenceAvailability = .available, contextSize: Int = 4096, respond: @escaping @Sendable (IntelligenceRequest) throws -> String) {236        self.availability = availability; self.contextSize = contextSize; self.respond = respond237    }238239    public func tokenCount(_ text: String) async -> Int { TextChunker.estimatedTokens(text) }240    public func supportsLanguage(_ code: String) -> Bool { true }241242    public func generate(_ request: IntelligenceRequest) -> AsyncThrowingStream<String, Error> {243        AsyncThrowingStream { continuation in244            do {245                if TextChunker.estimatedTokens(request.instructions) + TextChunker.estimatedTokens(request.prompt) > contextSize { throw IntelligenceError.contextExceeded }246                let answer = try respond(request)247                // Two snapshots, so that consumers see a partial text first.248                let half = answer.index(answer.startIndex, offsetBy: answer.count / 2)249                if answer.count > 1 { continuation.yield(String(answer[..<half])) }250                continuation.yield(answer)251                continuation.finish()252            } catch { continuation.finish(throwing: error) }253        }254    }255}256
Files main
Menu

Theme