Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocAutomation/Intelligence/IntelligenceRunner.swift3// Description: Runs an intelligence action on a text within the model's context window: directly when it fits, part by part4// or by map-reduce when it does not, then checks the result for citation-like passages absent from the source.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation910public struct IntelligenceOutcome: Sendable, Equatable {11 public enum Strategy: Sendable, Equatable {12 /// The whole text went to the model in one request.13 case direct14 /// A long text was rewritten part by part (parts cut between paragraphs) and the parts joined.15 case partByPart(parts: Int)16 /// A long text was condensed part by part, then the action ran on the notes. Detail is lost on the way.17 case mapReduce(parts: Int, passes: Int)18 }19 public var text: String20 public var strategy: Strategy21 /// Tokens of the source text: measured by the model when `tokensMeasured`, estimated otherwise.22 public var inputTokens: Int23 public var tokensMeasured: Bool24 public var contextSize: Int25 /// Citation-like passages of the result that are not in the source. Shown to the user; never removed silently.26 public var unsupported: [UnsupportedPassage]27 /// Seconds, and seconds until the first text arrived.28 public var duration: Double29 /// A rewrite came back identical to the source (ignoring spacing): there is nothing to accept.30 public var isUnchanged: Bool31 public var timeToFirstText: Double?32 public var requests: Int33}3435public enum IntelligenceEvent: Sendable {36 public enum Stage: Sendable, Equatable {37 case preparing38 case generating39 case readingPart(Int, of: Int)40 case rewritingPart(Int, of: Int)41 case combining42 }43 case stage(Stage)44 /// The proposal so far (the whole text, not a delta).45 case partial(String)46 case finished(IntelligenceOutcome)47}4849public struct IntelligenceRunner: Sendable {50 public let engine: any TextIntelligence51 public init(engine: any TextIntelligence) { self.engine = engine }5253 /// Runs `action` on `text`. The stream ends with `.finished`, or throws an `IntelligenceError`.54 /// Cancelling the task that consumes the stream stops the generation.55 public func run(_ action: IntelligenceAction, on text: String) -> AsyncThrowingStream<IntelligenceEvent, Error> {56 AsyncThrowingStream { continuation in57 let task = Task {58 do {59 let outcome = try await self.execute(action, text: text) { continuation.yield($0) }60 continuation.yield(.finished(outcome))61 continuation.finish()62 } catch is CancellationError {63 continuation.finish(throwing: CancellationError())64 } catch {65 continuation.finish(throwing: error)66 }67 }68 continuation.onTermination = { _ in task.cancel() }69 }70 }7172 /// The same, without streaming.73 public func result(_ action: IntelligenceAction, on text: String) async throws -> IntelligenceOutcome {74 try await execute(action, text: text) { _ in }75 }7677 // MARK: Implementation7879 private struct Counter { var requests = 0; var firstText: Date? }8081 private func execute(_ action: IntelligenceAction, text raw: String, emit: @Sendable (IntelligenceEvent) -> Void) async throws -> IntelligenceOutcome {82 let source = raw.trimmingCharacters(in: .whitespacesAndNewlines)83 guard !source.isEmpty else { throw IntelligenceError.emptyInput }84 let availability = engine.availability85 guard availability.isAvailable else { throw IntelligenceError.unavailable(availability) }86 let started = Date()87 emit(.stage(.preparing))8889 let context = max(1024, engine.contextSize)90 let instructions = IntelligencePrompts.instructions(for: action)91 let overhead = await engine.tokenCount(instructions) + 9692 let inputTokens = await engine.tokenCount(source)93 var counter = Counter()9495 // A rewrite returns about as much text as it receives: the window is shared in two. Other actions keep a quarter for the answer.96 func inputBudget(scale: Double) -> Int {97 let free = Double(context - overhead)98 return max(128, Int((action.replacesSource ? free / 2.2 : free * 0.72) * scale))99 }100101 // A bound on every answer: without one, a model that starts repeating itself writes until the window is full.102 // A rewrite may be somewhat longer than its source; other results are much shorter than theirs.103 func responseLimit(input: Int) -> Int {104 action.replacesSource ? min(context / 2, input + input / 2 + 160) : min(context / 4, max(400, input / 2 + 200))105 }106107 var scale = 1.0108 var forceParts = false109 while true {110 do {111 try Task.checkCancellation()112 let budget = inputBudget(scale: scale)113 let text: String114 let strategy: IntelligenceOutcome.Strategy115 if inputTokens <= budget, !forceParts {116 emit(.stage(.generating))117 let prompt = IntelligencePrompts.prompt(for: action, text: source)118 var answer = try await complete(IntelligenceRequest(instructions: instructions, prompt: prompt, maximumResponseTokens: responseLimit(input: inputTokens),119 temperature: action.replacesSource ? 0.6 : 0.2), counter: &counter) { emit(.partial($0)) }120 if action.replacesSource, IntelligenceRunner.sameText(answer, source) {121 // The model sometimes hands a rewrite back untouched. One more attempt, said plainly and sampled more freely.122 let insist = prompt + "\n\nDo not return the text as it is. Apply the task to it now and give only the new version."123 answer = try await complete(IntelligenceRequest(instructions: instructions, prompt: insist, maximumResponseTokens: responseLimit(input: inputTokens),124 temperature: 0.9), counter: &counter) { emit(.partial($0)) }125 }126 text = answer127 strategy = .direct128 } else if action.replacesSource {129 (text, strategy) = try await partByPart(action, source: source, instructions: instructions, budget: budget, limit: responseLimit(input: budget), counter: &counter, emit: emit)130 } else {131 (text, strategy) = try await mapReduce(action, source: source, instructions: instructions, context: context, scale: scale, counter: &counter, emit: emit)132 }133 let cleaned = IntelligenceRunner.clean(text)134 return IntelligenceOutcome(text: cleaned, strategy: strategy, inputTokens: inputTokens, tokensMeasured: engine.measuresTokens, contextSize: context,135 unsupported: CitationGuard.unsupportedPassages(in: cleaned, source: source), duration: Date().timeIntervalSince(started),136 isUnchanged: action.replacesSource && IntelligenceRunner.sameText(cleaned, source),137 timeToFirstText: counter.firstText.map { $0.timeIntervalSince(started) }, requests: counter.requests)138 } catch IntelligenceError.contextExceeded where scale > 0.3 {139 // The estimate was too optimistic: start again with smaller parts.140 scale *= 0.5141 forceParts = true142 }143 }144 }145146 private func complete(_ request: IntelligenceRequest, counter: inout Counter, partial: (String) -> Void) async throws -> String {147 counter.requests += 1148 var last = ""149 for try await text in engine.generate(request) {150 try Task.checkCancellation()151 if counter.firstText == nil, !text.isEmpty { counter.firstText = Date() }152 last = text153 partial(text)154 }155 try Task.checkCancellation()156 return last157 }158159 private func partByPart(_ action: IntelligenceAction, source: String, instructions: String, budget: Int, limit: Int, counter: inout Counter,160 emit: @Sendable (IntelligenceEvent) -> Void) async throws -> (String, IntelligenceOutcome.Strategy) {161 let parts = TextChunker.chunks(source, budget: budget)162 var done: [String] = []163 for (i, part) in parts.enumerated() {164 emit(.stage(.rewritingPart(i + 1, of: parts.count)))165 let prefix = done.joined(separator: "\n\n")166 let text = try await complete(IntelligenceRequest(instructions: instructions, prompt: IntelligencePrompts.prompt(for: action, text: part), maximumResponseTokens: limit, temperature: 0.6),167 counter: &counter) { emit(.partial(prefix.isEmpty ? $0 : prefix + "\n\n" + $0)) }168 done.append(IntelligenceRunner.clean(text))169 }170 return (done.joined(separator: "\n\n"), .partByPart(parts: parts.count))171 }172173 private func mapReduce(_ action: IntelligenceAction, source: String, instructions: String, context: Int, scale: Double, counter: inout Counter,174 emit: @Sendable (IntelligenceEvent) -> Void) async throws -> (String, IntelligenceOutcome.Strategy) {175 let mapInstructions = IntelligencePrompts.mapInstructions(for: action)176 let mapOverhead = await engine.tokenCount(mapInstructions) + 96177 // Notes are about a quarter of a part: the part may take most of the window.178 let mapBudget = max(128, Int(Double(context - mapOverhead) * 0.68 * scale))179 let reduceBudget = max(128, Int(Double(context - (await engine.tokenCount(instructions)) - 96) * 0.72 * scale))180181 var material = source182 var firstParts = 0183 var passes = 0184 var notes: [String] = []185 while true {186 passes += 1187 let parts = TextChunker.chunks(material, budget: mapBudget)188 if firstParts == 0 { firstParts = parts.count }189 notes = []190 for (i, part) in parts.enumerated() {191 emit(.stage(.readingPart(i + 1, of: parts.count)))192 let text = try await complete(IntelligenceRequest(instructions: mapInstructions, prompt: IntelligencePrompts.prompt(for: action, text: part),193 maximumResponseTokens: max(256, mapBudget / 3), temperature: 0.2),194 counter: &counter) { _ in }195 let note = IntelligenceRunner.clean(text)196 if !note.isEmpty, note.lowercased() != "(none)" { notes.append(note) }197 }198 let joined = notes.joined(separator: "\n\n")199 if notes.isEmpty || TextChunker.estimatedTokens(IntelligencePrompts.reducePrompt(for: action, notes: notes)) <= reduceBudget || passes >= 4 { break }200 // Still too long for one request: condense the notes once more.201 material = joined202 }203 emit(.stage(.combining))204 if notes.isEmpty, action == .actionItems {205 // Nothing found in any part: let the model phrase that in the language of the text, from a short sample.206 notes = [String(source.prefix(600))]207 }208 let text = try await complete(IntelligenceRequest(instructions: instructions, prompt: IntelligencePrompts.reducePrompt(for: action, notes: notes), maximumResponseTokens: context / 4, temperature: 0.2),209 counter: &counter) { emit(.partial($0)) }210 return (text, .mapReduce(parts: firstParts, passes: passes))211 }212213 static func sameText(_ a: String, _ b: String) -> Bool { a.filter { !$0.isWhitespace } == b.filter { !$0.isWhitespace } }214215 /// Trims the answer and removes a Markdown code fence wrapped around it.216 static func clean(_ text: String) -> String {217 var t = text.trimmingCharacters(in: .whitespacesAndNewlines)218 if t.hasPrefix("```"), t.hasSuffix("```"), t.count > 6 {219 var lines = t.components(separatedBy: "\n")220 lines.removeFirst()221 if lines.last?.trimmingCharacters(in: .whitespaces) == "```" { lines.removeLast() }222 t = lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)223 }224 return t225 }226}227228/// A scripted engine for tests and previews: answers from a closure, on this Mac, instantly.229public struct ScriptedIntelligence: TextIntelligence {230 public var availability: IntelligenceAvailability231 public var contextSize: Int232 public var measuresTokens: Bool { false }233 public var respond: @Sendable (IntelligenceRequest) throws -> String234235 public init(availability: IntelligenceAvailability = .available, contextSize: Int = 4096, respond: @escaping @Sendable (IntelligenceRequest) throws -> String) {236 self.availability = availability; self.contextSize = contextSize; self.respond = respond237 }238239 public func tokenCount(_ text: String) async -> Int { TextChunker.estimatedTokens(text) }240 public func supportsLanguage(_ code: String) -> Bool { true }241242 public func generate(_ request: IntelligenceRequest) -> AsyncThrowingStream<String, Error> {243 AsyncThrowingStream { continuation in244 do {245 if TextChunker.estimatedTokens(request.instructions) + TextChunker.estimatedTokens(request.prompt) > contextSize { throw IntelligenceError.contextExceeded }246 let answer = try respond(request)247 // Two snapshots, so that consumers see a partial text first.248 let half = answer.index(answer.startIndex, offsetBy: answer.count / 2)249 if answer.count > 1 { continuation.yield(String(answer[..<half])) }250 continuation.yield(answer)251 continuation.finish()252 } catch { continuation.finish(throwing: error) }253 }254 }255}256