SPB Git

Repository Public License

ChatTests.swift 10.1 KB 190 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXFeatureChat/Tests/ORIXFeatureChatTests/ChatTests.swift10// Description      : Chat logic against real services with a scripted (mock)11//                    provider: transcript building, a full send → tool →12//                    verified outcome cycle, steering, session persistence,13//                    rendering.14// Module           : ORIXFeatureChatTests15// Repository       : https://git.spboucher.ai/orix-automation16// Created          : 2026-11-2117// Last Modified    : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author           : Simon-Pierre Boucher24// Position         : Professor of Finance25// Institution      : Université du Québec en Outaouais (UQO)26// Department       : Département des sciences administratives27// Organization     : SP Boucher28//29// Website          : https://spboucher.ai30// Git              : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General          : info@spboucher.ai37// Professional     : contact@spboucher.ai38// Direct           : simon@spboucher.ai39// Research         : research@spboucher.ai40// Development      : dev@spboucher.ai41// Support          : support@spboucher.ai42// Security/Admin   : admin@spboucher.ai43// Notifications    : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification   : Public70// Distribution     : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License          : Proprietary, all rights reserved (see license file)84// License File     : LICENSE85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Foundation93import SwiftUI94import Testing95@testable import ORIXFeatureChat96import ORIXAppUI97import ORIXTestingSupport9899@Suite("Unit · transcript") struct TranscriptTests {100    @Test func eventsBecomeItemsAndOnlyTheLatestPlanIsShown() throws {101        let plan1 = try JSONValue.encoding(AgentPlan(objective: "o", steps: [PlanStep(id: "a", title: "one")], revision: 1))102        let plan2 = try JSONValue.encoding(AgentPlan(objective: "o", steps: [PlanStep(id: "a", title: "two")], revision: 2))103        let events = [104            RunEvent(runID: "r", kind: .userMessage, text: "Do it"), RunEvent(runID: "r", kind: .planUpdated, data: plan1),105            RunEvent(runID: "r", kind: .assistantText, text: "Looking"),106            RunEvent(runID: "r", kind: .toolFinished, data: ["callID": "c1", "toolID": "filesystem.list", "status": "succeeded", "summary": "3 entries", "durationMs": 4, "tier": 0]),107            RunEvent(runID: "r", kind: .planUpdated, data: plan2), RunEvent(runID: "r", kind: .usage), RunEvent(runID: "r", kind: .statusChanged, text: "SUCCEEDED"),108            RunEvent(runID: "r", kind: .verification, text: "PASS a.txt exists", data: ["passed": true]),109        ]110        let live = ["c1": ToolActivity(id: "c1", toolID: "filesystem.list", arguments: .null, status: nil, summary: "", durationMs: nil, tier: nil, liveOutput: ""),111                    "c2": ToolActivity(id: "c2", toolID: "shell.run", arguments: .null, status: nil, summary: "", durationMs: nil, tier: nil, liveOutput: "building")]112        let items = ChatModel.items(from: events, live: live)113        #expect(items.count == 6)114        let plans = items.compactMap { if case .plan(_, let p) = $0 { return p }; return nil }115        #expect(plans.count == 1 && plans[0].revision == 2)116        let tools = items.compactMap { if case .tool(let t) = $0 { return t }; return nil }117        #expect(tools.map(\.id) == ["c1", "c2"])           // finished call shown once, running call appended118        #expect(tools[0].status == .succeeded && tools[1].status == nil)119    }120}121122@Suite("Integration · chat on real services (mock provider)") @MainActor struct ChatIntegrationTests {123    @Test func sendRunsToolsVerifiesAndPersistsTheSession() async throws {124        let app = try await PreviewEnvironment.make(seed: false)125        let work = try TempDirectory(prefix: "orix-chat"); defer { work.remove() }126        try work.write("notes.txt", "alpha\nbeta\n")127        let provider = ScriptedProvider(turns: [128            ScriptedProvider.tool("filesystem.read", ["path": "notes.txt"]),129            ScriptedProvider.tool("filesystem.write", ["path": "summary.txt", "content": "2 lines"]),130            ScriptedProvider.tool("task.complete", ["summary": "Wrote the summary.", "artifacts": [.string(work.file("summary.txt").path)]]),131        ])132        await app.services.providers.register(provider)133        var settings = AppSettings(); settings.onboardingCompleted = true; settings.defaultModel = ModelRef(provider: provider.id, model: "scripted-1")134        try await app.services.runs.save(settings); app.settings = settings135136        let chat = ChatModel(app: app)137        await chat.load()138        chat.workspace = work.url.path; chat.access = .readWrite; chat.mode = .autonomous139        chat.draft = "Summarise notes.txt into summary.txt"140        await chat.send()141        #expect(chat.isRunning)142        let runID = try #require(chat.activeRunID)143        let final = await app.services.runs.wait(runID: runID)144        #expect(final?.status == .succeeded)145        try await Task.sleep(nanoseconds: 300_000_000)146        await chat.rebuild()147        #expect(!chat.isRunning)148        #expect(try String(contentsOf: work.file("summary.txt"), encoding: .utf8) == "2 lines")149        let tools = chat.items.compactMap { if case .tool(let t) = $0 { return t.toolID }; return nil }150        #expect(tools == ["filesystem.read", "filesystem.write"])151        #expect(chat.items.contains { if case .outcome(let o) = $0 { return o.status == .succeeded && o.artifacts.count == 1 }; return false })152        #expect(chat.items.contains { if case .verification(_, true, _) = $0 { return true }; return false })153154        // The session is stored and a second model sees it after a relaunch.155        let again = ChatModel(app: app)156        await again.load()157        #expect(again.sessions.count == 1 && again.sessions[0].runIDs == [runID])158        #expect(again.items.count == chat.items.count)159160        let out = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("orix-chat-conversation.png")161        try await ShotRenderer.render(AgentChatView().environment(app), size: CGSize(width: 1200, height: 760), to: out)162        #expect(((try FileManager.default.attributesOfItem(atPath: out.path)[.size] as? Int) ?? 0) > 20_000)163        print("SHOT \(out.path)")164    }165166    @Test func writeOutsideTheGrantWaitsForApprovalAndDenyLeavesNoFile() async throws {167        let app = try await PreviewEnvironment.make(seed: false)168        let work = try TempDirectory(prefix: "orix-chat"); defer { work.remove() }169        let provider = ScriptedProvider(turns: [ScriptedProvider.tool("filesystem.write", ["path": "x.txt", "content": "x"]), ScriptedProvider.text("I could not write the file.")])170        await app.services.providers.register(provider)171        var settings = AppSettings(); settings.defaultModel = ModelRef(provider: provider.id, model: "scripted-1")172        try await app.services.runs.save(settings); app.settings = settings173        let chat = ChatModel(app: app)174        await chat.load()175        chat.workspace = work.url.path; chat.access = .read; chat.mode = .autonomous   // read only: the write must ask176        chat.draft = "Write x.txt"177        await chat.send()178        let runID = try #require(chat.activeRunID)179        var waited = 0180        while app.pendingApprovals.isEmpty && waited < 100 { try await Task.sleep(nanoseconds: 50_000_000); waited += 1 }181        let request = try #require(app.pendingApprovals.first)182        #expect(request.access.toolID == "filesystem.write" && request.access.tier == .reversibleWrite)183        #expect(request.missing == [.fileWrite(path: work.url.path + "/x.txt")])184        app.resolve(request, .deny(reason: "no"))185        let final = await app.services.runs.wait(runID: runID)186        #expect(final?.status != .succeeded)187        #expect(!FileManager.default.fileExists(atPath: work.file("x.txt").path))188    }189}190
Files main
Menu

Theme