Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXFeatureManage/Tests/ORIXFeatureManageTests/ManageModelTests.swift3// Description: View models of the management sections against real services on a temporary data folder: keys, grants, agents, settings, skills, storage, memory.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9@testable import ORIXFeatureManage10import ORIXAppUI1112/// A provider that answers every turn with the same short text. Used only to drive a real agent run without a network.13struct CannedProvider: LLMProvider {14 let id = ProviderID(rawValue: "canned")15 func listModels() async throws -> [ModelInfo] { [ModelInfo(id: "canned-1", provider: id, inputPricePerMTok: 1, outputPricePerMTok: 1)] }16 func stream(_ request: LLMRequest) -> AsyncThrowingStream<LLMStreamEvent, Error> {17 AsyncThrowingStream { c in18 c.yield(.textDelta("Nothing to change."))19 c.yield(.usage(TokenUsage(inputTokens: 10, outputTokens: 4)))20 c.yield(.stop(.endTurn))21 c.finish()22 }23 }24}2526func tempFolder(_ name: String) throws -> URL {27 let url = URL(fileURLWithPath: PathResolver.resolveSymlinks(NSTemporaryDirectory())).appendingPathComponent("orix-manage-\(name)-\(UUID().uuidString.prefix(8))")28 try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)29 return url30}3132@Suite("Integration · providers (preview Keychain service)") @MainActor struct ProvidersModelTests {33 @Test func keyIsStoredMaskedAndDeletedWithoutEverBeingExposed() async throws {34 let model = try await PreviewEnvironment.make(seed: false)35 let providers = ProvidersModel(services: model.services)36 let fake = "sk-test-0000-unit-\(UUID().uuidString.prefix(8))"37 defer { try? model.services.keychain.delete(account: KeychainStore.providerAccount(.openrouter)); Redactor.shared.unregister(secret: fake) }3839 // Too short: refused, and the message does not echo the input.40 #expect(await providers.saveKey("abc", for: .openrouter) == false)41 #expect(providers.state(.openrouter).keyError?.contains("abc") == false)4243 #expect(await providers.saveKey(fake, for: .openrouter))44 let state = providers.state(.openrouter)45 #expect(state.source == .keychain)46 let masked = try #require(state.masked)47 #expect(masked != fake && !masked.contains(fake) && masked.count < fake.count)48 #expect(masked.hasSuffix(String(fake.suffix(4))))49 #expect(await model.services.providers.configuredProviders().contains(.openrouter))5051 await providers.removeKey(for: .openrouter)52 #expect(providers.state(.openrouter).source == .none)53 #expect(providers.state(.openrouter).masked == nil)54 #expect(model.services.keychain.exists(account: KeychainStore.providerAccount(.openrouter)) == false)55 }5657 @Test func catalogFallsBackToTheRegistryWithTheReasonWhenThereIsNoKey() async throws {58 let model = try await PreviewEnvironment.make(seed: false)59 let providers = ProvidersModel(services: model.services)60 await providers.reload()61 guard providers.state(.openai).source == .none else { print("SKIPPED(an OpenAI key exists in the preview Keychain service)"); return }62 await providers.loadCatalog(for: .openai)63 let s = providers.state(.openai)64 #expect(s.catalogLoaded && !s.models.isEmpty)65 #expect(s.catalogError != nil)66 #expect(!providers.catalogIsLive(.openai))67 #expect(providers.filtered(.openai, search: "zzzz-no-such-model").isEmpty)68 #expect(providers.testModel(for: .openai, settings: AppSettings()) != nil)69 // Without a key the test reports the precise problem and sends nothing.70 await providers.test(.openai, settings: AppSettings())71 #expect(providers.state(.openai).test?.ok == false)72 #expect(providers.state(.openai).test?.message?.isEmpty == false)73 }7475 @Test func budgetBoundsClampEveryLimit() {76 let wild = Budget(maxTurns: 0, maxWallSeconds: 1, maxTotalTokens: 1, maxCostUSD: 9999, maxToolCalls: -4, maxSubprocessSeconds: 99_999, maxConcurrentTasks: 400)77 let b = BudgetBounds.clamped(wild)78 #expect(b.maxTurns == 1 && b.maxWallSeconds == 30 && b.maxTotalTokens == 10_000 && b.maxCostUSD == 100)79 #expect(b.maxToolCalls == 1 && b.maxSubprocessSeconds == 3600 && b.maxConcurrentTasks == 16)80 #expect(BudgetBounds.clamped(.standard) == .standard)81 }82}8384@Suite("Integration · permissions") @MainActor struct PermissionsModelTests {85 @Test func addFilterRevokeAndRefuseProtectedLocations() async throws {86 let model = try await PreviewEnvironment.make(seed: false)87 let p = PermissionsModel(services: model.services)88 await p.reload()89 #expect(p.grants.isEmpty)90 let folder = try tempFolder("grant"); defer { try? FileManager.default.removeItem(at: folder) }9192 let g = try await p.addPersistentGrant(kind: .fileRead, value: folder.path, tier: .highRisk, note: " test ")93 #expect(g.maxTier == .consequential) // never above tier 294 #expect(g.scope == .persistent && g.note == "test")95 #expect(p.visible(in: .persistent).map(\.id) == [g.id])96 _ = try await p.addPersistentGrant(kind: .network, value: "API.Example.com", tier: .consequential, note: "")97 #expect(p.grants.contains { $0.capability == .network(host: "api.example.com") })9899 // Protected paths, the whole disk, wildcards and sudo are refused with a reason.100 for bad in ["~/.ssh", NSHomeDirectory() + "/Library/Keychains", "/System/Library", "/"] {101 await #expect(throws: GrantRefusal.self) { try await p.addPersistentGrant(kind: .fileWrite, value: bad, tier: .reversibleWrite, note: "") }102 }103 await #expect(throws: GrantRefusal.self) { try await p.addPersistentGrant(kind: .network, value: "*", tier: .consequential, note: "") }104 await #expect(throws: GrantRefusal.self) { try await p.addPersistentGrant(kind: .processExec, value: "sudo", tier: .consequential, note: "") }105 await #expect(throws: GrantRefusal.self) { try await p.addPersistentGrant(kind: .fileRead, value: folder.path + "/missing", tier: .readOnly, note: "") }106 #expect(p.grants.count == 2)107108 p.search = "example"109 #expect(p.visible.count == 1)110 p.search = ""111 await p.revoke(g.id)112 #expect(p.visible.count == 1) // active only by default113 p.activeOnly = false114 #expect(p.visible.count == 2)115 #expect(PermissionsModel.state(try #require(p.grants.first { $0.id == g.id })) == .revoked)116 await p.revokeAll()117 #expect(p.activeCount == 0)118 #expect(p.audit.contains { $0.kind == .grantRevoked } && p.audit.contains { $0.kind == .grantCreated })119 #expect(await model.services.policy.activeGrants().isEmpty)120 }121122 @Test func statusIsNeverInventedAndToolsAreMappedToPermissions() async throws {123 let model = try await PreviewEnvironment.make(seed: false)124 let p = PermissionsModel(services: model.services)125 await p.reload()126 #expect(p.snapshot.count == MacPermission.allCases.count)127 #expect(p.snapshot[.accessibility] == MacPermissions.status(.accessibility))128 #expect(p.snapshot[.appleEvents] == .unknown)129 #expect(p.toolsByPermission[.accessibility]?.contains("accessibility.press") == true)130 #expect(PermissionsModel.canRequest(.fullDiskAccess) == false && PermissionsModel.canRequest(.appleEvents) == false)131 #expect(ManualGrantKind.allCases.allSatisfy { $0.defaultTier <= .consequential })132 }133}134135@Suite("Unit · tool grant tree") struct ToolGrantTreeTests {136 let all = ["filesystem.list", "filesystem.read", "filesystem.write", "git.status", "git.push", "http.request"]137138 @Test func triStateFollowsPatterns() {139 #expect(ToolGrantTree.state(family: "filesystem", tools: Array(all[0...2]), patterns: ["filesystem.read"]) == .mixed)140 #expect(ToolGrantTree.state(family: "filesystem", tools: Array(all[0...2]), patterns: ["filesystem.*"]) == .on)141 #expect(ToolGrantTree.state(family: "git", tools: ["git.status", "git.push"], patterns: ["filesystem.*"]) == .off)142 #expect(ToolGrantTree.state(family: "git", tools: ["git.status", "git.push"], patterns: ["*"]) == .on)143 #expect(ToolGrantTree.families(all).map(\.family) == ["filesystem", "git", "http"])144 }145146 @Test func togglingMapsToFamilyWildcardsAndIndividualIdentifiers() {147 var p = ToolGrantTree.setFamily("filesystem", on: true, patterns: ["filesystem.read", "git.status"], all: all)148 #expect(p == ["filesystem.*", "git.status"])149 p = ToolGrantTree.setTool("filesystem.write", on: false, patterns: p, all: all)150 #expect(p == ["filesystem.list", "filesystem.read", "git.status"])151 p = ToolGrantTree.setTool("git.push", on: true, patterns: p, all: all)152 #expect(p.contains("git.push") && !p.contains("git.*"))153 p = ToolGrantTree.setFamily("git", on: false, patterns: p, all: all)154 #expect(p == ["filesystem.list", "filesystem.read"])155 // "*" expands to one wildcard per family before a single change.156 p = ToolGrantTree.setTool("http.request", on: false, patterns: ["*"], all: all)157 #expect(p == ["filesystem.*", "git.*"])158 #expect(ToolGrantTree.setEverything(true, patterns: p, all: all) == ["*"])159 #expect(ToolGrantTree.setEverything(false, patterns: ["*"], all: all) == ["filesystem.*", "git.*", "http.*"])160 #expect(ToolGrantTree.normalized(["git.status", "git.*", "git.status", ""]) == ["git.*"])161 // The mapping agrees with the runtime's own matching.162 let profile = AgentProfile(name: "x", instructions: "y", model: ModelRef(provider: .anthropic, model: ""), toolPatterns: ["filesystem.*", "git.status"])163 for id in all { #expect(profile.allows(toolID: id) == ToolGrantTree.allows(id, profile.toolPatterns)) }164 }165}166167@Suite("Integration · agents") @MainActor struct AgentsModelTests {168 @Test func createValidateSaveDuplicateDelete() async throws {169 let model = try await PreviewEnvironment.make(seed: false)170 let a = AgentsModel(services: model.services)171 await a.reload()172 #expect(a.profiles.count == BuiltInAgents.all.count && a.profiles.allSatisfy(\.builtIn))173 #expect(a.toolIDs.contains("filesystem.read"))174 #expect(await a.save() == false) // a built-in is selected: read-only175176 a.newProfile()177 #expect(a.isDirty && !a.problems.isEmpty) // instructions are missing178 a.draft?.name = "general"179 a.draft?.instructions = "Summarise."180 #expect(a.problems.count == 1) // the name is taken181 a.draft?.name = "Bad Name"182 #expect(!a.problems.isEmpty)183 a.draft?.name = "summary-writer"184 let before = a.draft?.toolPatterns ?? []185 a.draft?.toolPatterns = ToolGrantTree.setFamily("documents", on: true, patterns: before, all: a.toolIDs)186 a.draft?.budget.maxTurns = 100_000187 #expect(a.problems.isEmpty)188 #expect(await a.save())189 let stored = try #require(await model.services.agents.profile(named: "summary-writer"))190 #expect(!stored.builtIn && stored.toolPatterns.contains("documents.*") && stored.budget.maxTurns == 200)191 #expect(stored.model.model.isEmpty) // "use the default model"192 #expect(!a.isDirty)193194 let copy = try #require(await a.duplicate(BuiltInAgents.all[1].id))195 #expect(copy.name == BuiltInAgents.all[1].name + "-copy" && !copy.builtIn && a.selectedID == copy.id)196 await a.delete(copy.id)197 #expect(await model.services.agents.profile(named: copy.name) == nil)198 await a.delete(stored.id)199 #expect(a.profiles.count == BuiltInAgents.all.count)200 }201202 @Test func evaluationRunsRealReviewModeRunsAndStoresResults() async throws {203 let model = try await PreviewEnvironment.make(seed: false)204 await model.services.providers.register(CannedProvider())205 model.settings.defaultModel = ModelRef(provider: ProviderID(rawValue: "canned"), model: "canned-1")206 try await model.services.runs.save(model.settings)207 let folder = try tempFolder("eval"); defer { try? FileManager.default.removeItem(at: folder) }208 try "hello".write(to: folder.appendingPathComponent("present.txt"), atomically: true, encoding: .utf8)209210 let a = AgentsModel(services: model.services)211 await a.reload()212 a.newProfile()213 a.draft?.name = "eval-agent"; a.draft?.instructions = "Inspect only."; a.draft?.toolPatterns = ["filesystem.list"]214 #expect(await a.save())215 await a.addCase(AgentEvalCase(objective: "Confirm present.txt exists", workspace: folder.path, expectations: [Expectation(kind: "file_exists", path: folder.path + "/present.txt")]))216 await a.addCase(AgentEvalCase(objective: "Confirm missing.txt exists", workspace: folder.path, expectations: [Expectation(kind: "file_exists", path: folder.path + "/missing.txt")]))217 await a.runEvaluation(defaultWorkspace: folder.path)218219 let agentID = try #require(a.selectedID)220 let stored = try #require(try await model.services.store.get(AgentEvalProfile.self, collection: AgentEvalProfile.collection, id: agentID))221 #expect(stored.cases.count == 2 && stored.results.count == 2)222 for r in stored.results {223 let runID = try #require(r.runID)224 let run = try #require(try await model.services.store.run(id: runID))225 #expect(run.mode == .review && run.status.isTerminal && run.status.rawValue == r.status)226 }227 // The missing file can never pass: the check is made by the runtime, not by what the model says.228 #expect(stored.results[1].passed == false)229 #expect(stored.results[0].checksTotal == 1)230 await a.removeCase(stored.cases[1].id)231 #expect(a.evaluation?.cases.count == 1 && a.evaluation?.results.count == 1)232 }233}234235@Suite("Integration · settings, storage, memory") @MainActor struct SettingsStorageTests {236 @Test func settingsPersistThroughTheModel() async throws {237 let model = try await PreviewEnvironment.make(seed: false)238 model.setting(\.defaultMode).wrappedValue = .review239 model.setting(\.appearance).wrappedValue = "dark"240 model.setting(\.retentionDays).wrappedValue = 14241 model.setting(\.allowScreenContentToProvider).wrappedValue = true242 model.settings.plannerModel = ModelRef(provider: .openai, model: "planner-x")243 model.settings.defaultBudget = BudgetBounds.clamped(Budget(maxTurns: 7))244 model.settings.onboardingCompleted = true245 model.saveSettings()246 var stored = AppSettings()247 for _ in 0..<50 {248 stored = await model.services.runs.settings()249 if stored.plannerModel != nil && stored.onboardingCompleted { break }250 try await Task.sleep(nanoseconds: 40_000_000)251 }252 #expect(stored.defaultMode == .review && stored.appearance == "dark" && stored.retentionDays == 14)253 #expect(stored.allowScreenContentToProvider && stored.plannerModel?.model == "planner-x" && stored.defaultBudget.maxTurns == 7)254 #expect(stored.onboardingCompleted)255 #expect(AppSettings().allowScreenContentToProvider == false && AppSettings().backgroundAgentEnabled == false)256 }257258 @Test func backupExportImportRoundTripAndPrune() async throws {259 let model = try await PreviewEnvironment.make()260 let storage = StorageModel(services: model.services)261 await storage.reload()262 #expect((storage.databaseBytes ?? 0) > 0 && storage.artifactsBytes != nil)263 let out = try tempFolder("storage"); defer { try? FileManager.default.removeItem(at: out) }264265 let backup = out.appendingPathComponent("backup.sqlite")266 #expect(await storage.backup(to: backup))267 #expect(((try FileManager.default.attributesOfItem(atPath: backup.path)[.size] as? Int) ?? 0) > 10_000)268 #expect(await storage.backup(to: backup) == false) // never overwrites269 #expect(storage.message?.kind == .error)270271 let bundle = out.appendingPathComponent("export.json")272 let exported = try #require(await storage.exportBundle(to: bundle))273 #expect(exported.workflows == 3)274 let text = try String(contentsOf: bundle, encoding: .utf8)275 #expect(!text.contains("Sample grant")) // grants are not exported276 #expect(StorageModel.summaryLines(exported).count == 4)277278 let other = try await PreviewEnvironment.make(seed: false)279 #expect(await other.services.workflows.list().isEmpty)280 let imported = try #require(await StorageModel(services: other.services).importBundle(from: bundle))281 #expect(imported.workflows == 3)282 #expect(Set(await other.services.workflows.list().map(\.id)) == Set(await model.services.workflows.list().map(\.id)))283 #expect(await other.services.policy.grants().isEmpty)284285 #expect(await StorageModel(services: other.services).importBundle(from: backup) == nil) // not a bundle: precise failure, nothing imported286287 #expect(await storage.pruneRuns(olderThanDays: 3650) == 0)288 #expect(await storage.pruneRuns(olderThanDays: 0) == 6)289 #expect(try await model.services.store.runs(limit: 50).isEmpty)290 }291292 @Test func auditVerifyReportsValidThenTheFirstBrokenLine() async throws {293 let model = try await PreviewEnvironment.make()294 let storage = StorageModel(services: model.services)295 await storage.reload()296 #expect((storage.auditCount ?? 0) >= 2 && storage.auditEntries.count == storage.auditCount)297 await storage.verifyAudit()298 #expect(storage.verification?.isValid == true)299 let out = try tempFolder("audit"); defer { try? FileManager.default.removeItem(at: out) }300 #expect(await storage.exportAudit(to: out.appendingPathComponent("audit.jsonl")))301 #expect(try String(contentsOf: out.appendingPathComponent("audit.jsonl"), encoding: .utf8).split(separator: "\n").count == storage.auditCount)302303 // Tamper with the first line on disk: verification names it.304 let url = model.services.paths.auditLog305 var lines = try String(contentsOf: url, encoding: .utf8).components(separatedBy: "\n")306 lines[0] = lines[0].replacingOccurrences(of: "Grant", with: "Grunt")307 try lines.joined(separator: "\n").write(to: url, atomically: false, encoding: .utf8)308 await storage.verifyAudit()309 #expect(storage.verification?.isValid == false)310 #expect(storage.verification?.firstBrokenLine == 1)311 }312313 @Test func memoryApproveExportDelete() async throws {314 let model = try await PreviewEnvironment.make(seed: false)315 let memory = MemoryModel(services: model.services)316 await model.services.memory.save(MemoryRecord(id: "m1", scope: .user, kind: "preference", text: "Reports in French", provenance: "run r1"))317 await model.services.memory.save(MemoryRecord(id: "m2", scope: .agent("general"), kind: "fix", text: "Use absolute paths", provenance: "run r2"))318 await memory.reload()319 #expect(memory.records.count == 2)320 #expect(memory.records.filter(MemoryModel.isPending).map(\.id) == ["m1"])321 await memory.approve("m1")322 #expect(memory.records.filter(MemoryModel.isPending).isEmpty)323 let json = try JSONValue.parse(try memory.exportJSON())324 #expect(json.arrayValue?.count == 2 && json[0]?["provenance"] != nil)325 await memory.delete("m2")326 #expect(memory.records.map(\.id) == ["m1"])327 await memory.deleteAll()328 #expect(await model.services.memory.all().isEmpty)329 }330331 @Test func diagnosticsContainNoSecretAndOnlyShortPaths() async throws {332 let model = try await PreviewEnvironment.make(seed: false)333 let secret = "sk-test-diagnostics-\(UUID().uuidString)"334 Redactor.shared.register(secret: secret); defer { Redactor.shared.unregister(secret: secret) }335 let report = await DiagnosticsReport.make(model)336 #expect(report.toolTotal > 40 && report.toolsByFamily.contains { $0.family == "filesystem" })337 #expect(report.toolsByFamily.map(\.count).reduce(0, +) == report.toolTotal)338 #expect(report.permissions.count == MacPermission.allCases.count)339 #expect(!report.text.contains(secret) && !report.text.contains("sk-"))340 #expect(report.dataFolder == PathResolver.display(model.services.paths.root.path))341 }342}343344@Suite("Integration · skills") @MainActor struct SkillsModelTests {345 @Test func formBuildsAManifestOrExplainsTheProblem() throws {346 let built = try SkillForm.wordCountTemplate.build()347 #expect(built.manifest.id == "skill.wordCount" && built.manifest.tests.count == 1 && built.sources.keys.sorted() == ["main.py"])348 #expect(built.manifest.permissions.fileRead == ["$input.path"] && built.manifest.tier == .readOnly)349 var bad = SkillForm.wordCountTemplate350 bad.inputSchemaJSON = "{ not json"351 #expect(throws: SkillForm.Problem.self) { try bad.build() }352 bad = .wordCountTemplate; bad.fileRead = "~/.ssh"353 #expect(throws: SkillForm.Problem.self) { try bad.build() }354 bad = .wordCountTemplate; bad.version = "one"355 #expect(throws: SkillForm.Problem.self) { try bad.build() }356 #expect(throws: SkillForm.Problem.self) { try SkillForm().build() }357 #expect(SkillsModel.permissionLines(built.manifest).count == 3)358 }359360 @Test func draftInstallNewVersionRollbackTestUninstall() async throws {361 let model = try await PreviewEnvironment.make(seed: false)362 let skills = SkillsModel(services: model.services)363 let draft = try #require(await skills.propose(.wordCountTemplate))364 guard draft.installable else {365 print("SKIPPED(the word-count skill's own test did not pass here, python3 is probably unavailable: \(draft.tests.map(\.detail)))")366 return367 }368 #expect(skills.drafts.count == 1 && skills.installed.isEmpty)369 #expect(await model.services.registry.tool(id: "skill.wordCount") == nil) // a draft cannot be called370 let installed = try #require(await skills.install(draftID: draft.id))371 #expect(installed.provenance?.source == "user" && installed.provenance?.testsPassed == 1)372 #expect(skills.drafts.isEmpty && skills.installed.map(\.id) == ["skill.wordCount"])373 #expect(await model.services.registry.tool(id: "skill.wordCount") != nil) // the registry was refreshed374 #expect(await model.services.audit.entries(limit: 50).contains { $0.kind == .skillInstalled })375376 var v2 = SkillForm.wordCountTemplate377 v2.version = "1.1.0"; v2.source += "\n# second version\n"378 let d2 = try #require(await skills.propose(v2))379 _ = try #require(await skills.install(draftID: d2.id))380 #expect(skills.installed.first?.package.manifest.version == "1.1.0" && skills.installed.first?.versions == ["1.0.0", "1.1.0"])381 await skills.rollback("skill.wordCount")382 #expect(skills.installed.first?.package.manifest.version == "1.0.0")383 await skills.activate("skill.wordCount", version: "1.1.0")384 #expect(skills.installed.first?.package.manifest.version == "1.1.0")385386 await skills.test(try #require(skills.installed.first))387 #expect(skills.testResults["skill.wordCount"]?.allSatisfy(\.passed) == true)388389 await skills.setEnabled(false, id: "skill.wordCount")390 #expect(await model.services.registry.tool(id: "skill.wordCount") == nil)391392 // A draft with a blocker is never installed.393 var bad = SkillForm.wordCountTemplate394 bad.name = "Fetch"; bad.source = "import json, urllib.request\nprint(json.dumps({'words': 1, 'lines': 1}))\n"395 let blocked = try #require(await skills.propose(bad))396 #expect(!blocked.installable)397 #expect(await skills.install(draftID: blocked.id) == nil)398 #expect(skills.message?.kind == .error)399 await skills.discard(draftID: blocked.id)400 #expect(skills.drafts.isEmpty)401402 // Import always lands in drafts.403 let first = try #require(skills.installed.first)404 let imported = try #require(await skills.importFolder(first.package.directory))405 #expect(imported.source.hasPrefix("import:") && skills.drafts.count == 1)406407 await skills.uninstall("skill.wordCount")408 #expect(skills.installed.isEmpty)409 #expect(skills.catalog(search: "filesystem.move").first?.tools.map(\.identifier) == ["filesystem.move"])410 }411}412