SPB Git

Repository Public License

ManageModelTests.swift 24.2 KB 412 lines swift

1// ORIX OS Automation2// File: Packages/ORIXFeatureManage/Tests/ORIXFeatureManageTests/ManageModelTests.swift3// Description: View models of the management sections against real services on a temporary data folder: keys, grants, agents, settings, skills, storage, memory.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9@testable import ORIXFeatureManage10import ORIXAppUI1112/// A provider that answers every turn with the same short text. Used only to drive a real agent run without a network.13struct CannedProvider: LLMProvider {14    let id = ProviderID(rawValue: "canned")15    func listModels() async throws -> [ModelInfo] { [ModelInfo(id: "canned-1", provider: id, inputPricePerMTok: 1, outputPricePerMTok: 1)] }16    func stream(_ request: LLMRequest) -> AsyncThrowingStream<LLMStreamEvent, Error> {17        AsyncThrowingStream { c in18            c.yield(.textDelta("Nothing to change."))19            c.yield(.usage(TokenUsage(inputTokens: 10, outputTokens: 4)))20            c.yield(.stop(.endTurn))21            c.finish()22        }23    }24}2526func tempFolder(_ name: String) throws -> URL {27    let url = URL(fileURLWithPath: PathResolver.resolveSymlinks(NSTemporaryDirectory())).appendingPathComponent("orix-manage-\(name)-\(UUID().uuidString.prefix(8))")28    try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)29    return url30}3132@Suite("Integration · providers (preview Keychain service)") @MainActor struct ProvidersModelTests {33    @Test func keyIsStoredMaskedAndDeletedWithoutEverBeingExposed() async throws {34        let model = try await PreviewEnvironment.make(seed: false)35        let providers = ProvidersModel(services: model.services)36        let fake = "sk-test-0000-unit-\(UUID().uuidString.prefix(8))"37        defer { try? model.services.keychain.delete(account: KeychainStore.providerAccount(.openrouter)); Redactor.shared.unregister(secret: fake) }3839        // Too short: refused, and the message does not echo the input.40        #expect(await providers.saveKey("abc", for: .openrouter) == false)41        #expect(providers.state(.openrouter).keyError?.contains("abc") == false)4243        #expect(await providers.saveKey(fake, for: .openrouter))44        let state = providers.state(.openrouter)45        #expect(state.source == .keychain)46        let masked = try #require(state.masked)47        #expect(masked != fake && !masked.contains(fake) && masked.count < fake.count)48        #expect(masked.hasSuffix(String(fake.suffix(4))))49        #expect(await model.services.providers.configuredProviders().contains(.openrouter))5051        await providers.removeKey(for: .openrouter)52        #expect(providers.state(.openrouter).source == .none)53        #expect(providers.state(.openrouter).masked == nil)54        #expect(model.services.keychain.exists(account: KeychainStore.providerAccount(.openrouter)) == false)55    }5657    @Test func catalogFallsBackToTheRegistryWithTheReasonWhenThereIsNoKey() async throws {58        let model = try await PreviewEnvironment.make(seed: false)59        let providers = ProvidersModel(services: model.services)60        await providers.reload()61        guard providers.state(.openai).source == .none else { print("SKIPPED(an OpenAI key exists in the preview Keychain service)"); return }62        await providers.loadCatalog(for: .openai)63        let s = providers.state(.openai)64        #expect(s.catalogLoaded && !s.models.isEmpty)65        #expect(s.catalogError != nil)66        #expect(!providers.catalogIsLive(.openai))67        #expect(providers.filtered(.openai, search: "zzzz-no-such-model").isEmpty)68        #expect(providers.testModel(for: .openai, settings: AppSettings()) != nil)69        // Without a key the test reports the precise problem and sends nothing.70        await providers.test(.openai, settings: AppSettings())71        #expect(providers.state(.openai).test?.ok == false)72        #expect(providers.state(.openai).test?.message?.isEmpty == false)73    }7475    @Test func budgetBoundsClampEveryLimit() {76        let wild = Budget(maxTurns: 0, maxWallSeconds: 1, maxTotalTokens: 1, maxCostUSD: 9999, maxToolCalls: -4, maxSubprocessSeconds: 99_999, maxConcurrentTasks: 400)77        let b = BudgetBounds.clamped(wild)78        #expect(b.maxTurns == 1 && b.maxWallSeconds == 30 && b.maxTotalTokens == 10_000 && b.maxCostUSD == 100)79        #expect(b.maxToolCalls == 1 && b.maxSubprocessSeconds == 3600 && b.maxConcurrentTasks == 16)80        #expect(BudgetBounds.clamped(.standard) == .standard)81    }82}8384@Suite("Integration · permissions") @MainActor struct PermissionsModelTests {85    @Test func addFilterRevokeAndRefuseProtectedLocations() async throws {86        let model = try await PreviewEnvironment.make(seed: false)87        let p = PermissionsModel(services: model.services)88        await p.reload()89        #expect(p.grants.isEmpty)90        let folder = try tempFolder("grant"); defer { try? FileManager.default.removeItem(at: folder) }9192        let g = try await p.addPersistentGrant(kind: .fileRead, value: folder.path, tier: .highRisk, note: " test ")93        #expect(g.maxTier == .consequential)          // never above tier 294        #expect(g.scope == .persistent && g.note == "test")95        #expect(p.visible(in: .persistent).map(\.id) == [g.id])96        _ = try await p.addPersistentGrant(kind: .network, value: "API.Example.com", tier: .consequential, note: "")97        #expect(p.grants.contains { $0.capability == .network(host: "api.example.com") })9899        // Protected paths, the whole disk, wildcards and sudo are refused with a reason.100        for bad in ["~/.ssh", NSHomeDirectory() + "/Library/Keychains", "/System/Library", "/"] {101            await #expect(throws: GrantRefusal.self) { try await p.addPersistentGrant(kind: .fileWrite, value: bad, tier: .reversibleWrite, note: "") }102        }103        await #expect(throws: GrantRefusal.self) { try await p.addPersistentGrant(kind: .network, value: "*", tier: .consequential, note: "") }104        await #expect(throws: GrantRefusal.self) { try await p.addPersistentGrant(kind: .processExec, value: "sudo", tier: .consequential, note: "") }105        await #expect(throws: GrantRefusal.self) { try await p.addPersistentGrant(kind: .fileRead, value: folder.path + "/missing", tier: .readOnly, note: "") }106        #expect(p.grants.count == 2)107108        p.search = "example"109        #expect(p.visible.count == 1)110        p.search = ""111        await p.revoke(g.id)112        #expect(p.visible.count == 1)                  // active only by default113        p.activeOnly = false114        #expect(p.visible.count == 2)115        #expect(PermissionsModel.state(try #require(p.grants.first { $0.id == g.id })) == .revoked)116        await p.revokeAll()117        #expect(p.activeCount == 0)118        #expect(p.audit.contains { $0.kind == .grantRevoked } && p.audit.contains { $0.kind == .grantCreated })119        #expect(await model.services.policy.activeGrants().isEmpty)120    }121122    @Test func statusIsNeverInventedAndToolsAreMappedToPermissions() async throws {123        let model = try await PreviewEnvironment.make(seed: false)124        let p = PermissionsModel(services: model.services)125        await p.reload()126        #expect(p.snapshot.count == MacPermission.allCases.count)127        #expect(p.snapshot[.accessibility] == MacPermissions.status(.accessibility))128        #expect(p.snapshot[.appleEvents] == .unknown)129        #expect(p.toolsByPermission[.accessibility]?.contains("accessibility.press") == true)130        #expect(PermissionsModel.canRequest(.fullDiskAccess) == false && PermissionsModel.canRequest(.appleEvents) == false)131        #expect(ManualGrantKind.allCases.allSatisfy { $0.defaultTier <= .consequential })132    }133}134135@Suite("Unit · tool grant tree") struct ToolGrantTreeTests {136    let all = ["filesystem.list", "filesystem.read", "filesystem.write", "git.status", "git.push", "http.request"]137138    @Test func triStateFollowsPatterns() {139        #expect(ToolGrantTree.state(family: "filesystem", tools: Array(all[0...2]), patterns: ["filesystem.read"]) == .mixed)140        #expect(ToolGrantTree.state(family: "filesystem", tools: Array(all[0...2]), patterns: ["filesystem.*"]) == .on)141        #expect(ToolGrantTree.state(family: "git", tools: ["git.status", "git.push"], patterns: ["filesystem.*"]) == .off)142        #expect(ToolGrantTree.state(family: "git", tools: ["git.status", "git.push"], patterns: ["*"]) == .on)143        #expect(ToolGrantTree.families(all).map(\.family) == ["filesystem", "git", "http"])144    }145146    @Test func togglingMapsToFamilyWildcardsAndIndividualIdentifiers() {147        var p = ToolGrantTree.setFamily("filesystem", on: true, patterns: ["filesystem.read", "git.status"], all: all)148        #expect(p == ["filesystem.*", "git.status"])149        p = ToolGrantTree.setTool("filesystem.write", on: false, patterns: p, all: all)150        #expect(p == ["filesystem.list", "filesystem.read", "git.status"])151        p = ToolGrantTree.setTool("git.push", on: true, patterns: p, all: all)152        #expect(p.contains("git.push") && !p.contains("git.*"))153        p = ToolGrantTree.setFamily("git", on: false, patterns: p, all: all)154        #expect(p == ["filesystem.list", "filesystem.read"])155        // "*" expands to one wildcard per family before a single change.156        p = ToolGrantTree.setTool("http.request", on: false, patterns: ["*"], all: all)157        #expect(p == ["filesystem.*", "git.*"])158        #expect(ToolGrantTree.setEverything(true, patterns: p, all: all) == ["*"])159        #expect(ToolGrantTree.setEverything(false, patterns: ["*"], all: all) == ["filesystem.*", "git.*", "http.*"])160        #expect(ToolGrantTree.normalized(["git.status", "git.*", "git.status", ""]) == ["git.*"])161        // The mapping agrees with the runtime's own matching.162        let profile = AgentProfile(name: "x", instructions: "y", model: ModelRef(provider: .anthropic, model: ""), toolPatterns: ["filesystem.*", "git.status"])163        for id in all { #expect(profile.allows(toolID: id) == ToolGrantTree.allows(id, profile.toolPatterns)) }164    }165}166167@Suite("Integration · agents") @MainActor struct AgentsModelTests {168    @Test func createValidateSaveDuplicateDelete() async throws {169        let model = try await PreviewEnvironment.make(seed: false)170        let a = AgentsModel(services: model.services)171        await a.reload()172        #expect(a.profiles.count == BuiltInAgents.all.count && a.profiles.allSatisfy(\.builtIn))173        #expect(a.toolIDs.contains("filesystem.read"))174        #expect(await a.save() == false)                // a built-in is selected: read-only175176        a.newProfile()177        #expect(a.isDirty && !a.problems.isEmpty)        // instructions are missing178        a.draft?.name = "general"179        a.draft?.instructions = "Summarise."180        #expect(a.problems.count == 1)                   // the name is taken181        a.draft?.name = "Bad Name"182        #expect(!a.problems.isEmpty)183        a.draft?.name = "summary-writer"184        let before = a.draft?.toolPatterns ?? []185        a.draft?.toolPatterns = ToolGrantTree.setFamily("documents", on: true, patterns: before, all: a.toolIDs)186        a.draft?.budget.maxTurns = 100_000187        #expect(a.problems.isEmpty)188        #expect(await a.save())189        let stored = try #require(await model.services.agents.profile(named: "summary-writer"))190        #expect(!stored.builtIn && stored.toolPatterns.contains("documents.*") && stored.budget.maxTurns == 200)191        #expect(stored.model.model.isEmpty)              // "use the default model"192        #expect(!a.isDirty)193194        let copy = try #require(await a.duplicate(BuiltInAgents.all[1].id))195        #expect(copy.name == BuiltInAgents.all[1].name + "-copy" && !copy.builtIn && a.selectedID == copy.id)196        await a.delete(copy.id)197        #expect(await model.services.agents.profile(named: copy.name) == nil)198        await a.delete(stored.id)199        #expect(a.profiles.count == BuiltInAgents.all.count)200    }201202    @Test func evaluationRunsRealReviewModeRunsAndStoresResults() async throws {203        let model = try await PreviewEnvironment.make(seed: false)204        await model.services.providers.register(CannedProvider())205        model.settings.defaultModel = ModelRef(provider: ProviderID(rawValue: "canned"), model: "canned-1")206        try await model.services.runs.save(model.settings)207        let folder = try tempFolder("eval"); defer { try? FileManager.default.removeItem(at: folder) }208        try "hello".write(to: folder.appendingPathComponent("present.txt"), atomically: true, encoding: .utf8)209210        let a = AgentsModel(services: model.services)211        await a.reload()212        a.newProfile()213        a.draft?.name = "eval-agent"; a.draft?.instructions = "Inspect only."; a.draft?.toolPatterns = ["filesystem.list"]214        #expect(await a.save())215        await a.addCase(AgentEvalCase(objective: "Confirm present.txt exists", workspace: folder.path, expectations: [Expectation(kind: "file_exists", path: folder.path + "/present.txt")]))216        await a.addCase(AgentEvalCase(objective: "Confirm missing.txt exists", workspace: folder.path, expectations: [Expectation(kind: "file_exists", path: folder.path + "/missing.txt")]))217        await a.runEvaluation(defaultWorkspace: folder.path)218219        let agentID = try #require(a.selectedID)220        let stored = try #require(try await model.services.store.get(AgentEvalProfile.self, collection: AgentEvalProfile.collection, id: agentID))221        #expect(stored.cases.count == 2 && stored.results.count == 2)222        for r in stored.results {223            let runID = try #require(r.runID)224            let run = try #require(try await model.services.store.run(id: runID))225            #expect(run.mode == .review && run.status.isTerminal && run.status.rawValue == r.status)226        }227        // The missing file can never pass: the check is made by the runtime, not by what the model says.228        #expect(stored.results[1].passed == false)229        #expect(stored.results[0].checksTotal == 1)230        await a.removeCase(stored.cases[1].id)231        #expect(a.evaluation?.cases.count == 1 && a.evaluation?.results.count == 1)232    }233}234235@Suite("Integration · settings, storage, memory") @MainActor struct SettingsStorageTests {236    @Test func settingsPersistThroughTheModel() async throws {237        let model = try await PreviewEnvironment.make(seed: false)238        model.setting(\.defaultMode).wrappedValue = .review239        model.setting(\.appearance).wrappedValue = "dark"240        model.setting(\.retentionDays).wrappedValue = 14241        model.setting(\.allowScreenContentToProvider).wrappedValue = true242        model.settings.plannerModel = ModelRef(provider: .openai, model: "planner-x")243        model.settings.defaultBudget = BudgetBounds.clamped(Budget(maxTurns: 7))244        model.settings.onboardingCompleted = true245        model.saveSettings()246        var stored = AppSettings()247        for _ in 0..<50 {248            stored = await model.services.runs.settings()249            if stored.plannerModel != nil && stored.onboardingCompleted { break }250            try await Task.sleep(nanoseconds: 40_000_000)251        }252        #expect(stored.defaultMode == .review && stored.appearance == "dark" && stored.retentionDays == 14)253        #expect(stored.allowScreenContentToProvider && stored.plannerModel?.model == "planner-x" && stored.defaultBudget.maxTurns == 7)254        #expect(stored.onboardingCompleted)255        #expect(AppSettings().allowScreenContentToProvider == false && AppSettings().backgroundAgentEnabled == false)256    }257258    @Test func backupExportImportRoundTripAndPrune() async throws {259        let model = try await PreviewEnvironment.make()260        let storage = StorageModel(services: model.services)261        await storage.reload()262        #expect((storage.databaseBytes ?? 0) > 0 && storage.artifactsBytes != nil)263        let out = try tempFolder("storage"); defer { try? FileManager.default.removeItem(at: out) }264265        let backup = out.appendingPathComponent("backup.sqlite")266        #expect(await storage.backup(to: backup))267        #expect(((try FileManager.default.attributesOfItem(atPath: backup.path)[.size] as? Int) ?? 0) > 10_000)268        #expect(await storage.backup(to: backup) == false)        // never overwrites269        #expect(storage.message?.kind == .error)270271        let bundle = out.appendingPathComponent("export.json")272        let exported = try #require(await storage.exportBundle(to: bundle))273        #expect(exported.workflows == 3)274        let text = try String(contentsOf: bundle, encoding: .utf8)275        #expect(!text.contains("Sample grant"))                    // grants are not exported276        #expect(StorageModel.summaryLines(exported).count == 4)277278        let other = try await PreviewEnvironment.make(seed: false)279        #expect(await other.services.workflows.list().isEmpty)280        let imported = try #require(await StorageModel(services: other.services).importBundle(from: bundle))281        #expect(imported.workflows == 3)282        #expect(Set(await other.services.workflows.list().map(\.id)) == Set(await model.services.workflows.list().map(\.id)))283        #expect(await other.services.policy.grants().isEmpty)284285        #expect(await StorageModel(services: other.services).importBundle(from: backup) == nil)   // not a bundle: precise failure, nothing imported286287        #expect(await storage.pruneRuns(olderThanDays: 3650) == 0)288        #expect(await storage.pruneRuns(olderThanDays: 0) == 6)289        #expect(try await model.services.store.runs(limit: 50).isEmpty)290    }291292    @Test func auditVerifyReportsValidThenTheFirstBrokenLine() async throws {293        let model = try await PreviewEnvironment.make()294        let storage = StorageModel(services: model.services)295        await storage.reload()296        #expect((storage.auditCount ?? 0) >= 2 && storage.auditEntries.count == storage.auditCount)297        await storage.verifyAudit()298        #expect(storage.verification?.isValid == true)299        let out = try tempFolder("audit"); defer { try? FileManager.default.removeItem(at: out) }300        #expect(await storage.exportAudit(to: out.appendingPathComponent("audit.jsonl")))301        #expect(try String(contentsOf: out.appendingPathComponent("audit.jsonl"), encoding: .utf8).split(separator: "\n").count == storage.auditCount)302303        // Tamper with the first line on disk: verification names it.304        let url = model.services.paths.auditLog305        var lines = try String(contentsOf: url, encoding: .utf8).components(separatedBy: "\n")306        lines[0] = lines[0].replacingOccurrences(of: "Grant", with: "Grunt")307        try lines.joined(separator: "\n").write(to: url, atomically: false, encoding: .utf8)308        await storage.verifyAudit()309        #expect(storage.verification?.isValid == false)310        #expect(storage.verification?.firstBrokenLine == 1)311    }312313    @Test func memoryApproveExportDelete() async throws {314        let model = try await PreviewEnvironment.make(seed: false)315        let memory = MemoryModel(services: model.services)316        await model.services.memory.save(MemoryRecord(id: "m1", scope: .user, kind: "preference", text: "Reports in French", provenance: "run r1"))317        await model.services.memory.save(MemoryRecord(id: "m2", scope: .agent("general"), kind: "fix", text: "Use absolute paths", provenance: "run r2"))318        await memory.reload()319        #expect(memory.records.count == 2)320        #expect(memory.records.filter(MemoryModel.isPending).map(\.id) == ["m1"])321        await memory.approve("m1")322        #expect(memory.records.filter(MemoryModel.isPending).isEmpty)323        let json = try JSONValue.parse(try memory.exportJSON())324        #expect(json.arrayValue?.count == 2 && json[0]?["provenance"] != nil)325        await memory.delete("m2")326        #expect(memory.records.map(\.id) == ["m1"])327        await memory.deleteAll()328        #expect(await model.services.memory.all().isEmpty)329    }330331    @Test func diagnosticsContainNoSecretAndOnlyShortPaths() async throws {332        let model = try await PreviewEnvironment.make(seed: false)333        let secret = "sk-test-diagnostics-\(UUID().uuidString)"334        Redactor.shared.register(secret: secret); defer { Redactor.shared.unregister(secret: secret) }335        let report = await DiagnosticsReport.make(model)336        #expect(report.toolTotal > 40 && report.toolsByFamily.contains { $0.family == "filesystem" })337        #expect(report.toolsByFamily.map(\.count).reduce(0, +) == report.toolTotal)338        #expect(report.permissions.count == MacPermission.allCases.count)339        #expect(!report.text.contains(secret) && !report.text.contains("sk-"))340        #expect(report.dataFolder == PathResolver.display(model.services.paths.root.path))341    }342}343344@Suite("Integration · skills") @MainActor struct SkillsModelTests {345    @Test func formBuildsAManifestOrExplainsTheProblem() throws {346        let built = try SkillForm.wordCountTemplate.build()347        #expect(built.manifest.id == "skill.wordCount" && built.manifest.tests.count == 1 && built.sources.keys.sorted() == ["main.py"])348        #expect(built.manifest.permissions.fileRead == ["$input.path"] && built.manifest.tier == .readOnly)349        var bad = SkillForm.wordCountTemplate350        bad.inputSchemaJSON = "{ not json"351        #expect(throws: SkillForm.Problem.self) { try bad.build() }352        bad = .wordCountTemplate; bad.fileRead = "~/.ssh"353        #expect(throws: SkillForm.Problem.self) { try bad.build() }354        bad = .wordCountTemplate; bad.version = "one"355        #expect(throws: SkillForm.Problem.self) { try bad.build() }356        #expect(throws: SkillForm.Problem.self) { try SkillForm().build() }357        #expect(SkillsModel.permissionLines(built.manifest).count == 3)358    }359360    @Test func draftInstallNewVersionRollbackTestUninstall() async throws {361        let model = try await PreviewEnvironment.make(seed: false)362        let skills = SkillsModel(services: model.services)363        let draft = try #require(await skills.propose(.wordCountTemplate))364        guard draft.installable else {365            print("SKIPPED(the word-count skill's own test did not pass here, python3 is probably unavailable: \(draft.tests.map(\.detail)))")366            return367        }368        #expect(skills.drafts.count == 1 && skills.installed.isEmpty)369        #expect(await model.services.registry.tool(id: "skill.wordCount") == nil)      // a draft cannot be called370        let installed = try #require(await skills.install(draftID: draft.id))371        #expect(installed.provenance?.source == "user" && installed.provenance?.testsPassed == 1)372        #expect(skills.drafts.isEmpty && skills.installed.map(\.id) == ["skill.wordCount"])373        #expect(await model.services.registry.tool(id: "skill.wordCount") != nil)      // the registry was refreshed374        #expect(await model.services.audit.entries(limit: 50).contains { $0.kind == .skillInstalled })375376        var v2 = SkillForm.wordCountTemplate377        v2.version = "1.1.0"; v2.source += "\n# second version\n"378        let d2 = try #require(await skills.propose(v2))379        _ = try #require(await skills.install(draftID: d2.id))380        #expect(skills.installed.first?.package.manifest.version == "1.1.0" && skills.installed.first?.versions == ["1.0.0", "1.1.0"])381        await skills.rollback("skill.wordCount")382        #expect(skills.installed.first?.package.manifest.version == "1.0.0")383        await skills.activate("skill.wordCount", version: "1.1.0")384        #expect(skills.installed.first?.package.manifest.version == "1.1.0")385386        await skills.test(try #require(skills.installed.first))387        #expect(skills.testResults["skill.wordCount"]?.allSatisfy(\.passed) == true)388389        await skills.setEnabled(false, id: "skill.wordCount")390        #expect(await model.services.registry.tool(id: "skill.wordCount") == nil)391392        // A draft with a blocker is never installed.393        var bad = SkillForm.wordCountTemplate394        bad.name = "Fetch"; bad.source = "import json, urllib.request\nprint(json.dumps({'words': 1, 'lines': 1}))\n"395        let blocked = try #require(await skills.propose(bad))396        #expect(!blocked.installable)397        #expect(await skills.install(draftID: blocked.id) == nil)398        #expect(skills.message?.kind == .error)399        await skills.discard(draftID: blocked.id)400        #expect(skills.drafts.isEmpty)401402        // Import always lands in drafts.403        let first = try #require(skills.installed.first)404        let imported = try #require(await skills.importFolder(first.package.directory))405        #expect(imported.source.hasPrefix("import:") && skills.drafts.count == 1)406407        await skills.uninstall("skill.wordCount")408        #expect(skills.installed.isEmpty)409        #expect(skills.catalog(search: "filesystem.move").first?.tools.map(\.identifier) == ["filesystem.move"])410    }411}412
Files main
Menu

Theme