SPB Git

Repository Public License

RecordingAndRenderTests.swift 16.4 KB 276 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXFeatureManage/Tests/ORIXFeatureManageTests/RecordingAndRenderTests.swift10// Description      : Conversion of synthetic recordings to workflows validated11//                    against the real tool manifests, the Watch & Learn model,12//                    onboarding completion, and one render test per screen.13// Module           : ORIXFeatureManageTests14// Repository       : https://git.spboucher.ai/orix-automation15// Created          : 2026-11-2116// Last Modified    : 2026-11-2117//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author           : Simon-Pierre Boucher23// Position         : Professor of Finance24// Institution      : Université du Québec en Outaouais (UQO)25// Department       : Département des sciences administratives26// Organization     : SP Boucher27//28// Website          : https://spboucher.ai29// Git              : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General          : info@spboucher.ai36// Professional     : contact@spboucher.ai37// Direct           : simon@spboucher.ai38// Research         : research@spboucher.ai39// Development      : dev@spboucher.ai40// Support          : support@spboucher.ai41// Security/Admin   : admin@spboucher.ai42// Notifications    : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification   : Public69// Distribution     : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License          : Proprietary, all rights reserved (see license file)83// License File     : LICENSE84// Maintainer       : Simon-Pierre Boucher85// Technical Contact: dev@spboucher.ai86//87// =============================================================================88// END OF SP BOUCHER SOURCE HEADER89// =============================================================================9091import Foundation92import SwiftUI93import Testing94@testable import ORIXFeatureManage95import ORIXAppUI9697func syntheticRecording() -> [RecordedAction] {98    let app = "com.example.Editor"99    return [100        RecordedAction(sequence: 1, kind: .recordingStarted, summary: "Recording started"),101        RecordedAction(sequence: 2, kind: .limitation, summary: "Button clicks have no Accessibility notification"),102        RecordedAction(sequence: 3, kind: .appLaunched, bundleID: app, appName: "Editor", summary: "Launched Editor"),103        RecordedAction(sequence: 4, kind: .appActivated, bundleID: app, appName: "Editor", summary: "Switched to Editor"),104        RecordedAction(sequence: 5, kind: .menuItemSelected, bundleID: app, appName: "Editor", role: "AXMenuItem", title: "New Document", summary: "chose menu item"),105        RecordedAction(sequence: 6, kind: .windowCreated, bundleID: app, appName: "Editor", role: "AXWindow", title: "Untitled", summary: "window opened"),106        RecordedAction(sequence: 7, kind: .focusChanged, bundleID: app, appName: "Editor", role: "AXTextField", title: "Title", identifier: "title-field", summary: "focused"),107        RecordedAction(sequence: 8, kind: .valueChanged, bundleID: app, appName: "Editor", role: "AXTextField", title: "Title", summary: "typed text of length 9 in field \"Title\"", coalesced: 9),108        RecordedAction(sequence: 9, kind: .valueChanged, bundleID: app, appName: "Editor", role: "AXCheckBox", title: "Shared", summary: "set AXCheckBox to true", value: true),109        RecordedAction(sequence: 10, kind: .valueChanged, bundleID: app, appName: "Editor", role: "AXTextField", title: "Author", summary: "typed \"Ada\"", value: "Ada"),110        RecordedAction(sequence: 11, kind: .valueChanged, bundleID: app, appName: "Editor", role: "AXSecureTextField", title: "Password", summary: "edited secure field"),111        RecordedAction(sequence: 12, kind: .focusChanged, bundleID: app, appName: "Editor", role: "AXGroup", summary: "focused AXGroup"),112        RecordedAction(sequence: 13, kind: .selectionChanged, bundleID: app, appName: "Editor", role: "AXOutline", title: "Files", summary: "selection changed"),113        RecordedAction(sequence: 14, kind: .appTerminated, bundleID: app, appName: "Editor", summary: "Quit Editor"),114        RecordedAction(sequence: 15, kind: .recordingStopped, summary: "Recording stopped"),115    ]116}117118func toolSteps(_ doc: WorkflowDocument) -> [(id: String, tool: String, args: [String: JSONValue])] {119    doc.spec.steps.compactMap { s in if case .tool(let t) = s.kind { return (s.id, t.tool, t.arguments) }; return nil }120}121122@Suite("Unit · recording to workflow") struct RecordingToWorkflowTests {123    @Test func actionsBecomeSemanticStepsWithPostconditions() {124        let reviewed = syntheticRecording().map { ReviewedAction(action: $0, variable: $0.sequence == 10 ? "Author Name" : nil) }125        let p = RecordingToWorkflow.convert(reviewed, name: "Create a document")126        let steps = toolSteps(p.document)127        #expect(p.document.metadata.id == "create-a-document")128        // Launch + activation collapse into one step; quit maps to quitApp.129        #expect(steps.filter { $0.tool == "system.launchApp" }.count == 1 && steps.first?.tool == "system.launchApp")130        #expect(steps.last?.tool == "system.quitApp")131        // Menu → find + press; focus → find + focus + waitFor; values → find + setValue + waitFor.132        #expect(steps.filter { $0.tool == "accessibility.press" }.count == 1)133        #expect(steps.filter { $0.tool == "accessibility.focus" }.count == 1)134        #expect(steps.filter { $0.tool == "accessibility.setValue" }.count == 3)135        #expect(steps.filter { $0.tool == "accessibility.waitFor" }.count == 1 + 1 + 3)136        // Every action step addresses the element found by role/title, never a coordinate.137        for s in steps where ["accessibility.press", "accessibility.focus", "accessibility.setValue"].contains(s.tool) {138            #expect(s.args["element"]?.stringValue?.hasPrefix("${{ steps.find") == true)139        }140        for s in steps { #expect(Set(s.args.keys).isDisjoint(with: ["x", "y", "point", "position", "frame", "coordinates"])) }141        for s in steps where s.tool == "accessibility.find" { #expect(s.args["role"] != nil && (s.args["title"] != nil || s.args["identifier"] != nil)) }142        // The chain is linear.143        for (i, s) in p.document.spec.steps.enumerated() { #expect(s.needs == (i == 0 ? [] : [p.document.spec.steps[i - 1].id])) }144        // Unrecorded text → required parameter; labelled value → named parameter with the recorded default; plain value kept.145        let params = p.document.spec.parameters146        #expect(params.map(\.name) == ["title", "authorName"])147        #expect(params[0].required && params[0].defaultValue == nil)148        #expect(params[1].defaultValue == "Ada" && !params[1].required)149        #expect(steps.contains { $0.tool == "accessibility.setValue" && $0.args["value"] == true })150        #expect(steps.contains { $0.tool == "accessibility.setValue" && $0.args["value"] == "${{ params.authorName }}" })151        // Secure fields, unlabelled elements and selection changes are left out, with a reason.152        #expect(!p.yaml.contains("Password") && !p.yaml.contains("AXSecureTextField"))153        #expect(p.notes.count == 4)154        #expect(p.convertedActions == 8)155        // Permissions: only the recorded application, nothing else.156        let perms = p.document.spec.permissions157        #expect(perms.apps.accessibility == ["com.example.Editor"] && perms.apps.control == ["com.example.Editor"] && perms.apps.automation.isEmpty)158        var onlyApps = WorkflowPermissions(); onlyApps.apps = perms.apps159        #expect(perms == onlyApps)160        #expect(Set(p.capabilities) == [.accessibility(bundleID: "com.example.Editor"), .appControl(bundleID: "com.example.Editor")])161    }162163    @Test func redactionDropsValuesAndTitles() {164        let actions = syntheticRecording()165        let reviewed = actions.map { ReviewedAction(action: $0, redacted: $0.sequence == 10 || $0.sequence == 6) }166        let p = RecordingToWorkflow.convert(reviewed, name: "")167        #expect(!p.yaml.contains("Ada") && !p.yaml.contains("Untitled"))168        #expect(p.document.metadata.name == "Recorded workflow")169        #expect(p.notes.contains { $0.contains("Action 6") } && p.notes.contains { $0.contains("Action 10") })170        let empty = RecordingToWorkflow.convert([], name: "x")171        #expect(empty.document.spec.steps.isEmpty && empty.notes.count == 1 && empty.capabilities.isEmpty)172    }173174    @Test func parameterNamesAreValidIdentifiers() {175        #expect(RecordingToWorkflow.parameterName("Author Name") == "authorName")176        #expect(RecordingToWorkflow.parameterName("  2nd e-mail! ") == "v2ndEMail")177        #expect(RecordingToWorkflow.parameterName("§§") == "value")178        #expect(RecordingToWorkflow.parameterName("Word count") == "wordCount")179    }180}181182@Suite("Integration · watch and learn, onboarding") @MainActor struct WatchAndOnboardingTests {183    @Test func proposedWorkflowValidatesAgainstRealManifestsAndSavesOnlyOnApproval() async throws {184        let model = try await PreviewEnvironment.make(seed: false)185        let reviewed = syntheticRecording().map { ReviewedAction(action: $0) }186        let p = RecordingToWorkflow.convert(reviewed, name: "Recorded editor task")187        let result = await model.services.workflows.diagnose(p.yaml)188        let errors = result.diagnostics.filter { $0.severity == .error }189        #expect(errors.isEmpty, "\(errors)")190        #expect(result.document == p.document)                     // YAML round trip keeps the document191192        let w = WatchModel(services: model.services)193        w.beginReview(syntheticRecording(), sample: true)194        #expect(w.phase == .review && w.isSample)195        #expect(w.items.count == 12)                                // markers and limitations are not review items196        w.remove(try #require(w.items.first { $0.action.kind == .selectionChanged }).id)197        w.workflowName = "Recorded editor task"198        await w.propose()199        #expect(w.phase == .proposal && !w.hasErrors)200        #expect(await model.services.workflows.list().isEmpty)     // nothing is written before approval201        await w.save()202        let id = try #require(w.savedWorkflowID)203        #expect(await model.services.workflows.list().map(\.id) == [id])204        let review = try await model.services.workflows.permissionReview(workflowID: id, parameters: ["title": "Report"])205        #expect(!review.accepted && review.proposed.count == 2)    // saved, not granted206        w.discard()207        #expect(w.phase == .idle && w.items.isEmpty && w.actions.isEmpty && !RecordingStatus.shared.isRecording)208    }209210    @Test func recorderRefusesToStartWithoutAnApplication() async throws {211        let model = try await PreviewEnvironment.make(seed: false)212        let w = WatchModel(services: model.services)213        w.refreshApps()214        #expect(w.selected.isEmpty)                                 // nothing is preselected215        w.start()216        #expect(w.phase == .idle && w.error != nil && !RecordingStatus.shared.isRecording)217    }218219    @Test func onboardingFinishMarksCompletion() async throws {220        let model = try await PreviewEnvironment.make(seed: false)221        model.settings.onboardingCompleted = false222        let out = FileManager.default.temporaryDirectory.appendingPathComponent("orix-manage-onboarding-\(UUID().uuidString.prefix(6)).png")223        defer { try? FileManager.default.removeItem(at: out) }224        for step in 0..<5 {225            try await ShotRenderer.render(OnboardingView(step: step) {}.environment(model), size: CGSize(width: 1000, height: 720), settle: 0.5, to: out)226            #expect(((try FileManager.default.attributesOfItem(atPath: out.path)[.size] as? Int) ?? 0) > 20_000)227        }228        #expect(model.settings.onboardingCompleted == false)        // rendering never completes onboarding by itself229    }230}231232@Suite("Render · every screen") @MainActor struct RenderTests {233    func render<V: View>(_ name: String, _ view: V, _ model: AppModel, size: CGSize = CGSize(width: 1100, height: 760)) async throws {234        let out = FileManager.default.temporaryDirectory.appendingPathComponent("orix-manage-test-\(name)-\(UUID().uuidString.prefix(6)).png")235        defer { try? FileManager.default.removeItem(at: out) }236        try await ShotRenderer.render(view.environment(model), size: size, settle: 0.7, to: out)237        let bytes = (try FileManager.default.attributesOfItem(atPath: out.path)[.size] as? Int) ?? 0238        #expect(bytes > 25_000, "\(name) rendered only \(bytes) bytes")239    }240241    @Test func settingsPanes() async throws {242        let model = try await PreviewEnvironment.make()243        for tab in SettingsTab.allCases { try await render("settings-\(tab.rawValue)", SettingsView(tab: tab), model) }244        try await render("settings-default", SettingsView(), model)245    }246247    @Test func permissionsTabs() async throws {248        let model = try await PreviewEnvironment.make()249        for tab in PermissionsTab.allCases { try await render("permissions-\(tab.rawValue)", PermissionsView(tab: tab, showAddForm: tab == .grants), model) }250        try await render("permissions-default", PermissionsView(), model)251    }252253    @Test func agentsTabs() async throws {254        let model = try await PreviewEnvironment.make()255        try await render("agents", AgentsView(), model)256        try await render("agents-editor", AgentsView(tab: .profiles, selectAgentNamed: "verifier", expandedFamilies: ["git"]), model, size: CGSize(width: 1100, height: 1200))257        try await render("agents-memory", AgentsView(tab: .memory), model)258    }259260    @Test func skillsTabs() async throws {261        let model = try await PreviewEnvironment.make()262        let built = try SkillForm.wordCountTemplate.build()263        let draft = try await model.services.skills.propose(manifest: built.manifest, sources: built.sources, source: "user")264        try await render("skills-default", SkillsView(), model)265        for tab in SkillsTab.allCases { try await render("skills-\(tab.rawValue)", SkillsView(tab: tab, selectTool: "git.status"), model) }266        try await render("skills-install", InstallConfirmationPreview(draft: draft), model, size: CGSize(width: 560, height: 520))267    }268269    @Test func watchAndLearnPhases() async throws {270        let model = try await PreviewEnvironment.make()271        try await render("watch-idle", WatchAndLearnView(), model)272        try await render("watch-review", WatchAndLearnView(sampleRecording: syntheticRecording()), model)273        try await render("watch-proposal", WatchAndLearnView(sampleRecording: syntheticRecording(), showProposal: true), model, size: CGSize(width: 1100, height: 1000))274    }275}276
Files main
Menu

Theme