SPB Git

Repository Public License

IntegrationTests.swift 45.9 KB 734 lines swift

1// ORIX OS Automation2// File: Packages/ORIXAppleAutomation/Tests/ORIXAppleAutomationTests/IntegrationTests.swift3// Description: Integration tests against the real system: osascript, sdef, NSWorkspace, window server, Vision, pasteboard, Accessibility and ScreenCaptureKit (the last two gated on the actual TCC status, never prompting).4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import AppKit8import ApplicationServices9import CoreGraphics10import Foundation11import ORIXCore12import ORIXTestingSupport13import Testing14@testable import ORIXAppleAutomation1516@Suite("Integration · permissions (real TCC status, no prompt)")17struct PermissionIntegrationTests {18    @Test func everyPermissionHasAStatus() {19        var lines: [String] = []20        for p in MacPermission.allCases {21            let s = MacPermissions.status(p)22            #expect(PermissionStatus.allCases.contains(s))23            lines.append("\(p.rawValue)=\(s.rawValue)")24        }25        let finder = MacPermissions.automationStatus(bundleID: "com.apple.finder")26        lines.append("appleEvents(com.apple.finder)=\(finder.rawValue)")27        print("TCC status of the test process: " + lines.joined(separator: ", "))28        #expect(MacPermissions.status(.accessibility) == (AXIsProcessTrusted() ? .granted : .denied))29        #expect(MacPermissions.status(.screenRecording) == (CGPreflightScreenCaptureAccess() ? .granted : .denied))30        // `swift test` is not an application bundle: notifications cannot be evaluated, and that is what is reported.31        #expect(MacPermissions.hasAppBundle == false)32        #expect(MacPermissions.status(.notifications) == .unknown)33        #expect(MacPermissions.status(.appleEvents) == .unknown)34        #expect(MacPermissions.automationStatus(bundleID: "com.example.not-running-\(UUID().uuidString.prefix(6))") == .unknown)35        #expect(MacPermissions.snapshot().count == MacPermission.allCases.count)36    }3738    @Test func applicationResolutionUsesInstalledBundles() {39        #expect(AppResolver.bundleID(forName: "Finder") == "com.apple.finder")40        #expect(AppResolver.bundleID(forReference: "System Events") == "com.apple.systemevents")41        #expect(AppResolver.bundleID(forReference: "/System/Library/CoreServices/Finder.app") == "com.apple.finder")42        #expect(AppResolver.bundleID(forReference: "com.apple.finder") == "com.apple.finder")43        #expect(AppResolver.bundleID(forReference: "NoSuchApp-\(UUID().uuidString)") == nil)44        #expect(AppResolver.canonicalBundleID("COM.APPLE.FINDER") == "com.apple.finder")45    }46}4748@Suite("Integration · applescript.run / jxa.run (real osascript)")49struct ScriptIntegrationTests {50    let apple = ScriptRunTool(language: .applescript)51    let jxa = ScriptRunTool(language: .jxa)5253    @Test func appleScriptArithmeticStringsListsAndDates() async throws {54        let dir = try TempDirectory(); defer { dir.remove() }55        var o = await invoke(apple, ["script": "(6 * 7) + 0.5"], in: dir)56        #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")57        #expect(o.result.output["value"] == .number(42.5))58        #expect(o.result.output["parsed"] == true)59        #expect(o.result.untrusted)6061        o = await invoke(apple, ["script": "set s to \"orix\" & \" \" & \"automation\"\nreturn (count of s) as string & \":\" & s"], in: dir)62        #expect(o.result.output["value"] == "15:orix automation")63        #expect(o.result.output["result"] == "\"15:orix automation\"")6465        o = await invoke(apple, ["script": "set out to {}\nrepeat with i from 1 to 3\n set end of out to i * i\nend repeat\nreturn {squares:out, ok:true, label:\"é\"}"], in: dir)66        #expect(o.result.output["value"] == ["squares": [1, 4, 9], "ok": true, "label": "é"])6768        o = await invoke(apple, ["script": "return (year of (current date)) >= 2026"], in: dir)69        #expect(o.result.output["value"] == true)70        let year = Calendar.current.component(.year, from: Date())71        o = await invoke(apple, ["script": "year of ((current date) + 0)"], in: dir)72        #expect(o.result.output["value"] == .int(year))7374        // A date is not representable as JSON: the text is returned and the tool says it was not parsed.75        o = await invoke(apple, ["script": "current date"], in: dir)76        #expect(o.result.status == .succeeded)77        #expect(o.result.output["parsed"] == false)78        #expect(o.result.output["result"]?.stringValue?.hasPrefix("date ") == true)79    }8081    @Test func appleScriptSyntaxAndRuntimeErrorsAreTyped() async throws {82        let dir = try TempDirectory(); defer { dir.remove() }83        var o = await invoke(apple, ["script": "set x to 1\nset y to (2 +\nreturn x"], in: dir)84        #expect(o.result.status == .failed)85        #expect(o.result.failure?.code == "syntax_error")86        #expect(o.result.output["line"]?.intValue == 2 || o.result.output["line"]?.intValue == 3, "line \(String(describing: o.result.output["line"]))")87        #expect(o.result.output["errorNumber"]?.intValue == -2741)88        #expect(o.result.failure?.message.contains("line") == true)8990        o = await invoke(apple, ["script": "error \"boom\" number 4242"], in: dir)91        #expect(o.result.failure?.code == "script_error")92        #expect(o.result.output["errorNumber"] == 4242)93        #expect(o.result.failure?.message.contains("boom") == true)9495        o = await invoke(apple, ["script": "return 1 / 0"], in: dir)96        #expect(o.result.failure?.code == "script_error")97        #expect(o.result.output["errorNumber"] == -2701)98    }99100    @Test func scriptsAreStoppedAtTheTimeout() async throws {101        let dir = try TempDirectory(); defer { dir.remove() }102        let started = Date()103        var o = await invoke(apple, ["script": "repeat\nend repeat", "timeoutSeconds": 1.5], in: dir)104        #expect(o.result.status == .timedOut)105        #expect(o.error == .timeout(seconds: 1.5))106        o = await invoke(jxa, ["script": "while (true) {}", "timeoutSeconds": 1.5], in: dir)107        #expect(o.result.status == .timedOut)108        #expect(Date().timeIntervalSince(started) < 12)109    }110111    @Test func runningScriptIsTerminatedOnCancellation() async throws {112        let dir = try TempDirectory(); defer { dir.remove() }113        let tool = apple114        let task = Task { await invoke(tool, ["script": "repeat\nend repeat", "timeoutSeconds": 25], in: dir) }115        try await Task.sleep(nanoseconds: 700_000_000)116        let started = Date()117        task.cancel()118        let o = await task.value119        #expect(o.result.status == .canceled)120        #expect(Date().timeIntervalSince(started) < 5)121    }122123    @Test func jxaReturnsStructuredJSON() async throws {124        let dir = try TempDirectory(); defer { dir.remove() }125        var o = await invoke(jxa, ["script": "const xs = [1, 2, 3].map(x => x * x); ({sum: xs.reduce((a, b) => a + b, 0), xs: xs, name: 'orix'.toUpperCase()})"], in: dir)126        #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")127        #expect(o.result.output["value"] == ["sum": 14, "xs": [1, 4, 9], "name": "ORIX"])128        #expect(o.result.output["parsed"] == true)129130        o = await invoke(jxa, ["script": "new Date(86400000).toISOString()"], in: dir)131        #expect(o.result.output["value"] == "1970-01-02T00:00:00.000Z")132        #expect(o.result.output["result"] == "1970-01-02T00:00:00.000Z")133134        o = await invoke(jxa, ["script": "function run(argv) { return 6 * 7 }"], in: dir)135        #expect(o.result.output["value"] == 42)136137        o = await invoke(jxa, ["script": "var nothing; nothing"], in: dir)138        #expect(o.result.status == .succeeded)139        #expect(o.result.output["value"] == .null && o.result.output["parsed"] == false)140    }141142    @Test func jxaErrorsCarryLineInformation() async throws {143        let dir = try TempDirectory(); defer { dir.remove() }144        var o = await invoke(jxa, ["script": "var a = 1;\nvar b = (;\na"], in: dir)145        #expect(o.result.failure?.code == "syntax_error", "\(String(describing: o.result.failure))")146        #expect(o.result.output["line"] == 2)147148        // A SyntaxError raised while running (not while compiling) is a script error, not a syntax error.149        o = await invoke(jxa, ["script": "JSON.parse('{')"], in: dir)150        #expect(o.result.failure?.code == "script_error")151152        o = await invoke(jxa, ["script": "var a = 1;\n\nundefinedFunction(a)"], in: dir)153        #expect(o.result.failure?.code == "script_error")154        #expect(o.result.output["line"] == 3)155        #expect(o.result.failure?.message.contains("undefinedFunction") == true)156    }157158    @Test func undeclaredTargetNeverRuns() async throws {159        let dir = try TempDirectory(); defer { dir.remove() }160        let marker = dir.file("ran.txt")161        // If the refusal were not static, the shell command would create the marker before the Finder call.162        let script = "do shell script \"touch \" & quoted form of \"\(marker.path)\"\ntell application \"Finder\" to get name of startup disk"163        let o = await invoke(apple, ["script": .string(script), "targets": []], in: dir)164        #expect(o.result.status == .invalidInput)165        #expect(o.result.failure?.message.contains("Finder") == true)166        #expect(!FileManager.default.fileExists(atPath: marker.path))167168        let j = await invoke(jxa, ["script": "Application('Finder').startupDisk.name()", "targets": ["com.apple.mail"]], in: dir)169        #expect(j.result.status == .invalidInput)170    }171172    @Test func dryRunDoesNotExecute() async throws {173        let dir = try TempDirectory(); defer { dir.remove() }174        let marker = dir.file("ran.txt")175        let o = await invoke(apple, ["script": .string("do shell script \"touch \" & quoted form of \"\(marker.path)\"")], in: dir, dryRun: true)176        #expect(o.result.status == .simulated)177        #expect(!FileManager.default.fileExists(atPath: marker.path))178    }179180    @Test func shellEscapeRunsOnlyThroughTheDeclaredRuntime() async throws {181        let dir = try TempDirectory(); defer { dir.remove() }182        let o = await invoke(apple, ["script": "do shell script \"printf orix-shell\""], in: dir)183        #expect(o.requirements?.capabilities.contains(.scriptRuntime(name: "bash")) == true)184        #expect(o.requirements?.tier == .consequential)185        #expect(o.result.output["value"] == "orix-shell")186    }187188    @Test func finderScriptSucceedsOrReportsTheAppleEventsBlock() async throws {189        let dir = try TempDirectory(); defer { dir.remove() }190        let status = MacPermissions.automationStatus(bundleID: "com.apple.finder")191        if status == .notDetermined {192            // Running it would make macOS show the Automation consent dialog: tests never prompt.193            skipped("apple events consent for Finder is undetermined for this test process; running the script would show the system prompt")194            return195        }196        for (tool, script) in [(apple, "tell application \"Finder\" to get name of startup disk"), (jxa, "Application('Finder').startupDisk.name()")] {197            let o = await invoke(tool, ["script": .string(script), "targets": ["com.apple.finder"], "timeoutSeconds": 20], in: dir, timeout: 20)198            if o.result.status == .succeeded {199                let name = o.result.output["value"]?.stringValue ?? ""200                #expect(!name.isEmpty)201                #expect(o.result.output["targets"] == ["com.apple.finder"])202                verified("\(tool.identifier) talked to Finder: startup disk is \"\(name)\"")203            } else if case .tccBlocked(let permission, let message)? = o.error {204                #expect(permission == .appleEvents)205                #expect(message.contains("Finder"))206                #expect(o.result.status == .blockedPermission)207                skipped("apple events not authorized for this test process (\(tool.identifier))")208            } else {209                Issue.record("\(tool.identifier): expected success or tccBlocked(.appleEvents), got \(o.result.status) \(o.result.failure?.message ?? "")")210            }211        }212    }213}214215@Suite("Integration · applescript.dictionary (real sdef)")216struct DictionaryIntegrationTests {217    @Test func finderDictionaryIsSummarised() async throws {218        let dir = try TempDirectory(); defer { dir.remove() }219        let o = await invoke(ScriptingDictionaryTool(), ["bundleID": "com.apple.finder"], in: dir)220        #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")221        #expect(o.requirements?.capabilities == [.fileRead(path: "/System/Library/CoreServices/Finder.app")])222        let suites = o.result.output["suites"]?.arrayValue ?? []223        #expect(suites.count >= 3)224        #expect(suites.contains { $0["name"] == "Standard Suite" })225        let commands = suites.flatMap { $0["commands"]?.arrayValue ?? [] }226        let open = commands.first { $0["name"] == "open" }227        #expect(open != nil)228        #expect(open?["parameters"]?.arrayValue?.contains { $0["name"] == "using" } == true)229        let classes = suites.flatMap { $0["classes"]?.arrayValue ?? [] }230        let withProperties = classes.filter { !($0["properties"]?.arrayValue ?? []).isEmpty }231        #expect(!withProperties.isEmpty)232        #expect((o.result.output["commandCount"]?.intValue ?? 0) > 10)233        #expect(o.result.untrusted)234        #expect(o.result.output.serialized().utf8.count < 262_144)235    }236237    @Test func boundIsHonoured() async throws {238        let dir = try TempDirectory(); defer { dir.remove() }239        let o = await invoke(ScriptingDictionaryTool(), ["path": "/System/Library/CoreServices/Finder.app", "maxItems": 2], in: dir)240        #expect(o.result.status == .succeeded)241        for suite in o.result.output["suites"]?.arrayValue ?? [] {242            #expect((suite["commands"]?.arrayValue?.count ?? 0) <= 2)243            #expect((suite["classes"]?.arrayValue?.count ?? 0) <= 2)244        }245        #expect(o.result.truncated)246    }247248    @Test func appWithoutDictionaryIsReportedUnsupported() async throws {249        let dir = try TempDirectory(); defer { dir.remove() }250        let o = await invoke(ScriptingDictionaryTool(), ["path": "/System/Applications/Calculator.app"], in: dir)251        #expect(o.result.status == .unsupported, "\(o.result.status) \(o.result.failure?.message ?? "")")252        let missing = await invoke(ScriptingDictionaryTool(), ["bundleID": "com.example.absent.app"], in: dir)253        #expect(missing.result.failure?.code == "not_found")254    }255}256257// Serialized: one test launches Calculator while another asserts that a dry run launched nothing.258@Suite("Integration · system.* (real NSWorkspace and window server)", .serialized)259struct SystemIntegrationTests {260    @Test func runningApplicationsIncludeFinder() async throws {261        let dir = try TempDirectory(); defer { dir.remove() }262        let o = await invoke(SystemAppsTool(), ["includeBackground": true], in: dir)263        #expect(o.result.status == .succeeded)264        let apps = o.result.output["applications"]?.arrayValue ?? []265        let finder = apps.first { $0["bundleID"] == "com.apple.finder" }266        #expect(finder != nil)267        #expect((finder?["pid"]?.intValue ?? 0) > 0)268        #expect(finder?["name"]?.stringValue?.isEmpty == false)269        let regular = await invoke(SystemAppsTool(), [:], in: dir)270        #expect((regular.result.output["applications"]?.arrayValue?.count ?? 0) <= apps.count)271    }272273    @Test func frontmostDegradesHonestly() async throws {274        let dir = try TempDirectory(); defer { dir.remove() }275        let o = await invoke(SystemFrontmostTool(), [:], in: dir)276        #expect(o.result.status == .succeeded)277        let source = o.result.output["titleSource"]?.stringValue278        if AXIsProcessTrusted() { #expect(source == "accessibility") }279        else if CGPreflightScreenCaptureAccess() { #expect(source == "windowList") }280        else {281            #expect(source == "none")282            #expect(o.result.output["windowTitle"] == .null)283            #expect(o.result.output["note"]?.stringValue?.contains("Accessibility") == true)284        }285        print("system.frontmost: app=\(o.result.output["application"]?["name"]?.stringValue ?? "none") titleSource=\(source ?? "?")")286    }287288    @Test func windowsSayWhetherTitlesAreAvailable() async throws {289        let dir = try TempDirectory(); defer { dir.remove() }290        let o = await invoke(SystemWindowsTool(), ["includeAllLayers": true], in: dir)291        #expect(o.result.status == .succeeded)292        let titles = CGPreflightScreenCaptureAccess()293        #expect(o.result.output["titlesAvailable"] == .bool(titles))294        let windows = o.result.output["windows"]?.arrayValue ?? []295        if !titles {296            #expect(windows.allSatisfy { $0["title"] == nil })297            #expect(o.result.output["note"]?.stringValue?.contains("Screen Recording") == true)298        }299        for w in windows.prefix(20) {300            #expect((w["pid"]?.intValue ?? 0) > 0)301            #expect(w["bounds"]?["width"]?.doubleValue != nil)302        }303        #expect((o.result.output["displays"]?.arrayValue?.count ?? 0) >= (CGMainDisplayID() != 0 && !Displays.active().isEmpty ? 1 : 0))304        print("system.windows: \(windows.count) window(s), titlesAvailable=\(titles)")305        let filtered = await invoke(SystemWindowsTool(), ["bundleID": "com.apple.finder", "includeAllLayers": true], in: dir)306        #expect(filtered.result.output["windows"]?.arrayValue?.allSatisfy { $0["bundleID"] == "com.apple.finder" } == true)307    }308309    @Test func launchAndQuitDryRunsChangeNothing() async throws {310        let dir = try TempDirectory(); defer { dir.remove() }311        let before = AppResolver.running("com.apple.calculator").count312        let launch = await invoke(SystemLaunchAppTool(), ["bundleID": "com.apple.calculator"], in: dir, dryRun: true)313        #expect(launch.result.status == .simulated)314        let quit = await invoke(SystemQuitAppTool(), ["bundleID": "com.apple.finder"], in: dir, dryRun: true)315        #expect(quit.result.status == .simulated)316        #expect(quit.result.output["wasRunning"] == true)317        #expect(AppResolver.running("com.apple.calculator").count == before)318        #expect(!AppResolver.running("com.apple.finder").isEmpty)319        let absent = await invoke(SystemLaunchAppTool(), ["bundleID": "com.example.absent.app"], in: dir)320        #expect(absent.result.failure?.code == "not_found")321        let notRunning = await invoke(SystemQuitAppTool(), ["bundleID": "com.example.absent.app"], in: dir)322        #expect(notRunning.result.status == .succeeded && notRunning.result.output["wasRunning"] == false)323    }324325    @Test func launchWaitsForTheAppAndQuitIsPolite() async throws {326        let dir = try TempDirectory(); defer { dir.remove() }327        let id = "com.apple.calculator"328        guard AppResolver.url(forBundleID: id) != nil else { skipped("Calculator is not installed"); return }329        guard AppResolver.running(id).isEmpty else {330            skipped("Calculator is already running; the test will not quit an application the user opened")331            return332        }333        let launch = await invoke(SystemLaunchAppTool(), ["bundleID": .string(id), "activate": false, "timeoutSeconds": 20], in: dir)334        #expect(launch.result.status == .succeeded, "\(launch.result.failure?.message ?? "")")335        #expect(launch.result.output["alreadyRunning"] == false)336        #expect(launch.result.output["finishedLaunching"] == true)337        let pid = pid_t(launch.result.output["pid"]?.intValue ?? 0)338        #expect(pid > 0 && kill(pid, 0) == 0)339340        if AXIsProcessTrusted() { await pressRoundTrip(in: dir) }341        else { skipped("accessibility not granted to this test process: accessibility.press not exercised on a live application") }342343        let quit = await invoke(SystemQuitAppTool(), ["bundleID": .string(id), "timeoutSeconds": 15], in: dir)344        #expect(quit.result.status == .succeeded)345        #expect(quit.result.output["wasRunning"] == true)346        #expect(quit.result.output["terminated"] == true && quit.result.output["refused"] == false)347        #expect(kill(pid, 0) != 0)348        verified("system.launchApp started Calculator (pid \(pid)) in the background and system.quitApp ended it")349    }350}351352extension SystemIntegrationTests {353    /// find → press → observe → press back, on the Calculator this test launched in the background. The sidebar toggle is354    /// used because pressing it twice leaves the application exactly as it was (no calculation is entered or cleared).355    func pressRoundTrip(in dir: TempDirectory) async {356        let id = "com.apple.calculator"357        let ready = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "role": "AXToolbar", "timeoutSeconds": 10], in: dir)358        guard ready.result.output["satisfied"] == true, let toolbar = ready.result.output["element"]?["path"]?.stringValue else {359            skipped("this Calculator version exposes no toolbar through Accessibility: press round trip not exercised"); return360        }361        let button = toolbar + "/AXButton[0]"362        let tree = await invoke(AccessibilityTreeTool(), ["bundleID": .string(id), "element": .string(button), "maxDepth": 0], in: dir)363        let splitter = await invoke(AccessibilityFindTool(), ["bundleID": .string(id), "role": "AXSplitter", "maxResults": 1], in: dir)364        guard let label = tree.result.output["nodes"]?[0]?["description"]?.stringValue, !label.isEmpty,365              let splitterPath = splitter.result.output["matches"]?[0]?["path"]?.stringValue,366              let closedValue = splitter.result.output["matches"]?[0]?["value"]?.displayString else {367            skipped("this Calculator version has no labelled sidebar button or splitter: press round trip not exercised"); return368        }369        let dry = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(button), "expectedTitle": .string(label)], in: dir, dryRun: true)370        #expect(dry.result.status == .simulated)371        let untouched = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "element": .string(splitterPath), "condition": "valueEquals", "expected": .string(closedValue), "timeoutSeconds": 1], in: dir)372        #expect(untouched.result.output["satisfied"] == true)373374        let press = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(button), "expectedTitle": .string(label)], in: dir)375        #expect(press.result.status == .succeeded, "\(press.result.failure?.message ?? "")")376        #expect(press.result.output["pressed"] == true)377        // Observable postcondition: the button no longer carries its original label (it now offers the opposite action).378        let opened = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "role": "AXButton", "title": .string(label), "condition": "absent", "timeoutSeconds": 5], in: dir)379        #expect(opened.result.output["satisfied"] == true, "pressing the sidebar button changed nothing observable")380        // Same path, different meaning now: the expectedTitle guard refuses to press it.381        let guarded = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(button), "expectedTitle": .string(label)], in: dir)382        #expect(guarded.result.failure?.code == "element_changed")383        let back = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(button)], in: dir)384        #expect(back.result.status == .succeeded)385        let closed = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "role": "AXButton", "title": .string(label), "condition": "exists", "timeoutSeconds": 5], in: dir)386        #expect(closed.result.output["satisfied"] == true)387        let restored = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "element": .string(splitterPath), "condition": "valueEquals", "expected": .string(closedValue), "timeoutSeconds": 5], in: dir)388        #expect(restored.result.output["satisfied"] == true, "the sidebar did not return to its initial state")389        // A static text is not pressable, and its value cannot be set: both are honest refusals.390        let text = await invoke(AccessibilityFindTool(), ["bundleID": .string(id), "role": "AXStaticText", "maxResults": 1], in: dir)391        if let path = text.result.output["matches"]?[0]?["path"]?.stringValue {392            let p = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(path)], in: dir)393            #expect(p.result.status == .unsupported)394            let s = await invoke(AccessibilitySetValueTool(), ["bundleID": .string(id), "element": .string(path), "value": "1"], in: dir)395            #expect(s.result.status == .unsupported)396        }397        verified("accessibility.press toggled Calculator's sidebar and back in the background; waitFor observed both states and expectedTitle refused the changed button")398    }399}400401@Suite("Integration · screen.ocr / screen.compare (real Vision and CoreGraphics)")402struct ScreenFileIntegrationTests {403    @Test func ocrRecognisesRenderedWords() async throws {404        let dir = try TempDirectory(); defer { dir.remove() }405        let image = dir.file("words.png")406        try TestImages.text(["ORIX AUTOMATION", "Invoice 48213 approved"], to: image)407        let o = await invoke(ScreenOCRTool(), ["path": "words.png", "languages": ["en-US"]], in: dir, timeout: 90)408        #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")409        let text = o.result.output["text"]?.stringValue ?? ""410        for word in ["ORIX", "AUTOMATION", "Invoice", "48213", "approved"] { #expect(text.contains(word), "missing \(word) in \(text)") }411        #expect(o.result.untrusted)412        #expect(o.result.output["imageWidth"] == 1100 && o.result.output["imageHeight"] == 320)413        let blocks = o.result.output["blocks"]?.arrayValue ?? []414        #expect(blocks.count >= 2)415        // First line is above the second one (top-left origin), both inside the image.416        let first = blocks.first { $0["text"]?.stringValue?.contains("ORIX") == true }417        let second = blocks.first { $0["text"]?.stringValue?.contains("Invoice") == true }418        let y1 = first?["pixels"]?["y"]?.doubleValue ?? -1, y2 = second?["pixels"]?["y"]?.doubleValue ?? -1419        #expect(y1 >= 0 && y2 > y1 && y2 < 320)420        let n = first?["normalized"]421        #expect((n?["x"]?.doubleValue ?? -1) >= 0 && (n?["width"]?.doubleValue ?? 2) <= 1)422        #expect((first?["confidence"]?.doubleValue ?? 0) > 0.3)423    }424425    @Test func ocrRejectsMissingAndNonImageFiles() async throws {426        let dir = try TempDirectory(); defer { dir.remove() }427        let missing = await invoke(ScreenOCRTool(), ["path": "nope.png"], in: dir)428        #expect(missing.result.failure?.code == "not_found")429        try dir.write("text.png", "this is not an image")430        let bad = await invoke(ScreenOCRTool(), ["path": "text.png"], in: dir)431        #expect(bad.result.status == .invalidInput)432    }433434    @Test func compareFindsTheChangedRegion() async throws {435        let dir = try TempDirectory(); defer { dir.remove() }436        try TestImages.blocks(width: 200, height: 100, background: 1, rects: [], to: dir.file("a.png"))437        try TestImages.blocks(width: 200, height: 100, background: 1, rects: [(CGRect(x: 50, y: 20, width: 40, height: 30), 0)], to: dir.file("b.png"))438        try TestImages.blocks(width: 200, height: 100, background: 1, rects: [], to: dir.file("same.png"))439        try TestImages.blocks(width: 100, height: 100, background: 1, rects: [], to: dir.file("small.png"))440441        var o = await invoke(ScreenCompareTool(), ["before": "a.png", "after": "b.png", "diffPath": "out/diff.png"], in: dir)442        #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")443        #expect(o.result.output["identical"] == false)444        #expect(o.result.output["changedPixels"] == 1200)445        #expect(o.result.output["totalPixels"] == 20000)446        #expect(o.result.output["changedRatio"] == .number(0.06))447        #expect(o.result.output["boundingBox"] == ["x": .number(50), "y": .number(20), "width": .number(40), "height": .number(30)])448        let diff = dir.file("out/diff.png").path449        #expect(o.result.artifacts.first?.path == diff)450        #expect((o.result.artifacts.first?.bytes ?? 0) > 0 && o.result.artifacts.first?.sha256?.count == 64)451        // The annotated image is a real PNG of the same size, red where the change is.452        let rendered = try ImageIOHelper.rgba(try ImageIOHelper.load(diff))453        #expect(rendered.width == 200 && rendered.height == 100)454        let centre = (35 * 200 + 70) * 4455        #expect(rendered.pixels[centre] > 100 && rendered.pixels[centre + 1] < 40 && rendered.pixels[centre + 2] < 40)456457        o = await invoke(ScreenCompareTool(), ["before": "a.png", "after": "same.png"], in: dir)458        #expect(o.result.output["identical"] == true && o.result.output["changedPixels"] == 0)459        #expect(o.result.output["boundingBox"] == nil)460        #expect(o.requirements?.tier == .readOnly)461462        o = await invoke(ScreenCompareTool(), ["before": "a.png", "after": "small.png"], in: dir)463        #expect(o.result.failure?.code == "size_mismatch")464465        // An existing diff file is not replaced without overwrite.466        o = await invoke(ScreenCompareTool(), ["before": "a.png", "after": "b.png", "diffPath": "out/diff.png"], in: dir)467        #expect(o.result.failure?.code == "io")468    }469}470471@Suite("Integration · screen.capture (gated on Screen Recording)")472struct ScreenCaptureIntegrationTests {473    @Test func captureDisplayOrReportTheBlock() async throws {474        let dir = try TempDirectory(); defer { dir.remove() }475        let o = await invoke(ScreenCaptureTool(), ["scope": "display", "displayIndex": 0, "path": "shot.png"], in: dir)476        guard CGPreflightScreenCaptureAccess() else {477            #expect(o.error.map { if case .tccBlocked(.screenRecording, _) = $0 { return true } else { return false } } == true)478            #expect(o.result.status == .blockedPermission)479            #expect(!FileManager.default.fileExists(atPath: dir.file("shot.png").path))480            skipped("screen recording not granted to this test process: screen.capture returned tccBlocked(.screenRecording) as required")481            return482        }483        if case .permissionDenied? = o.error { skipped("secure keyboard input is active: capture correctly refused"); return }484        #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")485        let w = o.result.output["pixelWidth"]?.intValue ?? 0, h = o.result.output["pixelHeight"]?.intValue ?? 0486        #expect(w > 0 && h > 0 && (o.result.output["scale"]?.doubleValue ?? 0) >= 1)487        let image = try ImageIOHelper.load(dir.file("shot.png").path)488        #expect(image.width == w && image.height == h)489        #expect(o.result.artifacts.first?.kind == "screenshot")490491        let region = await invoke(ScreenCaptureTool(), ["scope": "region", "region": ["x": 10, "y": 10, "width": 120, "height": 80]], in: dir)492        #expect(region.result.status == .succeeded, "\(region.result.failure?.message ?? "")")493        let scale = region.result.output["scale"]?.doubleValue ?? 1494        #expect(region.result.output["pixelWidth"]?.intValue == Int(120 * scale))495        #expect(region.result.output["path"]?.stringValue?.hasPrefix(dir.url.path + "/artifacts/") == true)496        verified("screen.capture wrote a \(w)×\(h) px display screenshot and a region screenshot")497    }498}499500@Suite("Integration · clipboard (real pasteboard, restored afterwards)", .serialized)501struct ClipboardIntegrationTests {502    /// Every item and type on the pasteboard, to put back exactly what the user had.503    static func snapshot() -> [[(NSPasteboard.PasteboardType, Data)]] {504        (NSPasteboard.general.pasteboardItems ?? []).map { item in item.types.compactMap { t in item.data(forType: t).map { (t, $0) } } }505    }506507    static func restore(_ saved: [[(NSPasteboard.PasteboardType, Data)]]) {508        let pb = NSPasteboard.general509        pb.clearContents()510        let items = saved.map { entries -> NSPasteboardItem in511            let item = NSPasteboardItem()512            for (type, data) in entries { item.setData(data, forType: type) }513            return item514        }515        if !items.isEmpty { pb.writeObjects(items) }516    }517518    @Test func textAndFilesRoundTripAndConcealedEntriesAreWithheld() async throws {519        let dir = try TempDirectory(); defer { dir.remove() }520        let saved = Self.snapshot()521        defer { Self.restore(saved) }522523        let text = "orix clipboard \(UUID().uuidString) é✓"524        let dry = await invoke(ClipboardWriteTool(), ["content": .string(text)], in: dir, dryRun: true)525        #expect(dry.result.status == .simulated)526        #expect(NSPasteboard.general.string(forType: .string) != text)527528        let write = await invoke(ClipboardWriteTool(), ["content": .string(text)], in: dir)529        #expect(write.result.status == .succeeded)530        var read = await invoke(ClipboardReadTool(), [:], in: dir)531        #expect(read.result.output["text"] == .string(text))532        #expect(read.result.output["concealed"] == false)533        #expect(read.result.untrusted)534        #expect(read.result.output["changeCount"] == write.result.output["changeCount"])535536        let small = await invoke(ClipboardReadTool(), ["maxBytes": 8], in: dir)537        #expect(small.result.truncated)538        #expect(small.result.output["text"]?.stringValue?.hasPrefix("orix cli") == true)539540        let file = try dir.write("note.txt", "hello")541        let files = await invoke(ClipboardWriteTool(), ["paths": ["note.txt"]], in: dir)542        #expect(files.result.status == .succeeded)543        #expect(files.requirements?.capabilities == [.clipboardWrite, .fileRead(path: file.path)])544        read = await invoke(ClipboardReadTool(), [:], in: dir)545        #expect(read.result.output["fileURLs"] == [.string(file.path)])546547        // A password manager's entry: named, never read.548        let pb = NSPasteboard.general549        pb.clearContents()550        let item = NSPasteboardItem()551        item.setString("s3cret-password-value", forType: .string)552        item.setString("", forType: NSPasteboard.PasteboardType("org.nspasteboard.ConcealedType"))553        pb.writeObjects([item])554        read = await invoke(ClipboardReadTool(), [:], in: dir)555        #expect(read.result.output["concealed"] == true)556        #expect(read.result.output["text"] == .null)557        #expect(!read.result.output.serialized().contains("s3cret"))558559        let missing = await invoke(ClipboardWriteTool(), ["paths": ["absent.bin"]], in: dir)560        #expect(missing.result.failure?.code == "not_found")561    }562}563564@Suite("Integration · accessibility (gated on Accessibility trust)")565struct AccessibilityIntegrationTests {566    @Test func statusReportsTheRealTrust() async throws {567        let dir = try TempDirectory(); defer { dir.remove() }568        let o = await invoke(AccessibilityStatusTool(), [:], in: dir)569        #expect(o.result.status == .succeeded)570        #expect(o.result.output["trusted"] == .bool(AXIsProcessTrusted()))571        #expect(o.result.output["settingsURL"]?.stringValue?.contains("Privacy_Accessibility") == true)572    }573574    @Test func withoutTrustEveryToolIsBlockedNotWorkedAround() async throws {575        guard !AXIsProcessTrusted() else { skipped("accessibility is granted to this test process: the not-trusted path cannot be exercised"); return }576        let dir = try TempDirectory(); defer { dir.remove() }577        let path = "AXApplication/AXWindow[0]"578        let calls: [(any ORIXTool, JSONValue)] = [579            (AccessibilityTreeTool(), ["bundleID": "com.apple.finder"]),580            (AccessibilityFindTool(), ["bundleID": "com.apple.finder", "role": "AXWindow"]),581            (AccessibilityPressTool(), ["bundleID": "com.apple.finder", "element": .string(path)]),582            (AccessibilitySetValueTool(), ["bundleID": "com.apple.finder", "element": .string(path), "value": "x"]),583            (AccessibilityFocusTool(), ["bundleID": "com.apple.finder", "element": .string(path), "activate": false]),584            (AccessibilityTypeTextTool(), ["bundleID": "com.apple.finder", "element": .string(path), "text": "x"]),585            (AccessibilityWaitForTool(), ["bundleID": "com.apple.finder", "role": "AXWindow", "timeoutSeconds": 1]),586        ]587        for (tool, args) in calls {588            let o = await invoke(tool, args, in: dir)589            #expect(o.result.status == .blockedPermission, "\(tool.identifier): \(o.result.status)")590            if case .tccBlocked(let p, _)? = o.error { #expect(p == .accessibility) } else { Issue.record("\(tool.identifier) must return tccBlocked(.accessibility)") }591        }592        skipped("accessibility not granted to this test process: tree/find/press/setValue/focus/typeText/waitFor verified only up to their tccBlocked(.accessibility) result")593    }594595    @Test func treeFindAndWaitForOnFinder() async throws {596        guard AXIsProcessTrusted() else { skipped("accessibility not granted to this test process: live tree/find/waitFor not exercised"); return }597        let dir = try TempDirectory(); defer { dir.remove() }598        let tree = await invoke(AccessibilityTreeTool(), ["bundleID": "com.apple.finder", "maxDepth": 3, "maxNodes": 80, "includeMenuBar": true], in: dir)599        #expect(tree.result.status == .succeeded, "\(tree.result.failure?.message ?? "")")600        let nodes = tree.result.output["nodes"]?.arrayValue ?? []601        #expect(nodes.first?["path"] == "AXApplication" && nodes.first?["role"] == "AXApplication")602        #expect(nodes.count > 1 && nodes.count <= 80)603        #expect(nodes.allSatisfy { ($0["depth"]?.intValue ?? 99) <= 3 })604        #expect(nodes.filter { $0["secure"] == true }.allSatisfy { $0["value"] == "«secure»" })605606        let find = await invoke(AccessibilityFindTool(), ["bundleID": "com.apple.finder", "role": "AXMenuBarItem", "includeMenuBar": true, "maxResults": 5, "maxDepth": 3], in: dir)607        #expect(find.result.status == .succeeded, "\(find.result.status) \(find.result.failure?.message ?? "") \(find.result.output.serialized().prefix(300))")608        let matches = find.result.output["matches"]?.arrayValue ?? []609        #expect(!matches.isEmpty)610        guard let path = matches.first?["path"]?.stringValue else { return }611        #expect(path.hasPrefix("AXApplication/AXMenuBar[0]/AXMenuBarItem["))612613        // The path resolves again to the same element, and waitFor sees it without sleeping.614        let wait = await invoke(AccessibilityWaitForTool(), ["bundleID": "com.apple.finder", "element": .string(path), "condition": "exists", "timeoutSeconds": 3], in: dir)615        #expect(wait.result.output["satisfied"] == true)616        #expect(wait.result.output["element"]?["title"] == matches.first?["title"])617        let absent = await invoke(AccessibilityWaitForTool(), ["bundleID": "com.apple.finder", "identifier": "orix-no-such-element", "condition": "exists", "timeoutSeconds": 1], in: dir)618        #expect(absent.result.status == .timedOut && absent.result.output["satisfied"] == false)619        let gone = await invoke(AccessibilityWaitForTool(), ["bundleID": "com.apple.finder", "element": "AXApplication/AXWindow[97]", "condition": "absent", "timeoutSeconds": 2], in: dir)620        #expect(gone.result.output["satisfied"] == true)621622        // Stale or wrong paths are typed failures, and a title mismatch stops an action before it happens.623        let stale = await invoke(AccessibilityPressTool(), ["bundleID": "com.apple.finder", "element": "AXApplication/AXWindow[97]/AXButton[0]"], in: dir)624        #expect(stale.result.failure?.code == "element_not_found")625        let changed = await invoke(AccessibilityPressTool(), ["bundleID": "com.apple.finder", "element": .string(path), "expectedTitle": "orix-not-this-title"], in: dir, dryRun: true)626        #expect(changed.result.failure?.code == "element_changed")627        // Typing is refused because Finder's keyboard focus is not on that menu bar item (and Finder is usually not in front).628        let type = await invoke(AccessibilityTypeTextTool(), ["bundleID": "com.apple.finder", "element": .string(path), "text": "never typed"], in: dir)629        #expect(type.result.failure?.code == "focus_changed", "\(type.result.status) \(type.result.failure?.message ?? "")")630        verified("accessibility.tree/find/waitFor read Finder live (\(nodes.count) nodes); stale path, changed title and unfocused typing were refused")631    }632633    @Test func notRunningApplicationIsATypedFailure() async throws {634        guard AXIsProcessTrusted() else { skipped("accessibility not granted to this test process: app_not_running path not exercised"); return }635        let dir = try TempDirectory(); defer { dir.remove() }636        let o = await invoke(AccessibilityTreeTool(), ["bundleID": "com.example.absent.app"], in: dir)637        #expect(o.result.failure?.code == "app_not_running")638    }639}640641@Suite("Integration · notifications (no app bundle under swift test)")642struct NotificationIntegrationTests {643    @Test func postIsHonestlyUnsupportedWithoutABundle() async throws {644        let dir = try TempDirectory(); defer { dir.remove() }645        let o = await invoke(NotificationsPostTool(), ["title": "Run finished", "body": "3 files organised", "runID": "run-1", "approvalID": "ap-1"], in: dir)646        #expect(o.result.status == .unsupported)647        #expect(o.error == .unsupported(NotificationsPostTool.noBundleReason))648        #expect(o.result.failure?.message == "Notifications need an application bundle: this process has no bundle identifier.")649        #expect(o.requirements?.capabilities == [.notify])650        let bridge = NotificationCenterBridge { _ in }651        #expect(throws: ToolError.unsupported(NotificationsPostTool.noBundleReason)) { try bridge.register() }652        let empty = await invoke(NotificationsPostTool(), ["title": " ", "body": "x"], in: dir)653        #expect(empty.result.status == .invalidInput)654    }655}656657@Suite("Integration · InputRecorder (real NSWorkspace observers)")658@MainActor659struct InputRecorderIntegrationTests {660    @Test func startAndStopRecordTheirOwnLimits() async throws {661        let recorder = InputRecorder()662        #expect(recorder.state == .idle)663        #expect(throws: ToolError.self) { try recorder.start(targetBundleIDs: []) }664        #expect(throws: ToolError.self) { try recorder.start(targetBundleIDs: ["not a bundle id"]) }665        let seen = LockedBox<Int>(0)666        recorder.onAction = { _ in seen.mutate { $0 += 1 } }667        let limits = try recorder.start(targetBundleIDs: ["com.apple.finder"])668        #expect(recorder.state == .recording)669        #expect(recorder.targetBundleIDs == ["com.apple.finder"])670        #expect(!limits.isEmpty)671        #expect(limits.contains { $0.contains("Keystrokes") })672        if AXIsProcessTrusted() {673            #expect(!limits.contains { $0.contains("Accessibility is not enabled") })674            print("InputRecorder: accessibility granted, interface events of Finder are observed")675        } else {676            #expect(limits.contains { $0.contains("Accessibility is not enabled") && $0.contains("only application activations") })677            skipped("accessibility not granted to this test process: InputRecorder recorded app-level events and its limitation only")678        }679        #expect(throws: ToolError.self) { try recorder.start(targetBundleIDs: ["com.apple.finder"]) }680681        recorder.pause()682        #expect(recorder.state == .paused)683        recorder.resume()684        #expect(recorder.state == .recording)685        let actions = recorder.stop()686        #expect(recorder.state == .stopped)687        #expect(actions.first?.kind == .recordingStarted && actions.last?.kind == .recordingStopped)688        #expect(actions.contains { $0.kind == .recordingPaused } && actions.contains { $0.kind == .recordingResumed })689        #expect(actions.filter { $0.kind == .limitation }.count >= limits.count)690        #expect(actions.map(\.sequence) == Array(1...actions.count))691        #expect(zip(actions, actions.dropFirst()).allSatisfy { $0.timestamp <= $1.timestamp })692        #expect(seen.get() == actions.count)693        // Only the chosen application appears, and nothing carries a value.694        #expect(actions.allSatisfy { $0.bundleID == nil || $0.bundleID == "com.apple.finder" })695        #expect(actions.allSatisfy { $0.value == nil })696697        let decoded = try { () -> [RecordedAction] in698            let d = JSONDecoder(); d.dateDecodingStrategy = .iso8601699            return try d.decode([RecordedAction].self, from: try recorder.exportJSON())700        }()701        #expect(decoded.map(\.id) == actions.map(\.id) && decoded.map(\.kind) == actions.map(\.kind))702        #expect(recorder.stop().count == actions.count)703        // A stopped recorder can start a fresh recording.704        _ = try recorder.start(targetBundleIDs: ["com.apple.finder"])705        #expect(recorder.actions.first?.kind == .recordingStarted && recorder.actions.count < actions.count + 1)706        recorder.stop()707    }708709    @Test func activationsOfOtherApplicationsAreIgnored() async throws {710        let recorder = InputRecorder(options: .init(maxActions: 6))711        _ = try recorder.start(targetBundleIDs: ["com.apple.finder"])712        // Real notifications posted on the workspace centre: one for an application outside the targets, one for Finder.713        let center = NSWorkspace.shared.notificationCenter714        let finder = try #require(AppResolver.running("com.apple.finder").first)715        if let other = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier != nil && $0.bundleIdentifier != "com.apple.finder" }) {716            center.post(name: NSWorkspace.didActivateApplicationNotification, object: NSWorkspace.shared, userInfo: [NSWorkspace.applicationUserInfoKey: other])717        }718        center.post(name: NSWorkspace.didActivateApplicationNotification, object: NSWorkspace.shared, userInfo: [NSWorkspace.applicationUserInfoKey: finder])719        recorder.pause()720        center.post(name: NSWorkspace.didActivateApplicationNotification, object: NSWorkspace.shared, userInfo: [NSWorkspace.applicationUserInfoKey: finder])721        recorder.resume()722        let activations = recorder.actions.filter { $0.kind == .appActivated }723        #expect(activations.count == 1)724        #expect(activations.first?.bundleID == "com.apple.finder")725        // The action cap is enforced and recorded as a limitation.726        for _ in 0..<10 {727            center.post(name: NSWorkspace.didActivateApplicationNotification, object: NSWorkspace.shared, userInfo: [NSWorkspace.applicationUserInfoKey: finder])728        }729        let actions = recorder.stop()730        #expect(actions.count <= 8 && actions.last?.kind == .recordingStopped)731        #expect(recorder.limitations.contains { $0.contains("reached its limit") })732    }733}734
Files main
Menu

Theme