Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXAppleAutomation/Tests/ORIXAppleAutomationTests/IntegrationTests.swift3// Description: Integration tests against the real system: osascript, sdef, NSWorkspace, window server, Vision, pasteboard, Accessibility and ScreenCaptureKit (the last two gated on the actual TCC status, never prompting).4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import AppKit8import ApplicationServices9import CoreGraphics10import Foundation11import ORIXCore12import ORIXTestingSupport13import Testing14@testable import ORIXAppleAutomation1516@Suite("Integration · permissions (real TCC status, no prompt)")17struct PermissionIntegrationTests {18 @Test func everyPermissionHasAStatus() {19 var lines: [String] = []20 for p in MacPermission.allCases {21 let s = MacPermissions.status(p)22 #expect(PermissionStatus.allCases.contains(s))23 lines.append("\(p.rawValue)=\(s.rawValue)")24 }25 let finder = MacPermissions.automationStatus(bundleID: "com.apple.finder")26 lines.append("appleEvents(com.apple.finder)=\(finder.rawValue)")27 print("TCC status of the test process: " + lines.joined(separator: ", "))28 #expect(MacPermissions.status(.accessibility) == (AXIsProcessTrusted() ? .granted : .denied))29 #expect(MacPermissions.status(.screenRecording) == (CGPreflightScreenCaptureAccess() ? .granted : .denied))30 // `swift test` is not an application bundle: notifications cannot be evaluated, and that is what is reported.31 #expect(MacPermissions.hasAppBundle == false)32 #expect(MacPermissions.status(.notifications) == .unknown)33 #expect(MacPermissions.status(.appleEvents) == .unknown)34 #expect(MacPermissions.automationStatus(bundleID: "com.example.not-running-\(UUID().uuidString.prefix(6))") == .unknown)35 #expect(MacPermissions.snapshot().count == MacPermission.allCases.count)36 }3738 @Test func applicationResolutionUsesInstalledBundles() {39 #expect(AppResolver.bundleID(forName: "Finder") == "com.apple.finder")40 #expect(AppResolver.bundleID(forReference: "System Events") == "com.apple.systemevents")41 #expect(AppResolver.bundleID(forReference: "/System/Library/CoreServices/Finder.app") == "com.apple.finder")42 #expect(AppResolver.bundleID(forReference: "com.apple.finder") == "com.apple.finder")43 #expect(AppResolver.bundleID(forReference: "NoSuchApp-\(UUID().uuidString)") == nil)44 #expect(AppResolver.canonicalBundleID("COM.APPLE.FINDER") == "com.apple.finder")45 }46}4748@Suite("Integration · applescript.run / jxa.run (real osascript)")49struct ScriptIntegrationTests {50 let apple = ScriptRunTool(language: .applescript)51 let jxa = ScriptRunTool(language: .jxa)5253 @Test func appleScriptArithmeticStringsListsAndDates() async throws {54 let dir = try TempDirectory(); defer { dir.remove() }55 var o = await invoke(apple, ["script": "(6 * 7) + 0.5"], in: dir)56 #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")57 #expect(o.result.output["value"] == .number(42.5))58 #expect(o.result.output["parsed"] == true)59 #expect(o.result.untrusted)6061 o = await invoke(apple, ["script": "set s to \"orix\" & \" \" & \"automation\"\nreturn (count of s) as string & \":\" & s"], in: dir)62 #expect(o.result.output["value"] == "15:orix automation")63 #expect(o.result.output["result"] == "\"15:orix automation\"")6465 o = await invoke(apple, ["script": "set out to {}\nrepeat with i from 1 to 3\n set end of out to i * i\nend repeat\nreturn {squares:out, ok:true, label:\"é\"}"], in: dir)66 #expect(o.result.output["value"] == ["squares": [1, 4, 9], "ok": true, "label": "é"])6768 o = await invoke(apple, ["script": "return (year of (current date)) >= 2026"], in: dir)69 #expect(o.result.output["value"] == true)70 let year = Calendar.current.component(.year, from: Date())71 o = await invoke(apple, ["script": "year of ((current date) + 0)"], in: dir)72 #expect(o.result.output["value"] == .int(year))7374 // A date is not representable as JSON: the text is returned and the tool says it was not parsed.75 o = await invoke(apple, ["script": "current date"], in: dir)76 #expect(o.result.status == .succeeded)77 #expect(o.result.output["parsed"] == false)78 #expect(o.result.output["result"]?.stringValue?.hasPrefix("date ") == true)79 }8081 @Test func appleScriptSyntaxAndRuntimeErrorsAreTyped() async throws {82 let dir = try TempDirectory(); defer { dir.remove() }83 var o = await invoke(apple, ["script": "set x to 1\nset y to (2 +\nreturn x"], in: dir)84 #expect(o.result.status == .failed)85 #expect(o.result.failure?.code == "syntax_error")86 #expect(o.result.output["line"]?.intValue == 2 || o.result.output["line"]?.intValue == 3, "line \(String(describing: o.result.output["line"]))")87 #expect(o.result.output["errorNumber"]?.intValue == -2741)88 #expect(o.result.failure?.message.contains("line") == true)8990 o = await invoke(apple, ["script": "error \"boom\" number 4242"], in: dir)91 #expect(o.result.failure?.code == "script_error")92 #expect(o.result.output["errorNumber"] == 4242)93 #expect(o.result.failure?.message.contains("boom") == true)9495 o = await invoke(apple, ["script": "return 1 / 0"], in: dir)96 #expect(o.result.failure?.code == "script_error")97 #expect(o.result.output["errorNumber"] == -2701)98 }99100 @Test func scriptsAreStoppedAtTheTimeout() async throws {101 let dir = try TempDirectory(); defer { dir.remove() }102 let started = Date()103 var o = await invoke(apple, ["script": "repeat\nend repeat", "timeoutSeconds": 1.5], in: dir)104 #expect(o.result.status == .timedOut)105 #expect(o.error == .timeout(seconds: 1.5))106 o = await invoke(jxa, ["script": "while (true) {}", "timeoutSeconds": 1.5], in: dir)107 #expect(o.result.status == .timedOut)108 #expect(Date().timeIntervalSince(started) < 12)109 }110111 @Test func runningScriptIsTerminatedOnCancellation() async throws {112 let dir = try TempDirectory(); defer { dir.remove() }113 let tool = apple114 let task = Task { await invoke(tool, ["script": "repeat\nend repeat", "timeoutSeconds": 25], in: dir) }115 try await Task.sleep(nanoseconds: 700_000_000)116 let started = Date()117 task.cancel()118 let o = await task.value119 #expect(o.result.status == .canceled)120 #expect(Date().timeIntervalSince(started) < 5)121 }122123 @Test func jxaReturnsStructuredJSON() async throws {124 let dir = try TempDirectory(); defer { dir.remove() }125 var o = await invoke(jxa, ["script": "const xs = [1, 2, 3].map(x => x * x); ({sum: xs.reduce((a, b) => a + b, 0), xs: xs, name: 'orix'.toUpperCase()})"], in: dir)126 #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")127 #expect(o.result.output["value"] == ["sum": 14, "xs": [1, 4, 9], "name": "ORIX"])128 #expect(o.result.output["parsed"] == true)129130 o = await invoke(jxa, ["script": "new Date(86400000).toISOString()"], in: dir)131 #expect(o.result.output["value"] == "1970-01-02T00:00:00.000Z")132 #expect(o.result.output["result"] == "1970-01-02T00:00:00.000Z")133134 o = await invoke(jxa, ["script": "function run(argv) { return 6 * 7 }"], in: dir)135 #expect(o.result.output["value"] == 42)136137 o = await invoke(jxa, ["script": "var nothing; nothing"], in: dir)138 #expect(o.result.status == .succeeded)139 #expect(o.result.output["value"] == .null && o.result.output["parsed"] == false)140 }141142 @Test func jxaErrorsCarryLineInformation() async throws {143 let dir = try TempDirectory(); defer { dir.remove() }144 var o = await invoke(jxa, ["script": "var a = 1;\nvar b = (;\na"], in: dir)145 #expect(o.result.failure?.code == "syntax_error", "\(String(describing: o.result.failure))")146 #expect(o.result.output["line"] == 2)147148 // A SyntaxError raised while running (not while compiling) is a script error, not a syntax error.149 o = await invoke(jxa, ["script": "JSON.parse('{')"], in: dir)150 #expect(o.result.failure?.code == "script_error")151152 o = await invoke(jxa, ["script": "var a = 1;\n\nundefinedFunction(a)"], in: dir)153 #expect(o.result.failure?.code == "script_error")154 #expect(o.result.output["line"] == 3)155 #expect(o.result.failure?.message.contains("undefinedFunction") == true)156 }157158 @Test func undeclaredTargetNeverRuns() async throws {159 let dir = try TempDirectory(); defer { dir.remove() }160 let marker = dir.file("ran.txt")161 // If the refusal were not static, the shell command would create the marker before the Finder call.162 let script = "do shell script \"touch \" & quoted form of \"\(marker.path)\"\ntell application \"Finder\" to get name of startup disk"163 let o = await invoke(apple, ["script": .string(script), "targets": []], in: dir)164 #expect(o.result.status == .invalidInput)165 #expect(o.result.failure?.message.contains("Finder") == true)166 #expect(!FileManager.default.fileExists(atPath: marker.path))167168 let j = await invoke(jxa, ["script": "Application('Finder').startupDisk.name()", "targets": ["com.apple.mail"]], in: dir)169 #expect(j.result.status == .invalidInput)170 }171172 @Test func dryRunDoesNotExecute() async throws {173 let dir = try TempDirectory(); defer { dir.remove() }174 let marker = dir.file("ran.txt")175 let o = await invoke(apple, ["script": .string("do shell script \"touch \" & quoted form of \"\(marker.path)\"")], in: dir, dryRun: true)176 #expect(o.result.status == .simulated)177 #expect(!FileManager.default.fileExists(atPath: marker.path))178 }179180 @Test func shellEscapeRunsOnlyThroughTheDeclaredRuntime() async throws {181 let dir = try TempDirectory(); defer { dir.remove() }182 let o = await invoke(apple, ["script": "do shell script \"printf orix-shell\""], in: dir)183 #expect(o.requirements?.capabilities.contains(.scriptRuntime(name: "bash")) == true)184 #expect(o.requirements?.tier == .consequential)185 #expect(o.result.output["value"] == "orix-shell")186 }187188 @Test func finderScriptSucceedsOrReportsTheAppleEventsBlock() async throws {189 let dir = try TempDirectory(); defer { dir.remove() }190 let status = MacPermissions.automationStatus(bundleID: "com.apple.finder")191 if status == .notDetermined {192 // Running it would make macOS show the Automation consent dialog: tests never prompt.193 skipped("apple events consent for Finder is undetermined for this test process; running the script would show the system prompt")194 return195 }196 for (tool, script) in [(apple, "tell application \"Finder\" to get name of startup disk"), (jxa, "Application('Finder').startupDisk.name()")] {197 let o = await invoke(tool, ["script": .string(script), "targets": ["com.apple.finder"], "timeoutSeconds": 20], in: dir, timeout: 20)198 if o.result.status == .succeeded {199 let name = o.result.output["value"]?.stringValue ?? ""200 #expect(!name.isEmpty)201 #expect(o.result.output["targets"] == ["com.apple.finder"])202 verified("\(tool.identifier) talked to Finder: startup disk is \"\(name)\"")203 } else if case .tccBlocked(let permission, let message)? = o.error {204 #expect(permission == .appleEvents)205 #expect(message.contains("Finder"))206 #expect(o.result.status == .blockedPermission)207 skipped("apple events not authorized for this test process (\(tool.identifier))")208 } else {209 Issue.record("\(tool.identifier): expected success or tccBlocked(.appleEvents), got \(o.result.status) \(o.result.failure?.message ?? "")")210 }211 }212 }213}214215@Suite("Integration · applescript.dictionary (real sdef)")216struct DictionaryIntegrationTests {217 @Test func finderDictionaryIsSummarised() async throws {218 let dir = try TempDirectory(); defer { dir.remove() }219 let o = await invoke(ScriptingDictionaryTool(), ["bundleID": "com.apple.finder"], in: dir)220 #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")221 #expect(o.requirements?.capabilities == [.fileRead(path: "/System/Library/CoreServices/Finder.app")])222 let suites = o.result.output["suites"]?.arrayValue ?? []223 #expect(suites.count >= 3)224 #expect(suites.contains { $0["name"] == "Standard Suite" })225 let commands = suites.flatMap { $0["commands"]?.arrayValue ?? [] }226 let open = commands.first { $0["name"] == "open" }227 #expect(open != nil)228 #expect(open?["parameters"]?.arrayValue?.contains { $0["name"] == "using" } == true)229 let classes = suites.flatMap { $0["classes"]?.arrayValue ?? [] }230 let withProperties = classes.filter { !($0["properties"]?.arrayValue ?? []).isEmpty }231 #expect(!withProperties.isEmpty)232 #expect((o.result.output["commandCount"]?.intValue ?? 0) > 10)233 #expect(o.result.untrusted)234 #expect(o.result.output.serialized().utf8.count < 262_144)235 }236237 @Test func boundIsHonoured() async throws {238 let dir = try TempDirectory(); defer { dir.remove() }239 let o = await invoke(ScriptingDictionaryTool(), ["path": "/System/Library/CoreServices/Finder.app", "maxItems": 2], in: dir)240 #expect(o.result.status == .succeeded)241 for suite in o.result.output["suites"]?.arrayValue ?? [] {242 #expect((suite["commands"]?.arrayValue?.count ?? 0) <= 2)243 #expect((suite["classes"]?.arrayValue?.count ?? 0) <= 2)244 }245 #expect(o.result.truncated)246 }247248 @Test func appWithoutDictionaryIsReportedUnsupported() async throws {249 let dir = try TempDirectory(); defer { dir.remove() }250 let o = await invoke(ScriptingDictionaryTool(), ["path": "/System/Applications/Calculator.app"], in: dir)251 #expect(o.result.status == .unsupported, "\(o.result.status) \(o.result.failure?.message ?? "")")252 let missing = await invoke(ScriptingDictionaryTool(), ["bundleID": "com.example.absent.app"], in: dir)253 #expect(missing.result.failure?.code == "not_found")254 }255}256257// Serialized: one test launches Calculator while another asserts that a dry run launched nothing.258@Suite("Integration · system.* (real NSWorkspace and window server)", .serialized)259struct SystemIntegrationTests {260 @Test func runningApplicationsIncludeFinder() async throws {261 let dir = try TempDirectory(); defer { dir.remove() }262 let o = await invoke(SystemAppsTool(), ["includeBackground": true], in: dir)263 #expect(o.result.status == .succeeded)264 let apps = o.result.output["applications"]?.arrayValue ?? []265 let finder = apps.first { $0["bundleID"] == "com.apple.finder" }266 #expect(finder != nil)267 #expect((finder?["pid"]?.intValue ?? 0) > 0)268 #expect(finder?["name"]?.stringValue?.isEmpty == false)269 let regular = await invoke(SystemAppsTool(), [:], in: dir)270 #expect((regular.result.output["applications"]?.arrayValue?.count ?? 0) <= apps.count)271 }272273 @Test func frontmostDegradesHonestly() async throws {274 let dir = try TempDirectory(); defer { dir.remove() }275 let o = await invoke(SystemFrontmostTool(), [:], in: dir)276 #expect(o.result.status == .succeeded)277 let source = o.result.output["titleSource"]?.stringValue278 if AXIsProcessTrusted() { #expect(source == "accessibility") }279 else if CGPreflightScreenCaptureAccess() { #expect(source == "windowList") }280 else {281 #expect(source == "none")282 #expect(o.result.output["windowTitle"] == .null)283 #expect(o.result.output["note"]?.stringValue?.contains("Accessibility") == true)284 }285 print("system.frontmost: app=\(o.result.output["application"]?["name"]?.stringValue ?? "none") titleSource=\(source ?? "?")")286 }287288 @Test func windowsSayWhetherTitlesAreAvailable() async throws {289 let dir = try TempDirectory(); defer { dir.remove() }290 let o = await invoke(SystemWindowsTool(), ["includeAllLayers": true], in: dir)291 #expect(o.result.status == .succeeded)292 let titles = CGPreflightScreenCaptureAccess()293 #expect(o.result.output["titlesAvailable"] == .bool(titles))294 let windows = o.result.output["windows"]?.arrayValue ?? []295 if !titles {296 #expect(windows.allSatisfy { $0["title"] == nil })297 #expect(o.result.output["note"]?.stringValue?.contains("Screen Recording") == true)298 }299 for w in windows.prefix(20) {300 #expect((w["pid"]?.intValue ?? 0) > 0)301 #expect(w["bounds"]?["width"]?.doubleValue != nil)302 }303 #expect((o.result.output["displays"]?.arrayValue?.count ?? 0) >= (CGMainDisplayID() != 0 && !Displays.active().isEmpty ? 1 : 0))304 print("system.windows: \(windows.count) window(s), titlesAvailable=\(titles)")305 let filtered = await invoke(SystemWindowsTool(), ["bundleID": "com.apple.finder", "includeAllLayers": true], in: dir)306 #expect(filtered.result.output["windows"]?.arrayValue?.allSatisfy { $0["bundleID"] == "com.apple.finder" } == true)307 }308309 @Test func launchAndQuitDryRunsChangeNothing() async throws {310 let dir = try TempDirectory(); defer { dir.remove() }311 let before = AppResolver.running("com.apple.calculator").count312 let launch = await invoke(SystemLaunchAppTool(), ["bundleID": "com.apple.calculator"], in: dir, dryRun: true)313 #expect(launch.result.status == .simulated)314 let quit = await invoke(SystemQuitAppTool(), ["bundleID": "com.apple.finder"], in: dir, dryRun: true)315 #expect(quit.result.status == .simulated)316 #expect(quit.result.output["wasRunning"] == true)317 #expect(AppResolver.running("com.apple.calculator").count == before)318 #expect(!AppResolver.running("com.apple.finder").isEmpty)319 let absent = await invoke(SystemLaunchAppTool(), ["bundleID": "com.example.absent.app"], in: dir)320 #expect(absent.result.failure?.code == "not_found")321 let notRunning = await invoke(SystemQuitAppTool(), ["bundleID": "com.example.absent.app"], in: dir)322 #expect(notRunning.result.status == .succeeded && notRunning.result.output["wasRunning"] == false)323 }324325 @Test func launchWaitsForTheAppAndQuitIsPolite() async throws {326 let dir = try TempDirectory(); defer { dir.remove() }327 let id = "com.apple.calculator"328 guard AppResolver.url(forBundleID: id) != nil else { skipped("Calculator is not installed"); return }329 guard AppResolver.running(id).isEmpty else {330 skipped("Calculator is already running; the test will not quit an application the user opened")331 return332 }333 let launch = await invoke(SystemLaunchAppTool(), ["bundleID": .string(id), "activate": false, "timeoutSeconds": 20], in: dir)334 #expect(launch.result.status == .succeeded, "\(launch.result.failure?.message ?? "")")335 #expect(launch.result.output["alreadyRunning"] == false)336 #expect(launch.result.output["finishedLaunching"] == true)337 let pid = pid_t(launch.result.output["pid"]?.intValue ?? 0)338 #expect(pid > 0 && kill(pid, 0) == 0)339340 if AXIsProcessTrusted() { await pressRoundTrip(in: dir) }341 else { skipped("accessibility not granted to this test process: accessibility.press not exercised on a live application") }342343 let quit = await invoke(SystemQuitAppTool(), ["bundleID": .string(id), "timeoutSeconds": 15], in: dir)344 #expect(quit.result.status == .succeeded)345 #expect(quit.result.output["wasRunning"] == true)346 #expect(quit.result.output["terminated"] == true && quit.result.output["refused"] == false)347 #expect(kill(pid, 0) != 0)348 verified("system.launchApp started Calculator (pid \(pid)) in the background and system.quitApp ended it")349 }350}351352extension SystemIntegrationTests {353 /// find → press → observe → press back, on the Calculator this test launched in the background. The sidebar toggle is354 /// used because pressing it twice leaves the application exactly as it was (no calculation is entered or cleared).355 func pressRoundTrip(in dir: TempDirectory) async {356 let id = "com.apple.calculator"357 let ready = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "role": "AXToolbar", "timeoutSeconds": 10], in: dir)358 guard ready.result.output["satisfied"] == true, let toolbar = ready.result.output["element"]?["path"]?.stringValue else {359 skipped("this Calculator version exposes no toolbar through Accessibility: press round trip not exercised"); return360 }361 let button = toolbar + "/AXButton[0]"362 let tree = await invoke(AccessibilityTreeTool(), ["bundleID": .string(id), "element": .string(button), "maxDepth": 0], in: dir)363 let splitter = await invoke(AccessibilityFindTool(), ["bundleID": .string(id), "role": "AXSplitter", "maxResults": 1], in: dir)364 guard let label = tree.result.output["nodes"]?[0]?["description"]?.stringValue, !label.isEmpty,365 let splitterPath = splitter.result.output["matches"]?[0]?["path"]?.stringValue,366 let closedValue = splitter.result.output["matches"]?[0]?["value"]?.displayString else {367 skipped("this Calculator version has no labelled sidebar button or splitter: press round trip not exercised"); return368 }369 let dry = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(button), "expectedTitle": .string(label)], in: dir, dryRun: true)370 #expect(dry.result.status == .simulated)371 let untouched = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "element": .string(splitterPath), "condition": "valueEquals", "expected": .string(closedValue), "timeoutSeconds": 1], in: dir)372 #expect(untouched.result.output["satisfied"] == true)373374 let press = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(button), "expectedTitle": .string(label)], in: dir)375 #expect(press.result.status == .succeeded, "\(press.result.failure?.message ?? "")")376 #expect(press.result.output["pressed"] == true)377 // Observable postcondition: the button no longer carries its original label (it now offers the opposite action).378 let opened = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "role": "AXButton", "title": .string(label), "condition": "absent", "timeoutSeconds": 5], in: dir)379 #expect(opened.result.output["satisfied"] == true, "pressing the sidebar button changed nothing observable")380 // Same path, different meaning now: the expectedTitle guard refuses to press it.381 let guarded = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(button), "expectedTitle": .string(label)], in: dir)382 #expect(guarded.result.failure?.code == "element_changed")383 let back = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(button)], in: dir)384 #expect(back.result.status == .succeeded)385 let closed = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "role": "AXButton", "title": .string(label), "condition": "exists", "timeoutSeconds": 5], in: dir)386 #expect(closed.result.output["satisfied"] == true)387 let restored = await invoke(AccessibilityWaitForTool(), ["bundleID": .string(id), "element": .string(splitterPath), "condition": "valueEquals", "expected": .string(closedValue), "timeoutSeconds": 5], in: dir)388 #expect(restored.result.output["satisfied"] == true, "the sidebar did not return to its initial state")389 // A static text is not pressable, and its value cannot be set: both are honest refusals.390 let text = await invoke(AccessibilityFindTool(), ["bundleID": .string(id), "role": "AXStaticText", "maxResults": 1], in: dir)391 if let path = text.result.output["matches"]?[0]?["path"]?.stringValue {392 let p = await invoke(AccessibilityPressTool(), ["bundleID": .string(id), "element": .string(path)], in: dir)393 #expect(p.result.status == .unsupported)394 let s = await invoke(AccessibilitySetValueTool(), ["bundleID": .string(id), "element": .string(path), "value": "1"], in: dir)395 #expect(s.result.status == .unsupported)396 }397 verified("accessibility.press toggled Calculator's sidebar and back in the background; waitFor observed both states and expectedTitle refused the changed button")398 }399}400401@Suite("Integration · screen.ocr / screen.compare (real Vision and CoreGraphics)")402struct ScreenFileIntegrationTests {403 @Test func ocrRecognisesRenderedWords() async throws {404 let dir = try TempDirectory(); defer { dir.remove() }405 let image = dir.file("words.png")406 try TestImages.text(["ORIX AUTOMATION", "Invoice 48213 approved"], to: image)407 let o = await invoke(ScreenOCRTool(), ["path": "words.png", "languages": ["en-US"]], in: dir, timeout: 90)408 #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")409 let text = o.result.output["text"]?.stringValue ?? ""410 for word in ["ORIX", "AUTOMATION", "Invoice", "48213", "approved"] { #expect(text.contains(word), "missing \(word) in \(text)") }411 #expect(o.result.untrusted)412 #expect(o.result.output["imageWidth"] == 1100 && o.result.output["imageHeight"] == 320)413 let blocks = o.result.output["blocks"]?.arrayValue ?? []414 #expect(blocks.count >= 2)415 // First line is above the second one (top-left origin), both inside the image.416 let first = blocks.first { $0["text"]?.stringValue?.contains("ORIX") == true }417 let second = blocks.first { $0["text"]?.stringValue?.contains("Invoice") == true }418 let y1 = first?["pixels"]?["y"]?.doubleValue ?? -1, y2 = second?["pixels"]?["y"]?.doubleValue ?? -1419 #expect(y1 >= 0 && y2 > y1 && y2 < 320)420 let n = first?["normalized"]421 #expect((n?["x"]?.doubleValue ?? -1) >= 0 && (n?["width"]?.doubleValue ?? 2) <= 1)422 #expect((first?["confidence"]?.doubleValue ?? 0) > 0.3)423 }424425 @Test func ocrRejectsMissingAndNonImageFiles() async throws {426 let dir = try TempDirectory(); defer { dir.remove() }427 let missing = await invoke(ScreenOCRTool(), ["path": "nope.png"], in: dir)428 #expect(missing.result.failure?.code == "not_found")429 try dir.write("text.png", "this is not an image")430 let bad = await invoke(ScreenOCRTool(), ["path": "text.png"], in: dir)431 #expect(bad.result.status == .invalidInput)432 }433434 @Test func compareFindsTheChangedRegion() async throws {435 let dir = try TempDirectory(); defer { dir.remove() }436 try TestImages.blocks(width: 200, height: 100, background: 1, rects: [], to: dir.file("a.png"))437 try TestImages.blocks(width: 200, height: 100, background: 1, rects: [(CGRect(x: 50, y: 20, width: 40, height: 30), 0)], to: dir.file("b.png"))438 try TestImages.blocks(width: 200, height: 100, background: 1, rects: [], to: dir.file("same.png"))439 try TestImages.blocks(width: 100, height: 100, background: 1, rects: [], to: dir.file("small.png"))440441 var o = await invoke(ScreenCompareTool(), ["before": "a.png", "after": "b.png", "diffPath": "out/diff.png"], in: dir)442 #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")443 #expect(o.result.output["identical"] == false)444 #expect(o.result.output["changedPixels"] == 1200)445 #expect(o.result.output["totalPixels"] == 20000)446 #expect(o.result.output["changedRatio"] == .number(0.06))447 #expect(o.result.output["boundingBox"] == ["x": .number(50), "y": .number(20), "width": .number(40), "height": .number(30)])448 let diff = dir.file("out/diff.png").path449 #expect(o.result.artifacts.first?.path == diff)450 #expect((o.result.artifacts.first?.bytes ?? 0) > 0 && o.result.artifacts.first?.sha256?.count == 64)451 // The annotated image is a real PNG of the same size, red where the change is.452 let rendered = try ImageIOHelper.rgba(try ImageIOHelper.load(diff))453 #expect(rendered.width == 200 && rendered.height == 100)454 let centre = (35 * 200 + 70) * 4455 #expect(rendered.pixels[centre] > 100 && rendered.pixels[centre + 1] < 40 && rendered.pixels[centre + 2] < 40)456457 o = await invoke(ScreenCompareTool(), ["before": "a.png", "after": "same.png"], in: dir)458 #expect(o.result.output["identical"] == true && o.result.output["changedPixels"] == 0)459 #expect(o.result.output["boundingBox"] == nil)460 #expect(o.requirements?.tier == .readOnly)461462 o = await invoke(ScreenCompareTool(), ["before": "a.png", "after": "small.png"], in: dir)463 #expect(o.result.failure?.code == "size_mismatch")464465 // An existing diff file is not replaced without overwrite.466 o = await invoke(ScreenCompareTool(), ["before": "a.png", "after": "b.png", "diffPath": "out/diff.png"], in: dir)467 #expect(o.result.failure?.code == "io")468 }469}470471@Suite("Integration · screen.capture (gated on Screen Recording)")472struct ScreenCaptureIntegrationTests {473 @Test func captureDisplayOrReportTheBlock() async throws {474 let dir = try TempDirectory(); defer { dir.remove() }475 let o = await invoke(ScreenCaptureTool(), ["scope": "display", "displayIndex": 0, "path": "shot.png"], in: dir)476 guard CGPreflightScreenCaptureAccess() else {477 #expect(o.error.map { if case .tccBlocked(.screenRecording, _) = $0 { return true } else { return false } } == true)478 #expect(o.result.status == .blockedPermission)479 #expect(!FileManager.default.fileExists(atPath: dir.file("shot.png").path))480 skipped("screen recording not granted to this test process: screen.capture returned tccBlocked(.screenRecording) as required")481 return482 }483 if case .permissionDenied? = o.error { skipped("secure keyboard input is active: capture correctly refused"); return }484 #expect(o.result.status == .succeeded, "\(o.result.failure?.message ?? "")")485 let w = o.result.output["pixelWidth"]?.intValue ?? 0, h = o.result.output["pixelHeight"]?.intValue ?? 0486 #expect(w > 0 && h > 0 && (o.result.output["scale"]?.doubleValue ?? 0) >= 1)487 let image = try ImageIOHelper.load(dir.file("shot.png").path)488 #expect(image.width == w && image.height == h)489 #expect(o.result.artifacts.first?.kind == "screenshot")490491 let region = await invoke(ScreenCaptureTool(), ["scope": "region", "region": ["x": 10, "y": 10, "width": 120, "height": 80]], in: dir)492 #expect(region.result.status == .succeeded, "\(region.result.failure?.message ?? "")")493 let scale = region.result.output["scale"]?.doubleValue ?? 1494 #expect(region.result.output["pixelWidth"]?.intValue == Int(120 * scale))495 #expect(region.result.output["path"]?.stringValue?.hasPrefix(dir.url.path + "/artifacts/") == true)496 verified("screen.capture wrote a \(w)×\(h) px display screenshot and a region screenshot")497 }498}499500@Suite("Integration · clipboard (real pasteboard, restored afterwards)", .serialized)501struct ClipboardIntegrationTests {502 /// Every item and type on the pasteboard, to put back exactly what the user had.503 static func snapshot() -> [[(NSPasteboard.PasteboardType, Data)]] {504 (NSPasteboard.general.pasteboardItems ?? []).map { item in item.types.compactMap { t in item.data(forType: t).map { (t, $0) } } }505 }506507 static func restore(_ saved: [[(NSPasteboard.PasteboardType, Data)]]) {508 let pb = NSPasteboard.general509 pb.clearContents()510 let items = saved.map { entries -> NSPasteboardItem in511 let item = NSPasteboardItem()512 for (type, data) in entries { item.setData(data, forType: type) }513 return item514 }515 if !items.isEmpty { pb.writeObjects(items) }516 }517518 @Test func textAndFilesRoundTripAndConcealedEntriesAreWithheld() async throws {519 let dir = try TempDirectory(); defer { dir.remove() }520 let saved = Self.snapshot()521 defer { Self.restore(saved) }522523 let text = "orix clipboard \(UUID().uuidString) é✓"524 let dry = await invoke(ClipboardWriteTool(), ["content": .string(text)], in: dir, dryRun: true)525 #expect(dry.result.status == .simulated)526 #expect(NSPasteboard.general.string(forType: .string) != text)527528 let write = await invoke(ClipboardWriteTool(), ["content": .string(text)], in: dir)529 #expect(write.result.status == .succeeded)530 var read = await invoke(ClipboardReadTool(), [:], in: dir)531 #expect(read.result.output["text"] == .string(text))532 #expect(read.result.output["concealed"] == false)533 #expect(read.result.untrusted)534 #expect(read.result.output["changeCount"] == write.result.output["changeCount"])535536 let small = await invoke(ClipboardReadTool(), ["maxBytes": 8], in: dir)537 #expect(small.result.truncated)538 #expect(small.result.output["text"]?.stringValue?.hasPrefix("orix cli") == true)539540 let file = try dir.write("note.txt", "hello")541 let files = await invoke(ClipboardWriteTool(), ["paths": ["note.txt"]], in: dir)542 #expect(files.result.status == .succeeded)543 #expect(files.requirements?.capabilities == [.clipboardWrite, .fileRead(path: file.path)])544 read = await invoke(ClipboardReadTool(), [:], in: dir)545 #expect(read.result.output["fileURLs"] == [.string(file.path)])546547 // A password manager's entry: named, never read.548 let pb = NSPasteboard.general549 pb.clearContents()550 let item = NSPasteboardItem()551 item.setString("s3cret-password-value", forType: .string)552 item.setString("", forType: NSPasteboard.PasteboardType("org.nspasteboard.ConcealedType"))553 pb.writeObjects([item])554 read = await invoke(ClipboardReadTool(), [:], in: dir)555 #expect(read.result.output["concealed"] == true)556 #expect(read.result.output["text"] == .null)557 #expect(!read.result.output.serialized().contains("s3cret"))558559 let missing = await invoke(ClipboardWriteTool(), ["paths": ["absent.bin"]], in: dir)560 #expect(missing.result.failure?.code == "not_found")561 }562}563564@Suite("Integration · accessibility (gated on Accessibility trust)")565struct AccessibilityIntegrationTests {566 @Test func statusReportsTheRealTrust() async throws {567 let dir = try TempDirectory(); defer { dir.remove() }568 let o = await invoke(AccessibilityStatusTool(), [:], in: dir)569 #expect(o.result.status == .succeeded)570 #expect(o.result.output["trusted"] == .bool(AXIsProcessTrusted()))571 #expect(o.result.output["settingsURL"]?.stringValue?.contains("Privacy_Accessibility") == true)572 }573574 @Test func withoutTrustEveryToolIsBlockedNotWorkedAround() async throws {575 guard !AXIsProcessTrusted() else { skipped("accessibility is granted to this test process: the not-trusted path cannot be exercised"); return }576 let dir = try TempDirectory(); defer { dir.remove() }577 let path = "AXApplication/AXWindow[0]"578 let calls: [(any ORIXTool, JSONValue)] = [579 (AccessibilityTreeTool(), ["bundleID": "com.apple.finder"]),580 (AccessibilityFindTool(), ["bundleID": "com.apple.finder", "role": "AXWindow"]),581 (AccessibilityPressTool(), ["bundleID": "com.apple.finder", "element": .string(path)]),582 (AccessibilitySetValueTool(), ["bundleID": "com.apple.finder", "element": .string(path), "value": "x"]),583 (AccessibilityFocusTool(), ["bundleID": "com.apple.finder", "element": .string(path), "activate": false]),584 (AccessibilityTypeTextTool(), ["bundleID": "com.apple.finder", "element": .string(path), "text": "x"]),585 (AccessibilityWaitForTool(), ["bundleID": "com.apple.finder", "role": "AXWindow", "timeoutSeconds": 1]),586 ]587 for (tool, args) in calls {588 let o = await invoke(tool, args, in: dir)589 #expect(o.result.status == .blockedPermission, "\(tool.identifier): \(o.result.status)")590 if case .tccBlocked(let p, _)? = o.error { #expect(p == .accessibility) } else { Issue.record("\(tool.identifier) must return tccBlocked(.accessibility)") }591 }592 skipped("accessibility not granted to this test process: tree/find/press/setValue/focus/typeText/waitFor verified only up to their tccBlocked(.accessibility) result")593 }594595 @Test func treeFindAndWaitForOnFinder() async throws {596 guard AXIsProcessTrusted() else { skipped("accessibility not granted to this test process: live tree/find/waitFor not exercised"); return }597 let dir = try TempDirectory(); defer { dir.remove() }598 let tree = await invoke(AccessibilityTreeTool(), ["bundleID": "com.apple.finder", "maxDepth": 3, "maxNodes": 80, "includeMenuBar": true], in: dir)599 #expect(tree.result.status == .succeeded, "\(tree.result.failure?.message ?? "")")600 let nodes = tree.result.output["nodes"]?.arrayValue ?? []601 #expect(nodes.first?["path"] == "AXApplication" && nodes.first?["role"] == "AXApplication")602 #expect(nodes.count > 1 && nodes.count <= 80)603 #expect(nodes.allSatisfy { ($0["depth"]?.intValue ?? 99) <= 3 })604 #expect(nodes.filter { $0["secure"] == true }.allSatisfy { $0["value"] == "«secure»" })605606 let find = await invoke(AccessibilityFindTool(), ["bundleID": "com.apple.finder", "role": "AXMenuBarItem", "includeMenuBar": true, "maxResults": 5, "maxDepth": 3], in: dir)607 #expect(find.result.status == .succeeded, "\(find.result.status) \(find.result.failure?.message ?? "") \(find.result.output.serialized().prefix(300))")608 let matches = find.result.output["matches"]?.arrayValue ?? []609 #expect(!matches.isEmpty)610 guard let path = matches.first?["path"]?.stringValue else { return }611 #expect(path.hasPrefix("AXApplication/AXMenuBar[0]/AXMenuBarItem["))612613 // The path resolves again to the same element, and waitFor sees it without sleeping.614 let wait = await invoke(AccessibilityWaitForTool(), ["bundleID": "com.apple.finder", "element": .string(path), "condition": "exists", "timeoutSeconds": 3], in: dir)615 #expect(wait.result.output["satisfied"] == true)616 #expect(wait.result.output["element"]?["title"] == matches.first?["title"])617 let absent = await invoke(AccessibilityWaitForTool(), ["bundleID": "com.apple.finder", "identifier": "orix-no-such-element", "condition": "exists", "timeoutSeconds": 1], in: dir)618 #expect(absent.result.status == .timedOut && absent.result.output["satisfied"] == false)619 let gone = await invoke(AccessibilityWaitForTool(), ["bundleID": "com.apple.finder", "element": "AXApplication/AXWindow[97]", "condition": "absent", "timeoutSeconds": 2], in: dir)620 #expect(gone.result.output["satisfied"] == true)621622 // Stale or wrong paths are typed failures, and a title mismatch stops an action before it happens.623 let stale = await invoke(AccessibilityPressTool(), ["bundleID": "com.apple.finder", "element": "AXApplication/AXWindow[97]/AXButton[0]"], in: dir)624 #expect(stale.result.failure?.code == "element_not_found")625 let changed = await invoke(AccessibilityPressTool(), ["bundleID": "com.apple.finder", "element": .string(path), "expectedTitle": "orix-not-this-title"], in: dir, dryRun: true)626 #expect(changed.result.failure?.code == "element_changed")627 // Typing is refused because Finder's keyboard focus is not on that menu bar item (and Finder is usually not in front).628 let type = await invoke(AccessibilityTypeTextTool(), ["bundleID": "com.apple.finder", "element": .string(path), "text": "never typed"], in: dir)629 #expect(type.result.failure?.code == "focus_changed", "\(type.result.status) \(type.result.failure?.message ?? "")")630 verified("accessibility.tree/find/waitFor read Finder live (\(nodes.count) nodes); stale path, changed title and unfocused typing were refused")631 }632633 @Test func notRunningApplicationIsATypedFailure() async throws {634 guard AXIsProcessTrusted() else { skipped("accessibility not granted to this test process: app_not_running path not exercised"); return }635 let dir = try TempDirectory(); defer { dir.remove() }636 let o = await invoke(AccessibilityTreeTool(), ["bundleID": "com.example.absent.app"], in: dir)637 #expect(o.result.failure?.code == "app_not_running")638 }639}640641@Suite("Integration · notifications (no app bundle under swift test)")642struct NotificationIntegrationTests {643 @Test func postIsHonestlyUnsupportedWithoutABundle() async throws {644 let dir = try TempDirectory(); defer { dir.remove() }645 let o = await invoke(NotificationsPostTool(), ["title": "Run finished", "body": "3 files organised", "runID": "run-1", "approvalID": "ap-1"], in: dir)646 #expect(o.result.status == .unsupported)647 #expect(o.error == .unsupported(NotificationsPostTool.noBundleReason))648 #expect(o.result.failure?.message == "Notifications need an application bundle: this process has no bundle identifier.")649 #expect(o.requirements?.capabilities == [.notify])650 let bridge = NotificationCenterBridge { _ in }651 #expect(throws: ToolError.unsupported(NotificationsPostTool.noBundleReason)) { try bridge.register() }652 let empty = await invoke(NotificationsPostTool(), ["title": " ", "body": "x"], in: dir)653 #expect(empty.result.status == .invalidInput)654 }655}656657@Suite("Integration · InputRecorder (real NSWorkspace observers)")658@MainActor659struct InputRecorderIntegrationTests {660 @Test func startAndStopRecordTheirOwnLimits() async throws {661 let recorder = InputRecorder()662 #expect(recorder.state == .idle)663 #expect(throws: ToolError.self) { try recorder.start(targetBundleIDs: []) }664 #expect(throws: ToolError.self) { try recorder.start(targetBundleIDs: ["not a bundle id"]) }665 let seen = LockedBox<Int>(0)666 recorder.onAction = { _ in seen.mutate { $0 += 1 } }667 let limits = try recorder.start(targetBundleIDs: ["com.apple.finder"])668 #expect(recorder.state == .recording)669 #expect(recorder.targetBundleIDs == ["com.apple.finder"])670 #expect(!limits.isEmpty)671 #expect(limits.contains { $0.contains("Keystrokes") })672 if AXIsProcessTrusted() {673 #expect(!limits.contains { $0.contains("Accessibility is not enabled") })674 print("InputRecorder: accessibility granted, interface events of Finder are observed")675 } else {676 #expect(limits.contains { $0.contains("Accessibility is not enabled") && $0.contains("only application activations") })677 skipped("accessibility not granted to this test process: InputRecorder recorded app-level events and its limitation only")678 }679 #expect(throws: ToolError.self) { try recorder.start(targetBundleIDs: ["com.apple.finder"]) }680681 recorder.pause()682 #expect(recorder.state == .paused)683 recorder.resume()684 #expect(recorder.state == .recording)685 let actions = recorder.stop()686 #expect(recorder.state == .stopped)687 #expect(actions.first?.kind == .recordingStarted && actions.last?.kind == .recordingStopped)688 #expect(actions.contains { $0.kind == .recordingPaused } && actions.contains { $0.kind == .recordingResumed })689 #expect(actions.filter { $0.kind == .limitation }.count >= limits.count)690 #expect(actions.map(\.sequence) == Array(1...actions.count))691 #expect(zip(actions, actions.dropFirst()).allSatisfy { $0.timestamp <= $1.timestamp })692 #expect(seen.get() == actions.count)693 // Only the chosen application appears, and nothing carries a value.694 #expect(actions.allSatisfy { $0.bundleID == nil || $0.bundleID == "com.apple.finder" })695 #expect(actions.allSatisfy { $0.value == nil })696697 let decoded = try { () -> [RecordedAction] in698 let d = JSONDecoder(); d.dateDecodingStrategy = .iso8601699 return try d.decode([RecordedAction].self, from: try recorder.exportJSON())700 }()701 #expect(decoded.map(\.id) == actions.map(\.id) && decoded.map(\.kind) == actions.map(\.kind))702 #expect(recorder.stop().count == actions.count)703 // A stopped recorder can start a fresh recording.704 _ = try recorder.start(targetBundleIDs: ["com.apple.finder"])705 #expect(recorder.actions.first?.kind == .recordingStarted && recorder.actions.count < actions.count + 1)706 recorder.stop()707 }708709 @Test func activationsOfOtherApplicationsAreIgnored() async throws {710 let recorder = InputRecorder(options: .init(maxActions: 6))711 _ = try recorder.start(targetBundleIDs: ["com.apple.finder"])712 // Real notifications posted on the workspace centre: one for an application outside the targets, one for Finder.713 let center = NSWorkspace.shared.notificationCenter714 let finder = try #require(AppResolver.running("com.apple.finder").first)715 if let other = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier != nil && $0.bundleIdentifier != "com.apple.finder" }) {716 center.post(name: NSWorkspace.didActivateApplicationNotification, object: NSWorkspace.shared, userInfo: [NSWorkspace.applicationUserInfoKey: other])717 }718 center.post(name: NSWorkspace.didActivateApplicationNotification, object: NSWorkspace.shared, userInfo: [NSWorkspace.applicationUserInfoKey: finder])719 recorder.pause()720 center.post(name: NSWorkspace.didActivateApplicationNotification, object: NSWorkspace.shared, userInfo: [NSWorkspace.applicationUserInfoKey: finder])721 recorder.resume()722 let activations = recorder.actions.filter { $0.kind == .appActivated }723 #expect(activations.count == 1)724 #expect(activations.first?.bundleID == "com.apple.finder")725 // The action cap is enforced and recorded as a limitation.726 for _ in 0..<10 {727 center.post(name: NSWorkspace.didActivateApplicationNotification, object: NSWorkspace.shared, userInfo: [NSWorkspace.applicationUserInfoKey: finder])728 }729 let actions = recorder.stop()730 #expect(actions.count <= 8 && actions.last?.kind == .recordingStopped)731 #expect(recorder.limitations.contains { $0.contains("reached its limit") })732 }733}734