SPB Git

Repository Public License

BrowserToolsIntegrationTests.swift 44.6 KB 652 lines swift

1// ORIX OS Automation2// File: Packages/ORIXBrowserAutomation/Tests/ORIXBrowserAutomationTests/BrowserToolsIntegrationTests.swift3// Description: Integration tests of the browser.* tools: a real WKWebView driving the loopback test site (structure, data entry, secrets, navigation policy, dialogs, waits, screenshots, downloads).4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import AppKit8import Foundation9import ORIXCore10import ORIXTestingSupport11import Testing12@testable import ORIXBrowserAutomation1314struct MapSecrets: SecretResolving {15    let values: [String: String]16    func secret(reference: String) async throws -> String {17        guard let v = values[reference] else { throw ToolError.notFound("No secret named \(reference).") }18        return v19    }20}2122/// The site, a session manager, the tools and a miniature of the executor's pipeline (schema validation, capability23/// check against a fixed grant list, typed-error conversion, output-schema validation).24final class BrowserHarness: @unchecked Sendable {25    static let password = "S3cure-Horse-Battery-42"26    static let card = "4111111111111111"2728    let site: TestSite29    let dir: TempDirectory30    let manager: BrowserSessionManager31    let tools: [String: any ORIXTool]32    var grants: [Capability]33    var secrets: (any SecretResolving)? = MapSecrets(values: ["site/password": BrowserHarness.password, "site/card": BrowserHarness.card])34    private(set) var lastRequirements: ToolRequirements?3536    init() async throws {37        site = try TestSite()38        try await site.start()39        dir = try TempDirectory(prefix: "orix-browser")40        manager = BrowserSessionManager()41        tools = Dictionary(ORIXBrowserAutomationTools.all(manager: manager).map { ($0.identifier, $0) }, uniquingKeysWith: { a, _ in a })42        grants = [.browser(host: "127.0.0.1"), .secret(reference: "site/password"), .secret(reference: "site/card")]43    }4445    func tearDown() async {46        await manager.closeAll()47        site.stop()48        dir.remove()49    }5051    func context(runID: String = "run-1", timeout: Double = 20) -> ToolExecutionContext {52        let hosts = grants.compactMap { cap -> String? in if case .browser(let h) = cap { return h }; return nil }53        return ToolExecutionContext(runID: runID, callID: "call-\(UUID().uuidString.prefix(8))", mode: .guided, workingDirectory: dir.url,54                                    artifactsDirectory: dir.url.appendingPathComponent("artifacts"), timeoutSeconds: timeout,55                                    confinement: Confinement(browserHosts: hosts), secrets: secrets)56    }5758    @discardableResult59    func call(_ id: String, _ arguments: JSONValue = [:], runID: String = "run-1", timeout: Double = 20) async -> ToolResult {60        guard let tool = tools[id] else { return .failure(.failed, ToolFailure(code: "not_found", message: "Unknown tool \(id)")) }61        let violations = tool.manifest.inputSchema.validate(arguments)62        guard violations.isEmpty else {63            return .failure(.invalidInput, ToolFailure(code: "invalid_input", message: violations.map(\.description).joined(separator: "; ")))64        }65        let input = ToolInput(arguments: arguments)66        let ctx = context(runID: runID, timeout: timeout)67        do {68            let requirements = try tool.requirements(for: input, context: ctx)69            lastRequirements = requirements70            let artifacts = try PathResolver.canonical(ctx.artifactsDirectory.path, relativeTo: ctx.workingDirectory)71            for capability in requirements.capabilities {72                if case .fileWrite(let path) = capability, path.hasPrefix(artifacts + "/") { continue }73                guard grants.contains(where: { $0.covers(capability) }) else {74                    return .failure(.denied, ToolFailure(code: "permission_denied", message: "No grant covers \(capability.displayString)"))75                }76            }77            let result = try await tool.execute(input: input, context: ctx)78            if result.status == .succeeded {79                let problems = tool.manifest.outputSchema.validate(result.output)80                if !problems.isEmpty {81                    return .failure(.failed, ToolFailure(code: "output_invalid", message: problems.map(\.description).joined(separator: "; ")))82                }83            }84            return result85        } catch let error as ToolError {86            return ToolResult(status: error.status, summary: error.failure.message, failure: error.failure)87        } catch {88            return .failure(.failed, ToolFailure(code: "internal", message: "\(error)"))89        }90    }9192    /// Runs a snippet in the page (test-only inspection of real DOM state).93    @MainActor94    func pageValue(_ expression: String, runID: String = "run-1") async throws -> String {95        guard let session = await manager.session(runID: runID) else { return "" }96        // In the page's own world (not ORIX's isolated one), so the page's globals are visible.97        let r = try await session.webView.callAsyncJavaScript("return String(\(expression));", arguments: [:], in: nil, contentWorld: .page)98        return r as? String ?? ""99    }100}101102func withBrowser(_ body: (BrowserHarness) async throws -> Void) async throws {103    let harness = try await BrowserHarness()104    do { try await body(harness) } catch {105        await harness.tearDown()106        throw error107    }108    await harness.tearDown()109}110111@Suite("Browser tools (integration: real WKWebView against a loopback test site)", .serialized)112struct BrowserToolsIntegrationTests {113    @Test func openReportsFinalURLStatusAndTitle() async throws {114        try await withBrowser { (h: BrowserHarness) async throws -> Void in115            let opened = await h.call("browser.open", ["url": .string(h.site.origin + "/")])116            #expect(opened.status == .succeeded, "\(opened.failure?.message ?? "")")117            #expect(opened.output["url"]?.stringValue == h.site.origin + "/")118            #expect(opened.output["status"]?.intValue == 200)119            #expect(opened.output["title"]?.stringValue == "ORIX Test Site")120            #expect(opened.output["redirected"]?.boolValue == false)121            #expect(opened.untrusted)122            #expect(h.lastRequirements?.capabilities == [.browser(host: "127.0.0.1")] && h.lastRequirements?.tier == .reversibleWrite)123            #expect(h.manager.currentHost(runID: "run-1") == "127.0.0.1")124125            // A same-host server redirect is followed and reported.126            let redirected = await h.call("browser.open", ["url": .string(h.site.origin + "/redirect-home")])127            #expect(redirected.output["url"]?.stringValue == h.site.origin + "/" && redirected.output["redirected"]?.boolValue == true)128129            // An HTTP error page is still a page: the status says so.130            let missing = await h.call("browser.open", ["url": .string(h.site.origin + "/missing")])131            #expect(missing.status == .succeeded && missing.output["status"]?.intValue == 404)132133            // Nothing listening: a typed, retryable network failure.134            h.grants.append(.browser(host: "localhost"))135            let refused = await h.call("browser.open", ["url": "http://localhost:1/"])136            #expect(refused.status == .failed && refused.failure?.code == "network" && refused.failure?.retryable == true)137        }138    }139140    @Test func readReturnsTheStructureOfThePage() async throws {141        try await withBrowser { (h: BrowserHarness) async throws -> Void in142            await h.call("browser.open", ["url": .string(h.site.origin + "/")])143            let read = await h.call("browser.read")144            #expect(read.status == .succeeded, "\(read.failure?.message ?? "")")145            #expect(h.lastRequirements?.capabilities == [.browser(host: "127.0.0.1")] && h.lastRequirements?.tier == .readOnly)146            let page = read.output147            #expect(page["title"]?.stringValue == "ORIX Test Site" && page["url"]?.stringValue == h.site.origin + "/")148            let text = page["text"]?.stringValue ?? ""149            #expect(text.contains("Plain visible paragraph.") && !text.contains("hidden-text-should-not-be-visible"))150            #expect(page["headings"] == [["level": 1, "text": "Welcome to the test site"], ["level": 2, "text": "Navigation"], ["level": 2, "text": "Inventory"]])151152            let links = page["links"]?.arrayValue ?? []153            #expect(links.count == 7)154            let form = try #require(links.first { $0["text"]?.stringValue == "Data entry form" })155            #expect(form["href"]?.stringValue == h.site.origin + "/form" && form["selector"]?.stringValue == "#to-form")156            // A link without an id still gets a selector that resolves to exactly that element.157            let slow = try #require(links.first { $0["text"]?.stringValue == "Slow page" })158            let slowSelector = try #require(slow["selector"]?.stringValue)159            #expect(try await h.pageValue("document.querySelectorAll(\(JSONValue.string(slowSelector).serialized())).length") == "1")160            #expect(try await h.pageValue("document.querySelector(\(JSONValue.string(slowSelector).serialized())).textContent") == "Slow page")161162            let table = try #require(page["tables"]?.arrayValue?.first)163            #expect(table["selector"]?.stringValue == "#inventory" && table["caption"]?.stringValue == "Stock")164            #expect(table["headers"] == ["Item", "Quantity", "Price"])165            #expect(table["rows"] == [["Widget", "12", "4.50"], ["Gadget", "3", "19.99"]])166            #expect(table["rowCount"]?.intValue == 2)167168            let buttons = page["buttons"]?.arrayValue ?? []169            #expect(buttons.map { $0["selector"]?.stringValue } == ["#ghost", "#locked", "#counter"])170            #expect(buttons[0]["visible"]?.boolValue == false && buttons[1]["disabled"]?.boolValue == true && buttons[2]["text"]?.stringValue == "Count")171            #expect(page["forms"]?.arrayValue?.isEmpty == true && page["truncated"]?.boolValue == false)172        }173    }174175    @Test func promptInjectionOnThePageArrivesFlaggedUntrusted() async throws {176        try await withBrowser { (h: BrowserHarness) async throws -> Void in177            await h.call("browser.open", ["url": .string(h.site.origin + "/")])178            let read = await h.call("browser.read")179            // The hostile text is delivered as data, in a result marked untrusted by both the result and the manifest.180            #expect(read.output["text"]?.stringValue?.contains(TestSite.injection) == true)181            #expect(read.untrusted)182            #expect(h.tools["browser.read"]?.manifest.outputIsUntrusted == true)183            // Reading it changed nothing: no request left for the host the text names, and no file was read.184            #expect(h.site.requests.allSatisfy { $0.path != "/collect" })185            #expect(h.manager.currentHost(runID: "run-1") == "127.0.0.1")186        }187    }188189    @Test func readIsBoundedAndSaysWhenItTruncates() async throws {190        try await withBrowser { (h: BrowserHarness) async throws -> Void in191            await h.call("browser.open", ["url": .string(h.site.origin + "/")])192            let small = await h.call("browser.read", ["maxBytes": 1000, "maxItems": 2])193            #expect(small.status == .succeeded && small.truncated && small.output["truncated"]?.boolValue == true)194            #expect(small.output["links"]?.arrayValue?.count == 2 && small.output["buttons"]?.arrayValue?.count == 2)195            #expect(await h.call("browser.read", ["maxBytes": 10]).status == .invalidInput)196        }197    }198199    @Test func multiStepDataEntrySubmitsAndShowsTheValues() async throws {200        try await withBrowser { (h: BrowserHarness) async throws -> Void in201            await h.call("browser.open", ["url": .string(h.site.origin + "/form")])202            let before = await h.call("browser.read")203            let form = try #require(before.output["forms"]?.arrayValue?.first)204            #expect(form["selector"]?.stringValue == "#entry" && form["method"]?.stringValue == "post")205            #expect(form["action"]?.stringValue == h.site.origin + "/submit")206            let fields = form["fields"]?.arrayValue ?? []207            #expect(fields.map { $0["name"]?.stringValue } == ["fullname", "color", "subscribe", "plan", "plan", "notes", "password", "card", "csrf", "frozen"])208            let name = fields[0]209            #expect(name["label"]?.stringValue == "Full name" && name["placeholder"]?.stringValue == "Ada Lovelace" && name["type"]?.stringValue == "text")210            #expect(name["required"]?.boolValue == true && name["value"]?.stringValue == "")211            #expect(fields[1]["options"]?.arrayValue?.map { $0["text"]?.stringValue } == ["Red", "Green", "Blue"] && fields[1]["value"]?.stringValue == "r")212            #expect(fields[2]["checked"]?.boolValue == false && fields[3]["checked"]?.boolValue == true)213            #expect(before.output["fields"]?.arrayValue?.first?["label"]?.stringValue == "Search the site")214            let proRadio = try #require(fields[4]["selector"]?.stringValue)215216            // Fill every kind of control, using the selectors browser.read returned.217            let filled = await h.call("browser.fill", ["selector": name["selector"] ?? "", "value": "Ada Lovelace"])218            #expect(filled.status == .succeeded && filled.output["valueLength"]?.intValue == 12 && filled.output["secure"]?.boolValue == false)219            #expect(h.lastRequirements?.tier == .consequential && h.lastRequirements?.capabilities == [.browser(host: "127.0.0.1")])220            #expect(await h.call("browser.fill", ["selector": "#color", "value": "Blue"]).status == .succeeded)221            #expect(await h.call("browser.fill", ["selector": "#subscribe", "checked": true]).output["checked"]?.boolValue == true)222            #expect(await h.call("browser.fill", ["selector": .string(proRadio), "checked": true]).status == .succeeded)223            #expect(await h.call("browser.fill", ["selector": "#notes", "value": "Line one\nLine <two> & more"]).status == .succeeded)224            let secret = await h.call("browser.fill", ["selector": "#pw", "secretRef": "site/password"])225            #expect(secret.status == .succeeded && secret.output["secure"]?.boolValue == true)226            #expect(h.lastRequirements?.capabilities == [.browser(host: "127.0.0.1"), .secret(reference: "site/password")])227            #expect(await h.call("browser.fill", ["selector": "#card", "secretRef": "site/card"]).status == .succeeded)228229            // The page's own listeners saw real input and change events.230            #expect(try await h.pageValue("window.events.join('|')") == "input:Ada Lovelace")231            #expect(try await h.pageValue("document.title") == "changed:Ada Lovelace")232233            let click = await h.call("browser.click", ["selector": "#send"])234            #expect(click.status == .succeeded, "\(click.failure?.message ?? "")")235            #expect(click.output["navigated"]?.boolValue == true && click.output["url"]?.stringValue == h.site.origin + "/submit")236            #expect(click.output["status"]?.intValue == 200 && click.output["title"]?.stringValue == "Submission received")237            #expect(click.output["element"]?["text"]?.stringValue == "Send")238239            let waited = await h.call("browser.waitFor", ["selector": "#result", "state": "visible", "text": "Submission received", "urlMatches": "/submit$"])240            #expect(waited.status == .succeeded && waited.output["matched"]?.boolValue == true)241242            let after = await h.call("browser.read")243            let text = after.output["text"]?.stringValue ?? ""244            for expected in ["Ada Lovelace", "Line one Line <two> & more", "received \(BrowserHarness.password.count) characters", "received 16 digits"] {245                #expect(text.contains(expected), "result page lacks “\(expected)”: \(text)")246            }247            #expect(try await h.pageValue("document.getElementById('r-color').textContent") == "b")248            #expect(try await h.pageValue("document.getElementById('r-subscribe').textContent") == "yes")249            #expect(try await h.pageValue("document.getElementById('r-plan').textContent") == "pro")250251            // What the server actually received, including the secret that never appeared in any tool output.252            let posted = try #require(h.site.requests(to: "/submit").last?.form)253            #expect(posted["fullname"] == "Ada Lovelace" && posted["color"] == "b" && posted["subscribe"] == "yes" && posted["plan"] == "pro")254            #expect(posted["notes"] == "Line one\r\nLine <two> & more" && posted["csrf"] == "t0k3n" && posted["frozen"] == "fixed")255            #expect(posted["password"] == BrowserHarness.password && posted["card"] == BrowserHarness.card)256            for result in [secret, click, after] { #expect(!result.output.serialized().contains(BrowserHarness.password)) }257        }258    }259260    @Test func passwordAndCardValuesNeverAppearInReadOutput() async throws {261        try await withBrowser { (h: BrowserHarness) async throws -> Void in262            await h.call("browser.open", ["url": .string(h.site.origin + "/form")])263            let fill = await h.call("browser.fill", ["selector": "#pw", "secretRef": "site/password"])264            await h.call("browser.fill", ["selector": "#card", "secretRef": "site/card"])265            // A secret typed into an ordinary field is masked from then on as well.266            await h.call("browser.fill", ["selector": "#notes", "secretRef": "site/password"])267            #expect(try await h.pageValue("document.getElementById('pw').value") == BrowserHarness.password)268269            let read = await h.call("browser.read")270            let serialized = read.output.serialized() + fill.output.serialized() + fill.summary + read.summary271            #expect(!serialized.contains(BrowserHarness.password) && !serialized.contains(BrowserHarness.card))272            let fields = read.output["forms"]?.arrayValue?.first?["fields"]?.arrayValue ?? []273            let byName = Dictionary(fields.map { ($0["name"]?.stringValue ?? "", $0) }, uniquingKeysWith: { a, _ in a })274            for name in ["password", "card", "notes"] {275                #expect(byName[name]?["value"]?.stringValue == "«secure»" && byName[name]?["secure"]?.boolValue == true, "\(name)")276            }277            #expect(byName["fullname"]?["secure"]?.boolValue == false && byName["csrf"]?["value"]?.stringValue == "t0k3n")278            #expect(fill.output["valueLength"]?.isNull == true)279280            // Even after the page turns the password field into a text field ("show password").281            _ = try await h.pageValue("document.getElementById('pw').type = 'text'")282            let again = await h.call("browser.read")283            #expect(!again.output.serialized().contains(BrowserHarness.password))284        }285    }286287    @Test func fillingASecureFieldWithoutSecretRefIsRefused() async throws {288        try await withBrowser { (h: BrowserHarness) async throws -> Void in289            await h.call("browser.open", ["url": .string(h.site.origin + "/form")])290            for selector in ["#pw", "#card"] {291                let refused = await h.call("browser.fill", ["selector": .string(selector), "value": "typed-by-a-model"])292                #expect(refused.status == .denied && refused.failure?.code == "secret_required", "\(selector): \(refused.status)")293                #expect(refused.failure?.recovery?.contains("secretRef") == true)294            }295            let typed = try await h.pageValue("document.getElementById('pw').value + document.getElementById('card').value")296            #expect(typed.isEmpty)297298            // A secret the run was not granted is stopped by the capability check, before the tool runs.299            let notGranted = await h.call("browser.fill", ["selector": "#pw", "secretRef": "other/secret"])300            #expect(notGranted.status == .denied && notGranted.failure?.code == "permission_denied")301            // A granted reference that does not exist, and a run with no secret store at all: typed failures, field untouched.302            h.grants.append(.secret(reference: "other/secret"))303            #expect(await h.call("browser.fill", ["selector": "#pw", "secretRef": "other/secret"]).failure?.code == "not_found")304            h.secrets = nil305            #expect(await h.call("browser.fill", ["selector": "#pw", "secretRef": "site/password"]).status == .denied)306            let stillEmpty = try await h.pageValue("document.getElementById('pw').value")307            #expect(stillEmpty.isEmpty)308309            // Argument and element errors.310            #expect(await h.call("browser.fill", ["selector": "#fullname"]).status == .invalidInput)311            #expect(await h.call("browser.fill", ["selector": "#fullname", "value": "a", "checked": true]).status == .invalidInput)312            #expect(await h.call("browser.fill", ["selector": "#nope", "value": "a"]).failure?.code == "not_found")313            #expect(await h.call("browser.fill", ["selector": "((", "value": "a"]).status == .invalidInput)314            #expect(await h.call("browser.fill", ["selector": "#color", "value": "Purple"]).status == .invalidInput)315            #expect(await h.call("browser.fill", ["selector": "#frozen", "value": "x"]).failure?.code == "not_interactable")316            #expect(await h.call("browser.fill", ["selector": "input[name=\"csrf\"]", "value": "x"]).status == .unsupported)317            #expect(await h.call("browser.fill", ["selector": "#send", "value": "x"]).status == .unsupported)318        }319    }320321    @Test func openingANonGrantedHostIsBlocked() async throws {322        try await withBrowser { (h: BrowserHarness) async throws -> Void in323            // `localhost` and `127.0.0.1` reach the same server but are two different hosts for the policy.324            let direct = await h.call("browser.open", ["url": .string(h.site.foreignOrigin + "/")])325            #expect(direct.status == .denied)326            #expect(h.lastRequirements?.capabilities == [.browser(host: "localhost")])327            #expect(h.site.requests.isEmpty)328329            // A granted page that answers with a server redirect to the other host.330            let redirect = await h.call("browser.open", ["url": .string(h.site.origin + "/redirect-foreign")])331            #expect(redirect.status == .denied && redirect.failure?.code == "navigation_blocked")332            #expect(redirect.failure?.message.contains("localhost") == true)333            #expect(redirect.output["blocked"]?["host"]?.stringValue == "localhost" && redirect.output["blocked"]?["reason"]?.stringValue == "host_not_allowed")334335            // A granted page whose script sends the browser to the other host.336            let script = await h.call("browser.open", ["url": .string(h.site.origin + "/js-redirect")])337            #expect(script.status == .denied && script.failure?.code == "navigation_blocked" && script.failure?.message.contains("localhost") == true)338            #expect(h.manager.currentURL(runID: "run-1")?.absoluteString == h.site.origin + "/js-redirect")339340            // The server never saw a request addressed to the foreign host.341            #expect(h.site.requests.allSatisfy { ($0.headers["host"] ?? "").hasPrefix("127.0.0.1") })342            #expect(h.site.requests(to: "/").isEmpty)343344            // Granting the host makes the same navigation work.345            h.grants.append(.browser(host: "localhost"))346            let allowed = await h.call("browser.open", ["url": .string(h.site.origin + "/redirect-foreign")])347            #expect(allowed.status == .succeeded && allowed.output["url"]?.stringValue == h.site.foreignOrigin + "/")348        }349    }350351    @Test func clickingALinkToANonGrantedHostIsBlocked() async throws {352        try await withBrowser { (h: BrowserHarness) async throws -> Void in353            await h.call("browser.open", ["url": .string(h.site.origin + "/")])354            let before = h.site.requests.count355            for selector in ["#foreign-link", "#foreign-window"] {356                let click = await h.call("browser.click", ["selector": .string(selector)])357                #expect(click.status == .denied && click.failure?.code == "navigation_blocked", "\(selector): \(click.status) \(click.failure?.message ?? "")")358                #expect(click.failure?.message.contains("localhost") == true && click.output["clicked"]?.boolValue == true)359            }360            let local = await h.call("browser.click", ["selector": "#file-link"])361            // WebKit itself refuses to follow a file: link from a web page; when it does ask, the policy refuses.362            #expect(local.status == .denied || local.output["navigated"]?.boolValue == false, "\(local.status) \(local.output.serialized())")363            // Still on the granted page, and nothing was requested.364            #expect(h.manager.currentURL(runID: "run-1")?.absoluteString == h.site.origin + "/")365            #expect(h.site.requests.count == before)366367            // A form that posts to the other host: the submission is stopped before it is sent.368            await h.call("browser.open", ["url": .string(h.site.origin + "/form-foreign")])369            let post = await h.call("browser.click", ["selector": "#send"])370            #expect(post.status == .denied && post.failure?.code == "navigation_blocked" && post.failure?.message.contains("localhost") == true)371            #expect(h.site.requests(to: "/submit").isEmpty)372            await h.call("browser.open", ["url": .string(h.site.origin + "/")])373374            // A new-window link to a granted host loads in the same view.375            let sameView = await h.call("browser.click", ["selector": "#new-window"])376            #expect(sameView.status == .succeeded && sameView.output["url"]?.stringValue == h.site.origin + "/form")377            #expect(await h.manager.activeRunIDs == ["run-1"])378        }379    }380381    @Test func nonWebSchemesAreRefused() async throws {382        try await withBrowser { (h: BrowserHarness) async throws -> Void in383            h.grants.append(.browser(host: "example.com"))384            for url in ["file:///etc/hosts", "file://localhost/etc/passwd", "data:text/html,<h1>x</h1>", "javascript:alert(1)", "ftp://example.com/x", "http://example.com/"] {385                let result = await h.call("browser.open", ["url": .string(url)])386                #expect(result.status == .unsupported && result.failure?.code == "unsupported", "\(url): \(result.status)")387            }388            #expect(await h.call("browser.open", ["url": "file:///etc/hosts"]).failure?.message.contains("file:") == true)389            for url in ["not a url", "/relative/path", "https://user:pw@example.com/", "https:///nohost"] {390                #expect(await h.call("browser.open", ["url": .string(url)]).status == .invalidInput, "\(url)")391            }392            #expect(await h.call("browser.download", ["url": "file:///etc/hosts"]).status == .unsupported)393            #expect(await h.manager.activeRunIDs.isEmpty)394            #expect(h.site.requests.isEmpty)395        }396    }397398    @Test func javaScriptDialogsAreDismissedAndReported() async throws {399        try await withBrowser { (h: BrowserHarness) async throws -> Void in400            let start = Date()401            let opened = await h.call("browser.open", ["url": .string(h.site.origin + "/alert")])402            #expect(opened.status == .succeeded, "\(opened.failure?.message ?? "")")403            #expect(Date().timeIntervalSince(start) < 10)404            #expect(opened.output["dialogs"] == [405                ["kind": "alert", "message": "hello from the page", "response": "dismissed"],406                ["kind": "confirm", "message": "delete everything?", "response": "canceled"],407                ["kind": "prompt", "message": "your name?", "response": "canceled"],408            ])409            // The script ran to the end with the refusals as answers.410            #expect(opened.output["title"]?.stringValue == "confirm=false prompt=null")411            #expect(await h.call("browser.read").output["text"]?.stringValue?.contains("script finished") == true)412        }413    }414415    @Test func waitForObservesTheDOMAndTimesOutWithATypedFailure() async throws {416        try await withBrowser { (h: BrowserHarness) async throws -> Void in417            await h.call("browser.open", ["url": .string(h.site.origin + "/slow")])418            #expect(try await h.pageValue("!!document.getElementById('late')") == "false")419            let late = await h.call("browser.waitFor", ["selector": "#late", "timeoutSeconds": 10])420            #expect(late.status == .succeeded && late.output["matched"]?.boolValue == true, "\(late.failure?.message ?? "")")421            #expect((late.output["elapsedMs"]?.intValue ?? 0) < 9000)422            #expect(h.lastRequirements?.tier == .readOnly)423            #expect(await h.call("browser.waitFor", ["selector": "#spinner", "state": "absent", "text": "Late content has arrived"]).status == .succeeded)424            #expect(await h.call("browser.waitFor", ["urlMatches": "stage=done$", "timeoutSeconds": 10]).status == .succeeded)425426            let start = Date()427            let never = await h.call("browser.waitFor", ["selector": "#never", "timeoutSeconds": 0.6])428            let elapsed = Date().timeIntervalSince(start)429            #expect(never.status == .timedOut && never.failure?.code == "timeout" && never.failure?.retryable == true)430            #expect(never.failure?.message.contains("#never") == true)431            #expect(elapsed >= 0.55 && elapsed < 5, "elapsed \(elapsed)")432            #expect(await h.call("browser.waitFor", ["text": "text that never shows up", "timeoutSeconds": 0.3]).status == .timedOut)433            #expect(await h.call("browser.waitFor", ["urlMatches": "/elsewhere", "timeoutSeconds": 0.3]).status == .timedOut)434435            #expect(await h.call("browser.waitFor", [:]).status == .invalidInput)436            #expect(await h.call("browser.waitFor", ["state": "absent"]).status == .invalidInput)437            #expect(await h.call("browser.waitFor", ["urlMatches": "("]).status == .invalidInput)438            #expect(await h.call("browser.waitFor", ["selector": "((", "timeoutSeconds": 1]).status == .invalidInput)439            #expect(await h.call("browser.waitFor", ["selector": "#late", "state": "sideways"]).status == .invalidInput)440        }441    }442443    @Test func clickVerifiesTheElementAndReportsWhatFollowed() async throws {444        try await withBrowser { (h: BrowserHarness) async throws -> Void in445            await h.call("browser.open", ["url": .string(h.site.origin + "/")])446            let ghost = await h.call("browser.click", ["selector": "#ghost"])447            #expect(ghost.status == .failed && ghost.failure?.code == "not_interactable" && ghost.failure?.message.contains("not visible") == true)448            let locked = await h.call("browser.click", ["selector": "#locked"])449            #expect(locked.failure?.code == "not_interactable" && locked.failure?.message.contains("disabled") == true)450            #expect(await h.call("browser.click", ["selector": "#nope"]).failure?.code == "not_found")451            #expect(await h.call("browser.click", ["selector": "(("]).status == .invalidInput)452            #expect(await h.call("browser.click", ["selector": ""]).status == .invalidInput)453454            let counted = await h.call("browser.click", ["selector": "#counter"])455            #expect(counted.status == .succeeded && counted.output["navigated"]?.boolValue == false && counted.output["status"]?.isNull == true)456            await h.call("browser.click", ["selector": "#counter"])457            #expect(try await h.pageValue("document.getElementById('counter').textContent") == "Clicked 2")458459            let link = await h.call("browser.click", ["selector": "#to-form"])460            #expect(link.output["navigated"]?.boolValue == true && link.output["url"]?.stringValue == h.site.origin + "/form")461            #expect(link.output["element"]?["href"]?.stringValue == h.site.origin + "/form" && link.untrusted)462        }463    }464465    @Test func screenshotWritesARealPNGOfTheViewport() async throws {466        try await withBrowser { (h: BrowserHarness) async throws -> Void in467            await h.call("browser.open", ["url": .string(h.site.origin + "/")])468            let shot = await h.call("browser.screenshot")469            #expect(shot.status == .succeeded, "\(shot.failure?.message ?? "")")470            #expect(h.lastRequirements?.tier == .readOnly)471            let path = try #require(shot.output["path"]?.stringValue)472            #expect(path.hasPrefix(h.dir.url.path + "/artifacts/screenshots/") && path.hasSuffix(".png"))473            let data = try Data(contentsOf: URL(fileURLWithPath: path))474            #expect(data.count > 2000 && data.count == shot.output["bytes"]?.intValue)475            #expect(Array(data.prefix(8)) == [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A])476            // IHDR: width and height as big-endian 32-bit integers at offsets 16 and 20.477            let width = data[16..<20].reduce(0) { $0 << 8 | Int($1) }, height = data[20..<24].reduce(0) { $0 << 8 | Int($1) }478            #expect(width == 1280 && height == 800)479            #expect(shot.output["width"]?.intValue == 1280 && shot.output["height"]?.intValue == 800)480            #expect(shot.artifacts.first?.kind == "screenshot" && shot.artifacts.first?.path == path)481            #expect(shot.artifacts.first?.sha256 == shot.output["sha256"]?.stringValue && shot.output["sha256"]?.stringValue?.count == 64)482483            // The page was really rendered: dark text on a white background, not a blank image.484            let rep = try #require(NSBitmapImageRep(data: data))485            var dark = 0, light = 0486            for y in stride(from: 0, to: rep.pixelsHigh, by: 2) {487                for x in stride(from: 0, to: rep.pixelsWide, by: 2) {488                    guard let c = rep.colorAt(x: x, y: y)?.usingColorSpace(.deviceRGB) else { continue }489                    let luminance = (c.redComponent + c.greenComponent + c.blueComponent) / 3490                    if luminance < 0.4 { dark += 1 } else if luminance > 0.9 { light += 1 }491                }492            }493            #expect(dark > 200 && light > 50_000, "dark \(dark) light \(light)")494495            // An explicit path outside the artifacts folder needs a file-write grant and is a tier 1 write.496            let outside = h.dir.file("shots/page.png").path497            #expect(await h.call("browser.screenshot", ["path": .string(outside)]).status == .denied)498            #expect(h.lastRequirements?.tier == .reversibleWrite)499            #expect(h.lastRequirements?.capabilities == [.browser(host: "127.0.0.1"), .fileWrite(path: outside)])500            h.grants.append(.fileWrite(path: h.dir.url.path))501            #expect(await h.call("browser.screenshot", ["path": "shots/page.png"]).output["path"]?.stringValue == outside)502            #expect(await h.call("browser.screenshot", ["path": "shots/page.png"]).failure?.code == "io")503            #expect(await h.call("browser.screenshot", ["path": "shots/page.png", "overwrite": true]).status == .succeeded)504            #expect(await h.call("browser.screenshot", ["path": "shots/page.jpg"]).status == .invalidInput)505        }506    }507508    @Test func downloadSavesTheFileWithItsChecksum() async throws {509        try await withBrowser { (h: BrowserHarness) async throws -> Void in510            let result = await h.call("browser.download", ["url": .string(h.site.origin + "/file.bin")])511            #expect(result.status == .succeeded, "\(result.failure?.message ?? "")")512            #expect(h.lastRequirements?.tier == .consequential)513            let path = h.dir.url.path + "/artifacts/downloads/file.bin"514            #expect(h.lastRequirements?.capabilities == [.browser(host: "127.0.0.1"), .fileWrite(path: path)])515            #expect(result.output["path"]?.stringValue == path && result.output["bytes"]?.intValue == 65_536)516            #expect(result.output["sha256"]?.stringValue == TestSite.fileSHA256)517            #expect(result.output["status"]?.intValue == 200 && result.output["mimeType"]?.stringValue == "application/octet-stream")518            #expect(try Data(contentsOf: URL(fileURLWithPath: path)) == TestSite.fileBytes)519            #expect(result.artifacts.first?.sha256 == TestSite.fileSHA256 && result.artifacts.first?.bytes == 65_536 && result.artifacts.first?.kind == "file")520521            // Existing file: refused unless overwrite is asked for.522            #expect(await h.call("browser.download", ["url": .string(h.site.origin + "/file.bin")]).failure?.code == "io")523            #expect(await h.call("browser.download", ["url": .string(h.site.origin + "/file.bin"), "overwrite": true]).status == .succeeded)524525            // Size cap, HTTP errors and foreign redirects leave nothing behind.526            let folder = h.dir.url.appendingPathComponent("artifacts/downloads")527            let capped = await h.call("browser.download", ["url": .string(h.site.origin + "/file.bin"), "path": "artifacts/downloads/capped.bin", "maxBytes": 1000])528            #expect(capped.status == .failed && capped.failure?.code == "too_large")529            let missing = await h.call("browser.download", ["url": .string(h.site.origin + "/missing")])530            #expect(missing.status == .failed && missing.failure?.code == "http_error" && missing.failure?.message.contains("404") == true)531            let foreign = await h.call("browser.download", ["url": .string(h.site.origin + "/file-redirect-foreign")])532            #expect(foreign.status == .denied && foreign.failure?.code == "navigation_blocked" && foreign.failure?.message.contains("localhost") == true,533                    "\(foreign.status) \(foreign.failure?.code ?? "") \(foreign.failure?.message ?? "") \(foreign.output.serialized())")534            let left = try FileManager.default.contentsOfDirectory(atPath: folder.path)535            #expect(left == ["file.bin"])536            // The redirect was refused before any request reached the foreign host.537            #expect(h.site.requests.allSatisfy { ($0.headers["host"] ?? "").hasPrefix("127.0.0.1") })538539            // A destination outside the artifacts folder needs a file-write grant.540            #expect(await h.call("browser.download", ["url": .string(h.site.origin + "/file.bin"), "path": "elsewhere/f.bin"]).status == .denied)541        }542    }543544    @Test func downloadsUseTheSessionCookiesAndPagesCannotStartThem() async throws {545        try await withBrowser { (h: BrowserHarness) async throws -> Void in546            // Without the session cookie the server refuses.547            #expect(await h.call("browser.download", ["url": .string(h.site.origin + "/private.bin")], runID: "anonymous").failure?.code == "http_error")548            // Logged-in session: the cookie set by the page is sent with the download.549            await h.call("browser.open", ["url": .string(h.site.origin + "/login")])550            let authed = await h.call("browser.download", ["url": .string(h.site.origin + "/private.bin")])551            #expect(authed.status == .succeeded, "\(authed.failure?.message ?? "")")552            #expect(try String(contentsOfFile: authed.output["path"]?.stringValue ?? "", encoding: .utf8) == "private payload")553            // Sessions are isolated: another run does not have that cookie.554            #expect(await h.call("browser.download", ["url": .string(h.site.origin + "/private.bin"), "overwrite": true], runID: "other-run").failure?.code == "http_error")555556            // Navigating to something the server offers as a download is refused; nothing is written anywhere.557            let implicit = await h.call("browser.open", ["url": .string(h.site.origin + "/attachment")])558            #expect(implicit.status == .denied && implicit.failure?.code == "download_refused")559            let binary = await h.call("browser.open", ["url": .string(h.site.origin + "/file.bin")])560            #expect(binary.status == .denied && binary.failure?.code == "download_refused")561            let downloads = FileManager.default.urls(for: .downloadsDirectory, in: .userDomainMask).first562            #expect(downloads.map { !FileManager.default.fileExists(atPath: $0.appendingPathComponent("report.txt").path) } ?? true)563        }564    }565566    @Test func sessionsAreKeyedByRunAndCloseReleasesThem() async throws {567        try await withBrowser { (h: BrowserHarness) async throws -> Void in568            // No page yet: every page tool fails with a typed error naming the remedy.569            for (tool, arguments) in [("browser.read", [:]), ("browser.click", ["selector": "#x"]), ("browser.fill", ["selector": "#x", "value": "v"]),570                                      ("browser.waitFor", ["selector": "#x"]), ("browser.screenshot", [:])] as [(String, JSONValue)] {571                let result = await h.call(tool, arguments)572                #expect(result.status == .failed && result.failure?.code == "not_found" && result.failure?.message.contains("browser.open") == true, "\(tool)")573            }574            #expect(await h.manager.view(for: "run-1") == nil)575576            await h.call("browser.open", ["url": .string(h.site.origin + "/")], runID: "run-1")577            await h.call("browser.open", ["url": .string(h.site.origin + "/form")], runID: "run-2")578            #expect(await h.manager.activeRunIDs == ["run-1", "run-2"])579            #expect(await h.call("browser.read", runID: "run-1").output["title"]?.stringValue == "ORIX Test Site")580            #expect(await h.call("browser.read", runID: "run-2").output["title"]?.stringValue == "Data entry")581            let view = await h.manager.view(for: "run-2")582            #expect(await view?.url?.path == "/form")583            let persistent = await view?.configuration.websiteDataStore.isPersistent584            #expect(persistent == false)585586            let closed = await h.call("browser.close", runID: "run-1")587            #expect(closed.status == .succeeded && closed.output["closed"]?.boolValue == true)588            #expect(await h.call("browser.close", runID: "run-1").output["closed"]?.boolValue == false)589            #expect(await h.manager.activeRunIDs == ["run-2"] && h.manager.currentURL(runID: "run-1") == nil)590            #expect(await h.call("browser.read", runID: "run-1").failure?.code == "not_found")591            await h.manager.closeAll()592            #expect(await h.manager.activeRunIDs.isEmpty)593            #expect(await h.manager.close(runID: "run-2") == false)594        }595    }596597    @Test func slowLoadsTimeOutAndCallsCanBeCanceled() async throws {598        try await withBrowser { (h: BrowserHarness) async throws -> Void in599            // A server that accepts the request and never answers.600            let start = Date()601            let hung = await h.call("browser.open", ["url": .string(h.site.origin + "/hang"), "timeoutSeconds": 1])602            #expect(hung.status == .timedOut && hung.failure?.code == "timeout" && hung.failure?.retryable == true, "\(hung.status)")603            #expect(Date().timeIntervalSince(start) < 6)604            // The session is still usable afterwards.605            #expect(await h.call("browser.open", ["url": .string(h.site.origin + "/")]).status == .succeeded)606607            // Canceling the calling task ends a long wait promptly with a typed result.608            let waiting = Task { await h.call("browser.waitFor", ["selector": "#never", "timeoutSeconds": 15]) }609            try await Task.sleep(nanoseconds: 400_000_000)610            let canceledAt = Date()611            waiting.cancel()612            let canceled = await waiting.value613            #expect(canceled.status == .canceled && canceled.failure?.code == "canceled", "\(canceled.status)")614            #expect(Date().timeIntervalSince(canceledAt) < 3)615616            let loading = Task { await h.call("browser.open", ["url": .string(h.site.origin + "/hang"), "timeoutSeconds": 15]) }617            try await Task.sleep(nanoseconds: 400_000_000)618            loading.cancel()619            #expect(await loading.value.status == .canceled)620            #expect(await h.call("browser.read").status == .succeeded)621        }622    }623624    @Test func theOffscreenViewLaysOutAndRunsTimers() async throws {625        try await withBrowser { (h: BrowserHarness) async throws -> Void in626            await h.call("browser.open", ["url": .string(h.site.origin + "/visibility")])627            #expect(await h.call("browser.waitFor", ["urlMatches": "/visibility$", "selector": "h1", "state": "visible"]).status == .succeeded)628            #expect(try await h.pageValue("window.innerWidth + 'x' + window.innerHeight") == "1280x800")629            // The 50 ms timer of the page fired; report what WebKit says about visibility and how late the timer was.630            var title = ""631            for _ in 0..<100 {632                title = try await h.pageValue("document.title")633                if title.contains(":") { break }634                try await Task.sleep(nanoseconds: 50_000_000)635            }636            print("INFO offscreen WKWebView visibilityState:timerDelayMs = \(title)")637            #expect(title.contains(":"))638        }639    }640641    @Test func concurrentCallsOnOneSessionAreSerialised() async throws {642        try await withBrowser { (h: BrowserHarness) async throws -> Void in643            await h.call("browser.open", ["url": .string(h.site.origin + "/slow")])644            async let wait = h.call("browser.waitFor", ["selector": "#late", "timeoutSeconds": 10])645            async let read = h.call("browser.read")646            async let shot = h.call("browser.screenshot")647            let results = await [wait, read, shot]648            #expect(results.allSatisfy { $0.status == .succeeded }, "\(results.map(\.status))")649        }650    }651}652
Files main
Menu

Theme