SPB Git

Repository Public License

ExamplesEndToEndTests.swift 68.9 KB 1,062 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXServices/Tests/ORIXServicesTests/ExamplesEndToEndTests.swift10// Description      : Runs each of the 15 built-in example automations for real11//                    (real services on a temporary data folder, real tools,12//                    sample data copied into a temporary workspace) and checks13//                    the artifacts by content.14// Module           : ORIXServicesTests15// Repository       : https://git.spboucher.ai/orix-automation16// Created          : 2026-11-2117// Last Modified    : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author           : Simon-Pierre Boucher24// Position         : Professor of Finance25// Institution      : Université du Québec en Outaouais (UQO)26// Department       : Département des sciences administratives27// Organization     : SP Boucher28//29// Website          : https://spboucher.ai30// Git              : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General          : info@spboucher.ai37// Professional     : contact@spboucher.ai38// Direct           : simon@spboucher.ai39// Research         : research@spboucher.ai40// Development      : dev@spboucher.ai41// Support          : support@spboucher.ai42// Security/Admin   : admin@spboucher.ai43// Notifications    : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification   : Public70// Distribution     : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License          : Proprietary, all rights reserved (see license file)84// License File     : LICENSE85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import AppKit93import CoreGraphics94import CryptoKit95import Foundation96import ImageIO97import Network98import PDFKit99import Testing100import ORIXAppleAutomation101import ORIXCore102import ORIXServices103import ORIXWorkflowEngine104105// MARK: - Support106107/// What the test's "person at the screen" answered. Manual gates of a workflow are approved (or refused on request);108/// a *tool* approval is always refused and recorded: with the declared permissions accepted, no tool call of an109/// example may need one. This is stricter than `orixctl --approve all`, which would hide a missing permission.110final class ApprovalLog: @unchecked Sendable {111    private let lock = NSLock()112    private var _gates: [String] = []113    private var _tools: [String] = []114    var gates: [String] { lock.lock(); defer { lock.unlock() }; return _gates }115    var toolRequests: [String] { lock.lock(); defer { lock.unlock() }; return _tools }116    func gate(_ text: String) { lock.lock(); _gates.append(text); lock.unlock() }117    func tool(_ text: String) { lock.lock(); _tools.append(text); lock.unlock() }118}119120/// A loopback-only HTTP/1.1 server for the tests (GET only, one request per connection).121final class LoopbackServer: @unchecked Sendable {122    typealias Handler = @Sendable (_ path: String) -> (status: Int, type: String, body: Data)123    private let listener: NWListener124    private let queue = DispatchQueue(label: "orix.tests.loopback")125    private let lock = NSLock()126    private var _requests: [String] = []127    private var handler: Handler128    private(set) var port: UInt16 = 0129130    /// Request targets (path and query) received so far, in order.131    var requests: [String] { lock.lock(); defer { lock.unlock() }; return _requests }132    func setHandler(_ h: @escaping Handler) { lock.lock(); handler = h; lock.unlock() }133134    init(handler: @escaping Handler) async throws {135        self.handler = handler136        let parameters = NWParameters.tcp137        parameters.requiredLocalEndpoint = .hostPort(host: .ipv4(.loopback), port: .any)138        parameters.allowLocalEndpointReuse = true139        listener = try NWListener(using: parameters)140        listener.newConnectionHandler = { [weak self] connection in self?.serve(connection) }141        let started = AsyncStream<Bool>.makeStream()142        listener.stateUpdateHandler = { state in143            switch state {144            case .ready: started.continuation.yield(true); started.continuation.finish()145            case .failed, .cancelled: started.continuation.yield(false); started.continuation.finish()146            default: break147            }148        }149        listener.start(queue: queue)150        var ready = false151        for await ok in started.stream { ready = ok }152        guard ready, let p = listener.port?.rawValue else { throw CocoaError(.fileNoSuchFile) }153        port = p154    }155156    /// Serves the files of one folder (no directory listing, no path outside the folder).157    static func site(_ folder: URL) async throws -> LoopbackServer {158        let root = folder.resolvingSymlinksInPath().path159        return try await LoopbackServer { target in160            let path = String(target.split(separator: "?", maxSplits: 1, omittingEmptySubsequences: false).first ?? "")161            let file = URL(fileURLWithPath: root + path).standardizedFileURL.path162            guard file.hasPrefix(root + "/"), let data = FileManager.default.contents(atPath: file) else { return (404, "text/plain", Data("not found".utf8)) }163            let type = file.hasSuffix(".css") ? "text/css; charset=utf-8" : "text/html; charset=utf-8"164            return (200, type, data)165        }166    }167168    private func serve(_ connection: NWConnection) {169        connection.start(queue: queue)170        receive(connection, buffer: Data())171    }172173    private func receive(_ connection: NWConnection, buffer: Data) {174        connection.receive(minimumIncompleteLength: 1, maximumLength: 65_536) { [weak self] data, _, complete, error in175            guard let self else { connection.cancel(); return }176            var buffer = buffer177            if let data { buffer.append(data) }178            guard let end = buffer.range(of: Data("\r\n\r\n".utf8)) else {179                if complete || error != nil || buffer.count > 262_144 { connection.cancel() } else { self.receive(connection, buffer: buffer) }180                return181            }182            let head = String(decoding: buffer[..<end.lowerBound], as: UTF8.self)183            let parts = (head.split(separator: "\r\n").first ?? "").split(separator: " ")184            let target = parts.count >= 2 ? String(parts[1]) : "/"185            self.lock.lock(); self._requests.append(target); let handler = self.handler; self.lock.unlock()186            let reply = parts.first == "GET" ? handler(target) : (405, "text/plain", Data("GET only".utf8))187            var out = Data("HTTP/1.1 \(reply.0) \(reply.0 == 200 ? "OK" : "Error")\r\nContent-Type: \(reply.1)\r\nContent-Length: \(reply.2.count)\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n".utf8)188            out.append(reply.2)189            connection.send(content: out, completion: .contentProcessed { _ in connection.cancel() })190        }191    }192193    func stop() { listener.cancel() }194}195196enum Repo {197    static let root: URL = {198        var url = URL(fileURLWithPath: #filePath)199        for _ in 0..<5 { url.deleteLastPathComponent() }200        return url201    }()202    static var examples: URL { root.appendingPathComponent("Workflows/Examples") }203    static var mirror: URL { root.appendingPathComponent("Packages/ORIXWorkflowEngine/Sources/ORIXWorkflowEngine/Resources/Examples") }204    static var samples: URL { root.appendingPathComponent("Workflows/Samples") }205206    static func exampleFile(slug: String) throws -> URL {207        let files = try FileManager.default.contentsOfDirectory(at: examples, includingPropertiesForKeys: nil).filter { $0.pathExtension == "yaml" }208        guard let f = files.first(where: { $0.deletingPathExtension().lastPathComponent.dropFirst(3) == slug }) else { throw CocoaError(.fileNoSuchFile) }209        return f210    }211}212213enum Files {214    static func sha256(_ url: URL) throws -> String {215        SHA256.hash(data: try Data(contentsOf: url)).map { String(format: "%02x", $0) }.joined()216    }217218    /// Relative path → SHA-256 for every regular file under a folder (hidden files included); folders map to "dir".219    static func tree(_ root: URL) -> [String: String] {220        var out: [String: String] = [:]221        let base = root.resolvingSymlinksInPath().path222        guard let walker = FileManager.default.enumerator(atPath: base) else { return out }223        while let rel = walker.nextObject() as? String {224            var isDir: ObjCBool = false225            FileManager.default.fileExists(atPath: base + "/" + rel, isDirectory: &isDir)226            out[rel] = isDir.boolValue ? "dir" : ((try? sha256(URL(fileURLWithPath: base + "/" + rel))) ?? "unreadable")227        }228        return out229    }230231    static func exists(_ url: URL) -> Bool { FileManager.default.fileExists(atPath: url.path) }232    static func text(_ url: URL) throws -> String { try String(contentsOf: url, encoding: .utf8) }233234    static func imageSize(_ url: URL) -> (width: Int, height: Int, type: String)? {235        guard let source = CGImageSourceCreateWithURL(url as CFURL, nil), let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else { return nil }236        return (image.width, image.height, (CGImageSourceGetType(source) as String?) ?? "")237    }238239    /// Runs a program the test itself needs (git, ditto, the sample setup script) and returns its output.240    @discardableResult241    static func run(_ executable: String, _ arguments: [String], cwd: URL? = nil) throws -> (code: Int32, out: String) {242        let p = Process()243        p.executableURL = URL(fileURLWithPath: executable)244        p.arguments = arguments245        p.currentDirectoryURL = cwd246        var env = ["PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "LC_ALL": "C", "HOME": NSTemporaryDirectory()]247        env["GIT_CONFIG_GLOBAL"] = "/dev/null"; env["GIT_CONFIG_SYSTEM"] = "/dev/null"248        p.environment = env249        let pipe = Pipe()250        p.standardOutput = pipe; p.standardError = pipe251        try p.run()252        let data = pipe.fileHandleForReading.readDataToEndOfFile()253        p.waitUntilExit()254        return (p.terminationStatus, String(decoding: data, as: UTF8.self))255    }256}257258struct FinishedRun {259    let record: RunRecord260    let receipt: RunReceipt261    let outcome: WorkflowOutcome262    var status: RunStatus { record.status }263    func output(_ name: String) -> JSONValue? { outcome.outputs[name] }264    func step(_ id: String) -> StepResult? { outcome.step(id) }265    /// Why a run did not end as expected: errors, failed verifications and tool calls that did not succeed.266    var diagnosis: String {267        let checks = receipt.verifications.filter { !$0.passed }.map { "FAIL \($0.assertion) — \($0.evidence)" }268        let calls = receipt.toolCalls.filter { $0.status != .succeeded }.map { "\($0.toolID) \($0.status.rawValue): \($0.summary) \($0.arguments.serialized().prefix(300))" }269        return (receipt.errors + checks + calls).joined(separator: "\n")270    }271    /// Steps that ended in anything else than succeeded / skipped / simulated.272    var failedSteps: [String] { outcome.steps.filter { !$0.status.isSuccess && $0.status != .skipped }.map(\.stepID) }273}274275/// Real services on a temporary data folder plus a temporary workspace for the sample data.276final class ExampleHarness: @unchecked Sendable {277    let root: URL278    let workspace: URL279    let services: ORIXServices280    let approvals = ApprovalLog()281    private var tasks: [Task<Void, Never>] = []282    private var saved: Set<String> = []283284    static let devKeyName = "ORIX_DEV_ANTHROPIC_API_KEY"285    static var hasDevKey: Bool { !(ProcessInfo.processInfo.environment[devKeyName] ?? "").isEmpty }286    static let testModel = ModelRef(provider: .anthropic, model: "claude-haiku-5-5")287288    /// - Parameters:289    ///   - interactive: false = a headless process: nobody answers approvals.290    ///   - approveGates: what the simulated person answers to a workflow's manual gate.291    ///   - live: use the development provider key from the environment and Haiku as the default model.292    init(interactive: Bool = true, approveGates: Bool = true, live: Bool = false) async throws {293        let base = URL(fileURLWithPath: NSTemporaryDirectory()).resolvingSymlinksInPath().appendingPathComponent("orix-examples-\(UUID().uuidString.prefix(8))")294        root = base295        workspace = base.appendingPathComponent("workspace")296        try FileManager.default.createDirectory(at: workspace, withIntermediateDirectories: true)297        var options = ServiceOptions()298        options.interactive = interactive299        options.allowDevelopmentCredentials = live300        options.keychainService = "ai.spboucher.orix.tests.examples.\(UUID().uuidString.prefix(8))"   // never the user's real items301        services = try await ORIXServices.bootstrap(paths: ORIXPaths(root: base.appendingPathComponent("data")), options: options)302        if live {303            var settings = await services.runs.settings()304            settings.defaultModel = Self.testModel305            settings.dailySpendLimitUSD = 0.25306            try await services.runs.save(settings)307        }308        guard interactive else { return }309        let center = services.approvals, interaction = services.interaction, log = approvals310        tasks.append(Task {311            for await pending in center.updates() {312                for p in pending {313                    if p.access.toolID == "workflow.approval" {314                        log.gate(p.access.summary)315                        await center.resolve(id: p.id, response: approveGates ? .approveOnce : .deny(reason: "refused by the test"))316                    } else {317                        log.tool("\(p.access.toolID): \(p.reason)")318                        await center.resolve(id: p.id, response: .deny(reason: "the declared permissions of an example must be sufficient"))319                    }320                }321            }322        })323        tasks.append(Task {324            for await items in interaction.updates() {325                for item in items {326                    switch item {327                    case .plan(let p): await interaction.review(planID: p.id, .approved)328                    case .question(let q): await interaction.answer(questionID: q.id, nil)329                    }330                }331            }332        })333    }334335    func tearDown() async {336        for t in tasks { t.cancel() }337        await services.shutdown()338        try? FileManager.default.removeItem(at: root)339    }340341    /// Copies `Workflows/Samples/<slug>` into the workspace and returns the copy.342    func copySample(_ slug: String) throws -> URL {343        let target = workspace.appendingPathComponent(slug)344        try FileManager.default.copyItem(at: Repo.samples.appendingPathComponent(slug), to: target)345        return target346    }347348    /// Saves the example exactly as shipped (once) — or a variant with a tighter cost budget for live agent runs.349    @discardableResult350    func save(_ slug: String, costBudgetUSD: Double? = nil) async throws -> SavedWorkflow {351        var source = try String(contentsOf: Repo.exampleFile(slug: slug), encoding: .utf8)352        if let costBudgetUSD {353            // Same workflow, smaller spending ceiling: a test must not be able to spend what a user's real run may.354            let pattern = #"maxProviderCostUSD: [0-9.]+"#355            guard let range = source.range(of: pattern, options: .regularExpression) else { throw CocoaError(.formatting) }356            source.replaceSubrange(range, with: "maxProviderCostUSD: \(costBudgetUSD)")357        }358        let s = try await services.workflows.save(source: source)359        saved.insert(slug)360        return s361    }362363    func run(_ slug: String, _ parameters: [String: JSONValue] = [:], accept: Bool = true, dryRun: Bool = false, mode: ExecutionMode = .guided) async throws -> FinishedRun {364        if !saved.contains(slug) { try await save(slug) }365        if accept { try await services.workflows.acceptPermissions(workflowID: slug, parameters: parameters) }366        var request = WorkflowRunRequest(workflowID: slug)367        request.parameters = parameters368        request.dryRun = dryRun369        request.mode = mode370        request.startPaused = true   // hold the run until the test has its handle, so the outcome (outputs, step results) can be read371        let runID = try await services.workflows.startRun(request)372        let handle = try #require(await services.workflows.handle(runID: runID))373        await handle.resume()374        let outcome = await handle.outcome()375        let record = try #require(await services.runs.wait(runID: runID))376        let receipt = try #require(try await services.store.receipt(runID: runID))377        if ProcessInfo.processInfo.environment["ORIX_TEST_DUMP_EVENTS"] == "1" {378            // Debugging aid: the persisted (already redacted) event log of the run.379            for e in (try? await services.store.events(runID: runID, afterSeq: 0, limit: 5000)) ?? [] {380                print("EVENT \(slug) \(e.seq) \(e.kind.rawValue) \(e.stepID ?? "-") \(e.text.replacingOccurrences(of: "\n", with: " ⏎ ").prefix(700)) \(e.data.serialized().prefix(500))")381            }382        }383        return FinishedRun(record: record, receipt: receipt, outcome: outcome)384    }385386    func path(_ url: URL) -> JSONValue { .string(url.path) }387}388389/// The notification step of an example cannot work under `swift test` (no application bundle): it fails by itself,390/// honestly, and the run ends PARTIALLY_SUCCEEDED. This checks that it is the only thing that went wrong.391func expectOnlyTheNotificationFailed(_ run: FinishedRun, step: String, sourceLocation: SourceLocation = #_sourceLocation) {392    #expect(run.status == .partiallySucceeded, "\(run.diagnosis)", sourceLocation: sourceLocation)393    #expect(run.failedSteps == [step], sourceLocation: sourceLocation)394    #expect(run.step(step)?.error?.contains("application bundle") == true, "\(run.step(step)?.error ?? "nil")", sourceLocation: sourceLocation)395    #expect(run.step(step)?.continued == true, sourceLocation: sourceLocation)396}397398let allExampleSlugs = [399    "smart-downloads-organizer", "pdf-to-structured-report", "weekly-research-digest", "git-test-and-review", "csv-cleanup-and-charts",400    "batch-image-conversion", "calendar-preparation", "folder-backup-and-verify", "website-change-monitor", "system-health-report",401    "meeting-notes-organizer", "document-metadata-catalog", "scheduled-window-screenshot", "spreadsheet-validation", "browser-data-entry",402]403404// MARK: - Tests405406/// The suite's type name is a raw identifier so that `swift test --filter "Integration · examples"` selects it407/// (the filter matches identifiers, not display names). Real services, real tools, sample data; a live model and the408/// macOS privacy permissions are used only when they are available, and every skip prints `SKIPPED(<reason>)`.409@Suite(.serialized, .timeLimit(.minutes(10)))410struct `Integration · examples` {411412    // MARK: Validation against the real manifests413414    @Test("All 15 examples validate with zero errors and zero warnings against the registered tools, and the mirror is identical")415    func examplesValidateAgainstRealManifests() async throws {416        let h = try await ExampleHarness()417        defer { Task { await h.tearDown() } }418        let registered = Set(await h.services.executor.manifests().map(\.identifier))419        for slug in allExampleSlugs {420            let file = try Repo.exampleFile(slug: slug)421            let source = try String(contentsOf: file, encoding: .utf8)422            let (doc, diagnostics) = await h.services.workflows.diagnose(source)423            let problems = diagnostics.filter { $0.severity == .error || $0.severity == .warning }424            #expect(problems.isEmpty, "\(slug): \(problems.map { "line \($0.line): \($0.message)" }.joined(separator: "\n"))")425            let document = try #require(doc, "\(slug) does not parse")426            #expect(document.metadata.id == slug)427            for located in document.allSteps {428                if case .tool(let t) = located.step.kind { #expect(registered.contains(t.tool), "\(slug): \(t.tool) is not a registered tool") }429                if case .agent(let a) = located.step.kind, case .inline(let inline) = a.agent {430                    #expect(inline.tools.allSatisfy(registered.contains), "\(slug): agent tools \(inline.tools)")431                }432            }433            #expect(try Data(contentsOf: file) == Data(contentsOf: Repo.mirror.appendingPathComponent(file.lastPathComponent)), "\(slug): resource copy differs")434            #expect(Files.exists(Repo.samples.appendingPathComponent(slug)), "\(slug) has no sample folder")435            // Saving is what the template gallery does: it must accept the example as shipped.436            let saved = try await h.save(slug)437            #expect(saved.id == slug && saved.version == 1)438        }439    }440441    // MARK: 1 · Smart Downloads Organizer442443    @Test("01 smart-downloads-organizer: every file is moved to its category, nothing is deleted or altered")444    func smartDownloadsOrganizer() async throws {445        let h = try await ExampleHarness()446        defer { Task { await h.tearDown() } }447        let ws = try h.copySample("smart-downloads-organizer")448        let source = ws.appendingPathComponent("Downloads"), destination = ws.appendingPathComponent("Sorted")449        let before = Files.tree(source)450        let run = try await h.run("smart-downloads-organizer", ["source": h.path(source), "destination": h.path(destination)])451        #expect(run.status == .succeeded, "\(run.diagnosis)")452453        let expected: [String: [String]] = [454            "Documents": ["notes.txt", "reading-list.md", "SCAN 2026.TXT", "memo.pdf"], "Spreadsheets": ["budget.csv"], "Images": ["photo.png"],455            "Archives": ["old-project.zip"], "Audio": ["jingle.mp3"], "Video": ["clip.mov"], "Other": ["LICENSE", "data.xyz"],456        ]457        let after = Files.tree(destination)458        for (category, names) in expected {459            for name in names { #expect(after["\(category)/\(name)"] == before[name], "\(name) should be in \(category) with the same content") }460        }461        let moved = expected.values.reduce(0) { $0 + $1.count }462        #expect(after.values.filter { $0 != "dir" }.count == moved, "exactly the moved files are in the destination: \(after.keys.sorted())")463        // What stays: the subfolder and its content, untouched. Nothing was deleted: moved + remaining == before.464        let remaining = Files.tree(source)465        #expect(remaining == ["Projects": "dir", "Projects/keep-me.txt": before["Projects/keep-me.txt"]!])466        #expect(Set(after.values.filter { $0 != "dir" }).union(remaining.values.filter { $0 != "dir" }) == Set(before.values.filter { $0 != "dir" }))467        #expect(run.output("moved") == .int(moved))468        #expect(h.approvals.gates.count == 1, "the manual gate was asked once")469        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")470        #expect(run.receipt.verifications.count == 2 && run.receipt.verifications.allSatisfy(\.passed))471    }472473    @Test("01 smart-downloads-organizer, headless and without accepted permissions: waits for approval and moves nothing")474    func smartDownloadsOrganizerWithoutPermissions() async throws {475        let h = try await ExampleHarness(interactive: false)476        defer { Task { await h.tearDown() } }477        let ws = try h.copySample("smart-downloads-organizer")478        let before = Files.tree(ws)479        let run = try await h.run("smart-downloads-organizer", ["source": h.path(ws.appendingPathComponent("Downloads")), "destination": h.path(ws.appendingPathComponent("Sorted"))], accept: false)480        #expect(run.status == .waitingApproval || run.status == .blockedPermission, "\(run.status)")481        #expect(Files.tree(ws) == before, "nothing may change on disk")482        #expect(!run.receipt.toolCalls.contains { $0.status == .succeeded }, "no tool call may have run")483    }484485    // MARK: 2 · PDF to structured data and report486487    @Test("02 pdf-to-structured-report: CSV catalogue rows match the PDFs and the report PDF is readable")488    func pdfToStructuredReport() async throws {489        let h = try await ExampleHarness()490        defer { Task { await h.tearDown() } }491        let ws = try h.copySample("pdf-to-structured-report")492        let input = ws.appendingPathComponent("Input"), output = ws.appendingPathComponent("Output")493        let before = Files.tree(input)494        let run = try await h.run("pdf-to-structured-report", ["inputFolder": h.path(input), "outputFolder": h.path(output)])495        #expect(run.status == .succeeded, "\(run.diagnosis)")496497        let lines = try Files.text(output.appendingPathComponent("catalog.csv")).split(whereSeparator: \.isNewline).map(String.init)498        #expect(lines.first == "file,pages,title,textCharacters")499        var rows: [String: [String]] = [:]500        for line in lines.dropFirst() { let c = line.split(separator: ",", omittingEmptySubsequences: false).map(String.init); rows[c[0]] = c }501        #expect(rows.count == 4 && lines.count == 5)502        #expect(rows["field-notes.pdf"]?[1] == "3" && rows["field-notes.pdf"]?[2] == "Field notes")503        #expect(rows["quarterly-summary.pdf"]?[1] == "1" && rows["quarterly-summary.pdf"]?[2] == "Quarterly summary Q3")504        #expect(rows["untitled-memo.pdf"]?[2] == "", "a PDF without a Title property has an empty title")505        #expect(rows["scan-without-text.pdf"]?[3] == "0", "a page without a text layer has no text")506        for name in ["field-notes.pdf", "quarterly-summary.pdf", "untitled-memo.pdf"] { #expect((Int(rows[name]?[3] ?? "") ?? 0) > 20, "\(name) has a text layer") }507        // Page counts are checked against PDFKit directly, independently of the tool.508        for (rel, _) in before where rel.hasSuffix(".pdf") {509            let pages = PDFDocument(url: input.appendingPathComponent(rel))?.pageCount510            #expect(rows[(rel as NSString).lastPathComponent]?[1] == pages.map(String.init))511        }512        let report = try #require(PDFDocument(url: output.appendingPathComponent("report.pdf")))513        #expect(report.pageCount == 1)514        let text = report.string ?? ""515        #expect(text.contains("PDF report") && text.contains("field-notes.pdf") && text.contains("Documents: 4"))516        #expect(try Files.text(output.appendingPathComponent("report.md")).contains("| field-notes.pdf | 3 | Field notes |"))517        #expect(run.output("documents") == .int(4) && run.output("pages") == .int(6))518        #expect(Files.tree(input) == before, "the input folder is read only")519        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")520    }521522    // MARK: 3, 4, 11 · Examples with an agent step523524    /// How a dry run accounts for the agent step of an example. A dry run executes no read tool either, so what the525    /// agent step depends on decides how far the simulation gets.526    enum DryAgent {527        /// The step is reached and reported as "the agent was not run".528        case simulated529        /// The step's `when` reads the (not executed) listing and is false: reported as skipped.530        case skipped531        /// The step lives in a `foreach` over items a dry run cannot know: the loop body is not entered at all.532        case notEntered533    }534535    /// Dry run of an agent example: nothing is written, no model is called, and the report accounts for the agent step.536    private func expectDryRunSkipsTheAgent(_ h: ExampleHarness, slug: String, parameters: [String: JSONValue], agentStep: String, expecting: DryAgent, workspace: URL) async throws {537        let before = Files.tree(workspace)538        let dry = try await h.run(slug, parameters, dryRun: true)539        #expect(dry.outcome.dryRun)540        let report = dry.receipt.limitations541        #expect(report.first?.hasPrefix("Dry run: nothing was changed") == true, "\(report)")542        switch expecting {543        case .simulated:544            #expect(dry.step(agentStep)?.status == .simulated)545            #expect(report.contains { $0.hasPrefix("\(agentStep): simulated: the agent was not run") }, "\(report)")546        case .skipped:547            #expect(dry.step(agentStep)?.status == .skipped)548            #expect(report.contains { $0.hasPrefix("Step \(agentStep) was skipped") }, "\(report)")549        case .notEntered:550            #expect(dry.outcome.results(for: agentStep).isEmpty, "the loop body is not entered in a dry run")551        }552        #expect(!dry.outcome.steps.contains { $0.stepID == agentStep && $0.status == .succeeded }, "the agent step was not executed")553        #expect(dry.receipt.usage.turns == 0 && dry.receipt.usage.tokens.totalTokens == 0 && dry.receipt.usage.costUSD == 0, "a dry run calls no model")554        #expect(!dry.receipt.toolCalls.contains { $0.status == .succeeded }, "a dry run executes no tool: \(dry.receipt.toolCalls.map(\.toolID))")555        #expect(Files.tree(workspace) == before, "a dry run changes nothing on disk")556    }557558    @Test("03 weekly-research-digest: dry run never calls the model; live run (dev key) writes a cited digest and its PDF")559    func weeklyResearchDigest() async throws {560        let live = ExampleHarness.hasDevKey561        let h = try await ExampleHarness(live: live)562        defer { Task { await h.tearDown() } }563        let ws = try h.copySample("weekly-research-digest")564        let research = ws.appendingPathComponent("Research"), reports = ws.appendingPathComponent("Reports")565        let parameters: [String: JSONValue] = ["researchFolder": h.path(research), "reportsFolder": h.path(reports), "maxDocuments": 5]566        try await h.save("weekly-research-digest", costBudgetUSD: 0.04)567        try await expectDryRunSkipsTheAgent(h, slug: "weekly-research-digest", parameters: parameters, agentStep: "analyze", expecting: .skipped, workspace: ws)568        guard live else { print("SKIPPED(no ORIX_DEV_ANTHROPIC_API_KEY): weekly-research-digest live agent run; dry run verified"); return }569570        let before = Files.tree(research)571        let run = try await h.run("weekly-research-digest", parameters)572        expectOnlyTheNotificationFailed(run, step: "announce")573        let digestPath = try #require(run.output("digest")?.stringValue)574        let digest = try Files.text(URL(fileURLWithPath: digestPath))575        #expect(digest.contains("Highlights") && digest.count > 200)576        let cited = ["battery-storage-brief", "interview-operator", "field-notes"].filter(digest.contains)577        #expect(cited.count >= 2, "the digest names its source files (text and PDF): \(cited)")578        #expect(digest.lowercased().contains("inverter"), "the digest reflects the content of the sample documents")579        // The workflow read the documents itself; the agent's only tool call is the write of the digest.580        #expect(run.outcome.results(for: "content").count == 2 && run.outcome.results(for: "pdfContent").count == 1)581        #expect(run.step("pdfContent")?.output["pageCount"] == .int(3))582        #expect(!run.receipt.toolCalls.contains { $0.status != .succeeded && $0.toolID != "notifications.post" }, "\(run.diagnosis)")583        let pdfPath = try #require(run.output("pdf")?.stringValue)584        let pdf = try #require(PDFDocument(url: URL(fileURLWithPath: pdfPath)))585        #expect(pdf.pageCount >= 1 && (pdf.string ?? "").contains("Highlights"))586        #expect(run.output("documents") == .int(3))587        #expect(Files.tree(research) == before, "the research folder is read only")588        #expect(run.receipt.usage.tokens.totalTokens > 0 && run.receipt.usage.costUSD < 0.04, "cost \(run.receipt.usage.costUSD)")589        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")590        print(String(format: "VERIFIED(live anthropic:claude-haiku-5-5): weekly-research-digest, %d tokens, $%.4f", run.receipt.usage.tokens.totalTokens, run.receipt.usage.costUSD))591    }592593    @Test("04 git-test-and-review: tests run, patch saved, repository untouched; live review (dev key) reports the failing tests")594    func gitTestAndReview() async throws {595        let live = ExampleHarness.hasDevKey596        let h = try await ExampleHarness(live: live)597        defer { Task { await h.tearDown() } }598        let ws = try h.copySample("git-test-and-review")599        let setup = try Files.run("/bin/bash", [ws.appendingPathComponent("setup.sh").path, ws.path])600        #expect(setup.code == 0, "\(setup.out)")601        let repo = ws.appendingPathComponent("repo"), reviews = ws.appendingPathComponent("Reviews")602        func git(_ args: String...) throws -> String { try Files.run("/usr/bin/git", ["-C", repo.path] + args).out }603        let head = try git("rev-parse", "HEAD"), status = try git("status", "--porcelain"), commits = try git("rev-list", "--count", "HEAD")604        #expect(commits.trimmingCharacters(in: .whitespacesAndNewlines) == "2" && status.contains(" M slugify.sh"))605        let parameters: [String: JSONValue] = ["repository": h.path(repo), "reviewsFolder": h.path(reviews)]606        try await h.save("git-test-and-review", costBudgetUSD: 0.04)607        try await expectDryRunSkipsTheAgent(h, slug: "git-test-and-review", parameters: parameters, agentStep: "review", expecting: .simulated, workspace: ws)608        guard live else {609            // Without a model the non-agent half still runs for real: status, diff, log, the test command and the patch.610            let run = try await h.run("git-test-and-review", parameters)611            #expect(run.status == .failed, "the agent step cannot run without a model: \(run.status)")612            #expect(run.failedSteps == ["review"], "\(run.failedSteps)")613            #expect(run.step("tests")?.output["exitCode"] == .int(1), "the sample's uncommitted change breaks its tests")614            #expect(run.step("tests")?.output["stdout"]?.stringValue?.contains("FAIL - slugify 'Hello World'") == true)615            #expect(run.step("status")?.output["branch"] == .string("main") && run.step("status")?.output["clean"] == .bool(false))616            #expect(try Files.text(reviews.appendingPathComponent("changes.patch")).contains("tr -s ' ' '_'"))617            #expect(!Files.exists(reviews.appendingPathComponent("review.md")))618            #expect(try git("rev-parse", "HEAD") == head && git("status", "--porcelain") == status, "the repository is never modified")619            print("SKIPPED(no ORIX_DEV_ANTHROPIC_API_KEY): git-test-and-review agent review; git, test and patch steps verified for real")620            return621        }622        let run = try await h.run("git-test-and-review", parameters)623        #expect(run.status == .succeeded, "\(run.diagnosis)")624        #expect(run.output("testsPassed") == .bool(false) && run.output("branch") == .string("main") && run.output("patchWritten") == .bool(true))625        #expect(run.step("tests")?.output["exitCode"] == .int(1))626        let review = try Files.text(reviews.appendingPathComponent("review.md"))627        #expect(review.contains("## Tests") && review.lowercased().contains("fail"), "the review states that the tests failed")628        #expect(review.contains("slugify"), "the review talks about the changed file")629        let patch = try Files.text(reviews.appendingPathComponent("changes.patch"))630        #expect(patch.contains("diff --git a/slugify.sh b/slugify.sh") && patch.contains("tr -s ' ' '_'") && !patch.contains("notes.tmp"))631        #expect(try git("rev-parse", "HEAD") == head && git("status", "--porcelain") == status, "never commits, never changes the working tree")632        #expect(try git("rev-list", "--count", "HEAD") == commits)633        #expect(run.receipt.usage.tokens.totalTokens > 0 && run.receipt.usage.costUSD < 0.04, "cost \(run.receipt.usage.costUSD)")634        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")635        print(String(format: "VERIFIED(live anthropic:claude-haiku-5-5): git-test-and-review, %d tokens, $%.4f", run.receipt.usage.tokens.totalTokens, run.receipt.usage.costUSD))636    }637638    @Test("11 meeting-notes-organizer: dry run never calls the model; live run (dev key) writes notes per transcript and the index")639    func meetingNotesOrganizer() async throws {640        let live = ExampleHarness.hasDevKey641        let h = try await ExampleHarness(live: live)642        defer { Task { await h.tearDown() } }643        let ws = try h.copySample("meeting-notes-organizer")644        let transcripts = ws.appendingPathComponent("Transcripts"), notes = ws.appendingPathComponent("Notes")645        let parameters: [String: JSONValue] = ["transcriptsFolder": h.path(transcripts), "notesFolder": h.path(notes)]646        try await h.save("meeting-notes-organizer", costBudgetUSD: 0.05)647        try await expectDryRunSkipsTheAgent(h, slug: "meeting-notes-organizer", parameters: parameters, agentStep: "summarize", expecting: .notEntered, workspace: ws)648        guard live else { print("SKIPPED(no ORIX_DEV_ANTHROPIC_API_KEY): meeting-notes-organizer live agent run; dry run verified"); return }649650        let before = Files.tree(transcripts)651        let run = try await h.run("meeting-notes-organizer", parameters)652        #expect(run.status == .succeeded, "\(run.diagnosis)")653        let sync = try Files.text(notes.appendingPathComponent("2026-11-03-product-sync-notes.md"))654        let budget = try Files.text(notes.appendingPathComponent("2026-11-10-budget-review-notes.md"))655        for text in [sync, budget] { #expect(text.contains("## Summary") && text.contains("## Decisions") && text.contains("## Action items")) }656        #expect(sync.contains("2.4") && sync.contains("Sam"), "decisions and owners come from the transcript")657        #expect(budget.lowercased().contains("staging"))658        let index = try Files.text(notes.appendingPathComponent("index.md"))659        #expect(index.contains("# Meeting notes") && index.contains("2 meeting(s)"))660        #expect(index.contains("(2026-11-03-product-sync-notes.md)") && index.contains("(2026-11-10-budget-review-notes.md)"))661        #expect(Files.tree(transcripts) == before, "transcripts are read only")662        #expect(run.outcome.results(for: "summarize").count == 2)663        #expect(run.receipt.usage.tokens.totalTokens > 0 && run.receipt.usage.costUSD < 0.05, "cost \(run.receipt.usage.costUSD)")664        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")665        print(String(format: "VERIFIED(live anthropic:claude-haiku-5-5): meeting-notes-organizer, %d tokens, $%.4f", run.receipt.usage.tokens.totalTokens, run.receipt.usage.costUSD))666    }667668    // MARK: 5 · CSV cleanup and charts669670    @Test("05 csv-cleanup-and-charts: the sandboxed Python script cleans the dirty rows and draws one bar per row")671    func csvCleanupAndCharts() async throws {672        let h = try await ExampleHarness()673        defer { Task { await h.tearDown() } }674        let ws = try h.copySample("csv-cleanup-and-charts")675        let input = ws.appendingPathComponent("input.csv"), output = ws.appendingPathComponent("Output")676        let inputHash = try Files.sha256(input)677        let run = try await h.run("csv-cleanup-and-charts", ["inputFile": h.path(input), "outputFolder": h.path(output)])678        #expect(run.status == .succeeded, "\(run.receipt.errors) \(run.step("clean")?.output["stderr"]?.stringValue ?? "")")679680        let cleaned = try Files.text(output.appendingPathComponent("cleaned.csv")).replacingOccurrences(of: "\r\n", with: "\n")681        #expect(cleaned == "region,units,revenue\nNorth,12,1200\nSouth,7,\"1,050\"\nEast,3,\nWest,9,810\nCentral,15,2250\n")682        let svg = try Files.text(output.appendingPathComponent("chart.svg"))683        #expect(svg.hasPrefix("<svg") && svg.contains(">units</text>") && svg.components(separatedBy: "<rect ").count - 1 == 5)684        #expect(run.output("rows") == .int(5) && run.output("rowsBefore") == .int(9))685        #expect(run.output("log")?.stringValue == "rows_in=7 rows_out=5 charted_column=units bars=5")686        #expect(try Files.sha256(input) == inputHash, "the input file is never modified")687        #expect(Set(Files.tree(output).keys) == ["cleaned.csv", "chart.svg"], "the sandbox let the script write only what it should")688        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")689    }690691    // MARK: 6 · Batch image conversion692693    @Test("06 batch-image-conversion: one JPEG per image with sequential names and the original pixel sizes")694    func batchImageConversion() async throws {695        let h = try await ExampleHarness()696        defer { Task { await h.tearDown() } }697        let ws = try h.copySample("batch-image-conversion")698        let originals = ws.appendingPathComponent("Originals"), converted = ws.appendingPathComponent("Converted")699        let before = Files.tree(originals)700        let run = try await h.run("batch-image-conversion", ["sourceFolder": h.path(originals), "outputFolder": h.path(converted), "format": "jpeg", "prefix": "shot"])701        #expect(run.status == .succeeded, "\(run.diagnosis)")702        // Listing order is the order of the names as bytes: upper case first.703        let expected = [("Icon Large.PNG", 96, 96), ("badge.gif", 120, 80), ("diagram.png", 320, 200), ("sunset.jpg", 640, 360), ("tile.tiff", 200, 200)]704        for (i, (name, w, hgt)) in expected.enumerated() {705            let size = try #require(Files.imageSize(converted.appendingPathComponent("shot-\(i + 1).jpeg")), "shot-\(i + 1).jpeg (\(name)) is not a readable image")706            #expect(size.width == w && size.height == hgt && size.type == "public.jpeg", "\(name) → \(size)")707        }708        #expect(Set(Files.tree(converted).keys) == Set((1...5).map { "shot-\($0).jpeg" }), "captions.txt is not an image and is left out")709        #expect(Files.tree(originals) == before, "originals are never modified or removed")710        #expect(run.output("converted") == .int(5))711        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")712    }713714    // MARK: 7 · Calendar preparation (Apple Events)715716    @Test("07 calendar-preparation: read-only; runs for real only when Automation consent for Calendar is already granted")717    func calendarPreparation() async throws {718        let h = try await ExampleHarness()719        defer { Task { await h.tearDown() } }720        let folder = h.workspace.appendingPathComponent("calendar-preparation/Briefings")721        let parameters: [String: JSONValue] = ["outputFolder": h.path(folder), "daysAhead": 2]722723        // Read-only by construction: the only script addresses Calendar and contains no command that changes anything.724        let document = try await h.save("calendar-preparation").document725        let scripts = document.allSteps.compactMap { located -> String? in726            if case .tool(let t) = located.step.kind, t.tool == "applescript.run" { return t.arguments["script"]?.stringValue }; return nil727        }728        #expect(scripts.count == 1)729        for verb in ["make new", "delete", "duplicate", "move ", "set summary", "set start date", "set end date", "set description", "do shell script", "save"] {730            #expect(!scripts[0].lowercased().contains(verb), "the calendar script must stay read-only (found “\(verb)”)")731        }732        let review = try await h.services.workflows.permissionReview(workflowID: "calendar-preparation", parameters: parameters)733        #expect(review.proposed.map(\.capability).contains(.appAutomation(bundleID: "com.apple.iCal")))734        #expect(!review.proposed.contains { $0.capability.kind.hasPrefix("file.read") || $0.capability.kind.hasPrefix("file.delete") })735736        let status = MacPermissions.automationStatus(bundleID: "com.apple.iCal")737        switch status {738        case .granted:739            let run = try await h.run("calendar-preparation", parameters)740            expectOnlyTheNotificationFailed(run, step: "announce")741            let briefingPath = try #require(run.output("briefing")?.stringValue)742            let briefing = try Files.text(URL(fileURLWithPath: briefingPath))743            let events = try #require(run.output("events")?.intValue)744            #expect(briefing.contains("# Briefing for") && briefing.contains("Events in the next 2 day(s): \(events)"))745            #expect(briefing.components(separatedBy: "- [ ] ").count - 1 == events, "one checklist line per event")746            #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")747            print("VERIFIED(appleEvents → Calendar granted): calendar-preparation read \(events) event(s), briefing written")748        case .denied:749            let run = try await h.run("calendar-preparation", parameters)750            #expect(run.status == .blockedPermission, "\(run.status)")751            #expect(run.step("events")?.status == .blockedPermission)752            #expect(((try? FileManager.default.contentsOfDirectory(atPath: folder.path)) ?? []).isEmpty, "no briefing without consent")753            print("SKIPPED(appleEvents not granted): Automation consent for Calendar was refused; BLOCKED_PERMISSION verified")754        case .notDetermined, .unknown:755            // Running the script would make macOS show its consent prompt (or launch Calendar): not in a test. Dry run instead.756            let dry = try await h.run("calendar-preparation", parameters, dryRun: true)757            #expect(dry.step("events")?.status == .simulated)758            #expect(dry.receipt.limitations.contains { $0.contains("events") && $0.contains("AppleScript") && $0.contains("Calendar") }, "\(dry.receipt.limitations)")759            #expect(!Files.exists(folder), "a dry run writes nothing")760            print("SKIPPED(appleEvents not granted): Automation consent for Calendar is \(status.rawValue) (Calendar not running or never asked); dry run verified, no prompt triggered")761        }762    }763764    // MARK: 8 · Folder backup and verification765766    @Test("08 folder-backup-and-verify: the archive restores to an identical tree and its recorded checksum is right")767    func folderBackupAndVerify() async throws {768        let h = try await ExampleHarness()769        defer { Task { await h.tearDown() } }770        let ws = try h.copySample("folder-backup-and-verify")771        let source = ws.appendingPathComponent("Documents"), backups = ws.appendingPathComponent("Backups")772        let before = Files.tree(source)773        #expect(before[".settings"] != nil, "the sample has a hidden file")774        let run = try await h.run("folder-backup-and-verify", ["source": h.path(source), "backupRoot": h.path(backups)])775        // The temporary restore went to the user's Trash: take the test's own item out again, whatever happens next.776        let trashed = run.step("cleanup")?.output["trashPath"]?.stringValue777        defer { if let trashed, trashed.contains("/.Trash/"), (trashed as NSString).lastPathComponent.hasPrefix(".verify-") { try? FileManager.default.removeItem(atPath: trashed) } }778        expectOnlyTheNotificationFailed(run, step: "announce")779780        let archivePath = try #require(run.output("archive")?.stringValue)781        let archive = URL(fileURLWithPath: archivePath)782        #expect(archive.lastPathComponent.hasPrefix("backup-") && archive.pathExtension == "zip")783        let digest = try Files.sha256(archive)784        #expect(run.output("sha256") == .string(digest))785        #expect(try Files.text(URL(fileURLWithPath: archive.path + ".sha256")) == "\(digest)  \(archive.lastPathComponent)\n")786        // Independent restore with ditto: the archive really holds the whole tree, hidden file included.787        let check = h.root.appendingPathComponent("independent-restore")788        #expect(try Files.run("/usr/bin/ditto", ["-x", "-k", archive.path, check.path]).code == 0)789        #expect(Files.tree(check.appendingPathComponent("Documents")) == before)790        #expect(Files.tree(source) == before, "the source is read only")791        #expect(run.output("identical") == .bool(true) && run.output("entries")?.intValue == before.count + 1)792        #expect(Set(try FileManager.default.contentsOfDirectory(atPath: backups.path)) == [archive.lastPathComponent, archive.lastPathComponent + ".sha256"],793                "the temporary restore folder is gone from the backup root")794        #expect(trashed != nil && Files.exists(URL(fileURLWithPath: trashed ?? "/nonexistent")), "the temporary copy was moved to the Trash, not erased")795        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")796    }797798    @Test("08 folder-backup-and-verify, headless and without accepted permissions: waits for approval and writes nothing")799    func folderBackupWithoutPermissions() async throws {800        let h = try await ExampleHarness(interactive: false)801        defer { Task { await h.tearDown() } }802        let ws = try h.copySample("folder-backup-and-verify")803        let before = Files.tree(ws)804        let run = try await h.run("folder-backup-and-verify", ["source": h.path(ws.appendingPathComponent("Documents")), "backupRoot": h.path(ws.appendingPathComponent("Backups"))],805                                  accept: false, mode: .scheduled)806        #expect(run.status == .waitingApproval || run.status == .blockedPermission, "\(run.status)")807        #expect(Files.tree(ws) == before, "nothing may change on disk")808        #expect(!run.receipt.toolCalls.contains { $0.status == .succeeded })809    }810811    // MARK: 9 · Website change monitoring (loopback only)812813    @Test("09 website-change-monitor: baseline, unchanged and changed runs against a loopback server; the network permission follows the parameter")814    func websiteChangeMonitor() async throws {815        let h = try await ExampleHarness()816        defer { Task { await h.tearDown() } }817        let ws = try h.copySample("website-change-monitor")818        let v1 = try Data(contentsOf: ws.appendingPathComponent("site/page-v1.html")), v2 = try Data(contentsOf: ws.appendingPathComponent("site/page-v2.html"))819        let server = try await LoopbackServer { _ in (200, "text/html; charset=utf-8", v1) }820        defer { server.stop() }821        let state = ws.appendingPathComponent("State")822        let url = "http://127.0.0.1:\(server.port)/page.html"823        let parameters: [String: JSONValue] = ["host": "127.0.0.1", "url": .string(url), "stateFolder": h.path(state)]824825        try await h.save("website-change-monitor")826        let review = try await h.services.workflows.permissionReview(workflowID: "website-change-monitor", parameters: parameters)827        let hosts = review.proposed.compactMap { g -> String? in if case .network(let host) = g.capability { return host }; return nil }828        #expect(hosts == ["127.0.0.1"], "the declared network permission is the host parameter, not example.com: \(hosts)")829830        let first = try await h.run("website-change-monitor", parameters)831        #expect(first.status == .succeeded, "\(first.receipt.errors)")832        #expect(first.output("firstRun") == .bool(true) && first.output("changed") == .bool(false) && first.output("status") == .int(200))833        #expect(try Data(contentsOf: state.appendingPathComponent("previous.html")) == v1)834835        let second = try await h.run("website-change-monitor", parameters)836        #expect(second.status == .succeeded, "\(second.receipt.errors)")837        #expect(second.output("firstRun") == .bool(false) && second.output("changed") == .bool(false))838        #expect(second.step("alert")?.status == .skipped)839840        server.setHandler { _ in (200, "text/html; charset=utf-8", v2) }841        let third = try await h.run("website-change-monitor", parameters)842        #expect(third.output("changed") == .bool(true), "the change is detected")843        expectOnlyTheNotificationFailed(third, step: "alert")   // the alert is a notification: no app bundle under swift test844        #expect(try Data(contentsOf: state.appendingPathComponent("previous.html")) == v2, "the new copy became the reference")845        #expect(try Data(contentsOf: state.appendingPathComponent("current.html")) == v2)846        #expect(third.output("sha256")?.stringValue == SHA256.hash(data: v2).map { String(format: "%02x", $0) }.joined())847        #expect(server.requests == ["/page.html", "/page.html", "/page.html"], "exactly one GET per run: \(server.requests)")848849        // A URL on a host the accepted permission does not cover is not fetched.850        var elsewhere = parameters851        elsewhere["url"] = .string("http://localhost:\(server.port)/page.html")852        let refused = try await h.run("website-change-monitor", elsewhere)853        #expect(refused.status == .waitingApproval, "\(refused.status)")854        #expect(refused.step("fetch")?.status == .notApproved)855        #expect(server.requests.count == 3, "no request left for the other host")856        #expect(h.approvals.toolRequests.count == 1 && h.approvals.toolRequests.first?.hasPrefix("http.request") == true)857    }858859    @Test("09 website-change-monitor, headless and without accepted permissions: no request is sent and nothing is written")860    func websiteChangeMonitorWithoutPermissions() async throws {861        let h = try await ExampleHarness(interactive: false)862        defer { Task { await h.tearDown() } }863        let server = try await LoopbackServer { _ in (200, "text/html", Data("<p>never fetched</p>".utf8)) }864        defer { server.stop() }865        let state = h.workspace.appendingPathComponent("website-change-monitor/State")866        let run = try await h.run("website-change-monitor", ["host": "127.0.0.1", "url": .string("http://127.0.0.1:\(server.port)/page.html"), "stateFolder": h.path(state)],867                                  accept: false, mode: .scheduled)868        #expect(run.status == .waitingApproval || run.status == .blockedPermission, "\(run.status)")869        #expect(server.requests.isEmpty, "nothing may be sent without the network permission")870        #expect(!Files.exists(state), "nothing may be written")871    }872873    // MARK: 10 · System health report874875    @Test("10 system-health-report: the report holds the real machine facts and a plausible free-space fraction")876    func systemHealthReport() async throws {877        let h = try await ExampleHarness()878        defer { Task { await h.tearDown() } }879        let folder = h.workspace.appendingPathComponent("system-health-report/Reports")880        let run = try await h.run("system-health-report", ["outputFolder": h.path(folder)])881        let fraction = try #require(run.output("freeFraction")?.doubleValue)882        let low = fraction < 0.1883        if low { expectOnlyTheNotificationFailed(run, step: "warn") } else { #expect(run.status == .succeeded, "\(run.diagnosis)"); #expect(run.step("warn")?.status == .skipped) }884        #expect(run.output("lowDisk") == .bool(low))885        // The same figure, read by the test straight from the system.886        let values = try URL(fileURLWithPath: "/").resourceValues(forKeys: [.volumeTotalCapacityKey, .volumeAvailableCapacityForImportantUsageKey])887        let expected = Double(values.volumeAvailableCapacityForImportantUsage ?? 0) / Double(values.volumeTotalCapacity ?? 1)888        #expect(fraction > 0 && fraction <= 1 && abs(fraction - expected) < 0.05, "free fraction \(fraction) vs \(expected)")889        let reportPath = try #require(run.output("report")?.stringValue)890        let report = try Files.text(URL(fileURLWithPath: reportPath))891        for section in ["# System health", "## Machine", "## Disk", "## Busiest processes", "## Power", "## Network"] { #expect(report.contains(section)) }892        #expect(report.contains("\"cores\":\(ProcessInfo.processInfo.processorCount)") || report.contains("\"cores\": \(ProcessInfo.processInfo.processorCount)"))893        #expect(report.contains("\"memoryBytes\"") && report.contains("\"cpuPercent\"") && report.contains("\"onACPower\"") && report.contains("\"online\""))894        let v = ProcessInfo.processInfo.operatingSystemVersion895        #expect(run.output("osVersion")?.stringValue?.hasPrefix("\(v.majorVersion).\(v.minorVersion)") == true, "\(String(describing: run.output("osVersion")))")896        #expect(Set(Files.tree(folder).keys).count == 1, "the report is the only file written")897        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")898    }899900    // MARK: 12 · Document metadata catalogue901902    @Test("12 document-metadata-catalog: one row per file with the right size and SHA-256")903    func documentMetadataCatalog() async throws {904        let h = try await ExampleHarness()905        defer { Task { await h.tearDown() } }906        let ws = try h.copySample("document-metadata-catalog")907        let documents = ws.appendingPathComponent("Documents"), catalog = ws.appendingPathComponent("catalog.csv")908        let before = Files.tree(documents).filter { $0.value != "dir" }909        let run = try await h.run("document-metadata-catalog", ["documentsFolder": h.path(documents), "catalogFile": h.path(catalog)])910        #expect(run.status == .succeeded, "\(run.diagnosis)")911        let lines = try Files.text(catalog).split(whereSeparator: \.isNewline).map(String.init)912        #expect(lines.first == "path,extension,bytes,modified,contentType,sha256" && lines.count == before.count + 1)913        var total = 0914        for line in lines.dropFirst() {915            let c = line.split(separator: ",", omittingEmptySubsequences: false).map(String.init)916            #expect(c.count == 6, "\(line)")917            let file = documents.appendingPathComponent(c[0])918            let size = (try FileManager.default.attributesOfItem(atPath: file.path)[.size] as? Int) ?? -1919            #expect(before[c[0]] == c[5], "\(c[0]): SHA-256 in the catalogue matches the file")920            #expect(Int(c[2]) == size && c[1] == "." + file.pathExtension.lowercased())921            #expect(ISO8601DateFormatter().date(from: c[3]) != nil, "modified is an ISO 8601 date: \(c[3])")922            total += size923        }924        #expect(Set(lines.dropFirst().map { String($0.split(separator: ",")[0]) }) == Set(before.keys))925        #expect(run.output("files") == .int(before.count) && run.output("totalBytes") == .int(total))926        #expect(Files.tree(documents).filter { $0.value != "dir" } == before, "the documents are read only")927        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")928    }929930    // MARK: 13 · Scheduled window screenshot (Screen Recording)931932    /// An application that has a normal window on screen right now and is not one the capture tool refuses. Finder and933    /// Terminal are preferred over whatever else is open: the picture is deleted right after the assertion, but a934    /// file-manager or terminal window is the least personal thing to capture.935    private func applicationWithAWindow() -> String? {936        let list = (CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]]) ?? []937        var found: [String] = []938        for w in list where (w[kCGWindowLayer as String] as? Int) == 0 {939            guard let pid = w[kCGWindowOwnerPID as String] as? pid_t, let id = NSRunningApplication(processIdentifier: pid)?.bundleIdentifier else { continue }940            let bounds = w[kCGWindowBounds as String] as? [String: Any]941            guard (bounds?["Width"] as? Double ?? 0) > 100, (bounds?["Height"] as? Double ?? 0) > 100 else { continue }942            if !ScreenCaptureTool.defaultSensitiveBundleIDs.contains(id.lowercased()), !found.contains(id) { found.append(id) }943        }944        return ["com.apple.finder", "com.apple.Terminal"].first(where: found.contains) ?? found.first945    }946947    @Test("13 scheduled-window-screenshot: skips cleanly without a window; captures a real window only when Screen Recording is granted")948    func scheduledWindowScreenshot() async throws {949        let h = try await ExampleHarness()950        defer { Task { await h.tearDown() } }951        let folder = h.workspace.appendingPathComponent("scheduled-window-screenshot/Captures")952953        // An application that is not running: the run succeeds, says so, launches nothing and writes no picture.954        let absent = try await h.run("scheduled-window-screenshot", ["bundleID": "ai.spboucher.orix.tests.not-installed", "outputFolder": h.path(folder)])955        #expect(absent.status == .succeeded, "\(absent.receipt.errors)")956        #expect(absent.output("captured") == .bool(false) && absent.step("capture")?.status == .skipped)957        #expect(Files.tree(folder).isEmpty)958        let review = try await h.services.workflows.permissionReview(workflowID: "scheduled-window-screenshot", parameters: ["bundleID": "com.apple.finder", "outputFolder": h.path(folder)])959        #expect(review.proposed.map(\.capability).contains(.screenCapture(scope: "window:com.apple.finder")), "the screen scope follows the parameter, one window, never the display")960961        let granted = MacPermissions.status(.screenRecording) == .granted962        guard let bundleID = applicationWithAWindow() else {963            print("SKIPPED(no application window on screen): scheduled-window-screenshot capture; the no-window path was verified (screenRecording \(granted ? "granted" : "not granted"))")964            return965        }966        let run = try await h.run("scheduled-window-screenshot", ["bundleID": .string(bundleID), "outputFolder": h.path(folder)])967        if granted {968            #expect(run.status == .succeeded, "\(run.diagnosis)")969            #expect(run.output("captured") == .bool(true))970            let filePath = try #require(run.output("file")?.stringValue)971            let file = URL(fileURLWithPath: filePath)972            let size = try #require(Files.imageSize(file), "the capture is not a readable image")973            #expect(size.type == "public.png" && size.width > 50 && size.height > 50)974            #expect(run.step("capture")?.output["pixelWidth"] == .int(size.width) && run.step("capture")?.output["pixelHeight"] == .int(size.height))975            #expect(Files.tree(folder).count == 1)976            #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")977            try? FileManager.default.removeItem(at: file)   // a picture of a real window: not kept a moment longer than the assertion978            print("VERIFIED(screenRecording granted): scheduled-window-screenshot captured a \(size.width)×\(size.height) px window of \(bundleID)")979        } else {980            #expect(run.status == .blockedPermission, "\(run.status)")981            #expect(run.step("capture")?.status == .blockedPermission)982            #expect(Files.tree(folder).isEmpty, "no picture without Screen Recording")983            print("SKIPPED(screenRecording not granted): scheduled-window-screenshot; BLOCKED_PERMISSION verified, no prompt triggered")984        }985    }986987    // MARK: 14 · Spreadsheet validation988989    @Test("14 spreadsheet-validation: a valid workbook passes; a broken one fails the run and the report says why")990    func spreadsheetValidation() async throws {991        let h = try await ExampleHarness()992        defer { Task { await h.tearDown() } }993        let ws = try h.copySample("spreadsheet-validation")994        let reports = ws.appendingPathComponent("Reports")995        let good = ws.appendingPathComponent("data.xlsx"), bad = ws.appendingPathComponent("data-invalid.xlsx")996        let hashes = [try Files.sha256(good), try Files.sha256(bad)]997998        let valid = try await h.run("spreadsheet-validation", ["workbook": h.path(good), "reportFolder": h.path(reports)])999        #expect(valid.status == .succeeded, "\(valid.receipt.errors)")1000        #expect(valid.output("valid") == .bool(true) && valid.output("rows") == .int(4) && valid.output("columns") == ["id", "name", "amount"])1001        var report = try Files.text(reports.appendingPathComponent("validation.md"))1002        #expect(report.contains("Sheet: Orders") && report.contains("Data rows read: 4") && report.contains("Valid: true"))1003        #expect(valid.step("read")?.output["rows"]?.arrayValue?.last == [1004, "Delta", 195.5], "a formula cell is read as its cached value")10041005        let invalid = try await h.run("spreadsheet-validation", ["workbook": h.path(bad), "reportFolder": h.path(reports)])1006        #expect(invalid.status == .failed, "a broken workbook fails the run on purpose")1007        #expect(invalid.failedSteps == ["check"] && invalid.output("valid") == nil)1008        report = try Files.text(reports.appendingPathComponent("validation.md"))1009        #expect(report.contains("Missing required columns: 1 amount") && report.contains("Fully empty rows: 1") && report.contains("Rows without a key value: 2") && report.contains("Valid: false"))1010        let failed = invalid.receipt.verifications.filter { !$0.passed }1011        #expect(failed.count == 3, "\(failed.map(\.assertion))")1012        #expect([try Files.sha256(good), try Files.sha256(bad)] == hashes, "workbooks are never modified")1013        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")1014    }10151016    // MARK: 15 · Browser data entry (WKWebView against a loopback test site)10171018    @Test("15 browser-data-entry: fills both pages of the test site in the real WKWebView, submits after approval, keeps a screenshot")1019    func browserDataEntry() async throws {1020        let h = try await ExampleHarness()1021        defer { Task { await h.tearDown() } }1022        let ws = try h.copySample("browser-data-entry")1023        let server = try await LoopbackServer.site(ws.appendingPathComponent("site"))1024        defer { server.stop() }1025        let evidence = ws.appendingPathComponent("Evidence")1026        let parameters: [String: JSONValue] = ["host": "127.0.0.1", "pageURL": .string("http://127.0.0.1:\(server.port)/form.html"),1027                                               "fullName": "Grace Hopper", "email": "grace@example.org", "plan": "company", "evidenceFolder": h.path(evidence)]1028        let run = try await h.run("browser-data-entry", parameters)1029        #expect(run.status == .succeeded, "\(run.diagnosis)")1030        #expect(run.output("confirmation")?.stringValue?.contains("Registered Grace Hopper <grace@example.org> on plan company.") == true, "\(String(describing: run.output("confirmation")))")1031        #expect(run.output("page")?.stringValue?.contains("/done.html?") == true)1032        let pages = server.requests.filter { $0.contains(".html") }1033        #expect(pages.count == 3 && pages[0] == "/form.html", "\(pages)")1034        #expect(pages[1].hasPrefix("/details.html?") && pages[1].contains("name=Grace") && pages[1].contains("email=grace%40example.org"), "\(pages)")1035        #expect(pages[2].hasPrefix("/done.html?") && pages[2].contains("plan=company") && pages[2].contains("terms=yes") && pages[2].contains("name=Grace"), "\(pages)")1036        let shot = evidence.appendingPathComponent("confirmation.png")1037        let size = try #require(Files.imageSize(shot))1038        #expect(size.type == "public.png" && size.width > 100 && size.height > 100)1039        #expect(run.output("screenshotSha256")?.stringValue == (try Files.sha256(shot)))1040        #expect(h.approvals.gates.count == 1 && h.approvals.gates.first?.contains("Grace Hopper") == true, "the gate names what will be submitted")1041        #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")1042        #expect(await h.services.browser.session(runID: run.record.id) == nil, "the browser session is closed")1043    }10441045    @Test("15 browser-data-entry: a refused approval submits nothing")1046    func browserDataEntryRefused() async throws {1047        let h = try await ExampleHarness(approveGates: false)1048        defer { Task { await h.tearDown() } }1049        let ws = try h.copySample("browser-data-entry")1050        let server = try await LoopbackServer.site(ws.appendingPathComponent("site"))1051        defer { server.stop() }1052        let evidence = ws.appendingPathComponent("Evidence")1053        let run = try await h.run("browser-data-entry", ["host": "127.0.0.1", "pageURL": .string("http://127.0.0.1:\(server.port)/form.html"), "evidenceFolder": h.path(evidence)])1054        #expect(run.status == .failed, "a denied gate fails the run: \(run.status)")1055        #expect(run.failedSteps == ["confirm"] && run.step("submit")?.status != .succeeded)1056        #expect(!server.requests.contains { $0.hasPrefix("/done.html") }, "the form was not submitted: \(server.requests)")1057        #expect(server.requests.contains { $0.hasPrefix("/details.html?") }, "the first page was filled before the gate")1058        #expect(!Files.exists(evidence.appendingPathComponent("confirmation.png")))1059        #expect(await h.services.browser.session(runID: run.record.id) == nil, "the browser session is closed when the run ends")1060    }1061}1062
Files main
Menu

Theme