Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXServices/Tests/ORIXServicesTests/ExamplesEndToEndTests.swift10// Description : Runs each of the 15 built-in example automations for real11// (real services on a temporary data folder, real tools,12// sample data copied into a temporary workspace) and checks13// the artifacts by content.14// Module : ORIXServicesTests15// Repository : https://git.spboucher.ai/orix-automation16// Created : 2026-11-2117// Last Modified : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author : Simon-Pierre Boucher24// Position : Professor of Finance25// Institution : Université du Québec en Outaouais (UQO)26// Department : Département des sciences administratives27// Organization : SP Boucher28//29// Website : https://spboucher.ai30// Git : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General : info@spboucher.ai37// Professional : contact@spboucher.ai38// Direct : simon@spboucher.ai39// Research : research@spboucher.ai40// Development : dev@spboucher.ai41// Support : support@spboucher.ai42// Security/Admin : admin@spboucher.ai43// Notifications : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification : Public70// Distribution : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License : Proprietary, all rights reserved (see license file)84// License File : LICENSE85// Maintainer : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import AppKit93import CoreGraphics94import CryptoKit95import Foundation96import ImageIO97import Network98import PDFKit99import Testing100import ORIXAppleAutomation101import ORIXCore102import ORIXServices103import ORIXWorkflowEngine104105// MARK: - Support106107/// What the test's "person at the screen" answered. Manual gates of a workflow are approved (or refused on request);108/// a *tool* approval is always refused and recorded: with the declared permissions accepted, no tool call of an109/// example may need one. This is stricter than `orixctl --approve all`, which would hide a missing permission.110final class ApprovalLog: @unchecked Sendable {111 private let lock = NSLock()112 private var _gates: [String] = []113 private var _tools: [String] = []114 var gates: [String] { lock.lock(); defer { lock.unlock() }; return _gates }115 var toolRequests: [String] { lock.lock(); defer { lock.unlock() }; return _tools }116 func gate(_ text: String) { lock.lock(); _gates.append(text); lock.unlock() }117 func tool(_ text: String) { lock.lock(); _tools.append(text); lock.unlock() }118}119120/// A loopback-only HTTP/1.1 server for the tests (GET only, one request per connection).121final class LoopbackServer: @unchecked Sendable {122 typealias Handler = @Sendable (_ path: String) -> (status: Int, type: String, body: Data)123 private let listener: NWListener124 private let queue = DispatchQueue(label: "orix.tests.loopback")125 private let lock = NSLock()126 private var _requests: [String] = []127 private var handler: Handler128 private(set) var port: UInt16 = 0129130 /// Request targets (path and query) received so far, in order.131 var requests: [String] { lock.lock(); defer { lock.unlock() }; return _requests }132 func setHandler(_ h: @escaping Handler) { lock.lock(); handler = h; lock.unlock() }133134 init(handler: @escaping Handler) async throws {135 self.handler = handler136 let parameters = NWParameters.tcp137 parameters.requiredLocalEndpoint = .hostPort(host: .ipv4(.loopback), port: .any)138 parameters.allowLocalEndpointReuse = true139 listener = try NWListener(using: parameters)140 listener.newConnectionHandler = { [weak self] connection in self?.serve(connection) }141 let started = AsyncStream<Bool>.makeStream()142 listener.stateUpdateHandler = { state in143 switch state {144 case .ready: started.continuation.yield(true); started.continuation.finish()145 case .failed, .cancelled: started.continuation.yield(false); started.continuation.finish()146 default: break147 }148 }149 listener.start(queue: queue)150 var ready = false151 for await ok in started.stream { ready = ok }152 guard ready, let p = listener.port?.rawValue else { throw CocoaError(.fileNoSuchFile) }153 port = p154 }155156 /// Serves the files of one folder (no directory listing, no path outside the folder).157 static func site(_ folder: URL) async throws -> LoopbackServer {158 let root = folder.resolvingSymlinksInPath().path159 return try await LoopbackServer { target in160 let path = String(target.split(separator: "?", maxSplits: 1, omittingEmptySubsequences: false).first ?? "")161 let file = URL(fileURLWithPath: root + path).standardizedFileURL.path162 guard file.hasPrefix(root + "/"), let data = FileManager.default.contents(atPath: file) else { return (404, "text/plain", Data("not found".utf8)) }163 let type = file.hasSuffix(".css") ? "text/css; charset=utf-8" : "text/html; charset=utf-8"164 return (200, type, data)165 }166 }167168 private func serve(_ connection: NWConnection) {169 connection.start(queue: queue)170 receive(connection, buffer: Data())171 }172173 private func receive(_ connection: NWConnection, buffer: Data) {174 connection.receive(minimumIncompleteLength: 1, maximumLength: 65_536) { [weak self] data, _, complete, error in175 guard let self else { connection.cancel(); return }176 var buffer = buffer177 if let data { buffer.append(data) }178 guard let end = buffer.range(of: Data("\r\n\r\n".utf8)) else {179 if complete || error != nil || buffer.count > 262_144 { connection.cancel() } else { self.receive(connection, buffer: buffer) }180 return181 }182 let head = String(decoding: buffer[..<end.lowerBound], as: UTF8.self)183 let parts = (head.split(separator: "\r\n").first ?? "").split(separator: " ")184 let target = parts.count >= 2 ? String(parts[1]) : "/"185 self.lock.lock(); self._requests.append(target); let handler = self.handler; self.lock.unlock()186 let reply = parts.first == "GET" ? handler(target) : (405, "text/plain", Data("GET only".utf8))187 var out = Data("HTTP/1.1 \(reply.0) \(reply.0 == 200 ? "OK" : "Error")\r\nContent-Type: \(reply.1)\r\nContent-Length: \(reply.2.count)\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n".utf8)188 out.append(reply.2)189 connection.send(content: out, completion: .contentProcessed { _ in connection.cancel() })190 }191 }192193 func stop() { listener.cancel() }194}195196enum Repo {197 static let root: URL = {198 var url = URL(fileURLWithPath: #filePath)199 for _ in 0..<5 { url.deleteLastPathComponent() }200 return url201 }()202 static var examples: URL { root.appendingPathComponent("Workflows/Examples") }203 static var mirror: URL { root.appendingPathComponent("Packages/ORIXWorkflowEngine/Sources/ORIXWorkflowEngine/Resources/Examples") }204 static var samples: URL { root.appendingPathComponent("Workflows/Samples") }205206 static func exampleFile(slug: String) throws -> URL {207 let files = try FileManager.default.contentsOfDirectory(at: examples, includingPropertiesForKeys: nil).filter { $0.pathExtension == "yaml" }208 guard let f = files.first(where: { $0.deletingPathExtension().lastPathComponent.dropFirst(3) == slug }) else { throw CocoaError(.fileNoSuchFile) }209 return f210 }211}212213enum Files {214 static func sha256(_ url: URL) throws -> String {215 SHA256.hash(data: try Data(contentsOf: url)).map { String(format: "%02x", $0) }.joined()216 }217218 /// Relative path → SHA-256 for every regular file under a folder (hidden files included); folders map to "dir".219 static func tree(_ root: URL) -> [String: String] {220 var out: [String: String] = [:]221 let base = root.resolvingSymlinksInPath().path222 guard let walker = FileManager.default.enumerator(atPath: base) else { return out }223 while let rel = walker.nextObject() as? String {224 var isDir: ObjCBool = false225 FileManager.default.fileExists(atPath: base + "/" + rel, isDirectory: &isDir)226 out[rel] = isDir.boolValue ? "dir" : ((try? sha256(URL(fileURLWithPath: base + "/" + rel))) ?? "unreadable")227 }228 return out229 }230231 static func exists(_ url: URL) -> Bool { FileManager.default.fileExists(atPath: url.path) }232 static func text(_ url: URL) throws -> String { try String(contentsOf: url, encoding: .utf8) }233234 static func imageSize(_ url: URL) -> (width: Int, height: Int, type: String)? {235 guard let source = CGImageSourceCreateWithURL(url as CFURL, nil), let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else { return nil }236 return (image.width, image.height, (CGImageSourceGetType(source) as String?) ?? "")237 }238239 /// Runs a program the test itself needs (git, ditto, the sample setup script) and returns its output.240 @discardableResult241 static func run(_ executable: String, _ arguments: [String], cwd: URL? = nil) throws -> (code: Int32, out: String) {242 let p = Process()243 p.executableURL = URL(fileURLWithPath: executable)244 p.arguments = arguments245 p.currentDirectoryURL = cwd246 var env = ["PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "LC_ALL": "C", "HOME": NSTemporaryDirectory()]247 env["GIT_CONFIG_GLOBAL"] = "/dev/null"; env["GIT_CONFIG_SYSTEM"] = "/dev/null"248 p.environment = env249 let pipe = Pipe()250 p.standardOutput = pipe; p.standardError = pipe251 try p.run()252 let data = pipe.fileHandleForReading.readDataToEndOfFile()253 p.waitUntilExit()254 return (p.terminationStatus, String(decoding: data, as: UTF8.self))255 }256}257258struct FinishedRun {259 let record: RunRecord260 let receipt: RunReceipt261 let outcome: WorkflowOutcome262 var status: RunStatus { record.status }263 func output(_ name: String) -> JSONValue? { outcome.outputs[name] }264 func step(_ id: String) -> StepResult? { outcome.step(id) }265 /// Why a run did not end as expected: errors, failed verifications and tool calls that did not succeed.266 var diagnosis: String {267 let checks = receipt.verifications.filter { !$0.passed }.map { "FAIL \($0.assertion) — \($0.evidence)" }268 let calls = receipt.toolCalls.filter { $0.status != .succeeded }.map { "\($0.toolID) \($0.status.rawValue): \($0.summary) \($0.arguments.serialized().prefix(300))" }269 return (receipt.errors + checks + calls).joined(separator: "\n")270 }271 /// Steps that ended in anything else than succeeded / skipped / simulated.272 var failedSteps: [String] { outcome.steps.filter { !$0.status.isSuccess && $0.status != .skipped }.map(\.stepID) }273}274275/// Real services on a temporary data folder plus a temporary workspace for the sample data.276final class ExampleHarness: @unchecked Sendable {277 let root: URL278 let workspace: URL279 let services: ORIXServices280 let approvals = ApprovalLog()281 private var tasks: [Task<Void, Never>] = []282 private var saved: Set<String> = []283284 static let devKeyName = "ORIX_DEV_ANTHROPIC_API_KEY"285 static var hasDevKey: Bool { !(ProcessInfo.processInfo.environment[devKeyName] ?? "").isEmpty }286 static let testModel = ModelRef(provider: .anthropic, model: "claude-haiku-5-5")287288 /// - Parameters:289 /// - interactive: false = a headless process: nobody answers approvals.290 /// - approveGates: what the simulated person answers to a workflow's manual gate.291 /// - live: use the development provider key from the environment and Haiku as the default model.292 init(interactive: Bool = true, approveGates: Bool = true, live: Bool = false) async throws {293 let base = URL(fileURLWithPath: NSTemporaryDirectory()).resolvingSymlinksInPath().appendingPathComponent("orix-examples-\(UUID().uuidString.prefix(8))")294 root = base295 workspace = base.appendingPathComponent("workspace")296 try FileManager.default.createDirectory(at: workspace, withIntermediateDirectories: true)297 var options = ServiceOptions()298 options.interactive = interactive299 options.allowDevelopmentCredentials = live300 options.keychainService = "ai.spboucher.orix.tests.examples.\(UUID().uuidString.prefix(8))" // never the user's real items301 services = try await ORIXServices.bootstrap(paths: ORIXPaths(root: base.appendingPathComponent("data")), options: options)302 if live {303 var settings = await services.runs.settings()304 settings.defaultModel = Self.testModel305 settings.dailySpendLimitUSD = 0.25306 try await services.runs.save(settings)307 }308 guard interactive else { return }309 let center = services.approvals, interaction = services.interaction, log = approvals310 tasks.append(Task {311 for await pending in center.updates() {312 for p in pending {313 if p.access.toolID == "workflow.approval" {314 log.gate(p.access.summary)315 await center.resolve(id: p.id, response: approveGates ? .approveOnce : .deny(reason: "refused by the test"))316 } else {317 log.tool("\(p.access.toolID): \(p.reason)")318 await center.resolve(id: p.id, response: .deny(reason: "the declared permissions of an example must be sufficient"))319 }320 }321 }322 })323 tasks.append(Task {324 for await items in interaction.updates() {325 for item in items {326 switch item {327 case .plan(let p): await interaction.review(planID: p.id, .approved)328 case .question(let q): await interaction.answer(questionID: q.id, nil)329 }330 }331 }332 })333 }334335 func tearDown() async {336 for t in tasks { t.cancel() }337 await services.shutdown()338 try? FileManager.default.removeItem(at: root)339 }340341 /// Copies `Workflows/Samples/<slug>` into the workspace and returns the copy.342 func copySample(_ slug: String) throws -> URL {343 let target = workspace.appendingPathComponent(slug)344 try FileManager.default.copyItem(at: Repo.samples.appendingPathComponent(slug), to: target)345 return target346 }347348 /// Saves the example exactly as shipped (once) — or a variant with a tighter cost budget for live agent runs.349 @discardableResult350 func save(_ slug: String, costBudgetUSD: Double? = nil) async throws -> SavedWorkflow {351 var source = try String(contentsOf: Repo.exampleFile(slug: slug), encoding: .utf8)352 if let costBudgetUSD {353 // Same workflow, smaller spending ceiling: a test must not be able to spend what a user's real run may.354 let pattern = #"maxProviderCostUSD: [0-9.]+"#355 guard let range = source.range(of: pattern, options: .regularExpression) else { throw CocoaError(.formatting) }356 source.replaceSubrange(range, with: "maxProviderCostUSD: \(costBudgetUSD)")357 }358 let s = try await services.workflows.save(source: source)359 saved.insert(slug)360 return s361 }362363 func run(_ slug: String, _ parameters: [String: JSONValue] = [:], accept: Bool = true, dryRun: Bool = false, mode: ExecutionMode = .guided) async throws -> FinishedRun {364 if !saved.contains(slug) { try await save(slug) }365 if accept { try await services.workflows.acceptPermissions(workflowID: slug, parameters: parameters) }366 var request = WorkflowRunRequest(workflowID: slug)367 request.parameters = parameters368 request.dryRun = dryRun369 request.mode = mode370 request.startPaused = true // hold the run until the test has its handle, so the outcome (outputs, step results) can be read371 let runID = try await services.workflows.startRun(request)372 let handle = try #require(await services.workflows.handle(runID: runID))373 await handle.resume()374 let outcome = await handle.outcome()375 let record = try #require(await services.runs.wait(runID: runID))376 let receipt = try #require(try await services.store.receipt(runID: runID))377 if ProcessInfo.processInfo.environment["ORIX_TEST_DUMP_EVENTS"] == "1" {378 // Debugging aid: the persisted (already redacted) event log of the run.379 for e in (try? await services.store.events(runID: runID, afterSeq: 0, limit: 5000)) ?? [] {380 print("EVENT \(slug) \(e.seq) \(e.kind.rawValue) \(e.stepID ?? "-") \(e.text.replacingOccurrences(of: "\n", with: " ⏎ ").prefix(700)) \(e.data.serialized().prefix(500))")381 }382 }383 return FinishedRun(record: record, receipt: receipt, outcome: outcome)384 }385386 func path(_ url: URL) -> JSONValue { .string(url.path) }387}388389/// The notification step of an example cannot work under `swift test` (no application bundle): it fails by itself,390/// honestly, and the run ends PARTIALLY_SUCCEEDED. This checks that it is the only thing that went wrong.391func expectOnlyTheNotificationFailed(_ run: FinishedRun, step: String, sourceLocation: SourceLocation = #_sourceLocation) {392 #expect(run.status == .partiallySucceeded, "\(run.diagnosis)", sourceLocation: sourceLocation)393 #expect(run.failedSteps == [step], sourceLocation: sourceLocation)394 #expect(run.step(step)?.error?.contains("application bundle") == true, "\(run.step(step)?.error ?? "nil")", sourceLocation: sourceLocation)395 #expect(run.step(step)?.continued == true, sourceLocation: sourceLocation)396}397398let allExampleSlugs = [399 "smart-downloads-organizer", "pdf-to-structured-report", "weekly-research-digest", "git-test-and-review", "csv-cleanup-and-charts",400 "batch-image-conversion", "calendar-preparation", "folder-backup-and-verify", "website-change-monitor", "system-health-report",401 "meeting-notes-organizer", "document-metadata-catalog", "scheduled-window-screenshot", "spreadsheet-validation", "browser-data-entry",402]403404// MARK: - Tests405406/// The suite's type name is a raw identifier so that `swift test --filter "Integration · examples"` selects it407/// (the filter matches identifiers, not display names). Real services, real tools, sample data; a live model and the408/// macOS privacy permissions are used only when they are available, and every skip prints `SKIPPED(<reason>)`.409@Suite(.serialized, .timeLimit(.minutes(10)))410struct `Integration · examples` {411412 // MARK: Validation against the real manifests413414 @Test("All 15 examples validate with zero errors and zero warnings against the registered tools, and the mirror is identical")415 func examplesValidateAgainstRealManifests() async throws {416 let h = try await ExampleHarness()417 defer { Task { await h.tearDown() } }418 let registered = Set(await h.services.executor.manifests().map(\.identifier))419 for slug in allExampleSlugs {420 let file = try Repo.exampleFile(slug: slug)421 let source = try String(contentsOf: file, encoding: .utf8)422 let (doc, diagnostics) = await h.services.workflows.diagnose(source)423 let problems = diagnostics.filter { $0.severity == .error || $0.severity == .warning }424 #expect(problems.isEmpty, "\(slug): \(problems.map { "line \($0.line): \($0.message)" }.joined(separator: "\n"))")425 let document = try #require(doc, "\(slug) does not parse")426 #expect(document.metadata.id == slug)427 for located in document.allSteps {428 if case .tool(let t) = located.step.kind { #expect(registered.contains(t.tool), "\(slug): \(t.tool) is not a registered tool") }429 if case .agent(let a) = located.step.kind, case .inline(let inline) = a.agent {430 #expect(inline.tools.allSatisfy(registered.contains), "\(slug): agent tools \(inline.tools)")431 }432 }433 #expect(try Data(contentsOf: file) == Data(contentsOf: Repo.mirror.appendingPathComponent(file.lastPathComponent)), "\(slug): resource copy differs")434 #expect(Files.exists(Repo.samples.appendingPathComponent(slug)), "\(slug) has no sample folder")435 // Saving is what the template gallery does: it must accept the example as shipped.436 let saved = try await h.save(slug)437 #expect(saved.id == slug && saved.version == 1)438 }439 }440441 // MARK: 1 · Smart Downloads Organizer442443 @Test("01 smart-downloads-organizer: every file is moved to its category, nothing is deleted or altered")444 func smartDownloadsOrganizer() async throws {445 let h = try await ExampleHarness()446 defer { Task { await h.tearDown() } }447 let ws = try h.copySample("smart-downloads-organizer")448 let source = ws.appendingPathComponent("Downloads"), destination = ws.appendingPathComponent("Sorted")449 let before = Files.tree(source)450 let run = try await h.run("smart-downloads-organizer", ["source": h.path(source), "destination": h.path(destination)])451 #expect(run.status == .succeeded, "\(run.diagnosis)")452453 let expected: [String: [String]] = [454 "Documents": ["notes.txt", "reading-list.md", "SCAN 2026.TXT", "memo.pdf"], "Spreadsheets": ["budget.csv"], "Images": ["photo.png"],455 "Archives": ["old-project.zip"], "Audio": ["jingle.mp3"], "Video": ["clip.mov"], "Other": ["LICENSE", "data.xyz"],456 ]457 let after = Files.tree(destination)458 for (category, names) in expected {459 for name in names { #expect(after["\(category)/\(name)"] == before[name], "\(name) should be in \(category) with the same content") }460 }461 let moved = expected.values.reduce(0) { $0 + $1.count }462 #expect(after.values.filter { $0 != "dir" }.count == moved, "exactly the moved files are in the destination: \(after.keys.sorted())")463 // What stays: the subfolder and its content, untouched. Nothing was deleted: moved + remaining == before.464 let remaining = Files.tree(source)465 #expect(remaining == ["Projects": "dir", "Projects/keep-me.txt": before["Projects/keep-me.txt"]!])466 #expect(Set(after.values.filter { $0 != "dir" }).union(remaining.values.filter { $0 != "dir" }) == Set(before.values.filter { $0 != "dir" }))467 #expect(run.output("moved") == .int(moved))468 #expect(h.approvals.gates.count == 1, "the manual gate was asked once")469 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")470 #expect(run.receipt.verifications.count == 2 && run.receipt.verifications.allSatisfy(\.passed))471 }472473 @Test("01 smart-downloads-organizer, headless and without accepted permissions: waits for approval and moves nothing")474 func smartDownloadsOrganizerWithoutPermissions() async throws {475 let h = try await ExampleHarness(interactive: false)476 defer { Task { await h.tearDown() } }477 let ws = try h.copySample("smart-downloads-organizer")478 let before = Files.tree(ws)479 let run = try await h.run("smart-downloads-organizer", ["source": h.path(ws.appendingPathComponent("Downloads")), "destination": h.path(ws.appendingPathComponent("Sorted"))], accept: false)480 #expect(run.status == .waitingApproval || run.status == .blockedPermission, "\(run.status)")481 #expect(Files.tree(ws) == before, "nothing may change on disk")482 #expect(!run.receipt.toolCalls.contains { $0.status == .succeeded }, "no tool call may have run")483 }484485 // MARK: 2 · PDF to structured data and report486487 @Test("02 pdf-to-structured-report: CSV catalogue rows match the PDFs and the report PDF is readable")488 func pdfToStructuredReport() async throws {489 let h = try await ExampleHarness()490 defer { Task { await h.tearDown() } }491 let ws = try h.copySample("pdf-to-structured-report")492 let input = ws.appendingPathComponent("Input"), output = ws.appendingPathComponent("Output")493 let before = Files.tree(input)494 let run = try await h.run("pdf-to-structured-report", ["inputFolder": h.path(input), "outputFolder": h.path(output)])495 #expect(run.status == .succeeded, "\(run.diagnosis)")496497 let lines = try Files.text(output.appendingPathComponent("catalog.csv")).split(whereSeparator: \.isNewline).map(String.init)498 #expect(lines.first == "file,pages,title,textCharacters")499 var rows: [String: [String]] = [:]500 for line in lines.dropFirst() { let c = line.split(separator: ",", omittingEmptySubsequences: false).map(String.init); rows[c[0]] = c }501 #expect(rows.count == 4 && lines.count == 5)502 #expect(rows["field-notes.pdf"]?[1] == "3" && rows["field-notes.pdf"]?[2] == "Field notes")503 #expect(rows["quarterly-summary.pdf"]?[1] == "1" && rows["quarterly-summary.pdf"]?[2] == "Quarterly summary Q3")504 #expect(rows["untitled-memo.pdf"]?[2] == "", "a PDF without a Title property has an empty title")505 #expect(rows["scan-without-text.pdf"]?[3] == "0", "a page without a text layer has no text")506 for name in ["field-notes.pdf", "quarterly-summary.pdf", "untitled-memo.pdf"] { #expect((Int(rows[name]?[3] ?? "") ?? 0) > 20, "\(name) has a text layer") }507 // Page counts are checked against PDFKit directly, independently of the tool.508 for (rel, _) in before where rel.hasSuffix(".pdf") {509 let pages = PDFDocument(url: input.appendingPathComponent(rel))?.pageCount510 #expect(rows[(rel as NSString).lastPathComponent]?[1] == pages.map(String.init))511 }512 let report = try #require(PDFDocument(url: output.appendingPathComponent("report.pdf")))513 #expect(report.pageCount == 1)514 let text = report.string ?? ""515 #expect(text.contains("PDF report") && text.contains("field-notes.pdf") && text.contains("Documents: 4"))516 #expect(try Files.text(output.appendingPathComponent("report.md")).contains("| field-notes.pdf | 3 | Field notes |"))517 #expect(run.output("documents") == .int(4) && run.output("pages") == .int(6))518 #expect(Files.tree(input) == before, "the input folder is read only")519 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")520 }521522 // MARK: 3, 4, 11 · Examples with an agent step523524 /// How a dry run accounts for the agent step of an example. A dry run executes no read tool either, so what the525 /// agent step depends on decides how far the simulation gets.526 enum DryAgent {527 /// The step is reached and reported as "the agent was not run".528 case simulated529 /// The step's `when` reads the (not executed) listing and is false: reported as skipped.530 case skipped531 /// The step lives in a `foreach` over items a dry run cannot know: the loop body is not entered at all.532 case notEntered533 }534535 /// Dry run of an agent example: nothing is written, no model is called, and the report accounts for the agent step.536 private func expectDryRunSkipsTheAgent(_ h: ExampleHarness, slug: String, parameters: [String: JSONValue], agentStep: String, expecting: DryAgent, workspace: URL) async throws {537 let before = Files.tree(workspace)538 let dry = try await h.run(slug, parameters, dryRun: true)539 #expect(dry.outcome.dryRun)540 let report = dry.receipt.limitations541 #expect(report.first?.hasPrefix("Dry run: nothing was changed") == true, "\(report)")542 switch expecting {543 case .simulated:544 #expect(dry.step(agentStep)?.status == .simulated)545 #expect(report.contains { $0.hasPrefix("\(agentStep): simulated: the agent was not run") }, "\(report)")546 case .skipped:547 #expect(dry.step(agentStep)?.status == .skipped)548 #expect(report.contains { $0.hasPrefix("Step \(agentStep) was skipped") }, "\(report)")549 case .notEntered:550 #expect(dry.outcome.results(for: agentStep).isEmpty, "the loop body is not entered in a dry run")551 }552 #expect(!dry.outcome.steps.contains { $0.stepID == agentStep && $0.status == .succeeded }, "the agent step was not executed")553 #expect(dry.receipt.usage.turns == 0 && dry.receipt.usage.tokens.totalTokens == 0 && dry.receipt.usage.costUSD == 0, "a dry run calls no model")554 #expect(!dry.receipt.toolCalls.contains { $0.status == .succeeded }, "a dry run executes no tool: \(dry.receipt.toolCalls.map(\.toolID))")555 #expect(Files.tree(workspace) == before, "a dry run changes nothing on disk")556 }557558 @Test("03 weekly-research-digest: dry run never calls the model; live run (dev key) writes a cited digest and its PDF")559 func weeklyResearchDigest() async throws {560 let live = ExampleHarness.hasDevKey561 let h = try await ExampleHarness(live: live)562 defer { Task { await h.tearDown() } }563 let ws = try h.copySample("weekly-research-digest")564 let research = ws.appendingPathComponent("Research"), reports = ws.appendingPathComponent("Reports")565 let parameters: [String: JSONValue] = ["researchFolder": h.path(research), "reportsFolder": h.path(reports), "maxDocuments": 5]566 try await h.save("weekly-research-digest", costBudgetUSD: 0.04)567 try await expectDryRunSkipsTheAgent(h, slug: "weekly-research-digest", parameters: parameters, agentStep: "analyze", expecting: .skipped, workspace: ws)568 guard live else { print("SKIPPED(no ORIX_DEV_ANTHROPIC_API_KEY): weekly-research-digest live agent run; dry run verified"); return }569570 let before = Files.tree(research)571 let run = try await h.run("weekly-research-digest", parameters)572 expectOnlyTheNotificationFailed(run, step: "announce")573 let digestPath = try #require(run.output("digest")?.stringValue)574 let digest = try Files.text(URL(fileURLWithPath: digestPath))575 #expect(digest.contains("Highlights") && digest.count > 200)576 let cited = ["battery-storage-brief", "interview-operator", "field-notes"].filter(digest.contains)577 #expect(cited.count >= 2, "the digest names its source files (text and PDF): \(cited)")578 #expect(digest.lowercased().contains("inverter"), "the digest reflects the content of the sample documents")579 // The workflow read the documents itself; the agent's only tool call is the write of the digest.580 #expect(run.outcome.results(for: "content").count == 2 && run.outcome.results(for: "pdfContent").count == 1)581 #expect(run.step("pdfContent")?.output["pageCount"] == .int(3))582 #expect(!run.receipt.toolCalls.contains { $0.status != .succeeded && $0.toolID != "notifications.post" }, "\(run.diagnosis)")583 let pdfPath = try #require(run.output("pdf")?.stringValue)584 let pdf = try #require(PDFDocument(url: URL(fileURLWithPath: pdfPath)))585 #expect(pdf.pageCount >= 1 && (pdf.string ?? "").contains("Highlights"))586 #expect(run.output("documents") == .int(3))587 #expect(Files.tree(research) == before, "the research folder is read only")588 #expect(run.receipt.usage.tokens.totalTokens > 0 && run.receipt.usage.costUSD < 0.04, "cost \(run.receipt.usage.costUSD)")589 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")590 print(String(format: "VERIFIED(live anthropic:claude-haiku-5-5): weekly-research-digest, %d tokens, $%.4f", run.receipt.usage.tokens.totalTokens, run.receipt.usage.costUSD))591 }592593 @Test("04 git-test-and-review: tests run, patch saved, repository untouched; live review (dev key) reports the failing tests")594 func gitTestAndReview() async throws {595 let live = ExampleHarness.hasDevKey596 let h = try await ExampleHarness(live: live)597 defer { Task { await h.tearDown() } }598 let ws = try h.copySample("git-test-and-review")599 let setup = try Files.run("/bin/bash", [ws.appendingPathComponent("setup.sh").path, ws.path])600 #expect(setup.code == 0, "\(setup.out)")601 let repo = ws.appendingPathComponent("repo"), reviews = ws.appendingPathComponent("Reviews")602 func git(_ args: String...) throws -> String { try Files.run("/usr/bin/git", ["-C", repo.path] + args).out }603 let head = try git("rev-parse", "HEAD"), status = try git("status", "--porcelain"), commits = try git("rev-list", "--count", "HEAD")604 #expect(commits.trimmingCharacters(in: .whitespacesAndNewlines) == "2" && status.contains(" M slugify.sh"))605 let parameters: [String: JSONValue] = ["repository": h.path(repo), "reviewsFolder": h.path(reviews)]606 try await h.save("git-test-and-review", costBudgetUSD: 0.04)607 try await expectDryRunSkipsTheAgent(h, slug: "git-test-and-review", parameters: parameters, agentStep: "review", expecting: .simulated, workspace: ws)608 guard live else {609 // Without a model the non-agent half still runs for real: status, diff, log, the test command and the patch.610 let run = try await h.run("git-test-and-review", parameters)611 #expect(run.status == .failed, "the agent step cannot run without a model: \(run.status)")612 #expect(run.failedSteps == ["review"], "\(run.failedSteps)")613 #expect(run.step("tests")?.output["exitCode"] == .int(1), "the sample's uncommitted change breaks its tests")614 #expect(run.step("tests")?.output["stdout"]?.stringValue?.contains("FAIL - slugify 'Hello World'") == true)615 #expect(run.step("status")?.output["branch"] == .string("main") && run.step("status")?.output["clean"] == .bool(false))616 #expect(try Files.text(reviews.appendingPathComponent("changes.patch")).contains("tr -s ' ' '_'"))617 #expect(!Files.exists(reviews.appendingPathComponent("review.md")))618 #expect(try git("rev-parse", "HEAD") == head && git("status", "--porcelain") == status, "the repository is never modified")619 print("SKIPPED(no ORIX_DEV_ANTHROPIC_API_KEY): git-test-and-review agent review; git, test and patch steps verified for real")620 return621 }622 let run = try await h.run("git-test-and-review", parameters)623 #expect(run.status == .succeeded, "\(run.diagnosis)")624 #expect(run.output("testsPassed") == .bool(false) && run.output("branch") == .string("main") && run.output("patchWritten") == .bool(true))625 #expect(run.step("tests")?.output["exitCode"] == .int(1))626 let review = try Files.text(reviews.appendingPathComponent("review.md"))627 #expect(review.contains("## Tests") && review.lowercased().contains("fail"), "the review states that the tests failed")628 #expect(review.contains("slugify"), "the review talks about the changed file")629 let patch = try Files.text(reviews.appendingPathComponent("changes.patch"))630 #expect(patch.contains("diff --git a/slugify.sh b/slugify.sh") && patch.contains("tr -s ' ' '_'") && !patch.contains("notes.tmp"))631 #expect(try git("rev-parse", "HEAD") == head && git("status", "--porcelain") == status, "never commits, never changes the working tree")632 #expect(try git("rev-list", "--count", "HEAD") == commits)633 #expect(run.receipt.usage.tokens.totalTokens > 0 && run.receipt.usage.costUSD < 0.04, "cost \(run.receipt.usage.costUSD)")634 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")635 print(String(format: "VERIFIED(live anthropic:claude-haiku-5-5): git-test-and-review, %d tokens, $%.4f", run.receipt.usage.tokens.totalTokens, run.receipt.usage.costUSD))636 }637638 @Test("11 meeting-notes-organizer: dry run never calls the model; live run (dev key) writes notes per transcript and the index")639 func meetingNotesOrganizer() async throws {640 let live = ExampleHarness.hasDevKey641 let h = try await ExampleHarness(live: live)642 defer { Task { await h.tearDown() } }643 let ws = try h.copySample("meeting-notes-organizer")644 let transcripts = ws.appendingPathComponent("Transcripts"), notes = ws.appendingPathComponent("Notes")645 let parameters: [String: JSONValue] = ["transcriptsFolder": h.path(transcripts), "notesFolder": h.path(notes)]646 try await h.save("meeting-notes-organizer", costBudgetUSD: 0.05)647 try await expectDryRunSkipsTheAgent(h, slug: "meeting-notes-organizer", parameters: parameters, agentStep: "summarize", expecting: .notEntered, workspace: ws)648 guard live else { print("SKIPPED(no ORIX_DEV_ANTHROPIC_API_KEY): meeting-notes-organizer live agent run; dry run verified"); return }649650 let before = Files.tree(transcripts)651 let run = try await h.run("meeting-notes-organizer", parameters)652 #expect(run.status == .succeeded, "\(run.diagnosis)")653 let sync = try Files.text(notes.appendingPathComponent("2026-11-03-product-sync-notes.md"))654 let budget = try Files.text(notes.appendingPathComponent("2026-11-10-budget-review-notes.md"))655 for text in [sync, budget] { #expect(text.contains("## Summary") && text.contains("## Decisions") && text.contains("## Action items")) }656 #expect(sync.contains("2.4") && sync.contains("Sam"), "decisions and owners come from the transcript")657 #expect(budget.lowercased().contains("staging"))658 let index = try Files.text(notes.appendingPathComponent("index.md"))659 #expect(index.contains("# Meeting notes") && index.contains("2 meeting(s)"))660 #expect(index.contains("(2026-11-03-product-sync-notes.md)") && index.contains("(2026-11-10-budget-review-notes.md)"))661 #expect(Files.tree(transcripts) == before, "transcripts are read only")662 #expect(run.outcome.results(for: "summarize").count == 2)663 #expect(run.receipt.usage.tokens.totalTokens > 0 && run.receipt.usage.costUSD < 0.05, "cost \(run.receipt.usage.costUSD)")664 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")665 print(String(format: "VERIFIED(live anthropic:claude-haiku-5-5): meeting-notes-organizer, %d tokens, $%.4f", run.receipt.usage.tokens.totalTokens, run.receipt.usage.costUSD))666 }667668 // MARK: 5 · CSV cleanup and charts669670 @Test("05 csv-cleanup-and-charts: the sandboxed Python script cleans the dirty rows and draws one bar per row")671 func csvCleanupAndCharts() async throws {672 let h = try await ExampleHarness()673 defer { Task { await h.tearDown() } }674 let ws = try h.copySample("csv-cleanup-and-charts")675 let input = ws.appendingPathComponent("input.csv"), output = ws.appendingPathComponent("Output")676 let inputHash = try Files.sha256(input)677 let run = try await h.run("csv-cleanup-and-charts", ["inputFile": h.path(input), "outputFolder": h.path(output)])678 #expect(run.status == .succeeded, "\(run.receipt.errors) \(run.step("clean")?.output["stderr"]?.stringValue ?? "")")679680 let cleaned = try Files.text(output.appendingPathComponent("cleaned.csv")).replacingOccurrences(of: "\r\n", with: "\n")681 #expect(cleaned == "region,units,revenue\nNorth,12,1200\nSouth,7,\"1,050\"\nEast,3,\nWest,9,810\nCentral,15,2250\n")682 let svg = try Files.text(output.appendingPathComponent("chart.svg"))683 #expect(svg.hasPrefix("<svg") && svg.contains(">units</text>") && svg.components(separatedBy: "<rect ").count - 1 == 5)684 #expect(run.output("rows") == .int(5) && run.output("rowsBefore") == .int(9))685 #expect(run.output("log")?.stringValue == "rows_in=7 rows_out=5 charted_column=units bars=5")686 #expect(try Files.sha256(input) == inputHash, "the input file is never modified")687 #expect(Set(Files.tree(output).keys) == ["cleaned.csv", "chart.svg"], "the sandbox let the script write only what it should")688 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")689 }690691 // MARK: 6 · Batch image conversion692693 @Test("06 batch-image-conversion: one JPEG per image with sequential names and the original pixel sizes")694 func batchImageConversion() async throws {695 let h = try await ExampleHarness()696 defer { Task { await h.tearDown() } }697 let ws = try h.copySample("batch-image-conversion")698 let originals = ws.appendingPathComponent("Originals"), converted = ws.appendingPathComponent("Converted")699 let before = Files.tree(originals)700 let run = try await h.run("batch-image-conversion", ["sourceFolder": h.path(originals), "outputFolder": h.path(converted), "format": "jpeg", "prefix": "shot"])701 #expect(run.status == .succeeded, "\(run.diagnosis)")702 // Listing order is the order of the names as bytes: upper case first.703 let expected = [("Icon Large.PNG", 96, 96), ("badge.gif", 120, 80), ("diagram.png", 320, 200), ("sunset.jpg", 640, 360), ("tile.tiff", 200, 200)]704 for (i, (name, w, hgt)) in expected.enumerated() {705 let size = try #require(Files.imageSize(converted.appendingPathComponent("shot-\(i + 1).jpeg")), "shot-\(i + 1).jpeg (\(name)) is not a readable image")706 #expect(size.width == w && size.height == hgt && size.type == "public.jpeg", "\(name) → \(size)")707 }708 #expect(Set(Files.tree(converted).keys) == Set((1...5).map { "shot-\($0).jpeg" }), "captions.txt is not an image and is left out")709 #expect(Files.tree(originals) == before, "originals are never modified or removed")710 #expect(run.output("converted") == .int(5))711 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")712 }713714 // MARK: 7 · Calendar preparation (Apple Events)715716 @Test("07 calendar-preparation: read-only; runs for real only when Automation consent for Calendar is already granted")717 func calendarPreparation() async throws {718 let h = try await ExampleHarness()719 defer { Task { await h.tearDown() } }720 let folder = h.workspace.appendingPathComponent("calendar-preparation/Briefings")721 let parameters: [String: JSONValue] = ["outputFolder": h.path(folder), "daysAhead": 2]722723 // Read-only by construction: the only script addresses Calendar and contains no command that changes anything.724 let document = try await h.save("calendar-preparation").document725 let scripts = document.allSteps.compactMap { located -> String? in726 if case .tool(let t) = located.step.kind, t.tool == "applescript.run" { return t.arguments["script"]?.stringValue }; return nil727 }728 #expect(scripts.count == 1)729 for verb in ["make new", "delete", "duplicate", "move ", "set summary", "set start date", "set end date", "set description", "do shell script", "save"] {730 #expect(!scripts[0].lowercased().contains(verb), "the calendar script must stay read-only (found “\(verb)”)")731 }732 let review = try await h.services.workflows.permissionReview(workflowID: "calendar-preparation", parameters: parameters)733 #expect(review.proposed.map(\.capability).contains(.appAutomation(bundleID: "com.apple.iCal")))734 #expect(!review.proposed.contains { $0.capability.kind.hasPrefix("file.read") || $0.capability.kind.hasPrefix("file.delete") })735736 let status = MacPermissions.automationStatus(bundleID: "com.apple.iCal")737 switch status {738 case .granted:739 let run = try await h.run("calendar-preparation", parameters)740 expectOnlyTheNotificationFailed(run, step: "announce")741 let briefingPath = try #require(run.output("briefing")?.stringValue)742 let briefing = try Files.text(URL(fileURLWithPath: briefingPath))743 let events = try #require(run.output("events")?.intValue)744 #expect(briefing.contains("# Briefing for") && briefing.contains("Events in the next 2 day(s): \(events)"))745 #expect(briefing.components(separatedBy: "- [ ] ").count - 1 == events, "one checklist line per event")746 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")747 print("VERIFIED(appleEvents → Calendar granted): calendar-preparation read \(events) event(s), briefing written")748 case .denied:749 let run = try await h.run("calendar-preparation", parameters)750 #expect(run.status == .blockedPermission, "\(run.status)")751 #expect(run.step("events")?.status == .blockedPermission)752 #expect(((try? FileManager.default.contentsOfDirectory(atPath: folder.path)) ?? []).isEmpty, "no briefing without consent")753 print("SKIPPED(appleEvents not granted): Automation consent for Calendar was refused; BLOCKED_PERMISSION verified")754 case .notDetermined, .unknown:755 // Running the script would make macOS show its consent prompt (or launch Calendar): not in a test. Dry run instead.756 let dry = try await h.run("calendar-preparation", parameters, dryRun: true)757 #expect(dry.step("events")?.status == .simulated)758 #expect(dry.receipt.limitations.contains { $0.contains("events") && $0.contains("AppleScript") && $0.contains("Calendar") }, "\(dry.receipt.limitations)")759 #expect(!Files.exists(folder), "a dry run writes nothing")760 print("SKIPPED(appleEvents not granted): Automation consent for Calendar is \(status.rawValue) (Calendar not running or never asked); dry run verified, no prompt triggered")761 }762 }763764 // MARK: 8 · Folder backup and verification765766 @Test("08 folder-backup-and-verify: the archive restores to an identical tree and its recorded checksum is right")767 func folderBackupAndVerify() async throws {768 let h = try await ExampleHarness()769 defer { Task { await h.tearDown() } }770 let ws = try h.copySample("folder-backup-and-verify")771 let source = ws.appendingPathComponent("Documents"), backups = ws.appendingPathComponent("Backups")772 let before = Files.tree(source)773 #expect(before[".settings"] != nil, "the sample has a hidden file")774 let run = try await h.run("folder-backup-and-verify", ["source": h.path(source), "backupRoot": h.path(backups)])775 // The temporary restore went to the user's Trash: take the test's own item out again, whatever happens next.776 let trashed = run.step("cleanup")?.output["trashPath"]?.stringValue777 defer { if let trashed, trashed.contains("/.Trash/"), (trashed as NSString).lastPathComponent.hasPrefix(".verify-") { try? FileManager.default.removeItem(atPath: trashed) } }778 expectOnlyTheNotificationFailed(run, step: "announce")779780 let archivePath = try #require(run.output("archive")?.stringValue)781 let archive = URL(fileURLWithPath: archivePath)782 #expect(archive.lastPathComponent.hasPrefix("backup-") && archive.pathExtension == "zip")783 let digest = try Files.sha256(archive)784 #expect(run.output("sha256") == .string(digest))785 #expect(try Files.text(URL(fileURLWithPath: archive.path + ".sha256")) == "\(digest) \(archive.lastPathComponent)\n")786 // Independent restore with ditto: the archive really holds the whole tree, hidden file included.787 let check = h.root.appendingPathComponent("independent-restore")788 #expect(try Files.run("/usr/bin/ditto", ["-x", "-k", archive.path, check.path]).code == 0)789 #expect(Files.tree(check.appendingPathComponent("Documents")) == before)790 #expect(Files.tree(source) == before, "the source is read only")791 #expect(run.output("identical") == .bool(true) && run.output("entries")?.intValue == before.count + 1)792 #expect(Set(try FileManager.default.contentsOfDirectory(atPath: backups.path)) == [archive.lastPathComponent, archive.lastPathComponent + ".sha256"],793 "the temporary restore folder is gone from the backup root")794 #expect(trashed != nil && Files.exists(URL(fileURLWithPath: trashed ?? "/nonexistent")), "the temporary copy was moved to the Trash, not erased")795 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")796 }797798 @Test("08 folder-backup-and-verify, headless and without accepted permissions: waits for approval and writes nothing")799 func folderBackupWithoutPermissions() async throws {800 let h = try await ExampleHarness(interactive: false)801 defer { Task { await h.tearDown() } }802 let ws = try h.copySample("folder-backup-and-verify")803 let before = Files.tree(ws)804 let run = try await h.run("folder-backup-and-verify", ["source": h.path(ws.appendingPathComponent("Documents")), "backupRoot": h.path(ws.appendingPathComponent("Backups"))],805 accept: false, mode: .scheduled)806 #expect(run.status == .waitingApproval || run.status == .blockedPermission, "\(run.status)")807 #expect(Files.tree(ws) == before, "nothing may change on disk")808 #expect(!run.receipt.toolCalls.contains { $0.status == .succeeded })809 }810811 // MARK: 9 · Website change monitoring (loopback only)812813 @Test("09 website-change-monitor: baseline, unchanged and changed runs against a loopback server; the network permission follows the parameter")814 func websiteChangeMonitor() async throws {815 let h = try await ExampleHarness()816 defer { Task { await h.tearDown() } }817 let ws = try h.copySample("website-change-monitor")818 let v1 = try Data(contentsOf: ws.appendingPathComponent("site/page-v1.html")), v2 = try Data(contentsOf: ws.appendingPathComponent("site/page-v2.html"))819 let server = try await LoopbackServer { _ in (200, "text/html; charset=utf-8", v1) }820 defer { server.stop() }821 let state = ws.appendingPathComponent("State")822 let url = "http://127.0.0.1:\(server.port)/page.html"823 let parameters: [String: JSONValue] = ["host": "127.0.0.1", "url": .string(url), "stateFolder": h.path(state)]824825 try await h.save("website-change-monitor")826 let review = try await h.services.workflows.permissionReview(workflowID: "website-change-monitor", parameters: parameters)827 let hosts = review.proposed.compactMap { g -> String? in if case .network(let host) = g.capability { return host }; return nil }828 #expect(hosts == ["127.0.0.1"], "the declared network permission is the host parameter, not example.com: \(hosts)")829830 let first = try await h.run("website-change-monitor", parameters)831 #expect(first.status == .succeeded, "\(first.receipt.errors)")832 #expect(first.output("firstRun") == .bool(true) && first.output("changed") == .bool(false) && first.output("status") == .int(200))833 #expect(try Data(contentsOf: state.appendingPathComponent("previous.html")) == v1)834835 let second = try await h.run("website-change-monitor", parameters)836 #expect(second.status == .succeeded, "\(second.receipt.errors)")837 #expect(second.output("firstRun") == .bool(false) && second.output("changed") == .bool(false))838 #expect(second.step("alert")?.status == .skipped)839840 server.setHandler { _ in (200, "text/html; charset=utf-8", v2) }841 let third = try await h.run("website-change-monitor", parameters)842 #expect(third.output("changed") == .bool(true), "the change is detected")843 expectOnlyTheNotificationFailed(third, step: "alert") // the alert is a notification: no app bundle under swift test844 #expect(try Data(contentsOf: state.appendingPathComponent("previous.html")) == v2, "the new copy became the reference")845 #expect(try Data(contentsOf: state.appendingPathComponent("current.html")) == v2)846 #expect(third.output("sha256")?.stringValue == SHA256.hash(data: v2).map { String(format: "%02x", $0) }.joined())847 #expect(server.requests == ["/page.html", "/page.html", "/page.html"], "exactly one GET per run: \(server.requests)")848849 // A URL on a host the accepted permission does not cover is not fetched.850 var elsewhere = parameters851 elsewhere["url"] = .string("http://localhost:\(server.port)/page.html")852 let refused = try await h.run("website-change-monitor", elsewhere)853 #expect(refused.status == .waitingApproval, "\(refused.status)")854 #expect(refused.step("fetch")?.status == .notApproved)855 #expect(server.requests.count == 3, "no request left for the other host")856 #expect(h.approvals.toolRequests.count == 1 && h.approvals.toolRequests.first?.hasPrefix("http.request") == true)857 }858859 @Test("09 website-change-monitor, headless and without accepted permissions: no request is sent and nothing is written")860 func websiteChangeMonitorWithoutPermissions() async throws {861 let h = try await ExampleHarness(interactive: false)862 defer { Task { await h.tearDown() } }863 let server = try await LoopbackServer { _ in (200, "text/html", Data("<p>never fetched</p>".utf8)) }864 defer { server.stop() }865 let state = h.workspace.appendingPathComponent("website-change-monitor/State")866 let run = try await h.run("website-change-monitor", ["host": "127.0.0.1", "url": .string("http://127.0.0.1:\(server.port)/page.html"), "stateFolder": h.path(state)],867 accept: false, mode: .scheduled)868 #expect(run.status == .waitingApproval || run.status == .blockedPermission, "\(run.status)")869 #expect(server.requests.isEmpty, "nothing may be sent without the network permission")870 #expect(!Files.exists(state), "nothing may be written")871 }872873 // MARK: 10 · System health report874875 @Test("10 system-health-report: the report holds the real machine facts and a plausible free-space fraction")876 func systemHealthReport() async throws {877 let h = try await ExampleHarness()878 defer { Task { await h.tearDown() } }879 let folder = h.workspace.appendingPathComponent("system-health-report/Reports")880 let run = try await h.run("system-health-report", ["outputFolder": h.path(folder)])881 let fraction = try #require(run.output("freeFraction")?.doubleValue)882 let low = fraction < 0.1883 if low { expectOnlyTheNotificationFailed(run, step: "warn") } else { #expect(run.status == .succeeded, "\(run.diagnosis)"); #expect(run.step("warn")?.status == .skipped) }884 #expect(run.output("lowDisk") == .bool(low))885 // The same figure, read by the test straight from the system.886 let values = try URL(fileURLWithPath: "/").resourceValues(forKeys: [.volumeTotalCapacityKey, .volumeAvailableCapacityForImportantUsageKey])887 let expected = Double(values.volumeAvailableCapacityForImportantUsage ?? 0) / Double(values.volumeTotalCapacity ?? 1)888 #expect(fraction > 0 && fraction <= 1 && abs(fraction - expected) < 0.05, "free fraction \(fraction) vs \(expected)")889 let reportPath = try #require(run.output("report")?.stringValue)890 let report = try Files.text(URL(fileURLWithPath: reportPath))891 for section in ["# System health", "## Machine", "## Disk", "## Busiest processes", "## Power", "## Network"] { #expect(report.contains(section)) }892 #expect(report.contains("\"cores\":\(ProcessInfo.processInfo.processorCount)") || report.contains("\"cores\": \(ProcessInfo.processInfo.processorCount)"))893 #expect(report.contains("\"memoryBytes\"") && report.contains("\"cpuPercent\"") && report.contains("\"onACPower\"") && report.contains("\"online\""))894 let v = ProcessInfo.processInfo.operatingSystemVersion895 #expect(run.output("osVersion")?.stringValue?.hasPrefix("\(v.majorVersion).\(v.minorVersion)") == true, "\(String(describing: run.output("osVersion")))")896 #expect(Set(Files.tree(folder).keys).count == 1, "the report is the only file written")897 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")898 }899900 // MARK: 12 · Document metadata catalogue901902 @Test("12 document-metadata-catalog: one row per file with the right size and SHA-256")903 func documentMetadataCatalog() async throws {904 let h = try await ExampleHarness()905 defer { Task { await h.tearDown() } }906 let ws = try h.copySample("document-metadata-catalog")907 let documents = ws.appendingPathComponent("Documents"), catalog = ws.appendingPathComponent("catalog.csv")908 let before = Files.tree(documents).filter { $0.value != "dir" }909 let run = try await h.run("document-metadata-catalog", ["documentsFolder": h.path(documents), "catalogFile": h.path(catalog)])910 #expect(run.status == .succeeded, "\(run.diagnosis)")911 let lines = try Files.text(catalog).split(whereSeparator: \.isNewline).map(String.init)912 #expect(lines.first == "path,extension,bytes,modified,contentType,sha256" && lines.count == before.count + 1)913 var total = 0914 for line in lines.dropFirst() {915 let c = line.split(separator: ",", omittingEmptySubsequences: false).map(String.init)916 #expect(c.count == 6, "\(line)")917 let file = documents.appendingPathComponent(c[0])918 let size = (try FileManager.default.attributesOfItem(atPath: file.path)[.size] as? Int) ?? -1919 #expect(before[c[0]] == c[5], "\(c[0]): SHA-256 in the catalogue matches the file")920 #expect(Int(c[2]) == size && c[1] == "." + file.pathExtension.lowercased())921 #expect(ISO8601DateFormatter().date(from: c[3]) != nil, "modified is an ISO 8601 date: \(c[3])")922 total += size923 }924 #expect(Set(lines.dropFirst().map { String($0.split(separator: ",")[0]) }) == Set(before.keys))925 #expect(run.output("files") == .int(before.count) && run.output("totalBytes") == .int(total))926 #expect(Files.tree(documents).filter { $0.value != "dir" } == before, "the documents are read only")927 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")928 }929930 // MARK: 13 · Scheduled window screenshot (Screen Recording)931932 /// An application that has a normal window on screen right now and is not one the capture tool refuses. Finder and933 /// Terminal are preferred over whatever else is open: the picture is deleted right after the assertion, but a934 /// file-manager or terminal window is the least personal thing to capture.935 private func applicationWithAWindow() -> String? {936 let list = (CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]]) ?? []937 var found: [String] = []938 for w in list where (w[kCGWindowLayer as String] as? Int) == 0 {939 guard let pid = w[kCGWindowOwnerPID as String] as? pid_t, let id = NSRunningApplication(processIdentifier: pid)?.bundleIdentifier else { continue }940 let bounds = w[kCGWindowBounds as String] as? [String: Any]941 guard (bounds?["Width"] as? Double ?? 0) > 100, (bounds?["Height"] as? Double ?? 0) > 100 else { continue }942 if !ScreenCaptureTool.defaultSensitiveBundleIDs.contains(id.lowercased()), !found.contains(id) { found.append(id) }943 }944 return ["com.apple.finder", "com.apple.Terminal"].first(where: found.contains) ?? found.first945 }946947 @Test("13 scheduled-window-screenshot: skips cleanly without a window; captures a real window only when Screen Recording is granted")948 func scheduledWindowScreenshot() async throws {949 let h = try await ExampleHarness()950 defer { Task { await h.tearDown() } }951 let folder = h.workspace.appendingPathComponent("scheduled-window-screenshot/Captures")952953 // An application that is not running: the run succeeds, says so, launches nothing and writes no picture.954 let absent = try await h.run("scheduled-window-screenshot", ["bundleID": "ai.spboucher.orix.tests.not-installed", "outputFolder": h.path(folder)])955 #expect(absent.status == .succeeded, "\(absent.receipt.errors)")956 #expect(absent.output("captured") == .bool(false) && absent.step("capture")?.status == .skipped)957 #expect(Files.tree(folder).isEmpty)958 let review = try await h.services.workflows.permissionReview(workflowID: "scheduled-window-screenshot", parameters: ["bundleID": "com.apple.finder", "outputFolder": h.path(folder)])959 #expect(review.proposed.map(\.capability).contains(.screenCapture(scope: "window:com.apple.finder")), "the screen scope follows the parameter, one window, never the display")960961 let granted = MacPermissions.status(.screenRecording) == .granted962 guard let bundleID = applicationWithAWindow() else {963 print("SKIPPED(no application window on screen): scheduled-window-screenshot capture; the no-window path was verified (screenRecording \(granted ? "granted" : "not granted"))")964 return965 }966 let run = try await h.run("scheduled-window-screenshot", ["bundleID": .string(bundleID), "outputFolder": h.path(folder)])967 if granted {968 #expect(run.status == .succeeded, "\(run.diagnosis)")969 #expect(run.output("captured") == .bool(true))970 let filePath = try #require(run.output("file")?.stringValue)971 let file = URL(fileURLWithPath: filePath)972 let size = try #require(Files.imageSize(file), "the capture is not a readable image")973 #expect(size.type == "public.png" && size.width > 50 && size.height > 50)974 #expect(run.step("capture")?.output["pixelWidth"] == .int(size.width) && run.step("capture")?.output["pixelHeight"] == .int(size.height))975 #expect(Files.tree(folder).count == 1)976 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")977 try? FileManager.default.removeItem(at: file) // a picture of a real window: not kept a moment longer than the assertion978 print("VERIFIED(screenRecording granted): scheduled-window-screenshot captured a \(size.width)×\(size.height) px window of \(bundleID)")979 } else {980 #expect(run.status == .blockedPermission, "\(run.status)")981 #expect(run.step("capture")?.status == .blockedPermission)982 #expect(Files.tree(folder).isEmpty, "no picture without Screen Recording")983 print("SKIPPED(screenRecording not granted): scheduled-window-screenshot; BLOCKED_PERMISSION verified, no prompt triggered")984 }985 }986987 // MARK: 14 · Spreadsheet validation988989 @Test("14 spreadsheet-validation: a valid workbook passes; a broken one fails the run and the report says why")990 func spreadsheetValidation() async throws {991 let h = try await ExampleHarness()992 defer { Task { await h.tearDown() } }993 let ws = try h.copySample("spreadsheet-validation")994 let reports = ws.appendingPathComponent("Reports")995 let good = ws.appendingPathComponent("data.xlsx"), bad = ws.appendingPathComponent("data-invalid.xlsx")996 let hashes = [try Files.sha256(good), try Files.sha256(bad)]997998 let valid = try await h.run("spreadsheet-validation", ["workbook": h.path(good), "reportFolder": h.path(reports)])999 #expect(valid.status == .succeeded, "\(valid.receipt.errors)")1000 #expect(valid.output("valid") == .bool(true) && valid.output("rows") == .int(4) && valid.output("columns") == ["id", "name", "amount"])1001 var report = try Files.text(reports.appendingPathComponent("validation.md"))1002 #expect(report.contains("Sheet: Orders") && report.contains("Data rows read: 4") && report.contains("Valid: true"))1003 #expect(valid.step("read")?.output["rows"]?.arrayValue?.last == [1004, "Delta", 195.5], "a formula cell is read as its cached value")10041005 let invalid = try await h.run("spreadsheet-validation", ["workbook": h.path(bad), "reportFolder": h.path(reports)])1006 #expect(invalid.status == .failed, "a broken workbook fails the run on purpose")1007 #expect(invalid.failedSteps == ["check"] && invalid.output("valid") == nil)1008 report = try Files.text(reports.appendingPathComponent("validation.md"))1009 #expect(report.contains("Missing required columns: 1 amount") && report.contains("Fully empty rows: 1") && report.contains("Rows without a key value: 2") && report.contains("Valid: false"))1010 let failed = invalid.receipt.verifications.filter { !$0.passed }1011 #expect(failed.count == 3, "\(failed.map(\.assertion))")1012 #expect([try Files.sha256(good), try Files.sha256(bad)] == hashes, "workbooks are never modified")1013 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")1014 }10151016 // MARK: 15 · Browser data entry (WKWebView against a loopback test site)10171018 @Test("15 browser-data-entry: fills both pages of the test site in the real WKWebView, submits after approval, keeps a screenshot")1019 func browserDataEntry() async throws {1020 let h = try await ExampleHarness()1021 defer { Task { await h.tearDown() } }1022 let ws = try h.copySample("browser-data-entry")1023 let server = try await LoopbackServer.site(ws.appendingPathComponent("site"))1024 defer { server.stop() }1025 let evidence = ws.appendingPathComponent("Evidence")1026 let parameters: [String: JSONValue] = ["host": "127.0.0.1", "pageURL": .string("http://127.0.0.1:\(server.port)/form.html"),1027 "fullName": "Grace Hopper", "email": "grace@example.org", "plan": "company", "evidenceFolder": h.path(evidence)]1028 let run = try await h.run("browser-data-entry", parameters)1029 #expect(run.status == .succeeded, "\(run.diagnosis)")1030 #expect(run.output("confirmation")?.stringValue?.contains("Registered Grace Hopper <grace@example.org> on plan company.") == true, "\(String(describing: run.output("confirmation")))")1031 #expect(run.output("page")?.stringValue?.contains("/done.html?") == true)1032 let pages = server.requests.filter { $0.contains(".html") }1033 #expect(pages.count == 3 && pages[0] == "/form.html", "\(pages)")1034 #expect(pages[1].hasPrefix("/details.html?") && pages[1].contains("name=Grace") && pages[1].contains("email=grace%40example.org"), "\(pages)")1035 #expect(pages[2].hasPrefix("/done.html?") && pages[2].contains("plan=company") && pages[2].contains("terms=yes") && pages[2].contains("name=Grace"), "\(pages)")1036 let shot = evidence.appendingPathComponent("confirmation.png")1037 let size = try #require(Files.imageSize(shot))1038 #expect(size.type == "public.png" && size.width > 100 && size.height > 100)1039 #expect(run.output("screenshotSha256")?.stringValue == (try Files.sha256(shot)))1040 #expect(h.approvals.gates.count == 1 && h.approvals.gates.first?.contains("Grace Hopper") == true, "the gate names what will be submitted")1041 #expect(h.approvals.toolRequests.isEmpty, "\(h.approvals.toolRequests)")1042 #expect(await h.services.browser.session(runID: run.record.id) == nil, "the browser session is closed")1043 }10441045 @Test("15 browser-data-entry: a refused approval submits nothing")1046 func browserDataEntryRefused() async throws {1047 let h = try await ExampleHarness(approveGates: false)1048 defer { Task { await h.tearDown() } }1049 let ws = try h.copySample("browser-data-entry")1050 let server = try await LoopbackServer.site(ws.appendingPathComponent("site"))1051 defer { server.stop() }1052 let evidence = ws.appendingPathComponent("Evidence")1053 let run = try await h.run("browser-data-entry", ["host": "127.0.0.1", "pageURL": .string("http://127.0.0.1:\(server.port)/form.html"), "evidenceFolder": h.path(evidence)])1054 #expect(run.status == .failed, "a denied gate fails the run: \(run.status)")1055 #expect(run.failedSteps == ["confirm"] && run.step("submit")?.status != .succeeded)1056 #expect(!server.requests.contains { $0.hasPrefix("/done.html") }, "the form was not submitted: \(server.requests)")1057 #expect(server.requests.contains { $0.hasPrefix("/details.html?") }, "the first page was filled before the gate")1058 #expect(!Files.exists(evidence.appendingPathComponent("confirmation.png")))1059 #expect(await h.services.browser.session(runID: run.record.id) == nil, "the browser session is closed when the run ends")1060 }1061}1062