Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/ExamplesTests.swift3// Description: Integration tests of the 15 built-in automations (real files on disk and in the resource bundle) and of the published JSON Schema.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXCore10@testable import ORIXWorkflowEngine1112enum RepositoryLayout {13 /// Repository root, found from this file: <root>/Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/<file>.14 static func root(_ file: String = #filePath) -> URL {15 var url = URL(fileURLWithPath: file)16 for _ in 0..<5 { url.deleteLastPathComponent() }17 return url18 }19 static var examples: URL { root().appendingPathComponent("Workflows/Examples") }20 static var resources: URL { root().appendingPathComponent("Packages/ORIXWorkflowEngine/Sources/ORIXWorkflowEngine/Resources/Examples") }21 static var schemaFile: URL { root().appendingPathComponent("Workflows/Schemas/automation.v1.schema.json") }2223 static func yamlFiles(_ dir: URL) throws -> [URL] {24 try FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil)25 .filter { $0.pathExtension == "yaml" }.sorted { $0.lastPathComponent < $1.lastPathComponent }26 }27}2829let expectedExampleSlugs = [30 "smart-downloads-organizer", "pdf-to-structured-report", "weekly-research-digest", "git-test-and-review", "csv-cleanup-and-charts",31 "batch-image-conversion", "calendar-preparation", "folder-backup-and-verify", "website-change-monitor", "system-health-report",32 "meeting-notes-organizer", "document-metadata-catalog", "scheduled-window-screenshot", "spreadsheet-validation", "browser-data-entry",33]3435@Suite("Built-in examples (integration: real files and resource bundle)") struct ExamplesTests {36 private func load() throws -> [(file: String, source: String, doc: WorkflowDocument)] {37 try RepositoryLayout.yamlFiles(RepositoryLayout.examples).map { url in38 let source = try String(contentsOf: url, encoding: .utf8)39 do { return (url.lastPathComponent, source, try WorkflowCodec.parse(source)) }40 catch { Issue.record("\(url.lastPathComponent) does not parse: \(error.localizedDescription)"); throw error }41 }42 }4344 @Test func thereAreExactlyFifteenNumberedExamples() throws {45 let names = try RepositoryLayout.yamlFiles(RepositoryLayout.examples).map(\.lastPathComponent)46 #expect(names == expectedExampleSlugs.enumerated().map { String(format: "%02d-%@.yaml", $0.offset + 1, $0.element) })47 }4849 @Test func everyExampleParsesAndValidatesWithoutErrorsOrWarnings() throws {50 for (file, source, doc) in try load() {51 let found = WorkflowValidator.validate(doc, tools: CanonicalTools.manifests(), agents: [], workflows: [])52 let problems = found.filter { $0.severity != .info }53 #expect(problems.isEmpty, "\(file): \(problems.map(\.description).joined(separator: "\n"))")54 let diagnosed = WorkflowCodec.diagnose(source: source, tools: CanonicalTools.manifests())55 #expect(diagnosed.document == doc)56 #expect(try WorkflowCodec.parse(WorkflowCodec.emit(doc)) == doc, "\(file) does not survive canonical emission")57 }58 }5960 @Test func everyExampleIsCompleteAndUsesOnlyCanonicalTools() throws {61 let canonical = Set(CanonicalTools.identifiers)62 for (index, (file, source, doc)) in try load().enumerated() {63 #expect(doc.metadata.id == expectedExampleSlugs[index], "\(file)")64 #expect(doc.metadata.description.count > 80, "\(file) needs a real description")65 #expect(!doc.metadata.tags.isEmpty && !doc.spec.triggers.isEmpty, "\(file)")66 #expect(!doc.spec.parameters.isEmpty && doc.spec.parameters.allSatisfy { !$0.description.isEmpty && $0.defaultValue != nil }, "\(file)")67 #expect(!doc.spec.permissions.isEmpty && !doc.spec.budget.isEmpty, "\(file)")68 for p in doc.spec.parameters where p.type == .path {69 #expect(p.defaultValue?.stringValue?.hasPrefix("~/ORIX Samples/\(doc.metadata.id)/") == true, "\(file): \(p.name) must default under the sample workspace")70 }71 let steps = doc.allSteps.map(\.step)72 #expect(steps.contains { if case .verify(let v) = $0.kind { return !v.assertions.isEmpty }; return false }, "\(file) has no verify step")73 #expect(steps.contains { if case .output = $0.kind { return true }; return false }, "\(file) has no output step")74 for s in steps {75 if case .tool(let t) = s.kind { #expect(canonical.contains(t.tool), "\(file): \(t.tool) is not a canonical tool id") }76 if case .agent(let a) = s.kind {77 guard case .inline(let inline) = a.agent else { Issue.record("\(file): agents must be inline"); continue }78 #expect(inline.tools.allSatisfy(canonical.contains) && !a.expect.isEmpty, "\(file): agent \(s.id)")79 }80 }81 #expect(!steps.contains { if case .tool(let t) = $0.kind { return ["filesystem.delete", "git.push", "git.commit"].contains(t.tool) }; return false },82 "\(file) must not delete, commit or push")83 #expect(source.contains("# Limitations"), "\(file) must state its limitations in comments")84 #expect(source.hasPrefix("# ORIX OS Automation\n# File: Workflows/Examples/\(file)\n"), "\(file) header")85 }86 }8788 @Test func consequentialExamplesAskForApproval() throws {89 let docs = Dictionary(uniqueKeysWithValues: try load().map { ($0.doc.metadata.id, $0.doc) })90 for slug in ["smart-downloads-organizer", "browser-data-entry"] {91 let steps = docs[slug]?.allSteps.map(\.step) ?? []92 #expect(steps.contains { if case .approval = $0.kind { return true }; return false }, "\(slug) needs an approval gate")93 }94 }9596 @Test func resourcesMirrorTheRepositoryExamplesByteForByte() throws {97 let a = try RepositoryLayout.yamlFiles(RepositoryLayout.examples)98 let b = try FileManager.default.contentsOfDirectory(at: RepositoryLayout.resources, includingPropertiesForKeys: nil)99 .sorted { $0.lastPathComponent < $1.lastPathComponent }100 #expect(a.map(\.lastPathComponent) == b.map(\.lastPathComponent))101 for (x, y) in zip(a, b) { #expect(try Data(contentsOf: x) == Data(contentsOf: y), "\(x.lastPathComponent) differs from its resource copy") }102 }103104 @Test func builtInWorkflowsLoadFromTheBundle() throws {105 let all = BuiltInWorkflows.all()106 #expect(all.map(\.slug) == expectedExampleSlugs)107 let onDisk = try load()108 for (b, d) in zip(all, onDisk) {109 #expect(b.yaml == d.source && b.title == d.doc.metadata.name && b.summary == d.doc.metadata.description && b.id == b.slug)110 #expect(try b.document() == d.doc)111 }112 #expect(BuiltInWorkflows.workflow(slug: "system-health-report")?.title == "Local system health report")113 #expect(BuiltInWorkflows.workflow(slug: "nope") == nil)114 #expect(BuiltInWorkflows.slug(fileName: "03-weekly-research-digest.yaml") == "weekly-research-digest")115 #expect(BuiltInWorkflows.slug(fileName: "plain-name.yaml") == "plain-name")116 }117118 @Test func capabilitiesAndProposedGrantsAreCanonicalAndScoped() throws {119 let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("orix-home-\(UUID().uuidString.prefix(8))")120 try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true)121 defer { try? FileManager.default.removeItem(at: tmp) }122 let home = PathResolver.resolveSymlinks(tmp.path)123 for (file, _, doc) in try load() {124 let caps = doc.declaredCapabilities(home: home)125 #expect(!caps.isEmpty, "\(file)")126 for c in caps where c.kind.hasPrefix("file.") {127 #expect(c.scope.hasPrefix(home + "/ORIX Samples/\(doc.metadata.id)"), "\(file): \(c.displayString)")128 #expect(!c.scope.contains("~") && !c.scope.contains("${{"))129 }130 let grants = doc.proposedGrants(workflowID: "wf-1", home: home)131 #expect(grants.map(\.capability) == caps)132 for g in grants {133 #expect(g.scope == .workflow(workflowID: "wf-1") && g.maxTier <= .consequential && g.source == "workflow:wf-1", "\(file)")134 }135 }136 let backup = try #require(BuiltInWorkflows.workflow(slug: "folder-backup-and-verify")).document()137 let custom = backup.declaredCapabilities(home: home, parameters: ["source": "~/Elsewhere"])138 #expect(custom.contains(.fileRead(path: home + "/Elsewhere")))139 let shot = try #require(BuiltInWorkflows.workflow(slug: "scheduled-window-screenshot")).document()140 #expect(shot.declaredCapabilities(home: home).contains(.screenCapture(scope: "window:com.apple.Safari")))141 }142}143144@Suite("Automation JSON Schema (integration: schema file and Core validator)") struct WorkflowSchemaTests {145 private func canonicalJSON(_ doc: WorkflowDocument) throws -> JSONValue { try JSONValue.parse(WorkflowCodec.emit(doc, format: .json)) }146147 @Test func publishedSchemaFileMatchesTheGeneratedSchema() throws {148 let text = WorkflowSchema.automationV1JSON()149 if ProcessInfo.processInfo.environment["ORIX_WRITE_SCHEMA"] == "1" {150 try FileManager.default.createDirectory(at: RepositoryLayout.schemaFile.deletingLastPathComponent(), withIntermediateDirectories: true)151 try text.write(to: RepositoryLayout.schemaFile, atomically: true, encoding: .utf8)152 }153 let onDisk = try String(contentsOf: RepositoryLayout.schemaFile, encoding: .utf8)154 #expect(onDisk == text, "run the tests once with ORIX_WRITE_SCHEMA=1 to regenerate Workflows/Schemas/automation.v1.schema.json")155 let parsed = try JSONValue.parse(onDisk)156 #expect(parsed == WorkflowSchema.automationV1().raw)157 #expect(parsed["$id"] == "https://orix.ai/schemas/automation.v1.schema.json")158 }159160 @Test func schemaUsesOnlyKeywordsCoreEnforcesPlusAnnotations() {161 let allowed: Set<String> = ["$schema", "$id", "title", "description", "type", "properties", "required", "additionalProperties",162 "items", "enum", "minimum", "maximum", "minLength", "maxLength", "pattern", "minItems", "maxItems"]163 var seen: Set<String> = []164 func walk(_ v: JSONValue, isPropertyMap: Bool) {165 guard case .object(let o) = v else { return }166 for (k, child) in o {167 if isPropertyMap { walk(child, isPropertyMap: false); continue }168 seen.insert(k)169 switch k {170 case "properties": walk(child, isPropertyMap: true)171 case "items", "additionalProperties": walk(child, isPropertyMap: false)172 default: break173 }174 }175 }176 walk(WorkflowSchema.automationV1().raw, isPropertyMap: false)177 #expect(seen.subtracting(allowed).isEmpty, "unsupported keywords: \(seen.subtracting(allowed).sorted())")178 }179180 @Test func everyExampleValidatesAgainstTheSchema() throws {181 let schema = WorkflowSchema.automationV1()182 for b in BuiltInWorkflows.all() {183 let violations = schema.validate(try canonicalJSON(try b.document()))184 #expect(violations.isEmpty, "\(b.slug): \(violations.map(\.description).joined(separator: "; "))")185 }186 let section9 = try WorkflowCodec.parse(claudeSection9Workflow)187 #expect(schema.validate(try canonicalJSON(section9)).isEmpty)188 }189190 @Test func randomCanonicalDocumentsFitTheSchemaShape() throws {191 // Property check of the schema's structure: only id/name patterns and numeric bounds may be violated by random data.192 let schema = WorkflowSchema.automationV1()193 var generator = DocumentGenerator(seed: 2026)194 for _ in 0..<150 {195 let violations = schema.validate(try canonicalJSON(generator.document()))196 let structural = violations.filter { $0.message.contains("unexpected property") || $0.message.contains("expected ") || $0.message.contains("required") }197 #expect(structural.isEmpty, "\(structural.map(\.description).joined(separator: "; "))")198 }199 }200201 @Test func schemaRejectsUnknownKeysAndWrongTypes() throws {202 let schema = WorkflowSchema.automationV1()203 var value = try canonicalJSON(try WorkflowCodec.parse(claudeSection9Workflow))204 guard case .array(var steps)? = value["spec"]?["steps"] else { Issue.record("no steps"); return }205 steps[0]["wiht"] = ["path": "/x"]206 var spec = value["spec"]!207 spec["steps"] = .array(steps)208 value["spec"] = spec209 #expect(schema.validate(value).contains { $0.path == "spec.steps.0.wiht" && $0.message == "unexpected property" })210211 var wrong = try canonicalJSON(try WorkflowCodec.parse(claudeSection9Workflow))212 var spec2 = wrong["spec"]!213 spec2["budget"] = ["maxToolCalls": "many"]214 spec2["triggers"] = [["type": "sometimes"]]215 wrong["spec"] = spec2216 wrong["apiVersion"] = "orix.ai/v2"217 let found = schema.validate(wrong)218 #expect(found.contains { $0.path == "spec.budget.maxToolCalls" && $0.message.contains("expected integer") })219 #expect(found.contains { $0.path == "spec.triggers.0.type" })220 #expect(found.contains { $0.path == "apiVersion" })221 #expect(schema.validate(["kind": "Automation"]).contains { $0.message == "required property is missing" })222 }223224 @Test func nestedStepsAreDescribedToTheDocumentedDepth() throws {225 func nest(_ depth: Int, bad: Bool) -> JSONValue {226 if depth == 0 { return bad ? ["id": "leaf", "delay": 1, "bogus": true] : ["id": "leaf", "delay": 1] }227 return ["id": .string("level\(depth)"), "while": "false", "maxIterations": 1, "steps": [nest(depth - 1, bad: bad)]]228 }229 func doc(_ depth: Int, bad: Bool, key: String = "steps") -> JSONValue {230 var spec: [String: JSONValue] = ["steps": []]231 spec[key] = [nest(depth, bad: bad)]232 return ["apiVersion": "orix.ai/v1", "kind": "Automation", "metadata": ["id": "n", "name": "N"], "spec": .object(spec)]233 }234 let schema = WorkflowSchema.automationV1()235 let depth = WorkflowSchema.describedNestingDepth236 #expect(depth == 2 && WorkflowSchema.describedHandlerNestingDepth == 1)237 #expect(schema.validate(doc(depth, bad: false)).isEmpty)238 #expect(schema.validate(doc(depth, bad: true)).contains { $0.message == "unexpected property" })239 #expect(schema.validate(doc(depth + 1, bad: true)).isEmpty, "beyond the described depth a nested step is any object")240 #expect(schema.validate(doc(1, bad: true, key: "onError")).contains { $0.message == "unexpected property" })241 #expect(schema.validate(doc(2, bad: true, key: "onError")).isEmpty)242 // Parallel branches nest the same way.243 let branch: JSONValue = ["id": "p", "parallel": [["name": "b", "steps": [["id": "x", "delay": 1, "bogus": 1]]]]]244 let withBranch: JSONValue = ["apiVersion": "orix.ai/v1", "kind": "Automation", "metadata": ["id": "n", "name": "N"], "spec": ["steps": [branch]]]245 #expect(schema.validate(withBranch).contains { $0.path == "spec.steps.0.parallel.0.steps.0.bogus" })246 #expect(WorkflowSchema.automationV1JSON().utf8.count < 400_000, "the published schema must stay a reasonable size")247 }248249 @Test func canonicalToolCatalogueIsWellFormed() {250 let manifests = CanonicalTools.manifests()251 #expect(manifests.count == CanonicalTools.identifiers.count && Set(CanonicalTools.identifiers).count == manifests.count)252 for m in manifests { #expect(m.problems().isEmpty, "\(m.identifier): \(m.problems())") }253 #expect(manifests.first { $0.identifier == "filesystem.list" }?.idempotent == true)254 #expect(manifests.first { $0.identifier == "filesystem.move" }?.idempotent == false)255 #expect(manifests.first { $0.identifier == "filesystem.move" }?.supportsDryRun == true)256 }257}258