SPB Git

Repository Public License

ExamplesTests.swift 16.0 KB 258 lines swift

1// ORIX OS Automation2// File: Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/ExamplesTests.swift3// Description: Integration tests of the 15 built-in automations (real files on disk and in the resource bundle) and of the published JSON Schema.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXCore10@testable import ORIXWorkflowEngine1112enum RepositoryLayout {13    /// Repository root, found from this file: <root>/Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/<file>.14    static func root(_ file: String = #filePath) -> URL {15        var url = URL(fileURLWithPath: file)16        for _ in 0..<5 { url.deleteLastPathComponent() }17        return url18    }19    static var examples: URL { root().appendingPathComponent("Workflows/Examples") }20    static var resources: URL { root().appendingPathComponent("Packages/ORIXWorkflowEngine/Sources/ORIXWorkflowEngine/Resources/Examples") }21    static var schemaFile: URL { root().appendingPathComponent("Workflows/Schemas/automation.v1.schema.json") }2223    static func yamlFiles(_ dir: URL) throws -> [URL] {24        try FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil)25            .filter { $0.pathExtension == "yaml" }.sorted { $0.lastPathComponent < $1.lastPathComponent }26    }27}2829let expectedExampleSlugs = [30    "smart-downloads-organizer", "pdf-to-structured-report", "weekly-research-digest", "git-test-and-review", "csv-cleanup-and-charts",31    "batch-image-conversion", "calendar-preparation", "folder-backup-and-verify", "website-change-monitor", "system-health-report",32    "meeting-notes-organizer", "document-metadata-catalog", "scheduled-window-screenshot", "spreadsheet-validation", "browser-data-entry",33]3435@Suite("Built-in examples (integration: real files and resource bundle)") struct ExamplesTests {36    private func load() throws -> [(file: String, source: String, doc: WorkflowDocument)] {37        try RepositoryLayout.yamlFiles(RepositoryLayout.examples).map { url in38            let source = try String(contentsOf: url, encoding: .utf8)39            do { return (url.lastPathComponent, source, try WorkflowCodec.parse(source)) }40            catch { Issue.record("\(url.lastPathComponent) does not parse: \(error.localizedDescription)"); throw error }41        }42    }4344    @Test func thereAreExactlyFifteenNumberedExamples() throws {45        let names = try RepositoryLayout.yamlFiles(RepositoryLayout.examples).map(\.lastPathComponent)46        #expect(names == expectedExampleSlugs.enumerated().map { String(format: "%02d-%@.yaml", $0.offset + 1, $0.element) })47    }4849    @Test func everyExampleParsesAndValidatesWithoutErrorsOrWarnings() throws {50        for (file, source, doc) in try load() {51            let found = WorkflowValidator.validate(doc, tools: CanonicalTools.manifests(), agents: [], workflows: [])52            let problems = found.filter { $0.severity != .info }53            #expect(problems.isEmpty, "\(file): \(problems.map(\.description).joined(separator: "\n"))")54            let diagnosed = WorkflowCodec.diagnose(source: source, tools: CanonicalTools.manifests())55            #expect(diagnosed.document == doc)56            #expect(try WorkflowCodec.parse(WorkflowCodec.emit(doc)) == doc, "\(file) does not survive canonical emission")57        }58    }5960    @Test func everyExampleIsCompleteAndUsesOnlyCanonicalTools() throws {61        let canonical = Set(CanonicalTools.identifiers)62        for (index, (file, source, doc)) in try load().enumerated() {63            #expect(doc.metadata.id == expectedExampleSlugs[index], "\(file)")64            #expect(doc.metadata.description.count > 80, "\(file) needs a real description")65            #expect(!doc.metadata.tags.isEmpty && !doc.spec.triggers.isEmpty, "\(file)")66            #expect(!doc.spec.parameters.isEmpty && doc.spec.parameters.allSatisfy { !$0.description.isEmpty && $0.defaultValue != nil }, "\(file)")67            #expect(!doc.spec.permissions.isEmpty && !doc.spec.budget.isEmpty, "\(file)")68            for p in doc.spec.parameters where p.type == .path {69                #expect(p.defaultValue?.stringValue?.hasPrefix("~/ORIX Samples/\(doc.metadata.id)/") == true, "\(file): \(p.name) must default under the sample workspace")70            }71            let steps = doc.allSteps.map(\.step)72            #expect(steps.contains { if case .verify(let v) = $0.kind { return !v.assertions.isEmpty }; return false }, "\(file) has no verify step")73            #expect(steps.contains { if case .output = $0.kind { return true }; return false }, "\(file) has no output step")74            for s in steps {75                if case .tool(let t) = s.kind { #expect(canonical.contains(t.tool), "\(file): \(t.tool) is not a canonical tool id") }76                if case .agent(let a) = s.kind {77                    guard case .inline(let inline) = a.agent else { Issue.record("\(file): agents must be inline"); continue }78                    #expect(inline.tools.allSatisfy(canonical.contains) && !a.expect.isEmpty, "\(file): agent \(s.id)")79                }80            }81            #expect(!steps.contains { if case .tool(let t) = $0.kind { return ["filesystem.delete", "git.push", "git.commit"].contains(t.tool) }; return false },82                    "\(file) must not delete, commit or push")83            #expect(source.contains("# Limitations"), "\(file) must state its limitations in comments")84            #expect(source.hasPrefix("# ORIX OS Automation\n# File: Workflows/Examples/\(file)\n"), "\(file) header")85        }86    }8788    @Test func consequentialExamplesAskForApproval() throws {89        let docs = Dictionary(uniqueKeysWithValues: try load().map { ($0.doc.metadata.id, $0.doc) })90        for slug in ["smart-downloads-organizer", "browser-data-entry"] {91            let steps = docs[slug]?.allSteps.map(\.step) ?? []92            #expect(steps.contains { if case .approval = $0.kind { return true }; return false }, "\(slug) needs an approval gate")93        }94    }9596    @Test func resourcesMirrorTheRepositoryExamplesByteForByte() throws {97        let a = try RepositoryLayout.yamlFiles(RepositoryLayout.examples)98        let b = try FileManager.default.contentsOfDirectory(at: RepositoryLayout.resources, includingPropertiesForKeys: nil)99            .sorted { $0.lastPathComponent < $1.lastPathComponent }100        #expect(a.map(\.lastPathComponent) == b.map(\.lastPathComponent))101        for (x, y) in zip(a, b) { #expect(try Data(contentsOf: x) == Data(contentsOf: y), "\(x.lastPathComponent) differs from its resource copy") }102    }103104    @Test func builtInWorkflowsLoadFromTheBundle() throws {105        let all = BuiltInWorkflows.all()106        #expect(all.map(\.slug) == expectedExampleSlugs)107        let onDisk = try load()108        for (b, d) in zip(all, onDisk) {109            #expect(b.yaml == d.source && b.title == d.doc.metadata.name && b.summary == d.doc.metadata.description && b.id == b.slug)110            #expect(try b.document() == d.doc)111        }112        #expect(BuiltInWorkflows.workflow(slug: "system-health-report")?.title == "Local system health report")113        #expect(BuiltInWorkflows.workflow(slug: "nope") == nil)114        #expect(BuiltInWorkflows.slug(fileName: "03-weekly-research-digest.yaml") == "weekly-research-digest")115        #expect(BuiltInWorkflows.slug(fileName: "plain-name.yaml") == "plain-name")116    }117118    @Test func capabilitiesAndProposedGrantsAreCanonicalAndScoped() throws {119        let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("orix-home-\(UUID().uuidString.prefix(8))")120        try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true)121        defer { try? FileManager.default.removeItem(at: tmp) }122        let home = PathResolver.resolveSymlinks(tmp.path)123        for (file, _, doc) in try load() {124            let caps = doc.declaredCapabilities(home: home)125            #expect(!caps.isEmpty, "\(file)")126            for c in caps where c.kind.hasPrefix("file.") {127                #expect(c.scope.hasPrefix(home + "/ORIX Samples/\(doc.metadata.id)"), "\(file): \(c.displayString)")128                #expect(!c.scope.contains("~") && !c.scope.contains("${{"))129            }130            let grants = doc.proposedGrants(workflowID: "wf-1", home: home)131            #expect(grants.map(\.capability) == caps)132            for g in grants {133                #expect(g.scope == .workflow(workflowID: "wf-1") && g.maxTier <= .consequential && g.source == "workflow:wf-1", "\(file)")134            }135        }136        let backup = try #require(BuiltInWorkflows.workflow(slug: "folder-backup-and-verify")).document()137        let custom = backup.declaredCapabilities(home: home, parameters: ["source": "~/Elsewhere"])138        #expect(custom.contains(.fileRead(path: home + "/Elsewhere")))139        let shot = try #require(BuiltInWorkflows.workflow(slug: "scheduled-window-screenshot")).document()140        #expect(shot.declaredCapabilities(home: home).contains(.screenCapture(scope: "window:com.apple.Safari")))141    }142}143144@Suite("Automation JSON Schema (integration: schema file and Core validator)") struct WorkflowSchemaTests {145    private func canonicalJSON(_ doc: WorkflowDocument) throws -> JSONValue { try JSONValue.parse(WorkflowCodec.emit(doc, format: .json)) }146147    @Test func publishedSchemaFileMatchesTheGeneratedSchema() throws {148        let text = WorkflowSchema.automationV1JSON()149        if ProcessInfo.processInfo.environment["ORIX_WRITE_SCHEMA"] == "1" {150            try FileManager.default.createDirectory(at: RepositoryLayout.schemaFile.deletingLastPathComponent(), withIntermediateDirectories: true)151            try text.write(to: RepositoryLayout.schemaFile, atomically: true, encoding: .utf8)152        }153        let onDisk = try String(contentsOf: RepositoryLayout.schemaFile, encoding: .utf8)154        #expect(onDisk == text, "run the tests once with ORIX_WRITE_SCHEMA=1 to regenerate Workflows/Schemas/automation.v1.schema.json")155        let parsed = try JSONValue.parse(onDisk)156        #expect(parsed == WorkflowSchema.automationV1().raw)157        #expect(parsed["$id"] == "https://orix.ai/schemas/automation.v1.schema.json")158    }159160    @Test func schemaUsesOnlyKeywordsCoreEnforcesPlusAnnotations() {161        let allowed: Set<String> = ["$schema", "$id", "title", "description", "type", "properties", "required", "additionalProperties",162                                    "items", "enum", "minimum", "maximum", "minLength", "maxLength", "pattern", "minItems", "maxItems"]163        var seen: Set<String> = []164        func walk(_ v: JSONValue, isPropertyMap: Bool) {165            guard case .object(let o) = v else { return }166            for (k, child) in o {167                if isPropertyMap { walk(child, isPropertyMap: false); continue }168                seen.insert(k)169                switch k {170                case "properties": walk(child, isPropertyMap: true)171                case "items", "additionalProperties": walk(child, isPropertyMap: false)172                default: break173                }174            }175        }176        walk(WorkflowSchema.automationV1().raw, isPropertyMap: false)177        #expect(seen.subtracting(allowed).isEmpty, "unsupported keywords: \(seen.subtracting(allowed).sorted())")178    }179180    @Test func everyExampleValidatesAgainstTheSchema() throws {181        let schema = WorkflowSchema.automationV1()182        for b in BuiltInWorkflows.all() {183            let violations = schema.validate(try canonicalJSON(try b.document()))184            #expect(violations.isEmpty, "\(b.slug): \(violations.map(\.description).joined(separator: "; "))")185        }186        let section9 = try WorkflowCodec.parse(claudeSection9Workflow)187        #expect(schema.validate(try canonicalJSON(section9)).isEmpty)188    }189190    @Test func randomCanonicalDocumentsFitTheSchemaShape() throws {191        // Property check of the schema's structure: only id/name patterns and numeric bounds may be violated by random data.192        let schema = WorkflowSchema.automationV1()193        var generator = DocumentGenerator(seed: 2026)194        for _ in 0..<150 {195            let violations = schema.validate(try canonicalJSON(generator.document()))196            let structural = violations.filter { $0.message.contains("unexpected property") || $0.message.contains("expected ") || $0.message.contains("required") }197            #expect(structural.isEmpty, "\(structural.map(\.description).joined(separator: "; "))")198        }199    }200201    @Test func schemaRejectsUnknownKeysAndWrongTypes() throws {202        let schema = WorkflowSchema.automationV1()203        var value = try canonicalJSON(try WorkflowCodec.parse(claudeSection9Workflow))204        guard case .array(var steps)? = value["spec"]?["steps"] else { Issue.record("no steps"); return }205        steps[0]["wiht"] = ["path": "/x"]206        var spec = value["spec"]!207        spec["steps"] = .array(steps)208        value["spec"] = spec209        #expect(schema.validate(value).contains { $0.path == "spec.steps.0.wiht" && $0.message == "unexpected property" })210211        var wrong = try canonicalJSON(try WorkflowCodec.parse(claudeSection9Workflow))212        var spec2 = wrong["spec"]!213        spec2["budget"] = ["maxToolCalls": "many"]214        spec2["triggers"] = [["type": "sometimes"]]215        wrong["spec"] = spec2216        wrong["apiVersion"] = "orix.ai/v2"217        let found = schema.validate(wrong)218        #expect(found.contains { $0.path == "spec.budget.maxToolCalls" && $0.message.contains("expected integer") })219        #expect(found.contains { $0.path == "spec.triggers.0.type" })220        #expect(found.contains { $0.path == "apiVersion" })221        #expect(schema.validate(["kind": "Automation"]).contains { $0.message == "required property is missing" })222    }223224    @Test func nestedStepsAreDescribedToTheDocumentedDepth() throws {225        func nest(_ depth: Int, bad: Bool) -> JSONValue {226            if depth == 0 { return bad ? ["id": "leaf", "delay": 1, "bogus": true] : ["id": "leaf", "delay": 1] }227            return ["id": .string("level\(depth)"), "while": "false", "maxIterations": 1, "steps": [nest(depth - 1, bad: bad)]]228        }229        func doc(_ depth: Int, bad: Bool, key: String = "steps") -> JSONValue {230            var spec: [String: JSONValue] = ["steps": []]231            spec[key] = [nest(depth, bad: bad)]232            return ["apiVersion": "orix.ai/v1", "kind": "Automation", "metadata": ["id": "n", "name": "N"], "spec": .object(spec)]233        }234        let schema = WorkflowSchema.automationV1()235        let depth = WorkflowSchema.describedNestingDepth236        #expect(depth == 2 && WorkflowSchema.describedHandlerNestingDepth == 1)237        #expect(schema.validate(doc(depth, bad: false)).isEmpty)238        #expect(schema.validate(doc(depth, bad: true)).contains { $0.message == "unexpected property" })239        #expect(schema.validate(doc(depth + 1, bad: true)).isEmpty, "beyond the described depth a nested step is any object")240        #expect(schema.validate(doc(1, bad: true, key: "onError")).contains { $0.message == "unexpected property" })241        #expect(schema.validate(doc(2, bad: true, key: "onError")).isEmpty)242        // Parallel branches nest the same way.243        let branch: JSONValue = ["id": "p", "parallel": [["name": "b", "steps": [["id": "x", "delay": 1, "bogus": 1]]]]]244        let withBranch: JSONValue = ["apiVersion": "orix.ai/v1", "kind": "Automation", "metadata": ["id": "n", "name": "N"], "spec": ["steps": [branch]]]245        #expect(schema.validate(withBranch).contains { $0.path == "spec.steps.0.parallel.0.steps.0.bogus" })246        #expect(WorkflowSchema.automationV1JSON().utf8.count < 400_000, "the published schema must stay a reasonable size")247    }248249    @Test func canonicalToolCatalogueIsWellFormed() {250        let manifests = CanonicalTools.manifests()251        #expect(manifests.count == CanonicalTools.identifiers.count && Set(CanonicalTools.identifiers).count == manifests.count)252        for m in manifests { #expect(m.problems().isEmpty, "\(m.identifier): \(m.problems())") }253        #expect(manifests.first { $0.identifier == "filesystem.list" }?.idempotent == true)254        #expect(manifests.first { $0.identifier == "filesystem.move" }?.idempotent == false)255        #expect(manifests.first { $0.identifier == "filesystem.move" }?.supportsDryRun == true)256    }257}258
Files main
Menu

Theme