SPB Git

Repository Public MIT

DataTransformTests.swift 14.8 KB 273 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraAgentsTests/DataTransformTests.swift10// Module           : AstraAgentsTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraAgents9293@Suite struct DataTransformTests {94    static let sales = """95    region,rep,amount,units96    East,Ana,100,297    West,Bob,250.5,598    East,Cy,50,199    North,Dee,,3100    West,Eli,49.5,1101    """102103    func run(_ input: String, _ operations: [JSONValue] = [], output: DataTransform.Format? = nil, input format: DataTransform.Format? = nil) throws -> String {104        try DataTransform.run(input: input, inputFormat: format, operations: operations, outputFormat: output)105    }106107    // MARK: CSV108109    @Test func csvQuoting() throws {110        let text = "name,note\r\n\"Smith, John\",\"He said \"\"hi\"\"\"\r\n\"multi\nline\",x\r\n,\"\"\r\n  padded  ,\"a,b\"\"c\"\r\n"111        let rows = try CSV.parse(text)112        #expect(rows == [["name", "note"], ["Smith, John", "He said \"hi\""], ["multi\nline", "x"], ["", ""], ["  padded  ", "a,b\"c"]])113    }114115    @Test func csvEdgeCases() throws {116        #expect(try CSV.parse("a,b\n1,2") == [["a", "b"], ["1", "2"]])                      // no final line break117        #expect(try CSV.parse("a,b\n\n1,2\n\n") == [["a", "b"], ["1", "2"]])                // blank lines are not records118        #expect(try CSV.parse("\u{FEFF}a,b\n1,2\n") == [["a", "b"], ["1", "2"]])            // byte-order mark119        #expect(try CSV.parse("a,b\r1,2\r") == [["a", "b"], ["1", "2"]])                    // bare CR120        #expect(try CSV.parse("a,,c\n,,\n") == [["a", "", "c"], ["", "", ""]])              // empty fields121        #expect(try CSV.parse("a;b\n\"x;y\";2", delimiter: ";") == [["a", "b"], ["x;y", "2"]])122        #expect(try CSV.parse("a\tb\n1\t2", delimiter: "\t") == [["a", "b"], ["1", "2"]])123        #expect(try CSV.parse("a\n\"\"\n") == [["a"], [""]])                                // a quoted empty field is a record124        #expect(try CSV.parse("it's,5\" tall\n") == [["it's", "5\" tall"]])                 // a quote inside an unquoted field is literal125        #expect(try CSV.parse("\"a\r\nb\",c\r\n") == [["a\r\nb", "c"]])                     // CRLF kept inside quotes126        #expect(try CSV.parse("") == [])127    }128129    @Test func csvErrors() {130        #expect(throws: ToolError.self) { try CSV.parse("a,b\n\"unclosed,2\n3,4") }131        #expect(throws: ToolError.self) { try CSV.parse("a,b\n\"x\"y,2") }132        #expect(throws: ToolError.self) { try CSV.parse("a\n1\n2\n3\n4", maxRows: 2) }133        #expect(throws: ToolError.self) { try DataTransform.tableFromCSV("a,b\n1,2,3") }134        #expect(throws: ToolError.self) { try DataTransform.tableFromCSV("") }135    }136137    @Test func csvWritingRoundTrips() throws {138        let rows = [["name", "note"], ["Smith, John", "He said \"hi\""], ["multi\nline", " lead"], ["plain", ""]]139        let text = CSV.write(rows)140        #expect(text == "name,note\r\n\"Smith, John\",\"He said \"\"hi\"\"\"\r\n\"multi\nline\",\" lead\"\r\nplain,")141        #expect(try CSV.parse(text) == rows)142    }143144    @Test func csvToJSONTypesCells() throws {145        let json = JSONValue.parse(try run("id,code,price,note\n1,007,9.50,\n2,A1,-3e2,hi", output: .json))146        #expect(json == [["id": 1, "code": "007", "price": 9.5, "note": nil], ["id": 2, "code": "A1", "price": -300, "note": "hi"]])147    }148149    @Test func jsonToCSV() throws {150        let csv = try run(#"[{"b": "x, y", "a": 1}, {"a": 2.5, "c": true}]"#, output: .csv)151        #expect(csv == "a,b,c\r\n1,\"x, y\",\r\n2.5,,true")152    }153154    @Test func duplicateAndEmptyHeaders() throws {155        let table = try DataTransform.tableFromCSV("a,a,,a\n1,2,3,4")156        #expect(table.columns == ["a", "a_2", "column_3", "a_3"])157    }158159    // MARK: Operations160161    @Test func selectAndRename() throws {162        let out = try run(Self.sales, [["op": "select", "columns": ["rep", "amount"]], ["op": "rename", "mapping": ["rep": "name"]], ["op": "head", "n": 2]])163        #expect(out == "name,amount\r\nAna,100\r\nBob,250.5")164    }165166    @Test func filters() throws {167        func reps(_ comparison: String, _ value: JSONValue, column: String = "amount") throws -> String {168            let out = try run(Self.sales, [["op": "filter", "column": .string(column), "operator": .string(comparison), "value": value], ["op": "select", "columns": ["rep"]]])169            return out.split(separator: "\r\n").dropFirst().joined(separator: " ")170        }171        #expect(try reps("==", 100) == "Ana")172        #expect(try reps("==", "100") == "Ana")                 // numeric comparison when both sides are numbers173        #expect(try reps("!=", 100) == "Bob Cy Dee Eli")174        #expect(try reps("<", 50) == "Eli")175        #expect(try reps("<=", 50) == "Cy Eli")176        #expect(try reps(">", 100) == "Bob")177        #expect(try reps(">=", 100) == "Ana Bob")               // 100 < 250.5 as numbers, not as text178        #expect(try reps("==", nil) == "Dee")                   // empty cell179        #expect(try reps("!=", nil) == "Ana Bob Cy Eli")180        #expect(try reps("contains", "ST", column: "region") == "Ana Bob Cy Eli")181        #expect(try reps("==", "East", column: "region") == "Ana Cy")182        #expect(try reps(">", "M", column: "region") == "Bob Dee Eli")183    }184185    @Test func sorting() throws {186        func order(_ operation: JSONValue) throws -> String {187            try run(Self.sales, [operation, ["op": "select", "columns": ["rep"]]]).split(separator: "\r\n").dropFirst().joined(separator: " ")188        }189        #expect(try order(["op": "sort", "by": ["amount"]]) == "Eli Cy Ana Bob Dee")                       // numeric, nulls last190        #expect(try order(["op": "sort", "by": ["amount"], "descending": true]) == "Dee Bob Ana Cy Eli")191        #expect(try order(["op": "sort", "by": ["region"]]) == "Ana Cy Dee Bob Eli")                       // stable within a region192        #expect(try order(["op": "sort", "by": ["region", "amount"]]) == "Cy Ana Dee Eli Bob")193    }194195    @Test func groupByWithAggregates() throws {196        let out = try run(Self.sales, [[197            "op": "group_by", "by": ["region"],198            "aggregates": [199                ["function": "count"], ["function": "count", "column": "amount", "as": "priced"], ["function": "sum", "column": "amount"],200                ["function": "mean", "column": "amount"], ["function": "min", "column": "units"], ["function": "max", "column": "rep"],201            ],202        ]])203        #expect(out == """204        region,count,priced,sum_amount,mean_amount,min_units,max_rep\r205        East,2,2,150,75,1,Cy\r206        West,2,2,300,150,1,Eli\r207        North,1,0,0,,3,Dee208        """)209    }210211    @Test func groupByDefaultsAndWholeTable() throws {212        #expect(try run(Self.sales, [["op": "group_by", "by": ["region"]]]) == "region,count\r\nEast,2\r\nWest,2\r\nNorth,1")213        #expect(try run(Self.sales, [["op": "group_by", "aggregates": [["function": "sum", "column": "units", "as": "total"]]]]) == "total\r\n12")214        #expect(throws: ToolError.self) { try run(Self.sales, [["op": "group_by", "by": ["region"], "aggregates": [["function": "sum"]]]]) }215        #expect(throws: ToolError.self) { try run(Self.sales, [["op": "group_by", "by": ["region"], "aggregates": [["function": "count", "as": "region"]]]]) }216    }217218    @Test func uniqueHeadTail() throws {219        #expect(try run(Self.sales, [["op": "unique", "columns": ["region"]], ["op": "select", "columns": ["region"]]]) == "region\r\nEast\r\nWest\r\nNorth")220        #expect(try run("a,b\n1,2\n1,2\n1,3", [["op": "unique"]]) == "a,b\r\n1,2\r\n1,3")221        #expect(try run(Self.sales, [["op": "tail", "n": 1], ["op": "select", "columns": ["rep"]]]) == "rep\r\nEli")222        #expect(try run(Self.sales, [["op": "head", "n": 0]]) == "region,rep,amount,units")223    }224225    @Test func jsonPointer() throws {226        let doc = #"{"data": {"items": [{"name": "a", "n": 3}, {"name": "b/c", "n": 1}], "k~y": 7, "a/b": [10, 20]}}"#227        #expect(try run(doc, [["op": "pointer", "path": "/data/items/1/name"]]) == "\"b/c\"")228        #expect(try run(doc, [["op": "pointer", "path": "/data/k~0y"]]) == "7")229        #expect(try run(doc, [["op": "pointer", "path": "/data/a~1b/1"]]) == "20")230        #expect(try run(doc, [["op": "pointer", "path": "/data/items"], ["op": "sort", "by": ["n"]], ["op": "select", "columns": ["name"]]], output: .csv) == "name\r\nb/c\r\na")231        #expect(try DataTransform.pointer("", in: ["a": 1]) == ["a": 1])232        for bad in ["/data/missing", "/data/items/9", "data/items", "/data/k~0y/deeper"] {233            #expect(throws: ToolError.self, "\(bad)") { try run(doc, [["op": "pointer", "path": .string(bad)]]) }234        }235    }236237    @Test func errorsAreSpecific() {238        func message(_ input: String, _ operations: [JSONValue]) -> String {239            do { return try run(input, operations) } catch { return (error as? ToolError)?.message ?? "\(error)" }240        }241        #expect(message(Self.sales, [["op": "select", "columns": ["nope"]]]) == "Operation 1 (select): there is no column “nope”. Columns: region, rep, amount, units.")242        #expect(message(Self.sales, [["op": "head", "n": 1], ["op": "filter", "column": "amount"]]).hasPrefix("Operation 2 (filter): “operator” is required"))243        #expect(message(#"{"a": 1}"#, [["op": "head", "n": 1]]).contains("not a table"))244        #expect(message("{not json", []).isEmpty == false)245        #expect(throws: ToolError.self) { try run("{broken", input: .json) }246        #expect(message(Self.sales, [["op": "rename", "mapping": ["rep": "region"]]]).contains("already exists"))247        #expect(message(Self.sales, [["op": "explode"]]).contains("unknown operation"))248    }249250    @Test func limits() {251        let big = "a\n" + String(repeating: "1234567890\n", count: 100_000)252        #expect(throws: ToolError.self) { try run(big) }                                             // bytes253        let rows = "a\n" + String(repeating: "1\n", count: DataTransform.maxRows + 5)254        #expect(throws: ToolError.self) { try run(rows) }                                            // rows255        let operations = [JSONValue](repeating: ["op": "head", "n": 1], count: DataTransform.maxOperations + 1)256        #expect(throws: ToolError.self) { try run(Self.sales, operations) }257        #expect((try? run("a\n" + String(repeating: "1\n", count: 5_000), [["op": "group_by"]])) == "count\r\n5000")258    }259260    @Test func toolValidatesItsSchema() async throws {261        let tool = DataTransformTool()262        #expect(tool.risk == .safe)263        let out = try await runTool(tool, ["input": .string(Self.sales), "output_format": "json",264                                           "operations": [["op": "filter", "column": "region", "operator": "==", "value": "West"], ["op": "select", "columns": ["rep", "units"]]]])265        #expect(JSONValue.parse(out) == [["rep": "Bob", "units": 5], ["rep": "Eli", "units": 1]])266        #expect(await toolError(tool, ["input": "a\n1", "operations": [["op": "filter", "column": "a", "operator": "~=", "value": 1]]])?.contains("“operations[0].operator” must be one of") == true)267        #expect(await toolError(tool, ["input": "a\n1", "operations": [["op": "eval", "code": "rm -rf"]]])?.contains("“operations[0].code” is not a known property") == true)268        #expect(await toolError(tool, ["input": "a\n1", "operations": [["op": "head", "n": -1]]])?.contains("at least 0") == true)269        #expect(await toolError(tool, ["input": "a;b\n1;2", "delimiter": "::"])?.contains("must be one of") == true)270        #expect(try await runTool(tool, ["input": "a;b\n1;2", "delimiter": ";", "output_format": "json"]).contains("\"b\" : 2"))271    }272}273
Files main
Menu

Theme