Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraAgentsTests/DataTransformTests.swift10// Module : AstraAgentsTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraAgents9293@Suite struct DataTransformTests {94 static let sales = """95 region,rep,amount,units96 East,Ana,100,297 West,Bob,250.5,598 East,Cy,50,199 North,Dee,,3100 West,Eli,49.5,1101 """102103 func run(_ input: String, _ operations: [JSONValue] = [], output: DataTransform.Format? = nil, input format: DataTransform.Format? = nil) throws -> String {104 try DataTransform.run(input: input, inputFormat: format, operations: operations, outputFormat: output)105 }106107 // MARK: CSV108109 @Test func csvQuoting() throws {110 let text = "name,note\r\n\"Smith, John\",\"He said \"\"hi\"\"\"\r\n\"multi\nline\",x\r\n,\"\"\r\n padded ,\"a,b\"\"c\"\r\n"111 let rows = try CSV.parse(text)112 #expect(rows == [["name", "note"], ["Smith, John", "He said \"hi\""], ["multi\nline", "x"], ["", ""], [" padded ", "a,b\"c"]])113 }114115 @Test func csvEdgeCases() throws {116 #expect(try CSV.parse("a,b\n1,2") == [["a", "b"], ["1", "2"]]) // no final line break117 #expect(try CSV.parse("a,b\n\n1,2\n\n") == [["a", "b"], ["1", "2"]]) // blank lines are not records118 #expect(try CSV.parse("\u{FEFF}a,b\n1,2\n") == [["a", "b"], ["1", "2"]]) // byte-order mark119 #expect(try CSV.parse("a,b\r1,2\r") == [["a", "b"], ["1", "2"]]) // bare CR120 #expect(try CSV.parse("a,,c\n,,\n") == [["a", "", "c"], ["", "", ""]]) // empty fields121 #expect(try CSV.parse("a;b\n\"x;y\";2", delimiter: ";") == [["a", "b"], ["x;y", "2"]])122 #expect(try CSV.parse("a\tb\n1\t2", delimiter: "\t") == [["a", "b"], ["1", "2"]])123 #expect(try CSV.parse("a\n\"\"\n") == [["a"], [""]]) // a quoted empty field is a record124 #expect(try CSV.parse("it's,5\" tall\n") == [["it's", "5\" tall"]]) // a quote inside an unquoted field is literal125 #expect(try CSV.parse("\"a\r\nb\",c\r\n") == [["a\r\nb", "c"]]) // CRLF kept inside quotes126 #expect(try CSV.parse("") == [])127 }128129 @Test func csvErrors() {130 #expect(throws: ToolError.self) { try CSV.parse("a,b\n\"unclosed,2\n3,4") }131 #expect(throws: ToolError.self) { try CSV.parse("a,b\n\"x\"y,2") }132 #expect(throws: ToolError.self) { try CSV.parse("a\n1\n2\n3\n4", maxRows: 2) }133 #expect(throws: ToolError.self) { try DataTransform.tableFromCSV("a,b\n1,2,3") }134 #expect(throws: ToolError.self) { try DataTransform.tableFromCSV("") }135 }136137 @Test func csvWritingRoundTrips() throws {138 let rows = [["name", "note"], ["Smith, John", "He said \"hi\""], ["multi\nline", " lead"], ["plain", ""]]139 let text = CSV.write(rows)140 #expect(text == "name,note\r\n\"Smith, John\",\"He said \"\"hi\"\"\"\r\n\"multi\nline\",\" lead\"\r\nplain,")141 #expect(try CSV.parse(text) == rows)142 }143144 @Test func csvToJSONTypesCells() throws {145 let json = JSONValue.parse(try run("id,code,price,note\n1,007,9.50,\n2,A1,-3e2,hi", output: .json))146 #expect(json == [["id": 1, "code": "007", "price": 9.5, "note": nil], ["id": 2, "code": "A1", "price": -300, "note": "hi"]])147 }148149 @Test func jsonToCSV() throws {150 let csv = try run(#"[{"b": "x, y", "a": 1}, {"a": 2.5, "c": true}]"#, output: .csv)151 #expect(csv == "a,b,c\r\n1,\"x, y\",\r\n2.5,,true")152 }153154 @Test func duplicateAndEmptyHeaders() throws {155 let table = try DataTransform.tableFromCSV("a,a,,a\n1,2,3,4")156 #expect(table.columns == ["a", "a_2", "column_3", "a_3"])157 }158159 // MARK: Operations160161 @Test func selectAndRename() throws {162 let out = try run(Self.sales, [["op": "select", "columns": ["rep", "amount"]], ["op": "rename", "mapping": ["rep": "name"]], ["op": "head", "n": 2]])163 #expect(out == "name,amount\r\nAna,100\r\nBob,250.5")164 }165166 @Test func filters() throws {167 func reps(_ comparison: String, _ value: JSONValue, column: String = "amount") throws -> String {168 let out = try run(Self.sales, [["op": "filter", "column": .string(column), "operator": .string(comparison), "value": value], ["op": "select", "columns": ["rep"]]])169 return out.split(separator: "\r\n").dropFirst().joined(separator: " ")170 }171 #expect(try reps("==", 100) == "Ana")172 #expect(try reps("==", "100") == "Ana") // numeric comparison when both sides are numbers173 #expect(try reps("!=", 100) == "Bob Cy Dee Eli")174 #expect(try reps("<", 50) == "Eli")175 #expect(try reps("<=", 50) == "Cy Eli")176 #expect(try reps(">", 100) == "Bob")177 #expect(try reps(">=", 100) == "Ana Bob") // 100 < 250.5 as numbers, not as text178 #expect(try reps("==", nil) == "Dee") // empty cell179 #expect(try reps("!=", nil) == "Ana Bob Cy Eli")180 #expect(try reps("contains", "ST", column: "region") == "Ana Bob Cy Eli")181 #expect(try reps("==", "East", column: "region") == "Ana Cy")182 #expect(try reps(">", "M", column: "region") == "Bob Dee Eli")183 }184185 @Test func sorting() throws {186 func order(_ operation: JSONValue) throws -> String {187 try run(Self.sales, [operation, ["op": "select", "columns": ["rep"]]]).split(separator: "\r\n").dropFirst().joined(separator: " ")188 }189 #expect(try order(["op": "sort", "by": ["amount"]]) == "Eli Cy Ana Bob Dee") // numeric, nulls last190 #expect(try order(["op": "sort", "by": ["amount"], "descending": true]) == "Dee Bob Ana Cy Eli")191 #expect(try order(["op": "sort", "by": ["region"]]) == "Ana Cy Dee Bob Eli") // stable within a region192 #expect(try order(["op": "sort", "by": ["region", "amount"]]) == "Cy Ana Dee Eli Bob")193 }194195 @Test func groupByWithAggregates() throws {196 let out = try run(Self.sales, [[197 "op": "group_by", "by": ["region"],198 "aggregates": [199 ["function": "count"], ["function": "count", "column": "amount", "as": "priced"], ["function": "sum", "column": "amount"],200 ["function": "mean", "column": "amount"], ["function": "min", "column": "units"], ["function": "max", "column": "rep"],201 ],202 ]])203 #expect(out == """204 region,count,priced,sum_amount,mean_amount,min_units,max_rep\r205 East,2,2,150,75,1,Cy\r206 West,2,2,300,150,1,Eli\r207 North,1,0,0,,3,Dee208 """)209 }210211 @Test func groupByDefaultsAndWholeTable() throws {212 #expect(try run(Self.sales, [["op": "group_by", "by": ["region"]]]) == "region,count\r\nEast,2\r\nWest,2\r\nNorth,1")213 #expect(try run(Self.sales, [["op": "group_by", "aggregates": [["function": "sum", "column": "units", "as": "total"]]]]) == "total\r\n12")214 #expect(throws: ToolError.self) { try run(Self.sales, [["op": "group_by", "by": ["region"], "aggregates": [["function": "sum"]]]]) }215 #expect(throws: ToolError.self) { try run(Self.sales, [["op": "group_by", "by": ["region"], "aggregates": [["function": "count", "as": "region"]]]]) }216 }217218 @Test func uniqueHeadTail() throws {219 #expect(try run(Self.sales, [["op": "unique", "columns": ["region"]], ["op": "select", "columns": ["region"]]]) == "region\r\nEast\r\nWest\r\nNorth")220 #expect(try run("a,b\n1,2\n1,2\n1,3", [["op": "unique"]]) == "a,b\r\n1,2\r\n1,3")221 #expect(try run(Self.sales, [["op": "tail", "n": 1], ["op": "select", "columns": ["rep"]]]) == "rep\r\nEli")222 #expect(try run(Self.sales, [["op": "head", "n": 0]]) == "region,rep,amount,units")223 }224225 @Test func jsonPointer() throws {226 let doc = #"{"data": {"items": [{"name": "a", "n": 3}, {"name": "b/c", "n": 1}], "k~y": 7, "a/b": [10, 20]}}"#227 #expect(try run(doc, [["op": "pointer", "path": "/data/items/1/name"]]) == "\"b/c\"")228 #expect(try run(doc, [["op": "pointer", "path": "/data/k~0y"]]) == "7")229 #expect(try run(doc, [["op": "pointer", "path": "/data/a~1b/1"]]) == "20")230 #expect(try run(doc, [["op": "pointer", "path": "/data/items"], ["op": "sort", "by": ["n"]], ["op": "select", "columns": ["name"]]], output: .csv) == "name\r\nb/c\r\na")231 #expect(try DataTransform.pointer("", in: ["a": 1]) == ["a": 1])232 for bad in ["/data/missing", "/data/items/9", "data/items", "/data/k~0y/deeper"] {233 #expect(throws: ToolError.self, "\(bad)") { try run(doc, [["op": "pointer", "path": .string(bad)]]) }234 }235 }236237 @Test func errorsAreSpecific() {238 func message(_ input: String, _ operations: [JSONValue]) -> String {239 do { return try run(input, operations) } catch { return (error as? ToolError)?.message ?? "\(error)" }240 }241 #expect(message(Self.sales, [["op": "select", "columns": ["nope"]]]) == "Operation 1 (select): there is no column “nope”. Columns: region, rep, amount, units.")242 #expect(message(Self.sales, [["op": "head", "n": 1], ["op": "filter", "column": "amount"]]).hasPrefix("Operation 2 (filter): “operator” is required"))243 #expect(message(#"{"a": 1}"#, [["op": "head", "n": 1]]).contains("not a table"))244 #expect(message("{not json", []).isEmpty == false)245 #expect(throws: ToolError.self) { try run("{broken", input: .json) }246 #expect(message(Self.sales, [["op": "rename", "mapping": ["rep": "region"]]]).contains("already exists"))247 #expect(message(Self.sales, [["op": "explode"]]).contains("unknown operation"))248 }249250 @Test func limits() {251 let big = "a\n" + String(repeating: "1234567890\n", count: 100_000)252 #expect(throws: ToolError.self) { try run(big) } // bytes253 let rows = "a\n" + String(repeating: "1\n", count: DataTransform.maxRows + 5)254 #expect(throws: ToolError.self) { try run(rows) } // rows255 let operations = [JSONValue](repeating: ["op": "head", "n": 1], count: DataTransform.maxOperations + 1)256 #expect(throws: ToolError.self) { try run(Self.sales, operations) }257 #expect((try? run("a\n" + String(repeating: "1\n", count: 5_000), [["op": "group_by"]])) == "count\r\n5000")258 }259260 @Test func toolValidatesItsSchema() async throws {261 let tool = DataTransformTool()262 #expect(tool.risk == .safe)263 let out = try await runTool(tool, ["input": .string(Self.sales), "output_format": "json",264 "operations": [["op": "filter", "column": "region", "operator": "==", "value": "West"], ["op": "select", "columns": ["rep", "units"]]]])265 #expect(JSONValue.parse(out) == [["rep": "Bob", "units": 5], ["rep": "Eli", "units": 1]])266 #expect(await toolError(tool, ["input": "a\n1", "operations": [["op": "filter", "column": "a", "operator": "~=", "value": 1]]])?.contains("“operations[0].operator” must be one of") == true)267 #expect(await toolError(tool, ["input": "a\n1", "operations": [["op": "eval", "code": "rm -rf"]]])?.contains("“operations[0].code” is not a known property") == true)268 #expect(await toolError(tool, ["input": "a\n1", "operations": [["op": "head", "n": -1]]])?.contains("at least 0") == true)269 #expect(await toolError(tool, ["input": "a;b\n1;2", "delimiter": "::"])?.contains("must be one of") == true)270 #expect(try await runTool(tool, ["input": "a;b\n1;2", "delimiter": ";", "output_format": "json"]).contains("\"b\" : 2"))271 }272}273